Capture UC dependencies for vector search indexes

.nextchanges/bundles/vector-search-index-uc-deps.md

@@ -0,0 +1 @@
+* Capture the implicit dependency a vector search index has on a catalog or schema defined in the same bundle, so the catalog and schema are deployed first. ([#6655](https://github.com/databricks/cli/pull/6655))

acceptance/bundle/resource_deps/implicit_deps_vector_search_index/databricks.yml

@@ -0,0 +1,25 @@
+bundle:
+  name: test-bundle
+
+# Tests implicit dependency detection for vector search indexes:
+# - name (format "catalog.schema.index") should resolve the catalog and schema parts
+resources:
+  catalogs:
+    my_catalog:
+      name: mycatalog
+  schemas:
+    my_schema:
+      catalog_name: mycatalog
+      name: myschema
+  vector_search_indexes:
+    my_index:
+      name: mycatalog.myschema.myindex
+      endpoint_name: my-endpoint
+      primary_key: id
+      index_type: DIRECT_ACCESS
+      direct_access_index_spec:
+        schema_json: '{"id":"integer","vector":"array<float>"}'
+
+targets:
+  dev:
+    mode: development

acceptance/bundle/resource_deps/implicit_deps_vector_search_index/output.txt

@@ -0,0 +1,26 @@
+
+>>> [CLI] bundle validate -t dev -o json
+{
+  "catalogs": {
+    "my_catalog": {
+      "name": "mycatalog"
+    }
+  },
+  "schemas": {
+    "my_schema": {
+      "catalog_name": "${resources.catalogs.my_catalog.name}",
+      "name": "dev_[USERNAME]_myschema"
+    }
+  },
+  "vector_search_indexes": {
+    "my_index": {
+      "direct_access_index_spec": {
+        "schema_json": "{\"id\":\"integer\",\"vector\":\"array<float>\"}"
+      },
+      "endpoint_name": "my-endpoint",
+      "index_type": "DIRECT_ACCESS",
+      "name": "${resources.catalogs.my_catalog.name}.${resources.schemas.my_schema.name}.myindex",
+      "primary_key": "id"
+    }
+  }
+}

acceptance/bundle/resource_deps/implicit_deps_vector_search_index/script

@@ -0,0 +1 @@
+trace $CLI bundle validate -t dev -o json | jq .resources

acceptance/bundle/resource_deps/implicit_deps_vector_search_index/test.toml

@@ -0,0 +1 @@
+RecordRequests = false

bundle/config/mutator/resourcemutator/capture_uc_dependencies.go

@@ -13,13 +13,13 @@ import (
 type captureUCDependencies struct{}
 
 // If a user defines a UC schema in the bundle, they can refer to it in SDP pipelines,
-// UC Volumes, Registered Models, Quality Monitors, or Model Serving Endpoints using the
-// `${resources.schemas.<schema_key>.name}` syntax. Using this syntax allows TF to capture
-// the deploy time dependency this resource has on the schema and deploy changes to the
-// schema before deploying the dependent resource.
+// UC Volumes, Registered Models, Quality Monitors, Model Serving Endpoints, or Vector Search
+// Indexes using the `${resources.schemas.<schema_key>.name}` syntax. Using this syntax allows
+// TF to capture the deploy time dependency this resource has on the schema and deploy changes
+// to the schema before deploying the dependent resource.
 //
 // Similarly, if a user defines a UC catalog in the bundle, they can refer to it in UC schemas,
-// UC Volumes, Registered Models, or Model Serving Endpoints using the
+// UC Volumes, Registered Models, Model Serving Endpoints, or Vector Search Indexes using the
 // `${resources.catalogs.<catalog_key>.name}` syntax. This captures the deploy time
 // dependency the resource has on the catalog.
 //
@@ -155,6 +155,18 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia
 			qm.OutputSchemaName = resolved
 		}
 	}
+	for _, idx := range b.Config.Resources.VectorSearchIndexes {
+		if idx == nil {
+			continue
+		}
+		// Name is a three-part "catalog.schema.index" UC identifier.
+		parts := strings.SplitN(idx.Name, ".", 3)
+		if len(parts) != 3 {
+			continue
+		}
+		catalogName, schemaName := parts[0], parts[1]
+		idx.Name = resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) + "." + parts[2]
+	}
 	for _, mse := range b.Config.Resources.ModelServingEndpoints {
 		if mse == nil {
 			continue

bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
 	"github.com/databricks/databricks-sdk-go/service/serving"
+	"github.com/databricks/databricks-sdk-go/service/vectorsearch"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -136,6 +137,11 @@ func TestCaptureUCDependencies(t *testing.T) {
 						Parent: "schemas/mycatalog.myschema", ModelServiceId: "myservice",
 					}},
 				},
+				VectorSearchIndexes: map[string]*resources.VectorSearchIndex{
+					"my_index": {CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{
+						Name: "mycatalog.myschema.myindex",
+					}},
+				},
 			},
 		},
 	}
@@ -171,6 +177,9 @@ func TestCaptureUCDependencies(t *testing.T) {
 
 	// Model service (compound "schemas/{catalog}.{schema}" parent field).
 	assert.Equal(t, "schemas/"+catalogRef+"."+schemaRef, b.Config.Resources.ModelServices["my_model_service"].Parent)
+
+	// Vector search index (three-part "catalog.schema.index" name).
+	assert.Equal(t, catalogRef+"."+schemaRef+".myindex", b.Config.Resources.VectorSearchIndexes["my_index"].Name)
 }
 
 // Pipeline schema and target are mutually exclusive; only the populated field
@@ -287,6 +296,37 @@ func TestCaptureUCDependenciesModelServingEndpointEdgeCases(t *testing.T) {
 	assert.Nil(t, b.Config.Resources.ModelServingEndpoints["nil_endpoint"])
 }
 
+func TestCaptureUCDependenciesVectorSearchIndexEdgeCases(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Catalogs: map[string]*resources.Catalog{
+					"my_catalog": {CreateCatalog: catalog.CreateCatalog{Name: "mycatalog"}},
+				},
+				Schemas: map[string]*resources.Schema{
+					"my_schema": {CreateSchema: catalog.CreateSchema{CatalogName: "mycatalog", Name: "myschema"}},
+				},
+				VectorSearchIndexes: map[string]*resources.VectorSearchIndex{
+					"catalog_only": {CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{Name: "mycatalog.other.myindex"}},
+					"no_match":     {CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{Name: "other.other.myindex"}},
+					"empty":        {CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{Name: ""}},
+					"two_part":     {CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{Name: "mycatalog.myschema"}},
+					"nil_index":    nil,
+				},
+			},
+		},
+	}
+
+	d := bundle.Apply(t.Context(), b, CaptureUCDependencies())
+	require.Nil(t, d)
+
+	assert.Equal(t, "${resources.catalogs.my_catalog.name}.other.myindex", b.Config.Resources.VectorSearchIndexes["catalog_only"].Name)
+	assert.Equal(t, "other.other.myindex", b.Config.Resources.VectorSearchIndexes["no_match"].Name)
+	assert.Empty(t, b.Config.Resources.VectorSearchIndexes["empty"].Name)
+	assert.Equal(t, "mycatalog.myschema", b.Config.Resources.VectorSearchIndexes["two_part"].Name)
+	assert.Nil(t, b.Config.Resources.VectorSearchIndexes["nil_index"])
+}
+
 // Nil and empty resources should not panic.
 func TestCaptureUCDependenciesNilResources(t *testing.T) {
 	b := &bundle.Bundle{
@@ -299,6 +339,7 @@ func TestCaptureUCDependenciesNilResources(t *testing.T) {
 				Pipelines:             map[string]*resources.Pipeline{"nil": nil, "empty": {}},
 				QualityMonitors:       map[string]*resources.QualityMonitor{"nil": nil, "empty": {}},
 				ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{"nil": nil, "empty": {}},
+				VectorSearchIndexes:   map[string]*resources.VectorSearchIndex{"nil": nil, "empty": {}},
 			},
 		},
 	}