diff --git a/.nextchanges/bundles/vector-search-index-uc-deps.md b/.nextchanges/bundles/vector-search-index-uc-deps.md new file mode 100644 index 00000000000..df032138610 --- /dev/null +++ b/.nextchanges/bundles/vector-search-index-uc-deps.md @@ -0,0 +1 @@ +* Capture the implicit dependency a vector search index has on a catalog or schema defined in the same bundle, so the catalog and schema are deployed first. ([#6655](https://github.com/databricks/cli/pull/6655)) diff --git a/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/databricks.yml b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/databricks.yml new file mode 100644 index 00000000000..97c420f1a52 --- /dev/null +++ b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/databricks.yml @@ -0,0 +1,25 @@ +bundle: + name: test-bundle + +# Tests implicit dependency detection for vector search indexes: +# - name (format "catalog.schema.index") should resolve the catalog and schema parts +resources: + catalogs: + my_catalog: + name: mycatalog + schemas: + my_schema: + catalog_name: mycatalog + name: myschema + vector_search_indexes: + my_index: + name: mycatalog.myschema.myindex + endpoint_name: my-endpoint + primary_key: id + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"id":"integer","vector":"array"}' + +targets: + dev: + mode: development diff --git a/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/out.test.toml new file mode 100644 index 00000000000..e1af1a235ad --- /dev/null +++ b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/output.txt b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/output.txt new file mode 100644 index 00000000000..862b7e25ea5 --- /dev/null +++ b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/output.txt @@ -0,0 +1,26 @@ + +>>> [CLI] bundle validate -t dev -o json +{ + "catalogs": { + "my_catalog": { + "name": "mycatalog" + } + }, + "schemas": { + "my_schema": { + "catalog_name": "${resources.catalogs.my_catalog.name}", + "name": "dev_[USERNAME]_myschema" + } + }, + "vector_search_indexes": { + "my_index": { + "direct_access_index_spec": { + "schema_json": "{\"id\":\"integer\",\"vector\":\"array\"}" + }, + "endpoint_name": "my-endpoint", + "index_type": "DIRECT_ACCESS", + "name": "${resources.catalogs.my_catalog.name}.${resources.schemas.my_schema.name}.myindex", + "primary_key": "id" + } + } +} diff --git a/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/script b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/script new file mode 100644 index 00000000000..30cb3ec2e5d --- /dev/null +++ b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/script @@ -0,0 +1 @@ +trace $CLI bundle validate -t dev -o json | jq .resources diff --git a/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/test.toml b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/test.toml new file mode 100644 index 00000000000..a030353d571 --- /dev/null +++ b/acceptance/bundle/resource_deps/implicit_deps_vector_search_index/test.toml @@ -0,0 +1 @@ +RecordRequests = false diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go index 1373978bfb9..61b7467df9f 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go @@ -13,13 +13,13 @@ import ( type captureUCDependencies struct{} // If a user defines a UC schema in the bundle, they can refer to it in SDP pipelines, -// UC Volumes, Registered Models, Quality Monitors, or Model Serving Endpoints using the -// `${resources.schemas..name}` syntax. Using this syntax allows TF to capture -// the deploy time dependency this resource has on the schema and deploy changes to the -// schema before deploying the dependent resource. +// UC Volumes, Registered Models, Quality Monitors, Model Serving Endpoints, or Vector Search +// Indexes using the `${resources.schemas..name}` syntax. Using this syntax allows +// TF to capture the deploy time dependency this resource has on the schema and deploy changes +// to the schema before deploying the dependent resource. // // Similarly, if a user defines a UC catalog in the bundle, they can refer to it in UC schemas, -// UC Volumes, Registered Models, or Model Serving Endpoints using the +// UC Volumes, Registered Models, Model Serving Endpoints, or Vector Search Indexes using the // `${resources.catalogs..name}` syntax. This captures the deploy time // dependency the resource has on the catalog. // @@ -155,6 +155,18 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia qm.OutputSchemaName = resolved } } + for _, idx := range b.Config.Resources.VectorSearchIndexes { + if idx == nil { + continue + } + // Name is a three-part "catalog.schema.index" UC identifier. + parts := strings.SplitN(idx.Name, ".", 3) + if len(parts) != 3 { + continue + } + catalogName, schemaName := parts[0], parts[1] + idx.Name = resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) + "." + parts[2] + } for _, mse := range b.Config.Resources.ModelServingEndpoints { if mse == nil { continue diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go index a99db361848..db4f888c7c4 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -136,6 +137,11 @@ func TestCaptureUCDependencies(t *testing.T) { Parent: "schemas/mycatalog.myschema", ModelServiceId: "myservice", }}, }, + VectorSearchIndexes: map[string]*resources.VectorSearchIndex{ + "my_index": {CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "mycatalog.myschema.myindex", + }}, + }, }, }, } @@ -171,6 +177,9 @@ func TestCaptureUCDependencies(t *testing.T) { // Model service (compound "schemas/{catalog}.{schema}" parent field). assert.Equal(t, "schemas/"+catalogRef+"."+schemaRef, b.Config.Resources.ModelServices["my_model_service"].Parent) + + // Vector search index (three-part "catalog.schema.index" name). + assert.Equal(t, catalogRef+"."+schemaRef+".myindex", b.Config.Resources.VectorSearchIndexes["my_index"].Name) } // Pipeline schema and target are mutually exclusive; only the populated field @@ -287,6 +296,37 @@ func TestCaptureUCDependenciesModelServingEndpointEdgeCases(t *testing.T) { assert.Nil(t, b.Config.Resources.ModelServingEndpoints["nil_endpoint"]) } +func TestCaptureUCDependenciesVectorSearchIndexEdgeCases(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Catalogs: map[string]*resources.Catalog{ + "my_catalog": {CreateCatalog: catalog.CreateCatalog{Name: "mycatalog"}}, + }, + Schemas: map[string]*resources.Schema{ + "my_schema": {CreateSchema: catalog.CreateSchema{CatalogName: "mycatalog", Name: "myschema"}}, + }, + VectorSearchIndexes: map[string]*resources.VectorSearchIndex{ + "catalog_only": {CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{Name: "mycatalog.other.myindex"}}, + "no_match": {CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{Name: "other.other.myindex"}}, + "empty": {CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{Name: ""}}, + "two_part": {CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{Name: "mycatalog.myschema"}}, + "nil_index": nil, + }, + }, + }, + } + + d := bundle.Apply(t.Context(), b, CaptureUCDependencies()) + require.Nil(t, d) + + assert.Equal(t, "${resources.catalogs.my_catalog.name}.other.myindex", b.Config.Resources.VectorSearchIndexes["catalog_only"].Name) + assert.Equal(t, "other.other.myindex", b.Config.Resources.VectorSearchIndexes["no_match"].Name) + assert.Empty(t, b.Config.Resources.VectorSearchIndexes["empty"].Name) + assert.Equal(t, "mycatalog.myschema", b.Config.Resources.VectorSearchIndexes["two_part"].Name) + assert.Nil(t, b.Config.Resources.VectorSearchIndexes["nil_index"]) +} + // Nil and empty resources should not panic. func TestCaptureUCDependenciesNilResources(t *testing.T) { b := &bundle.Bundle{ @@ -299,6 +339,7 @@ func TestCaptureUCDependenciesNilResources(t *testing.T) { Pipelines: map[string]*resources.Pipeline{"nil": nil, "empty": {}}, QualityMonitors: map[string]*resources.QualityMonitor{"nil": nil, "empty": {}}, ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{"nil": nil, "empty": {}}, + VectorSearchIndexes: map[string]*resources.VectorSearchIndex{"nil": nil, "empty": {}}, }, }, }