From 17f0207d9df84a72af599d4d85d130882d342b32 Mon Sep 17 00:00:00 2001 From: Dan Noble Date: Fri, 17 Jul 2026 03:48:59 -0700 Subject: [PATCH] refactor(infra): deprecate namespace in favor of instance-name --- README.md | 2 +- infra/dcp/README.md | 12 +-- infra/dcp/main.tf | 5 +- infra/dcp/modules/auth/main.tf | 4 +- infra/dcp/modules/auth/variables.tf | 2 +- .../dcp/modules/datacommons_services/main.tf | 2 +- .../modules/datacommons_services/variables.tf | 2 +- infra/dcp/modules/ingestion/dataflow/main.tf | 4 +- .../modules/ingestion/dataflow/variables.tf | 2 +- .../modules/ingestion/helper_service/main.tf | 2 +- .../ingestion/helper_service/variables.tf | 2 +- .../ingestion/postprocessing_job/main.tf | 2 +- .../ingestion/postprocessing_job/variables.tf | 2 +- .../ingestion/preprocessing_job/main.tf | 2 +- .../ingestion/preprocessing_job/variables.tf | 2 +- infra/dcp/modules/ingestion/workflow/main.tf | 6 +- .../modules/ingestion/workflow/variables.tf | 4 +- infra/dcp/modules/redis/main.tf | 6 +- infra/dcp/modules/redis/variables.tf | 4 +- infra/dcp/modules/spanner/main.tf | 2 +- infra/dcp/modules/spanner/variables.tf | 2 +- infra/dcp/modules/stack/main.tf | 27 +++--- infra/dcp/modules/stack/variables.tf | 2 +- infra/dcp/modules/storage/main.tf | 2 +- infra/dcp/modules/storage/variables.tf | 2 +- infra/dcp/terraform.tfvars.template | 6 +- infra/dcp/variables.tf | 16 ++-- .../datacommons_admin/admin_cli.py | 86 +++++++++++-------- .../datacommons_admin/infra_templates.py | 2 +- .../datacommons-admin/tests/test_admin_cli.py | 40 +++++++++ packages/datacommons-cli/README.md | 2 +- 31 files changed, 161 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index 292b60a4..7c3ac842 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ uv run datacommons admin init The command will prompt for: - GCP project id -- namespace +- instance name - Data Commons API key It then creates a new folder with `main.tf`, `terraform.tfvars`, and a deployment `README.md`. diff --git a/infra/dcp/README.md b/infra/dcp/README.md index eada5b1b..79c92ebe 100644 --- a/infra/dcp/README.md +++ b/infra/dcp/README.md @@ -41,7 +41,7 @@ cp terraform.tfvars.template terraform.tfvars Edit `terraform.tfvars` and fill in at least the following required variables: * `project_id`: Your GCP Project ID. -* `namespace`: A unique identifier for resource naming (e.g., your name or team name). +* `instance_name`: A unique identifier for resource naming (e.g., your name or team name). ### 2. Run the Setup Script @@ -103,17 +103,17 @@ Upon successful apply, Terraform displays key endpoints and resource names: After successful deployment with `deploy_ingestion_workflow = true`, you can run the automated ingestion pipeline. ### Step 1: Upload your Schema file -Upload your custom graph nodes file (`.mcf`) to the provisioned ingestion bucket. By default, the bucket name format is: `-ingestion-bucket-`. +Upload your custom graph nodes file (`.mcf`) to the provisioned ingestion bucket. By default, the bucket name format is: `-ingestion-bucket-`. ```bash -gcloud storage cp path/to/your/sample.mcf gs://-ingestion-bucket-/imports/sample.mcf +gcloud storage cp path/to/your/sample.mcf gs://-ingestion-bucket-/imports/sample.mcf ``` ### Step 2: Trigger the Workflow Orchestrator Trigger the Cloud Workflow to start the Dataflow job that will read the file and insert it into Spanner. ```bash -gcloud workflows run -ingestion-orchestrator \ +gcloud workflows run -ingestion-orchestrator \ --project= \ --location= \ --data='{ @@ -121,8 +121,8 @@ gcloud workflows run -ingestion-orchestrator \ "region": "", "spannerInstanceId": "", "spannerDatabaseId": "", - "importList": "[{\"importName\": \"SampleTestCase\", \"graphPath\": \"gs://-ingestion-bucket-/imports/sample.mcf\"}]", - "tempLocation": "gs://-ingestion-bucket-/temp" + "importList": "[{\"importName\": \"SampleTestCase\", \"graphPath\": \"gs://-ingestion-bucket-/imports/sample.mcf\"}]", + "tempLocation": "gs://-ingestion-bucket-/temp" }' ``` diff --git a/infra/dcp/main.tf b/infra/dcp/main.tf index b35157cc..67751b06 100644 --- a/infra/dcp/main.tf +++ b/infra/dcp/main.tf @@ -59,13 +59,16 @@ resource "google_project_service" "apis" { } locals { + # For backward compatibility, fallback to namespace if instance_name is empty + effective_instance_name = var.instance_name != "" ? var.instance_name : var.namespace + # Redirect abandoned 'latest' alias to 'stable' for Dataflow templates df_template_version = var.dcp_version == "latest" ? "stable" : var.dcp_version global_config = { project_id = var.project_id region = var.region - namespace = var.namespace + instance_name = local.effective_instance_name stateful_deletion_protection = var.stateful_deletion_protection stateless_deletion_protection = var.stateless_deletion_protection } diff --git a/infra/dcp/modules/auth/main.tf b/infra/dcp/modules/auth/main.tf index be78a5e1..1421e300 100644 --- a/infra/dcp/modules/auth/main.tf +++ b/infra/dcp/modules/auth/main.tf @@ -1,5 +1,5 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" google_maps_api_key_value = var.google_maps_api_key != null ? var.google_maps_api_key : try(google_apikeys_key.maps_api_key[0].key_string, "") } @@ -24,7 +24,7 @@ resource "google_apikeys_key" "maps_api_key" { count = var.google_maps_api_key == null && var.create_google_maps_key ? 1 : 0 name = "${local.name_prefix}dc-google-maps-key-${random_id.api_key_suffix.hex}" - display_name = "Maps API Key for ${var.namespace != "" ? var.namespace : "Data Commons"}" + display_name = "Maps API Key for ${var.instance_name != "" ? var.instance_name : "Data Commons"}" project = var.project_id restrictions { diff --git a/infra/dcp/modules/auth/variables.tf b/infra/dcp/modules/auth/variables.tf index 09d894b3..2396a7f4 100644 --- a/infra/dcp/modules/auth/variables.tf +++ b/infra/dcp/modules/auth/variables.tf @@ -2,7 +2,7 @@ variable "project_id" { type = string } -variable "namespace" { +variable "instance_name" { type = string } diff --git a/infra/dcp/modules/datacommons_services/main.tf b/infra/dcp/modules/datacommons_services/main.tf index a95ae93e..5febde14 100644 --- a/infra/dcp/modules/datacommons_services/main.tf +++ b/infra/dcp/modules/datacommons_services/main.tf @@ -1,5 +1,5 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" } resource "google_service_account" "serving_sa" { diff --git a/infra/dcp/modules/datacommons_services/variables.tf b/infra/dcp/modules/datacommons_services/variables.tf index dff9a37a..b1da6c49 100644 --- a/infra/dcp/modules/datacommons_services/variables.tf +++ b/infra/dcp/modules/datacommons_services/variables.tf @@ -5,7 +5,7 @@ variable "project_id" { type = string } -variable "namespace" { +variable "instance_name" { type = string } diff --git a/infra/dcp/modules/ingestion/dataflow/main.tf b/infra/dcp/modules/ingestion/dataflow/main.tf index 31a1fa33..ad4c7167 100644 --- a/infra/dcp/modules/ingestion/dataflow/main.tf +++ b/infra/dcp/modules/ingestion/dataflow/main.tf @@ -1,6 +1,6 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" - display_name_prefix = var.namespace != "" ? "(${var.namespace}) " : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" + display_name_prefix = var.instance_name != "" ? "(${var.instance_name}) " : "" } resource "google_service_account" "dataflow_sa" { diff --git a/infra/dcp/modules/ingestion/dataflow/variables.tf b/infra/dcp/modules/ingestion/dataflow/variables.tf index e0cd1066..a95b0a48 100644 --- a/infra/dcp/modules/ingestion/dataflow/variables.tf +++ b/infra/dcp/modules/ingestion/dataflow/variables.tf @@ -6,7 +6,7 @@ variable "project_id" { type = string } -variable "namespace" { +variable "instance_name" { type = string } diff --git a/infra/dcp/modules/ingestion/helper_service/main.tf b/infra/dcp/modules/ingestion/helper_service/main.tf index 3e08f3e4..c4819b10 100644 --- a/infra/dcp/modules/ingestion/helper_service/main.tf +++ b/infra/dcp/modules/ingestion/helper_service/main.tf @@ -1,5 +1,5 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" } resource "google_service_account" "helper_sa" { diff --git a/infra/dcp/modules/ingestion/helper_service/variables.tf b/infra/dcp/modules/ingestion/helper_service/variables.tf index cdf98061..3ff1db00 100644 --- a/infra/dcp/modules/ingestion/helper_service/variables.tf +++ b/infra/dcp/modules/ingestion/helper_service/variables.tf @@ -6,7 +6,7 @@ variable "project_id" { type = string } -variable "namespace" { +variable "instance_name" { type = string } diff --git a/infra/dcp/modules/ingestion/postprocessing_job/main.tf b/infra/dcp/modules/ingestion/postprocessing_job/main.tf index 77a82f4a..f9de83e2 100644 --- a/infra/dcp/modules/ingestion/postprocessing_job/main.tf +++ b/infra/dcp/modules/ingestion/postprocessing_job/main.tf @@ -1,5 +1,5 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" } resource "google_service_account" "postprocessing_sa" { diff --git a/infra/dcp/modules/ingestion/postprocessing_job/variables.tf b/infra/dcp/modules/ingestion/postprocessing_job/variables.tf index dd50f053..41bb4fe0 100644 --- a/infra/dcp/modules/ingestion/postprocessing_job/variables.tf +++ b/infra/dcp/modules/ingestion/postprocessing_job/variables.tf @@ -1,5 +1,5 @@ variable "project_id" { type = string } -variable "namespace" { type = string } +variable "instance_name" { type = string } variable "region" { type = string } variable "stateless_deletion_protection" { type = bool diff --git a/infra/dcp/modules/ingestion/preprocessing_job/main.tf b/infra/dcp/modules/ingestion/preprocessing_job/main.tf index 67d58af4..ba2df454 100644 --- a/infra/dcp/modules/ingestion/preprocessing_job/main.tf +++ b/infra/dcp/modules/ingestion/preprocessing_job/main.tf @@ -1,5 +1,5 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" } resource "google_service_account" "preprocessing_sa" { diff --git a/infra/dcp/modules/ingestion/preprocessing_job/variables.tf b/infra/dcp/modules/ingestion/preprocessing_job/variables.tf index 36b60041..8264cdf3 100644 --- a/infra/dcp/modules/ingestion/preprocessing_job/variables.tf +++ b/infra/dcp/modules/ingestion/preprocessing_job/variables.tf @@ -1,5 +1,5 @@ variable "project_id" { type = string } -variable "namespace" { type = string } +variable "instance_name" { type = string } variable "region" { type = string } variable "stateless_deletion_protection" { type = bool diff --git a/infra/dcp/modules/ingestion/workflow/main.tf b/infra/dcp/modules/ingestion/workflow/main.tf index 34f0bc78..1542abff 100644 --- a/infra/dcp/modules/ingestion/workflow/main.tf +++ b/infra/dcp/modules/ingestion/workflow/main.tf @@ -1,7 +1,7 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" should_run_postprocessing = var.enable_bigquery_postprocessing || var.enable_embeddings_generation - clean_namespace_prefix = var.namespace != "" ? "${replace(lower(var.namespace), "_", "-")}-" : "" + clean_namespace_prefix = var.instance_name != "" ? "${replace(lower(var.instance_name), "_", "-")}-" : "" } resource "google_service_account" "workflow_sa" { @@ -31,6 +31,8 @@ resource "google_workflows_workflow" "ingestion_orchestrator" { dataflow_subnetwork = var.dataflow_subnetwork embeddings_timeout = var.embeddings_timeout clean_namespace_prefix = local.clean_namespace_prefix + + enable_redis_cache_clearing = var.enable_redis_cache_clearing preprocessing_job_name = var.preprocessing_job_name postprocessing_job_name = var.postprocessing_job_name diff --git a/infra/dcp/modules/ingestion/workflow/variables.tf b/infra/dcp/modules/ingestion/workflow/variables.tf index 04c5a752..5d8e4fff 100644 --- a/infra/dcp/modules/ingestion/workflow/variables.tf +++ b/infra/dcp/modules/ingestion/workflow/variables.tf @@ -2,7 +2,7 @@ variable "deploy" { type = bool } -variable "namespace" { +variable "instance_name" { type = string } @@ -47,6 +47,8 @@ variable "embeddings_timeout" { default = 1800 } + + variable "ingestion_helper_service_name" { type = string description = "Name of the ingestion helper Cloud Run service" diff --git a/infra/dcp/modules/redis/main.tf b/infra/dcp/modules/redis/main.tf index 4f80a250..7843d6d5 100644 --- a/infra/dcp/modules/redis/main.tf +++ b/infra/dcp/modules/redis/main.tf @@ -1,10 +1,10 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" - display_name_prefix = var.namespace != "" ? "(${var.namespace}) " : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" + display_name_prefix = var.instance_name != "" ? "(${var.instance_name}) " : "" } resource "google_redis_instance" "redis_instance" { - name = var.instance_name != "" ? "${local.name_prefix}${var.instance_name}" : "${local.name_prefix}dc-redis-instance" + name = var.redis_instance_name != "" ? "${local.name_prefix}${var.redis_instance_name}" : "${local.name_prefix}dc-redis-instance" memory_size_gb = var.memory_size_gb tier = var.tier region = var.region diff --git a/infra/dcp/modules/redis/variables.tf b/infra/dcp/modules/redis/variables.tf index 6c88e7a7..a74e1ea1 100644 --- a/infra/dcp/modules/redis/variables.tf +++ b/infra/dcp/modules/redis/variables.tf @@ -1,4 +1,4 @@ -variable "namespace" { +variable "instance_name" { type = string } @@ -6,7 +6,7 @@ variable "region" { type = string } -variable "instance_name" { +variable "redis_instance_name" { type = string default = "" } diff --git a/infra/dcp/modules/spanner/main.tf b/infra/dcp/modules/spanner/main.tf index 85889e8f..e17bdeed 100644 --- a/infra/dcp/modules/spanner/main.tf +++ b/infra/dcp/modules/spanner/main.tf @@ -1,5 +1,5 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" effective_instance_id = var.create_instance ? (var.instance_id != "" ? "${local.name_prefix}${var.instance_id}" : "${local.name_prefix}dc-instance") : var.instance_id effective_database_id = var.create_database ? (var.database_id != "" ? "${local.name_prefix}${var.database_id}" : "${local.name_prefix}dc-db") : var.database_id } diff --git a/infra/dcp/modules/spanner/variables.tf b/infra/dcp/modules/spanner/variables.tf index a47c66eb..d702fbdf 100644 --- a/infra/dcp/modules/spanner/variables.tf +++ b/infra/dcp/modules/spanner/variables.tf @@ -3,7 +3,7 @@ variable "project_id" { description = "GCP Project ID" } -variable "namespace" { +variable "instance_name" { type = string } diff --git a/infra/dcp/modules/stack/main.tf b/infra/dcp/modules/stack/main.tf index a6e75975..4f83385d 100644 --- a/infra/dcp/modules/stack/main.tf +++ b/infra/dcp/modules/stack/main.tf @@ -80,7 +80,7 @@ module "spanner" { count = var.spanner_config.enable ? 1 : 0 project_id = var.global.project_id - namespace = var.global.namespace + instance_name = var.global.instance_name region = var.global.region create_instance = var.spanner_config.create_instance create_database = var.spanner_config.create_db @@ -106,8 +106,8 @@ module "storage" { stateful_deletion_protection = var.global.stateful_deletion_protection # Shared vars - project_id = var.global.project_id - namespace = var.global.namespace + project_id = var.global.project_id + instance_name = var.global.instance_name } module "ingestion_preprocessing_job" { @@ -115,7 +115,7 @@ module "ingestion_preprocessing_job" { count = var.ingestion_config.enable_ingestion ? 1 : 0 project_id = var.global.project_id - namespace = var.global.namespace + instance_name = var.global.instance_name region = var.global.region stateless_deletion_protection = var.global.stateless_deletion_protection image = var.ingestion_config.preprocessing_job_image @@ -149,7 +149,7 @@ module "ingestion_postprocessing_job" { count = var.ingestion_config.enable_ingestion ? 1 : 0 project_id = var.global.project_id - namespace = var.global.namespace + instance_name = var.global.instance_name region = var.global.region stateless_deletion_protection = var.global.stateless_deletion_protection image = var.ingestion_config.postprocessing_job_image @@ -171,7 +171,7 @@ module "ingestion_dataflow" { deploy = var.ingestion_config.enable_ingestion project_id = var.global.project_id - namespace = var.global.namespace + instance_name = var.global.instance_name ingestion_bucket_name = module.storage.artifacts_bucket_name use_spanner = var.spanner_config.enable } @@ -181,7 +181,7 @@ module "ingestion_helper_service" { deploy = var.ingestion_config.enable_ingestion project_id = var.global.project_id - namespace = var.global.namespace + instance_name = var.global.instance_name region = var.global.region stateless_deletion_protection = var.global.stateless_deletion_protection # Use index [0] because module.spanner is conditional. Fallback to empty string if disabled. @@ -207,7 +207,7 @@ module "ingestion_workflow" { source = "../ingestion/workflow" deploy = var.ingestion_config.enable_ingestion - namespace = var.global.namespace + instance_name = var.global.instance_name region = var.global.region stateless_deletion_protection = var.global.stateless_deletion_protection project_id = var.global.project_id @@ -216,7 +216,7 @@ module "ingestion_workflow" { dataflow_service_account_email = module.ingestion_dataflow.service_account_email enable_bigquery_postprocessing = var.ingestion_config.workflow_enable_bigquery_postprocessing enable_embeddings_generation = var.spanner_config.enable_embeddings_generation - ingestion_helper_service_name = "${var.global.namespace != "" ? "${var.global.namespace}-" : ""}dc-ingestion-helper" + ingestion_helper_service_name = "${var.global.instance_name != "" ? "${var.global.instance_name}-" : ""}dc-ingestion-helper" enable_redis_cache_clearing = var.redis_config.enable ingestion_artifacts_path = "${var.ingestion_config.ingestion_artifacts_path}/metadata" dataflow_ip_configuration = var.ingestion_config.dataflow_ip_configuration @@ -228,13 +228,14 @@ module "ingestion_workflow" { depends_on = [module.ingestion_helper_service] } + module "redis" { source = "../redis" count = var.redis_config.enable ? 1 : 0 - namespace = var.global.namespace + instance_name = var.global.instance_name region = var.global.region - instance_name = var.redis_config.instance_name + redis_instance_name = var.redis_config.instance_name memory_size_gb = var.redis_config.memory_size_gb tier = var.redis_config.tier location_id = var.redis_config.location_id @@ -249,7 +250,7 @@ module "auth" { source = "../auth" project_id = var.global.project_id - namespace = var.global.namespace + instance_name = var.global.instance_name dc_api_key = var.auth_config.google_datacommons_api_key google_maps_api_key = var.auth_config.google_maps_api_key create_google_maps_key = var.auth_config.create_google_maps_key @@ -260,7 +261,7 @@ module "datacommons_services" { count = var.datacommons_services_config.enable ? 1 : 0 project_id = var.global.project_id - namespace = var.global.namespace + instance_name = var.global.instance_name region = var.global.region stateless_deletion_protection = var.global.stateless_deletion_protection image = var.datacommons_services_config.image diff --git a/infra/dcp/modules/stack/variables.tf b/infra/dcp/modules/stack/variables.tf index 90210c48..4086b03d 100644 --- a/infra/dcp/modules/stack/variables.tf +++ b/infra/dcp/modules/stack/variables.tf @@ -2,7 +2,7 @@ variable "global" { type = object({ project_id = string region = string - namespace = string + instance_name = string stateful_deletion_protection = bool stateless_deletion_protection = bool }) diff --git a/infra/dcp/modules/storage/main.tf b/infra/dcp/modules/storage/main.tf index 6f2afde2..d1cb9d84 100644 --- a/infra/dcp/modules/storage/main.tf +++ b/infra/dcp/modules/storage/main.tf @@ -1,5 +1,5 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" artifacts_bucket_name = var.artifacts_bucket_name != "" ? var.artifacts_bucket_name : "${local.name_prefix}dc-artifacts-${var.project_id}" } diff --git a/infra/dcp/modules/storage/variables.tf b/infra/dcp/modules/storage/variables.tf index a3512823..42f4104c 100644 --- a/infra/dcp/modules/storage/variables.tf +++ b/infra/dcp/modules/storage/variables.tf @@ -2,7 +2,7 @@ variable "project_id" { type = string } -variable "namespace" { +variable "instance_name" { type = string } diff --git a/infra/dcp/terraform.tfvars.template b/infra/dcp/terraform.tfvars.template index 948efda0..bedc6287 100644 --- a/infra/dcp/terraform.tfvars.template +++ b/infra/dcp/terraform.tfvars.template @@ -6,9 +6,9 @@ # ============================================================================= # Global Configuration # ============================================================================= -project_id = "$$PROJECT_ID$$" -namespace = "$$NAMESPACE$$" -region = "us-central1" +project_id = "$$PROJECT_ID$$" +instance_name = "$$INSTANCE_NAME$$" +region = "us-central1" # Deletion protection for stateful resources holding persistent data # (Spanner and Cloud Storage). diff --git a/infra/dcp/variables.tf b/infra/dcp/variables.tf index 0af5cd60..2e4a216b 100644 --- a/infra/dcp/variables.tf +++ b/infra/dcp/variables.tf @@ -13,12 +13,18 @@ variable "region" { default = "us-central1" } -variable "namespace" { +variable "instance_name" { description = "A unique identifier used as a prefix for resource naming. This prevents naming conflicts when deploying multiple isolated environments (like dev, staging, or feature branches) within the same GCP project." type = string default = "" } +variable "namespace" { + description = "DEPRECATED: Use instance_name instead. A unique identifier used as a prefix for resource naming." + type = string + default = "" +} + variable "stateful_deletion_protection" { description = "Enable deletion protection for stateful resources (Spanner, GCS) to prevent data loss." type = bool @@ -82,7 +88,7 @@ variable "storage_create_artifacts_bucket" { } variable "storage_artifacts_bucket_name" { - description = "The name of the unified GCS bucket for artifacts (serving and ingestion). If not provided, a name will be automatically generated following the pattern [namespace-]dc-artifacts-[project_id]" + description = "The name of the unified GCS bucket for artifacts (serving and ingestion). If not provided, a name will be automatically generated following the pattern [instance-name-]dc-artifacts-[project_id]" type = string default = "" } @@ -98,7 +104,7 @@ variable "enable_redis" { } variable "redis_instance_name" { - description = "The name of the Redis instance. If not provided, a name will be automatically generated following the pattern [namespace-]dc-redis-instance" + description = "The name of the Redis instance. If not provided, a name will be automatically generated following the pattern [instance-name-]dc-redis-instance" type = string default = "" } @@ -168,13 +174,13 @@ variable "spanner_create_database" { } variable "spanner_instance_id" { - description = "The ID of the Spanner instance. If not provided, a name will be automatically generated following the pattern [namespace-]dc-instance" + description = "The ID of the Spanner instance. If not provided, a name will be automatically generated following the pattern [instance-name-]dc-instance" type = string default = "" } variable "spanner_database_id" { - description = "The ID of the Spanner database. If not provided, a name will be automatically generated following the pattern [namespace-]dc-db" + description = "The ID of the Spanner database. If not provided, a name will be automatically generated following the pattern [instance-name-]dc-db" type = string default = "" } diff --git a/packages/datacommons-admin/datacommons_admin/admin_cli.py b/packages/datacommons-admin/datacommons_admin/admin_cli.py index d2dd327e..9a6e591e 100644 --- a/packages/datacommons-admin/datacommons_admin/admin_cli.py +++ b/packages/datacommons-admin/datacommons_admin/admin_cli.py @@ -35,14 +35,14 @@ GITHUB_REPO_URL = "https://github.com/datacommonsorg/datacommons.git" -def _get_default_bucket_name(namespace: str, project_id: str) -> str: +def _get_default_bucket_name(instance_name: str, project_id: str) -> str: """Returns the default Google Cloud Storage bucket name for Terraform state.""" - return f"tf-state-{namespace}-{project_id}" + return f"tf-state-{instance_name}-{project_id}" -def _get_default_state_prefix(namespace: str) -> str: +def _get_default_state_prefix(instance_name: str) -> str: """Returns the default Google Cloud Storage object prefix for Terraform state.""" - return f"terraform/state/{namespace}" + return f"terraform/state/{instance_name}" def _log_resolved_value(label: str, value: str, is_default: bool, indent: int = 2): @@ -229,30 +229,30 @@ def admin() -> None: """Manage a Data Commons Platform instance in Google Cloud""" -def _validate_namespace(ns: str) -> Tuple[bool, str]: - if not ns: - return False, "Namespace must not be empty." - if len(ns) > 16: +def _validate_instance_name(name: str) -> Tuple[bool, str]: + if not name: + return False, "Instance name must not be empty." + if len(name) > 16: return ( False, - f"Namespace must be 16 characters or less (currently {len(ns)} characters).", + f"Instance name must be 16 characters or less (currently {len(name)} characters).", ) - if not re.match(r"^[a-z]([-a-z0-9]*[a-z0-9])?$", ns): + if not re.match(r"^[a-z]([-a-z0-9]*[a-z0-9])?$", name): return ( False, - "Namespace must start with a lowercase letter, end with a lowercase letter or number, and contain only lowercase alphanumeric characters and dashes.", + "Instance name must start with a lowercase letter, end with a lowercase letter or number, and contain only lowercase alphanumeric characters and dashes.", ) return True, "" def _resolve_project_config( - project_id: str, namespace: str, force: bool + project_id: str, instance_name: str, force: bool ) -> Tuple[str, str, Path]: - """Resolves project ID and namespace, and determines target directory.""" + """Resolves project ID and instance name, and determines target directory.""" if project_id: _log_resolved_value("Project ID", project_id, is_default=False) - if namespace: - _log_resolved_value("Namespace", namespace, is_default=False) + if instance_name: + _log_resolved_value("Instance Name", instance_name, is_default=False) resolved_project_id = project_id.strip() if not resolved_project_id: @@ -262,34 +262,34 @@ def _resolve_project_config( if not resolved_project_id: raise click.ClickException("GCP project ID must not be empty.") - resolved_namespace = namespace.strip() - if resolved_namespace: - is_valid, err_msg = _validate_namespace(resolved_namespace) + resolved_instance_name = instance_name.strip() + if resolved_instance_name: + is_valid, err_msg = _validate_instance_name(resolved_instance_name) if not is_valid: raise click.ClickException(err_msg) while True: - if not resolved_namespace: - resolved_namespace = _prompt("Namespace", type=str).strip() - is_valid, err_msg = _validate_namespace(resolved_namespace) + if not resolved_instance_name: + resolved_instance_name = _prompt("Instance Name", type=str).strip() + is_valid, err_msg = _validate_instance_name(resolved_instance_name) if not is_valid: click.secho(f"Error: {err_msg}", fg="red") - resolved_namespace = "" + resolved_instance_name = "" continue - target_dir = Path.cwd() / resolved_namespace + target_dir = Path.cwd() / resolved_instance_name if target_dir.exists() and not force: click.secho( - f"Error: Folder '{resolved_namespace}' already exists locally. " - "Please specify a different namespace, or use --force to overwrite.", + f"Error: Folder '{resolved_instance_name}' already exists locally. " + "Please specify a different instance name, or use --force to overwrite.", fg="yellow", ) - resolved_namespace = "" + resolved_instance_name = "" continue break - return resolved_project_id, resolved_namespace, target_dir + return resolved_project_id, resolved_instance_name, target_dir def _check_existing_files(target_dir: Path, use_remote_state: bool, force: bool): @@ -315,7 +315,7 @@ def _check_existing_files(target_dir: Path, use_remote_state: bool, force: bool) def _setup_dcp_config_dir( target_dir: Path, project_id: str, - namespace: str, + instance_name: str, bucket_name: str, tf_state_prefix: str, dc_api_key: str, @@ -364,7 +364,8 @@ def _setup_dcp_config_dir( # Modify tfvars_example with actual values tfvars_content = tfvars_example tfvars_content = tfvars_content.replace('"$$PROJECT_ID$$"', f'"{project_id}"') - tfvars_content = tfvars_content.replace('"$$NAMESPACE$$"', f'"{namespace}"') + tfvars_content = tfvars_content.replace('"$$NAMESPACE$$"', f'"{instance_name}"') + tfvars_content = tfvars_content.replace('"$$INSTANCE_NAME$$"', f'"{instance_name}"') if api_key: tfvars_content = tfvars_content.replace('"$$DC_API_KEY$$"', f'"{api_key}"') @@ -401,7 +402,14 @@ def _setup_dcp_config_dir( help="Google Cloud Platform project ID used for all resources related to your Data Commons instance.", ) @click.option( - "--namespace", default="", help="Namespace prefix for provisioned resources." + "--instance-name", + default="", + help="Unique identifier used as a prefix for resource naming (replaces --namespace).", +) +@click.option( + "--namespace", + default="", + help="DEPRECATED: Use --instance-name instead.", ) @click.option("--dc-api-key", default="", help="Data Commons API key.") @click.option( @@ -432,11 +440,12 @@ def _setup_dcp_config_dir( @click.option( "--tf-state-prefix", default="", - help="Google Cloud Storage object prefix for Terraform state file (default: terraform/state/{namespace}).", + help="Google Cloud Storage object prefix for Terraform state file (default: terraform/state/{instance_name}).", ) def init( project_id: str, namespace: str, + instance_name: str, dc_api_key: str, tf_git_ref: str, force: bool, @@ -448,11 +457,14 @@ def init( """Initialize Terraform scaffolding for Data Commons administration/infrastructure.""" click.secho("Data Commons Admin Init", fg="cyan", bold=True) + # For backward compatibility, fallback to namespace if instance_name is empty + effective_instance_name = instance_name.strip() or namespace.strip() + # 1. Project Configs click.secho("\n[Project configuration]", fg="cyan", bold=True) click.secho("Configuring project settings...", fg="bright_black") - resolved_project_id, resolved_namespace, target_dir = _resolve_project_config( - project_id, namespace, force + resolved_project_id, resolved_instance_name, target_dir = _resolve_project_config( + project_id, effective_instance_name, force ) # 2. Terraform Setup @@ -467,7 +479,7 @@ def init( resolved_bucket_name = ( _configure_remote_state( resolved_project_id, - resolved_namespace, + resolved_instance_name, tf_state_bucket, tf_state_bucket_location, ) @@ -476,7 +488,7 @@ def init( ) resolved_tf_state_prefix = tf_state_prefix.strip() or _get_default_state_prefix( - resolved_namespace + resolved_instance_name ) # 3. DCP config dir setup @@ -485,7 +497,7 @@ def init( _setup_dcp_config_dir( target_dir, resolved_project_id, - resolved_namespace, + resolved_instance_name, resolved_bucket_name, resolved_tf_state_prefix, dc_api_key, @@ -494,7 +506,7 @@ def init( ) click.secho( - f"Customize variables in {resolved_namespace}/terraform.tfvars as needed.", + f"Customize variables in {resolved_instance_name}/terraform.tfvars as needed.", fg="green", ) click.secho( diff --git a/packages/datacommons-admin/datacommons_admin/infra_templates.py b/packages/datacommons-admin/datacommons_admin/infra_templates.py index 4f5999f0..05a2d2be 100644 --- a/packages/datacommons-admin/datacommons_admin/infra_templates.py +++ b/packages/datacommons-admin/datacommons_admin/infra_templates.py @@ -36,7 +36,7 @@ ## Configure Variables -Set environment-specific values in `terraform.tfvars` (for example `project_id`, `namespace`, and `dc_api_key`), and update module arguments in `main.tf` if you want to enable or tune additional features. +Set environment-specific values in `terraform.tfvars` (for example `project_id`, `instance_name`, and `dc_api_key`), and update module arguments in `main.tf` if you want to enable or tune additional features. ## Learn More About Variables diff --git a/packages/datacommons-admin/tests/test_admin_cli.py b/packages/datacommons-admin/tests/test_admin_cli.py index 2827a40a..1b8a5866 100644 --- a/packages/datacommons-admin/tests/test_admin_cli.py +++ b/packages/datacommons-admin/tests/test_admin_cli.py @@ -66,6 +66,46 @@ def test_init_success_with_options( assert 'dc_api_key = "test-key"' in tfvars_content +@patch("datacommons_admin.admin_cli._get_github_templates") +def test_init_success_with_instance_name( + mock_get_templates, runner: CliRunner, tmp_path: Path +) -> None: + mock_get_templates.return_value = ( + 'variable "test" {}', + 'module "stack" {\n source = "./modules/stack"\n}', + 'output "test" {}', + 'project_id = "$$PROJECT_ID$$"\ninstance_name = "$$INSTANCE_NAME$$"\n# dc_api_key = "$$DC_API_KEY$$"', + ) + with runner.isolated_filesystem(temp_dir=tmp_path): + result = runner.invoke( + admin, + [ + "init", + "--project-id", + "test-project", + "--instance-name", + "test-inst", + "--dc-api-key", + "test-key", + "--no-tf-remote-state", + ], + ) + assert result.exit_code == 0 + assert "Downloaded and populated Terraform templates." in result.output + + target_dir = Path.cwd() / "test-inst" + assert target_dir.exists() + assert (target_dir / "main.tf").exists() + assert (target_dir / "terraform.tfvars").exists() + assert (target_dir / "README.md").exists() + assert not (target_dir / "backend.tf").exists() + + tfvars_content = (target_dir / "terraform.tfvars").read_text() + assert 'project_id = "test-project"' in tfvars_content + assert 'instance_name = "test-inst"' in tfvars_content + assert 'dc_api_key = "test-key"' in tfvars_content + + @patch("datacommons_admin.admin_cli._get_github_templates") def test_init_success_with_prompts( mock_get_templates, runner: CliRunner, tmp_path: Path diff --git a/packages/datacommons-cli/README.md b/packages/datacommons-cli/README.md index ac3e0f3a..f868da8f 100644 --- a/packages/datacommons-cli/README.md +++ b/packages/datacommons-cli/README.md @@ -95,7 +95,7 @@ datacommons admin [COMMAND] --help 1. Initialize your project configuration and scaffold Terraform templates: ```bash - datacommons admin init --project-id my-gcp-project --namespace prod + datacommons admin init --project-id my-gcp-project --instance-name prod ``` 2. Change directories to the scaffolded directory (e.g. `cd prod`).