diff --git a/README.md b/README.md index 292b60a4..7c3ac842 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ uv run datacommons admin init The command will prompt for: - GCP project id -- namespace +- instance name - Data Commons API key It then creates a new folder with `main.tf`, `terraform.tfvars`, and a deployment `README.md`. diff --git a/infra/dcp/README.md b/infra/dcp/README.md index eada5b1b..79c92ebe 100644 --- a/infra/dcp/README.md +++ b/infra/dcp/README.md @@ -41,7 +41,7 @@ cp terraform.tfvars.template terraform.tfvars Edit `terraform.tfvars` and fill in at least the following required variables: * `project_id`: Your GCP Project ID. -* `namespace`: A unique identifier for resource naming (e.g., your name or team name). +* `instance_name`: A unique identifier for resource naming (e.g., your name or team name). ### 2. Run the Setup Script @@ -103,17 +103,17 @@ Upon successful apply, Terraform displays key endpoints and resource names: After successful deployment with `deploy_ingestion_workflow = true`, you can run the automated ingestion pipeline. ### Step 1: Upload your Schema file -Upload your custom graph nodes file (`.mcf`) to the provisioned ingestion bucket. By default, the bucket name format is: `-ingestion-bucket-`. +Upload your custom graph nodes file (`.mcf`) to the provisioned ingestion bucket. By default, the bucket name format is: `-ingestion-bucket-`. ```bash -gcloud storage cp path/to/your/sample.mcf gs://-ingestion-bucket-/imports/sample.mcf +gcloud storage cp path/to/your/sample.mcf gs://-ingestion-bucket-/imports/sample.mcf ``` ### Step 2: Trigger the Workflow Orchestrator Trigger the Cloud Workflow to start the Dataflow job that will read the file and insert it into Spanner. ```bash -gcloud workflows run -ingestion-orchestrator \ +gcloud workflows run -ingestion-orchestrator \ --project= \ --location= \ --data='{ @@ -121,8 +121,8 @@ gcloud workflows run -ingestion-orchestrator \ "region": "", "spannerInstanceId": "", "spannerDatabaseId": "", - "importList": "[{\"importName\": \"SampleTestCase\", \"graphPath\": \"gs://-ingestion-bucket-/imports/sample.mcf\"}]", - "tempLocation": "gs://-ingestion-bucket-/temp" + "importList": "[{\"importName\": \"SampleTestCase\", \"graphPath\": \"gs://-ingestion-bucket-/imports/sample.mcf\"}]", + "tempLocation": "gs://-ingestion-bucket-/temp" }' ``` diff --git a/infra/dcp/main.tf b/infra/dcp/main.tf index b35157cc..67751b06 100644 --- a/infra/dcp/main.tf +++ b/infra/dcp/main.tf @@ -59,13 +59,16 @@ resource "google_project_service" "apis" { } locals { + # For backward compatibility, fallback to namespace if instance_name is empty + effective_instance_name = var.instance_name != "" ? var.instance_name : var.namespace + # Redirect abandoned 'latest' alias to 'stable' for Dataflow templates df_template_version = var.dcp_version == "latest" ? "stable" : var.dcp_version global_config = { project_id = var.project_id region = var.region - namespace = var.namespace + instance_name = local.effective_instance_name stateful_deletion_protection = var.stateful_deletion_protection stateless_deletion_protection = var.stateless_deletion_protection } diff --git a/infra/dcp/modules/auth/main.tf b/infra/dcp/modules/auth/main.tf index be78a5e1..1421e300 100644 --- a/infra/dcp/modules/auth/main.tf +++ b/infra/dcp/modules/auth/main.tf @@ -1,5 +1,5 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" google_maps_api_key_value = var.google_maps_api_key != null ? var.google_maps_api_key : try(google_apikeys_key.maps_api_key[0].key_string, "") } @@ -24,7 +24,7 @@ resource "google_apikeys_key" "maps_api_key" { count = var.google_maps_api_key == null && var.create_google_maps_key ? 1 : 0 name = "${local.name_prefix}dc-google-maps-key-${random_id.api_key_suffix.hex}" - display_name = "Maps API Key for ${var.namespace != "" ? var.namespace : "Data Commons"}" + display_name = "Maps API Key for ${var.instance_name != "" ? var.instance_name : "Data Commons"}" project = var.project_id restrictions { diff --git a/infra/dcp/modules/auth/variables.tf b/infra/dcp/modules/auth/variables.tf index 09d894b3..2396a7f4 100644 --- a/infra/dcp/modules/auth/variables.tf +++ b/infra/dcp/modules/auth/variables.tf @@ -2,7 +2,7 @@ variable "project_id" { type = string } -variable "namespace" { +variable "instance_name" { type = string } diff --git a/infra/dcp/modules/datacommons_services/main.tf b/infra/dcp/modules/datacommons_services/main.tf index a95ae93e..5febde14 100644 --- a/infra/dcp/modules/datacommons_services/main.tf +++ b/infra/dcp/modules/datacommons_services/main.tf @@ -1,5 +1,5 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" } resource "google_service_account" "serving_sa" { diff --git a/infra/dcp/modules/datacommons_services/variables.tf b/infra/dcp/modules/datacommons_services/variables.tf index dff9a37a..b1da6c49 100644 --- a/infra/dcp/modules/datacommons_services/variables.tf +++ b/infra/dcp/modules/datacommons_services/variables.tf @@ -5,7 +5,7 @@ variable "project_id" { type = string } -variable "namespace" { +variable "instance_name" { type = string } diff --git a/infra/dcp/modules/ingestion/dataflow/main.tf b/infra/dcp/modules/ingestion/dataflow/main.tf index 31a1fa33..ad4c7167 100644 --- a/infra/dcp/modules/ingestion/dataflow/main.tf +++ b/infra/dcp/modules/ingestion/dataflow/main.tf @@ -1,6 +1,6 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" - display_name_prefix = var.namespace != "" ? "(${var.namespace}) " : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" + display_name_prefix = var.instance_name != "" ? "(${var.instance_name}) " : "" } resource "google_service_account" "dataflow_sa" { diff --git a/infra/dcp/modules/ingestion/dataflow/variables.tf b/infra/dcp/modules/ingestion/dataflow/variables.tf index e0cd1066..a95b0a48 100644 --- a/infra/dcp/modules/ingestion/dataflow/variables.tf +++ b/infra/dcp/modules/ingestion/dataflow/variables.tf @@ -6,7 +6,7 @@ variable "project_id" { type = string } -variable "namespace" { +variable "instance_name" { type = string } diff --git a/infra/dcp/modules/ingestion/helper_service/main.tf b/infra/dcp/modules/ingestion/helper_service/main.tf index 3e08f3e4..c4819b10 100644 --- a/infra/dcp/modules/ingestion/helper_service/main.tf +++ b/infra/dcp/modules/ingestion/helper_service/main.tf @@ -1,5 +1,5 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" } resource "google_service_account" "helper_sa" { diff --git a/infra/dcp/modules/ingestion/helper_service/variables.tf b/infra/dcp/modules/ingestion/helper_service/variables.tf index cdf98061..3ff1db00 100644 --- a/infra/dcp/modules/ingestion/helper_service/variables.tf +++ b/infra/dcp/modules/ingestion/helper_service/variables.tf @@ -6,7 +6,7 @@ variable "project_id" { type = string } -variable "namespace" { +variable "instance_name" { type = string } diff --git a/infra/dcp/modules/ingestion/postprocessing_job/main.tf b/infra/dcp/modules/ingestion/postprocessing_job/main.tf index 77a82f4a..f9de83e2 100644 --- a/infra/dcp/modules/ingestion/postprocessing_job/main.tf +++ b/infra/dcp/modules/ingestion/postprocessing_job/main.tf @@ -1,5 +1,5 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" } resource "google_service_account" "postprocessing_sa" { diff --git a/infra/dcp/modules/ingestion/postprocessing_job/variables.tf b/infra/dcp/modules/ingestion/postprocessing_job/variables.tf index dd50f053..41bb4fe0 100644 --- a/infra/dcp/modules/ingestion/postprocessing_job/variables.tf +++ b/infra/dcp/modules/ingestion/postprocessing_job/variables.tf @@ -1,5 +1,5 @@ variable "project_id" { type = string } -variable "namespace" { type = string } +variable "instance_name" { type = string } variable "region" { type = string } variable "stateless_deletion_protection" { type = bool diff --git a/infra/dcp/modules/ingestion/preprocessing_job/main.tf b/infra/dcp/modules/ingestion/preprocessing_job/main.tf index 67d58af4..ba2df454 100644 --- a/infra/dcp/modules/ingestion/preprocessing_job/main.tf +++ b/infra/dcp/modules/ingestion/preprocessing_job/main.tf @@ -1,5 +1,5 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" } resource "google_service_account" "preprocessing_sa" { diff --git a/infra/dcp/modules/ingestion/preprocessing_job/variables.tf b/infra/dcp/modules/ingestion/preprocessing_job/variables.tf index 36b60041..8264cdf3 100644 --- a/infra/dcp/modules/ingestion/preprocessing_job/variables.tf +++ b/infra/dcp/modules/ingestion/preprocessing_job/variables.tf @@ -1,5 +1,5 @@ variable "project_id" { type = string } -variable "namespace" { type = string } +variable "instance_name" { type = string } variable "region" { type = string } variable "stateless_deletion_protection" { type = bool diff --git a/infra/dcp/modules/ingestion/workflow/main.tf b/infra/dcp/modules/ingestion/workflow/main.tf index 34f0bc78..1542abff 100644 --- a/infra/dcp/modules/ingestion/workflow/main.tf +++ b/infra/dcp/modules/ingestion/workflow/main.tf @@ -1,7 +1,7 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" should_run_postprocessing = var.enable_bigquery_postprocessing || var.enable_embeddings_generation - clean_namespace_prefix = var.namespace != "" ? "${replace(lower(var.namespace), "_", "-")}-" : "" + clean_namespace_prefix = var.instance_name != "" ? "${replace(lower(var.instance_name), "_", "-")}-" : "" } resource "google_service_account" "workflow_sa" { @@ -31,6 +31,8 @@ resource "google_workflows_workflow" "ingestion_orchestrator" { dataflow_subnetwork = var.dataflow_subnetwork embeddings_timeout = var.embeddings_timeout clean_namespace_prefix = local.clean_namespace_prefix + + enable_redis_cache_clearing = var.enable_redis_cache_clearing preprocessing_job_name = var.preprocessing_job_name postprocessing_job_name = var.postprocessing_job_name diff --git a/infra/dcp/modules/ingestion/workflow/variables.tf b/infra/dcp/modules/ingestion/workflow/variables.tf index 04c5a752..5d8e4fff 100644 --- a/infra/dcp/modules/ingestion/workflow/variables.tf +++ b/infra/dcp/modules/ingestion/workflow/variables.tf @@ -2,7 +2,7 @@ variable "deploy" { type = bool } -variable "namespace" { +variable "instance_name" { type = string } @@ -47,6 +47,8 @@ variable "embeddings_timeout" { default = 1800 } + + variable "ingestion_helper_service_name" { type = string description = "Name of the ingestion helper Cloud Run service" diff --git a/infra/dcp/modules/redis/main.tf b/infra/dcp/modules/redis/main.tf index 4f80a250..7843d6d5 100644 --- a/infra/dcp/modules/redis/main.tf +++ b/infra/dcp/modules/redis/main.tf @@ -1,10 +1,10 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" - display_name_prefix = var.namespace != "" ? "(${var.namespace}) " : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" + display_name_prefix = var.instance_name != "" ? "(${var.instance_name}) " : "" } resource "google_redis_instance" "redis_instance" { - name = var.instance_name != "" ? "${local.name_prefix}${var.instance_name}" : "${local.name_prefix}dc-redis-instance" + name = var.redis_instance_name != "" ? "${local.name_prefix}${var.redis_instance_name}" : "${local.name_prefix}dc-redis-instance" memory_size_gb = var.memory_size_gb tier = var.tier region = var.region diff --git a/infra/dcp/modules/redis/variables.tf b/infra/dcp/modules/redis/variables.tf index 6c88e7a7..a74e1ea1 100644 --- a/infra/dcp/modules/redis/variables.tf +++ b/infra/dcp/modules/redis/variables.tf @@ -1,4 +1,4 @@ -variable "namespace" { +variable "instance_name" { type = string } @@ -6,7 +6,7 @@ variable "region" { type = string } -variable "instance_name" { +variable "redis_instance_name" { type = string default = "" } diff --git a/infra/dcp/modules/spanner/main.tf b/infra/dcp/modules/spanner/main.tf index 85889e8f..e17bdeed 100644 --- a/infra/dcp/modules/spanner/main.tf +++ b/infra/dcp/modules/spanner/main.tf @@ -1,5 +1,5 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" effective_instance_id = var.create_instance ? (var.instance_id != "" ? "${local.name_prefix}${var.instance_id}" : "${local.name_prefix}dc-instance") : var.instance_id effective_database_id = var.create_database ? (var.database_id != "" ? "${local.name_prefix}${var.database_id}" : "${local.name_prefix}dc-db") : var.database_id } diff --git a/infra/dcp/modules/spanner/variables.tf b/infra/dcp/modules/spanner/variables.tf index a47c66eb..d702fbdf 100644 --- a/infra/dcp/modules/spanner/variables.tf +++ b/infra/dcp/modules/spanner/variables.tf @@ -3,7 +3,7 @@ variable "project_id" { description = "GCP Project ID" } -variable "namespace" { +variable "instance_name" { type = string } diff --git a/infra/dcp/modules/stack/main.tf b/infra/dcp/modules/stack/main.tf index a6e75975..4f83385d 100644 --- a/infra/dcp/modules/stack/main.tf +++ b/infra/dcp/modules/stack/main.tf @@ -80,7 +80,7 @@ module "spanner" { count = var.spanner_config.enable ? 1 : 0 project_id = var.global.project_id - namespace = var.global.namespace + instance_name = var.global.instance_name region = var.global.region create_instance = var.spanner_config.create_instance create_database = var.spanner_config.create_db @@ -106,8 +106,8 @@ module "storage" { stateful_deletion_protection = var.global.stateful_deletion_protection # Shared vars - project_id = var.global.project_id - namespace = var.global.namespace + project_id = var.global.project_id + instance_name = var.global.instance_name } module "ingestion_preprocessing_job" { @@ -115,7 +115,7 @@ module "ingestion_preprocessing_job" { count = var.ingestion_config.enable_ingestion ? 1 : 0 project_id = var.global.project_id - namespace = var.global.namespace + instance_name = var.global.instance_name region = var.global.region stateless_deletion_protection = var.global.stateless_deletion_protection image = var.ingestion_config.preprocessing_job_image @@ -149,7 +149,7 @@ module "ingestion_postprocessing_job" { count = var.ingestion_config.enable_ingestion ? 1 : 0 project_id = var.global.project_id - namespace = var.global.namespace + instance_name = var.global.instance_name region = var.global.region stateless_deletion_protection = var.global.stateless_deletion_protection image = var.ingestion_config.postprocessing_job_image @@ -171,7 +171,7 @@ module "ingestion_dataflow" { deploy = var.ingestion_config.enable_ingestion project_id = var.global.project_id - namespace = var.global.namespace + instance_name = var.global.instance_name ingestion_bucket_name = module.storage.artifacts_bucket_name use_spanner = var.spanner_config.enable } @@ -181,7 +181,7 @@ module "ingestion_helper_service" { deploy = var.ingestion_config.enable_ingestion project_id = var.global.project_id - namespace = var.global.namespace + instance_name = var.global.instance_name region = var.global.region stateless_deletion_protection = var.global.stateless_deletion_protection # Use index [0] because module.spanner is conditional. Fallback to empty string if disabled. @@ -207,7 +207,7 @@ module "ingestion_workflow" { source = "../ingestion/workflow" deploy = var.ingestion_config.enable_ingestion - namespace = var.global.namespace + instance_name = var.global.instance_name region = var.global.region stateless_deletion_protection = var.global.stateless_deletion_protection project_id = var.global.project_id @@ -216,7 +216,7 @@ module "ingestion_workflow" { dataflow_service_account_email = module.ingestion_dataflow.service_account_email enable_bigquery_postprocessing = var.ingestion_config.workflow_enable_bigquery_postprocessing enable_embeddings_generation = var.spanner_config.enable_embeddings_generation - ingestion_helper_service_name = "${var.global.namespace != "" ? "${var.global.namespace}-" : ""}dc-ingestion-helper" + ingestion_helper_service_name = "${var.global.instance_name != "" ? "${var.global.instance_name}-" : ""}dc-ingestion-helper" enable_redis_cache_clearing = var.redis_config.enable ingestion_artifacts_path = "${var.ingestion_config.ingestion_artifacts_path}/metadata" dataflow_ip_configuration = var.ingestion_config.dataflow_ip_configuration @@ -228,13 +228,14 @@ module "ingestion_workflow" { depends_on = [module.ingestion_helper_service] } + module "redis" { source = "../redis" count = var.redis_config.enable ? 1 : 0 - namespace = var.global.namespace + instance_name = var.global.instance_name region = var.global.region - instance_name = var.redis_config.instance_name + redis_instance_name = var.redis_config.instance_name memory_size_gb = var.redis_config.memory_size_gb tier = var.redis_config.tier location_id = var.redis_config.location_id @@ -249,7 +250,7 @@ module "auth" { source = "../auth" project_id = var.global.project_id - namespace = var.global.namespace + instance_name = var.global.instance_name dc_api_key = var.auth_config.google_datacommons_api_key google_maps_api_key = var.auth_config.google_maps_api_key create_google_maps_key = var.auth_config.create_google_maps_key @@ -260,7 +261,7 @@ module "datacommons_services" { count = var.datacommons_services_config.enable ? 1 : 0 project_id = var.global.project_id - namespace = var.global.namespace + instance_name = var.global.instance_name region = var.global.region stateless_deletion_protection = var.global.stateless_deletion_protection image = var.datacommons_services_config.image diff --git a/infra/dcp/modules/stack/variables.tf b/infra/dcp/modules/stack/variables.tf index 90210c48..4086b03d 100644 --- a/infra/dcp/modules/stack/variables.tf +++ b/infra/dcp/modules/stack/variables.tf @@ -2,7 +2,7 @@ variable "global" { type = object({ project_id = string region = string - namespace = string + instance_name = string stateful_deletion_protection = bool stateless_deletion_protection = bool }) diff --git a/infra/dcp/modules/storage/main.tf b/infra/dcp/modules/storage/main.tf index 6f2afde2..d1cb9d84 100644 --- a/infra/dcp/modules/storage/main.tf +++ b/infra/dcp/modules/storage/main.tf @@ -1,5 +1,5 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.instance_name != "" ? "${var.instance_name}-" : "" artifacts_bucket_name = var.artifacts_bucket_name != "" ? var.artifacts_bucket_name : "${local.name_prefix}dc-artifacts-${var.project_id}" } diff --git a/infra/dcp/modules/storage/variables.tf b/infra/dcp/modules/storage/variables.tf index a3512823..42f4104c 100644 --- a/infra/dcp/modules/storage/variables.tf +++ b/infra/dcp/modules/storage/variables.tf @@ -2,7 +2,7 @@ variable "project_id" { type = string } -variable "namespace" { +variable "instance_name" { type = string } diff --git a/infra/dcp/terraform.tfvars.template b/infra/dcp/terraform.tfvars.template index 948efda0..bedc6287 100644 --- a/infra/dcp/terraform.tfvars.template +++ b/infra/dcp/terraform.tfvars.template @@ -6,9 +6,9 @@ # ============================================================================= # Global Configuration # ============================================================================= -project_id = "$$PROJECT_ID$$" -namespace = "$$NAMESPACE$$" -region = "us-central1" +project_id = "$$PROJECT_ID$$" +instance_name = "$$INSTANCE_NAME$$" +region = "us-central1" # Deletion protection for stateful resources holding persistent data # (Spanner and Cloud Storage). diff --git a/infra/dcp/variables.tf b/infra/dcp/variables.tf index 0af5cd60..2e4a216b 100644 --- a/infra/dcp/variables.tf +++ b/infra/dcp/variables.tf @@ -13,12 +13,18 @@ variable "region" { default = "us-central1" } -variable "namespace" { +variable "instance_name" { description = "A unique identifier used as a prefix for resource naming. This prevents naming conflicts when deploying multiple isolated environments (like dev, staging, or feature branches) within the same GCP project." type = string default = "" } +variable "namespace" { + description = "DEPRECATED: Use instance_name instead. A unique identifier used as a prefix for resource naming." + type = string + default = "" +} + variable "stateful_deletion_protection" { description = "Enable deletion protection for stateful resources (Spanner, GCS) to prevent data loss." type = bool @@ -82,7 +88,7 @@ variable "storage_create_artifacts_bucket" { } variable "storage_artifacts_bucket_name" { - description = "The name of the unified GCS bucket for artifacts (serving and ingestion). If not provided, a name will be automatically generated following the pattern [namespace-]dc-artifacts-[project_id]" + description = "The name of the unified GCS bucket for artifacts (serving and ingestion). If not provided, a name will be automatically generated following the pattern [instance-name-]dc-artifacts-[project_id]" type = string default = "" } @@ -98,7 +104,7 @@ variable "enable_redis" { } variable "redis_instance_name" { - description = "The name of the Redis instance. If not provided, a name will be automatically generated following the pattern [namespace-]dc-redis-instance" + description = "The name of the Redis instance. If not provided, a name will be automatically generated following the pattern [instance-name-]dc-redis-instance" type = string default = "" } @@ -168,13 +174,13 @@ variable "spanner_create_database" { } variable "spanner_instance_id" { - description = "The ID of the Spanner instance. If not provided, a name will be automatically generated following the pattern [namespace-]dc-instance" + description = "The ID of the Spanner instance. If not provided, a name will be automatically generated following the pattern [instance-name-]dc-instance" type = string default = "" } variable "spanner_database_id" { - description = "The ID of the Spanner database. If not provided, a name will be automatically generated following the pattern [namespace-]dc-db" + description = "The ID of the Spanner database. If not provided, a name will be automatically generated following the pattern [instance-name-]dc-db" type = string default = "" } diff --git a/packages/datacommons-admin/datacommons_admin/admin_cli.py b/packages/datacommons-admin/datacommons_admin/admin_cli.py index d2dd327e..139c8b7e 100644 --- a/packages/datacommons-admin/datacommons_admin/admin_cli.py +++ b/packages/datacommons-admin/datacommons_admin/admin_cli.py @@ -35,14 +35,14 @@ GITHUB_REPO_URL = "https://github.com/datacommonsorg/datacommons.git" -def _get_default_bucket_name(namespace: str, project_id: str) -> str: +def _get_default_bucket_name(instance_name: str, project_id: str) -> str: """Returns the default Google Cloud Storage bucket name for Terraform state.""" - return f"tf-state-{namespace}-{project_id}" + return f"tf-state-{instance_name}-{project_id}" -def _get_default_state_prefix(namespace: str) -> str: +def _get_default_state_prefix(instance_name: str) -> str: """Returns the default Google Cloud Storage object prefix for Terraform state.""" - return f"terraform/state/{namespace}" + return f"terraform/state/{instance_name}" def _log_resolved_value(label: str, value: str, is_default: bool, indent: int = 2): @@ -225,34 +225,59 @@ def fetch(filename: str) -> str: @click.group() -def admin() -> None: +@click.option( + "--project-id", + default=None, + help="GCP project ID used to locate the remote Terraform state bucket.", +) +@click.option( + "--instance-name", + default=None, + help="DCP instance name (prefix) used to locate the remote Terraform state bucket.", +) +@click.option( + "--tf-state-location", + default=None, + help="Direct GCS URI pointing to the Terraform state file (gs://bucket/prefix/default.tfstate).", +) +@click.pass_context +def admin( + ctx: click.Context, + project_id: str | None, + instance_name: str | None, + tf_state_location: str | None, +) -> None: """Manage a Data Commons Platform instance in Google Cloud""" + ctx.ensure_object(dict) + ctx.obj["project_id"] = project_id + ctx.obj["instance_name"] = instance_name + ctx.obj["tf_state_location"] = tf_state_location -def _validate_namespace(ns: str) -> Tuple[bool, str]: - if not ns: - return False, "Namespace must not be empty." - if len(ns) > 16: +def _validate_instance_name(name: str) -> Tuple[bool, str]: + if not name: + return False, "Instance name must not be empty." + if len(name) > 16: return ( False, - f"Namespace must be 16 characters or less (currently {len(ns)} characters).", + f"Instance name must be 16 characters or less (currently {len(name)} characters).", ) - if not re.match(r"^[a-z]([-a-z0-9]*[a-z0-9])?$", ns): + if not re.match(r"^[a-z]([-a-z0-9]*[a-z0-9])?$", name): return ( False, - "Namespace must start with a lowercase letter, end with a lowercase letter or number, and contain only lowercase alphanumeric characters and dashes.", + "Instance name must start with a lowercase letter, end with a lowercase letter or number, and contain only lowercase alphanumeric characters and dashes.", ) return True, "" def _resolve_project_config( - project_id: str, namespace: str, force: bool + project_id: str, instance_name: str, force: bool ) -> Tuple[str, str, Path]: - """Resolves project ID and namespace, and determines target directory.""" + """Resolves project ID and instance name, and determines target directory.""" if project_id: _log_resolved_value("Project ID", project_id, is_default=False) - if namespace: - _log_resolved_value("Namespace", namespace, is_default=False) + if instance_name: + _log_resolved_value("Instance Name", instance_name, is_default=False) resolved_project_id = project_id.strip() if not resolved_project_id: @@ -262,34 +287,34 @@ def _resolve_project_config( if not resolved_project_id: raise click.ClickException("GCP project ID must not be empty.") - resolved_namespace = namespace.strip() - if resolved_namespace: - is_valid, err_msg = _validate_namespace(resolved_namespace) + resolved_instance_name = instance_name.strip() + if resolved_instance_name: + is_valid, err_msg = _validate_instance_name(resolved_instance_name) if not is_valid: raise click.ClickException(err_msg) while True: - if not resolved_namespace: - resolved_namespace = _prompt("Namespace", type=str).strip() - is_valid, err_msg = _validate_namespace(resolved_namespace) + if not resolved_instance_name: + resolved_instance_name = _prompt("Instance Name", type=str).strip() + is_valid, err_msg = _validate_instance_name(resolved_instance_name) if not is_valid: click.secho(f"Error: {err_msg}", fg="red") - resolved_namespace = "" + resolved_instance_name = "" continue - target_dir = Path.cwd() / resolved_namespace + target_dir = Path.cwd() / resolved_instance_name if target_dir.exists() and not force: click.secho( - f"Error: Folder '{resolved_namespace}' already exists locally. " - "Please specify a different namespace, or use --force to overwrite.", + f"Error: Folder '{resolved_instance_name}' already exists locally. " + "Please specify a different instance name, or use --force to overwrite.", fg="yellow", ) - resolved_namespace = "" + resolved_instance_name = "" continue break - return resolved_project_id, resolved_namespace, target_dir + return resolved_project_id, resolved_instance_name, target_dir def _check_existing_files(target_dir: Path, use_remote_state: bool, force: bool): @@ -315,7 +340,7 @@ def _check_existing_files(target_dir: Path, use_remote_state: bool, force: bool) def _setup_dcp_config_dir( target_dir: Path, project_id: str, - namespace: str, + instance_name: str, bucket_name: str, tf_state_prefix: str, dc_api_key: str, @@ -364,7 +389,8 @@ def _setup_dcp_config_dir( # Modify tfvars_example with actual values tfvars_content = tfvars_example tfvars_content = tfvars_content.replace('"$$PROJECT_ID$$"', f'"{project_id}"') - tfvars_content = tfvars_content.replace('"$$NAMESPACE$$"', f'"{namespace}"') + tfvars_content = tfvars_content.replace('"$$NAMESPACE$$"', f'"{instance_name}"') + tfvars_content = tfvars_content.replace('"$$INSTANCE_NAME$$"', f'"{instance_name}"') if api_key: tfvars_content = tfvars_content.replace('"$$DC_API_KEY$$"', f'"{api_key}"') @@ -401,7 +427,14 @@ def _setup_dcp_config_dir( help="Google Cloud Platform project ID used for all resources related to your Data Commons instance.", ) @click.option( - "--namespace", default="", help="Namespace prefix for provisioned resources." + "--instance-name", + default="", + help="Unique identifier used as a prefix for resource naming (replaces --namespace).", +) +@click.option( + "--namespace", + default="", + help="DEPRECATED: Use --instance-name instead.", ) @click.option("--dc-api-key", default="", help="Data Commons API key.") @click.option( @@ -432,11 +465,12 @@ def _setup_dcp_config_dir( @click.option( "--tf-state-prefix", default="", - help="Google Cloud Storage object prefix for Terraform state file (default: terraform/state/{namespace}).", + help="Google Cloud Storage object prefix for Terraform state file (default: terraform/state/{instance_name}).", ) def init( project_id: str, namespace: str, + instance_name: str, dc_api_key: str, tf_git_ref: str, force: bool, @@ -448,11 +482,14 @@ def init( """Initialize Terraform scaffolding for Data Commons administration/infrastructure.""" click.secho("Data Commons Admin Init", fg="cyan", bold=True) + # For backward compatibility, fallback to namespace if instance_name is empty + effective_instance_name = instance_name.strip() or namespace.strip() + # 1. Project Configs click.secho("\n[Project configuration]", fg="cyan", bold=True) click.secho("Configuring project settings...", fg="bright_black") - resolved_project_id, resolved_namespace, target_dir = _resolve_project_config( - project_id, namespace, force + resolved_project_id, resolved_instance_name, target_dir = _resolve_project_config( + project_id, effective_instance_name, force ) # 2. Terraform Setup @@ -467,7 +504,7 @@ def init( resolved_bucket_name = ( _configure_remote_state( resolved_project_id, - resolved_namespace, + resolved_instance_name, tf_state_bucket, tf_state_bucket_location, ) @@ -476,7 +513,7 @@ def init( ) resolved_tf_state_prefix = tf_state_prefix.strip() or _get_default_state_prefix( - resolved_namespace + resolved_instance_name ) # 3. DCP config dir setup @@ -485,7 +522,7 @@ def init( _setup_dcp_config_dir( target_dir, resolved_project_id, - resolved_namespace, + resolved_instance_name, resolved_bucket_name, resolved_tf_state_prefix, dc_api_key, @@ -494,7 +531,7 @@ def init( ) click.secho( - f"Customize variables in {resolved_namespace}/terraform.tfvars as needed.", + f"Customize variables in {resolved_instance_name}/terraform.tfvars as needed.", fg="green", ) click.secho( diff --git a/packages/datacommons-admin/datacommons_admin/infra_templates.py b/packages/datacommons-admin/datacommons_admin/infra_templates.py index 4f5999f0..05a2d2be 100644 --- a/packages/datacommons-admin/datacommons_admin/infra_templates.py +++ b/packages/datacommons-admin/datacommons_admin/infra_templates.py @@ -36,7 +36,7 @@ ## Configure Variables -Set environment-specific values in `terraform.tfvars` (for example `project_id`, `namespace`, and `dc_api_key`), and update module arguments in `main.tf` if you want to enable or tune additional features. +Set environment-specific values in `terraform.tfvars` (for example `project_id`, `instance_name`, and `dc_api_key`), and update module arguments in `main.tf` if you want to enable or tune additional features. ## Learn More About Variables diff --git a/packages/datacommons-admin/datacommons_admin/tf_utils.py b/packages/datacommons-admin/datacommons_admin/tf_utils.py index ed6ed9c2..7550fefa 100644 --- a/packages/datacommons-admin/datacommons_admin/tf_utils.py +++ b/packages/datacommons-admin/datacommons_admin/tf_utils.py @@ -31,60 +31,154 @@ TF_OUTPUT_INGESTION_WORKFLOW_NAME = "ingestion_workflow_name" -def get_terraform_output(key: str) -> str: - """Fetches a specific key from `terraform output -json` with graceful error handling.""" - if not shutil.which("terraform"): - raise click.ClickException( - "Terraform CLI not found. Please ensure Terraform is installed and available in your PATH." - ) +def _get_outputs_from_gcs( + tf_state_location: str | None, + project_id: str | None, + instance_name: str | None, +) -> dict: + from google.cloud import storage + from google.cloud.exceptions import GoogleCloudError + + # Resolve the GCS URI + if tf_state_location: + gcs_uri = tf_state_location.strip() + if not gcs_uri.endswith(".tfstate"): + gcs_uri = gcs_uri.rstrip("/") + "/default.tfstate" + elif project_id and instance_name: + p_id = project_id.strip() + i_name = instance_name.strip() + gcs_uri = f"gs://tf-state-{i_name}-{p_id}/terraform/state/{i_name}/default.tfstate" + else: + return {} try: - result = subprocess.run( - ["terraform", "output", "-json"], - capture_output=True, - text=True, - check=True, - ) - except subprocess.CalledProcessError as e: - raise click.ClickException( - f"Failed to run 'terraform output'. Are you in an initialized Terraform deployment directory?\n" - f"Error details: {e.stderr.strip() or e.stdout.strip()}" - ) + if not gcs_uri.startswith("gs://"): + raise click.ClickException(f"Invalid GCS URI '{gcs_uri}'. Must start with 'gs://'.") + parts = gcs_uri[5:].split("/", 1) + if len(parts) < 2 or not parts[1]: + raise click.ClickException(f"Invalid GCS URI '{gcs_uri}'. Must specify bucket and object path.") + bucket_name, blob_name = parts[0], parts[1] + + try: + client = storage.Client() + except Exception as e: + raise click.ClickException( + f"Failed to initialize Google Cloud Storage client: {e}.\n" + "Please ensure you are authenticated via 'gcloud auth application-default login'." + ) - try: - outputs = json.loads(result.stdout) - except json.JSONDecodeError: - raise click.ClickException( - "Failed to parse 'terraform output -json'. The output was not valid JSON." - ) + bucket = client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + try: + content = blob.download_as_text() + except GoogleCloudError as e: + if "404" in str(e) or "Not Found" in str(e): + raise click.ClickException( + f"Terraform state file not found at '{gcs_uri}'.\n" + "Please verify that the --project-id, --instance-name, or --tf-state-location flags are correct and that resources were deployed." + ) + raise click.ClickException( + f"Failed to download Terraform state from GCS at '{gcs_uri}': {e}" + ) - if not outputs: - from pathlib import Path + try: + state_data = json.loads(content) + except json.JSONDecodeError: + raise click.ClickException(f"Failed to parse Terraform state at '{gcs_uri}' as valid JSON.") + + outputs = state_data.get("outputs", {}) + if not outputs: + raise click.ClickException( + f"No outputs found in the Terraform state file at '{gcs_uri}'.\n" + "Please verify that your deployment is active and that variables are exported." + ) + return outputs + + except click.ClickException: + raise + except Exception as e: + raise click.ClickException(f"An unexpected error occurred while fetching remote state: {e}") - cwd = Path.cwd() - has_tf_files = ( - (cwd / ".terraform").exists() - or (cwd / "terraform.tfstate").exists() - or (cwd / "main.tf").exists() - ) - if not has_tf_files: +def get_terraform_output(key: str) -> str: + """Fetches a specific key from `terraform output -json` (local or remote GCS state).""" + ctx = click.get_current_context(silent=True) + project_id = None + instance_name = None + tf_state_location = None + + if ctx and ctx.obj: + cur_ctx = ctx + while cur_ctx: + if cur_ctx.obj: + project_id = project_id or cur_ctx.obj.get("project_id") + instance_name = instance_name or cur_ctx.obj.get("instance_name") + tf_state_location = tf_state_location or cur_ctx.obj.get("tf_state_location") + cur_ctx = cur_ctx.parent + + use_remote = bool(tf_state_location or (project_id and instance_name)) + + if use_remote: + outputs = _get_outputs_from_gcs(tf_state_location, project_id, instance_name) + else: + if not shutil.which("terraform"): raise click.ClickException( - f"No Terraform outputs found in '{cwd}'.\n" - "Please navigate to your initialized DCP Terraform directory (e.g., 'cd my-namespace') and ensure 'terraform apply' has been run." + "Terraform CLI not found. Please ensure Terraform is installed and available in your PATH." + ) + + try: + result = subprocess.run( + ["terraform", "output", "-json"], + capture_output=True, + text=True, + check=True, ) - else: + except subprocess.CalledProcessError as e: raise click.ClickException( - f"No Terraform outputs found in '{cwd}'.\n" - "Please ensure you have successfully run 'terraform apply' to generate the deployment state." + f"Failed to run 'terraform output'.\n" + f"To resolve your deployment configuration, either:\n" + f"a. Run this command inside your initialized DCP Terraform deployment directory (where 'terraform apply' has been run).\n" + f"b. Specify the GCS remote state flags on the 'admin' group: --project-id and --instance-name (or --tf-state-location ).\n\n" + f"Error details: {e.stderr.strip() or e.stdout.strip()}" ) + try: + outputs = json.loads(result.stdout) + except json.JSONDecodeError: + raise click.ClickException( + "Failed to parse 'terraform output -json'. The output was not valid JSON." + ) + + if not outputs: + from pathlib import Path + + cwd = Path.cwd() + has_tf_files = ( + (cwd / ".terraform").exists() + or (cwd / "terraform.tfstate").exists() + or (cwd / "main.tf").exists() + ) + + if not has_tf_files: + raise click.ClickException( + f"No Terraform deployment state found in '{cwd}'.\n" + "To resolve your deployment configuration, either:\n" + "a. Navigate to your initialized DCP Terraform directory (where 'terraform apply' has been run).\n" + "b. Run the command with GCS remote state flags on the 'admin' group: --project-id and --instance-name (or --tf-state-location )." + ) + else: + raise click.ClickException( + f"No Terraform outputs found in '{cwd}'.\n" + "Please ensure you have successfully run 'terraform apply' to generate the deployment state." + ) + if key not in outputs: from pathlib import Path - + location_desc = tf_state_location or f"GCP project: '{project_id}' / instance: '{instance_name}'" if use_remote else f"'{Path.cwd()}'" raise click.ClickException( - f"Terraform output key '{key}' not found in '{Path.cwd()}'.\n" - "Please verify that your Terraform configuration exports this output and that 'terraform apply' was fully completed." + f"Terraform output key '{key}' not found in {location_desc}.\n" + "Please verify that your Terraform configuration exports this output." ) value = outputs[key].get("value") diff --git a/packages/datacommons-admin/tests/test_admin_cli.py b/packages/datacommons-admin/tests/test_admin_cli.py index 2827a40a..6128dca2 100644 --- a/packages/datacommons-admin/tests/test_admin_cli.py +++ b/packages/datacommons-admin/tests/test_admin_cli.py @@ -66,6 +66,46 @@ def test_init_success_with_options( assert 'dc_api_key = "test-key"' in tfvars_content +@patch("datacommons_admin.admin_cli._get_github_templates") +def test_init_success_with_instance_name( + mock_get_templates, runner: CliRunner, tmp_path: Path +) -> None: + mock_get_templates.return_value = ( + 'variable "test" {}', + 'module "stack" {\n source = "./modules/stack"\n}', + 'output "test" {}', + 'project_id = "$$PROJECT_ID$$"\ninstance_name = "$$INSTANCE_NAME$$"\n# dc_api_key = "$$DC_API_KEY$$"', + ) + with runner.isolated_filesystem(temp_dir=tmp_path): + result = runner.invoke( + admin, + [ + "init", + "--project-id", + "test-project", + "--instance-name", + "test-inst", + "--dc-api-key", + "test-key", + "--no-tf-remote-state", + ], + ) + assert result.exit_code == 0 + assert "Downloaded and populated Terraform templates." in result.output + + target_dir = Path.cwd() / "test-inst" + assert target_dir.exists() + assert (target_dir / "main.tf").exists() + assert (target_dir / "terraform.tfvars").exists() + assert (target_dir / "README.md").exists() + assert not (target_dir / "backend.tf").exists() + + tfvars_content = (target_dir / "terraform.tfvars").read_text() + assert 'project_id = "test-project"' in tfvars_content + assert 'instance_name = "test-inst"' in tfvars_content + assert 'dc_api_key = "test-key"' in tfvars_content + + @patch("datacommons_admin.admin_cli._get_github_templates") def test_init_success_with_prompts( mock_get_templates, runner: CliRunner, tmp_path: Path @@ -540,3 +580,64 @@ def test_ingest_start_with_imports_success( called_payload = called_args["json"] assert "argument" in called_payload assert json.loads(called_payload["argument"]) == expected_arg + + +@patch("google.cloud.storage.Client") +def test_get_terraform_output_from_gcs(mock_storage_client, runner: CliRunner) -> None: + from unittest.mock import MagicMock + from datacommons_admin.tf_utils import get_terraform_output + import click + + # Mock storage client + mock_client_inst = MagicMock() + mock_storage_client.return_value = mock_client_inst + mock_bucket = MagicMock() + mock_client_inst.bucket.return_value = mock_bucket + mock_blob = MagicMock() + mock_bucket.blob.return_value = mock_blob + + state_content = """{ + "version": 4, + "outputs": { + "test_key": { + "value": "gcs-resolved-val", + "type": "string" + } + } + }""" + mock_blob.download_as_text.return_value = state_content + + @admin.command(name="test-get-output") + def test_get_output_cmd(): + val = get_terraform_output("test_key") + click.echo(f"VAL={val}") + + # Test with project-id and instance-name + result = runner.invoke( + admin, + [ + "--project-id", "mock-project", + "--instance-name", "mock-instance", + "test-get-output" + ] + ) + assert result.exit_code == 0 + assert "VAL=gcs-resolved-val" in result.output + mock_client_inst.bucket.assert_called_once_with("tf-state-mock-instance-mock-project") + mock_bucket.blob.assert_called_once_with("terraform/state/mock-instance/default.tfstate") + + # Test with tf-state-location + mock_client_inst.reset_mock() + mock_bucket.reset_mock() + result = runner.invoke( + admin, + [ + "--tf-state-location", "gs://custom-bucket/custom-prefix/state.tfstate", + "test-get-output" + ] + ) + assert result.exit_code == 0 + assert "VAL=gcs-resolved-val" in result.output + mock_client_inst.bucket.assert_called_once_with("custom-bucket") + mock_bucket.blob.assert_called_once_with("custom-prefix/state.tfstate") + diff --git a/packages/datacommons-cli/README.md b/packages/datacommons-cli/README.md index ac3e0f3a..f868da8f 100644 --- a/packages/datacommons-cli/README.md +++ b/packages/datacommons-cli/README.md @@ -95,7 +95,7 @@ datacommons admin [COMMAND] --help 1. Initialize your project configuration and scaffold Terraform templates: ```bash - datacommons admin init --project-id my-gcp-project --namespace prod + datacommons admin init --project-id my-gcp-project --instance-name prod ``` 2. Change directories to the scaffolded directory (e.g. `cd prod`). diff --git a/packages/datacommons-cli/datacommons_cli/cli.py b/packages/datacommons-cli/datacommons_cli/cli.py index 1e69e381..4bcae815 100644 --- a/packages/datacommons-cli/datacommons_cli/cli.py +++ b/packages/datacommons-cli/datacommons_cli/cli.py @@ -16,6 +16,17 @@ import click +# Monkeypatch ClickException to format errors in red +def _custom_click_exception_show(self, file=None): + if file is None: + import sys + file = sys.stderr + click.echo( + click.style("Error: ", fg="red", bold=True) + click.style(self.format_message(), fg="red"), + file=file + ) +click.ClickException.show = _custom_click_exception_show + from datacommons_admin.admin_cli import admin as admin_cli from . import __version__