From 5e69c0f741b686376d056df9e289e04f0d2e8e6f Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Fri, 22 May 2026 13:59:33 -0700
Subject: [PATCH 01/23] Add dcp setup/ops skills and configure dynamic CLI
 overrides

---
 .gitignore                                    |   4 +-
 .../datacommons_admin/admin_cli.py            |   6 +-
 .../datacommons_cli/skills/dcp-ops/SKILL.md   | 171 +++++++++++++
 .../datacommons_cli/skills/dcp-setup/SKILL.md | 231 ++++++++++++++++++
 .../skills/dcp-setup/scripts/poll_iam.sh      |  91 +++++++
 .../dcp-setup/scripts/preflight_check.sh      | 153 ++++++++++++
 .../datacommons-cli/scripts/install-agent.sh  | 231 ++++++++++++++++++
 7 files changed, 884 insertions(+), 3 deletions(-)
 create mode 100644 packages/datacommons-cli/datacommons_cli/skills/dcp-ops/SKILL.md
 create mode 100644 packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
 create mode 100755 packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/poll_iam.sh
 create mode 100755 packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh
 create mode 100755 packages/datacommons-cli/scripts/install-agent.sh

diff --git a/.gitignore b/.gitignore
index e0695b11..c0ee9ef3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -40,4 +40,6 @@ infra/dcp/.env
 
 AGENTS.md
 docs/plans
-docs/designs
\ No newline at end of file
+docs/designs
+
+tmp/
\ No newline at end of file
diff --git a/packages/datacommons-admin/datacommons_admin/admin_cli.py b/packages/datacommons-admin/datacommons_admin/admin_cli.py
index 8700f191..c3092429 100644
--- a/packages/datacommons-admin/datacommons_admin/admin_cli.py
+++ b/packages/datacommons-admin/datacommons_admin/admin_cli.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from pathlib import Path
+import os
 import re
 import sys
 import urllib.request
@@ -30,8 +31,9 @@
 
 
 DEFAULT_BUCKET_LOCATION = "US"
-GITHUB_RAW_BASE_URL = "https://raw.githubusercontent.com/datacommonsorg/datacommons"
-GITHUB_REPO_URL = "https://github.com/datacommonsorg/datacommons.git"
+# Overridable repository sources (allows developers to point to branches/forks)
+GITHUB_RAW_BASE_URL = os.getenv("DCP_GITHUB_RAW_BASE", "https://raw.githubusercontent.com/datacommonsorg/datacommons")
+GITHUB_REPO_URL = os.getenv("DCP_GIT_REPO", "https://github.com/datacommonsorg/datacommons.git")
 
 
 def _get_default_bucket_name(namespace: str, project_id: str) -> str:
diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-ops/SKILL.md b/packages/datacommons-cli/datacommons_cli/skills/dcp-ops/SKILL.md
new file mode 100644
index 00000000..9e56347e
--- /dev/null
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-ops/SKILL.md
@@ -0,0 +1,171 @@
+---
+name: dcp-ops
+description: Procedural guide and strict safety guidelines for managing, upgrading, and maintaining an active Data Commons Platform (DCP) instance. Enforces absolute transparency, step-by-step pacing, and user approval gates.
+---
+
+# Data Commons Platform Operations (dcp-ops)
+
+This skill guides an agentic coding assistant through Day-2 administration, data loading, configuration upgrades, and troubleshooting of an existing Data Commons Platform (DCP) instance.
+
+> [!IMPORTANT]
+> **CRITICAL COMPLIANCE DIRECTIVE: USER CONTROL & TRANSPARENCY**
+> * The agent must **NEVER** silently automate or modify cloud resources (GCS, Spanner, Cloud Run) without explaining the exact proposed changes and waiting for explicit verbal confirmation.
+> * Before executing *any* command, the agent must present the proposed command line to the user and ask for confirmation.
+> * **After executing any command**, the agent must immediately provide a clear, useful summary of exactly what was done, what resources were changed, and what status or outputs were returned.
+> * **GCP Console Deep-Links**: Whenever a resource is created or modified, the agent must construct and print the direct Google Cloud Console deep-links (e.g. GCS browser, Spanner console, Cloud Run dashboard) to enable instant user verification and audit.
+> * Keep the user fully informed of background tasks (e.g., logs monitoring, proxy tunnels), providing direct console URLs for observability.
+
+---
+
+## Phase 1: Active Environment Discovery
+
+Before performing any operations, the agent must inspect the current directory to map out the active infrastructure state.
+
+### 1. Verify Active Provisioning
+* Verify that `main.tf` and `terraform.tfvars` exist and that a valid Terraform state is present.
+* If no infrastructure has been provisioned yet, stop immediately and direct the user to run **`dcp-setup`** first.
+
+### 2. Read Active GCP Project Context
+* Parse `terraform.tfvars` to extract `project_id`, `namespace`, and target instance sizes.
+* Set the active gcloud project context and show the user:
+  ```bash
+  gcloud config set project [project_id]
+  ```
+* Verify the active user has permission to query project services.
+
+---
+
+## Phase 2: Updating System Configuration & Deployments (Gate 1 Approval)
+
+Use this workflow whenever you need to deploy a new Docker image or update configuration variables.
+
+### 1. Edit Configuration
+* Instruct the user that we are about to update the configuration.
+* Present the proposed modifications inside `terraform.tfvars` (e.g., changing `cdc_web_service_image`, `cdc_data_job_image`, or `gcs_data_bucket_input_folder`) and wait for approval before writing any changes.
+
+### 2. Apply Configuration Changes (Gate 1 Approval)
+* Run a dry-run plan to inspect the proposed modifications:
+  ```bash
+  terraform init  # Confirm plugins are initialized
+  terraform plan -out=tfplan.out
+  ```
+* Generate a detailed **Impact Report** listing:
+  * **Modified Variables**: Old value vs. proposed new value.
+  * **Added/Changed Cloud Resources**: Resources affected by the plan.
+  * **Safety Guarantee**: Verify that no Spanner instances or active production databases are being destroyed or modified in a destructive manner.
+* **Do not run `terraform apply` until the user reviews this Impact Report and gives explicit verbal confirmation (e.g., "Yes", "Approve").**
+* Upon approval, execute the deployment:
+  ```bash
+  terraform apply tfplan.out
+  ```
+
+---
+
+## Phase 3: Seed Data Updates & Custom Ingestions
+
+Use this workflow to copy new data files and trigger a new ingestion execution to reload the Spanner database.
+
+### 1. Copy Custom Data to GCS Ingestion Bucket
+* Read the target input bucket and input folder from the active configuration.
+* Show the user the exact GCS destination path: `gs://[data_bucket_name]/[gcs_data_bucket_input_folder]/`.
+* Ask the user for approval to copy the new data files from the local `./data` folder to GCS:
+  ```bash
+  gcloud storage cp -R ./data/* "gs://${MY_BUCKET}/${INPUT_FOLDER}/"
+  ```
+
+### 2. Trigger the Ingestion Pipeline
+* Explain that we are about to trigger the Cloud Run Ingestion Job, show the command, and ask for permission to execute:
+  ```bash
+  uv run datacommons admin ingest start
+  ```
+* **Provide Ingestion Logs Link**: Once executed, capture the **Job Console Link** from the logs and present it clearly in the chat so the user has direct observability in their browser console.
+
+---
+
+## Phase 4: Operational Monitoring & Health Audits (Gate 2 Verification)
+
+The agent must actively verify that operations complete successfully and that the platform remains highly responsive, keeping the user updated on every step.
+
+### 1. Monitor Ingestion Progress
+* Explicitly inform the user you are watching the Cloud Run Job execution logs.
+* Output regular status updates (e.g., *"Ingestion job is currently running. Status: Active..."*).
+* If errors appear, present the exact error log trace and recommend remediation steps.
+
+### 2. Validate Database Health (Timeout & Sampling Heuristic)
+Verify that the ingestion succeeded by querying the Spanner database table **`Observation`** (singular). 
+
+🚨 **Performance Precaution**: Full-table counts (`SELECT COUNT(*)`) can cause expensive full scans and timeouts on large Spanner tables. Follow this safe, observable query flow:
+
+1. **Data Presence Probe**: Tell the user you are running a fast probe to confirm data is active:
+   ```bash
+   gcloud spanner databases execute-sql [spanner_db_name] \
+       --instance=[dcp_spanner_instance_id] \
+       --sql="SELECT 1 FROM Observation LIMIT 1"
+   ```
+2. **Fast Count Scan**: Run a count query:
+   ```bash
+   gcloud spanner databases execute-sql [spanner_db_name] \
+       --instance=[dcp_spanner_instance_id] \
+       --sql="SELECT COUNT(1) FROM Observation"
+   ```
+   * *Query Timeout Handling*: If this query takes more than **15-30 seconds**, abort/cancel the query immediately.
+   * *Fall Back to Sampling*: If aborted, report to the user that the table is too large for an immediate full count, and query a small recent sample as proof of successful data load:
+     ```bash
+     gcloud spanner databases execute-sql [spanner_db_name] \
+         --instance=[dcp_spanner_instance_id] \
+         --sql="SELECT * FROM Observation LIMIT 10"
+     ```
+     Present the sample records table directly in the chat.
+
+### 3. Local Health Checks
+Establish a background proxy connection to the Cloud Run Website:
+```bash
+gcloud run services proxy [cloud-run-service-name] --region=us-central1
+```
+Inform the user and query local port `8080`:
+```bash
+curl -i -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8080/healthz
+```
+Confirm a HTTP `200 OK` response.
+
+---
+
+## Phase 5: Troubleshooting & Diagnosing Common Failures
+
+When components fail, explain the diagnostic path and ask for approval before reading logs:
+
+### 1. Mixer Service Crashes (HTTP 502/503 Errors)
+* *Action*: Propose reading the latest 50 lines of Cloud Run service logs:
+  ```bash
+  gcloud logging read "resource.type=cloud_run_revision AND resource.labels.service_name=[cloud-run-service-name]" --limit=50 --format=json
+  ```
+* Show logs in the chat, identify Spanner configuration or credentials mismatch errors, and propose fixes.
+
+### 2. Ingestion Job Terminated/Failed
+* *Action*: Propose describing the failed Cloud Run job execution:
+  ```bash
+  gcloud run jobs executions describe [execution_name]
+  ```
+* Analyze permissions and GCS bucket bindings, and present clear fixes to the user.
+
+---
+
+## Phase 6: Tear-down / Full Clean Wipe
+
+If the partner wishes to completely tear down an environment, guide them safely to avoid data loss.
+
+1. **Backup Alert**: Warn the user to backup any custom Spanner database schemas or GCS input data. **Do not run any destructive commands until they acknowledge.**
+2. **Run Terraform Destroy**: Show the exact plan of resources to be destroyed and wait for verbal confirmation before executing:
+   ```bash
+   terraform destroy
+   ```
+3. **Erase Remote State**: Ask for permission to delete the Terraform backend state bucket:
+   ```bash
+   STATE_BUCKET="tf-state-backend-bucket"
+   gcloud storage rm --recursive "gs://${STATE_BUCKET}"
+   ```
+4. **Remove Directories**: Delete generated scaffolding folder:
+   ```bash
+   rm -rf [namespace_folder]
+   ```
+5. **Audit Log Update**: Log the wipe out action in the `ops_audit_log.md` file.
diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
new file mode 100644
index 00000000..ede13039
--- /dev/null
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
@@ -0,0 +1,231 @@
+---
+name: dcp-setup
+description: Procedural guide and strict agent safety guidelines for provisioning a new or existing Data Commons Platform (DCP) instance. Enforces absolute transparency, explicit user approval gates, and interactive param harvesting.
+---
+
+# Data Commons Platform Setup (dcp-setup)
+
+This skill guides an agentic coding assistant through the setup and provisioning of a Data Commons Platform (DCP) instance on Google Cloud Platform (GCP).
+
+> [!IMPORTANT]
+> **CRITICAL COMPLIANCE DIRECTIVE: USER CONTROL & TRANSPARENCY**
+> * The agent must **NEVER** silently automate or submit interactive inputs to CLI commands running in the background (no silent pipelines).
+> * The agent must **NEVER** create or modify GCP resources (GCS buckets, Spanner, Cloud Run) without first explaining the exact proposed action and waiting for explicit verbal confirmation.
+> * Before executing *any* command, the agent must present the proposed command line to the user and ask for confirmation.
+> * **After executing any command**, the agent must immediately provide a clear, useful summary of exactly what was done, what resources were created/changed, and what status or outputs were returned.
+> * **GCP Console Deep-Links**: Whenever a resource is created or modified, the agent must construct and print the direct Google Cloud Console deep-links (e.g. GCS browser, Spanner console, Cloud Run dashboard) to enable instant user verification and audit.
+
+---
+
+## Phase 1: Prerequisites Verification & Installation
+
+Before running any setup tasks, the agent must verify that all required local dependencies are installed and authenticated.
+
+### 1. OS Detection & Shell Verification
+* Detect the host operating system (macOS vs. Linux).
+* Ensure the execution environment has standard terminal tools like `curl`, `unzip`, and shell capabilities.
+
+### 2. CLI Dependencies Check & Local Virtual Env Setup
+Verify the availability of `uv`, `terraform`, and `gcloud` by running version checks. If any tool is missing, present a summary to the user and ask if they want the agent to install it automatically:
+
+* **`uv` (Python Package Manager)**:
+  * *Check*: `uv --version`
+  * *Action if missing*: Ask the user for permission to install it via:
+    ```bash
+    curl -LsSf https://astral.sh/uv/install.sh | sh
+    ```
+
+* **Permanent Workspace CLI Installation**:
+  * Inform the user that a workspace-local virtual environment will be created to avoid slow `uvx` downlads.
+  * Ask for approval, then run:
+    ```bash
+    uv venv
+    uv pip install "git+https://github.com/datacommonsorg/datacommons.git@main#subdirectory=packages/datacommons-cli"
+    ```
+  * **Mandatory Tool Call**: For all subsequent steps, invoke the CLI instantly using `uv run datacommons` to prevent execution delays.
+
+* **`terraform` (Infrastructure as Code)**:
+  * *Check*: `terraform -version`
+  * *Action if missing*: Present the proposed installation command (macOS Homebrew or Linux apt) and wait for approval before running.
+
+* **`gcloud` (Google Cloud SDK)**:
+  * *Check*: `gcloud --version`
+  * *Action if missing*: Guide the user through the official installer, explaining exactly what package to download.
+
+### 3. GCP Authentication Check
+* **Verify active login**: Run `gcloud auth list`. 
+* **Re-authenticate if needed**: If no active Application Default Credentials (ADC) are found, inform the user and ask for approval to trigger:
+  ```bash
+  gcloud auth application-default login
+  ```
+  Explain that an interactive browser window will open for authentication.
+
+---
+
+## Phase 2: Scaffolding & Configuration Validation (User Approval Gates)
+
+The agent must guide the user through the scaffolding phase with complete transparency, prompting for every parameter.
+
+### 1. Parameter Harvesting with Smart Environment Defaults
+Before running the scaffold generator, the agent **MUST run discovery checks to harvest environment defaults, and then present them to the user for explicit confirmation or modification**:
+
+1. **GCP Project ID**:
+   * *Discovery*: Run `gcloud config get-value project 2>/dev/null`.
+   * *Prompt*: *"I detected that your active Google Cloud project context is set to `[detected_project_id]`. Would you like to use this project for deployment, or would you prefer to specify a different GCP project ID?"*
+2. **Namespace**:
+   * *Discovery*: Query system username (via `$USER`) or folder name.
+   * *Prompt*: *"What unique namespace identifier should we use for this environment? [Default: dcp-$USER]"*
+3. **Data Commons API Key**:
+   * *Discovery*: Check active environment variables (`echo $DC_API_KEY` or `$MIXER_API_KEY`).
+   * *Prompt*: If found: *"I detected a `DC_API_KEY` environment variable in your active session. Should I use this key, or would you like to provide a different one?"*
+   * If not found: *"I did not detect a `DC_API_KEY` in your active environment. Please provide a valid API Key from apikeys.datacommons.org (or let me know if we should use a dummy key 'fake-key' for this scaffolding phase)."*
+
+**Wait for the user's explicit confirmation or custom inputs for all three parameters.**
+
+
+### 2. Scaffolding Generation (Explicit Step)
+Once parameters are harvested, present the command and the inputs to the user:
+* *"I am going to run `uv run datacommons admin init` to generate your Terraform templates and set up your remote GCS state bucket. The parameters I will submit are Project: [project_id], Namespace: [namespace], and API Key: [api_key]. Do you approve?"*
+* **Do not run the command in the background silently.** Run it with explicit inputs so that the command executes without prompting in the background:
+  ```bash
+  # Force inputs non-interactively to prevent silent background prompt automation
+  uv run datacommons admin init --project=[project_id] --namespace=[namespace] --key=[api_key] --auto-approve
+  ```
+
+### 3. Variable Verification
+* Read `terraform.tfvars` and show the user the generated configuration variables.
+* Confirm if they want to make any manual adjustments (e.g., changing machine sizes or regions) before proceeding.
+
+---
+
+## Phase 3: Existing GCP Project & Conflict Validation
+
+Because the partner might deploy to a shared or pre-existing GCP project, the agent must actively identify and resolve resource conflicts, keeping the user fully informed.
+
+### 1. GCP Project Verification
+Set the active project context and verify access:
+```bash
+gcloud config set project [PROJECT_ID]
+```
+
+### 2. Enable Required GCP APIs
+List the necessary APIs (Spanner, Cloud Run, Secret Manager, Artifact Registry) and ask for approval to enable them:
+```bash
+gcloud services enable spanner.googleapis.com run.googleapis.com secretmanager.googleapis.com artifactregistry.googleapis.com cloudbuild.googleapis.com iamcredentials.googleapis.com
+```
+
+### 3. Spanner Database Conflict Check
+* Run `gcloud spanner instances list --project=[PROJECT_ID]` to check if the configured `dcp_spanner_instance_id` already exists.
+* **If it exists, report immediately**:
+  > ⚠️ **Existing Spanner Instance Detected**: The instance ID `[dcp_spanner_instance_id]` already exists in GCP project `[project_id]`.
+  > * **Reusing** this instance may overwrite existing database tables.
+  > * **Creating a new one** requires modifying `terraform.tfvars` to use a unique instance ID.
+  >
+  > *Please tell me how you want to proceed: [Option A: Reuse Instance] or [Option B: Configure New Unique ID].*
+
+---
+
+## Phase 4: Dry-Run & IaC Provisioning (Gate 1 Approval)
+
+Before applying infrastructure changes, compile the exact plan and require verbal user approval.
+
+### 1. Terraform Initialization
+Explain and run:
+```bash
+terraform init
+```
+
+### 2. Generate the "Impact Report" (Gate 1 Approval)
+Run `terraform plan` and output the changes to a temporary file. Parse this plan to build a detailed **Impact Report** containing:
+* **Resources to Create**: List of added resources (e.g. GCS buckets, Spanner instance, Secret Manager keys).
+* **Resources to Modify**: List of changed settings.
+* **Resources to Destroy**: 🚨 Explicit warnings if any existing resources are set to be destroyed.
+* **Estimated Cost Category**: Indicate if it uses production-scale Spanner nodes or budget-friendly processing units.
+
+**Do not run `terraform apply` until the user reviews the Impact Report and replies with explicit verbal approval (e.g., "Yes", "Approve").**
+
+### 3. Apply Infrastructure Changes
+Upon receiving approval, execute the deployment:
+```bash
+terraform apply -auto-approve
+```
+Display the outputs to the user, highlighting `data_bucket_name` and `cloud-run-service-name`.
+
+---
+
+## Phase 5: IAM Role Impersonation & Latency Polling
+
+### 1. Bind Token Creator Role
+Explain that the user must be bound to the newly created service account to initialize the database:
+```bash
+gcloud iam service-accounts add-iam-policy-binding [dcp_orchestrator_service_account_email] \
+    --member="user:[active-gcloud-user]" \
+    --role="roles/iam.serviceAccountTokenCreator" \
+    --project=[project_id]
+```
+
+### 2. Active IAM Propagation Polling (Safety Gate)
+Inform the user that GCP IAM replication can take a few minutes. Run a visible loop that checks impersonation status every 15 seconds, showing a progress indicator so the user knows the status.
+
+---
+
+## Phase 6: Flexible Data Loading & Database Initialization
+
+The database seed step should accommodate the partner's custom data rather than defaulting to hardcoded UN-centric files.
+
+### 1. Custom Data Path Selection
+Ask the user for their ingestion data source:
+* **Option A: Local Folder**: Copy local CSV/JSON files to GCS:
+  ```bash
+  gcloud storage cp -R ./data/* "gs://[data_bucket_name]/[gcs_data_bucket_input_folder]/"
+  ```
+* **Option B: Custom GCS Source**: Sync from an existing external GCS bucket:
+  ```bash
+  gcloud storage cp -R gs://[custom-source-path]/* "gs://[data_bucket_name]/[gcs_data_bucket_input_folder]/"
+  ```
+* **Option C: Default Sandbox Data**: Copy standard Data Commons public sandbox files for a quick test.
+
+### 2. Initialize Spanner Tables (init-db)
+Explain that we are initializing the Spanner schema tables and run:
+```bash
+uv run datacommons admin init-db
+```
+
+### 3. Start the Ingestion Job
+Present the command to trigger the Cloud Run pipeline to ingest the data:
+```bash
+uv run datacommons admin ingest start
+```
+* **Provide Console Links**: Print the printed **Job Console Link** clearly in the chat so the user can watch the live logs in their browser console.
+
+---
+
+## Phase 7: Health Verification & Local Proxy
+
+Verify that the entire platform is healthy and operational.
+
+### 1. Local Cloud Run Proxy
+Establish a local proxy tunnel in the background to expose the Cloud Run Website service:
+```bash
+gcloud run services proxy [cloud-run-service-name] \
+    --project=[project_id] \
+    --region=us-central1
+```
+
+### 2. Automatic Health Check (Gate 2 Verification)
+Show the user you are running automated health tests:
+* Hit the `/healthz` endpoint and report the HTTP status code.
+* Run a test query to verify the V2 API and present the JSON response.
+
+---
+
+## Phase 8: Operations Handoff
+
+1. **Write Audit Log**: Append a record to `ops_audit_log.md` with timestamps, active project, and success statuses of the setup phases.
+2. **Provide Dashboard Links**: Output the direct GCP Console URLs for:
+   * Spanner Database Console
+   * Cloud Run Service Logs
+3. **Instruct Transition**: Present the operational options and advise the user:
+   > 🎉 **DCP Bootstrap Setup Complete!**
+   > Your Data Commons Platform is fully provisioned and serving custom data on `http://127.0.0.1:8080/`.
+   > For all future day-to-day modifications, schema upgrades, and ingestion runs, please load and execute the **`dcp-ops`** skill!
diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/poll_iam.sh b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/poll_iam.sh
new file mode 100755
index 00000000..40cfc44b
--- /dev/null
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/poll_iam.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+#
+# Data Commons Platform (DCP) - IAM Token Impersonation Poller
+# Binds serviceAccountTokenCreator privileges and polls until propagation succeeds.
+#
+
+set -euo pipefail
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0;0m'
+
+log_info() {
+    echo -e "${BLUE}[INFO]${NC} $1"
+}
+
+log_success() {
+    echo -e "${GREEN}[SUCCESS]${NC} $1"
+}
+
+log_warning() {
+    echo -e "${YELLOW}[WARNING]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+if [[ $# -lt 3 ]]; then
+    log_error "Usage: $0 <PROJECT_ID> <ORCHESTRATOR_SA_EMAIL> <ACTIVE_USER_EMAIL>"
+    exit 1
+fi
+
+PROJECT_ID="$1"
+SA_EMAIL="$2"
+USER_EMAIL="$3"
+
+log_info "Configuring IAM Service Account Impersonation bindings..."
+log_info "Project: ${PROJECT_ID}"
+log_info "Service Account: ${SA_EMAIL}"
+log_info "User: ${USER_EMAIL}"
+
+# Apply the binding
+if gcloud iam service-accounts add-iam-policy-binding "${SA_EMAIL}" \
+    --member="user:${USER_EMAIL}" \
+    --role="roles/iam.serviceAccountTokenCreator" \
+    --project="${PROJECT_ID}" &>/dev/null; then
+    log_success "Successfully applied Token Creator binding to service account!"
+else
+    log_error "Failed to bind roles/iam.serviceAccountTokenCreator on service account."
+    exit 1
+fi
+
+log_info "IAM permission changes submitted to GCP."
+log_info "Starting active verification loop (impersonation token polling)..."
+
+MAX_ATTEMPTS=16
+WAIT_INTERVAL=15
+SUCCESS=false
+
+for ((attempt=1; attempt<=MAX_ATTEMPTS; attempt++)); do
+    log_info "Attempt $attempt of $MAX_ATTEMPTS: Requesting access token via impersonation..."
+    
+    # Redirect stderr to a temporary file to check for permission errors
+    ERR_OUT=$(mktemp)
+    
+    if gcloud auth print-access-token --impersonate-service-account="${SA_EMAIL}" &>/dev/null 2>"${ERR_OUT}"; then
+        log_success "GCP impersonation propagated successfully! Access token generated."
+        SUCCESS=true
+        rm -f "${ERR_OUT}"
+        break
+    else
+        ERR_MSG=$(cat "${ERR_OUT}")
+        rm -f "${ERR_OUT}"
+        
+        log_warning "Permission propagation pending. Waiting ${WAIT_INTERVAL}s before retry..."
+        sleep ${WAIT_INTERVAL}
+    fi
+done
+
+if [ "$SUCCESS" = true ]; then
+    log_success "IAM bindings are completely propagated and active!"
+    exit 0
+else
+    log_error "IAM permissions failed to propagate within $((MAX_ATTEMPTS * WAIT_INTERVAL)) seconds."
+    log_error "Please manually verify that user has impersonation privileges on ${SA_EMAIL}."
+    exit 1
+fi
diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh
new file mode 100755
index 00000000..7be8c4cb
--- /dev/null
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh
@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+#
+# Data Commons Platform (DCP) - Setup Pre-flight Validation Script
+# Automates operating system detection, CLI dependency checks, auto-installers,
+# authentication verification, and GCP API enablement.
+#
+
+set -euo pipefail
+
+# Standard formatting variables
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0;0m' # No Color
+
+log_info() {
+    echo -e "${BLUE}[INFO]${NC} $1"
+}
+
+log_success() {
+    echo -e "${GREEN}[SUCCESS]${NC} $1"
+}
+
+log_warning() {
+    echo -e "${YELLOW}[WARNING]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+# 1. OS Detection
+OS_TYPE="$(uname -s)"
+log_info "Detected operating system: ${OS_TYPE}"
+
+if [[ "${OS_TYPE}" != "Darwin" && "${OS_TYPE}" != "Linux" ]]; then
+    log_error "Unsupported operating system: ${OS_TYPE}. This setup requires macOS or Linux."
+    exit 1
+fi
+
+# Ensure base commands exist
+for cmd in curl unzip grep; do
+    if ! command -v "$cmd" &> /dev/null; then
+        log_error "Required system utility '$cmd' is missing. Please install it first."
+        exit 1
+    fi
+done
+
+# 2. Check & Install uv
+if command -v uv &> /dev/null; then
+    log_success "uv package manager is already installed: $(uv --version)"
+else
+    log_warning "uv is missing. Attempting standalone installation..."
+    if curl -LsSf https://astral.sh/uv/install.sh | sh; then
+        # Source the environment to update PATH immediately
+        if [ -f "$HOME/.local/bin/env" ]; then
+            source "$HOME/.local/bin/env"
+        elif [ -f "$HOME/.cargo/env" ]; then
+            source "$HOME/.cargo/env"
+        fi
+        export PATH="$HOME/.local/bin:$PATH"
+        if command -v uv &> /dev/null; then
+            log_success "uv was successfully installed: $(uv --version)"
+        else
+            log_error "uv was installed but could not be found in PATH. Please restart your terminal and re-run."
+            exit 1
+        fi
+    else
+        log_error "Failed to install uv. Please install it manually from https://astral.sh/uv"
+        exit 1
+    fi
+fi
+
+# 3. Check & Install Terraform
+if command -v terraform &> /dev/null; then
+    log_success "Terraform is already installed: $(terraform -version | head -n 1)"
+else
+    log_warning "Terraform is missing. Attempting installation..."
+    if [[ "${OS_TYPE}" == "Darwin" ]]; then
+        if command -v brew &> /dev/null; then
+            log_info "Using Homebrew to install Terraform..."
+            brew tap hashicorp/tap
+            brew install hashicorp/tap/terraform
+        else
+            log_warning "Homebrew not found. Downloading standalone Terraform binary..."
+            TF_URL="https://releases.hashicorp.com/terraform/1.8.0/terraform_1.8.0_darwin_amd64.zip"
+            curl -Lo /tmp/terraform.zip "${TF_URL}"
+            unzip -o /tmp/terraform.zip -d /usr/local/bin/ || unzip -o /tmp/terraform.zip -d "$HOME/.local/bin/"
+            rm /tmp/terraform.zip
+        fi
+    elif [[ "${OS_TYPE}" == "Linux" ]]; then
+        if command -v apt-get &> /dev/null; then
+            log_info "Using APT to install Terraform..."
+            sudo apt-get update && sudo apt-get install -y gnupg software-properties-common wget
+            wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg > /dev/null
+            echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+            sudo apt-get update && sudo apt-get install -y terraform
+        else
+            log_warning "APT package manager not found. Downloading standalone Terraform binary..."
+            TF_URL="https://releases.hashicorp.com/terraform/1.8.0/terraform_1.8.0_linux_amd64.zip"
+            curl -Lo /tmp/terraform.zip "${TF_URL}"
+            mkdir -p "$HOME/.local/bin"
+            unzip -o /tmp/terraform.zip -d "$HOME/.local/bin"
+            export PATH="$HOME/.local/bin:$PATH"
+            rm /tmp/terraform.zip
+        fi
+    fi
+
+    # Verify install
+    if command -v terraform &> /dev/null; then
+        log_success "Terraform was successfully installed: $(terraform -version | head -n 1)"
+    else
+        log_error "Terraform installation failed. Please install it manually."
+        exit 1
+    fi
+fi
+
+# 4. Check & Guide gcloud SDK
+if command -v gcloud &> /dev/null; then
+    log_success "gcloud CLI is already installed: $(gcloud --version | head -n 1)"
+else
+    log_error "gcloud CLI is missing."
+    log_info "To proceed, please download and install Google Cloud SDK by running the following commands or visiting: https://cloud.google.com/sdk/docs/install"
+    if [[ "${OS_TYPE}" == "Darwin" ]]; then
+        log_info "macOS command: brew install --cask google-cloud-sdk"
+    elif [[ "${OS_TYPE}" == "Linux" ]]; then
+        log_info "Linux quick-install: curl https://sdk.cloud.google.com | bash"
+    fi
+    exit 1
+fi
+
+# 5. Validate active GCP project setting
+ACTIVE_PROJECT="$(gcloud config get-value project 2>/dev/null || echo "")"
+if [[ -z "${ACTIVE_PROJECT}" || "${ACTIVE_PROJECT}" == "(unset)" ]]; then
+    log_warning "No active GCP project is configured in gcloud."
+    log_info "Please set your project context using: gcloud config set project [PROJECT_ID]"
+else
+    log_success "Active GCP project context verified: ${ACTIVE_PROJECT}"
+fi
+
+# 6. Verify Authentication State
+log_info "Checking GCP authentication state..."
+if gcloud auth application-default print-access-token &> /dev/null; then
+    log_success "Active Application Default Credentials (ADC) verified."
+else
+    log_warning "No active Application Default Credentials (ADC) found."
+    log_info "Please login by running: gcloud auth application-default login"
+    log_info "Once authentication completes, re-run this pre-flight validation."
+    exit 2
+fi
+
+log_success "DCP Setup Pre-flight checks completed successfully!"
diff --git a/packages/datacommons-cli/scripts/install-agent.sh b/packages/datacommons-cli/scripts/install-agent.sh
new file mode 100755
index 00000000..068872fe
--- /dev/null
+++ b/packages/datacommons-cli/scripts/install-agent.sh
@@ -0,0 +1,231 @@
+#!/usr/bin/env bash
+#
+# Data Commons Platform (DCP) - Agentic Assistant Installer
+# Autonomously provisions a workspace with partner setup and operations skills,
+# virtual environment, CLI, and triggers validation.
+#
+
+set -euo pipefail
+
+# ==========================================
+# Overridable Repository Settings (For dev branch & fork overrides)
+# ==========================================
+DCP_GIT_REPO="${DCP_GIT_REPO:-https://github.com/datacommonsorg/datacommons.git}"
+DCP_GIT_BRANCH="${DCP_GIT_BRANCH:-main}"
+DCP_GITHUB_RAW_BASE="${DCP_GITHUB_RAW_BASE:-https://raw.githubusercontent.com/datacommonsorg/datacommons}"
+
+# ==========================================
+# 1. Initialization & Logging Functions
+# ==========================================
+
+setup_colors() {
+    RED='\033[0;31m'
+    GREEN='\033[0;32m'
+    YELLOW='\033[1;33m'
+    BLUE='\033[0;34m'
+    NC='\033[0;0m' # No Color
+}
+
+log_info() {
+    echo -e "${BLUE}[INFO]${NC} $1"
+}
+
+log_success() {
+    echo -e "${GREEN}[SUCCESS]${NC} $1"
+}
+
+log_warning() {
+    echo -e "${YELLOW}[WARNING]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+# ==========================================
+# 2. Target Directory Resolution
+# ==========================================
+
+resolve_target_dir() {
+    local current_dir
+    current_dir="$(pwd)"
+    echo -e -n "${YELLOW}[INPUT]${NC} Where would you like to install the Data Commons agent? [Default: ${current_dir}]: "
+    read -r user_input
+    
+    # Use current directory if input is empty
+    local target_path="${user_input:-${current_dir}}"
+    
+    # Resolve absolute path
+    mkdir -p "${target_path}"
+    TARGET_DIR="$(cd "${target_path}" && pwd)"
+    log_info "Target workspace resolved to: ${TARGET_DIR}"
+}
+
+# ==========================================
+# 3. Environment & Mode Detection
+# ==========================================
+
+detect_mode() {
+    log_info "Detecting running environment mode..."
+    
+    # Get current script path
+    local script_dir
+    script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    
+    # Check if we are running inside a local cloned dc-datacommons repository
+    if [[ "${script_dir}" == *"packages/datacommons-cli/scripts"* ]]; then
+        DEV_MODE=true
+        # Root repo directory is 3 levels up from the script folder
+        REPO_ROOT="$(cd "${script_dir}/../../.." && pwd)"
+        log_success "Development Mode Detected! Local Repository Root: ${REPO_ROOT}"
+    else
+        DEV_MODE=false
+        log_info "Production/Remote Mode Detected. Assets will be fetched from GitHub."
+    fi
+}
+
+# ==========================================
+# 4. Skill Deployment (Local vs. Remote)
+# ==========================================
+
+copy_local_skills() {
+    log_info "Copying local skill assets in Development Mode..."
+    
+    local local_skills_source="${REPO_ROOT}/packages/datacommons-cli/datacommons_cli/skills"
+    local target_skills_dest="${TARGET_DIR}/.agent/skills"
+    
+    if [[ ! -d "${local_skills_source}" ]]; then
+        log_error "Source skills folder not found at: ${local_skills_source}"
+        exit 1
+    fi
+    
+    mkdir -p "${target_skills_dest}"
+    cp -R "${local_skills_source}/dcp-setup" "${target_skills_dest}/"
+    cp -R "${local_skills_source}/dcp-ops" "${target_skills_dest}/"
+    
+    # Ensure scripts are marked executable
+    chmod +x "${target_skills_dest}/dcp-setup/scripts/"*.sh 2>/dev/null || true
+    
+    log_success "Skills successfully copied to: ${target_skills_dest}"
+}
+
+download_remote_skills() {
+    log_info "Downloading remote skill assets from GitHub..."
+    
+    local github_raw_url="${DCP_GITHUB_RAW_BASE}/${DCP_GIT_BRANCH}/packages/datacommons-cli/datacommons_cli/skills"
+    local target_skills_dest="${TARGET_DIR}/.agent/skills"
+    
+    local files_to_fetch=(
+        "dcp-setup/SKILL.md"
+        "dcp-setup/scripts/preflight_check.sh"
+        "dcp-setup/scripts/poll_iam.sh"
+        "dcp-ops/SKILL.md"
+    )
+    
+    for file_path in "${files_to_fetch[@]}"; do
+        local dest_file="${target_skills_dest}/${file_path}"
+        local source_url="${github_raw_url}/${file_path}"
+        
+        # Create parent directory for the file
+        mkdir -p "$(dirname "${dest_file}")"
+        
+        log_info "Fetching: ${file_path}..."
+        if ! curl -sSfL "${source_url}" -o "${dest_file}"; then
+            log_error "Failed to fetch file from: ${source_url}"
+            exit 1
+        fi
+    done
+    
+    # Ensure scripts are marked executable
+    chmod +x "${target_skills_dest}/dcp-setup/scripts/"*.sh 2>/dev/null || true
+    
+    log_success "Skills successfully downloaded to: ${target_skills_dest}"
+}
+
+deploy_skills() {
+    if [ "${DEV_MODE}" = true ]; then
+        copy_local_skills
+    else
+        download_remote_skills
+    fi
+}
+
+# ==========================================
+# 5. Python Virtual Env & Package Setup
+# ==========================================
+
+ensure_uv_installed() {
+    if ! command -v uv &> /dev/null; then
+        log_warning "uv package manager is missing. Attempting automatic installation..."
+        if curl -LsSf https://astral.sh/uv/install.sh | sh; then
+            export PATH="$HOME/.local/bin:$PATH"
+        else
+            log_error "Failed to install uv automatically. Please install it manually from https://astral.sh/uv"
+            exit 1
+        fi
+    fi
+}
+
+initialize_python_env() {
+    log_info "Provisioning Python virtual environment..."
+    ensure_uv_installed
+    
+    cd "${TARGET_DIR}"
+    uv venv
+    
+    # Install package in target virtual environment
+    if [ "${DEV_MODE}" = true ]; then
+        log_info "Installing local datacommons CLI package in editable mode..."
+        # Resolve relative package paths from local repository root
+        uv pip install -e "${REPO_ROOT}/packages/datacommons-cli" -e "${REPO_ROOT}/packages/datacommons-admin"
+    else
+        log_info "Installing datacommons CLI package from GitHub release..."
+        uv pip install "git+${DCP_GIT_REPO}@${DCP_GIT_BRANCH}#subdirectory=packages/datacommons-cli"
+    fi
+    
+    log_success "Python virtual environment and CLI package successfully configured!"
+}
+
+# ==========================================
+# 6. Pre-flight Orchestration
+# ==========================================
+
+run_preflight_validation() {
+    log_info "Triggering local pre-flight checks..."
+    
+    local preflight_script="${TARGET_DIR}/.agent/skills/dcp-setup/scripts/preflight_check.sh"
+    
+    if [[ -f "${preflight_script}" ]]; then
+        bash "${preflight_script}"
+    else
+        log_warning "Pre-flight script not found at: ${preflight_script}. Skipping."
+    fi
+}
+
+# ==========================================
+# 7. Main Execution Sequence
+# ==========================================
+
+main() {
+    setup_colors
+    log_info "========================================="
+    log_info "   Data Commons Platform Agent Installer "
+    log_info "========================================="
+    
+    resolve_target_dir
+    detect_mode
+    deploy_skills
+    initialize_python_env
+    run_preflight_validation
+    
+    log_success "===================================================="
+    log_success "   DCP Agent Installation Completed Successfully!  "
+    log_success "===================================================="
+    log_info "Your workspace at '${TARGET_DIR}' is fully configured."
+    log_info "Please open your agentic assistant inside this workspace and ask:"
+    log_info "\"Help me set up my Data Commons\""
+    log_info "===================================================="
+}
+
+# Trigger Main
+main

From 2a8f8f96df378cd8fc61889048d6b94caf7ef155 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Fri, 22 May 2026 14:06:34 -0700
Subject: [PATCH 02/23] Fix maps_api_key compile syntax error in auth template

---
 infra/dcp/modules/auth/main.tf | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/infra/dcp/modules/auth/main.tf b/infra/dcp/modules/auth/main.tf
index ff04f557..17c738b8 100644
--- a/infra/dcp/modules/auth/main.tf
+++ b/infra/dcp/modules/auth/main.tf
@@ -1,6 +1,8 @@
 locals {
   name_prefix        = var.namespace != "" ? "${var.namespace}-" : ""
-  maps_api_key_value = var.maps_api_key != null ? var.maps_api_key : try(google_apikeys_key.maps_api_key[0].key_string, "")
+  # Temporary compile-time fix: Removed '[0]' index reference because 'google_apikeys_key.maps_api_key' count is commented out upstream.
+  # If upstream re-enables count in the future, restore the '[0]' list index reference here.
+  maps_api_key_value = var.maps_api_key != null ? var.maps_api_key : try(google_apikeys_key.maps_api_key.key_string, "")
 }
 
 

From 39176c6974e0caaa1deda5e1358957618ac480fe Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Fri, 22 May 2026 17:59:23 -0700
Subject: [PATCH 03/23] Update setup and ops skills progress telemetry
 guidelines

---
 .../datacommons_cli/skills/dcp-ops/SKILL.md   | 26 ++++-
 .../datacommons_cli/skills/dcp-setup/SKILL.md | 96 +++++++++++++++----
 2 files changed, 102 insertions(+), 20 deletions(-)

diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-ops/SKILL.md b/packages/datacommons-cli/datacommons_cli/skills/dcp-ops/SKILL.md
index 9e56347e..9cce766e 100644
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-ops/SKILL.md
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-ops/SKILL.md
@@ -12,7 +12,25 @@ This skill guides an agentic coding assistant through Day-2 administration, data
 > * The agent must **NEVER** silently automate or modify cloud resources (GCS, Spanner, Cloud Run) without explaining the exact proposed changes and waiting for explicit verbal confirmation.
 > * Before executing *any* command, the agent must present the proposed command line to the user and ask for confirmation.
 > * **After executing any command**, the agent must immediately provide a clear, useful summary of exactly what was done, what resources were changed, and what status or outputs were returned.
-> * **GCP Console Deep-Links**: Whenever a resource is created or modified, the agent must construct and print the direct Google Cloud Console deep-links (e.g. GCS browser, Spanner console, Cloud Run dashboard) to enable instant user verification and audit.
+> * **GCP Console Deep-Links**: Whenever a resource is created, modified, or queried, the agent must **generously construct and print direct GCP Console deep-links** (such as Spanner database tables, GCS folder paths, Cloud Run metrics, Cloud Workflows executions, and Dataflow pipeline graphs) to enable the user to instantly inspect and audit the live resources by themselves at any time.
+> * **Active Progress Updates (Quiet Polling & Percent Spinner)**: During long-running operations (such as `terraform apply` or data ingestion pipelines), the agent must never go silent, but **must avoid noisy output**. Print only a **highly-condensed, 3-line progress block** using **Exponential Backoff Polling Heuristics** (checking at 10s, 20s, 40s, and then scaling up to a maximum quiet heartbeat of 60s for the remainder of the long-running job).
+>   * **Structured Progress Layout (Compact 2-Column Table)**: All progress checks must be formatted in a clean, headerless, 2-column Markdown table (with values wrapped in inline backticks to render as yellow/amber pills):
+>     | | |
+>     | :--- | :--- |
+>     | **Phase** | `Phase X/3: [Global Phase Name]` |
+>     | **Status** | `[Factual, highly-condensed numerical metrics/counters]` |
+>     | **Elapsed** | `[phase_time] (Total pipeline time: [total_ingestion_time])` |
+>   * **Unified Ingestion Phase Labeling**:
+>     * 🛠️ **Phase 1 of 3: Preprocessing Container**
+>       * *Sub-stage 1/3 (CSV Import)*: `Status: Sub-stage 1/3 - CSV Import: 62.6% (979 / 1,562 files processed)`
+>       * *Sub-stage 2/3 (Indexing)*: `Status: Sub-stage 2/3 - Database Indexing: Indexing 13.9M observations`
+>       * *Sub-stage 3/3 (Export)*: `Status: Sub-stage 3/3 - JSON-LD Export: Staging 13.98M observations. 41 Node & 318+ Observation shards completed`
+>     * 🔗 **Phase 2 of 3: Cloud Workflows Orchestration**
+>       * `Status: Container completed successfully. Workflows state machine orchestrating Dataflow trigger`
+>     * ⚡ **Phase 3 of 3: Dataflow Parallel Loader**
+>       * **Dynamic Spanner Ingest Heuristic**: Extract active mutation metrics: `gcloud beta dataflow metrics list [JOB_ID] --project=[PROJECT_ID] --region=[REGION]`. Look up `mutation_groups_write_success` (staged mutation count) and `spanner_write_latency_ms_MEAN` (average batch latency) and print: `Status: Written graph mutations: [mutation_count] successfully loaded to Spanner (mean batch latency: [latency]ms)`.
+>   * Only print a detailed block or link when a **major pipeline transition occurs** or upon final success/failure. **Silent Scheduler Rule**: When scheduling these background timers, the agent must never print redundant closing text (such as "The progress timer is active. I will stand by..."). Simply print the 3-line status update, call the `schedule` tool, and end the turn immediately.
+> * **Brevity & Tabular Formatting**: Infrastructure operations require absolute clarity. Avoid verbose, narrative, or flowery summaries. Present complex details (such as configuration changes, resource impacts, plans, or status reports) using clean markdown tables, bullet points, or short, direct facts. Keep narrative text blocks extremely concise.
 > * Keep the user fully informed of background tasks (e.g., logs monitoring, proxy tunnels), providing direct console URLs for observability.
 
 ---
@@ -74,11 +92,13 @@ Use this workflow to copy new data files and trigger a new ingestion execution t
   ```
 
 ### 2. Trigger the Ingestion Pipeline
-* Explain that we are about to trigger the Cloud Run Ingestion Job, show the command, and ask for permission to execute:
+* **Parameter-free Ingestion Warning**: Explain that the `ingest start` command is completely **parameter-free**. It dynamically resolves variables from Terraform outputs and reads staged configurations directly from GCS.
+* **Strict Command Constraint**: **NEVER** append flags like `--import-name` or `--import-list` (handled automatically).
+* Present the command and trigger:
   ```bash
   uv run datacommons admin ingest start
   ```
-* **Provide Ingestion Logs Link**: Once executed, capture the **Job Console Link** from the logs and present it clearly in the chat so the user has direct observability in their browser console.
+* **Provide Observability Links**: Print both the **Job Console Link** and **Workflow Console Link** in the chat so the user has direct observability.
 
 ---
 
diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
index ede13039..1360ea24 100644
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
@@ -13,7 +13,25 @@ This skill guides an agentic coding assistant through the setup and provisioning
 > * The agent must **NEVER** create or modify GCP resources (GCS buckets, Spanner, Cloud Run) without first explaining the exact proposed action and waiting for explicit verbal confirmation.
 > * Before executing *any* command, the agent must present the proposed command line to the user and ask for confirmation.
 > * **After executing any command**, the agent must immediately provide a clear, useful summary of exactly what was done, what resources were created/changed, and what status or outputs were returned.
-> * **GCP Console Deep-Links**: Whenever a resource is created or modified, the agent must construct and print the direct Google Cloud Console deep-links (e.g. GCS browser, Spanner console, Cloud Run dashboard) to enable instant user verification and audit.
+> * **GCP Console Deep-Links**: Whenever a resource is created, modified, or queried, the agent must **generously construct and print direct GCP Console deep-links** (such as Spanner database tables, GCS folder paths, Cloud Run metrics, Cloud Workflows executions, and Dataflow pipeline graphs) to enable the user to instantly inspect and audit the live resources by themselves at any time.
+> * **Active Progress Updates (Quiet Polling & Percent Spinner)**: During long-running operations (such as `terraform apply` or data ingestion pipelines), the agent must never go silent, but **must avoid noisy output**. Print only a **highly-condensed, 3-line progress block** using **Exponential Backoff Polling Heuristics** (checking at 10s, 20s, 40s, and then scaling up to a maximum quiet heartbeat of 60s for the remainder of the long-running job).
+>   * **Structured Progress Layout (Compact 2-Column Table)**: All progress checks must be formatted in a clean, headerless, 2-column Markdown table (with values wrapped in inline backticks to render as yellow/amber pills):
+>     | | |
+>     | :--- | :--- |
+>     | **Phase** | `Phase X/3: [Global Phase Name]` |
+>     | **Status** | `[Factual, highly-condensed numerical metrics/counters]` |
+>     | **Elapsed** | `[phase_time] (Total pipeline time: [total_ingestion_time])` |
+>   * **Unified Ingestion Phase Labeling**:
+>     * 🛠️ **Phase 1 of 3: Preprocessing Container**
+>       * *Sub-stage 1/3 (CSV Import)*: `Status: Sub-stage 1/3 - CSV Import: 62.6% (979 / 1,562 files processed)`
+>       * *Sub-stage 2/3 (Indexing)*: `Status: Sub-stage 2/3 - Database Indexing: Indexing 13.9M observations`
+>       * *Sub-stage 3/3 (Export)*: `Status: Sub-stage 3/3 - JSON-LD Export: Staging 13.98M observations. 41 Node & 318+ Observation shards completed`
+>     * 🔗 **Phase 2 of 3: Cloud Workflows Orchestration**
+>       * `Status: Container completed successfully. Workflows state machine orchestrating Dataflow trigger`
+>     * ⚡ **Phase 3 of 3: Dataflow Parallel Loader**
+>       * **Dynamic Spanner Ingest Heuristic**: Extract active mutation metrics: `gcloud beta dataflow metrics list [JOB_ID] --project=[PROJECT_ID] --region=[REGION]`. Look up `mutation_groups_write_success` (staged mutation count) and `spanner_write_latency_ms_MEAN` (average batch latency) and print: `Status: Written graph mutations: [mutation_count] successfully loaded to Spanner (mean batch latency: [latency]ms)`.
+>   * Only print a detailed block or link when a **major pipeline transition occurs** or upon final success/failure. **Silent Scheduler Rule**: When scheduling these background timers, the agent must never print redundant closing text (such as "The progress timer is active. I will stand by..."). Simply print the 3-line status update, call the `schedule` tool, and end the turn immediately.
+> * **Brevity & Tabular Formatting**: Infrastructure operations require absolute clarity. Avoid verbose, narrative, or flowery summaries. Present complex details (such as configuration changes, resource impacts, plans, or status reports) using clean markdown tables, bullet points, or short, direct facts. Keep narrative text blocks extremely concise.
 
 ---
 
@@ -94,7 +112,7 @@ Once parameters are harvested, present the command and the inputs to the user:
 
 ### 3. Variable Verification
 * Read `terraform.tfvars` and show the user the generated configuration variables.
-* Confirm if they want to make any manual adjustments (e.g., changing machine sizes or regions) before proceeding.
+* Confirm if the user wants to make any other manual adjustments (e.g., changing machine sizes or regions) before proceeding.
 
 ---
 
@@ -114,14 +132,44 @@ List the necessary APIs (Spanner, Cloud Run, Secret Manager, Artifact Registry)
 gcloud services enable spanner.googleapis.com run.googleapis.com secretmanager.googleapis.com artifactregistry.googleapis.com cloudbuild.googleapis.com iamcredentials.googleapis.com
 ```
 
-### 3. Spanner Database Conflict Check
-* Run `gcloud spanner instances list --project=[PROJECT_ID]` to check if the configured `dcp_spanner_instance_id` already exists.
-* **If it exists, report immediately**:
-  > ⚠️ **Existing Spanner Instance Detected**: The instance ID `[dcp_spanner_instance_id]` already exists in GCP project `[project_id]`.
-  > * **Reusing** this instance may overwrite existing database tables.
-  > * **Creating a new one** requires modifying `terraform.tfvars` to use a unique instance ID.
-  >
-  > *Please tell me how you want to proceed: [Option A: Reuse Instance] or [Option B: Configure New Unique ID].*
+### 3. Spanner Instance Discovery & Proactive Configuration (Step 1)
+* **Mandatory Order**: This check must be executed and finalized *before* initiating the BigQuery check.
+* Run `gcloud spanner instances list --project=[PROJECT_ID]` to discover active Spanner resources in the project.
+* **If active Spanner instances are found**:
+  * Present the list clearly to the user in the chat:
+    > 🔍 **Active Spanner Instances Discovered**:
+    > I found these active Spanner instances in project `[project_id]`:
+    > 1. `dcp-shared-spanner-instance`
+    > 2. `dcp-instance-dev`
+    > ...
+    > Would you like to:
+    > * **Option A: Reuse an existing instance (Highly Recommended to save project capacity and costs)**? Please tell me which number/ID to use.
+    > * **Option B: Create a brand-new Spanner instance**? (Will generate `[namespace]-dc-instance` and increase project billing).
+  * **Action based on Choice**:
+    * **If Option A (Reuse)**: Ask the user if they approve, then automatically update `terraform.tfvars` to set `spanner_create_instance = false` and `spanner_instance_id = "[chosen_instance_id]"`. Inform the user once the file is updated.
+    * **If Option B (Create New)**:
+      * **Display Name Length Safety Check**: Calculate the resulting display name length: `${namespace}-${spanner_instance_id}`.
+      * If this length exceeds the GCP limit of **30 characters** (e.g., `dcp-keyurs-dcp-keyurs-dc-instance` is 34 characters):
+        * Prompt the user immediately with a warning and recommend a shortened `spanner_instance_id` like `dc-inst` so the resulting name (`[namespace]-dc-inst`) remains safe under 30 characters.
+      * Ask the user if they approve the verified setting, then automatically update `terraform.tfvars` to set `spanner_create_instance = true` and `spanner_instance_id = "[chosen_shortened_instance_id]"`. Inform the user once the file is updated.
+* **If no active Spanner instances are found**:
+  * Automatically update `terraform.tfvars` to set `spanner_create_instance = true` and `spanner_instance_id = "[namespace]-dc-inst"` to ensure a fresh database instance is provisioned cleanly and inform the user.
+
+* **Wait for Spanner configuration to be finalized and written before moving to the next step.**
+
+### 4. BigQuery Slot Reservation Conflict Check (Step 2)
+* **Mandatory Order**: Initiate this check *only* after the Spanner configuration above is fully completed and written to disk.
+* **Retrieve Target Region**: Parse the `region` variable from `terraform.tfvars` or `variables.tf` (defaulting to `us-central1` if it is commented out or blank). Use this string as the target `[REGION]` location.
+* Run a query to check for existing BigQuery reservations in that target region location:
+  ```bash
+  bq ls --reservation --project_id=[PROJECT_ID] --location=[REGION]
+  ```
+* **If a reservation named `default` already exists**:
+  * Present this discovery to the user and explain that we should reuse it to avoid conflicts and save slots.
+  * Upon user confirmation, write/append `spanner_create_bigquery_reservation = false` to `terraform.tfvars` and confirm the write to the user.
+* **If no `default` reservation is found**:
+  * Inform the user that no reservation was found, and ask if they want to create a new one (sets `spanner_create_bigquery_reservation = true`) or reuse a pre-existing default one if they know it exists.
+  * Write the selected value to `terraform.tfvars` and confirm.
 
 ---
 
@@ -144,12 +192,24 @@ Run `terraform plan` and output the changes to a temporary file. Parse this plan
 
 **Do not run `terraform apply` until the user reviews the Impact Report and replies with explicit verbal approval (e.g., "Yes", "Approve").**
 
-### 3. Apply Infrastructure Changes
+### 3. Apply Infrastructure Changes & Interactive Fail-safe
 Upon receiving approval, execute the deployment:
 ```bash
 terraform apply -auto-approve
 ```
-Display the outputs to the user, highlighting `data_bucket_name` and `cloud-run-service-name`.
+* **Transient GCP Propagation Fail-safe**:
+  * If the deployment fails due to transient GCP resource replication latency (e.g., `404 Instance/Database not found` or `409 Already exists` during IAM database user bindings):
+    * **Immediately inform the user of the failure** in the chat.
+    * Explain that this is a normal, expected GCP resource replication latency issue, and that the previous plan binary is now stale due to a partial apply.
+    * Prompt the user in the chat:
+      > 🔄 **GCP Resource Propagation Delay Detected**:
+      > The deployment hit a transient GCP replication delay. This is completely normal and expected.
+      > Since a partial apply occurred, our old plan is now stale. 
+      > * I will compile a fresh plan using `terraform plan -out=tfplan.binary` immediately.
+      > * I will then present the new short Impact Report of the remaining resources for your review.
+      > * Do you approve compiling the fresh plan and retrying?
+    * **Wait for the user's explicit verbal confirmation** before executing the new plan and apply sequence.
+* Display the final successful outputs to the user, highlighting `data_bucket_name` and `cloud-run-service-name`.
 
 ---
 
@@ -192,11 +252,13 @@ uv run datacommons admin init-db
 ```
 
 ### 3. Start the Ingestion Job
-Present the command to trigger the Cloud Run pipeline to ingest the data:
-```bash
-uv run datacommons admin ingest start
-```
-* **Provide Console Links**: Print the printed **Job Console Link** clearly in the chat so the user can watch the live logs in their browser console.
+* **Parameter-free Ingestion Warning**: Explain that the `ingest start` command is completely **parameter-free**. It dynamically resolves Cloud Run variables from Terraform outputs, and reads staged config files directly from GCS.
+* **Strict Command Constraint**: **NEVER** append flags like `--import-name` or `--import-list` (the pipeline handles this automatically).
+* Present the command and trigger:
+  ```bash
+  uv run datacommons admin ingest start
+  ```
+* **Provide Console Links**: Print the **Job Console Link** and **Workflow Console Link** clearly in the chat so the user can watch the live executions.
 
 ---
 

From 5f5abd48568bb7573e88d124daac35142d62290c Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sat, 23 May 2026 00:26:46 -0700
Subject: [PATCH 04/23] Improve ingestion workflow permissions, setup ingress
 options, and maps API key security prompt

---
 infra/dcp/modules/ingestion/workflow/main.tf         |  8 ++++++++
 .../datacommons_cli/skills/dcp-setup/SKILL.md        | 12 +++++++++++-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/infra/dcp/modules/ingestion/workflow/main.tf b/infra/dcp/modules/ingestion/workflow/main.tf
index a87ca268..cdefe198 100644
--- a/infra/dcp/modules/ingestion/workflow/main.tf
+++ b/infra/dcp/modules/ingestion/workflow/main.tf
@@ -213,3 +213,11 @@ resource "google_project_iam_member" "workflow_run_viewer" {
   role    = "roles/run.viewer"
   member  = "serviceAccount:${google_service_account.workflow_sa[0].email}"
 }
+
+resource "google_cloud_run_v2_service_iam_member" "workflow_sa_service_developer" {
+  count    = var.deploy && var.enable_datacommons_services ? 1 : 0
+  location = var.region
+  name     = "${local.name_prefix}dc-datacommons-service"
+  role     = "roles/run.developer"
+  member   = "serviceAccount:${google_service_account.workflow_sa[0].email}"
+}
diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
index 1360ea24..c3db954c 100644
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
@@ -110,8 +110,18 @@ Once parameters are harvested, present the command and the inputs to the user:
   uv run datacommons admin init --project=[project_id] --namespace=[namespace] --key=[api_key] --auto-approve
   ```
 
-### 3. Variable Verification
+### 3. Variable Verification & Ingress Security Prompt
 * Read `terraform.tfvars` and show the user the generated configuration variables.
+* **Interactive Ingress Security Choice (Mandatory Prompt)**:
+  * Ask the user explicitly in the chat: *"Would you like to make your Custom Data Commons website publicly accessible to the open internet (best for sharing and quick testing), or keep it private and secured behind IAM OIDC authentication (default, best for secure enterprise data)?"*
+  * **If Public is selected**: Locate `datacommons_services_allow_unauthenticated_access` inside `terraform.tfvars`, uncomment it, and set it to `true`.
+  * **If Private/Default is selected**: Ensure `datacommons_services_allow_unauthenticated_access` remains commented out or set to `false`.
+* **Explicit Google Maps API Key Choice Heuristic (Mandatory Prompt)**:
+  * The agent must inspect `terraform.tfvars` for Google Maps credentials.
+  * If `auth_google_maps_api_key` is set to `"TODO"` and `auth_create_google_maps_api_key` is set to `false`, the agent **must explicitly prompt the user in the chat**:
+    * *"A valid Google Maps API key is required for using Data Commons. How would you like to proceed?"*
+    * **Option 1: Use Existing Key**: Ask the user to provide their existing Maps API key string. Once provided, the agent must write it directly to `auth_google_maps_api_key` inside `terraform.tfvars`.
+    * **Option 2: Auto-generate restricted key**: Ask the user if they want Terraform to create a new restricted key natively. Once approved, the agent must write `auth_create_google_maps_api_key = true` and set `auth_google_maps_api_key = null` inside `terraform.tfvars` so Terraform provisions it securely.
 * Confirm if the user wants to make any other manual adjustments (e.g., changing machine sizes or regions) before proceeding.
 
 ---

From cab770914bd5a51513df86d1596c6ce492fd5314 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sat, 23 May 2026 00:30:42 -0700
Subject: [PATCH 05/23] Remove Day-2 operations skill

---
 .../datacommons_cli/skills/dcp-ops/SKILL.md   | 191 ------------------
 1 file changed, 191 deletions(-)
 delete mode 100644 packages/datacommons-cli/datacommons_cli/skills/dcp-ops/SKILL.md

diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-ops/SKILL.md b/packages/datacommons-cli/datacommons_cli/skills/dcp-ops/SKILL.md
deleted file mode 100644
index 9cce766e..00000000
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-ops/SKILL.md
+++ /dev/null
@@ -1,191 +0,0 @@
----
-name: dcp-ops
-description: Procedural guide and strict safety guidelines for managing, upgrading, and maintaining an active Data Commons Platform (DCP) instance. Enforces absolute transparency, step-by-step pacing, and user approval gates.
----
-
-# Data Commons Platform Operations (dcp-ops)
-
-This skill guides an agentic coding assistant through Day-2 administration, data loading, configuration upgrades, and troubleshooting of an existing Data Commons Platform (DCP) instance.
-
-> [!IMPORTANT]
-> **CRITICAL COMPLIANCE DIRECTIVE: USER CONTROL & TRANSPARENCY**
-> * The agent must **NEVER** silently automate or modify cloud resources (GCS, Spanner, Cloud Run) without explaining the exact proposed changes and waiting for explicit verbal confirmation.
-> * Before executing *any* command, the agent must present the proposed command line to the user and ask for confirmation.
-> * **After executing any command**, the agent must immediately provide a clear, useful summary of exactly what was done, what resources were changed, and what status or outputs were returned.
-> * **GCP Console Deep-Links**: Whenever a resource is created, modified, or queried, the agent must **generously construct and print direct GCP Console deep-links** (such as Spanner database tables, GCS folder paths, Cloud Run metrics, Cloud Workflows executions, and Dataflow pipeline graphs) to enable the user to instantly inspect and audit the live resources by themselves at any time.
-> * **Active Progress Updates (Quiet Polling & Percent Spinner)**: During long-running operations (such as `terraform apply` or data ingestion pipelines), the agent must never go silent, but **must avoid noisy output**. Print only a **highly-condensed, 3-line progress block** using **Exponential Backoff Polling Heuristics** (checking at 10s, 20s, 40s, and then scaling up to a maximum quiet heartbeat of 60s for the remainder of the long-running job).
->   * **Structured Progress Layout (Compact 2-Column Table)**: All progress checks must be formatted in a clean, headerless, 2-column Markdown table (with values wrapped in inline backticks to render as yellow/amber pills):
->     | | |
->     | :--- | :--- |
->     | **Phase** | `Phase X/3: [Global Phase Name]` |
->     | **Status** | `[Factual, highly-condensed numerical metrics/counters]` |
->     | **Elapsed** | `[phase_time] (Total pipeline time: [total_ingestion_time])` |
->   * **Unified Ingestion Phase Labeling**:
->     * 🛠️ **Phase 1 of 3: Preprocessing Container**
->       * *Sub-stage 1/3 (CSV Import)*: `Status: Sub-stage 1/3 - CSV Import: 62.6% (979 / 1,562 files processed)`
->       * *Sub-stage 2/3 (Indexing)*: `Status: Sub-stage 2/3 - Database Indexing: Indexing 13.9M observations`
->       * *Sub-stage 3/3 (Export)*: `Status: Sub-stage 3/3 - JSON-LD Export: Staging 13.98M observations. 41 Node & 318+ Observation shards completed`
->     * 🔗 **Phase 2 of 3: Cloud Workflows Orchestration**
->       * `Status: Container completed successfully. Workflows state machine orchestrating Dataflow trigger`
->     * ⚡ **Phase 3 of 3: Dataflow Parallel Loader**
->       * **Dynamic Spanner Ingest Heuristic**: Extract active mutation metrics: `gcloud beta dataflow metrics list [JOB_ID] --project=[PROJECT_ID] --region=[REGION]`. Look up `mutation_groups_write_success` (staged mutation count) and `spanner_write_latency_ms_MEAN` (average batch latency) and print: `Status: Written graph mutations: [mutation_count] successfully loaded to Spanner (mean batch latency: [latency]ms)`.
->   * Only print a detailed block or link when a **major pipeline transition occurs** or upon final success/failure. **Silent Scheduler Rule**: When scheduling these background timers, the agent must never print redundant closing text (such as "The progress timer is active. I will stand by..."). Simply print the 3-line status update, call the `schedule` tool, and end the turn immediately.
-> * **Brevity & Tabular Formatting**: Infrastructure operations require absolute clarity. Avoid verbose, narrative, or flowery summaries. Present complex details (such as configuration changes, resource impacts, plans, or status reports) using clean markdown tables, bullet points, or short, direct facts. Keep narrative text blocks extremely concise.
-> * Keep the user fully informed of background tasks (e.g., logs monitoring, proxy tunnels), providing direct console URLs for observability.
-
----
-
-## Phase 1: Active Environment Discovery
-
-Before performing any operations, the agent must inspect the current directory to map out the active infrastructure state.
-
-### 1. Verify Active Provisioning
-* Verify that `main.tf` and `terraform.tfvars` exist and that a valid Terraform state is present.
-* If no infrastructure has been provisioned yet, stop immediately and direct the user to run **`dcp-setup`** first.
-
-### 2. Read Active GCP Project Context
-* Parse `terraform.tfvars` to extract `project_id`, `namespace`, and target instance sizes.
-* Set the active gcloud project context and show the user:
-  ```bash
-  gcloud config set project [project_id]
-  ```
-* Verify the active user has permission to query project services.
-
----
-
-## Phase 2: Updating System Configuration & Deployments (Gate 1 Approval)
-
-Use this workflow whenever you need to deploy a new Docker image or update configuration variables.
-
-### 1. Edit Configuration
-* Instruct the user that we are about to update the configuration.
-* Present the proposed modifications inside `terraform.tfvars` (e.g., changing `cdc_web_service_image`, `cdc_data_job_image`, or `gcs_data_bucket_input_folder`) and wait for approval before writing any changes.
-
-### 2. Apply Configuration Changes (Gate 1 Approval)
-* Run a dry-run plan to inspect the proposed modifications:
-  ```bash
-  terraform init  # Confirm plugins are initialized
-  terraform plan -out=tfplan.out
-  ```
-* Generate a detailed **Impact Report** listing:
-  * **Modified Variables**: Old value vs. proposed new value.
-  * **Added/Changed Cloud Resources**: Resources affected by the plan.
-  * **Safety Guarantee**: Verify that no Spanner instances or active production databases are being destroyed or modified in a destructive manner.
-* **Do not run `terraform apply` until the user reviews this Impact Report and gives explicit verbal confirmation (e.g., "Yes", "Approve").**
-* Upon approval, execute the deployment:
-  ```bash
-  terraform apply tfplan.out
-  ```
-
----
-
-## Phase 3: Seed Data Updates & Custom Ingestions
-
-Use this workflow to copy new data files and trigger a new ingestion execution to reload the Spanner database.
-
-### 1. Copy Custom Data to GCS Ingestion Bucket
-* Read the target input bucket and input folder from the active configuration.
-* Show the user the exact GCS destination path: `gs://[data_bucket_name]/[gcs_data_bucket_input_folder]/`.
-* Ask the user for approval to copy the new data files from the local `./data` folder to GCS:
-  ```bash
-  gcloud storage cp -R ./data/* "gs://${MY_BUCKET}/${INPUT_FOLDER}/"
-  ```
-
-### 2. Trigger the Ingestion Pipeline
-* **Parameter-free Ingestion Warning**: Explain that the `ingest start` command is completely **parameter-free**. It dynamically resolves variables from Terraform outputs and reads staged configurations directly from GCS.
-* **Strict Command Constraint**: **NEVER** append flags like `--import-name` or `--import-list` (handled automatically).
-* Present the command and trigger:
-  ```bash
-  uv run datacommons admin ingest start
-  ```
-* **Provide Observability Links**: Print both the **Job Console Link** and **Workflow Console Link** in the chat so the user has direct observability.
-
----
-
-## Phase 4: Operational Monitoring & Health Audits (Gate 2 Verification)
-
-The agent must actively verify that operations complete successfully and that the platform remains highly responsive, keeping the user updated on every step.
-
-### 1. Monitor Ingestion Progress
-* Explicitly inform the user you are watching the Cloud Run Job execution logs.
-* Output regular status updates (e.g., *"Ingestion job is currently running. Status: Active..."*).
-* If errors appear, present the exact error log trace and recommend remediation steps.
-
-### 2. Validate Database Health (Timeout & Sampling Heuristic)
-Verify that the ingestion succeeded by querying the Spanner database table **`Observation`** (singular). 
-
-🚨 **Performance Precaution**: Full-table counts (`SELECT COUNT(*)`) can cause expensive full scans and timeouts on large Spanner tables. Follow this safe, observable query flow:
-
-1. **Data Presence Probe**: Tell the user you are running a fast probe to confirm data is active:
-   ```bash
-   gcloud spanner databases execute-sql [spanner_db_name] \
-       --instance=[dcp_spanner_instance_id] \
-       --sql="SELECT 1 FROM Observation LIMIT 1"
-   ```
-2. **Fast Count Scan**: Run a count query:
-   ```bash
-   gcloud spanner databases execute-sql [spanner_db_name] \
-       --instance=[dcp_spanner_instance_id] \
-       --sql="SELECT COUNT(1) FROM Observation"
-   ```
-   * *Query Timeout Handling*: If this query takes more than **15-30 seconds**, abort/cancel the query immediately.
-   * *Fall Back to Sampling*: If aborted, report to the user that the table is too large for an immediate full count, and query a small recent sample as proof of successful data load:
-     ```bash
-     gcloud spanner databases execute-sql [spanner_db_name] \
-         --instance=[dcp_spanner_instance_id] \
-         --sql="SELECT * FROM Observation LIMIT 10"
-     ```
-     Present the sample records table directly in the chat.
-
-### 3. Local Health Checks
-Establish a background proxy connection to the Cloud Run Website:
-```bash
-gcloud run services proxy [cloud-run-service-name] --region=us-central1
-```
-Inform the user and query local port `8080`:
-```bash
-curl -i -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8080/healthz
-```
-Confirm a HTTP `200 OK` response.
-
----
-
-## Phase 5: Troubleshooting & Diagnosing Common Failures
-
-When components fail, explain the diagnostic path and ask for approval before reading logs:
-
-### 1. Mixer Service Crashes (HTTP 502/503 Errors)
-* *Action*: Propose reading the latest 50 lines of Cloud Run service logs:
-  ```bash
-  gcloud logging read "resource.type=cloud_run_revision AND resource.labels.service_name=[cloud-run-service-name]" --limit=50 --format=json
-  ```
-* Show logs in the chat, identify Spanner configuration or credentials mismatch errors, and propose fixes.
-
-### 2. Ingestion Job Terminated/Failed
-* *Action*: Propose describing the failed Cloud Run job execution:
-  ```bash
-  gcloud run jobs executions describe [execution_name]
-  ```
-* Analyze permissions and GCS bucket bindings, and present clear fixes to the user.
-
----
-
-## Phase 6: Tear-down / Full Clean Wipe
-
-If the partner wishes to completely tear down an environment, guide them safely to avoid data loss.
-
-1. **Backup Alert**: Warn the user to backup any custom Spanner database schemas or GCS input data. **Do not run any destructive commands until they acknowledge.**
-2. **Run Terraform Destroy**: Show the exact plan of resources to be destroyed and wait for verbal confirmation before executing:
-   ```bash
-   terraform destroy
-   ```
-3. **Erase Remote State**: Ask for permission to delete the Terraform backend state bucket:
-   ```bash
-   STATE_BUCKET="tf-state-backend-bucket"
-   gcloud storage rm --recursive "gs://${STATE_BUCKET}"
-   ```
-4. **Remove Directories**: Delete generated scaffolding folder:
-   ```bash
-   rm -rf [namespace_folder]
-   ```
-5. **Audit Log Update**: Log the wipe out action in the `ops_audit_log.md` file.

From e6db4335d835531d486c14ce9403d5e2cb1c2e47 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sat, 23 May 2026 00:36:01 -0700
Subject: [PATCH 06/23] Modularize setup preflight check script and structure
 main orchestrator

---
 .../dcp-setup/scripts/preflight_check.sh      | 131 +++++++++++-------
 1 file changed, 81 insertions(+), 50 deletions(-)

diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh
index 7be8c4cb..a153303e 100755
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh
@@ -30,27 +30,45 @@ log_error() {
     echo -e "${RED}[ERROR]${NC} $1"
 }
 
-# 1. OS Detection
-OS_TYPE="$(uname -s)"
-log_info "Detected operating system: ${OS_TYPE}"
+# Main orchestrator (Table of Contents)
+main() {
+    log_info "Starting Data Commons Platform Setup Pre-flight checks..."
+    check_os_and_utilities
+    check_and_install_uv
+    check_and_install_terraform
+    check_gcloud_sdk
+    validate_gcp_context
+    verify_gcp_adc
+    log_success "DCP Setup Pre-flight validation completed successfully!"
+}
 
-if [[ "${OS_TYPE}" != "Darwin" && "${OS_TYPE}" != "Linux" ]]; then
-    log_error "Unsupported operating system: ${OS_TYPE}. This setup requires macOS or Linux."
-    exit 1
-fi
+# OS and Base Utilities Check
+check_os_and_utilities() {
+    OS_TYPE="$(uname -s)"
+    log_info "Detected operating system: ${OS_TYPE}"
 
-# Ensure base commands exist
-for cmd in curl unzip grep; do
-    if ! command -v "$cmd" &> /dev/null; then
-        log_error "Required system utility '$cmd' is missing. Please install it first."
+    if [[ "${OS_TYPE}" != "Darwin" && "${OS_TYPE}" != "Linux" ]]; then
+        log_error "Unsupported operating system: ${OS_TYPE}. This setup requires macOS or Linux."
         exit 1
     fi
-done
 
-# 2. Check & Install uv
-if command -v uv &> /dev/null; then
-    log_success "uv package manager is already installed: $(uv --version)"
-else
+    # Ensure base commands exist
+    for cmd in curl unzip grep; do
+        if ! command -v "$cmd" &> /dev/null; then
+            log_error "Required system utility '$cmd' is missing. Please install it first."
+            exit 1
+        fi
+    done
+    log_success "Operating system and core utilities verified successfully."
+}
+
+# Check & Install uv
+check_and_install_uv() {
+    if command -v uv &> /dev/null; then
+        log_success "uv package manager is already installed: $(uv --version)"
+        return 0
+    fi
+
     log_warning "uv is missing. Attempting standalone installation..."
     if curl -LsSf https://astral.sh/uv/install.sh | sh; then
         # Source the environment to update PATH immediately
@@ -70,13 +88,17 @@ else
         log_error "Failed to install uv. Please install it manually from https://astral.sh/uv"
         exit 1
     fi
-fi
+}
+
+# Check & Install Terraform
+check_and_install_terraform() {
+    if command -v terraform &> /dev/null; then
+        log_success "Terraform is already installed: $(terraform -version | head -n 1)"
+        return 0
+    fi
 
-# 3. Check & Install Terraform
-if command -v terraform &> /dev/null; then
-    log_success "Terraform is already installed: $(terraform -version | head -n 1)"
-else
     log_warning "Terraform is missing. Attempting installation..."
+    OS_TYPE="$(uname -s)"
     if [[ "${OS_TYPE}" == "Darwin" ]]; then
         if command -v brew &> /dev/null; then
             log_info "Using Homebrew to install Terraform..."
@@ -94,7 +116,7 @@ else
             log_info "Using APT to install Terraform..."
             sudo apt-get update && sudo apt-get install -y gnupg software-properties-common wget
             wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg > /dev/null
-            echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+            echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com \$(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
             sudo apt-get update && sudo apt-get install -y terraform
         else
             log_warning "APT package manager not found. Downloading standalone Terraform binary..."
@@ -114,40 +136,49 @@ else
         log_error "Terraform installation failed. Please install it manually."
         exit 1
     fi
-fi
+}
+
+# Check gcloud SDK
+check_gcloud_sdk() {
+    if command -v gcloud &> /dev/null; then
+        log_success "gcloud CLI is already installed: $(gcloud --version | head -n 1)"
+        return 0
+    fi
 
-# 4. Check & Guide gcloud SDK
-if command -v gcloud &> /dev/null; then
-    log_success "gcloud CLI is already installed: $(gcloud --version | head -n 1)"
-else
     log_error "gcloud CLI is missing."
     log_info "To proceed, please download and install Google Cloud SDK by running the following commands or visiting: https://cloud.google.com/sdk/docs/install"
+    OS_TYPE="$(uname -s)"
     if [[ "${OS_TYPE}" == "Darwin" ]]; then
         log_info "macOS command: brew install --cask google-cloud-sdk"
     elif [[ "${OS_TYPE}" == "Linux" ]]; then
         log_info "Linux quick-install: curl https://sdk.cloud.google.com | bash"
     fi
     exit 1
-fi
-
-# 5. Validate active GCP project setting
-ACTIVE_PROJECT="$(gcloud config get-value project 2>/dev/null || echo "")"
-if [[ -z "${ACTIVE_PROJECT}" || "${ACTIVE_PROJECT}" == "(unset)" ]]; then
-    log_warning "No active GCP project is configured in gcloud."
-    log_info "Please set your project context using: gcloud config set project [PROJECT_ID]"
-else
-    log_success "Active GCP project context verified: ${ACTIVE_PROJECT}"
-fi
-
-# 6. Verify Authentication State
-log_info "Checking GCP authentication state..."
-if gcloud auth application-default print-access-token &> /dev/null; then
-    log_success "Active Application Default Credentials (ADC) verified."
-else
-    log_warning "No active Application Default Credentials (ADC) found."
-    log_info "Please login by running: gcloud auth application-default login"
-    log_info "Once authentication completes, re-run this pre-flight validation."
-    exit 2
-fi
-
-log_success "DCP Setup Pre-flight checks completed successfully!"
+}
+
+# Validate Active GCP Project context
+validate_gcp_context() {
+    ACTIVE_PROJECT="$(gcloud config get-value project 2>/dev/null || echo "")"
+    if [[ -z "${ACTIVE_PROJECT}" || "${ACTIVE_PROJECT}" == "(unset)" ]]; then
+        log_warning "No active GCP project is configured in gcloud."
+        log_info "Please set your project context using: gcloud config set project [PROJECT_ID]"
+    else
+        log_success "Active GCP project context verified: ${ACTIVE_PROJECT}"
+    fi
+}
+
+# Verify GCP Application Default Credentials
+verify_gcp_adc() {
+    log_info "Checking GCP authentication state..."
+    if gcloud auth application-default print-access-token &> /dev/null; then
+        log_success "Active Application Default Credentials (ADC) verified."
+    else
+        log_warning "No active Application Default Credentials (ADC) found."
+        log_info "Please login by running: gcloud auth application-default login"
+        log_info "Once authentication completes, re-run this pre-flight validation."
+        exit 2
+    fi
+}
+
+# Execution Call (Must remain at the bottom of the script)
+main "$@"

From 5bafe8ea2e6f5c68e5259dfecd5579a9d2d8fdf6 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sat, 23 May 2026 00:50:55 -0700
Subject: [PATCH 07/23] Fully modularize setup installer and refine onboarding
 prompt

---
 .../datacommons-cli/scripts/install-agent.sh  | 105 +++++++++++-------
 1 file changed, 65 insertions(+), 40 deletions(-)

diff --git a/packages/datacommons-cli/scripts/install-agent.sh b/packages/datacommons-cli/scripts/install-agent.sh
index 068872fe..3da1c167 100755
--- a/packages/datacommons-cli/scripts/install-agent.sh
+++ b/packages/datacommons-cli/scripts/install-agent.sh
@@ -8,14 +8,66 @@
 set -euo pipefail
 
 # ==========================================
-# Overridable Repository Settings (For dev branch & fork overrides)
+# Main Execution Sequence (Table of Contents)
 # ==========================================
-DCP_GIT_REPO="${DCP_GIT_REPO:-https://github.com/datacommonsorg/datacommons.git}"
-DCP_GIT_BRANCH="${DCP_GIT_BRANCH:-main}"
-DCP_GITHUB_RAW_BASE="${DCP_GITHUB_RAW_BASE:-https://raw.githubusercontent.com/datacommonsorg/datacommons}"
+
+main() {
+    load_env_overrides
+    resolve_repository_settings
+    setup_colors
+    
+    log_info "========================================="
+    log_info "   Data Commons Platform Agent Installer "
+    log_info "========================================="
+    
+    resolve_target_dir
+    detect_mode
+    deploy_skills
+    initialize_python_env
+    run_preflight_validation
+    
+    log_success "===================================================="
+    log_success "   DCP Agent Installation Completed Successfully!  "
+    log_success "===================================================="
+    log_info "Your workspace at '${TARGET_DIR}' is fully configured."
+    log_info "Please open your agentic assistant inside this workspace and ask:"
+    log_info "\"Help me set up my DCP instance\""
+    log_info "===================================================="
+}
+
+# ==========================================
+# Local Environment Override Support
+# ==========================================
+
+load_env_overrides() {
+    if [ -f .env ]; then
+        echo "[INFO] Found local .env file. Sourcing environment overrides..."
+        # Export overrides while discarding comment lines and trailing spaces
+        export $(grep -v '^#' .env | xargs)
+    fi
+}
+
+# ==========================================
+# Repository Settings Resolution
+# ==========================================
+
+resolve_repository_settings() {
+    # Establish overridable Git repository defaults
+    DCP_GIT_REPO="${DCP_GIT_REPO:-https://github.com/datacommonsorg/datacommons.git}"
+    DCP_GIT_BRANCH="${DCP_GIT_BRANCH:-main}"
+
+    # Dynamically resolve the raw Github URL base based on the targeted fork and branch
+    if [[ "$DCP_GIT_REPO" =~ github\.com[:/]([^/]+)/([^/.]+)(\.git)?$ ]]; then
+        local git_owner="${BASH_REMATCH[1]}"
+        local git_name="${BASH_REMATCH[2]}"
+        DCP_GITHUB_RAW_BASE="${DCP_GITHUB_RAW_BASE:-https://raw.githubusercontent.com/${git_owner}/${git_name}/${DCP_GIT_BRANCH}}"
+    else
+        DCP_GITHUB_RAW_BASE="${DCP_GITHUB_RAW_BASE:-https://raw.githubusercontent.com/datacommonsorg/datacommons/${DCP_GIT_BRANCH}}"
+    fi
+}
 
 # ==========================================
-# 1. Initialization & Logging Functions
+# Initialization & Logging Functions
 # ==========================================
 
 setup_colors() {
@@ -43,7 +95,7 @@ log_error() {
 }
 
 # ==========================================
-# 2. Target Directory Resolution
+# Target Directory Resolution
 # ==========================================
 
 resolve_target_dir() {
@@ -62,7 +114,7 @@ resolve_target_dir() {
 }
 
 # ==========================================
-# 3. Environment & Mode Detection
+# Environment & Mode Detection
 # ==========================================
 
 detect_mode() {
@@ -85,7 +137,7 @@ detect_mode() {
 }
 
 # ==========================================
-# 4. Skill Deployment (Local vs. Remote)
+# Skill Deployment (Local vs. Remote)
 # ==========================================
 
 copy_local_skills() {
@@ -101,7 +153,6 @@ copy_local_skills() {
     
     mkdir -p "${target_skills_dest}"
     cp -R "${local_skills_source}/dcp-setup" "${target_skills_dest}/"
-    cp -R "${local_skills_source}/dcp-ops" "${target_skills_dest}/"
     
     # Ensure scripts are marked executable
     chmod +x "${target_skills_dest}/dcp-setup/scripts/"*.sh 2>/dev/null || true
@@ -112,14 +163,13 @@ copy_local_skills() {
 download_remote_skills() {
     log_info "Downloading remote skill assets from GitHub..."
     
-    local github_raw_url="${DCP_GITHUB_RAW_BASE}/${DCP_GIT_BRANCH}/packages/datacommons-cli/datacommons_cli/skills"
+    local github_raw_url="${DCP_GITHUB_RAW_BASE}/packages/datacommons-cli/datacommons_cli/skills"
     local target_skills_dest="${TARGET_DIR}/.agent/skills"
     
     local files_to_fetch=(
         "dcp-setup/SKILL.md"
         "dcp-setup/scripts/preflight_check.sh"
         "dcp-setup/scripts/poll_iam.sh"
-        "dcp-ops/SKILL.md"
     )
     
     for file_path in "${files_to_fetch[@]}"; do
@@ -151,7 +201,7 @@ deploy_skills() {
 }
 
 # ==========================================
-# 5. Python Virtual Env & Package Setup
+# Python Virtual Env & Package Setup
 # ==========================================
 
 ensure_uv_installed() {
@@ -187,7 +237,7 @@ initialize_python_env() {
 }
 
 # ==========================================
-# 6. Pre-flight Orchestration
+# Pre-flight Orchestration
 # ==========================================
 
 run_preflight_validation() {
@@ -202,30 +252,5 @@ run_preflight_validation() {
     fi
 }
 
-# ==========================================
-# 7. Main Execution Sequence
-# ==========================================
-
-main() {
-    setup_colors
-    log_info "========================================="
-    log_info "   Data Commons Platform Agent Installer "
-    log_info "========================================="
-    
-    resolve_target_dir
-    detect_mode
-    deploy_skills
-    initialize_python_env
-    run_preflight_validation
-    
-    log_success "===================================================="
-    log_success "   DCP Agent Installation Completed Successfully!  "
-    log_success "===================================================="
-    log_info "Your workspace at '${TARGET_DIR}' is fully configured."
-    log_info "Please open your agentic assistant inside this workspace and ask:"
-    log_info "\"Help me set up my Data Commons\""
-    log_info "===================================================="
-}
-
-# Trigger Main
-main
+# Execution Trigger (Must remain at the bottom of the script)
+main "$@"

From e8fafd238c77d9aa0554c6d8fc634ac3c73a6695 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sat, 23 May 2026 01:03:48 -0700
Subject: [PATCH 08/23] Rename sandbox audit log to dcp_audit_log.md and sync
 setup guidelines

---
 .../datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
index c3db954c..9ea2ef40 100644
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
@@ -293,7 +293,7 @@ Show the user you are running automated health tests:
 
 ## Phase 8: Operations Handoff
 
-1. **Write Audit Log**: Append a record to `ops_audit_log.md` with timestamps, active project, and success statuses of the setup phases.
+1. **Write Audit Log**: Append a record to `dcp_audit_log.md` with timestamps, active project, and success statuses of the setup phases.
 2. **Provide Dashboard Links**: Output the direct GCP Console URLs for:
    * Spanner Database Console
    * Cloud Run Service Logs

From 3afb02a7b5e9b635f707cd7ba003f2d8c6f7946a Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sat, 23 May 2026 01:10:13 -0700
Subject: [PATCH 09/23] Refactor onboarding installer and validation scripts
 for pipeline robustness

---
 .../datacommons_cli/skills/dcp-setup/SKILL.md | 11 +++++-----
 .../datacommons-cli/scripts/install-agent.sh  | 21 +++++++++++++++----
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
index 9ea2ef40..567e396f 100644
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
@@ -14,13 +14,14 @@ This skill guides an agentic coding assistant through the setup and provisioning
 > * Before executing *any* command, the agent must present the proposed command line to the user and ask for confirmation.
 > * **After executing any command**, the agent must immediately provide a clear, useful summary of exactly what was done, what resources were created/changed, and what status or outputs were returned.
 > * **GCP Console Deep-Links**: Whenever a resource is created, modified, or queried, the agent must **generously construct and print direct GCP Console deep-links** (such as Spanner database tables, GCS folder paths, Cloud Run metrics, Cloud Workflows executions, and Dataflow pipeline graphs) to enable the user to instantly inspect and audit the live resources by themselves at any time.
-> * **Active Progress Updates (Quiet Polling & Percent Spinner)**: During long-running operations (such as `terraform apply` or data ingestion pipelines), the agent must never go silent, but **must avoid noisy output**. Print only a **highly-condensed, 3-line progress block** using **Exponential Backoff Polling Heuristics** (checking at 10s, 20s, 40s, and then scaling up to a maximum quiet heartbeat of 60s for the remainder of the long-running job).
+> * **Active Progress Updates (Mandatory Heartbeat Feedback)**: During long-running operations (such as `terraform apply` or data ingestion pipelines), the agent must never go completely silent. **At every single heartbeat check (checking at 10s, 20s, 40s, and then scaling up to a maximum quiet heartbeat of 60s), the agent MUST output the highly-condensed 2-column progress table directly into the chat** to provide constant, reassuring visual feedback.
 >   * **Structured Progress Layout (Compact 2-Column Table)**: All progress checks must be formatted in a clean, headerless, 2-column Markdown table (with values wrapped in inline backticks to render as yellow/amber pills):
 >     | | |
 >     | :--- | :--- |
 >     | **Phase** | `Phase X/3: [Global Phase Name]` |
 >     | **Status** | `[Factual, highly-condensed numerical metrics/counters]` |
 >     | **Elapsed** | `[phase_time] (Total pipeline time: [total_ingestion_time])` |
+>   * **Silent Scheduler & Mandatory Timer Registration**: Schedulers are silent (do not print closing conversational filler like "Scheduling a 30-second timer now..."), but **the agent MUST strictly register a progress timer using the `schedule` tool at the end of every single turn** during background monitoring, and then immediately end its turn by calling no more tools. Doing so ensures the system reactively wakes the agent up in a new turn to push the next progress card automatically. **Never go to sleep without an active scheduled timer!**
 >   * **Unified Ingestion Phase Labeling**:
 >     * 🛠️ **Phase 1 of 3: Preprocessing Container**
 >       * *Sub-stage 1/3 (CSV Import)*: `Status: Sub-stage 1/3 - CSV Import: 62.6% (979 / 1,562 files processed)`
@@ -103,11 +104,11 @@ Before running the scaffold generator, the agent **MUST run discovery checks to
 
 ### 2. Scaffolding Generation (Explicit Step)
 Once parameters are harvested, present the command and the inputs to the user:
-* *"I am going to run `uv run datacommons admin init` to generate your Terraform templates and set up your remote GCS state bucket. The parameters I will submit are Project: [project_id], Namespace: [namespace], and API Key: [api_key]. Do you approve?"*
+* *"I am going to run `uv run datacommons admin init` to generate your Terraform templates and set up your remote GCS state bucket. The parameters I will submit are Project ID: [project-id], Namespace: [namespace], and DC API Key: [dc-api-key]. Do you approve?"*
 * **Do not run the command in the background silently.** Run it with explicit inputs so that the command executes without prompting in the background:
   ```bash
   # Force inputs non-interactively to prevent silent background prompt automation
-  uv run datacommons admin init --project=[project_id] --namespace=[namespace] --key=[api_key] --auto-approve
+  uv run datacommons admin init --project-id=[project-id] --namespace=[namespace] --dc-api-key=[dc-api-key]
   ```
 
 ### 3. Variable Verification & Ingress Security Prompt
@@ -299,5 +300,5 @@ Show the user you are running automated health tests:
    * Cloud Run Service Logs
 3. **Instruct Transition**: Present the operational options and advise the user:
    > 🎉 **DCP Bootstrap Setup Complete!**
-   > Your Data Commons Platform is fully provisioned and serving custom data on `http://127.0.0.1:8080/`.
-   > For all future day-to-day modifications, schema upgrades, and ingestion runs, please load and execute the **`dcp-ops`** skill!
+   > Your Data Commons Platform is fully provisioned and serving custom data.
+   > For all future day-to-day data loads, schema modifications, and ingestion runs, please use the standard CLI: `datacommons admin ingest start`!
diff --git a/packages/datacommons-cli/scripts/install-agent.sh b/packages/datacommons-cli/scripts/install-agent.sh
index 3da1c167..d6a34777 100755
--- a/packages/datacommons-cli/scripts/install-agent.sh
+++ b/packages/datacommons-cli/scripts/install-agent.sh
@@ -30,9 +30,13 @@ main() {
     log_success "   DCP Agent Installation Completed Successfully!  "
     log_success "===================================================="
     log_info "Your workspace at '${TARGET_DIR}' is fully configured."
-    log_info "Please open your agentic assistant inside this workspace and ask:"
-    log_info "\"Help me set up my DCP instance\""
-    log_info "===================================================="
+    echo -e ""
+    echo -e "👉 ${YELLOW}WHAT TO DO NEXT:${NC}"
+    echo -e "   1. Open your agentic coding assistant inside this workspace folder."
+    echo -e "   2. Send this exact onboarding prompt to bootstrap the environment:"
+    echo -e "      ${GREEN}\"Help me set up my DCP instance\"${NC}"
+    echo -e ""
+    log_success "===================================================="
 }
 
 # ==========================================
@@ -102,7 +106,16 @@ resolve_target_dir() {
     local current_dir
     current_dir="$(pwd)"
     echo -e -n "${YELLOW}[INPUT]${NC} Where would you like to install the Data Commons agent? [Default: ${current_dir}]: "
-    read -r user_input
+    
+    # Read from TTY keyboard if available, otherwise fail loudly with manual instructions
+    if [ -c /dev/tty ] && [ -r /dev/tty ]; then
+        read -r user_input < /dev/tty
+    else
+        log_error "Interactive keyboard terminal (TTY) is blocked or unavailable."
+        log_info "To bypass this terminal warning block, please download and run the installer locally:"
+        log_info "  curl -sSfL \"${DCP_GITHUB_RAW_BASE}/packages/datacommons-cli/scripts/install-agent.sh\" -o install.sh && bash install.sh"
+        exit 1
+    fi
     
     # Use current directory if input is empty
     local target_path="${user_input:-${current_dir}}"

From e335710495bf2cb0d0fe46af38850ce3fa5e59d3 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sat, 23 May 2026 09:57:53 -0700
Subject: [PATCH 10/23] Format files

---
 .../datacommons-admin/datacommons_admin/admin_cli.py     | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/packages/datacommons-admin/datacommons_admin/admin_cli.py b/packages/datacommons-admin/datacommons_admin/admin_cli.py
index 74257916..3df37715 100644
--- a/packages/datacommons-admin/datacommons_admin/admin_cli.py
+++ b/packages/datacommons-admin/datacommons_admin/admin_cli.py
@@ -32,8 +32,13 @@
 
 DEFAULT_BUCKET_LOCATION = "US"
 # Overridable repository sources (allows developers to point to branches/forks)
-GITHUB_RAW_BASE_URL = os.getenv("DCP_GITHUB_RAW_BASE", "https://raw.githubusercontent.com/datacommonsorg/datacommons")
-GITHUB_REPO_URL = os.getenv("DCP_GIT_REPO", "https://github.com/datacommonsorg/datacommons.git")
+GITHUB_RAW_BASE_URL = os.getenv(
+    "DCP_GITHUB_RAW_BASE",
+    "https://raw.githubusercontent.com/datacommonsorg/datacommons",
+)
+GITHUB_REPO_URL = os.getenv(
+    "DCP_GIT_REPO", "https://github.com/datacommonsorg/datacommons.git"
+)
 
 
 def _get_default_bucket_name(namespace: str, project_id: str) -> str:

From 557e80d0714a95a8d0c7fcb300d06081693ddbf5 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sat, 23 May 2026 10:06:55 -0700
Subject: [PATCH 11/23] Address gemini comments

---
 .../datacommons_cli/skills/dcp-setup/SKILL.md |   8 +-
 .../skills/dcp-setup/scripts/poll_iam.sh      | 130 +++++++++++-------
 .../dcp-setup/scripts/preflight_check.sh      |  19 ++-
 .../datacommons-cli/scripts/install-agent.sh  |  11 +-
 4 files changed, 103 insertions(+), 65 deletions(-)

diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
index 567e396f..dd985770 100644
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
@@ -54,12 +54,10 @@ Verify the availability of `uv`, `terraform`, and `gcloud` by running version ch
     curl -LsSf https://astral.sh/uv/install.sh | sh
     ```
 
-* **Permanent Workspace CLI Installation**:
-  * Inform the user that a workspace-local virtual environment will be created to avoid slow `uvx` downlads.
-  * Ask for approval, then run:
+* **Workspace CLI Verification**:
+  * Confirm that the workspace-local virtual environment and CLI are fully functional by verifying the help output:
     ```bash
-    uv venv
-    uv pip install "git+https://github.com/datacommonsorg/datacommons.git@main#subdirectory=packages/datacommons-cli"
+    uv run datacommons --help
     ```
   * **Mandatory Tool Call**: For all subsequent steps, invoke the CLI instantly using `uv run datacommons` to prevent execution delays.
 
diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/poll_iam.sh b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/poll_iam.sh
index 40cfc44b..818026e2 100755
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/poll_iam.sh
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/poll_iam.sh
@@ -29,63 +29,89 @@ log_error() {
     echo -e "${RED}[ERROR]${NC} $1"
 }
 
-if [[ $# -lt 3 ]]; then
-    log_error "Usage: $0 <PROJECT_ID> <ORCHESTRATOR_SA_EMAIL> <ACTIVE_USER_EMAIL>"
-    exit 1
-fi
-
-PROJECT_ID="$1"
-SA_EMAIL="$2"
-USER_EMAIL="$3"
-
-log_info "Configuring IAM Service Account Impersonation bindings..."
-log_info "Project: ${PROJECT_ID}"
-log_info "Service Account: ${SA_EMAIL}"
-log_info "User: ${USER_EMAIL}"
-
-# Apply the binding
-if gcloud iam service-accounts add-iam-policy-binding "${SA_EMAIL}" \
-    --member="user:${USER_EMAIL}" \
-    --role="roles/iam.serviceAccountTokenCreator" \
-    --project="${PROJECT_ID}" &>/dev/null; then
-    log_success "Successfully applied Token Creator binding to service account!"
-else
-    log_error "Failed to bind roles/iam.serviceAccountTokenCreator on service account."
-    exit 1
-fi
-
-log_info "IAM permission changes submitted to GCP."
-log_info "Starting active verification loop (impersonation token polling)..."
+# Main orchestrator (Table of Contents)
+main() {
+    validate_args "$@"
+    
+    local project_id="$1"
+    local sa_email="$2"
+    local user_email="$3"
+    
+    apply_policy_binding "${project_id}" "${sa_email}" "${user_email}"
+    poll_token_propagation "${sa_email}"
+}
 
-MAX_ATTEMPTS=16
-WAIT_INTERVAL=15
-SUCCESS=false
+# Validate CLI Arguments
+validate_args() {
+    if [[ $# -lt 3 ]]; then
+        log_error "Usage: $0 <PROJECT_ID> <ORCHESTRATOR_SA_EMAIL> <ACTIVE_USER_EMAIL>"
+        exit 1
+    fi
+}
 
-for ((attempt=1; attempt<=MAX_ATTEMPTS; attempt++)); do
-    log_info "Attempt $attempt of $MAX_ATTEMPTS: Requesting access token via impersonation..."
+# Apply IAM Impersonation Policy Binding
+apply_policy_binding() {
+    local project_id="$1"
+    local sa_email="$2"
+    local user_email="$3"
     
-    # Redirect stderr to a temporary file to check for permission errors
-    ERR_OUT=$(mktemp)
+    log_info "Configuring IAM Service Account Impersonation bindings..."
+    log_info "Project: ${project_id}"
+    log_info "Service Account: ${sa_email}"
+    log_info "User: ${user_email}"
     
-    if gcloud auth print-access-token --impersonate-service-account="${SA_EMAIL}" &>/dev/null 2>"${ERR_OUT}"; then
-        log_success "GCP impersonation propagated successfully! Access token generated."
-        SUCCESS=true
-        rm -f "${ERR_OUT}"
-        break
+    if gcloud iam service-accounts add-iam-policy-binding "${sa_email}" \
+        --member="user:${user_email}" \
+        --role="roles/iam.serviceAccountTokenCreator" \
+        --project="${project_id}" &>/dev/null; then
+        log_success "Successfully applied Token Creator binding to service account!"
     else
-        ERR_MSG=$(cat "${ERR_OUT}")
-        rm -f "${ERR_OUT}"
+        log_error "Failed to bind roles/iam.serviceAccountTokenCreator on service account."
+        exit 1
+    fi
+    
+    log_info "IAM permission changes submitted to GCP."
+}
+
+# Poll for IAM propagation until success
+poll_token_propagation() {
+    local sa_email="$1"
+    
+    log_info "Starting active verification loop (impersonation token polling)..."
+    
+    local max_attempts=16
+    local wait_interval=15
+    local success=false
+    
+    # Create a single temporary file outside the loop and ensure cleanup on exit
+    local err_out
+    err_out="$(mktemp)"
+    trap 'rm -f "${err_out}"' EXIT
+    
+    for ((attempt=1; attempt<=max_attempts; attempt++)); do
+        log_info "Attempt $attempt of $max_attempts: Requesting access token via impersonation..."
         
-        log_warning "Permission propagation pending. Waiting ${WAIT_INTERVAL}s before retry..."
-        sleep ${WAIT_INTERVAL}
+        if gcloud auth print-access-token --impersonate-service-account="${sa_email}" &>/dev/null 2>"${err_out}"; then
+            log_success "GCP impersonation propagated successfully! Access token generated."
+            success=true
+            break
+        else
+            local err_msg
+            err_msg="$(cat "${err_out}" | tr -d '\n')"
+            log_warning "Propagation pending (Details: ${err_msg}). Waiting ${wait_interval}s before retry..."
+            sleep ${wait_interval}
+        fi
+    done
+    
+    if [ "${success}" = true ]; then
+        log_success "IAM bindings are completely propagated and active!"
+        exit 0
+    else
+        log_error "IAM permissions failed to propagate within $((max_attempts * wait_interval)) seconds."
+        log_error "Please manually verify that user has impersonation privileges on ${sa_email}."
+        exit 1
     fi
-done
+}
 
-if [ "$SUCCESS" = true ]; then
-    log_success "IAM bindings are completely propagated and active!"
-    exit 0
-else
-    log_error "IAM permissions failed to propagate within $((MAX_ATTEMPTS * WAIT_INTERVAL)) seconds."
-    log_error "Please manually verify that user has impersonation privileges on ${SA_EMAIL}."
-    exit 1
-fi
+# Execution Trigger (Must remain at the bottom of the script)
+main "$@"
diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh
index a153303e..d205d486 100755
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh
@@ -98,6 +98,17 @@ check_and_install_terraform() {
     fi
 
     log_warning "Terraform is missing. Attempting installation..."
+    
+    # Dynamically detect CPU architecture and map to HashiCorp's naming standard
+    local arch
+    local tf_arch
+    arch="$(uname -m)"
+    case "${arch}" in
+        x86_64)        tf_arch="amd64" ;;
+        arm64|aarch64) tf_arch="arm64" ;;
+        *)             tf_arch="amd64" ;; # Fallback
+    esac
+    
     OS_TYPE="$(uname -s)"
     if [[ "${OS_TYPE}" == "Darwin" ]]; then
         if command -v brew &> /dev/null; then
@@ -105,8 +116,8 @@ check_and_install_terraform() {
             brew tap hashicorp/tap
             brew install hashicorp/tap/terraform
         else
-            log_warning "Homebrew not found. Downloading standalone Terraform binary..."
-            TF_URL="https://releases.hashicorp.com/terraform/1.8.0/terraform_1.8.0_darwin_amd64.zip"
+            log_warning "Homebrew not found. Downloading standalone Terraform binary (${tf_arch})..."
+            local TF_URL="https://releases.hashicorp.com/terraform/1.8.0/terraform_1.8.0_darwin_${tf_arch}.zip"
             curl -Lo /tmp/terraform.zip "${TF_URL}"
             unzip -o /tmp/terraform.zip -d /usr/local/bin/ || unzip -o /tmp/terraform.zip -d "$HOME/.local/bin/"
             rm /tmp/terraform.zip
@@ -119,8 +130,8 @@ check_and_install_terraform() {
             echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com \$(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
             sudo apt-get update && sudo apt-get install -y terraform
         else
-            log_warning "APT package manager not found. Downloading standalone Terraform binary..."
-            TF_URL="https://releases.hashicorp.com/terraform/1.8.0/terraform_1.8.0_linux_amd64.zip"
+            log_warning "APT package manager not found. Downloading standalone Terraform binary (${tf_arch})..."
+            local TF_URL="https://releases.hashicorp.com/terraform/1.8.0/terraform_1.8.0_linux_${tf_arch}.zip"
             curl -Lo /tmp/terraform.zip "${TF_URL}"
             mkdir -p "$HOME/.local/bin"
             unzip -o /tmp/terraform.zip -d "$HOME/.local/bin"
diff --git a/packages/datacommons-cli/scripts/install-agent.sh b/packages/datacommons-cli/scripts/install-agent.sh
index d6a34777..6db40958 100755
--- a/packages/datacommons-cli/scripts/install-agent.sh
+++ b/packages/datacommons-cli/scripts/install-agent.sh
@@ -46,8 +46,9 @@ main() {
 load_env_overrides() {
     if [ -f .env ]; then
         echo "[INFO] Found local .env file. Sourcing environment overrides..."
-        # Export overrides while discarding comment lines and trailing spaces
-        export $(grep -v '^#' .env | xargs)
+        set -a
+        source .env
+        set +a
     fi
 }
 
@@ -242,8 +243,10 @@ initialize_python_env() {
         # Resolve relative package paths from local repository root
         uv pip install -e "${REPO_ROOT}/packages/datacommons-cli" -e "${REPO_ROOT}/packages/datacommons-admin"
     else
-        log_info "Installing datacommons CLI package from GitHub release..."
-        uv pip install "git+${DCP_GIT_REPO}@${DCP_GIT_BRANCH}#subdirectory=packages/datacommons-cli"
+        log_info "Installing datacommons CLI and admin packages from GitHub release..."
+        uv pip install \
+            "git+${DCP_GIT_REPO}@${DCP_GIT_BRANCH}#subdirectory=packages/datacommons-cli" \
+            "git+${DCP_GIT_REPO}@${DCP_GIT_BRANCH}#subdirectory=packages/datacommons-admin"
     fi
     
     log_success "Python virtual environment and CLI package successfully configured!"

From e64bccc03a2986ed6df7457612542375b9de4873 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sat, 23 May 2026 10:21:06 -0700
Subject: [PATCH 12/23] Add interactive selection gates recommendation to setup
 skill

---
 .../datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
index dd985770..62f1676e 100644
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
@@ -11,6 +11,7 @@ This skill guides an agentic coding assistant through the setup and provisioning
 > **CRITICAL COMPLIANCE DIRECTIVE: USER CONTROL & TRANSPARENCY**
 > * The agent must **NEVER** silently automate or submit interactive inputs to CLI commands running in the background (no silent pipelines).
 > * The agent must **NEVER** create or modify GCP resources (GCS buckets, Spanner, Cloud Run) without first explaining the exact proposed action and waiting for explicit verbal confirmation.
+> * **Interactive Selection Gates (UI Recommendation)**: For all user approval gates, resource configuration decisions, and options (such as Spanner instance reuse, ingress privacy, and BigQuery slot reservations), the agent should leverage interactive UI selectors or structured multi-choice questionnaire tools (such as `ask_question` if supported by the platform) to present clean, selectable options in the client instead of requesting the user to manually type their response.
 > * Before executing *any* command, the agent must present the proposed command line to the user and ask for confirmation.
 > * **After executing any command**, the agent must immediately provide a clear, useful summary of exactly what was done, what resources were created/changed, and what status or outputs were returned.
 > * **GCP Console Deep-Links**: Whenever a resource is created, modified, or queried, the agent must **generously construct and print direct GCP Console deep-links** (such as Spanner database tables, GCS folder paths, Cloud Run metrics, Cloud Workflows executions, and Dataflow pipeline graphs) to enable the user to instantly inspect and audit the live resources by themselves at any time.

From 4c00d98d7e5404a994e135b7b3a697e5023f9287 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sun, 24 May 2026 09:14:11 -0700
Subject: [PATCH 13/23] Restore BigQuery post-processing safeguard to setup
 skill

---
 .../datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
index 62f1676e..3a2c04da 100644
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
@@ -122,6 +122,10 @@ Once parameters are harvested, present the command and the inputs to the user:
     * *"A valid Google Maps API key is required for using Data Commons. How would you like to proceed?"*
     * **Option 1: Use Existing Key**: Ask the user to provide their existing Maps API key string. Once provided, the agent must write it directly to `auth_google_maps_api_key` inside `terraform.tfvars`.
     * **Option 2: Auto-generate restricted key**: Ask the user if they want Terraform to create a new restricted key natively. Once approved, the agent must write `auth_create_google_maps_api_key = true` and set `auth_google_maps_api_key = null` inside `terraform.tfvars` so Terraform provisions it securely.
+* **BigQuery Post-Processing Check (Mandatory Safeguard)**:
+  * Verify the configuration of `ingestion_workflow_enable_bigquery_postprocessing` inside `terraform.tfvars`.
+  * **Mandatory Constraint**: Due to active upstream BigQuery federation and post-processing limitations, this variable **MUST be set to `false`** to ensure a stable and successful data ingestion execution.
+  * If it is missing, commented out, or set to `true`, the agent must explain this limitation to the user in the chat, ask for approval, and automatically write `ingestion_workflow_enable_bigquery_postprocessing = false` to `terraform.tfvars`.
 * Confirm if the user wants to make any other manual adjustments (e.g., changing machine sizes or regions) before proceeding.
 
 ---

From 95d6405f903bd57e4f0f9b8f8f4067aa64f516f9 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sun, 24 May 2026 09:34:27 -0700
Subject: [PATCH 14/23] Update preflight checks to guide developers to rerun
 the installer

---
 .../skills/dcp-setup/scripts/preflight_check.sh               | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh
index d205d486..90aa4bf1 100755
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh
@@ -81,7 +81,7 @@ check_and_install_uv() {
         if command -v uv &> /dev/null; then
             log_success "uv was successfully installed: $(uv --version)"
         else
-            log_error "uv was installed but could not be found in PATH. Please restart your terminal and re-run."
+            log_error "uv was installed but could not be found in PATH. Please restart your terminal and re-run the installer command!"
             exit 1
         fi
     else
@@ -186,7 +186,7 @@ verify_gcp_adc() {
     else
         log_warning "No active Application Default Credentials (ADC) found."
         log_info "Please login by running: gcloud auth application-default login"
-        log_info "Once authentication completes, re-run this pre-flight validation."
+        log_info "Once authentication completes, please re-run the one-line installer command!"
         exit 2
     fi
 }

From 5134e652a3d8c757da51415893a8ff505df20414 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sun, 24 May 2026 09:51:19 -0700
Subject: [PATCH 15/23] Make python virtual environment creation non-blocking
 in installer

---
 packages/datacommons-cli/scripts/install-agent.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/datacommons-cli/scripts/install-agent.sh b/packages/datacommons-cli/scripts/install-agent.sh
index 6db40958..19ab689a 100755
--- a/packages/datacommons-cli/scripts/install-agent.sh
+++ b/packages/datacommons-cli/scripts/install-agent.sh
@@ -235,7 +235,7 @@ initialize_python_env() {
     ensure_uv_installed
     
     cd "${TARGET_DIR}"
-    uv venv
+    uv venv --clear
     
     # Install package in target virtual environment
     if [ "${DEV_MODE}" = true ]; then

From e73ab13ba013b1fa2b4b63bfc7ca9012b62e3753 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sun, 24 May 2026 17:24:16 -0700
Subject: [PATCH 16/23] Fix plan-time count error using Spanner static
 connection_id

---
 infra/dcp/modules/spanner/outputs.tf                          | 2 +-
 .../datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/infra/dcp/modules/spanner/outputs.tf b/infra/dcp/modules/spanner/outputs.tf
index e5df1687..33e60e6f 100644
--- a/infra/dcp/modules/spanner/outputs.tf
+++ b/infra/dcp/modules/spanner/outputs.tf
@@ -7,5 +7,5 @@ output "spanner_database_id" {
 }
 
 output "bigquery_connection_id" {
-  value = var.enable_bigquery_connection ? google_bigquery_connection.spanner_connection[0].id : ""
+  value = var.enable_bigquery_connection ? google_bigquery_connection.spanner_connection[0].connection_id : ""
 }
diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
index 3a2c04da..62f1676e 100644
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
@@ -122,10 +122,6 @@ Once parameters are harvested, present the command and the inputs to the user:
     * *"A valid Google Maps API key is required for using Data Commons. How would you like to proceed?"*
     * **Option 1: Use Existing Key**: Ask the user to provide their existing Maps API key string. Once provided, the agent must write it directly to `auth_google_maps_api_key` inside `terraform.tfvars`.
     * **Option 2: Auto-generate restricted key**: Ask the user if they want Terraform to create a new restricted key natively. Once approved, the agent must write `auth_create_google_maps_api_key = true` and set `auth_google_maps_api_key = null` inside `terraform.tfvars` so Terraform provisions it securely.
-* **BigQuery Post-Processing Check (Mandatory Safeguard)**:
-  * Verify the configuration of `ingestion_workflow_enable_bigquery_postprocessing` inside `terraform.tfvars`.
-  * **Mandatory Constraint**: Due to active upstream BigQuery federation and post-processing limitations, this variable **MUST be set to `false`** to ensure a stable and successful data ingestion execution.
-  * If it is missing, commented out, or set to `true`, the agent must explain this limitation to the user in the chat, ask for approval, and automatically write `ingestion_workflow_enable_bigquery_postprocessing = false` to `terraform.tfvars`.
 * Confirm if the user wants to make any other manual adjustments (e.g., changing machine sizes or regions) before proceeding.
 
 ---

From 1c8f000ba5b560160f24b5c8665e1a87d791fff7 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sun, 24 May 2026 17:30:29 -0700
Subject: [PATCH 17/23] Intercept python registry 401 errors and guide SSO
 refresh

---
 .../datacommons-cli/scripts/install-agent.sh  | 34 +++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/packages/datacommons-cli/scripts/install-agent.sh b/packages/datacommons-cli/scripts/install-agent.sh
index 19ab689a..480718de 100755
--- a/packages/datacommons-cli/scripts/install-agent.sh
+++ b/packages/datacommons-cli/scripts/install-agent.sh
@@ -230,6 +230,36 @@ ensure_uv_installed() {
     fi
 }
 
+run_pip_install() {
+    local err_log
+    err_log="$(mktemp)"
+    trap 'rm -f "${err_log}"' EXIT
+    
+    # Run uv pip install and redirect standard error to log
+    if ! uv pip install "$@" 2>"${err_log}"; then
+        local err_msg
+        err_msg="$(cat "${err_log}")"
+        
+        # Print the raw error to screen so the developer sees the original trace
+        echo -e "${err_msg}" >&2
+        
+        # Check for index 401 credentials blocks
+        if [[ "${err_msg}" == *"401 Unauthorized"* || "${err_msg}" == *"credentials"* ]]; then
+            log_error "===================================================="
+            log_error "   Python Package Registry Authentication Failure!  "
+            log_error "===================================================="
+            log_warning "It seems that your Python package index or private registry credentials have expired (returned HTTP 401 Unauthorized)."
+            log_info "If you are running inside a corporate network with private mirrors:"
+            log_info "  1. Please refresh your SSO/corporate package manager login tokens."
+            log_info "  2. Once authenticated, please re-run this one-line installer command!"
+            log_error "===================================================="
+        else
+            log_error "Python package installation failed. Please check your Python environment and try again."
+        fi
+        exit 1
+    fi
+}
+
 initialize_python_env() {
     log_info "Provisioning Python virtual environment..."
     ensure_uv_installed
@@ -241,10 +271,10 @@ initialize_python_env() {
     if [ "${DEV_MODE}" = true ]; then
         log_info "Installing local datacommons CLI package in editable mode..."
         # Resolve relative package paths from local repository root
-        uv pip install -e "${REPO_ROOT}/packages/datacommons-cli" -e "${REPO_ROOT}/packages/datacommons-admin"
+        run_pip_install -e "${REPO_ROOT}/packages/datacommons-cli" -e "${REPO_ROOT}/packages/datacommons-admin"
     else
         log_info "Installing datacommons CLI and admin packages from GitHub release..."
-        uv pip install \
+        run_pip_install \
             "git+${DCP_GIT_REPO}@${DCP_GIT_BRANCH}#subdirectory=packages/datacommons-cli" \
             "git+${DCP_GIT_REPO}@${DCP_GIT_BRANCH}#subdirectory=packages/datacommons-admin"
     fi

From 322b181c727ecb3fab0d5060d21259f12d528dd2 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sun, 24 May 2026 17:32:53 -0700
Subject: [PATCH 18/23] Ensure POSIX tty -s terminal check during setup prompts

---
 packages/datacommons-cli/scripts/install-agent.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/datacommons-cli/scripts/install-agent.sh b/packages/datacommons-cli/scripts/install-agent.sh
index 480718de..b93460de 100755
--- a/packages/datacommons-cli/scripts/install-agent.sh
+++ b/packages/datacommons-cli/scripts/install-agent.sh
@@ -109,7 +109,7 @@ resolve_target_dir() {
     echo -e -n "${YELLOW}[INPUT]${NC} Where would you like to install the Data Commons agent? [Default: ${current_dir}]: "
     
     # Read from TTY keyboard if available, otherwise fail loudly with manual instructions
-    if [ -c /dev/tty ] && [ -r /dev/tty ]; then
+    if tty -s && [ -c /dev/tty ] && [ -r /dev/tty ]; then
         read -r user_input < /dev/tty
     else
         log_error "Interactive keyboard terminal (TTY) is blocked or unavailable."

From 8f09705b9530c84f13727dd539b51a8b17bc53cb Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sun, 24 May 2026 17:36:40 -0700
Subject: [PATCH 19/23] Resolve unbound variable in installer exit paths

---
 packages/datacommons-cli/scripts/install-agent.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/datacommons-cli/scripts/install-agent.sh b/packages/datacommons-cli/scripts/install-agent.sh
index b93460de..e5f90d6c 100755
--- a/packages/datacommons-cli/scripts/install-agent.sh
+++ b/packages/datacommons-cli/scripts/install-agent.sh
@@ -233,12 +233,12 @@ ensure_uv_installed() {
 run_pip_install() {
     local err_log
     err_log="$(mktemp)"
-    trap 'rm -f "${err_log}"' EXIT
     
     # Run uv pip install and redirect standard error to log
     if ! uv pip install "$@" 2>"${err_log}"; then
         local err_msg
         err_msg="$(cat "${err_log}")"
+        rm -f "${err_log}"
         
         # Print the raw error to screen so the developer sees the original trace
         echo -e "${err_msg}" >&2
@@ -258,6 +258,9 @@ run_pip_install() {
         fi
         exit 1
     fi
+    
+    # Clean up temp file on success path
+    rm -f "${err_log}"
 }
 
 initialize_python_env() {

From f9fc0302b7bef2926c415891158edb4a53596f78 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sun, 24 May 2026 17:41:26 -0700
Subject: [PATCH 20/23] Fix piped TTY redirection regression using true <
 /dev/tty

---
 packages/datacommons-cli/scripts/install-agent.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/datacommons-cli/scripts/install-agent.sh b/packages/datacommons-cli/scripts/install-agent.sh
index e5f90d6c..26538e2d 100755
--- a/packages/datacommons-cli/scripts/install-agent.sh
+++ b/packages/datacommons-cli/scripts/install-agent.sh
@@ -109,7 +109,7 @@ resolve_target_dir() {
     echo -e -n "${YELLOW}[INPUT]${NC} Where would you like to install the Data Commons agent? [Default: ${current_dir}]: "
     
     # Read from TTY keyboard if available, otherwise fail loudly with manual instructions
-    if tty -s && [ -c /dev/tty ] && [ -r /dev/tty ]; then
+    if true < /dev/tty 2>/dev/null && [ -c /dev/tty ] && [ -r /dev/tty ]; then
         read -r user_input < /dev/tty
     else
         log_error "Interactive keyboard terminal (TTY) is blocked or unavailable."

From 98a6b07392710eae4065895cf8256816c5652727 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sun, 24 May 2026 17:48:01 -0700
Subject: [PATCH 21/23] Restore loud-failing TTY check for interactive setups

---
 packages/datacommons-cli/scripts/install-agent.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/datacommons-cli/scripts/install-agent.sh b/packages/datacommons-cli/scripts/install-agent.sh
index 26538e2d..db30756a 100755
--- a/packages/datacommons-cli/scripts/install-agent.sh
+++ b/packages/datacommons-cli/scripts/install-agent.sh
@@ -109,7 +109,7 @@ resolve_target_dir() {
     echo -e -n "${YELLOW}[INPUT]${NC} Where would you like to install the Data Commons agent? [Default: ${current_dir}]: "
     
     # Read from TTY keyboard if available, otherwise fail loudly with manual instructions
-    if true < /dev/tty 2>/dev/null && [ -c /dev/tty ] && [ -r /dev/tty ]; then
+    if [ -c /dev/tty ] && [ -r /dev/tty ]; then
         read -r user_input < /dev/tty
     else
         log_error "Interactive keyboard terminal (TTY) is blocked or unavailable."

From dc7ca45a01b827ed478467a7a28e85ec3032a7d1 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Sun, 24 May 2026 18:12:43 -0700
Subject: [PATCH 22/23] Fix Spanner planning count ID and apply progress tables
 to IAM polling

---
 infra/dcp/modules/spanner/outputs.tf                            | 2 +-
 .../datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/infra/dcp/modules/spanner/outputs.tf b/infra/dcp/modules/spanner/outputs.tf
index 33e60e6f..5a347b0c 100644
--- a/infra/dcp/modules/spanner/outputs.tf
+++ b/infra/dcp/modules/spanner/outputs.tf
@@ -7,5 +7,5 @@ output "spanner_database_id" {
 }
 
 output "bigquery_connection_id" {
-  value = var.enable_bigquery_connection ? google_bigquery_connection.spanner_connection[0].connection_id : ""
+  value = var.enable_bigquery_connection ? "projects/${var.project_id}/locations/${var.region}/connections/${replace("${local.name_prefix}dc_${var.bigquery_connection_name}", "-", "_")}" : ""
 }
diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
index 62f1676e..716e3dd4 100644
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
@@ -235,7 +235,7 @@ gcloud iam service-accounts add-iam-policy-binding [dcp_orchestrator_service_acc
 ```
 
 ### 2. Active IAM Propagation Polling (Safety Gate)
-Inform the user that GCP IAM replication can take a few minutes. Run a visible loop that checks impersonation status every 15 seconds, showing a progress indicator so the user knows the status.
+Inform the user that GCP IAM replication can take a few minutes. The agent **MUST** run a visible, non-blocking loop using the **Exponential Backoff Polling Heuristics** (checking at 10s, 20s, 40s, and then scaling up to a maximum quiet heartbeat of 60s), **outputting the compact, headerless 2-column progress table at every single heartbeat check** to show elapsed time, current state, and prevent redundant conversational filler.
 
 ---
 

From c27237fa40c5aea738c5d2a5cb9d9a858caede43 Mon Sep 17 00:00:00 2001
From: Keyur Shah <keyurs@google.com>
Date: Tue, 26 May 2026 00:23:57 -0700
Subject: [PATCH 23/23] Incorporate feedback from gabe

---
 .../modules/ingestion/helper_service/main.tf  |   2 +-
 infra/dcp/modules/spanner/outputs.tf          |   2 +-
 .../datacommons_cli/skills/dcp-setup/SKILL.md | 136 +++++++++---------
 3 files changed, 74 insertions(+), 66 deletions(-)

diff --git a/infra/dcp/modules/ingestion/helper_service/main.tf b/infra/dcp/modules/ingestion/helper_service/main.tf
index 135e7602..8f46001f 100644
--- a/infra/dcp/modules/ingestion/helper_service/main.tf
+++ b/infra/dcp/modules/ingestion/helper_service/main.tf
@@ -87,7 +87,7 @@ resource "google_project_iam_member" "helper_bq_roles" {
 }
 
 resource "google_bigquery_connection_iam_member" "helper_connection_user" {
-  count         = var.deploy && var.bigquery_connection_id != "" ? 1 : 0
+  count         = var.deploy && var.enable_bigquery_postprocessing && var.use_spanner ? 1 : 0
   project       = var.project_id
   location      = var.region
   connection_id = var.bigquery_connection_id
diff --git a/infra/dcp/modules/spanner/outputs.tf b/infra/dcp/modules/spanner/outputs.tf
index 5a347b0c..e5df1687 100644
--- a/infra/dcp/modules/spanner/outputs.tf
+++ b/infra/dcp/modules/spanner/outputs.tf
@@ -7,5 +7,5 @@ output "spanner_database_id" {
 }
 
 output "bigquery_connection_id" {
-  value = var.enable_bigquery_connection ? "projects/${var.project_id}/locations/${var.region}/connections/${replace("${local.name_prefix}dc_${var.bigquery_connection_name}", "-", "_")}" : ""
+  value = var.enable_bigquery_connection ? google_bigquery_connection.spanner_connection[0].id : ""
 }
diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
index 716e3dd4..f36701be 100644
--- a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
+++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md
@@ -11,7 +11,8 @@ This skill guides an agentic coding assistant through the setup and provisioning
 > **CRITICAL COMPLIANCE DIRECTIVE: USER CONTROL & TRANSPARENCY**
 > * The agent must **NEVER** silently automate or submit interactive inputs to CLI commands running in the background (no silent pipelines).
 > * The agent must **NEVER** create or modify GCP resources (GCS buckets, Spanner, Cloud Run) without first explaining the exact proposed action and waiting for explicit verbal confirmation.
-> * **Interactive Selection Gates (UI Recommendation)**: For all user approval gates, resource configuration decisions, and options (such as Spanner instance reuse, ingress privacy, and BigQuery slot reservations), the agent should leverage interactive UI selectors or structured multi-choice questionnaire tools (such as `ask_question` if supported by the platform) to present clean, selectable options in the client instead of requesting the user to manually type their response.
+> * **Interactive Selection Gates (Mandatory Tool Call)**: For all user approval gates, resource configuration decisions, and options (such as Spanner instance reuse, ingress privacy, and BigQuery slot reservations), **if the agent's active platform equips it with an interactive questionnaire tool (like `ask_question`), the agent MUST invoke that tool** to present the choices dynamically in the client UI instead of requesting the user to manually type their response. Only print plain text options if no such tool is supported by the active platform.
+> * **Clickable File Links (UX Requirement)**: Whenever referring to a local file or directory (such as `terraform.tfvars` or the project workspace folder), the agent **MUST** format it as a clickable Markdown link using its absolute file path: `[filename](file:///absolute/path/to/file)`. Never print plain-text filenames.
 > * Before executing *any* command, the agent must present the proposed command line to the user and ask for confirmation.
 > * **After executing any command**, the agent must immediately provide a clear, useful summary of exactly what was done, what resources were created/changed, and what status or outputs were returned.
 > * **GCP Console Deep-Links**: Whenever a resource is created, modified, or queried, the agent must **generously construct and print direct GCP Console deep-links** (such as Spanner database tables, GCS folder paths, Cloud Run metrics, Cloud Workflows executions, and Dataflow pipeline graphs) to enable the user to instantly inspect and audit the live resources by themselves at any time.
@@ -84,21 +85,23 @@ Verify the availability of `uv`, `terraform`, and `gcloud` by running version ch
 
 The agent must guide the user through the scaffolding phase with complete transparency, prompting for every parameter.
 
-### 1. Parameter Harvesting with Smart Environment Defaults
-Before running the scaffold generator, the agent **MUST run discovery checks to harvest environment defaults, and then present them to the user for explicit confirmation or modification**:
+### 1. Parameter Harvesting with Environment Defaults
+Before running the scaffold generator, the agent **MUST first run discovery checks to harvest default configuration settings, present them as a neat list, and prompt the user for their choice**:
 
-1. **GCP Project ID**:
-   * *Discovery*: Run `gcloud config get-value project 2>/dev/null`.
-   * *Prompt*: *"I detected that your active Google Cloud project context is set to `[detected_project_id]`. Would you like to use this project for deployment, or would you prefer to specify a different GCP project ID?"*
-2. **Namespace**:
-   * *Discovery*: Query system username (via `$USER`) or folder name.
-   * *Prompt*: *"What unique namespace identifier should we use for this environment? [Default: dcp-$USER]"*
-3. **Data Commons API Key**:
-   * *Discovery*: Check active environment variables (`echo $DC_API_KEY` or `$MIXER_API_KEY`).
-   * *Prompt*: If found: *"I detected a `DC_API_KEY` environment variable in your active session. Should I use this key, or would you like to provide a different one?"*
-   * If not found: *"I did not detect a `DC_API_KEY` in your active environment. Please provide a valid API Key from apikeys.datacommons.org (or let me know if we should use a dummy key 'fake-key' for this scaffolding phase)."*
+* **Environment Discoveries**:
+  1. **GCP Project ID**: Run `gcloud config get-value project 2>/dev/null`.
+  2. **Namespace**: Query system username (via `$USER`) or directory folder name.
+  3. **Data Commons API Key**: Check active environment variables (`echo $DC_API_KEY` or `$MIXER_API_KEY`).
 
-**Wait for the user's explicit confirmation or custom inputs for all three parameters.**
+* **Interactive Choice (UI Recommendation)**: Present the harvested defaults and explicitly prompt the user:
+  > *"Would you like to proceed with these default configuration settings, or would you prefer to customize specific variables?"*
+  * **Option A: Proceed with Defaults**: Directly write the defaults to `terraform.tfvars`, asking only for any required credentials that are missing (like a blank Project ID), and proceed straight to the dry-run plan.
+  * **Option B: Customize Specific Settings**: Sequentially prompt for Project ID, Custom Namespace overrides, and API Keys.
+
+* **Mandatory Context Block**: Before executing the scaffolding generator (or any major infrastructure phase), the agent **MUST** print a highly concise blockquote headered **`### Context`**, using empty blockquote lines (`>`) to ensure clean vertical formatting:
+  * **What**: What the step/command does.
+  * **Why**: Why this is architecturally necessary.
+  * **Where**: Where in their directory structure they can inspect or customize the resources.
 
 
 ### 2. Scaffolding Generation (Explicit Step)
@@ -113,15 +116,17 @@ Once parameters are harvested, present the command and the inputs to the user:
 ### 3. Variable Verification & Ingress Security Prompt
 * Read `terraform.tfvars` and show the user the generated configuration variables.
 * **Interactive Ingress Security Choice (Mandatory Prompt)**:
-  * Ask the user explicitly in the chat: *"Would you like to make your Custom Data Commons website publicly accessible to the open internet (best for sharing and quick testing), or keep it private and secured behind IAM OIDC authentication (default, best for secure enterprise data)?"*
-  * **If Public is selected**: Locate `datacommons_services_allow_unauthenticated_access` inside `terraform.tfvars`, uncomment it, and set it to `true`.
-  * **If Private/Default is selected**: Ensure `datacommons_services_allow_unauthenticated_access` remains commented out or set to `false`.
+  * Ask the user explicitly in the chat (leveraging the `ask_question` selection tool): *"Would you like to make your Custom Data Commons website publicly accessible to the open internet (best for sharing and quick testing), or keep it private and secured behind IAM OIDC authentication (default, best for secure enterprise data)?"*
+  * **Action & Approval Gate**: Once the choice is selected, **the agent must explain the changes, present the proposed modifications as a clean file diff block, ask for explicit verbal approval, and write the update to `terraform.tfvars` only upon receiving approval.**
+    * **If Public**: uncomment `datacommons_services_allow_unauthenticated_access = true`.
+    * **If Private/Default**: ensure it remains commented out or set to `false`.
 * **Explicit Google Maps API Key Choice Heuristic (Mandatory Prompt)**:
   * The agent must inspect `terraform.tfvars` for Google Maps credentials.
-  * If `auth_google_maps_api_key` is set to `"TODO"` and `auth_create_google_maps_api_key` is set to `false`, the agent **must explicitly prompt the user in the chat**:
+  * If `auth_google_maps_api_key` is set to `"TODO"` and `auth_create_google_maps_api_key` is set to `false`, the agent **must explicitly prompt the user in the chat (leveraging the `ask_question` tool)**:
     * *"A valid Google Maps API key is required for using Data Commons. How would you like to proceed?"*
-    * **Option 1: Use Existing Key**: Ask the user to provide their existing Maps API key string. Once provided, the agent must write it directly to `auth_google_maps_api_key` inside `terraform.tfvars`.
-    * **Option 2: Auto-generate restricted key**: Ask the user if they want Terraform to create a new restricted key natively. Once approved, the agent must write `auth_create_google_maps_api_key = true` and set `auth_google_maps_api_key = null` inside `terraform.tfvars` so Terraform provisions it securely.
+    * **Option 1: Use Existing Key**: Ask the user to provide their existing Maps API key string.
+    * **Option 2: Auto-generate restricted key**: Ask the user if they want Terraform to create a new restricted key natively.
+  * **Action & Approval Gate**: Once the option is selected, **the agent must explain the changes, present the proposed modifications as a clean file diff block, ask for explicit verbal approval, and write the variables to `terraform.tfvars` only upon receiving approval.**
 * Confirm if the user wants to make any other manual adjustments (e.g., changing machine sizes or regions) before proceeding.
 
 ---
@@ -131,54 +136,52 @@ Once parameters are harvested, present the command and the inputs to the user:
 Because the partner might deploy to a shared or pre-existing GCP project, the agent must actively identify and resolve resource conflicts, keeping the user fully informed.
 
 ### 1. GCP Project Verification
-Set the active project context and verify access:
-```bash
-gcloud config set project [PROJECT_ID]
-```
-
-### 2. Enable Required GCP APIs
-List the necessary APIs (Spanner, Cloud Run, Secret Manager, Artifact Registry) and ask for approval to enable them:
-```bash
-gcloud services enable spanner.googleapis.com run.googleapis.com secretmanager.googleapis.com artifactregistry.googleapis.com cloudbuild.googleapis.com iamcredentials.googleapis.com
-```
+* **Action & Approval Gate**: Explain that the active project context needs to be set to `[project-id]`. Present the proposed command, ask for explicit verbal approval, and execute **only** upon receiving approval:
+  ```bash
+  gcloud config set project [project-id]
+  ```
 
-### 3. Spanner Instance Discovery & Proactive Configuration (Step 1)
+### 2. Spanner Instance Discovery & Proactive Configuration (Step 1)
 * **Mandatory Order**: This check must be executed and finalized *before* initiating the BigQuery check.
-* Run `gcloud spanner instances list --project=[PROJECT_ID]` to discover active Spanner resources in the project.
+* **Action & Approval Gate**: Explain that we need to query active Spanner database instances on GCP to check for name conflicts or reuse opportunities. Present the proposed command, ask for explicit verbal approval, and execute **only** upon receiving approval:
+  ```bash
+  gcloud spanner instances list --project=[project-id]
+  ```
 * **If active Spanner instances are found**:
-  * Present the list clearly to the user in the chat:
+  * Present the list clearly to the user in the chat, and explicitly prompt them (leveraging the `ask_question` tool if available):
     > 🔍 **Active Spanner Instances Discovered**:
-    > I found these active Spanner instances in project `[project_id]`:
+    > I found these active Spanner instances in project `[project-id]`:
     > 1. `dcp-shared-spanner-instance`
     > 2. `dcp-instance-dev`
     > ...
     > Would you like to:
-    > * **Option A: Reuse an existing instance (Highly Recommended to save project capacity and costs)**? Please tell me which number/ID to use.
+    > * **Option A: Reuse an existing instance**? (Highly Recommended to save project capacity and costs).
     > * **Option B: Create a brand-new Spanner instance**? (Will generate `[namespace]-dc-instance` and increase project billing).
-  * **Action based on Choice**:
-    * **If Option A (Reuse)**: Ask the user if they approve, then automatically update `terraform.tfvars` to set `spanner_create_instance = false` and `spanner_instance_id = "[chosen_instance_id]"`. Inform the user once the file is updated.
+  * **Action & Approval Gate**: Once the option is selected, **the agent must explain the changes, present the proposed modifications as a clean file diff block, ask for explicit verbal approval, and write the update to `terraform.tfvars` only upon receiving approval.**
+    * **If Option A (Reuse)**: set `spanner_create_instance = false` and `spanner_instance_id = "[chosen_instance_id]"`.
     * **If Option B (Create New)**:
       * **Display Name Length Safety Check**: Calculate the resulting display name length: `${namespace}-${spanner_instance_id}`.
       * If this length exceeds the GCP limit of **30 characters** (e.g., `dcp-keyurs-dcp-keyurs-dc-instance` is 34 characters):
         * Prompt the user immediately with a warning and recommend a shortened `spanner_instance_id` like `dc-inst` so the resulting name (`[namespace]-dc-inst`) remains safe under 30 characters.
-      * Ask the user if they approve the verified setting, then automatically update `terraform.tfvars` to set `spanner_create_instance = true` and `spanner_instance_id = "[chosen_shortened_instance_id]"`. Inform the user once the file is updated.
+      * Once the shortened name is verified, present the proposed HCL diff block, ask for approval, and set `spanner_create_instance = true` and `spanner_instance_id = "[chosen_shortened_instance_id]"`.
 * **If no active Spanner instances are found**:
-  * Automatically update `terraform.tfvars` to set `spanner_create_instance = true` and `spanner_instance_id = "[namespace]-dc-inst"` to ensure a fresh database instance is provisioned cleanly and inform the user.
+  * **Action & Approval Gate**: Explain that a new database instance will be created. Present the proposed modifications as a clean file diff block, ask for explicit verbal approval, and write `spanner_create_instance = true` and `spanner_instance_id = "[namespace]-dc-inst"` to `terraform.tfvars` **only** upon receiving approval.
 
 * **Wait for Spanner configuration to be finalized and written before moving to the next step.**
 
-### 4. BigQuery Slot Reservation Conflict Check (Step 2)
+### 3. BigQuery Slot Reservation Conflict Check (Step 2)
 * **Mandatory Order**: Initiate this check *only* after the Spanner configuration above is fully completed and written to disk.
-* **Retrieve Target Region**: Parse the `region` variable from `terraform.tfvars` or `variables.tf` (defaulting to `us-central1` if it is commented out or blank). Use this string as the target `[REGION]` location.
-* Run a query to check for existing BigQuery reservations in that target region location:
+* **Retrieve Target Region**: Parse the `region` variable from `terraform.tfvars` or `variables.tf` (defaulting to `us-central1` if it is commented out or blank). Use this string as the target `[region]` location.
+* **Action & Approval Gate**: Explain that we need to check for existing BigQuery slot reservations in the target region to avoid provisioning conflicts. Present the proposed command, ask for explicit verbal approval, and execute **only** upon receiving approval:
   ```bash
-  bq ls --reservation --project_id=[PROJECT_ID] --location=[REGION]
+  bq ls --reservation --project_id=[project-id] --location=[region]
   ```
 * **If a reservation named `default` already exists**:
-  * Present this discovery to the user and explain that we should reuse it to avoid conflicts and save slots.
-  * Upon user confirmation, write/append `spanner_create_bigquery_reservation = false` to `terraform.tfvars` and confirm the write to the user.
+  * Present this discovery to the user and explain that we should reuse it to save slots.
+  * **Action & Approval Gate**: Present the proposed modifications as a clean file diff block, ask for explicit verbal approval, and write `spanner_create_bigquery_reservation = false` to `terraform.tfvars` **only** upon receiving approval.
 * **If no `default` reservation is found**:
-  * Inform the user that no reservation was found, and ask if they want to create a new one (sets `spanner_create_bigquery_reservation = true`) or reuse a pre-existing default one if they know it exists.
+  * Inform the user that no reservation was found, and ask if they want to create a new one or reuse an existing one.
+  * **Action & Approval Gate**: Once selected, present the proposed modifications as a clean file diff block, ask for explicit verbal approval, and write the variables to `terraform.tfvars` **only** upon receiving approval.
   * Write the selected value to `terraform.tfvars` and confirm.
 
 ---
@@ -202,24 +205,29 @@ Run `terraform plan` and output the changes to a temporary file. Parse this plan
 
 **Do not run `terraform apply` until the user reviews the Impact Report and replies with explicit verbal approval (e.g., "Yes", "Approve").**
 
-### 3. Apply Infrastructure Changes & Interactive Fail-safe
-Upon receiving approval, execute the deployment:
-```bash
-terraform apply -auto-approve
-```
-* **Transient GCP Propagation Fail-safe**:
-  * If the deployment fails due to transient GCP resource replication latency (e.g., `404 Instance/Database not found` or `409 Already exists` during IAM database user bindings):
-    * **Immediately inform the user of the failure** in the chat.
-    * Explain that this is a normal, expected GCP resource replication latency issue, and that the previous plan binary is now stale due to a partial apply.
-    * Prompt the user in the chat:
-      > 🔄 **GCP Resource Propagation Delay Detected**:
-      > The deployment hit a transient GCP replication delay. This is completely normal and expected.
-      > Since a partial apply occurred, our old plan is now stale. 
-      > * I will compile a fresh plan using `terraform plan -out=tfplan.binary` immediately.
-      > * I will then present the new short Impact Report of the remaining resources for your review.
-      > * Do you approve compiling the fresh plan and retrying?
-    * **Wait for the user's explicit verbal confirmation** before executing the new plan and apply sequence.
-* Display the final successful outputs to the user, highlighting `data_bucket_name` and `cloud-run-service-name`.
+### 3. User-Led Infrastructure Provisioning (Gate 2 Hand-Off)
+* **Strict Compliance Directive**: The agent **must never** execute `terraform apply` automatically or in the background. It must hand over execution control directly to the developer.
+* **Set Time and Component Expectations**: Explain to the developer that `terraform apply` is a long-running operation taking **10 to 15 minutes or longer** depending on the size of the custom dataset. Outline the physical execution phases:
+  * **Infrastructure Provisioning**: Creating the Spanner instance, Secret Manager secrets, and Cloud Run serving container/API gateway proxy revisions.
+  * **Data Pre-processing**: Spinning up the stager container to parse local CSV datasets, index observations into SQLite databases, and compile JSON-LD graph shards.
+  * **Parallel Database Seeding**: Triggering Cloud Workflows and deploying Apache Beam parallel Dataflow templates to load mutations and build Spanner indexes.
+* **Mandatory Context Block**: Print a highly concise blockquote headered **`### Context`**, using empty blockquote lines (`>`) to ensure clean vertical formatting:
+  * **What**: What `terraform apply` is creating.
+  * **Why**: Why this multi-layer cloud architecture is required to serve Custom Data Commons SPARQL queries securely and at scale.
+  * **Where**: Where they can review the module definitions in `/infra/dcp/modules/`.
+* **Instruct Execution**: Advise the developer to run this command in their terminal:
+  ```bash
+  terraform apply
+  ```
+
+### 4. Conditional Telemetry Tracking (Timing Guard)
+* **The Timing Guard**: The agent must **only** start telemetry after the developer has actively kicked off the command. It must ask:
+  > *"Please run `terraform apply` in your terminal. **Once you have actively kicked off the command**, you can steer my tracking in one of two ways:*
+  > * 📊 **If you want me to actively monitor progress**: Reply to me with **`\"Start monitoring\"`** and I will register background polling loops to track your GCS uploads, Workflows executions, and Dataflow Loader jobs reactively!
+  > * ⚙️ **If you prefer to run it independently**: Simply let me know **once the command has finished successfully** (so we can proceed to the verification steps), or **if you run into any issues** (so we can troubleshoot)."*
+* **Safety poller activation**:
+  * **If `"Start monitoring"` is requested**: Immediately register the `schedule` progress checking task.
+  * **Otherwise**: Go to sleep immediately without registering a timer, waiting for the user's next manual update.
 
 ---