diff --git a/.gitignore b/.gitignore index e0695b11..c0ee9ef3 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,6 @@ infra/dcp/.env AGENTS.md docs/plans -docs/designs \ No newline at end of file +docs/designs + +tmp/ \ No newline at end of file diff --git a/infra/dcp/modules/ingestion/helper_service/main.tf b/infra/dcp/modules/ingestion/helper_service/main.tf index 135e7602..8f46001f 100644 --- a/infra/dcp/modules/ingestion/helper_service/main.tf +++ b/infra/dcp/modules/ingestion/helper_service/main.tf @@ -87,7 +87,7 @@ resource "google_project_iam_member" "helper_bq_roles" { } resource "google_bigquery_connection_iam_member" "helper_connection_user" { - count = var.deploy && var.bigquery_connection_id != "" ? 1 : 0 + count = var.deploy && var.enable_bigquery_postprocessing && var.use_spanner ? 1 : 0 project = var.project_id location = var.region connection_id = var.bigquery_connection_id diff --git a/infra/dcp/modules/ingestion/workflow/main.tf b/infra/dcp/modules/ingestion/workflow/main.tf index a87ca268..cdefe198 100644 --- a/infra/dcp/modules/ingestion/workflow/main.tf +++ b/infra/dcp/modules/ingestion/workflow/main.tf @@ -213,3 +213,11 @@ resource "google_project_iam_member" "workflow_run_viewer" { role = "roles/run.viewer" member = "serviceAccount:${google_service_account.workflow_sa[0].email}" } + +resource "google_cloud_run_v2_service_iam_member" "workflow_sa_service_developer" { + count = var.deploy && var.enable_datacommons_services ? 1 : 0 + location = var.region + name = "${local.name_prefix}dc-datacommons-service" + role = "roles/run.developer" + member = "serviceAccount:${google_service_account.workflow_sa[0].email}" +} diff --git a/packages/datacommons-admin/datacommons_admin/admin_cli.py b/packages/datacommons-admin/datacommons_admin/admin_cli.py index af32ddc7..3df37715 100644 --- a/packages/datacommons-admin/datacommons_admin/admin_cli.py +++ b/packages/datacommons-admin/datacommons_admin/admin_cli.py @@ -13,6 +13,7 @@ # limitations under the License. from pathlib import Path +import os import re import sys import urllib.request @@ -30,8 +31,14 @@ DEFAULT_BUCKET_LOCATION = "US" -GITHUB_RAW_BASE_URL = "https://raw.githubusercontent.com/datacommonsorg/datacommons" -GITHUB_REPO_URL = "https://github.com/datacommonsorg/datacommons.git" +# Overridable repository sources (allows developers to point to branches/forks) +GITHUB_RAW_BASE_URL = os.getenv( + "DCP_GITHUB_RAW_BASE", + "https://raw.githubusercontent.com/datacommonsorg/datacommons", +) +GITHUB_REPO_URL = os.getenv( + "DCP_GIT_REPO", "https://github.com/datacommonsorg/datacommons.git" +) def _get_default_bucket_name(namespace: str, project_id: str) -> str: diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md new file mode 100644 index 00000000..f36701be --- /dev/null +++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/SKILL.md @@ -0,0 +1,311 @@ +--- +name: dcp-setup +description: Procedural guide and strict agent safety guidelines for provisioning a new or existing Data Commons Platform (DCP) instance. Enforces absolute transparency, explicit user approval gates, and interactive param harvesting. +--- + +# Data Commons Platform Setup (dcp-setup) + +This skill guides an agentic coding assistant through the setup and provisioning of a Data Commons Platform (DCP) instance on Google Cloud Platform (GCP). + +> [!IMPORTANT] +> **CRITICAL COMPLIANCE DIRECTIVE: USER CONTROL & TRANSPARENCY** +> * The agent must **NEVER** silently automate or submit interactive inputs to CLI commands running in the background (no silent pipelines). +> * The agent must **NEVER** create or modify GCP resources (GCS buckets, Spanner, Cloud Run) without first explaining the exact proposed action and waiting for explicit verbal confirmation. +> * **Interactive Selection Gates (Mandatory Tool Call)**: For all user approval gates, resource configuration decisions, and options (such as Spanner instance reuse, ingress privacy, and BigQuery slot reservations), **if the agent's active platform equips it with an interactive questionnaire tool (like `ask_question`), the agent MUST invoke that tool** to present the choices dynamically in the client UI instead of requesting the user to manually type their response. Only print plain text options if no such tool is supported by the active platform. +> * **Clickable File Links (UX Requirement)**: Whenever referring to a local file or directory (such as `terraform.tfvars` or the project workspace folder), the agent **MUST** format it as a clickable Markdown link using its absolute file path: `[filename](file:///absolute/path/to/file)`. Never print plain-text filenames. +> * Before executing *any* command, the agent must present the proposed command line to the user and ask for confirmation. +> * **After executing any command**, the agent must immediately provide a clear, useful summary of exactly what was done, what resources were created/changed, and what status or outputs were returned. +> * **GCP Console Deep-Links**: Whenever a resource is created, modified, or queried, the agent must **generously construct and print direct GCP Console deep-links** (such as Spanner database tables, GCS folder paths, Cloud Run metrics, Cloud Workflows executions, and Dataflow pipeline graphs) to enable the user to instantly inspect and audit the live resources by themselves at any time. +> * **Active Progress Updates (Mandatory Heartbeat Feedback)**: During long-running operations (such as `terraform apply` or data ingestion pipelines), the agent must never go completely silent. **At every single heartbeat check (checking at 10s, 20s, 40s, and then scaling up to a maximum quiet heartbeat of 60s), the agent MUST output the highly-condensed 2-column progress table directly into the chat** to provide constant, reassuring visual feedback. +> * **Structured Progress Layout (Compact 2-Column Table)**: All progress checks must be formatted in a clean, headerless, 2-column Markdown table (with values wrapped in inline backticks to render as yellow/amber pills): +> | | | +> | :--- | :--- | +> | **Phase** | `Phase X/3: [Global Phase Name]` | +> | **Status** | `[Factual, highly-condensed numerical metrics/counters]` | +> | **Elapsed** | `[phase_time] (Total pipeline time: [total_ingestion_time])` | +> * **Silent Scheduler & Mandatory Timer Registration**: Schedulers are silent (do not print closing conversational filler like "Scheduling a 30-second timer now..."), but **the agent MUST strictly register a progress timer using the `schedule` tool at the end of every single turn** during background monitoring, and then immediately end its turn by calling no more tools. Doing so ensures the system reactively wakes the agent up in a new turn to push the next progress card automatically. **Never go to sleep without an active scheduled timer!** +> * **Unified Ingestion Phase Labeling**: +> * 🛠️ **Phase 1 of 3: Preprocessing Container** +> * *Sub-stage 1/3 (CSV Import)*: `Status: Sub-stage 1/3 - CSV Import: 62.6% (979 / 1,562 files processed)` +> * *Sub-stage 2/3 (Indexing)*: `Status: Sub-stage 2/3 - Database Indexing: Indexing 13.9M observations` +> * *Sub-stage 3/3 (Export)*: `Status: Sub-stage 3/3 - JSON-LD Export: Staging 13.98M observations. 41 Node & 318+ Observation shards completed` +> * 🔗 **Phase 2 of 3: Cloud Workflows Orchestration** +> * `Status: Container completed successfully. Workflows state machine orchestrating Dataflow trigger` +> * ⚡ **Phase 3 of 3: Dataflow Parallel Loader** +> * **Dynamic Spanner Ingest Heuristic**: Extract active mutation metrics: `gcloud beta dataflow metrics list [JOB_ID] --project=[PROJECT_ID] --region=[REGION]`. Look up `mutation_groups_write_success` (staged mutation count) and `spanner_write_latency_ms_MEAN` (average batch latency) and print: `Status: Written graph mutations: [mutation_count] successfully loaded to Spanner (mean batch latency: [latency]ms)`. +> * Only print a detailed block or link when a **major pipeline transition occurs** or upon final success/failure. **Silent Scheduler Rule**: When scheduling these background timers, the agent must never print redundant closing text (such as "The progress timer is active. I will stand by..."). Simply print the 3-line status update, call the `schedule` tool, and end the turn immediately. +> * **Brevity & Tabular Formatting**: Infrastructure operations require absolute clarity. Avoid verbose, narrative, or flowery summaries. Present complex details (such as configuration changes, resource impacts, plans, or status reports) using clean markdown tables, bullet points, or short, direct facts. Keep narrative text blocks extremely concise. + +--- + +## Phase 1: Prerequisites Verification & Installation + +Before running any setup tasks, the agent must verify that all required local dependencies are installed and authenticated. + +### 1. OS Detection & Shell Verification +* Detect the host operating system (macOS vs. Linux). +* Ensure the execution environment has standard terminal tools like `curl`, `unzip`, and shell capabilities. + +### 2. CLI Dependencies Check & Local Virtual Env Setup +Verify the availability of `uv`, `terraform`, and `gcloud` by running version checks. If any tool is missing, present a summary to the user and ask if they want the agent to install it automatically: + +* **`uv` (Python Package Manager)**: + * *Check*: `uv --version` + * *Action if missing*: Ask the user for permission to install it via: + ```bash + curl -LsSf https://astral.sh/uv/install.sh | sh + ``` + +* **Workspace CLI Verification**: + * Confirm that the workspace-local virtual environment and CLI are fully functional by verifying the help output: + ```bash + uv run datacommons --help + ``` + * **Mandatory Tool Call**: For all subsequent steps, invoke the CLI instantly using `uv run datacommons` to prevent execution delays. + +* **`terraform` (Infrastructure as Code)**: + * *Check*: `terraform -version` + * *Action if missing*: Present the proposed installation command (macOS Homebrew or Linux apt) and wait for approval before running. + +* **`gcloud` (Google Cloud SDK)**: + * *Check*: `gcloud --version` + * *Action if missing*: Guide the user through the official installer, explaining exactly what package to download. + +### 3. GCP Authentication Check +* **Verify active login**: Run `gcloud auth list`. +* **Re-authenticate if needed**: If no active Application Default Credentials (ADC) are found, inform the user and ask for approval to trigger: + ```bash + gcloud auth application-default login + ``` + Explain that an interactive browser window will open for authentication. + +--- + +## Phase 2: Scaffolding & Configuration Validation (User Approval Gates) + +The agent must guide the user through the scaffolding phase with complete transparency, prompting for every parameter. + +### 1. Parameter Harvesting with Environment Defaults +Before running the scaffold generator, the agent **MUST first run discovery checks to harvest default configuration settings, present them as a neat list, and prompt the user for their choice**: + +* **Environment Discoveries**: + 1. **GCP Project ID**: Run `gcloud config get-value project 2>/dev/null`. + 2. **Namespace**: Query system username (via `$USER`) or directory folder name. + 3. **Data Commons API Key**: Check active environment variables (`echo $DC_API_KEY` or `$MIXER_API_KEY`). + +* **Interactive Choice (UI Recommendation)**: Present the harvested defaults and explicitly prompt the user: + > *"Would you like to proceed with these default configuration settings, or would you prefer to customize specific variables?"* + * **Option A: Proceed with Defaults**: Directly write the defaults to `terraform.tfvars`, asking only for any required credentials that are missing (like a blank Project ID), and proceed straight to the dry-run plan. + * **Option B: Customize Specific Settings**: Sequentially prompt for Project ID, Custom Namespace overrides, and API Keys. + +* **Mandatory Context Block**: Before executing the scaffolding generator (or any major infrastructure phase), the agent **MUST** print a highly concise blockquote headered **`### Context`**, using empty blockquote lines (`>`) to ensure clean vertical formatting: + * **What**: What the step/command does. + * **Why**: Why this is architecturally necessary. + * **Where**: Where in their directory structure they can inspect or customize the resources. + + +### 2. Scaffolding Generation (Explicit Step) +Once parameters are harvested, present the command and the inputs to the user: +* *"I am going to run `uv run datacommons admin init` to generate your Terraform templates and set up your remote GCS state bucket. The parameters I will submit are Project ID: [project-id], Namespace: [namespace], and DC API Key: [dc-api-key]. Do you approve?"* +* **Do not run the command in the background silently.** Run it with explicit inputs so that the command executes without prompting in the background: + ```bash + # Force inputs non-interactively to prevent silent background prompt automation + uv run datacommons admin init --project-id=[project-id] --namespace=[namespace] --dc-api-key=[dc-api-key] + ``` + +### 3. Variable Verification & Ingress Security Prompt +* Read `terraform.tfvars` and show the user the generated configuration variables. +* **Interactive Ingress Security Choice (Mandatory Prompt)**: + * Ask the user explicitly in the chat (leveraging the `ask_question` selection tool): *"Would you like to make your Custom Data Commons website publicly accessible to the open internet (best for sharing and quick testing), or keep it private and secured behind IAM OIDC authentication (default, best for secure enterprise data)?"* + * **Action & Approval Gate**: Once the choice is selected, **the agent must explain the changes, present the proposed modifications as a clean file diff block, ask for explicit verbal approval, and write the update to `terraform.tfvars` only upon receiving approval.** + * **If Public**: uncomment `datacommons_services_allow_unauthenticated_access = true`. + * **If Private/Default**: ensure it remains commented out or set to `false`. +* **Explicit Google Maps API Key Choice Heuristic (Mandatory Prompt)**: + * The agent must inspect `terraform.tfvars` for Google Maps credentials. + * If `auth_google_maps_api_key` is set to `"TODO"` and `auth_create_google_maps_api_key` is set to `false`, the agent **must explicitly prompt the user in the chat (leveraging the `ask_question` tool)**: + * *"A valid Google Maps API key is required for using Data Commons. How would you like to proceed?"* + * **Option 1: Use Existing Key**: Ask the user to provide their existing Maps API key string. + * **Option 2: Auto-generate restricted key**: Ask the user if they want Terraform to create a new restricted key natively. + * **Action & Approval Gate**: Once the option is selected, **the agent must explain the changes, present the proposed modifications as a clean file diff block, ask for explicit verbal approval, and write the variables to `terraform.tfvars` only upon receiving approval.** +* Confirm if the user wants to make any other manual adjustments (e.g., changing machine sizes or regions) before proceeding. + +--- + +## Phase 3: Existing GCP Project & Conflict Validation + +Because the partner might deploy to a shared or pre-existing GCP project, the agent must actively identify and resolve resource conflicts, keeping the user fully informed. + +### 1. GCP Project Verification +* **Action & Approval Gate**: Explain that the active project context needs to be set to `[project-id]`. Present the proposed command, ask for explicit verbal approval, and execute **only** upon receiving approval: + ```bash + gcloud config set project [project-id] + ``` + +### 2. Spanner Instance Discovery & Proactive Configuration (Step 1) +* **Mandatory Order**: This check must be executed and finalized *before* initiating the BigQuery check. +* **Action & Approval Gate**: Explain that we need to query active Spanner database instances on GCP to check for name conflicts or reuse opportunities. Present the proposed command, ask for explicit verbal approval, and execute **only** upon receiving approval: + ```bash + gcloud spanner instances list --project=[project-id] + ``` +* **If active Spanner instances are found**: + * Present the list clearly to the user in the chat, and explicitly prompt them (leveraging the `ask_question` tool if available): + > 🔍 **Active Spanner Instances Discovered**: + > I found these active Spanner instances in project `[project-id]`: + > 1. `dcp-shared-spanner-instance` + > 2. `dcp-instance-dev` + > ... + > Would you like to: + > * **Option A: Reuse an existing instance**? (Highly Recommended to save project capacity and costs). + > * **Option B: Create a brand-new Spanner instance**? (Will generate `[namespace]-dc-instance` and increase project billing). + * **Action & Approval Gate**: Once the option is selected, **the agent must explain the changes, present the proposed modifications as a clean file diff block, ask for explicit verbal approval, and write the update to `terraform.tfvars` only upon receiving approval.** + * **If Option A (Reuse)**: set `spanner_create_instance = false` and `spanner_instance_id = "[chosen_instance_id]"`. + * **If Option B (Create New)**: + * **Display Name Length Safety Check**: Calculate the resulting display name length: `${namespace}-${spanner_instance_id}`. + * If this length exceeds the GCP limit of **30 characters** (e.g., `dcp-keyurs-dcp-keyurs-dc-instance` is 34 characters): + * Prompt the user immediately with a warning and recommend a shortened `spanner_instance_id` like `dc-inst` so the resulting name (`[namespace]-dc-inst`) remains safe under 30 characters. + * Once the shortened name is verified, present the proposed HCL diff block, ask for approval, and set `spanner_create_instance = true` and `spanner_instance_id = "[chosen_shortened_instance_id]"`. +* **If no active Spanner instances are found**: + * **Action & Approval Gate**: Explain that a new database instance will be created. Present the proposed modifications as a clean file diff block, ask for explicit verbal approval, and write `spanner_create_instance = true` and `spanner_instance_id = "[namespace]-dc-inst"` to `terraform.tfvars` **only** upon receiving approval. + +* **Wait for Spanner configuration to be finalized and written before moving to the next step.** + +### 3. BigQuery Slot Reservation Conflict Check (Step 2) +* **Mandatory Order**: Initiate this check *only* after the Spanner configuration above is fully completed and written to disk. +* **Retrieve Target Region**: Parse the `region` variable from `terraform.tfvars` or `variables.tf` (defaulting to `us-central1` if it is commented out or blank). Use this string as the target `[region]` location. +* **Action & Approval Gate**: Explain that we need to check for existing BigQuery slot reservations in the target region to avoid provisioning conflicts. Present the proposed command, ask for explicit verbal approval, and execute **only** upon receiving approval: + ```bash + bq ls --reservation --project_id=[project-id] --location=[region] + ``` +* **If a reservation named `default` already exists**: + * Present this discovery to the user and explain that we should reuse it to save slots. + * **Action & Approval Gate**: Present the proposed modifications as a clean file diff block, ask for explicit verbal approval, and write `spanner_create_bigquery_reservation = false` to `terraform.tfvars` **only** upon receiving approval. +* **If no `default` reservation is found**: + * Inform the user that no reservation was found, and ask if they want to create a new one or reuse an existing one. + * **Action & Approval Gate**: Once selected, present the proposed modifications as a clean file diff block, ask for explicit verbal approval, and write the variables to `terraform.tfvars` **only** upon receiving approval. + * Write the selected value to `terraform.tfvars` and confirm. + +--- + +## Phase 4: Dry-Run & IaC Provisioning (Gate 1 Approval) + +Before applying infrastructure changes, compile the exact plan and require verbal user approval. + +### 1. Terraform Initialization +Explain and run: +```bash +terraform init +``` + +### 2. Generate the "Impact Report" (Gate 1 Approval) +Run `terraform plan` and output the changes to a temporary file. Parse this plan to build a detailed **Impact Report** containing: +* **Resources to Create**: List of added resources (e.g. GCS buckets, Spanner instance, Secret Manager keys). +* **Resources to Modify**: List of changed settings. +* **Resources to Destroy**: 🚨 Explicit warnings if any existing resources are set to be destroyed. +* **Estimated Cost Category**: Indicate if it uses production-scale Spanner nodes or budget-friendly processing units. + +**Do not run `terraform apply` until the user reviews the Impact Report and replies with explicit verbal approval (e.g., "Yes", "Approve").** + +### 3. User-Led Infrastructure Provisioning (Gate 2 Hand-Off) +* **Strict Compliance Directive**: The agent **must never** execute `terraform apply` automatically or in the background. It must hand over execution control directly to the developer. +* **Set Time and Component Expectations**: Explain to the developer that `terraform apply` is a long-running operation taking **10 to 15 minutes or longer** depending on the size of the custom dataset. Outline the physical execution phases: + * **Infrastructure Provisioning**: Creating the Spanner instance, Secret Manager secrets, and Cloud Run serving container/API gateway proxy revisions. + * **Data Pre-processing**: Spinning up the stager container to parse local CSV datasets, index observations into SQLite databases, and compile JSON-LD graph shards. + * **Parallel Database Seeding**: Triggering Cloud Workflows and deploying Apache Beam parallel Dataflow templates to load mutations and build Spanner indexes. +* **Mandatory Context Block**: Print a highly concise blockquote headered **`### Context`**, using empty blockquote lines (`>`) to ensure clean vertical formatting: + * **What**: What `terraform apply` is creating. + * **Why**: Why this multi-layer cloud architecture is required to serve Custom Data Commons SPARQL queries securely and at scale. + * **Where**: Where they can review the module definitions in `/infra/dcp/modules/`. +* **Instruct Execution**: Advise the developer to run this command in their terminal: + ```bash + terraform apply + ``` + +### 4. Conditional Telemetry Tracking (Timing Guard) +* **The Timing Guard**: The agent must **only** start telemetry after the developer has actively kicked off the command. It must ask: + > *"Please run `terraform apply` in your terminal. **Once you have actively kicked off the command**, you can steer my tracking in one of two ways:* + > * 📊 **If you want me to actively monitor progress**: Reply to me with **`\"Start monitoring\"`** and I will register background polling loops to track your GCS uploads, Workflows executions, and Dataflow Loader jobs reactively! + > * ⚙️ **If you prefer to run it independently**: Simply let me know **once the command has finished successfully** (so we can proceed to the verification steps), or **if you run into any issues** (so we can troubleshoot)."* +* **Safety poller activation**: + * **If `"Start monitoring"` is requested**: Immediately register the `schedule` progress checking task. + * **Otherwise**: Go to sleep immediately without registering a timer, waiting for the user's next manual update. + +--- + +## Phase 5: IAM Role Impersonation & Latency Polling + +### 1. Bind Token Creator Role +Explain that the user must be bound to the newly created service account to initialize the database: +```bash +gcloud iam service-accounts add-iam-policy-binding [dcp_orchestrator_service_account_email] \ + --member="user:[active-gcloud-user]" \ + --role="roles/iam.serviceAccountTokenCreator" \ + --project=[project_id] +``` + +### 2. Active IAM Propagation Polling (Safety Gate) +Inform the user that GCP IAM replication can take a few minutes. The agent **MUST** run a visible, non-blocking loop using the **Exponential Backoff Polling Heuristics** (checking at 10s, 20s, 40s, and then scaling up to a maximum quiet heartbeat of 60s), **outputting the compact, headerless 2-column progress table at every single heartbeat check** to show elapsed time, current state, and prevent redundant conversational filler. + +--- + +## Phase 6: Flexible Data Loading & Database Initialization + +The database seed step should accommodate the partner's custom data rather than defaulting to hardcoded UN-centric files. + +### 1. Custom Data Path Selection +Ask the user for their ingestion data source: +* **Option A: Local Folder**: Copy local CSV/JSON files to GCS: + ```bash + gcloud storage cp -R ./data/* "gs://[data_bucket_name]/[gcs_data_bucket_input_folder]/" + ``` +* **Option B: Custom GCS Source**: Sync from an existing external GCS bucket: + ```bash + gcloud storage cp -R gs://[custom-source-path]/* "gs://[data_bucket_name]/[gcs_data_bucket_input_folder]/" + ``` +* **Option C: Default Sandbox Data**: Copy standard Data Commons public sandbox files for a quick test. + +### 2. Initialize Spanner Tables (init-db) +Explain that we are initializing the Spanner schema tables and run: +```bash +uv run datacommons admin init-db +``` + +### 3. Start the Ingestion Job +* **Parameter-free Ingestion Warning**: Explain that the `ingest start` command is completely **parameter-free**. It dynamically resolves Cloud Run variables from Terraform outputs, and reads staged config files directly from GCS. +* **Strict Command Constraint**: **NEVER** append flags like `--import-name` or `--import-list` (the pipeline handles this automatically). +* Present the command and trigger: + ```bash + uv run datacommons admin ingest start + ``` +* **Provide Console Links**: Print the **Job Console Link** and **Workflow Console Link** clearly in the chat so the user can watch the live executions. + +--- + +## Phase 7: Health Verification & Local Proxy + +Verify that the entire platform is healthy and operational. + +### 1. Local Cloud Run Proxy +Establish a local proxy tunnel in the background to expose the Cloud Run Website service: +```bash +gcloud run services proxy [cloud-run-service-name] \ + --project=[project_id] \ + --region=us-central1 +``` + +### 2. Automatic Health Check (Gate 2 Verification) +Show the user you are running automated health tests: +* Hit the `/healthz` endpoint and report the HTTP status code. +* Run a test query to verify the V2 API and present the JSON response. + +--- + +## Phase 8: Operations Handoff + +1. **Write Audit Log**: Append a record to `dcp_audit_log.md` with timestamps, active project, and success statuses of the setup phases. +2. **Provide Dashboard Links**: Output the direct GCP Console URLs for: + * Spanner Database Console + * Cloud Run Service Logs +3. **Instruct Transition**: Present the operational options and advise the user: + > 🎉 **DCP Bootstrap Setup Complete!** + > Your Data Commons Platform is fully provisioned and serving custom data. + > For all future day-to-day data loads, schema modifications, and ingestion runs, please use the standard CLI: `datacommons admin ingest start`! diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/poll_iam.sh b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/poll_iam.sh new file mode 100755 index 00000000..818026e2 --- /dev/null +++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/poll_iam.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# +# Data Commons Platform (DCP) - IAM Token Impersonation Poller +# Binds serviceAccountTokenCreator privileges and polls until propagation succeeds. +# + +set -euo pipefail + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0;0m' + +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Main orchestrator (Table of Contents) +main() { + validate_args "$@" + + local project_id="$1" + local sa_email="$2" + local user_email="$3" + + apply_policy_binding "${project_id}" "${sa_email}" "${user_email}" + poll_token_propagation "${sa_email}" +} + +# Validate CLI Arguments +validate_args() { + if [[ $# -lt 3 ]]; then + log_error "Usage: $0 " + exit 1 + fi +} + +# Apply IAM Impersonation Policy Binding +apply_policy_binding() { + local project_id="$1" + local sa_email="$2" + local user_email="$3" + + log_info "Configuring IAM Service Account Impersonation bindings..." + log_info "Project: ${project_id}" + log_info "Service Account: ${sa_email}" + log_info "User: ${user_email}" + + if gcloud iam service-accounts add-iam-policy-binding "${sa_email}" \ + --member="user:${user_email}" \ + --role="roles/iam.serviceAccountTokenCreator" \ + --project="${project_id}" &>/dev/null; then + log_success "Successfully applied Token Creator binding to service account!" + else + log_error "Failed to bind roles/iam.serviceAccountTokenCreator on service account." + exit 1 + fi + + log_info "IAM permission changes submitted to GCP." +} + +# Poll for IAM propagation until success +poll_token_propagation() { + local sa_email="$1" + + log_info "Starting active verification loop (impersonation token polling)..." + + local max_attempts=16 + local wait_interval=15 + local success=false + + # Create a single temporary file outside the loop and ensure cleanup on exit + local err_out + err_out="$(mktemp)" + trap 'rm -f "${err_out}"' EXIT + + for ((attempt=1; attempt<=max_attempts; attempt++)); do + log_info "Attempt $attempt of $max_attempts: Requesting access token via impersonation..." + + if gcloud auth print-access-token --impersonate-service-account="${sa_email}" &>/dev/null 2>"${err_out}"; then + log_success "GCP impersonation propagated successfully! Access token generated." + success=true + break + else + local err_msg + err_msg="$(cat "${err_out}" | tr -d '\n')" + log_warning "Propagation pending (Details: ${err_msg}). Waiting ${wait_interval}s before retry..." + sleep ${wait_interval} + fi + done + + if [ "${success}" = true ]; then + log_success "IAM bindings are completely propagated and active!" + exit 0 + else + log_error "IAM permissions failed to propagate within $((max_attempts * wait_interval)) seconds." + log_error "Please manually verify that user has impersonation privileges on ${sa_email}." + exit 1 + fi +} + +# Execution Trigger (Must remain at the bottom of the script) +main "$@" diff --git a/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh new file mode 100755 index 00000000..90aa4bf1 --- /dev/null +++ b/packages/datacommons-cli/datacommons_cli/skills/dcp-setup/scripts/preflight_check.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# +# Data Commons Platform (DCP) - Setup Pre-flight Validation Script +# Automates operating system detection, CLI dependency checks, auto-installers, +# authentication verification, and GCP API enablement. +# + +set -euo pipefail + +# Standard formatting variables +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0;0m' # No Color + +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Main orchestrator (Table of Contents) +main() { + log_info "Starting Data Commons Platform Setup Pre-flight checks..." + check_os_and_utilities + check_and_install_uv + check_and_install_terraform + check_gcloud_sdk + validate_gcp_context + verify_gcp_adc + log_success "DCP Setup Pre-flight validation completed successfully!" +} + +# OS and Base Utilities Check +check_os_and_utilities() { + OS_TYPE="$(uname -s)" + log_info "Detected operating system: ${OS_TYPE}" + + if [[ "${OS_TYPE}" != "Darwin" && "${OS_TYPE}" != "Linux" ]]; then + log_error "Unsupported operating system: ${OS_TYPE}. This setup requires macOS or Linux." + exit 1 + fi + + # Ensure base commands exist + for cmd in curl unzip grep; do + if ! command -v "$cmd" &> /dev/null; then + log_error "Required system utility '$cmd' is missing. Please install it first." + exit 1 + fi + done + log_success "Operating system and core utilities verified successfully." +} + +# Check & Install uv +check_and_install_uv() { + if command -v uv &> /dev/null; then + log_success "uv package manager is already installed: $(uv --version)" + return 0 + fi + + log_warning "uv is missing. Attempting standalone installation..." + if curl -LsSf https://astral.sh/uv/install.sh | sh; then + # Source the environment to update PATH immediately + if [ -f "$HOME/.local/bin/env" ]; then + source "$HOME/.local/bin/env" + elif [ -f "$HOME/.cargo/env" ]; then + source "$HOME/.cargo/env" + fi + export PATH="$HOME/.local/bin:$PATH" + if command -v uv &> /dev/null; then + log_success "uv was successfully installed: $(uv --version)" + else + log_error "uv was installed but could not be found in PATH. Please restart your terminal and re-run the installer command!" + exit 1 + fi + else + log_error "Failed to install uv. Please install it manually from https://astral.sh/uv" + exit 1 + fi +} + +# Check & Install Terraform +check_and_install_terraform() { + if command -v terraform &> /dev/null; then + log_success "Terraform is already installed: $(terraform -version | head -n 1)" + return 0 + fi + + log_warning "Terraform is missing. Attempting installation..." + + # Dynamically detect CPU architecture and map to HashiCorp's naming standard + local arch + local tf_arch + arch="$(uname -m)" + case "${arch}" in + x86_64) tf_arch="amd64" ;; + arm64|aarch64) tf_arch="arm64" ;; + *) tf_arch="amd64" ;; # Fallback + esac + + OS_TYPE="$(uname -s)" + if [[ "${OS_TYPE}" == "Darwin" ]]; then + if command -v brew &> /dev/null; then + log_info "Using Homebrew to install Terraform..." + brew tap hashicorp/tap + brew install hashicorp/tap/terraform + else + log_warning "Homebrew not found. Downloading standalone Terraform binary (${tf_arch})..." + local TF_URL="https://releases.hashicorp.com/terraform/1.8.0/terraform_1.8.0_darwin_${tf_arch}.zip" + curl -Lo /tmp/terraform.zip "${TF_URL}" + unzip -o /tmp/terraform.zip -d /usr/local/bin/ || unzip -o /tmp/terraform.zip -d "$HOME/.local/bin/" + rm /tmp/terraform.zip + fi + elif [[ "${OS_TYPE}" == "Linux" ]]; then + if command -v apt-get &> /dev/null; then + log_info "Using APT to install Terraform..." + sudo apt-get update && sudo apt-get install -y gnupg software-properties-common wget + wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg > /dev/null + echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com \$(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo apt-get update && sudo apt-get install -y terraform + else + log_warning "APT package manager not found. Downloading standalone Terraform binary (${tf_arch})..." + local TF_URL="https://releases.hashicorp.com/terraform/1.8.0/terraform_1.8.0_linux_${tf_arch}.zip" + curl -Lo /tmp/terraform.zip "${TF_URL}" + mkdir -p "$HOME/.local/bin" + unzip -o /tmp/terraform.zip -d "$HOME/.local/bin" + export PATH="$HOME/.local/bin:$PATH" + rm /tmp/terraform.zip + fi + fi + + # Verify install + if command -v terraform &> /dev/null; then + log_success "Terraform was successfully installed: $(terraform -version | head -n 1)" + else + log_error "Terraform installation failed. Please install it manually." + exit 1 + fi +} + +# Check gcloud SDK +check_gcloud_sdk() { + if command -v gcloud &> /dev/null; then + log_success "gcloud CLI is already installed: $(gcloud --version | head -n 1)" + return 0 + fi + + log_error "gcloud CLI is missing." + log_info "To proceed, please download and install Google Cloud SDK by running the following commands or visiting: https://cloud.google.com/sdk/docs/install" + OS_TYPE="$(uname -s)" + if [[ "${OS_TYPE}" == "Darwin" ]]; then + log_info "macOS command: brew install --cask google-cloud-sdk" + elif [[ "${OS_TYPE}" == "Linux" ]]; then + log_info "Linux quick-install: curl https://sdk.cloud.google.com | bash" + fi + exit 1 +} + +# Validate Active GCP Project context +validate_gcp_context() { + ACTIVE_PROJECT="$(gcloud config get-value project 2>/dev/null || echo "")" + if [[ -z "${ACTIVE_PROJECT}" || "${ACTIVE_PROJECT}" == "(unset)" ]]; then + log_warning "No active GCP project is configured in gcloud." + log_info "Please set your project context using: gcloud config set project [PROJECT_ID]" + else + log_success "Active GCP project context verified: ${ACTIVE_PROJECT}" + fi +} + +# Verify GCP Application Default Credentials +verify_gcp_adc() { + log_info "Checking GCP authentication state..." + if gcloud auth application-default print-access-token &> /dev/null; then + log_success "Active Application Default Credentials (ADC) verified." + else + log_warning "No active Application Default Credentials (ADC) found." + log_info "Please login by running: gcloud auth application-default login" + log_info "Once authentication completes, please re-run the one-line installer command!" + exit 2 + fi +} + +# Execution Call (Must remain at the bottom of the script) +main "$@" diff --git a/packages/datacommons-cli/scripts/install-agent.sh b/packages/datacommons-cli/scripts/install-agent.sh new file mode 100755 index 00000000..db30756a --- /dev/null +++ b/packages/datacommons-cli/scripts/install-agent.sh @@ -0,0 +1,305 @@ +#!/usr/bin/env bash +# +# Data Commons Platform (DCP) - Agentic Assistant Installer +# Autonomously provisions a workspace with partner setup and operations skills, +# virtual environment, CLI, and triggers validation. +# + +set -euo pipefail + +# ========================================== +# Main Execution Sequence (Table of Contents) +# ========================================== + +main() { + load_env_overrides + resolve_repository_settings + setup_colors + + log_info "=========================================" + log_info " Data Commons Platform Agent Installer " + log_info "=========================================" + + resolve_target_dir + detect_mode + deploy_skills + initialize_python_env + run_preflight_validation + + log_success "====================================================" + log_success " DCP Agent Installation Completed Successfully! " + log_success "====================================================" + log_info "Your workspace at '${TARGET_DIR}' is fully configured." + echo -e "" + echo -e "👉 ${YELLOW}WHAT TO DO NEXT:${NC}" + echo -e " 1. Open your agentic coding assistant inside this workspace folder." + echo -e " 2. Send this exact onboarding prompt to bootstrap the environment:" + echo -e " ${GREEN}\"Help me set up my DCP instance\"${NC}" + echo -e "" + log_success "====================================================" +} + +# ========================================== +# Local Environment Override Support +# ========================================== + +load_env_overrides() { + if [ -f .env ]; then + echo "[INFO] Found local .env file. Sourcing environment overrides..." + set -a + source .env + set +a + fi +} + +# ========================================== +# Repository Settings Resolution +# ========================================== + +resolve_repository_settings() { + # Establish overridable Git repository defaults + DCP_GIT_REPO="${DCP_GIT_REPO:-https://github.com/datacommonsorg/datacommons.git}" + DCP_GIT_BRANCH="${DCP_GIT_BRANCH:-main}" + + # Dynamically resolve the raw Github URL base based on the targeted fork and branch + if [[ "$DCP_GIT_REPO" =~ github\.com[:/]([^/]+)/([^/.]+)(\.git)?$ ]]; then + local git_owner="${BASH_REMATCH[1]}" + local git_name="${BASH_REMATCH[2]}" + DCP_GITHUB_RAW_BASE="${DCP_GITHUB_RAW_BASE:-https://raw.githubusercontent.com/${git_owner}/${git_name}/${DCP_GIT_BRANCH}}" + else + DCP_GITHUB_RAW_BASE="${DCP_GITHUB_RAW_BASE:-https://raw.githubusercontent.com/datacommonsorg/datacommons/${DCP_GIT_BRANCH}}" + fi +} + +# ========================================== +# Initialization & Logging Functions +# ========================================== + +setup_colors() { + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + BLUE='\033[0;34m' + NC='\033[0;0m' # No Color +} + +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# ========================================== +# Target Directory Resolution +# ========================================== + +resolve_target_dir() { + local current_dir + current_dir="$(pwd)" + echo -e -n "${YELLOW}[INPUT]${NC} Where would you like to install the Data Commons agent? [Default: ${current_dir}]: " + + # Read from TTY keyboard if available, otherwise fail loudly with manual instructions + if [ -c /dev/tty ] && [ -r /dev/tty ]; then + read -r user_input < /dev/tty + else + log_error "Interactive keyboard terminal (TTY) is blocked or unavailable." + log_info "To bypass this terminal warning block, please download and run the installer locally:" + log_info " curl -sSfL \"${DCP_GITHUB_RAW_BASE}/packages/datacommons-cli/scripts/install-agent.sh\" -o install.sh && bash install.sh" + exit 1 + fi + + # Use current directory if input is empty + local target_path="${user_input:-${current_dir}}" + + # Resolve absolute path + mkdir -p "${target_path}" + TARGET_DIR="$(cd "${target_path}" && pwd)" + log_info "Target workspace resolved to: ${TARGET_DIR}" +} + +# ========================================== +# Environment & Mode Detection +# ========================================== + +detect_mode() { + log_info "Detecting running environment mode..." + + # Get current script path + local script_dir + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + + # Check if we are running inside a local cloned dc-datacommons repository + if [[ "${script_dir}" == *"packages/datacommons-cli/scripts"* ]]; then + DEV_MODE=true + # Root repo directory is 3 levels up from the script folder + REPO_ROOT="$(cd "${script_dir}/../../.." && pwd)" + log_success "Development Mode Detected! Local Repository Root: ${REPO_ROOT}" + else + DEV_MODE=false + log_info "Production/Remote Mode Detected. Assets will be fetched from GitHub." + fi +} + +# ========================================== +# Skill Deployment (Local vs. Remote) +# ========================================== + +copy_local_skills() { + log_info "Copying local skill assets in Development Mode..." + + local local_skills_source="${REPO_ROOT}/packages/datacommons-cli/datacommons_cli/skills" + local target_skills_dest="${TARGET_DIR}/.agent/skills" + + if [[ ! -d "${local_skills_source}" ]]; then + log_error "Source skills folder not found at: ${local_skills_source}" + exit 1 + fi + + mkdir -p "${target_skills_dest}" + cp -R "${local_skills_source}/dcp-setup" "${target_skills_dest}/" + + # Ensure scripts are marked executable + chmod +x "${target_skills_dest}/dcp-setup/scripts/"*.sh 2>/dev/null || true + + log_success "Skills successfully copied to: ${target_skills_dest}" +} + +download_remote_skills() { + log_info "Downloading remote skill assets from GitHub..." + + local github_raw_url="${DCP_GITHUB_RAW_BASE}/packages/datacommons-cli/datacommons_cli/skills" + local target_skills_dest="${TARGET_DIR}/.agent/skills" + + local files_to_fetch=( + "dcp-setup/SKILL.md" + "dcp-setup/scripts/preflight_check.sh" + "dcp-setup/scripts/poll_iam.sh" + ) + + for file_path in "${files_to_fetch[@]}"; do + local dest_file="${target_skills_dest}/${file_path}" + local source_url="${github_raw_url}/${file_path}" + + # Create parent directory for the file + mkdir -p "$(dirname "${dest_file}")" + + log_info "Fetching: ${file_path}..." + if ! curl -sSfL "${source_url}" -o "${dest_file}"; then + log_error "Failed to fetch file from: ${source_url}" + exit 1 + fi + done + + # Ensure scripts are marked executable + chmod +x "${target_skills_dest}/dcp-setup/scripts/"*.sh 2>/dev/null || true + + log_success "Skills successfully downloaded to: ${target_skills_dest}" +} + +deploy_skills() { + if [ "${DEV_MODE}" = true ]; then + copy_local_skills + else + download_remote_skills + fi +} + +# ========================================== +# Python Virtual Env & Package Setup +# ========================================== + +ensure_uv_installed() { + if ! command -v uv &> /dev/null; then + log_warning "uv package manager is missing. Attempting automatic installation..." + if curl -LsSf https://astral.sh/uv/install.sh | sh; then + export PATH="$HOME/.local/bin:$PATH" + else + log_error "Failed to install uv automatically. Please install it manually from https://astral.sh/uv" + exit 1 + fi + fi +} + +run_pip_install() { + local err_log + err_log="$(mktemp)" + + # Run uv pip install and redirect standard error to log + if ! uv pip install "$@" 2>"${err_log}"; then + local err_msg + err_msg="$(cat "${err_log}")" + rm -f "${err_log}" + + # Print the raw error to screen so the developer sees the original trace + echo -e "${err_msg}" >&2 + + # Check for index 401 credentials blocks + if [[ "${err_msg}" == *"401 Unauthorized"* || "${err_msg}" == *"credentials"* ]]; then + log_error "====================================================" + log_error " Python Package Registry Authentication Failure! " + log_error "====================================================" + log_warning "It seems that your Python package index or private registry credentials have expired (returned HTTP 401 Unauthorized)." + log_info "If you are running inside a corporate network with private mirrors:" + log_info " 1. Please refresh your SSO/corporate package manager login tokens." + log_info " 2. Once authenticated, please re-run this one-line installer command!" + log_error "====================================================" + else + log_error "Python package installation failed. Please check your Python environment and try again." + fi + exit 1 + fi + + # Clean up temp file on success path + rm -f "${err_log}" +} + +initialize_python_env() { + log_info "Provisioning Python virtual environment..." + ensure_uv_installed + + cd "${TARGET_DIR}" + uv venv --clear + + # Install package in target virtual environment + if [ "${DEV_MODE}" = true ]; then + log_info "Installing local datacommons CLI package in editable mode..." + # Resolve relative package paths from local repository root + run_pip_install -e "${REPO_ROOT}/packages/datacommons-cli" -e "${REPO_ROOT}/packages/datacommons-admin" + else + log_info "Installing datacommons CLI and admin packages from GitHub release..." + run_pip_install \ + "git+${DCP_GIT_REPO}@${DCP_GIT_BRANCH}#subdirectory=packages/datacommons-cli" \ + "git+${DCP_GIT_REPO}@${DCP_GIT_BRANCH}#subdirectory=packages/datacommons-admin" + fi + + log_success "Python virtual environment and CLI package successfully configured!" +} + +# ========================================== +# Pre-flight Orchestration +# ========================================== + +run_preflight_validation() { + log_info "Triggering local pre-flight checks..." + + local preflight_script="${TARGET_DIR}/.agent/skills/dcp-setup/scripts/preflight_check.sh" + + if [[ -f "${preflight_script}" ]]; then + bash "${preflight_script}" + else + log_warning "Pre-flight script not found at: ${preflight_script}. Skipping." + fi +} + +# Execution Trigger (Must remain at the bottom of the script) +main "$@"