diff --git a/infra/dcp/main.tf b/infra/dcp/main.tf index a2251614..84ec0ebc 100644 --- a/infra/dcp/main.tf +++ b/infra/dcp/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0" + version = ">= 7.30.0" } null = { source = "hashicorp/null" @@ -131,10 +131,11 @@ locals { ingestion_artifacts_path = trimsuffix(var.ingestion_artifacts_path, "/") # Preprocessing Job - preprocessing_job_image = coalesce(var.ingestion_preprocessing_job_image, "gcr.io/datcom-ci/datacommons-data:${var.dcp_version}") - preprocessing_job_cpu = var.ingestion_preprocessing_job_cpu - preprocessing_job_memory = var.ingestion_preprocessing_job_memory - preprocessing_job_timeout = var.ingestion_preprocessing_job_timeout + preprocessing_job_image = coalesce(var.ingestion_preprocessing_job_image, "gcr.io/datcom-ci/datacommons-data:${var.dcp_version}") + preprocessing_job_cpu = var.ingestion_preprocessing_job_cpu + preprocessing_job_memory = var.ingestion_preprocessing_job_memory + preprocessing_job_timeout = var.ingestion_preprocessing_job_timeout + preprocessing_job_tmp_disk_size = var.ingestion_preprocessing_job_tmp_disk_size # Workflow & Helper Service workflow_lock_acquisition_timeout = var.ingestion_workflow_lock_acquisition_timeout diff --git a/infra/dcp/modules/ingestion/preprocessing_job/main.tf b/infra/dcp/modules/ingestion/preprocessing_job/main.tf index 484bf290..400fdde0 100644 --- a/infra/dcp/modules/ingestion/preprocessing_job/main.tf +++ b/infra/dcp/modules/ingestion/preprocessing_job/main.tf @@ -1,5 +1,8 @@ locals { - name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + tmp_disk_mount_path = "/mnt/preprocessing-tmp" + tmp_disk_size = var.tmp_disk_size == null ? "" : trimspace(var.tmp_disk_size) + tmp_disk_enabled = local.tmp_disk_size != "" } resource "google_service_account" "preprocessing_sa" { @@ -10,6 +13,7 @@ resource "google_service_account" "preprocessing_sa" { resource "google_cloud_run_v2_job" "dc_data_job" { name = "${local.name_prefix}dc-ingestion-preprocessing-job" location = var.region + launch_stage = local.tmp_disk_enabled ? "BETA" : null deletion_protection = var.stateless_deletion_protection template { @@ -23,6 +27,14 @@ resource "google_cloud_run_v2_job" "dc_data_job" { } } + dynamic "volume_mounts" { + for_each = local.tmp_disk_enabled ? [1] : [] + content { + name = "preprocessing-tmp" + mount_path = local.tmp_disk_mount_path + } + } + dynamic "env" { for_each = var.env_vars content { @@ -44,6 +56,14 @@ resource "google_cloud_run_v2_job" "dc_data_job" { } } + dynamic "env" { + for_each = local.tmp_disk_enabled ? [1] : [] + content { + name = "TMPDIR" + value = local.tmp_disk_mount_path + } + } + env { name = "GCS_BUCKET" value = var.bucket_name @@ -66,6 +86,16 @@ resource "google_cloud_run_v2_job" "dc_data_job" { } } + dynamic "volumes" { + for_each = local.tmp_disk_enabled ? [local.tmp_disk_size] : [] + content { + name = "preprocessing-tmp" + empty_dir { + medium = "DISK" + size_limit = volumes.value + } + } + } dynamic "vpc_access" { for_each = var.vpc_connector_id != null && var.vpc_connector_id != "" ? [1] : [] content { diff --git a/infra/dcp/modules/ingestion/preprocessing_job/variables.tf b/infra/dcp/modules/ingestion/preprocessing_job/variables.tf index 8b860917..e0251731 100644 --- a/infra/dcp/modules/ingestion/preprocessing_job/variables.tf +++ b/infra/dcp/modules/ingestion/preprocessing_job/variables.tf @@ -13,6 +13,11 @@ variable "image" { variable "cpu" { type = string } variable "memory" { type = string } variable "timeout" { type = string } +variable "tmp_disk_size" { + description = "Optional size of the disk-backed temporary volume. Null or empty values disable the volume." + type = string + default = null +} variable "vpc_connector_id" { type = string } variable "bucket_name" { type = string } variable "input_path" { type = string } diff --git a/infra/dcp/modules/stack/main.tf b/infra/dcp/modules/stack/main.tf index 1d730151..eb09f01b 100644 --- a/infra/dcp/modules/stack/main.tf +++ b/infra/dcp/modules/stack/main.tf @@ -133,6 +133,7 @@ module "ingestion_preprocessing_job" { secret_env_vars = local.datacommons_services_secrets dc_api_key_secret_id = module.auth.dc_api_key_secret_id maps_api_key_secret_id = module.auth.maps_api_key_secret_id + tmp_disk_size = var.ingestion_config.preprocessing_job_tmp_disk_size depends_on = [module.auth] } diff --git a/infra/dcp/modules/stack/variables.tf b/infra/dcp/modules/stack/variables.tf index 3d34fc4b..c77bf1d9 100644 --- a/infra/dcp/modules/stack/variables.tf +++ b/infra/dcp/modules/stack/variables.tf @@ -90,10 +90,11 @@ variable "ingestion_config" { ingestion_artifacts_path = string # Preprocessing Job - preprocessing_job_image = optional(string) - preprocessing_job_cpu = string - preprocessing_job_memory = string - preprocessing_job_timeout = string + preprocessing_job_image = optional(string) + preprocessing_job_cpu = string + preprocessing_job_memory = string + preprocessing_job_timeout = string + preprocessing_job_tmp_disk_size = optional(string) # Workflow & Helper Service workflow_lock_acquisition_timeout = number diff --git a/infra/dcp/terraform.tfvars.template b/infra/dcp/terraform.tfvars.template index 948efda0..991498b5 100644 --- a/infra/dcp/terraform.tfvars.template +++ b/infra/dcp/terraform.tfvars.template @@ -69,6 +69,11 @@ datacommons_services_allow_unauthenticated_access = false # files are uploaded for processing. ingestion_input_path = "ingestion/input" +# Optional: use disk-backed temporary storage for large preprocessing jobs. +# Cloud Run ephemeral disks require at least 10Gi and a supported region. The +# default per-instance quota is 10 GB; request a quota increase for larger disks. +# ingestion_preprocessing_job_tmp_disk_size = "10Gi" + # ============================================================================= # Ingestion - Postprocessing # ============================================================================= diff --git a/infra/dcp/variables.tf b/infra/dcp/variables.tf index c0759d4b..18dbffaf 100644 --- a/infra/dcp/variables.tf +++ b/infra/dcp/variables.tf @@ -344,6 +344,12 @@ variable "ingestion_preprocessing_job_timeout" { default = "21600s" } +variable "ingestion_preprocessing_job_tmp_disk_size" { + description = "Optional ephemeral disk size for pre-processing temporary files. Null or empty values disable disk-backed storage; enabled volumes must be at least 10Gi." + type = string + default = null +} + variable "ingestion_input_path" { description = "Path within the bucket where raw files are uploaded" type = string @@ -413,4 +419,3 @@ variable "ingestion_dataflow_template_gcs_path" { type = string default = null } -