feat: add support for ephemeral disk-backed storage to preprocessing jobs

infra/dcp/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 5.11.0"
+      version = ">= 7.30.0"
     }
     null = {
       source  = "hashicorp/null"
@@ -131,10 +131,11 @@ locals {
     ingestion_artifacts_path = trimsuffix(var.ingestion_artifacts_path, "/")
 
     # Preprocessing Job
-    preprocessing_job_image   = coalesce(var.ingestion_preprocessing_job_image, "gcr.io/datcom-ci/datacommons-data:${var.dcp_version}")
-    preprocessing_job_cpu     = var.ingestion_preprocessing_job_cpu
-    preprocessing_job_memory  = var.ingestion_preprocessing_job_memory
-    preprocessing_job_timeout = var.ingestion_preprocessing_job_timeout
+    preprocessing_job_image         = coalesce(var.ingestion_preprocessing_job_image, "gcr.io/datcom-ci/datacommons-data:${var.dcp_version}")
+    preprocessing_job_cpu           = var.ingestion_preprocessing_job_cpu
+    preprocessing_job_memory        = var.ingestion_preprocessing_job_memory
+    preprocessing_job_timeout       = var.ingestion_preprocessing_job_timeout
+    preprocessing_job_tmp_disk_size = var.ingestion_preprocessing_job_tmp_disk_size
 
     # Workflow & Helper Service
     workflow_lock_acquisition_timeout = var.ingestion_workflow_lock_acquisition_timeout

infra/dcp/modules/ingestion/preprocessing_job/main.tf

@@ -1,5 +1,8 @@
 locals {
-  name_prefix = var.namespace != "" ? "${var.namespace}-" : ""
+  name_prefix         = var.namespace != "" ? "${var.namespace}-" : ""
+  tmp_disk_mount_path = "/mnt/preprocessing-tmp"
+  tmp_disk_size       = var.tmp_disk_size == null ? "" : trimspace(var.tmp_disk_size)
+  tmp_disk_enabled    = local.tmp_disk_size != ""
 }
 
 resource "google_service_account" "preprocessing_sa" {
@@ -10,6 +13,7 @@ resource "google_service_account" "preprocessing_sa" {
 resource "google_cloud_run_v2_job" "dc_data_job" {
   name                = "${local.name_prefix}dc-ingestion-preprocessing-job"
   location            = var.region
+  launch_stage         = local.tmp_disk_enabled ? "BETA" : null
   deletion_protection = var.stateless_deletion_protection
 
   template {
@@ -23,6 +27,14 @@ resource "google_cloud_run_v2_job" "dc_data_job" {
           }
         }
 
+        dynamic "volume_mounts" {
+          for_each = local.tmp_disk_enabled ? [1] : []
+          content {
+            name       = "preprocessing-tmp"
+            mount_path = local.tmp_disk_mount_path
+          }
+        }
+
         dynamic "env" {
           for_each = var.env_vars
           content {
@@ -44,6 +56,14 @@ resource "google_cloud_run_v2_job" "dc_data_job" {
           }
         }
 
+        dynamic "env" {
+          for_each = local.tmp_disk_enabled ? [1] : []
+          content {
+            name  = "TMPDIR"
+            value = local.tmp_disk_mount_path
+          }
+        }
+
         env {
           name  = "GCS_BUCKET"
           value = var.bucket_name
@@ -66,6 +86,16 @@ resource "google_cloud_run_v2_job" "dc_data_job" {
         }
 
       }
+      dynamic "volumes" {
+        for_each = local.tmp_disk_enabled ? [local.tmp_disk_size] : []
+        content {
+          name = "preprocessing-tmp"
+          empty_dir {
+            medium     = "DISK"
+            size_limit = volumes.value
+          }
+        }
+      }
       dynamic "vpc_access" {
         for_each = var.vpc_connector_id != null && var.vpc_connector_id != "" ? [1] : []
         content {

infra/dcp/modules/ingestion/preprocessing_job/variables.tf

@@ -13,6 +13,11 @@ variable "image" {
 variable "cpu" { type = string }
 variable "memory" { type = string }
 variable "timeout" { type = string }
+variable "tmp_disk_size" {
+  description = "Optional size of the disk-backed temporary volume. Null or empty values disable the volume."
+  type        = string
+  default     = null
+}
 variable "vpc_connector_id" { type = string }
 variable "bucket_name" { type = string }
 variable "input_path" { type = string }

infra/dcp/modules/stack/main.tf

@@ -133,6 +133,7 @@ module "ingestion_preprocessing_job" {
   secret_env_vars               = local.datacommons_services_secrets
   dc_api_key_secret_id          = module.auth.dc_api_key_secret_id
   maps_api_key_secret_id        = module.auth.maps_api_key_secret_id
+  tmp_disk_size                 = var.ingestion_config.preprocessing_job_tmp_disk_size
 
   depends_on = [module.auth]
 }

infra/dcp/modules/stack/variables.tf

@@ -90,10 +90,11 @@ variable "ingestion_config" {
     ingestion_artifacts_path = string
 
     # Preprocessing Job
-    preprocessing_job_image   = optional(string)
-    preprocessing_job_cpu     = string
-    preprocessing_job_memory  = string
-    preprocessing_job_timeout = string
+    preprocessing_job_image         = optional(string)
+    preprocessing_job_cpu           = string
+    preprocessing_job_memory        = string
+    preprocessing_job_timeout       = string
+    preprocessing_job_tmp_disk_size = optional(string)
 
     # Workflow & Helper Service
     workflow_lock_acquisition_timeout = number

infra/dcp/terraform.tfvars.template

@@ -69,6 +69,11 @@ datacommons_services_allow_unauthenticated_access = false
 # files are uploaded for processing.
 ingestion_input_path = "ingestion/input"
 
+# Optional: use disk-backed temporary storage for large preprocessing jobs.
+# Cloud Run ephemeral disks require at least 10Gi and a supported region. The
+# default per-instance quota is 10 GB; request a quota increase for larger disks.
+# ingestion_preprocessing_job_tmp_disk_size = "10Gi"
+
 # =============================================================================
 # Ingestion - Postprocessing
 # =============================================================================

infra/dcp/variables.tf

@@ -344,6 +344,12 @@ variable "ingestion_preprocessing_job_timeout" {
   default     = "21600s"
 }
 
+variable "ingestion_preprocessing_job_tmp_disk_size" {
+  description = "Optional ephemeral disk size for pre-processing temporary files. Null or empty values disable disk-backed storage; enabled volumes must be at least 10Gi."
+  type        = string
+  default     = null
+}
+
 variable "ingestion_input_path" {
   description = "Path within the bucket where raw files are uploaded"
   type        = string
@@ -413,4 +419,3 @@ variable "ingestion_dataflow_template_gcs_path" {
   type        = string
   default     = null
 }
-