feat: Add --purge option to delete command to physically delete data files

examples/scratch/README.md

@@ -39,6 +39,10 @@ ice insert nyc.taxis_p_by_day -p \
 ice delete nyc.taxis_p_by_day  \
   --partition '[{"name": "tpep_pickup_datetime", "values": ["2024-12-31T23:51:20"]}]' --dry-run=false
 
+# delete partition and physically purge the data files from storage
+ice delete nyc.taxis_p_by_day  \
+  --partition '[{"name": "tpep_pickup_datetime", "values": ["2024-12-31T23:51:20"]}]' --dry-run=false --purge
+
 # insert data ordered by tpep_pickup_datetime column
 ice insert nyc.taxis_s_by_day -p \
   https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-01.parquet \

ice-rest-catalog/src/test/resources/scenarios/delete-partition-purge/run.sh.tmpl

@@ -0,0 +1,89 @@
+#!/bin/bash
+set -e
+
+echo "Running delete partition with purge test..."
+
+# Create namespace
+{{ICE_CLI}} --config {{CLI_CONFIG}} create-namespace ${NAMESPACE_NAME}
+echo "OK Created namespace: ${NAMESPACE_NAME}"
+
+# Reuse the tripdata.parquet input from the delete-partition scenario
+SCENARIO_DIR="{{SCENARIO_DIR}}"
+INPUT_PATH="${SCENARIO_DIR}/../delete-partition/${INPUT_FILE}"
+
+# Create table with partitioning and insert data
+{{ICE_CLI}} --config {{CLI_CONFIG}} insert --create-table ${TABLE_NAME} ${INPUT_PATH} --partition="${PARTITION_SPEC}"
+echo "OK Inserted data with partitioning into table ${TABLE_NAME}"
+
+# Validate partition exists before deletion
+list_before=$({{ICE_CLI}} --config {{CLI_CONFIG}} list-partitions ${TABLE_NAME})
+if [[ "$list_before" != *"${TRANSFORMED_PARTITION_COLUMN}=${DELETE_PARTITION_DAY}"* ]]; then
+  echo "FAIL Expected partition key ${TRANSFORMED_PARTITION_COLUMN}=${DELETE_PARTITION_DAY} in list-partitions before delete"
+  exit 1
+fi
+echo "OK Partition exists before deletion"
+
+# Physical data file prefix for the partition being deleted
+DATA_PREFIX="s3://${S3_BUCKET}/warehouse/${NAMESPACE_NAME}/tripdata_p_by_day/data/${TRANSFORMED_PARTITION_COLUMN}=${DELETE_PARTITION_DAY}/"
+
+# Verify the partition's data files physically exist before purge (best-effort; requires aws + MinIO)
+CHECK_PHYSICAL=0
+if command -v aws &>/dev/null && [ -n "{{MINIO_ENDPOINT}}" ]; then
+  export AWS_ACCESS_KEY_ID=minioadmin
+  export AWS_SECRET_ACCESS_KEY=minioadmin
+  export AWS_REGION=us-east-1
+  export AWS_DEFAULT_REGION=us-east-1
+  CHECK_PHYSICAL=1
+
+  before_count=$(aws --endpoint-url "{{MINIO_ENDPOINT}}" s3 ls "${DATA_PREFIX}" 2>/dev/null | grep -c '\.parquet' || true)
+  if [ "${before_count}" -lt 1 ]; then
+    echo "FAIL Expected at least one .parquet under ${DATA_PREFIX} before purge, found ${before_count}"
+    exit 1
+  fi
+  echo "OK Found ${before_count} physical data file(s) before purge"
+else
+  echo "SKIP physical file check (aws unavailable or MINIO_ENDPOINT unset)"
+fi
+
+# Delete partition for a specific day and physically purge the data files
+output=$({{ICE_CLI}} --config {{CLI_CONFIG}} delete ${TABLE_NAME} --dry-run=false --purge --partition "[{\"name\": \"${TRANSFORMED_PARTITION_COLUMN}\", \"values\": [\"${DELETE_PARTITION_DAY}\"]}]" 2>&1)
+
+echo "$output"
+
+if ! echo "$output" | grep -qF "${TRANSFORMED_PARTITION_COLUMN}=${DELETE_PARTITION_DAY}"; then
+  echo "FAIL Expected output to contain '${TRANSFORMED_PARTITION_COLUMN}=${DELETE_PARTITION_DAY}'"
+  exit 1
+fi
+
+if ! echo "$output" | grep -qF "Purging"; then
+  echo "FAIL Expected output to contain 'Purging' when using --purge"
+  exit 1
+fi
+echo "OK Deleted and purged partition with value ${DELETE_PARTITION_DAY}"
+
+# Validate partition was deleted (logical)
+list_after=$({{ICE_CLI}} --config {{CLI_CONFIG}} list-partitions ${TABLE_NAME})
+if [[ "$list_after" == *"${TRANSFORMED_PARTITION_COLUMN}=${DELETE_PARTITION_DAY}"* ]]; then
+  echo "FAIL Deleted partition key ${TRANSFORMED_PARTITION_COLUMN}=${DELETE_PARTITION_DAY} in list-partitions after delete"
+  exit 1
+fi
+echo "OK Validated partition was logically deleted"
+
+# Validate the partition's data files were physically removed from storage
+if [ "${CHECK_PHYSICAL}" -eq 1 ]; then
+  after_count=$(aws --endpoint-url "{{MINIO_ENDPOINT}}" s3 ls "${DATA_PREFIX}" 2>/dev/null | grep -c '\.parquet' || true)
+  if [ "${after_count}" -ne 0 ]; then
+    echo "FAIL Expected no .parquet under ${DATA_PREFIX} after purge, found ${after_count}"
+    exit 1
+  fi
+  echo "OK Validated partition data files were physically purged"
+fi
+
+# Cleanup
+{{ICE_CLI}} --config {{CLI_CONFIG}} delete-table ${TABLE_NAME}
+echo "OK Deleted table: ${TABLE_NAME}"
+
+{{ICE_CLI}} --config {{CLI_CONFIG}} delete-namespace ${NAMESPACE_NAME}
+echo "OK Deleted namespace: ${NAMESPACE_NAME}"
+
+echo "Delete partition with purge test completed successfully"

ice-rest-catalog/src/test/resources/scenarios/delete-partition-purge/scenario.yaml

@@ -0,0 +1,15 @@
+name: Delete partition with purge
+description: Tests deleting a partition with --purge, physically removing its data files from storage
+
+catalogConfig:
+  warehouse: s3://test-bucket/warehouse
+
+env:
+  NAMESPACE_NAME: test_delete_partition_purge
+  TABLE_NAME: test_delete_partition_purge.tripdata_p_by_day
+  INPUT_FILE: tripdata.parquet
+  PARTITION_SPEC: '[{"column":"tpep_pickup_datetime","transform":"day"}]'
+  PARTITION_COLUMN: tpep_pickup_datetime
+  TRANSFORMED_PARTITION_COLUMN: tpep_pickup_datetime_day
+  DELETE_PARTITION_DAY: 2025-01-02
+  S3_BUCKET: test-bucket

ice/src/main/java/com/altinity/ice/cli/Main.java

@@ -970,7 +970,12 @@ void delete(
               names = "--dry-run",
               description =
                   "Log files that would be deleted without actually deleting them (true by default)")
-          Boolean dryRun)
+          Boolean dryRun,
+      @CommandLine.Option(
+              names = {"--purge"},
+              description =
+                  "Physically delete the matched data files from storage (breaks time-travel to snapshots that referenced them). No effect during a dry run.")
+          boolean purge)
       throws IOException {
     if (dryRun == null) {
       dryRun = true;
@@ -985,7 +990,7 @@ void delete(
       }
       TableIdentifier tableId = TableIdentifier.parse(name);
 
-      Delete.run(catalog, tableId, partitions, dryRun);
+      Delete.run(catalog, tableId, partitions, dryRun, purge);
     } catch (URISyntaxException e) {
       throw new RuntimeException(e);
     }

ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java

@@ -42,7 +42,8 @@ public static void run(
       RESTCatalog catalog,
       TableIdentifier tableId,
       List<PartitionFilter> partitions,
-      boolean dryRun)
+      boolean dryRun,
+      boolean purge)
       throws IOException, URISyntaxException {
 
     Table table = catalog.loadTable(tableId);
@@ -98,7 +99,11 @@ public static void run(
     if (!filesToDelete.isEmpty()) {
       if (dryRun) {
         for (DataFile file : filesToDelete) {
-          logger.info("To be deleted: {}", file.location());
+          if (purge) {
+            logger.info("To be deleted and purged: {}", file.location());
+          } else {
+            logger.info("To be deleted: {}", file.location());
+          }
         }
       } else {
         RewriteFiles rewrite = table.newRewrite();
@@ -107,6 +112,13 @@ public static void run(
           rewrite.deleteFile(deleteFile);
         }
         rewrite.commit();
+
+        if (purge) {
+          for (DataFile deleteFile : filesToDelete) {
+            logger.info("Purging {}", deleteFile.location());
+            io.deleteFile(deleteFile.location());
+          }
+        }
       }
     } else {
       logger.info("No files to delete");