From a922a4eef99683edf42339befaaeee68f28353cc Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Mon, 20 Jul 2026 16:48:19 -0400 Subject: [PATCH] Added logic to physically data files on deletion of partition. --- examples/scratch/README.md | 4 + .../delete-partition-purge/run.sh.tmpl | 89 +++++++++++++++++++ .../delete-partition-purge/scenario.yaml | 15 ++++ .../main/java/com/altinity/ice/cli/Main.java | 9 +- .../altinity/ice/cli/internal/cmd/Delete.java | 16 +++- 5 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 ice-rest-catalog/src/test/resources/scenarios/delete-partition-purge/run.sh.tmpl create mode 100644 ice-rest-catalog/src/test/resources/scenarios/delete-partition-purge/scenario.yaml diff --git a/examples/scratch/README.md b/examples/scratch/README.md index 011fd715..0d8a4262 100644 --- a/examples/scratch/README.md +++ b/examples/scratch/README.md @@ -39,6 +39,10 @@ ice insert nyc.taxis_p_by_day -p \ ice delete nyc.taxis_p_by_day \ --partition '[{"name": "tpep_pickup_datetime", "values": ["2024-12-31T23:51:20"]}]' --dry-run=false +# delete partition and physically purge the data files from storage +ice delete nyc.taxis_p_by_day \ + --partition '[{"name": "tpep_pickup_datetime", "values": ["2024-12-31T23:51:20"]}]' --dry-run=false --purge + # insert data ordered by tpep_pickup_datetime column ice insert nyc.taxis_s_by_day -p \ https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-01.parquet \ diff --git a/ice-rest-catalog/src/test/resources/scenarios/delete-partition-purge/run.sh.tmpl b/ice-rest-catalog/src/test/resources/scenarios/delete-partition-purge/run.sh.tmpl new file mode 100644 index 00000000..c37a0c86 --- /dev/null +++ b/ice-rest-catalog/src/test/resources/scenarios/delete-partition-purge/run.sh.tmpl @@ -0,0 +1,89 @@ +#!/bin/bash +set -e + +echo "Running delete partition with purge test..." + +# Create namespace +{{ICE_CLI}} --config {{CLI_CONFIG}} create-namespace ${NAMESPACE_NAME} +echo "OK Created namespace: ${NAMESPACE_NAME}" + +# Reuse the tripdata.parquet input from the delete-partition scenario +SCENARIO_DIR="{{SCENARIO_DIR}}" +INPUT_PATH="${SCENARIO_DIR}/../delete-partition/${INPUT_FILE}" + +# Create table with partitioning and insert data +{{ICE_CLI}} --config {{CLI_CONFIG}} insert --create-table ${TABLE_NAME} ${INPUT_PATH} --partition="${PARTITION_SPEC}" +echo "OK Inserted data with partitioning into table ${TABLE_NAME}" + +# Validate partition exists before deletion +list_before=$({{ICE_CLI}} --config {{CLI_CONFIG}} list-partitions ${TABLE_NAME}) +if [[ "$list_before" != *"${TRANSFORMED_PARTITION_COLUMN}=${DELETE_PARTITION_DAY}"* ]]; then + echo "FAIL Expected partition key ${TRANSFORMED_PARTITION_COLUMN}=${DELETE_PARTITION_DAY} in list-partitions before delete" + exit 1 +fi +echo "OK Partition exists before deletion" + +# Physical data file prefix for the partition being deleted +DATA_PREFIX="s3://${S3_BUCKET}/warehouse/${NAMESPACE_NAME}/tripdata_p_by_day/data/${TRANSFORMED_PARTITION_COLUMN}=${DELETE_PARTITION_DAY}/" + +# Verify the partition's data files physically exist before purge (best-effort; requires aws + MinIO) +CHECK_PHYSICAL=0 +if command -v aws &>/dev/null && [ -n "{{MINIO_ENDPOINT}}" ]; then + export AWS_ACCESS_KEY_ID=minioadmin + export AWS_SECRET_ACCESS_KEY=minioadmin + export AWS_REGION=us-east-1 + export AWS_DEFAULT_REGION=us-east-1 + CHECK_PHYSICAL=1 + + before_count=$(aws --endpoint-url "{{MINIO_ENDPOINT}}" s3 ls "${DATA_PREFIX}" 2>/dev/null | grep -c '\.parquet' || true) + if [ "${before_count}" -lt 1 ]; then + echo "FAIL Expected at least one .parquet under ${DATA_PREFIX} before purge, found ${before_count}" + exit 1 + fi + echo "OK Found ${before_count} physical data file(s) before purge" +else + echo "SKIP physical file check (aws unavailable or MINIO_ENDPOINT unset)" +fi + +# Delete partition for a specific day and physically purge the data files +output=$({{ICE_CLI}} --config {{CLI_CONFIG}} delete ${TABLE_NAME} --dry-run=false --purge --partition "[{\"name\": \"${TRANSFORMED_PARTITION_COLUMN}\", \"values\": [\"${DELETE_PARTITION_DAY}\"]}]" 2>&1) + +echo "$output" + +if ! echo "$output" | grep -qF "${TRANSFORMED_PARTITION_COLUMN}=${DELETE_PARTITION_DAY}"; then + echo "FAIL Expected output to contain '${TRANSFORMED_PARTITION_COLUMN}=${DELETE_PARTITION_DAY}'" + exit 1 +fi + +if ! echo "$output" | grep -qF "Purging"; then + echo "FAIL Expected output to contain 'Purging' when using --purge" + exit 1 +fi +echo "OK Deleted and purged partition with value ${DELETE_PARTITION_DAY}" + +# Validate partition was deleted (logical) +list_after=$({{ICE_CLI}} --config {{CLI_CONFIG}} list-partitions ${TABLE_NAME}) +if [[ "$list_after" == *"${TRANSFORMED_PARTITION_COLUMN}=${DELETE_PARTITION_DAY}"* ]]; then + echo "FAIL Deleted partition key ${TRANSFORMED_PARTITION_COLUMN}=${DELETE_PARTITION_DAY} in list-partitions after delete" + exit 1 +fi +echo "OK Validated partition was logically deleted" + +# Validate the partition's data files were physically removed from storage +if [ "${CHECK_PHYSICAL}" -eq 1 ]; then + after_count=$(aws --endpoint-url "{{MINIO_ENDPOINT}}" s3 ls "${DATA_PREFIX}" 2>/dev/null | grep -c '\.parquet' || true) + if [ "${after_count}" -ne 0 ]; then + echo "FAIL Expected no .parquet under ${DATA_PREFIX} after purge, found ${after_count}" + exit 1 + fi + echo "OK Validated partition data files were physically purged" +fi + +# Cleanup +{{ICE_CLI}} --config {{CLI_CONFIG}} delete-table ${TABLE_NAME} +echo "OK Deleted table: ${TABLE_NAME}" + +{{ICE_CLI}} --config {{CLI_CONFIG}} delete-namespace ${NAMESPACE_NAME} +echo "OK Deleted namespace: ${NAMESPACE_NAME}" + +echo "Delete partition with purge test completed successfully" diff --git a/ice-rest-catalog/src/test/resources/scenarios/delete-partition-purge/scenario.yaml b/ice-rest-catalog/src/test/resources/scenarios/delete-partition-purge/scenario.yaml new file mode 100644 index 00000000..9bc7075c --- /dev/null +++ b/ice-rest-catalog/src/test/resources/scenarios/delete-partition-purge/scenario.yaml @@ -0,0 +1,15 @@ +name: Delete partition with purge +description: Tests deleting a partition with --purge, physically removing its data files from storage + +catalogConfig: + warehouse: s3://test-bucket/warehouse + +env: + NAMESPACE_NAME: test_delete_partition_purge + TABLE_NAME: test_delete_partition_purge.tripdata_p_by_day + INPUT_FILE: tripdata.parquet + PARTITION_SPEC: '[{"column":"tpep_pickup_datetime","transform":"day"}]' + PARTITION_COLUMN: tpep_pickup_datetime + TRANSFORMED_PARTITION_COLUMN: tpep_pickup_datetime_day + DELETE_PARTITION_DAY: 2025-01-02 + S3_BUCKET: test-bucket diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index 7ea5257b..01b9ce32 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -970,7 +970,12 @@ void delete( names = "--dry-run", description = "Log files that would be deleted without actually deleting them (true by default)") - Boolean dryRun) + Boolean dryRun, + @CommandLine.Option( + names = {"--purge"}, + description = + "Physically delete the matched data files from storage (breaks time-travel to snapshots that referenced them). No effect during a dry run.") + boolean purge) throws IOException { if (dryRun == null) { dryRun = true; @@ -985,7 +990,7 @@ void delete( } TableIdentifier tableId = TableIdentifier.parse(name); - Delete.run(catalog, tableId, partitions, dryRun); + Delete.run(catalog, tableId, partitions, dryRun, purge); } catch (URISyntaxException e) { throw new RuntimeException(e); } diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java index b88484a8..ce49b53d 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java @@ -42,7 +42,8 @@ public static void run( RESTCatalog catalog, TableIdentifier tableId, List partitions, - boolean dryRun) + boolean dryRun, + boolean purge) throws IOException, URISyntaxException { Table table = catalog.loadTable(tableId); @@ -98,7 +99,11 @@ public static void run( if (!filesToDelete.isEmpty()) { if (dryRun) { for (DataFile file : filesToDelete) { - logger.info("To be deleted: {}", file.location()); + if (purge) { + logger.info("To be deleted and purged: {}", file.location()); + } else { + logger.info("To be deleted: {}", file.location()); + } } } else { RewriteFiles rewrite = table.newRewrite(); @@ -107,6 +112,13 @@ public static void run( rewrite.deleteFile(deleteFile); } rewrite.commit(); + + if (purge) { + for (DataFile deleteFile : filesToDelete) { + logger.info("Purging {}", deleteFile.location()); + io.deleteFile(deleteFile.location()); + } + } } } else { logger.info("No files to delete");