From a897bfa53dac6ec0ccd92470d9f844e471e5195a Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Mon, 20 Jul 2026 20:48:28 -0400 Subject: [PATCH 1/2] Added support for deleting parititions with logic operators other than equals. --- examples/scratch/README.md | 4 + .../scenarios/delete-partition/run.sh.tmpl | 11 +++ ice/README.md | 21 +++++ .../main/java/com/altinity/ice/cli/Main.java | 45 ++++++++++- .../altinity/ice/cli/internal/cmd/Delete.java | 12 ++- .../ice/cli/internal/cmd/DeleteTest.java | 81 +++++++++++++++++++ 6 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 ice/src/test/java/com/altinity/ice/cli/internal/cmd/DeleteTest.java diff --git a/examples/scratch/README.md b/examples/scratch/README.md index 011fd715..0f091ea7 100644 --- a/examples/scratch/README.md +++ b/examples/scratch/README.md @@ -39,6 +39,10 @@ ice insert nyc.taxis_p_by_day -p \ ice delete nyc.taxis_p_by_day \ --partition '[{"name": "tpep_pickup_datetime", "values": ["2024-12-31T23:51:20"]}]' --dry-run=false +# delete a range of partitions using a comparison operator (default op is equals) +ice delete nyc.taxis_p_by_day \ + --partition '[{"name": "tpep_pickup_datetime", "op": "greater_than_or_equal", "values": ["2024-12-31T23:51:20"]}]' --dry-run=false + # insert data ordered by tpep_pickup_datetime column ice insert nyc.taxis_s_by_day -p \ https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-01.parquet \ diff --git a/ice-rest-catalog/src/test/resources/scenarios/delete-partition/run.sh.tmpl b/ice-rest-catalog/src/test/resources/scenarios/delete-partition/run.sh.tmpl index 22227aad..46523d5a 100644 --- a/ice-rest-catalog/src/test/resources/scenarios/delete-partition/run.sh.tmpl +++ b/ice-rest-catalog/src/test/resources/scenarios/delete-partition/run.sh.tmpl @@ -43,6 +43,17 @@ if [[ "$list_after" == *"${TRANSFORMED_PARTITION_COLUMN}=${DELETE_PARTITION_DAY} fi echo "OK Validated partition was deleted" +# Exercise a comparison operator (greater_than_or_equal) via a dry run. +# >= 2000-01-01 matches all remaining data, so candidate files must be found. +range_output=$({{ICE_CLI}} --config {{CLI_CONFIG}} delete ${TABLE_NAME} \ + --partition "[{\"name\": \"${TRANSFORMED_PARTITION_COLUMN}\", \"op\": \"greater_than_or_equal\", \"values\": [\"2000-01-01\"]}]" 2>&1) +echo "$range_output" +if ! echo "$range_output" | grep -qF "To be deleted"; then + echo "FAIL Expected range (>= 2000-01-01) dry-run to find candidate files" + exit 1 +fi +echo "OK Range operator (greater_than_or_equal) matched files" + # Cleanup {{ICE_CLI}} --config {{CLI_CONFIG}} delete-table ${TABLE_NAME} echo "OK Deleted table: ${TABLE_NAME}" diff --git a/ice/README.md b/ice/README.md index c5d5f5b9..8e0bfe16 100644 --- a/ice/README.md +++ b/ice/README.md @@ -123,8 +123,29 @@ ice delete nyc.taxis_p_by_day \ ice delete nyc.taxis_p_by_day \ --partition '[{"name": "tpep_pickup_datetime_day", "values": ["2024-12-31"]}]' \ --dry-run=false + +# actually delete and physically purge the data files from storage +ice delete nyc.taxis_p_by_day \ + --partition '[{"name": "tpep_pickup_datetime_day", "values": ["2024-12-31"]}]' \ + --dry-run=false --purge + +# delete a range of partitions using a comparison operator +ice delete nyc.taxis_p_by_day \ + --partition '[{"name": "tpep_pickup_datetime_day", "op": "greater_than_or_equal", "values": ["2024-12-31"]}]' \ + --dry-run=false ``` +Each partition filter accepts an optional `op` field (default `equals`): one of +`equals`, `less_than`, `greater_than`, `less_than_or_equal`, `greater_than_or_equal`. +Values within a filter are OR'd together and filters across fields are AND'd, so a +range delete typically uses a single value. + +Without `--purge`, `ice delete` only de-references the data files (a new snapshot is +created); the physical files remain until catalog maintenance reclaims them via +`SNAPSHOT_CLEANUP` / `ORPHAN_CLEANUP`. With `--purge`, the matched data files are +deleted from storage immediately, which breaks time-travel to snapshots that +referenced them. + ### Insert Without Copy Insert an S3 file by reference (no data copy) when it's already in the warehouse: diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index 7ea5257b..ed34c053 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -36,7 +36,9 @@ import com.altinity.ice.internal.jetty.DebugServer; import com.altinity.ice.internal.picocli.VersionProvider; import com.altinity.ice.internal.strings.Strings; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; @@ -964,7 +966,10 @@ void delete( names = {"--partition"}, description = "JSON array of partition filters: [{\"name\": \"vendorId\", \"values\": [5, 6]}]. " - + "For timestamp columns, use ISO Datetime format YYYY-MM-ddTHH:mm:ss") + + "For timestamp columns, use ISO Datetime format YYYY-MM-ddTHH:mm:ss. " + + "Each filter may set an optional \"op\": one of equals (default), less_than, " + + "greater_than, less_than_or_equal, greater_than_or_equal, e.g. " + + "[{\"name\": \"vendorId\", \"op\": \"greater_than_or_equal\", \"values\": [5]}]") String partitionJson, @CommandLine.Option( names = "--dry-run", @@ -992,7 +997,43 @@ void delete( } public record PartitionFilter( - @JsonProperty("name") String name, @JsonProperty("values") List values) {} + @JsonProperty("name") String name, + @JsonProperty("values") List values, + @JsonProperty("op") Op op) { + + public PartitionFilter { + op = op == null ? Op.EQUALS : op; + } + + public enum Op { + EQUALS("equals"), + LESS_THAN("less_than"), + GREATER_THAN("greater_than"), + LESS_THAN_OR_EQUAL("less_than_or_equal"), + GREATER_THAN_OR_EQUAL("greater_than_or_equal"); + + private final String json; + + Op(String json) { + this.json = json; + } + + @JsonValue + public String json() { + return json; + } + + @JsonCreator + public static Op fromJson(String v) { + for (Op op : values()) { + if (op.json.equalsIgnoreCase(v) || op.name().equalsIgnoreCase(v)) { + return op; + } + } + throw new IllegalArgumentException("unknown partition filter op: " + v); + } + } + } private RESTCatalog loadCatalog() throws IOException { return loadCatalog(this.configFile()); diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java index b88484a8..fe4aa834 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/Delete.java @@ -72,7 +72,7 @@ public static void run( value = transformed; } - Expression singleValueExpr = Expressions.equal(fieldName, value); + Expression singleValueExpr = toPredicate(fieldName, value, pf.op()); fieldExpr = fieldExpr == null ? singleValueExpr : Expressions.or(fieldExpr, singleValueExpr); } @@ -112,4 +112,14 @@ public static void run( logger.info("No files to delete"); } } + + static Expression toPredicate(String fieldName, Object value, PartitionFilter.Op op) { + return switch (op) { + case EQUALS -> Expressions.equal(fieldName, value); + case LESS_THAN -> Expressions.lessThan(fieldName, value); + case GREATER_THAN -> Expressions.greaterThan(fieldName, value); + case LESS_THAN_OR_EQUAL -> Expressions.lessThanOrEqual(fieldName, value); + case GREATER_THAN_OR_EQUAL -> Expressions.greaterThanOrEqual(fieldName, value); + }; + } } diff --git a/ice/src/test/java/com/altinity/ice/cli/internal/cmd/DeleteTest.java b/ice/src/test/java/com/altinity/ice/cli/internal/cmd/DeleteTest.java new file mode 100644 index 00000000..5cef323c --- /dev/null +++ b/ice/src/test/java/com/altinity/ice/cli/internal/cmd/DeleteTest.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.cli.internal.cmd; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.altinity.ice.cli.Main.PartitionFilter; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.List; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.UnboundPredicate; +import org.testng.annotations.Test; + +public class DeleteTest { + + private final ObjectMapper mapper = new ObjectMapper(); + + @Test + public void testToPredicateOperations() { + assertThat(((UnboundPredicate) Delete.toPredicate("f", 5, PartitionFilter.Op.EQUALS)).op()) + .isEqualTo(Expression.Operation.EQ); + assertThat( + ((UnboundPredicate) Delete.toPredicate("f", 5, PartitionFilter.Op.LESS_THAN)).op()) + .isEqualTo(Expression.Operation.LT); + assertThat( + ((UnboundPredicate) Delete.toPredicate("f", 5, PartitionFilter.Op.GREATER_THAN)) + .op()) + .isEqualTo(Expression.Operation.GT); + assertThat( + ((UnboundPredicate) + Delete.toPredicate("f", 5, PartitionFilter.Op.LESS_THAN_OR_EQUAL)) + .op()) + .isEqualTo(Expression.Operation.LT_EQ); + assertThat( + ((UnboundPredicate) + Delete.toPredicate("f", 5, PartitionFilter.Op.GREATER_THAN_OR_EQUAL)) + .op()) + .isEqualTo(Expression.Operation.GT_EQ); + } + + @Test + public void testOpDefaultsToEquals() { + PartitionFilter pf = new PartitionFilter("f", List.of(1), null); + assertThat(pf.op()).isEqualTo(PartitionFilter.Op.EQUALS); + } + + @Test + public void testDeserializeWithoutOpDefaultsToEquals() throws Exception { + PartitionFilter pf = + mapper.readValue("{\"name\": \"f\", \"values\": [1]}", PartitionFilter.class); + assertThat(pf.name()).isEqualTo("f"); + assertThat(pf.op()).isEqualTo(PartitionFilter.Op.EQUALS); + } + + @Test + public void testDeserializeWithOp() throws Exception { + PartitionFilter pf = + mapper.readValue( + "{\"name\": \"f\", \"op\": \"greater_than_or_equal\", \"values\": [1]}", + PartitionFilter.class); + assertThat(pf.op()).isEqualTo(PartitionFilter.Op.GREATER_THAN_OR_EQUAL); + } + + @Test + public void testDeserializeUnknownOpThrows() { + assertThatThrownBy( + () -> + mapper.readValue( + "{\"name\": \"f\", \"op\": \"between\", \"values\": [1]}", + PartitionFilter.class)) + .hasMessageContaining("unknown partition filter op"); + } +} From 34b2174ee9ca6b42a9d37321bac7a9e359ee8f65 Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Tue, 21 Jul 2026 14:51:30 -0500 Subject: [PATCH 2/2] Update README to include partition name --- examples/scratch/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/scratch/README.md b/examples/scratch/README.md index 0f091ea7..df4a6f47 100644 --- a/examples/scratch/README.md +++ b/examples/scratch/README.md @@ -37,11 +37,11 @@ ice insert nyc.taxis_p_by_day -p \ # delete partition ice delete nyc.taxis_p_by_day \ - --partition '[{"name": "tpep_pickup_datetime", "values": ["2024-12-31T23:51:20"]}]' --dry-run=false + --partition '[{"name": "tpep_pickup_datetime_day", "values": ["2024-12-31T23:51:20"]}]' --dry-run=false # delete a range of partitions using a comparison operator (default op is equals) ice delete nyc.taxis_p_by_day \ - --partition '[{"name": "tpep_pickup_datetime", "op": "greater_than_or_equal", "values": ["2024-12-31T23:51:20"]}]' --dry-run=false + --partition '[{"name": "tpep_pickup_datetime_day", "op": "greater_than_or_equal", "values": ["2024-12-31T23:51:20"]}]' --dry-run=false # insert data ordered by tpep_pickup_datetime column ice insert nyc.taxis_s_by_day -p \ @@ -79,7 +79,7 @@ CREATE DATABASE ice ENGINE = DataLakeCatalog('http://localhost:5000') SETTINGS catalog_type = 'rest', auth_header = 'Authorization: Bearer foo', - storage_endpoint = 'http://localhost:9000', + storage_endpoint = 'http://localhost:8999', warehouse = 's3://bucket1'; -- inspect