diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java index 018ba8bb7551..bcde6f3575d0 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java @@ -72,6 +72,7 @@ import java.lang.reflect.Modifier; import java.lang.reflect.Type; import java.math.BigDecimal; +import java.math.BigInteger; import java.math.RoundingMode; import java.sql.Date; import java.sql.Time; @@ -112,6 +113,39 @@ private EnumUtils() {} public static final List LEFT_RIGHT = ImmutableList.of("left", "right"); + /** Converts a FETCH or OFFSET runtime value to {@link BigDecimal}. + * + *

The value must be numeric and non-negative. */ + public static BigDecimal numberToBigDecimal(Object value, String kind) { + return numberToBigDecimal(value, kind, FetchOffsetRoundingPolicy.NONE); + } + + /** Converts a FETCH or OFFSET runtime value to {@link BigDecimal}. + * + *

The value must be numeric and non-negative. The result is adjusted by + * the configured rounding policy. */ + public static BigDecimal numberToBigDecimal(Object value, String kind, + FetchOffsetRoundingPolicy roundingPolicy) { + if (!(value instanceof Number)) { + throw new IllegalArgumentException(kind + " must be a number"); + } + final Number number = (Number) value; + final BigDecimal decimal; + if (number instanceof BigDecimal) { + decimal = (BigDecimal) number; + } else if (number instanceof BigInteger) { + decimal = new BigDecimal((BigInteger) number); + } else if (number instanceof Float || number instanceof Double) { + decimal = BigDecimal.valueOf(number.doubleValue()); + } else { + decimal = BigDecimal.valueOf(number.longValue()); + } + if (decimal.signum() < 0) { + throw new IllegalArgumentException(kind + " must not be negative"); + } + return roundingPolicy.round(decimal); + } + /** Declares a method that overrides another method. */ public static MethodDeclaration overridingMethodDecl(Method method, Iterable parameters, diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java index 3c046de9228d..02fd54bdad86 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimit.java @@ -101,36 +101,50 @@ public static EnumerableLimit create(final RelNode input, @Nullable RexNode offs result.format); Expression v = builder.append("child", result.block); + Expression roundingPolicyExp = getRoundingPolicy(implementor); if (offset != null) { v = builder.append("offset", - Expressions.call(v, BuiltInMethod.SKIP.method, - getExpression(offset))); + Expressions.call(BuiltInMethod.SKIP_BIG_DECIMAL.method, v, + getExpression(offset, "OFFSET", roundingPolicyExp))); } if (fetch != null) { v = builder.append("fetch", - Expressions.call(v, BuiltInMethod.TAKE.method, - getExpression(fetch))); + Expressions.call(BuiltInMethod.TAKE_BIG_DECIMAL.method, v, + getExpression(fetch, "FETCH", roundingPolicyExp))); } builder.add(Expressions.return_(null, v)); return implementor.result(physType, builder.toBlock()); } - static Expression getExpression(RexNode rexNode) { + static Expression getExpression(RexNode rexNode, String kind, + Expression roundingPolicy) { + final Expression value; if (rexNode instanceof RexDynamicParam) { final RexDynamicParam param = (RexDynamicParam) rexNode; - return Expressions.convert_( + value = Expressions.call(DataContext.ROOT, BuiltInMethod.DATA_CONTEXT_GET.method, - Expressions.constant("?" + param.getIndex())), - Integer.class); + Expressions.constant("?" + param.getIndex())); } else { - // TODO: Enumerable runtime only supports INT types for FETCH and OFFSET, not BIGINT types. - // Currently, using BIGINT types for execution will result in an error message. - // This issue needs to be fixed. For more information, see CALCITE-7156. - return Expressions.constant(RexLiteral.intValue(rexNode)); + value = Expressions.constant(RexLiteral.bigDecimalValue(rexNode)); } + return Expressions.call( + BuiltInMethod.NUMBER_TO_BIG_DECIMAL_LIMIT.method, + value, + Expressions.constant(kind), + roundingPolicy); + } + + static Expression getRoundingPolicy(EnumerableRelImplementor implementor) { + final Object roundingPolicy = + implementor.map.get(EnumerableRelImplementor.FETCH_OFFSET_ROUNDING_POLICY); + if (roundingPolicy == null) { + return Expressions.field(null, FetchOffsetRoundingPolicy.class, "NONE"); + } + return implementor.stash((FetchOffsetRoundingPolicy) roundingPolicy, + FetchOffsetRoundingPolicy.class); } } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java index defc28f69a53..325fe687ba4d 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java @@ -30,7 +30,10 @@ import org.checkerframework.checker.nullness.qual.Nullable; +import java.math.BigDecimal; + import static org.apache.calcite.adapter.enumerable.EnumerableLimit.getExpression; +import static org.apache.calcite.adapter.enumerable.EnumerableLimit.getRoundingPolicy; /** * Implementation of {@link org.apache.calcite.rel.core.Sort} in @@ -95,19 +98,20 @@ public static EnumerableLimitSort create( final PhysType inputPhysType = result.physType; final Pair pair = inputPhysType.generateCollationKey(this.collation.getFieldCollations()); + final Expression roundingPolicyExp = getRoundingPolicy(implementor); final Expression fetchVal; if (this.fetch == null) { - fetchVal = Expressions.constant(Integer.MAX_VALUE); + fetchVal = Expressions.constant(BigDecimal.valueOf(Integer.MAX_VALUE)); } else { - fetchVal = getExpression(this.fetch); + fetchVal = getExpression(this.fetch, "FETCH", roundingPolicyExp); } final Expression offsetVal; if (this.offset == null) { - offsetVal = Expressions.constant(0); + offsetVal = Expressions.constant(BigDecimal.ZERO); } else { - offsetVal = getExpression(this.offset); + offsetVal = getExpression(this.offset, "OFFSET", roundingPolicyExp); } builder.add( diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java index 53d0296f7a6b..d55ecbc23e46 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableRelImplementor.java @@ -78,6 +78,9 @@ * operators of {@link EnumerableConvention} calling convention. */ public class EnumerableRelImplementor extends JavaRelImplementor { + public static final String FETCH_OFFSET_ROUNDING_POLICY = + "_fetchOffsetRoundingPolicy"; + public final Map map; private final Map corrVars = new HashMap<>(); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/FetchOffsetRoundingPolicy.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/FetchOffsetRoundingPolicy.java new file mode 100644 index 000000000000..027abcade63e --- /dev/null +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/FetchOffsetRoundingPolicy.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import java.math.BigDecimal; + +/** Defines how enumerable FETCH and OFFSET values are rounded. */ +public interface FetchOffsetRoundingPolicy { + /** Default policy that preserves the value produced by validation or + * parameter binding. */ + FetchOffsetRoundingPolicy NONE = value -> value; + + /** Rounds a FETCH or OFFSET value. */ + BigDecimal round(BigDecimal value); +} diff --git a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java index 8439fabd5f3c..06e27e361cff 100644 --- a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java @@ -21,7 +21,9 @@ import org.apache.calcite.adapter.enumerable.EnumerableConvention; import org.apache.calcite.adapter.enumerable.EnumerableInterpretable; import org.apache.calcite.adapter.enumerable.EnumerableRel; +import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; import org.apache.calcite.adapter.enumerable.EnumerableRules; +import org.apache.calcite.adapter.enumerable.FetchOffsetRoundingPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.adapter.java.JavaTypeFactory; import org.apache.calcite.avatica.AvaticaParameter; @@ -1196,6 +1198,13 @@ protected SqlValidator createSqlValidator(CatalogReader catalogReader, CatalogReader.THREAD_LOCAL.set(catalogReader); final SqlConformance conformance = context.config().conformance(); internalParameters.put("_conformance", conformance); + final FetchOffsetRoundingPolicy fetchOffsetRoundingPolicy = + planner.getContext().unwrap(FetchOffsetRoundingPolicy.class); + if (fetchOffsetRoundingPolicy != null) { + internalParameters.put( + EnumerableRelImplementor.FETCH_OFFSET_ROUNDING_POLICY, + fetchOffsetRoundingPolicy); + } bindable = EnumerableInterpretable.toBindable(internalParameters, context.spark(), enumerable, diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 37f6712e90a7..b891fe76a1f6 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -1743,11 +1743,11 @@ SqlValidatorNamespace getNamespaceOrThrow(SqlIdentifier id, private void handleOffsetFetch(@Nullable SqlNode offset, @Nullable SqlNode fetch) { if (offset instanceof SqlDynamicParam) { setValidatedNodeType(offset, - typeFactory.createSqlType(SqlTypeName.INTEGER)); + typeFactory.createSqlType(SqlTypeName.DECIMAL)); } if (fetch instanceof SqlDynamicParam) { setValidatedNodeType(fetch, - typeFactory.createSqlType(SqlTypeName.INTEGER)); + typeFactory.createSqlType(SqlTypeName.DECIMAL)); } } diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 295d7d662e62..2b07069b6598 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -21,6 +21,7 @@ import org.apache.calcite.adapter.enumerable.BasicAggregateLambdaFactory; import org.apache.calcite.adapter.enumerable.BasicLazyAccumulator; import org.apache.calcite.adapter.enumerable.EnumUtils; +import org.apache.calcite.adapter.enumerable.FetchOffsetRoundingPolicy; import org.apache.calcite.adapter.enumerable.LazyAggregateLambdaFactory; import org.apache.calcite.adapter.enumerable.MatchUtils; import org.apache.calcite.adapter.enumerable.SourceSorter; @@ -296,7 +297,7 @@ public enum BuiltInMethod { ORDER_BY(ExtendedEnumerable.class, "orderBy", Function1.class, Comparator.class), ORDER_BY_WITH_FETCH_AND_OFFSET(EnumerableDefaults.class, "orderBy", Enumerable.class, - Function1.class, Comparator.class, int.class, int.class), + Function1.class, Comparator.class, BigDecimal.class, BigDecimal.class), UNION(ExtendedEnumerable.class, "union", Enumerable.class), CONCAT(ExtendedEnumerable.class, "concat", Enumerable.class), REPEAT_UNION(EnumerableDefaults.class, "repeatUnion", Enumerable.class, @@ -308,7 +309,11 @@ public enum BuiltInMethod { INTERSECT(ExtendedEnumerable.class, "intersect", Enumerable.class, boolean.class), EXCEPT(ExtendedEnumerable.class, "except", Enumerable.class, boolean.class), SKIP(ExtendedEnumerable.class, "skip", int.class), + SKIP_BIG_DECIMAL(EnumerableDefaults.class, "skip", Enumerable.class, + BigDecimal.class), TAKE(ExtendedEnumerable.class, "take", int.class), + TAKE_BIG_DECIMAL(EnumerableDefaults.class, "take", Enumerable.class, + BigDecimal.class), SINGLETON_ENUMERABLE(Linq4j.class, "singletonEnumerable", Object.class), EMPTY_ENUMERABLE(Linq4j.class, "emptyEnumerable"), NULLS_COMPARATOR(Functions.class, "nullsComparator", boolean.class, @@ -390,6 +395,8 @@ public enum BuiltInMethod { Calendar.class), TIME_ZONE_GET_OFFSET(TimeZone.class, "getOffset", long.class), LONG_VALUE(Number.class, "longValue"), + NUMBER_TO_BIG_DECIMAL_LIMIT(EnumUtils.class, "numberToBigDecimal", + Object.class, String.class, FetchOffsetRoundingPolicy.class), STRING_TO_UPPER(String.class, "toUpperCase"), COMPARATOR_COMPARE(Comparator.class, "compare", Object.class, Object.class), COLLECTIONS_REVERSE_ORDER(Collections.class, "reverseOrder"), diff --git a/core/src/test/java/org/apache/calcite/adapter/enumerable/CodeGeneratorTest.java b/core/src/test/java/org/apache/calcite/adapter/enumerable/CodeGeneratorTest.java index 1a32d8de0197..d8e605e39176 100644 --- a/core/src/test/java/org/apache/calcite/adapter/enumerable/CodeGeneratorTest.java +++ b/core/src/test/java/org/apache/calcite/adapter/enumerable/CodeGeneratorTest.java @@ -16,7 +16,9 @@ */ package org.apache.calcite.adapter.enumerable; +import org.apache.calcite.DataContexts; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.linq4j.Enumerable; import org.apache.calcite.linq4j.tree.ClassDeclaration; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.plan.ConventionTraitDef; @@ -26,19 +28,32 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelRoot; import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.runtime.Bindable; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.test.SqlTestFactory; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql2rel.SqlToRelConverter; +import com.google.common.collect.ImmutableList; + import org.junit.jupiter.api.Test; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.HashMap; +import java.util.List; +import java.util.Map; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasSize; import static org.junit.jupiter.api.Assertions.assertFalse; /** @@ -97,4 +112,49 @@ public class CodeGeneratorTest { assertFalse(javaCode.contains("case_when_value3")); assertFalse(javaCode.contains("case_when_value4")); } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test public void testFetchOffsetRoundingPolicy() { + final JavaTypeFactoryImpl typeFactory = + new JavaTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + final RexBuilder rexBuilder = new RexBuilder(typeFactory); + final VolcanoPlanner planner = new VolcanoPlanner(); + planner.addRelTraitDef(ConventionTraitDef.INSTANCE); + final RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); + final RelDataType rowType = + typeFactory.builder().add("i", SqlTypeName.INTEGER).build(); + final RelDataType integerType = rowType.getFieldList().get(0).getType(); + final ImmutableList> tuples = + ImmutableList.of( + ImmutableList.of( + rexBuilder.makeExactLiteral(BigDecimal.ONE, + integerType)), + ImmutableList.of( + rexBuilder.makeExactLiteral(BigDecimal.valueOf(2), + integerType)), + ImmutableList.of( + rexBuilder.makeExactLiteral(BigDecimal.valueOf(3), + integerType))); + final EnumerableValues values = + EnumerableValues.create(cluster, rowType, tuples); + final RelDataType decimalType = + typeFactory.createSqlType(SqlTypeName.DECIMAL, 2, 1); + final EnumerableLimit limit = + EnumerableLimit.create(values, null, + rexBuilder.makeExactLiteral(new BigDecimal("1.5"), decimalType)); + + final Map parameters = new HashMap<>(); + parameters.put(EnumerableRelImplementor.FETCH_OFFSET_ROUNDING_POLICY, + (FetchOffsetRoundingPolicy) value -> + value.setScale(0, RoundingMode.DOWN)); + final Bindable bindable = + EnumerableInterpretable.toBindable(parameters, null, limit, + EnumerableRel.Prefer.ARRAY); + + final Enumerable enumerable = bindable.bind(DataContexts.of(parameters)); + final List result = enumerable.toList(); + assertThat(result, hasSize(1)); + } } diff --git a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java index 267aa39ca162..40b825939df9 100644 --- a/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java +++ b/core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java @@ -29,6 +29,7 @@ import org.junit.jupiter.api.Test; import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.Arrays; import static org.hamcrest.CoreMatchers.is; @@ -171,6 +172,20 @@ public final class EnumUtilsTest { assertThat(Expressions.toString(e2), is("(String) (Object) null")); } + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testNumberToBigDecimalPreservesFractionalNumber() { + final FetchOffsetRoundingPolicy ceiling = + value -> value.setScale(0, RoundingMode.CEILING); + assertThat(EnumUtils.numberToBigDecimal(1.5D, "FETCH"), + is(new BigDecimal("1.5"))); + assertThat(EnumUtils.numberToBigDecimal(1.5F, "OFFSET"), + is(new BigDecimal("1.5"))); + assertThat(EnumUtils.numberToBigDecimal(1.5D, "FETCH", ceiling), + is(BigDecimal.valueOf(2))); + } + @Test void testMethodCallExpression() { // test for Object.class method parameter type final ConstantExpression arg0 = Expressions.constant(1, int.class); diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index a3c004c441ad..6bf15ecdbce5 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -5793,6 +5793,256 @@ private CalciteAssert.AssertQuery withEmpDept(String sql) { Matchers.returnsUnordered("name=Eric")); } + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedOffsetFetchWithBigDecimal() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, BigDecimal.valueOf(3)); + p.setBigDecimal(2, BigDecimal.valueOf(4)); + }) + .returnsUnordered("name=Eric"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedFetchWithFractionalBigDecimal() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, BigDecimal.ZERO); + p.setBigDecimal(2, new BigDecimal("1.5")); + }) + .returnsUnordered("name=Bill", "name=Theodore"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedOffsetWithFractionalBigDecimal() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, new BigDecimal("1.5")); + p.setBigDecimal(2, BigDecimal.ONE); + }) + .returnsUnordered("name=Sebastian"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testFetchLiteralFractionalBigDecimal() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset 0 fetch next 1.5 rows only") + .explainContains("EnumerableLimit(offset=[0], fetch=[1.5:DECIMAL(2, 1)])") + .returnsUnordered("name=Bill", "name=Theodore"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testOffsetLiteralFractionalBigDecimal() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset 1.5 fetch next 1 rows only") + .explainContains("EnumerableLimit(offset=[1.5:DECIMAL(2, 1)], fetch=[1])") + .returnsUnordered("name=Sebastian"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedOffsetFetchWithIntegerParameters() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setInt(1, 1); + p.setInt(2, 2); + }) + .returnsUnordered("name=Theodore", "name=Sebastian"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedOffsetFetchWithLongParameters() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setLong(1, 1L); + p.setLong(2, 2L); + }) + .returnsUnordered("name=Theodore", "name=Sebastian"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedOffsetWithBigDecimalAboveIntegerMax() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, + BigDecimal.valueOf(Integer.MAX_VALUE).add(BigDecimal.ONE)); + p.setBigDecimal(2, BigDecimal.ONE); + }) + .returnsUnordered(); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testOffsetLiteralAboveIntegerMax() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "order by \"empid\" offset 2147483648 fetch next 1 rows only") + .explainContains("EnumerableLimit(offset=[2147483648:BIGINT], fetch=[1])") + .returnsUnordered(); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedOffsetFetchWithBigDecimalWithoutOrderBy() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, BigDecimal.ZERO); + p.setBigDecimal(2, BigDecimal.ZERO); + }) + .returnsUnordered(); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedNegativeFetchWithBigDecimalWithoutOrderBy() + throws Exception { + CalciteAssert.hr() + .doWithConnection(connection -> { + final String sql = "select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "offset ? fetch next ? rows only"; + try (PreparedStatement p = connection.prepareStatement(sql)) { + p.setBigDecimal(1, BigDecimal.ZERO); + p.setBigDecimal(2, BigDecimal.valueOf(-1)); + final SQLException e = assertThrows(SQLException.class, p::executeQuery); + assertThat(e.getMessage(), containsString("FETCH must not be negative")); + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + }); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedNegativeOffsetWithBigDecimalWithoutOrderBy() + throws Exception { + CalciteAssert.hr() + .doWithConnection(connection -> { + final String sql = "select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "offset ? fetch next ? rows only"; + try (PreparedStatement p = connection.prepareStatement(sql)) { + p.setBigDecimal(1, BigDecimal.valueOf(-1)); + p.setBigDecimal(2, BigDecimal.valueOf(2)); + final SQLException e = assertThrows(SQLException.class, p::executeQuery); + assertThat(e.getMessage(), containsString("OFFSET must not be negative")); + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + }); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedNegativeLimitWithBigDecimalWithoutOrderBy() + throws Exception { + CalciteAssert.hr() + .doWithConnection(connection -> { + final String sql = "select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "limit ? offset ?"; + try (PreparedStatement p = connection.prepareStatement(sql)) { + p.setBigDecimal(1, BigDecimal.valueOf(-1)); + p.setBigDecimal(2, BigDecimal.ZERO); + final SQLException e = assertThrows(SQLException.class, p::executeQuery); + assertThat(e.getMessage(), containsString("FETCH must not be negative")); + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + }); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testPreparedFetchWithBigDecimalAboveLongMaxWithoutOrderBy() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "offset ? fetch next ? rows only") + .explainContains("EnumerableLimit(offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, BigDecimal.ZERO); + p.setBigDecimal(2, + BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE)); + }) + .returnsUnordered( + "name=Bill", + "name=Eric", + "name=Sebastian", + "name=Theodore"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testFetchLiteralAboveLongMaxWithoutOrderBy() { + CalciteAssert.hr() + .query("select \"name\"\n" + + "from \"hr\".\"emps\"\n" + + "offset 0 fetch next 9223372036854775808 rows only") + .explainContains("EnumerableLimit(offset=[0], " + + "fetch=[9223372036854775808:DECIMAL(19, 0)])") + .returnsUnordered( + "name=Bill", + "name=Eric", + "name=Sebastian", + "name=Theodore"); + } + private void checkPreparedOffsetFetch(final int offset, final int fetch, final Matcher matcher) throws Exception { CalciteAssert.hr() @@ -5804,8 +6054,8 @@ private void checkPreparedOffsetFetch(final int offset, final int fetch, connection.prepareStatement(sql)) { final ParameterMetaData pmd = p.getParameterMetaData(); assertThat(pmd.getParameterCount(), is(2)); - assertThat(pmd.getParameterType(1), is(Types.INTEGER)); - assertThat(pmd.getParameterType(2), is(Types.INTEGER)); + assertThat(pmd.getParameterType(1), is(Types.DECIMAL)); + assertThat(pmd.getParameterType(2), is(Types.DECIMAL)); p.setInt(1, offset); p.setInt(2, fetch); try (ResultSet r = p.executeQuery()) { diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index f85c6c75bf30..867a7f954b9b 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -10368,6 +10368,24 @@ void testGroupExpressionEquivalenceParams() { .rewritesTo(expected); } + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testNegativeFetchOffsetLimit() { + sql("select name from dept limit ^-^1") + .fails("(?s).*Encountered \"-\".*"); + sql("select name from dept offset ^-^1") + .fails("(?s).*Encountered \"-\".*"); + sql("select name from dept fetch next ^-^1 rows only") + .fails("(?s).*Encountered \"-\".*"); + sql("select name from dept order by name limit ^-^1") + .fails("(?s).*Encountered \"-\".*"); + sql("select name from dept order by name offset ^-^1") + .fails("(?s).*Encountered \"-\".*"); + sql("select name from dept order by name fetch next ^-^1 rows only") + .fails("(?s).*Encountered \"-\".*"); + } + @Test void testRewriteWithUnionFetchWithoutOrderBy() { final String sql = "select name from dept union all select name from dept limit 2"; diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableLimitSortTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableLimitSortTest.java index 5c0d35ca4b5f..90256f8fdd4c 100644 --- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableLimitSortTest.java +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableLimitSortTest.java @@ -24,11 +24,19 @@ import org.apache.calcite.runtime.Hook; import org.apache.calcite.test.CalciteAssert; import org.apache.calcite.test.schemata.hr.HrSchemaBig; +import org.apache.calcite.util.TestUtil; import org.junit.jupiter.api.Test; +import java.math.BigDecimal; +import java.sql.PreparedStatement; +import java.sql.SQLException; import java.util.function.Consumer; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + /** Tests for * {@link org.apache.calcite.adapter.enumerable.EnumerableLimitSort}. */ public class EnumerableLimitSortTest { @@ -158,6 +166,165 @@ public class EnumerableLimitSortTest { "commission=250; empid=36"); } + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void limitWithoutOffsetUsesBigDecimalDefaultOffset() { + tester("select empid from emps order by empid limit 2") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "fetch=[2])") + .returnsOrdered( + "empid=1", + "empid=2"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void offsetWithoutLimitUsesBigDecimalDefaultFetch() { + tester("select empid from emps where empid <= 4 order by empid offset 2") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[2])") + .returnsOrdered( + "empid=3", + "empid=4"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void fractionalOffsetAndFetchUseIndependentRowCounts() { + tester("select empid from emps where empid <= 4 order by empid " + + "offset 1.5 fetch next 1.5 rows only") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[1.5:DECIMAL(2, 1)], fetch=[1.5:DECIMAL(2, 1)])") + .returnsOrdered( + "empid=3", + "empid=4"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void dynamicParametersWithNegativeBigDecimalFetch() throws Exception { + final String sql = "select empid from emps where empid <= 2 order by empid " + + "offset ? fetch next ? rows only"; + tester(sql) + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[?0], fetch=[?1])"); + failsWithNegativeParameter(sql, p -> { + p.setBigDecimal(1, BigDecimal.ZERO); + p.setBigDecimal(2, BigDecimal.valueOf(-1)); + }, "FETCH must not be negative"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void dynamicParametersWithNegativeBigDecimalOffset() throws Exception { + final String sql = "select empid from emps where empid <= 3 order by empid " + + "offset ? fetch next ? rows only"; + tester(sql) + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[?0], fetch=[?1])"); + failsWithNegativeParameter(sql, p -> { + p.setBigDecimal(1, BigDecimal.valueOf(-1)); + p.setBigDecimal(2, BigDecimal.valueOf(2)); + }, "OFFSET must not be negative"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void dynamicParametersWithNegativeBigDecimalLimit() throws Exception { + final String sql = "select empid from emps where empid <= 2 order by empid " + + "limit ? offset ?"; + tester(sql) + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[?1], fetch=[?0])"); + failsWithNegativeParameter(sql, p -> { + p.setBigDecimal(1, BigDecimal.valueOf(-1)); + p.setBigDecimal(2, BigDecimal.ZERO); + }, "FETCH must not be negative"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void dynamicParametersWithBigDecimal() { + tester("select commission from emps order by commission nulls last " + + "offset ? fetch next ? rows only") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[?0], fetch=[?1])\n" + + " EnumerableCalc(expr#0..4=[{inputs}], commission=[$t4])\n" + + " EnumerableTableScan(table=[[s, emps]])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, BigDecimal.valueOf(1)); + p.setBigDecimal(2, BigDecimal.valueOf(2)); + }) + .returnsOrdered( + "commission=250", + "commission=250"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void dynamicParametersWithBigDecimalAboveIntegerMax() { + tester("select commission from emps order by commission nulls last " + + "offset ? fetch next ? rows only") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, + BigDecimal.valueOf(Integer.MAX_VALUE).add(BigDecimal.ONE)); + p.setBigDecimal(2, BigDecimal.ONE); + }) + .returnsOrdered(); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void offsetLiteralAboveIntegerMax() { + tester("select commission from emps order by commission nulls last " + + "offset 2147483648 fetch next 1 rows only") + .explainContains("EnumerableLimitSort(sort0=[$4], dir0=[ASC], " + + "offset=[2147483648:BIGINT], fetch=[1])") + .returnsOrdered(); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void dynamicParametersWithBigDecimalAboveLongMax() { + tester("select empid from emps where empid <= 2 order by empid " + + "offset ? fetch next ? rows only") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[?0], fetch=[?1])") + .consumesPreparedStatement(p -> { + p.setBigDecimal(1, BigDecimal.ZERO); + p.setBigDecimal(2, + BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE)); + }) + .returnsOrdered( + "empid=1", + "empid=2"); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void fetchLiteralAboveLongMax() { + tester("select empid from emps where empid <= 2 order by empid " + + "offset 0 fetch next 9223372036854775808 rows only") + .explainContains("EnumerableLimitSort(sort0=[$0], dir0=[ASC], " + + "offset=[0], fetch=[9223372036854775808:DECIMAL(19, 0)])") + .returnsOrdered( + "empid=1", + "empid=2"); + } + private CalciteAssert.AssertQuery tester(String sqlQuery) { return CalciteAssert.that() .with(CalciteConnectionProperty.LEX, Lex.JAVA) @@ -169,4 +336,27 @@ private CalciteAssert.AssertQuery tester(String sqlQuery) { planner.addRule(EnumerableRules.ENUMERABLE_LIMIT_SORT_RULE); }); } + + private void failsWithNegativeParameter(String sql, + CalciteAssert.PreparedStatementConsumer consumer, String message) + throws Exception { + CalciteAssert.that() + .with(CalciteConnectionProperty.LEX, Lex.JAVA) + .with(CalciteConnectionProperty.FORCE_DECORRELATE, false) + .withSchema("s", new ReflectiveSchema(new HrSchemaBig())) + .doWithConnection(connection -> { + try (Hook.Closeable ignored = + Hook.PLANNER.addThread((Consumer) planner -> { + planner.removeRule(EnumerableRules.ENUMERABLE_SORT_RULE); + planner.addRule(EnumerableRules.ENUMERABLE_LIMIT_SORT_RULE); + }); + PreparedStatement p = connection.prepareStatement(sql)) { + consumer.accept(p); + final SQLException e = assertThrows(SQLException.class, p::executeQuery); + assertThat(e.getMessage(), containsString(message)); + } catch (SQLException e) { + throw TestUtil.rethrow(e); + } + }); + } } diff --git a/core/src/test/resources/sql/sort.iq b/core/src/test/resources/sql/sort.iq index f9149a54822a..0e8d84cff947 100644 --- a/core/src/test/resources/sql/sort.iq +++ b/core/src/test/resources/sql/sort.iq @@ -419,8 +419,13 @@ order by arr desc; # [CALCITE-7156] OFFSET and FETCH in EnumerableLimit need to support BIGINT select * from "hr"."emps" limit 3000000000 offset 2500000000; -java.lang.ArithmeticException: Integer overflow: 2500000000 is out of range for INT -!error ++-------+--------+------+--------+------------+ +| empid | deptno | name | salary | commission | ++-------+--------+------+--------+------------+ ++-------+--------+------+--------+------------+ +(0 rows) + +!ok # [CALCITE-7367] NULLS FIRST throws ClassCastException when sorting arrays select * from diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java index 54dcc20f41e7..c02001b346d4 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/DefaultEnumerable.java @@ -693,6 +693,10 @@ protected OrderedQueryable asOrderedQueryable() { return EnumerableDefaults.skip(getThis(), count); } + @Override public Enumerable skip(BigDecimal count) { + return EnumerableDefaults.skip(getThis(), count); + } + @Override public Enumerable skipWhile(Predicate1 predicate) { return EnumerableDefaults.skipWhile(getThis(), predicate); } @@ -701,6 +705,10 @@ protected OrderedQueryable asOrderedQueryable() { return EnumerableDefaults.skipWhile(getThis(), predicate); } + @Override public Enumerable skipWhileBigDecimal(Predicate2 predicate) { + return EnumerableDefaults.skipWhileBigDecimal(getThis(), predicate); + } + @Override public BigDecimal sum(BigDecimalFunction1 selector) { return EnumerableDefaults.sum(getThis(), selector); } @@ -745,6 +753,10 @@ protected OrderedQueryable asOrderedQueryable() { return EnumerableDefaults.take(getThis(), count); } + @Override public Enumerable take(BigDecimal count) { + return EnumerableDefaults.take(getThis(), count); + } + @Override public Enumerable takeWhile(Predicate1 predicate) { return EnumerableDefaults.takeWhile(getThis(), predicate); } @@ -753,6 +765,10 @@ protected OrderedQueryable asOrderedQueryable() { return EnumerableDefaults.takeWhile(getThis(), predicate); } + @Override public Enumerable takeWhileBigDecimal(Predicate2 predicate) { + return EnumerableDefaults.takeWhileBigDecimal(getThis(), predicate); + } + @Override public > OrderedEnumerable thenBy( Function1 keySelector) { return EnumerableDefaults.thenBy(getThisOrdered(), keySelector); diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java index ae26a602132d..0ff4afddb6dd 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java @@ -48,6 +48,7 @@ import org.checkerframework.framework.qual.HasQualifierParameter; import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.AbstractList; import java.util.AbstractMap; import java.util.AbstractSet; @@ -3244,26 +3245,43 @@ public static Enumerable orderBy( Function1 keySelector, Comparator comparator, int offset, int fetch) { + return orderBy(source, keySelector, comparator, + BigDecimal.valueOf(offset), BigDecimal.valueOf(fetch)); + } + + /** + * A sort implementation optimized for a sort with a fetch size (LIMIT). + * + * @param offset how many rows are skipped from the sorted output. + * Must be greater than or equal to 0. + * @param fetch how many rows are retrieved. Must be greater than or equal to 0. + */ + public static Enumerable orderBy( + Enumerable source, + Function1 keySelector, + Comparator comparator, + BigDecimal offset, BigDecimal fetch) { // As discussed in CALCITE-3920 and CALCITE-4157, this method avoids to sort the complete input, // if only the first N rows are actually needed. A TreeMap implementation has been chosen, // so that it behaves similar to the orderBy method without fetch/offset. // The TreeMap has a better performance if there are few distinct sort keys. return new AbstractEnumerable() { @Override public Enumerator enumerator() { - if (fetch == 0) { + if (fetch.compareTo(BigDecimal.ZERO) <= 0) { return Linq4j.emptyEnumerator(); } TreeMap> map = new TreeMap<>(comparator); - long size = 0; - long needed = fetch + (long) offset; + BigDecimal size = BigDecimal.ZERO; + BigDecimal actualOffset = offset.max(BigDecimal.ZERO); + BigDecimal needed = rowsRequired(actualOffset).add(rowsRequired(fetch)); // read the input into a tree map try (Enumerator os = source.enumerator()) { while (os.moveNext()) { TSource o = os.current(); TKey key = keySelector.apply(o); - if (needed >= 0 && size >= needed) { + if (needed.signum() >= 0 && size.compareTo(needed) >= 0) { // the current row will never appear in the output, so just skip it @KeyFor("map") TKey lastKey = map.lastKey(); if (comparator.compare(key, lastKey) >= 0) { @@ -3277,7 +3295,7 @@ public static Enumerable orderBy( } else { l.remove(l.size() - 1); } - size--; + size = size.subtract(BigDecimal.ONE); } // add the current element to the map map.compute(key, (k, l) -> { @@ -3292,24 +3310,30 @@ public static Enumerable orderBy( l.add(o); return l; }); - size++; + size = size.add(BigDecimal.ONE); } } // skip the first 'offset' rows by deleting them from the map - if (offset > 0) { - // search the key up to (but excluding) which we have to remove entries from the map - int skipped = 0; + if (actualOffset.compareTo(BigDecimal.ZERO) > 0) { + // search the key up to which we have to remove entries from the map + BigDecimal skipped = BigDecimal.ZERO; + BigDecimal rowsToSkip = rowsRequired(actualOffset); TKey until = (TKey) DUMMY; + boolean removeUntilInclusive = false; for (Map.Entry> e : map.entrySet()) { - skipped += e.getValue().size(); + skipped = skipped.add(BigDecimal.valueOf(e.getValue().size())); - if (skipped > offset) { + if (skipped.compareTo(rowsToSkip) >= 0) { // we might need to remove entries from the list List l = e.getValue(); - int toKeep = skipped - offset; - if (toKeep < l.size()) { - l.subList(0, l.size() - toKeep).clear(); + BigDecimal toKeep = skipped.subtract(rowsToSkip); + if (toKeep.compareTo(BigDecimal.valueOf(l.size())) < 0) { + if (toKeep.signum() == 0) { + removeUntilInclusive = true; + } else { + l.subList(0, l.size() - toKeep.intValueExact()).clear(); + } } until = e.getKey(); @@ -3320,7 +3344,7 @@ public static Enumerable orderBy( // the offset is bigger than the number of rows in the map return Linq4j.emptyEnumerator(); } - map.headMap(until, false).clear(); + map.headMap(until, removeUntilInclusive).clear(); } return new LookupImpl<>(map).valuesEnumerable().enumerator(); @@ -3790,6 +3814,10 @@ public static TSource single(Enumerable source, return toRet; } + private static BigDecimal rowsRequired(BigDecimal count) { + return count.max(BigDecimal.ZERO).setScale(0, RoundingMode.CEILING); + } + /** * Bypasses a specified number of elements in a * sequence and then returns the remaining elements. @@ -3802,6 +3830,18 @@ public static Enumerable skip(Enumerable source, }); } + /** + * Bypasses a specified number of elements in a + * sequence and then returns the remaining elements. + */ + public static Enumerable skip(Enumerable source, + final BigDecimal count) { + return skipWhileBigDecimal(source, (v1, v2) -> { + // Count is 1-based + return v2.compareTo(count) < 0; + }); + } + /** * Bypasses elements in a sequence as long as a * specified condition is true and then returns the remaining @@ -3957,6 +3997,19 @@ public static Enumerable take(Enumerable source, }); } + /** + * Returns a specified number of contiguous elements + * from the start of a sequence. + */ + public static Enumerable take(Enumerable source, + final BigDecimal count) { + return takeWhileBigDecimal( + source, (v1, v2) -> { + // Count is 1-based + return v2.compareTo(count) < 0; + }); + } + /** * Returns elements from a sequence as long as a * specified condition is true. @@ -3997,6 +4050,36 @@ public static Enumerable takeWhileLong( }; } + /** + * Returns elements from a sequence as long as a + * specified condition is true. The element's index is used in the + * logic of the predicate function. + */ + public static Enumerable takeWhileBigDecimal( + final Enumerable source, + final Predicate2 predicate) { + return new AbstractEnumerable() { + @Override public Enumerator enumerator() { + return new TakeWhileBigDecimalEnumerator<>(source.enumerator(), predicate); + } + }; + } + + /** + * Bypasses elements in a sequence as long as a + * specified condition is true. The element's index is used in the + * logic of the predicate function. + */ + public static Enumerable skipWhileBigDecimal( + final Enumerable source, + final Predicate2 predicate) { + return new AbstractEnumerable() { + @Override public Enumerator enumerator() { + return new SkipWhileBigDecimalEnumerator<>(source.enumerator(), predicate); + } + }; + } + /** * Performs a subsequent ordering of the elements in a sequence according * to a key. @@ -4577,6 +4660,50 @@ static class TakeWhileLongEnumerator implements Enumerator { } } + /** Enumerable that implements take-while. + * + * @param element type */ + static class TakeWhileBigDecimalEnumerator implements Enumerator { + private final Enumerator enumerator; + private final Predicate2 predicate; + + boolean done = false; + BigDecimal n = BigDecimal.valueOf(-1); + + TakeWhileBigDecimalEnumerator(Enumerator enumerator, + Predicate2 predicate) { + this.enumerator = enumerator; + this.predicate = predicate; + } + + @Override public TSource current() { + return enumerator.current(); + } + + @Override public boolean moveNext() { + if (!done) { + if (enumerator.moveNext()) { + n = n.add(BigDecimal.ONE); + if (predicate.apply(enumerator.current(), n)) { + return true; + } + } + done = true; + } + return false; + } + + @Override public void reset() { + enumerator.reset(); + done = false; + n = BigDecimal.valueOf(-1); + } + + @Override public void close() { + enumerator.close(); + } + } + /** Enumerator that implements skip-while. * * @param element type */ @@ -4623,6 +4750,53 @@ static class SkipWhileEnumerator implements Enumerator { } } + /** Enumerator that implements skip-while. + * + * @param element type */ + static class SkipWhileBigDecimalEnumerator implements Enumerator { + private final Enumerator enumerator; + private final Predicate2 predicate; + + boolean started = false; + BigDecimal n = BigDecimal.valueOf(-1); + + SkipWhileBigDecimalEnumerator(Enumerator enumerator, + Predicate2 predicate) { + this.enumerator = enumerator; + this.predicate = predicate; + } + + @Override public TSource current() { + return enumerator.current(); + } + + @Override public boolean moveNext() { + for (;;) { + if (!enumerator.moveNext()) { + return false; + } + if (started) { + return true; + } + n = n.add(BigDecimal.ONE); + if (!predicate.apply(enumerator.current(), n)) { + started = true; + return true; + } + } + } + + @Override public void reset() { + enumerator.reset(); + started = false; + n = BigDecimal.valueOf(-1); + } + + @Override public void close() { + enumerator.close(); + } + } + /** Enumerator that casts each value. * *

If the source type {@code F} is not nullable, the target element type diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java index ff28c3fe822c..362372464ac4 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/ExtendedEnumerable.java @@ -1095,6 +1095,12 @@ boolean sequenceEqual(Enumerable enumerable1, */ Enumerable skip(int count); + /** + * Bypasses a specified number of elements in a + * sequence and then returns the remaining elements. + */ + Enumerable skip(BigDecimal count); + /** * Bypasses elements in a sequence as long as a * specified condition is true and then returns the remaining @@ -1110,6 +1116,14 @@ boolean sequenceEqual(Enumerable enumerable1, */ Enumerable skipWhile(Predicate2 predicate); + /** + * Bypasses elements in a sequence as long as a + * specified condition is true and then returns the remaining + * elements. The element's index is used in the logic of the + * predicate function. + */ + Enumerable skipWhileBigDecimal(Predicate2 predicate); + /** * Computes the sum of the sequence of Decimal values * that are obtained by invoking a transform function on each @@ -1186,6 +1200,12 @@ boolean sequenceEqual(Enumerable enumerable1, */ Enumerable take(int count); + /** + * Returns a specified number of contiguous elements + * from the start of a sequence. + */ + Enumerable take(BigDecimal count); + /** * Returns elements from a sequence as long as a * specified condition is true. @@ -1199,6 +1219,13 @@ boolean sequenceEqual(Enumerable enumerable1, */ Enumerable takeWhile(Predicate2 predicate); + /** + * Returns elements from a sequence as long as a + * specified condition is true. The element's index is used in the + * logic of the predicate function. + */ + Enumerable takeWhileBigDecimal(Predicate2 predicate); + /** * Creates a {@code Map} from an * {@code Enumerable} according to a specified key selector diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java index 9e2fa64a4c03..df9e39e35bd2 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/Linq4j.java @@ -21,6 +21,8 @@ import org.checkerframework.checker.nullness.qual.Nullable; import java.lang.reflect.Method; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -591,6 +593,16 @@ static class ListEnumerable extends CollectionEnumerable { return new ListEnumerable<>(list.subList(count, list.size())); } + @Override public Enumerable skip(BigDecimal count) { + final List list = toList(); + BigDecimal rows = count.max(BigDecimal.ZERO) + .setScale(0, RoundingMode.CEILING); + if (rows.compareTo(BigDecimal.valueOf(list.size())) >= 0) { + return Linq4j.emptyEnumerable(); + } + return new ListEnumerable<>(list.subList(rows.intValueExact(), list.size())); + } + @Override public Enumerable take(int count) { final List list = toList(); if (count >= list.size()) { @@ -599,6 +611,16 @@ static class ListEnumerable extends CollectionEnumerable { return new ListEnumerable<>(list.subList(0, count)); } + @Override public Enumerable take(BigDecimal count) { + final List list = toList(); + BigDecimal rows = count.max(BigDecimal.ZERO) + .setScale(0, RoundingMode.CEILING); + if (rows.compareTo(BigDecimal.valueOf(list.size())) >= 0) { + return this; + } + return new ListEnumerable<>(list.subList(0, rows.intValueExact())); + } + @Override public T elementAt(int index) { return toList().get(index); } diff --git a/linq4j/src/test/java/org/apache/calcite/linq4j/test/Linq4jTest.java b/linq4j/src/test/java/org/apache/calcite/linq4j/test/Linq4jTest.java index bacb6da3c8b1..62e7dd9f5eef 100644 --- a/linq4j/src/test/java/org/apache/calcite/linq4j/test/Linq4jTest.java +++ b/linq4j/src/test/java/org/apache/calcite/linq4j/test/Linq4jTest.java @@ -1264,6 +1264,28 @@ private List contentsOf(Enumerator enumerator) { .toList(), hasSize(0)); } + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testTakeEnumerableDefaultsBigDecimalSize() { + assertThat( + EnumerableDefaults.take(Linq4j.asEnumerable(depts), + new BigDecimal("1.5")).toList(), hasSize(2)); + assertThat( + EnumerableDefaults.take(Linq4j.asEnumerable(depts), + BigDecimal.valueOf(-2)).toList(), hasSize(0)); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testTakeListEnumerableBigDecimalSize() { + assertThat(Linq4j.asEnumerable(depts).take(new BigDecimal("1.5")) + .toList(), hasSize(2)); + assertThat(Linq4j.asEnumerable(depts).take(BigDecimal.valueOf(-2)) + .toList(), hasSize(0)); + } + @Test void testTakeQueryableZeroOrNegativeSize() { assertThat(QueryableDefaults.take(Linq4j.asEnumerable(depts).asQueryable(), 0) .toList(), hasSize(0)); @@ -1424,6 +1446,28 @@ public boolean apply(Department v1, Integer v2) { || v2 == 1)).count(), is(1)); } + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testSkipEnumerableDefaultsBigDecimalSize() { + assertThat( + EnumerableDefaults.skip(Linq4j.asEnumerable(depts), + new BigDecimal("1.5")).count(), is(1)); + assertThat( + EnumerableDefaults.skip(Linq4j.asEnumerable(depts), + BigDecimal.valueOf(-2)).count(), is(3)); + } + + /** Test case for + * [CALCITE-7624] + * Support BigDecimal for FETCH and OFFSET in Enumerable. */ + @Test void testSkipListEnumerableBigDecimalSize() { + assertThat(Linq4j.asEnumerable(depts).skip(new BigDecimal("1.5")) + .count(), is(1)); + assertThat(Linq4j.asEnumerable(depts).skip(BigDecimal.valueOf(-2)) + .count(), is(3)); + } + @Test void testOrderBy() { // Note: sort is stable. Records occur Fred, Eric, Janet in input. assertThat(Linq4j.asEnumerable(emps).orderBy(EMP_DEPTNO_SELECTOR) diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 6250a6d63943..2e20f867abb1 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -427,8 +427,9 @@ in the order that they appear in the list; for example: "SELECT x, y FROM t ORDER BY x, y" An optional trailing ASC / DESC and NULLS FIRST / NULLS LAST applies to all keys. -In *query*, *count* and *start* may each be either an unsigned integer literal -or a dynamic parameter whose value is an integer. +In *query*, *count* and *start* may each be either an unsigned numeric literal +or a dynamic parameter whose value is numeric. +Support for decimal or non-integer values is adapter-dependent. An aggregate query is a query that contains a GROUP BY or a HAVING clause, or aggregate functions in the SELECT clause. In the SELECT,