diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java index de167cd08e4..768c580513d 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java @@ -28,8 +28,7 @@ import org.apache.calcite.runtime.SqlFunctions.FlatProductInputType; import org.apache.calcite.sql.type.MapSqlType; import org.apache.calcite.util.BuiltInMethod; - -import com.google.common.primitives.Ints; +import org.apache.calcite.util.ImmutableBitSet; import java.util.ArrayList; import java.util.Collections; @@ -56,6 +55,19 @@ public EnumerableUncollect(RelOptCluster cluster, RelTraitSet traitSet, assert getConvention() == child.getConvention(); } + /** Creates an EnumerableUncollect with explicit control over which input + * fields are passed through unchanged and which are unnested. + * + *

Use {@link #create} unless you know what you're doing. */ + public EnumerableUncollect(RelOptCluster cluster, RelTraitSet traitSet, + RelNode child, boolean withOrdinality, ImmutableBitSet passthroughFieldIndices, + ImmutableBitSet collectionFieldIndices, boolean outer, boolean expandStructFields) { + super(cluster, traitSet, child, withOrdinality, Collections.emptyList(), + passthroughFieldIndices, collectionFieldIndices, outer, expandStructFields); + assert getConvention() instanceof EnumerableConvention; + assert getConvention() == child.getConvention(); + } + /** * Creates an EnumerableUncollect. * @@ -72,70 +84,98 @@ public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, return new EnumerableUncollect(cluster, traitSet, input, withOrdinality); } + /** + * Creates an EnumerableUncollect that unnests the collection-typed fields at + * {@code collectionFieldIndices}, passing through the fields at + * {@code passthroughFieldIndices} unchanged and dropping all others. + * + * @param traitSet Trait set + * @param input Input relational expression + * @param withOrdinality Whether output should contain an ORDINALITY column + * @param passthroughFieldIndices Fields from the input copied unchanged to the output + * @param collectionFieldIndices Fields from the input which are unnested + * @param outer if true, preserves input rows with null/empty collections + * (LEFT JOIN); if false, drops them (INNER) + */ + public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, + boolean withOrdinality, ImmutableBitSet passthroughFieldIndices, + ImmutableBitSet collectionFieldIndices, boolean outer, boolean expandStructFields) { + final RelOptCluster cluster = input.getCluster(); + return new EnumerableUncollect(cluster, traitSet, input, withOrdinality, + passthroughFieldIndices, collectionFieldIndices, outer, expandStructFields); + } + @Override public EnumerableUncollect copy(RelTraitSet traitSet, RelNode newInput) { - return new EnumerableUncollect(getCluster(), traitSet, newInput, - withOrdinality); + return new EnumerableUncollect(getCluster(), traitSet, newInput, withOrdinality, + getPassthroughFieldIndices(), getCollectionFieldIndices(), isOuter, + expandStructFields); } + /** Implements Uncollect: a subset of input fields (possibly all of them) + * are unnested, the rest are either passed through unchanged or dropped, + * and (if {@link #isOuter}) rows with null/empty collections are preserved + * with NULL-padded element columns. */ @Override public Result implement(EnumerableRelImplementor implementor, Prefer pref) { + final ImmutableBitSet passthroughIndices = getPassthroughFieldIndices(); + final ImmutableBitSet collIndices = getCollectionFieldIndices(); final BlockBuilder builder = new BlockBuilder(); final EnumerableRel child = (EnumerableRel) getInput(); - final Result result = implementor.visitChild(this, 0, child, pref); + final Result childResult = implementor.visitChild(this, 0, child, pref); final PhysType physType = - PhysTypeImpl.of( - implementor.getTypeFactory(), - getRowType(), - JavaRowFormat.LIST); - - // final Enumerable> child = <>; - // return child.selectMany(FLAT_ZIP); - final Expression child_ = - builder.append( - "child", result.block); + PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), JavaRowFormat.LIST); + // Compute element width and kind for each collection field. + final List inputFields = + getInput().getRowType().getFieldList(); final List fieldCounts = new ArrayList<>(); final List inputTypes = new ArrayList<>(); - - Expression lambdaForStructWithSingleItem = null; - for (RelDataTypeField field : child.getRowType().getFieldList()) { - final RelDataType type = field.getType(); - if (type instanceof MapSqlType) { + for (int idx : collIndices) { + final RelDataType ft = inputFields.get(idx).getType(); + if (ft instanceof MapSqlType) { fieldCounts.add(2); inputTypes.add(FlatProductInputType.MAP); } else { - final RelDataType elementType = getComponentTypeOrThrow(type); - if (elementType.isStruct()) { - if (elementType.getFieldCount() == 1 && child.getRowType().getFieldList().size() == 1 - && !withOrdinality) { - // Solves CALCITE-4063: if we are processing a single field, which is a struct with a - // single item inside, and no ordinality; the result must be a scalar, hence use a - // special lambda that does not return lists, but the (single) items within those lists - lambdaForStructWithSingleItem = Expressions.call(BuiltInMethod.FLAT_LIST.method); - } else { - fieldCounts.add(elementType.getFieldCount()); - inputTypes.add(FlatProductInputType.LIST); - } + final RelDataType ct = getComponentTypeOrThrow(ft); + if (ct.isStruct() && expandStructFields) { + fieldCounts.add(ct.getFieldCount()); + inputTypes.add(FlatProductInputType.LIST); + } else if (ct.isStruct()) { + // A struct element kept whole occupies a single output column, + // like a scalar element, but its row value must be converted from + // the collection's internal list representation to Object[]. + fieldCounts.add(-1); + inputTypes.add(FlatProductInputType.STRUCT); } else { + // Scalar elements occupy a single output column, unconverted. fieldCounts.add(-1); inputTypes.add(FlatProductInputType.SCALAR); } } } - final Expression lambda = lambdaForStructWithSingleItem != null - ? lambdaForStructWithSingleItem - : Expressions.call(BuiltInMethod.FLAT_ZIP.method, - Expressions.constant(Ints.toArray(fieldCounts)), - Expressions.constant(withOrdinality), + final Expression childExpression = builder.append("child", childResult.block); + + // SqlFunctions.flatUncollect(passthroughIndices, collectionIndices, + // fieldCounts, inputTypes, withOrdinality, inputFieldCount, outer) + final Expression flatUncollectFn = + Expressions.call(BuiltInMethod.FLAT_UNCOLLECT.method, + Expressions.constant(passthroughIndices.toArray()), + Expressions.constant(collIndices.toArray()), + // Following converts from a List to int[] (not Integer[]). + Expressions.constant(fieldCounts.stream().mapToInt(i -> i).toArray()), Expressions.constant( - inputTypes.toArray(new FlatProductInputType[0]))); + inputTypes.toArray(new FlatProductInputType[0])), + Expressions.constant(withOrdinality), + Expressions.constant(inputFields.size()), + Expressions.constant(isOuter)); + builder.add( Expressions.return_(null, - Expressions.call(child_, + Expressions.call(childExpression, BuiltInMethod.SELECT_MANY.method, - lambda))); + flatUncollectFn))); + return implementor.result(physType, builder.toBlock()); } - } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java index 9079964897d..c4df71d26b3 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java @@ -49,6 +49,8 @@ protected EnumerableUncollectRule(Config config) { convert(input, input.getTraitSet().replace(EnumerableConvention.INSTANCE)); return EnumerableUncollect.create(traitSet, newInput, - uncollect.withOrdinality); + uncollect.withOrdinality, uncollect.getPassthroughFieldIndices(), + uncollect.getCollectionFieldIndices(), uncollect.isOuter, + uncollect.expandStructFields); } } diff --git a/core/src/main/java/org/apache/calcite/interpreter/UncollectNode.java b/core/src/main/java/org/apache/calcite/interpreter/UncollectNode.java index 01f4d578f46..047ee9433f5 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/UncollectNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/UncollectNode.java @@ -17,11 +17,20 @@ package org.apache.calcite.interpreter; import org.apache.calcite.rel.core.Uncollect; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.sql.type.SqlTypeName; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; +import static java.util.Objects.requireNonNull; + /** * Interpreter node that implements a * {@link org.apache.calcite.rel.core.Uncollect}. @@ -33,6 +42,17 @@ public UncollectNode(Compiler compiler, Uncollect uncollect) { } @Override public void run() throws InterruptedException { + final int inputFieldCount = rel.getInput().getRowType().getFieldCount(); + if (rel.getPassthroughFieldIndices().isEmpty() + && rel.getCollectionFieldIndices().cardinality() == inputFieldCount) { + runPlain(); + } else { + runGeneralized(); + } + } + + /** Unnests every input field; every input field must be a collection. */ + private void runPlain() throws InterruptedException { Row row = null; while ((row = source.receive()) != null) { for (Object value : row.getValues()) { @@ -67,4 +87,116 @@ public UncollectNode(Compiler compiler, Uncollect uncollect) { } } } + + /** + * Unnests the collection fields and prepends the passthrough fields, + * mirroring {@code SqlFunctions.flatUncollect}: multiple collections zip + * with {@code null} padding, a {@code null} collection counts as empty, + * and when all collections are empty {@code isOuter} decides between one + * {@code null}-padded row and dropping the input row. + */ + private void runGeneralized() throws InterruptedException { + final List inputFields = rel.getInput().getRowType().getFieldList(); + final List passthroughs = rel.getPassthroughFieldIndices().toList(); + final List collections = rel.getCollectionFieldIndices().toList(); + + // Output width of each collection: 2 for maps (key, value), the component + // field count for struct elements, otherwise 1. + final int[] widths = new int[collections.size()]; + int elementWidth = 0; + for (int c = 0; c < collections.size(); c++) { + final RelDataType type = inputFields.get(collections.get(c)).getType(); + if (type.getSqlTypeName() == SqlTypeName.MAP) { + widths[c] = 2; + } else { + final RelDataType component = + requireNonNull(type.getComponentType(), "componentType"); + widths[c] = component.isStruct() ? component.getFieldCount() : 1; + } + elementWidth += widths[c]; + } + final int outputWidth = + passthroughs.size() + elementWidth + (rel.withOrdinality ? 1 : 0); + + Row row = null; + while ((row = source.receive()) != null) { + final @Nullable Object[] values = row.getValues(); + final List> elements = new ArrayList<>(); + int maxSize = 0; + for (int index : collections) { + final Object value = values[index]; + final List list; + if (value == null) { + list = Collections.emptyList(); + } else if (value instanceof List) { + list = (List) value; + } else if (value instanceof Map) { + list = new ArrayList<>(((Map) value).entrySet()); + } else { + throw new UnsupportedOperationException( + String.format(Locale.ROOT, + "Invalid type: %s for unnest.", + value.getClass().getCanonicalName())); + } + elements.add(list); + maxSize = Math.max(maxSize, list.size()); + } + + if (maxSize == 0) { + if (rel.isOuter) { + final @Nullable Object[] out = new Object[outputWidth]; + int outCol = 0; + for (int p : passthroughs) { + out[outCol++] = values[p]; + } + sink.send(Row.of(out)); + } + continue; + } + + for (int i = 0; i < maxSize; i++) { + final @Nullable Object[] out = new Object[outputWidth]; + int outCol = 0; + for (int p : passthroughs) { + out[outCol++] = values[p]; + } + for (int c = 0; c < collections.size(); c++) { + final List list = elements.get(c); + final Object element = i < list.size() ? list.get(i) : null; + outCol = appendElement(out, outCol, widths[c], element); + } + if (rel.withOrdinality) { + out[outCol] = i + 1; + } + sink.send(Row.of(out)); + } + } + } + + /** Writes one collection element into {@code out} starting at {@code outCol}, + * expanding map entries and struct elements into their columns; a {@code null} + * element leaves its columns {@code null}. Returns the next free column. */ + private static int appendElement(@Nullable Object [] out, int outCol, int width, + @Nullable Object element) { + if (element instanceof Map.Entry) { + final Map.Entry entry = (Map.Entry) element; + out[outCol] = entry.getKey(); + out[outCol + 1] = entry.getValue(); + } else if (width == 1 || element == null) { + out[outCol] = element; + } else if (element instanceof Object[]) { + System.arraycopy((Object[]) element, 0, out, outCol, width); + } else if (element instanceof List) { + final List struct = (List) element; + for (int i = 0; i < width; i++) { + out[outCol + i] = struct.get(i); + } + } else { + throw new UnsupportedOperationException( + String.format(Locale.ROOT, + "Invalid element type: %s for unnest.", + element.getClass().getCanonicalName())); + } + return outCol + width; + } } diff --git a/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java b/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java index 2d4c3620a74..80ecf6a48b3 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java @@ -30,9 +30,11 @@ import org.apache.calcite.sql.SqlUnnestOperator; import org.apache.calcite.sql.type.MapSqlType; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.ImmutableBitSet; import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -50,9 +52,33 @@ *

Like its inverse operation {@link Collect}, Uncollect is generally * invoked in a nested loop, driven by * {@link org.apache.calcite.rel.logical.LogicalCorrelate} or similar. + * + *

By default, every input field is unnested and no input field is dropped + * or passed through unchanged. {@code Uncollect} also supports a more + * general mode, in which {@code passthroughFieldIndices} names input fields + * that are copied unchanged to the output and {@code collectionFieldIndices} + * names the (possibly smaller) set of input fields to unnest; any input + * field in neither set is dropped. + * + *

{@code isOuter} controls the + * behavior for empty or {@code NULL} collections: if {@code true} (LEFT JOIN + * semantics) the input row is preserved with {@code NULL}-padded element + * columns; if {@code false} (INNER semantics) the input row is dropped. + * + *

{@code expandStructFields} controls the shape of the element columns: + * if {@code true} a collection whose element type is a struct produces one + * output column per struct field; if {@code false} it produces a single + * column typed as the whole element. Maps always expand into a key and a + * value column, regardless of this flag. */ public class Uncollect extends SingleRel { public final boolean withOrdinality; + public final boolean isOuter; + + /** If true, a collection whose element type is a struct expands into one + * output column per struct field; if false, it produces a single column + * typed as the whole element. */ + public final boolean expandStructFields; // To alias the items in Uncollect list, // i.e., "UNNEST(a, b, c) as T(d, e, f)" @@ -63,6 +89,12 @@ public class Uncollect extends SingleRel { // same with element type of "a". private final List itemAliases; + /** 0-based indices of the input fields to pass through unchanged. Empty by default. */ + private final ImmutableBitSet passthroughFieldIndices; + /** 0-based indices of the input fields whose values are collections to unnest. + * All input fields by default. */ + private final ImmutableBitSet collectionFieldIndices; + //~ Constructors ----------------------------------------------------------- @Deprecated // to be removed before 2.0 @@ -73,13 +105,53 @@ public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, /** Creates an Uncollect. * - *

Use {@link #create} unless you know what you're doing. */ - @SuppressWarnings("method.invocation.invalid") + *

Use {@link #create} unless you know what you're doing. + * + *

Item aliases historically implied that struct elements are not + * expanded (Presto dialect), so this constructor derives + * {@code expandStructFields} from their absence. */ public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, boolean withOrdinality, List itemAliases) { + this(cluster, traitSet, input, withOrdinality, itemAliases, + ImmutableBitSet.of(), ImmutableBitSet.range(input.getRowType().getFieldCount()), + false, itemAliases.isEmpty()); + } + + /** Creates an Uncollect, with explicit control over which input fields are + * passed through unchanged and which are unnested. + * + * @param input Input relational expression + * @param withOrdinality Whether output should contain an ORDINALITY column + * @param itemAliases Aliases for the operand items; only meaningful when + * {@code passthroughFieldIndices} is empty and + * {@code collectionFieldIndices} covers every input field + * @param passthroughFieldIndices 0-based indices of the input fields to pass through + * unchanged + * @param collectionFieldIndices 0-based indices of the input fields whose values are + * collections to unnest + * @param isOuter If true, preserves input rows with null/empty + * collections (LEFT JOIN); if false, drops them (INNER) + * @param expandStructFields If true, a collection whose element type is a struct + * produces one output column per struct field; if false, + * a single column typed as the whole element + */ + @SuppressWarnings("method.invocation.invalid") + public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, + boolean withOrdinality, List itemAliases, + ImmutableBitSet passthroughFieldIndices, ImmutableBitSet collectionFieldIndices, + boolean isOuter, boolean expandStructFields) { super(cluster, traitSet, input); this.withOrdinality = withOrdinality; this.itemAliases = ImmutableList.copyOf(itemAliases); + this.passthroughFieldIndices = + requireNonNull(passthroughFieldIndices, "passthroughFieldIndices"); + this.collectionFieldIndices = + requireNonNull(collectionFieldIndices, "collectionFieldIndices"); + if (collectionFieldIndices.isEmpty()) { + throw new IllegalArgumentException("collectionFieldIndices must not be empty"); + } + this.isOuter = isOuter; + this.expandStructFields = expandStructFields; requireNonNull(deriveRowType(), "invalid child rowType"); } @@ -88,7 +160,31 @@ public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, */ public Uncollect(RelInput input) { this(input.getCluster(), input.getTraitSet(), input.getInput(), - input.getBoolean("withOrdinality", false), Collections.emptyList()); + input.getBoolean("withOrdinality", false), Collections.emptyList(), + fieldIndices(input, "passthrough", ImmutableBitSet.of()), + fieldIndices(input, "collectionFields", + ImmutableBitSet.range(input.getInput().getRowType().getFieldCount())), + input.getBoolean("isOuter", false), + input.getBoolean("expandStructFields", true)); + } + + /** Maps the input-field names serialized under {@code tag} (see + * {@link #explainTerms}) back to field indices; returns {@code defaultSet} + * when the attribute is absent. */ + private static ImmutableBitSet fieldIndices(RelInput input, String tag, + ImmutableBitSet defaultSet) { + final List names = input.getStringList(tag); + if (names == null) { + return defaultSet; + } + final RelDataType inputRowType = input.getInput().getRowType(); + final ImmutableBitSet.Builder builder = ImmutableBitSet.builder(); + for (String name : names) { + builder.set( + requireNonNull(inputRowType.getField(name, true, false), + () -> "field " + name).getIndex()); + } + return builder.build(); } /** @@ -111,6 +207,37 @@ public static Uncollect create( return new Uncollect(cluster, traitSet, input, withOrdinality, itemAliases); } + /** + * Creates an Uncollect that unnests the collection-typed fields at + * {@code collectionFieldIndices}, passing through the fields at + * {@code passthroughFieldIndices} unchanged and dropping all others. + * + * @param traitSet Trait set + * @param input Input relational expression + * @param withOrdinality Whether output should contain an ORDINALITY column + * @param passthroughFieldIndices 0-based indices of the input fields to pass through + * unchanged + * @param collectionFieldIndices 0-based indices of the input fields whose values are + * collections to unnest + * @param isOuter If true, preserves input rows with null/empty + * collections (LEFT JOIN); if false, drops them (INNER) + * @param expandStructFields If true, a collection whose element type is a struct + * produces one output column per struct field; if false, + * a single column typed as the whole element + */ + public static Uncollect create( + RelTraitSet traitSet, + RelNode input, + boolean withOrdinality, + ImmutableBitSet passthroughFieldIndices, + ImmutableBitSet collectionFieldIndices, + boolean isOuter, + boolean expandStructFields) { + final RelOptCluster cluster = input.getCluster(); + return new Uncollect(cluster, traitSet, input, withOrdinality, Collections.emptyList(), + passthroughFieldIndices, collectionFieldIndices, isOuter, expandStructFields); + } + //~ Methods ---------------------------------------------------------------- @Override public RelNode accept(RelShuttle shuttle) { @@ -118,8 +245,23 @@ public static Uncollect create( } @Override public RelWriter explainTerms(RelWriter pw) { + final List inputFields = getInput().getRowType().getFieldList(); + final List passthroughNames = new ArrayList<>(); + for (int i : passthroughFieldIndices) { + passthroughNames.add(inputFields.get(i).getName()); + } + final boolean isPassthroughMode = !passthroughFieldIndices.isEmpty() + || collectionFieldIndices.cardinality() != inputFields.size(); + final List collFieldNames = new ArrayList<>(); + for (int i : collectionFieldIndices) { + collFieldNames.add(inputFields.get(i).getName()); + } return super.explainTerms(pw) - .itemIf("withOrdinality", withOrdinality, withOrdinality); + .itemIf("passthrough", passthroughNames, !passthroughNames.isEmpty()) + .itemIf("collectionFields", collFieldNames, isPassthroughMode) + .itemIf("withOrdinality", withOrdinality, withOrdinality) + .itemIf("isOuter", isOuter, isOuter) + .itemIf("expandStructFields", expandStructFields, !expandStructFields); } @Override public final RelNode copy(RelTraitSet traitSet, @@ -129,34 +271,55 @@ public static Uncollect create( public RelNode copy(RelTraitSet traitSet, RelNode input) { assert traitSet.containsIfApplicable(Convention.NONE); - return new Uncollect(getCluster(), traitSet, input, withOrdinality, itemAliases); - } - - @Override protected RelDataType deriveRowType() { - return deriveUncollectRowType(input, withOrdinality, itemAliases); + return new Uncollect(getCluster(), traitSet, input, withOrdinality, itemAliases, + passthroughFieldIndices, collectionFieldIndices, isOuter, expandStructFields); } /** * Returns the row type returned by applying the 'UNNEST' operation to a - * relational expression. + * relational expression whose every field is a collection to unnest. * - *

Each column in the relational expression must be a multiset of - * structs or an array. The return type is the combination of expanding - * element types from each column, plus an ORDINALITY column if {@code - * withOrdinality}. If {@code itemAliases} is not empty, the element types - * would not expand, each column element outputs as a whole (the return - * type has same column types as input type). + * @deprecated Construct an {@link Uncollect} and call + * {@link #getRowType()} instead. */ + @Deprecated // to be removed before 2.0 public static RelDataType deriveUncollectRowType(RelNode rel, boolean withOrdinality, List itemAliases) { - RelDataType inputType = rel.getRowType(); + return new Uncollect(rel.getCluster(), rel.getTraitSet(), rel, + withOrdinality, itemAliases).getRowType(); + } + + /** + * Returns the row type of the 'UNNEST' operation. + * + *

Pass-through fields (named by {@code passthroughFieldIndices}) are + * copied to the output as-is, followed by the expansion of the + * collection-typed fields (named by {@code collectionFieldIndices}): each + * column in {@code collectionFieldIndices} must be a multiset of structs + * or an array. The return type is the combination of expanding element + * types from each such column, plus an ORDINALITY column if {@code + * withOrdinality}. Any input field in neither set is dropped. + * + *

{@code expandStructFields} controls the expansion of struct element + * types: if {@code true}, one output column per struct field; if {@code + * false}, a single column typed as the whole element. Maps always expand + * into a key and a value column. {@code itemAliases}, when not empty, + * names the non-expanded element columns. + * + *

Element columns are nullable when {@code isOuter} is {@code true} + * (empty/null collections still emit a null-padded row) or when there is + * more than one collection field (shorter collections are padded with + * {@code NULL} to align with SQL {@code UNNEST(a, b)} zip semantics). + */ + @Override protected RelDataType deriveRowType() { + RelDataType inputType = input.getRowType(); assert inputType.isStruct() : inputType + " is not a struct"; boolean requireAlias = !itemAliases.isEmpty(); assert !requireAlias || itemAliases.size() == inputType.getFieldCount(); final List fields = inputType.getFieldList(); - final RelDataTypeFactory typeFactory = rel.getCluster().getTypeFactory(); + final RelDataTypeFactory typeFactory = getCluster().getTypeFactory(); final RelDataTypeFactory.Builder builder = typeFactory.builder(); if (fields.size() == 1 @@ -169,19 +332,31 @@ public static RelDataType deriveUncollectRowType(RelNode rel, .build(); } - // With multiple collections, zip semantics pads shorter collections with - // NULL, so all output columns from a multi-collection UNNEST are nullable. - final boolean padNullable = fields.size() > 1; + // Element columns are nullable when: + // - isOuter=true: empty/null collections emit a null-padded row, or + // - multiple collections: zip pads shorter collections with NULL. + final boolean elemNullable = isOuter || collectionFieldIndices.cardinality() > 1; + + // Pass-through fields first (in input-index order). + for (int i = 0; i < fields.size(); i++) { + if (passthroughFieldIndices.get(i)) { + builder.add(fields.get(i)); + } + } + // Expanded collection fields second (in input-index order). for (int i = 0; i < fields.size(); i++) { + if (!collectionFieldIndices.get(i)) { + continue; + } RelDataTypeField field = fields.get(i); if (field.getType() instanceof MapSqlType) { // This code is similar to SqlUnnestOperator::inferReturnType. MapSqlType mapType = (MapSqlType) field.getType(); - RelDataType keyType = padNullable + RelDataType keyType = elemNullable ? typeFactory.enforceTypeWithNullability(mapType.getKeyType(), true) : mapType.getKeyType(); - RelDataType valueType = padNullable + RelDataType valueType = elemNullable ? typeFactory.enforceTypeWithNullability(mapType.getValueType(), true) : mapType.getValueType(); builder.add(SqlUnnestOperator.MAP_KEY_COLUMN_NAME, keyType); @@ -191,13 +366,8 @@ public static RelDataType deriveUncollectRowType(RelNode rel, if (null == componentType) { throw RESOURCE.unnestArgument().ex(); } - boolean isNullable = componentType.isNullable() || padNullable; - if (requireAlias) { - RelDataType colType = padNullable - ? typeFactory.enforceTypeWithNullability(componentType, true) - : componentType; - builder.add(itemAliases.get(i), colType); - } else if (componentType.isStruct()) { + boolean isNullable = componentType.isNullable() || elemNullable; + if (expandStructFields && componentType.isStruct()) { for (RelDataTypeField fieldInfo : componentType.getFieldList()) { RelDataType fieldType = fieldInfo.getType(); if (isNullable) { @@ -206,18 +376,25 @@ public static RelDataType deriveUncollectRowType(RelNode rel, builder.add(fieldInfo.getName(), fieldType); } } else { - // Element type is not a record, use the field name of the element directly - RelDataType colType = padNullable - ? typeFactory.enforceTypeWithNullability(componentType, true) + // A single column typed as the whole element, named by the item + // alias when present, otherwise by the collection field's name. + RelDataType elementType = componentType.isStruct() + ? typeFactory.builder().kind(componentType.getStructKind()) + .addAll(componentType.getFieldList()).build() : componentType; - builder.add(field.getName(), colType); + RelDataType colType = elemNullable + ? typeFactory.enforceTypeWithNullability(elementType, true) + : elementType; + builder.add(requireAlias ? itemAliases.get(i) : field.getName(), colType); } } } if (withOrdinality) { + final RelDataType ordType = typeFactory.createSqlType(SqlTypeName.INTEGER); + // For outer JOINS even the ordinality is nullable builder.add(SqlUnnestOperator.ORDINALITY_COLUMN_NAME, - SqlTypeName.INTEGER); + isOuter ? typeFactory.createTypeWithNullability(ordType, true) : ordType); } return builder.build(); } @@ -226,4 +403,14 @@ public static RelDataType deriveUncollectRowType(RelNode rel, public List getItemAliases() { return itemAliases; } + + /** Returns the 0-based indices of the input fields passed through unchanged. */ + public ImmutableBitSet getPassthroughFieldIndices() { + return passthroughFieldIndices; + } + + /** Returns the 0-based indices of the input fields to be unnested. */ + public ImmutableBitSet getCollectionFieldIndices() { + return collectionFieldIndices; + } } diff --git a/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java b/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java index d7cee2dae7a..5d6e16e604d 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java @@ -41,8 +41,6 @@ import org.apache.calcite.rel.core.Window; import org.apache.calcite.tools.RelBuilder; -import java.util.Collections; - /** * Shuttle to convert any rel plan to a plan with all logical nodes. */ @@ -191,7 +189,9 @@ public ToLogicalConverter(RelBuilder relBuilder) { final Uncollect uncollect = (Uncollect) relNode; final RelNode input = visit(uncollect.getInput()); return Uncollect.create(input.getTraitSet(), input, - uncollect.withOrdinality, Collections.emptyList()); + uncollect.withOrdinality, uncollect.getPassthroughFieldIndices(), + uncollect.getCollectionFieldIndices(), uncollect.isOuter, + uncollect.expandStructFields); } throw new AssertionError("Need to implement logical converter for " diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java index ed509b6d60b..e586cf383be 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java @@ -65,7 +65,6 @@ import java.util.AbstractList; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -257,7 +256,8 @@ public static RelNode fromMutable(MutableRel node, RelBuilder relBuilder) { final MutableUncollect uncollect = (MutableUncollect) node; final RelNode child = fromMutable(uncollect.getInput(), relBuilder); return Uncollect.create(child.getTraitSet(), child, uncollect.withOrdinality, - Collections.emptyList()); + uncollect.passthroughFieldIndices, uncollect.collectionFieldIndices, + uncollect.isOuter, uncollect.expandStructFields); } case WINDOW: { final MutableWindow window = (MutableWindow) node; @@ -378,7 +378,10 @@ public static MutableRel toMutable(RelNode rel) { if (rel instanceof Uncollect) { final Uncollect uncollect = (Uncollect) rel; final MutableRel input = toMutable(uncollect.getInput()); - return MutableUncollect.of(uncollect.getRowType(), input, uncollect.withOrdinality); + return MutableUncollect.of(uncollect.getRowType(), input, + uncollect.withOrdinality, uncollect.getPassthroughFieldIndices(), + uncollect.getCollectionFieldIndices(), uncollect.isOuter, + uncollect.expandStructFields); } if (rel instanceof Window) { final Window window = (Window) rel; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java index 594d109b5d6..af9998a164d 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java @@ -17,6 +17,7 @@ package org.apache.calcite.rel.mutable; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.util.ImmutableBitSet; import org.checkerframework.checker.nullness.qual.Nullable; @@ -25,15 +26,26 @@ /** Mutable equivalent of {@link org.apache.calcite.rel.core.Uncollect}. */ public class MutableUncollect extends MutableSingleRel { public final boolean withOrdinality; + public final ImmutableBitSet passthroughFieldIndices; + public final ImmutableBitSet collectionFieldIndices; + public final boolean isOuter; + public final boolean expandStructFields; private MutableUncollect(RelDataType rowType, - MutableRel input, boolean withOrdinality) { + MutableRel input, boolean withOrdinality, + ImmutableBitSet passthroughFieldIndices, + ImmutableBitSet collectionFieldIndices, boolean isOuter, + boolean expandStructFields) { super(MutableRelType.UNCOLLECT, rowType, input); this.withOrdinality = withOrdinality; + this.passthroughFieldIndices = passthroughFieldIndices; + this.collectionFieldIndices = collectionFieldIndices; + this.isOuter = isOuter; + this.expandStructFields = expandStructFields; } /** - * Creates a MutableUncollect. + * Creates a MutableUncollect that unnests every input field. * * @param rowType Row type * @param input Input relational expression @@ -42,26 +54,69 @@ private MutableUncollect(RelDataType rowType, */ public static MutableUncollect of(RelDataType rowType, MutableRel input, boolean withOrdinality) { - return new MutableUncollect(rowType, input, withOrdinality); + return of(rowType, input, withOrdinality, ImmutableBitSet.of(), + ImmutableBitSet.range(input.rowType.getFieldCount()), false, true); + } + + /** + * Creates a MutableUncollect. + * + * @param rowType Row type + * @param input Input relational expression + * @param withOrdinality Whether the output contains an extra + * {@code ORDINALITY} column + * @param passthroughFieldIndices 0-based indices of the input fields to pass + * through unchanged + * @param collectionFieldIndices 0-based indices of the input fields whose + * values are collections to unnest + * @param isOuter If true, preserves input rows with + * null/empty collections (LEFT JOIN); if + * false, drops them (INNER) + * @param expandStructFields If true, a collection whose element type + * is a struct produces one output column per + * struct field; if false, a single column + * typed as the whole element + */ + public static MutableUncollect of(RelDataType rowType, + MutableRel input, boolean withOrdinality, + ImmutableBitSet passthroughFieldIndices, + ImmutableBitSet collectionFieldIndices, boolean isOuter, + boolean expandStructFields) { + return new MutableUncollect(rowType, input, withOrdinality, + passthroughFieldIndices, collectionFieldIndices, isOuter, + expandStructFields); } @Override public boolean equals(@Nullable Object obj) { return obj == this || obj instanceof MutableUncollect && withOrdinality == ((MutableUncollect) obj).withOrdinality + && passthroughFieldIndices.equals( + ((MutableUncollect) obj).passthroughFieldIndices) + && collectionFieldIndices.equals( + ((MutableUncollect) obj).collectionFieldIndices) + && isOuter == ((MutableUncollect) obj).isOuter + && expandStructFields == ((MutableUncollect) obj).expandStructFields && input.equals(((MutableUncollect) obj).input); } @Override public int hashCode() { - return Objects.hash(input, withOrdinality); + return Objects.hash(input, withOrdinality, passthroughFieldIndices, + collectionFieldIndices, isOuter, expandStructFields); } @Override public StringBuilder digest(StringBuilder buf) { - return buf.append("Uncollect(withOrdinality: ") - .append(withOrdinality).append(")"); + return buf.append("Uncollect(withOrdinality: ").append(withOrdinality) + .append(", passthrough: ").append(passthroughFieldIndices) + .append(", collectionFields: ").append(collectionFieldIndices) + .append(", isOuter: ").append(isOuter) + .append(", expandStructFields: ").append(expandStructFields) + .append(")"); } @Override public MutableRel clone() { - return MutableUncollect.of(rowType, input.clone(), withOrdinality); + return MutableUncollect.of(rowType, input.clone(), withOrdinality, + passthroughFieldIndices, collectionFieldIndices, isOuter, + expandStructFields); } } diff --git a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java index 0404fa7bc51..150c46fce9a 100644 --- a/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/rel2sql/RelToSqlConverter.java @@ -19,6 +19,7 @@ import org.apache.calcite.adapter.jdbc.JdbcTable; import org.apache.calcite.linq4j.Ord; import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptSamplingParameters; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.rel.RelCollation; @@ -48,8 +49,10 @@ import org.apache.calcite.rel.core.Values; import org.apache.calcite.rel.core.Window; import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.logical.LogicalCorrelate; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.logical.LogicalSort; +import org.apache.calcite.rel.logical.LogicalValues; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; @@ -536,8 +539,17 @@ public Result visit(Correlate e) { rightLateralAs = SqlStdOperatorTable.AS.createCall(POS, rightLateral, id); } - final SqlNode join = - new SqlJoin(POS, + // A LEFT correlate preserves left rows with an empty right side, which a + // comma join would drop; emit LEFT JOIN LATERAL ... ON TRUE for it. + final SqlNode join = e.getJoinType() == JoinRelType.LEFT + ? new SqlJoin(POS, + leftResult.asFrom(), + SqlLiteral.createBoolean(false, POS), + JoinType.LEFT.symbol(POS), + rightLateralAs, + JoinConditionType.ON.symbol(POS), + SqlLiteral.createBoolean(true, POS)) + : new SqlJoin(POS, leftResult.asFrom(), SqlLiteral.createBoolean(false, POS), JoinType.COMMA.symbol(POS), @@ -1505,6 +1517,15 @@ private static SqlCall as(SqlNode e, String alias) { } public Result visit(Uncollect e) { + final int inputFieldCount = e.getInput().getRowType().getFieldCount(); + if (!e.getPassthroughFieldIndices().isEmpty() + || e.getCollectionFieldIndices().cardinality() != inputFieldCount + || e.isOuter) { + // A generalized Uncollect needs LATERAL join syntax: rewrite it to a + // Correlate over a plain Uncollect (the inverse of + // CoreRules#CORRELATE_UNCOLLECT_MERGE) and dispatch that instead. + return dispatch(generalizedUncollectToCorrelate(e)); + } final Result x = visitInput(e, 0); final SqlOperator operator = e.withOrdinality ? SqlStdOperatorTable.UNNEST_WITH_ORDINALITY : SqlStdOperatorTable.UNNEST; @@ -1517,6 +1538,52 @@ public Result visit(Uncollect e) { return result(asNode, ImmutableList.of(Clause.FROM), e, null); } + /** + * Rewrites a generalized {@link Uncollect} (passthrough fields, a subset of + * collection fields, or outer semantics) into the equivalent + * {@code Project(Correlate(input, Uncollect(Project($cor.c...))))}. + */ + private static RelNode generalizedUncollectToCorrelate(Uncollect e) { + final RelOptCluster cluster = e.getCluster(); + final RexBuilder rexBuilder = cluster.getRexBuilder(); + final RelNode input = e.getInput(); + final int inputFieldCount = input.getRowType().getFieldCount(); + + final CorrelationId corrId = cluster.createCorrel(); + final RexNode corr = rexBuilder.makeCorrel(input.getRowType(), corrId); + final List collectionExprs = new ArrayList<>(); + final List collectionNames = new ArrayList<>(); + for (int i : e.getCollectionFieldIndices()) { + collectionExprs.add(rexBuilder.makeFieldAccess(corr, i)); + collectionNames.add(input.getRowType().getFieldList().get(i).getName()); + } + final RelNode innerProject = + LogicalProject.create(LogicalValues.createOneRow(cluster), + ImmutableList.of(), collectionExprs, collectionNames, + ImmutableSet.of()); + final Uncollect plainUncollect = + Uncollect.create(e.getTraitSet(), innerProject, e.withOrdinality, + Collections.emptyList()); + final RelNode correlate = + LogicalCorrelate.create(input, plainUncollect, ImmutableList.of(), + corrId, e.getCollectionFieldIndices(), + e.isOuter ? JoinRelType.LEFT : JoinRelType.INNER); + + // Keep only the passthrough fields and the unnested columns, in the + // generalized Uncollect's output order. + final List projects = new ArrayList<>(); + for (int i : e.getPassthroughFieldIndices()) { + projects.add(rexBuilder.makeInputRef(correlate, i)); + } + final int elementFieldCount = + e.getRowType().getFieldCount() - e.getPassthroughFieldIndices().cardinality(); + for (int k = 0; k < elementFieldCount; k++) { + projects.add(rexBuilder.makeInputRef(correlate, inputFieldCount + k)); + } + return LogicalProject.create(correlate, ImmutableList.of(), projects, + e.getRowType().getFieldNames(), ImmutableSet.of()); + } + public Result visit(TableFunctionScan e) { final List inputSqlNodes = new ArrayList<>(); final List fieldNodes = new ArrayList<>(); diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java index 104e34bfaeb..da211701f50 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java @@ -990,11 +990,23 @@ private CoreRules() {} public static final AggregateGroupingSetsToUnionRule AGGREGATE_GROUPING_SETS_TO_UNION = AggregateGroupingSetsToUnionRule.Config.DEFAULT.toRule(); + /** Rule that removes {@code LITERAL_AGG} aggregate calls by replacing them * with literal expressions in a {@link Project}. */ public static final AggregateRemoveLiteralAggRule AGGREGATE_REMOVE_LITERAL_AGG = AggregateRemoveLiteralAggRule.Config.DEFAULT.toRule(); + /** Rule that converts a {@link Correlate} over an {@link Uncollect} into a + * single, generalized {@link Uncollect} that preserves the correlate's full + * output. */ + public static final RelOptRule CORRELATE_UNCOLLECT_MERGE = + CorrelateUncollectMergeRule.Config.DEFAULT.toRule(); + + /** Rule that narrows an {@link Uncollect} to the + * columns referenced by the {@link Project} above it. */ + public static final RelOptRule PROJECT_UNCOLLECT_MERGE = + ProjectUncollectMergeRule.Config.DEFAULT.toRule(); + /** Rule that converts a {@link Correlate} after an {@link Uncollect} into a simple * Uncollect, if possible. */ public static final RelOptRule UNNEST_DECORRELATE = diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CorrelateUncollectMergeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/CorrelateUncollectMergeRule.java new file mode 100644 index 00000000000..470d1a9e64f --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/CorrelateUncollectMergeRule.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Correlate; +import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Uncollect; +import org.apache.calcite.rel.logical.LogicalValues; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexFieldAccess; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.type.MapSqlType; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.ImmutableBitSet; + +import org.immutables.value.Value; + +import java.util.ArrayList; +import java.util.List; + +/** + * Rule that converts a {@link Correlate} over an {@link Uncollect} into a + * single, generalized {@link Uncollect} that reads directly from the + * correlate's left input, when possible. + * + *

Input plan: + * 

+ * LogicalCorrelate(cor=[$cor0], joinType=[inner|left])
+ *   left (any RelNode)
+ *   Uncollect
+ *     LogicalProject($cor0.f_i, $cor0.f_j, ...)  (innerProject)
+ *       LogicalValues
+ *
+ * + *

Converted to: + * 

+ * LogicalProject (restores the correlate's columns)
+ *   Uncollect(passthrough=[all left fields], collectionFields=[f_i, f_j, ...])
+ *     left
+ *
+ * + *

Every left field becomes a passthrough field, so the merged + * {@code Uncollect} preserves the correlate's full output. The merged + * {@code Uncollect} expands collections in ascending input-index order; the + * projection on top restores the order in which {@code innerProject} lists + * them (collections may repeat). {@link ProjectMergeRule} and + * {@link ProjectUncollectMergeRule} can then prune passthrough fields that no + * consumer references. + * + *

The rule fires only when every expression in {@code innerProject} is a + * direct field access on the correlation variable (no nested struct paths). + * + *

This rule is a generalization of {@link UnnestDecorrelateRule}. + */ +@Value.Enclosing +public class CorrelateUncollectMergeRule + extends RelRule + implements TransformationRule { + + protected CorrelateUncollectMergeRule(Config config) { + super(config); + } + + /** Returns the number of output columns produced by unnesting a collection + * of {@code type}, which must be a collection (validated by the matched + * {@link Uncollect} when it derived its row type). + * + *

Must mirror the row-type derivation of {@link Uncollect}: a map expands to + * a key and a value column; a struct element expands to one column per + * struct field when {@code expandStructFields}, else to a single column. */ + private static int expansionWidth(RelDataType type, boolean expandStructFields) { + if (type instanceof MapSqlType) { + return 2; + } + final RelDataType componentType = type.getComponentType(); + assert componentType != null : type + " is not a collection"; + if (expandStructFields && componentType.isStruct()) { + return componentType.getFieldCount(); + } + return 1; + } + + @Override public void onMatch(RelOptRuleCall call) { + final Correlate correlate = call.rel(0); + final RelNode left = call.rel(1); + final Uncollect uncollect = call.rel(2); + final Project innerProject = call.rel(3); + + // Only INNER and LEFT join types map to Uncollect's passthrough/outer semantics. + // Other types should have been rejected by the validator. + final JoinRelType joinType = correlate.getJoinType(); + if (joinType != JoinRelType.INNER && joinType != JoinRelType.LEFT) { + return; + } + + final CorrelationId corId = correlate.getCorrelationId(); + final int leftCount = left.getRowType().getFieldCount(); + + // Check that each expression in innerProject is a direct $cor0.field access; + // bail out if not + final List collectionFieldIndices = new ArrayList<>(); + for (RexNode expr : innerProject.getProjects()) { + if (!(expr instanceof RexFieldAccess)) { + return; + } + final RexFieldAccess fa = (RexFieldAccess) expr; + if (!(fa.getReferenceExpr() instanceof RexCorrelVariable)) { + return; + } + if (((RexCorrelVariable) fa.getReferenceExpr()).id != corId) { + return; + } + collectionFieldIndices.add(fa.getField().getIndex()); + } + + if (collectionFieldIndices.isEmpty()) { + return; + } + + final ImmutableBitSet collBitSet = ImmutableBitSet.of(collectionFieldIndices); + final List leftFields = left.getRowType().getFieldList(); + final boolean expandStructFields = uncollect.expandStructFields; + + // Start offset of each collection's expansion block in the merged + // Uncollect's output, which expands collections in ascending input-index + // order after the passthrough fields. + final int[] blockStart = new int[leftCount]; + int nextStart = leftCount; + for (int c : collBitSet) { + final RelDataType collType = leftFields.get(c).getType(); + if (collType.getSqlTypeName() == SqlTypeName.ANY) { + // Legal when the collection type is unknown (schema-less inputs), but + // the merged Uncollect cannot represent it: its row-type derivation + // handles ANY only for a single-field input. + return; + } + blockStart[c] = nextStart; + nextStart += expansionWidth(collType, expandStructFields); + } + final int mergedOrdinality = nextStart; + + // Merged output column feeding each correlate output column. innerProject + // may list the collections in any order, and may repeat them; the + // projection built below restores the correlate's column order. + final List sources = new ArrayList<>(); + for (int i = 0; i < leftCount; i++) { + sources.add(i); + } + for (int c : collectionFieldIndices) { + final int width = expansionWidth(leftFields.get(c).getType(), expandStructFields); + for (int j = 0; j < width; j++) { + sources.add(blockStart[c] + j); + } + } + if (uncollect.withOrdinality) { + sources.add(mergedOrdinality); + } + + final List corFields = correlate.getRowType().getFieldList(); + if (sources.size() != corFields.size()) { + return; + } + + // An outer Uncollect emits a NULL-padded row for an empty collection even + // under an INNER correlate, so the merged Uncollect must stay outer. + final boolean isOuter = uncollect.isOuter || joinType == JoinRelType.LEFT; + final RelBuilder builder = call.builder(); + builder.push(left) + .uncollect(ImmutableBitSet.range(leftCount), collBitSet, + uncollect.withOrdinality, isOuter, expandStructFields); + + final RexBuilder rexBuilder = correlate.getCluster().getRexBuilder(); + final List mergedFields = builder.peek().getRowType().getFieldList(); + final List exprs = new ArrayList<>(); + for (int i = 0; i < sources.size(); i++) { + final int source = sources.get(i); + final RelDataType mergedType = mergedFields.get(source).getType(); + final RelDataType corType = corFields.get(i).getType(); + RexNode ref = rexBuilder.makeInputRef(mergedType, source); + if (!mergedType.equals(corType)) { + // The replacement must reproduce the correlate's column types. The + // merged Uncollect may derive a column that differs only by being + // less nullable (a repeated collection collapses to one and no + // longer zips); a no-op cast widens it. Any other mismatch means a + // plan this rule does not understand. + if (!corType.isNullable() + || !SqlTypeUtil.equalSansNullability( + correlate.getCluster().getTypeFactory(), mergedType, corType)) { + return; + } + ref = rexBuilder.makeCast(corType, ref); + } + exprs.add(ref); + } + + // Restore the correlate's column order and field names; rename() skips + // the projection entirely when it would be a no-op. + if (RexUtil.isIdentity(exprs, builder.peek().getRowType())) { + builder.rename(correlate.getRowType().getFieldNames()); + } else { + builder.project(exprs, correlate.getRowType().getFieldNames()); + } + call.transformTo(builder.build()); + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + Config DEFAULT = ImmutableCorrelateUncollectMergeRule.Config.of() + .withOperandSupplier(b0 -> b0.operand(Correlate.class) + .inputs( + b1 -> b1.operand(RelNode.class).anyInputs(), + b2 -> b2.operand(Uncollect.class) + .oneInput(b3 -> b3.operand(Project.class) + .oneInput(b4 -> b4.operand(LogicalValues.class) + .anyInputs())))) + .as(Config.class); + + @Override default CorrelateUncollectMergeRule toRule() { + return new CorrelateUncollectMergeRule(this); + } + } +} diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectUncollectMergeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectUncollectMergeRule.java new file mode 100644 index 00000000000..ae4134fe34c --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectUncollectMergeRule.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Uncollect; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.Util; + +import org.immutables.value.Value; + +import java.util.Arrays; +import java.util.List; + +/** + * Rule that narrows an {@link Uncollect} to the columns actually used by the + * {@link Project} above it. + * + *

Passthrough fields whose output column the project does not reference + * are removed from the {@code Uncollect}, and an unreferenced + * {@code ORDINALITY} column is dropped. The project's input references are + * remapped to the narrowed output. + * + *

Collection fields are never removed: with multiple collections, + * {@code UNNEST(a, b)} emits {@code max(|a|, |b|)} rows (zip semantics), so + * dropping a collection could change the row count. Even when all collection + * fields are dropped, the collection cardinalities are important, so + * they cannot be dropped from the UNCOLLECT. + */ +@Value.Enclosing +public class ProjectUncollectMergeRule + extends RelRule + implements TransformationRule { + + protected ProjectUncollectMergeRule(Config config) { + super(config); + } + + @Override public void onMatch(RelOptRuleCall call) { + final Project project = call.rel(0); + final Uncollect uncollect = call.rel(1); + + final ImmutableBitSet passthrough = uncollect.getPassthroughFieldIndices(); + final int passthroughCount = passthrough.cardinality(); + final int outputCount = uncollect.getRowType().getFieldCount(); + + final ImmutableBitSet refs = + RelOptUtil.InputFinder.bits(project.getProjects(), null); + + // Keep the passthrough fields referenced by the project. + // Passthrough fields occupy output positions 0 to passthroughCount-1, in + // ascending input-index order. + final ImmutableBitSet.Builder keptBuilder = ImmutableBitSet.builder(); + int outPos = 0; + for (int field : passthrough) { + if (refs.get(outPos)) { + keptBuilder.set(field); + } + outPos++; + } + final ImmutableBitSet kept = keptBuilder.build(); + final boolean keepOrdinality = + uncollect.withOrdinality && refs.get(outputCount - 1); + + if (kept.cardinality() == passthroughCount + && keepOrdinality == uncollect.withOrdinality) { + // Nothing to prune + return; + } + + final Uncollect newUncollect = + new Uncollect(uncollect.getCluster(), uncollect.getTraitSet(), + uncollect.getInput(), keepOrdinality, uncollect.getItemAliases(), + kept, uncollect.getCollectionFieldIndices(), uncollect.isOuter, + uncollect.expandStructFields); + + // Map old output indices to new ones; dropped columns map to -1 and are + // never referenced. + final int keptCount = kept.cardinality(); + final int[] map = new int[outputCount]; + Arrays.fill(map, -1); + int oldPos = 0; + int newPos = 0; + for (int field : passthrough) { + if (kept.get(field)) { + map[oldPos] = newPos++; + } + oldPos++; + } + // Element columns (and the ordinality column, if kept) shift down uniformly. + for (int i = passthroughCount; i < outputCount; i++) { + map[i] = keptCount + i - passthroughCount; + } + if (uncollect.withOrdinality && !keepOrdinality) { + map[outputCount - 1] = -1; + } + + final RexBuilder rexBuilder = project.getCluster().getRexBuilder(); + final List newFields = newUncollect.getRowType().getFieldList(); + final RexShuttle remapper = new RexShuttle() { + @Override public RexNode visitInputRef(RexInputRef inputRef) { + final int newIdx = map[inputRef.getIndex()]; + return rexBuilder.makeInputRef(newFields.get(newIdx).getType(), newIdx); + } + }; + + final RelBuilder builder = call.builder(); + // preserve the field names. + builder.push(newUncollect) + .project(Util.transform(project.getProjects(), e -> e.accept(remapper)), + project.getRowType().getFieldNames(), true); + call.transformTo(builder.build()); + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + Config DEFAULT = ImmutableProjectUncollectMergeRule.Config.of() + .withOperandSupplier(b0 -> b0.operand(Project.class) + .oneInput(b1 -> b1.operand(Uncollect.class).anyInputs())) + .as(Config.class); + + @Override default ProjectUncollectMergeRule toRule() { + return new ProjectUncollectMergeRule(this); + } + } +} diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index ea8e1772c67..8a0a419e985 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -331,6 +331,9 @@ ExInst invalidCompare(String a0, String a1, String a2, @BaseMessage("Cannot specify condition (NATURAL keyword, or ON or USING clause) following CROSS JOIN") ExInst crossJoinDisallowsCondition(); + @BaseMessage("UNNEST is only supported with INNER, LEFT, CROSS, or COMMA join, not ''{0}''") + ExInst unnestInvalidJoinType(String a0); + @BaseMessage("Cannot specify NATURAL keyword with ON or USING clause") ExInst naturalDisallowsOnOrUsing(); diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 789fb9d6c79..c13a01ae891 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -205,9 +205,6 @@ public class SqlFunctions { // Note: this variable handling is inspired by Apache Spark private static final int MAX_ARRAY_LENGTH = Integer.MAX_VALUE - 15; - private static final Function1, Enumerable> LIST_AS_ENUMERABLE = - a0 -> a0 == null ? Linq4j.emptyEnumerable() : Linq4j.asEnumerable(a0); - @SuppressWarnings("unused") private static final Function1> ARRAY_CARTESIAN_PRODUCT = lists -> { @@ -7421,94 +7418,239 @@ public static String arrayToString(List list, String delimiter, @Nullable String } /** - * Function that, given a certain List containing single-item structs (i.e. arrays / lists with - * a single item), builds an Enumerable that returns those single items inside the structs. + * Returns a function that, for each input row from an + * {@link org.apache.calcite.rel.core.Uncollect} operator yields an enumerable + * of output rows by unnesting the designated + * collection fields and appending them to the explicitly specified + * pass-through fields. + * + *

Multiple collections are combined with {@link ZipPaddedEnumerator}: elements at + * the same ordinal position are combined, and shorter collections are padded + * with {@code null}. This matches SQL {@code UNNEST(a, b)} semantics. + * + * @param passthroughIndices indices of the input fields to pass through, + * in ascending order + * @param collectionIndices indices of the input fields that are collections, + * in the order they are passed to {@code zipPadded} + * @param fieldCounts For each collection it's element-row width; + * -1 means the element is scalar (contributes 1 column) + * @param inputTypes collection kind per slot + * @param withOrdinality whether to append a 1-based ORDINALITY column + * @param inputFieldCount total number of fields in each input row + * (used to detect SCALAR row format when inputFieldCount == 1) + * @param outer if {@code true}, emit one NULL-padded row when all + * collections are null or empty (LEFT JOIN semantics); + * if {@code false}, drop the input row (INNER semantics) */ - public static Function1, Enumerable> flatList() { - return inputList -> Linq4j.asEnumerable(inputList).select(v -> structAccess(v, 0, null)); + @SuppressWarnings({"rawtypes", "unchecked"}) + public static Function1>> + flatUncollect( + final int[] passthroughIndices, + final int[] collectionIndices, final int[] fieldCounts, + final FlatProductInputType[] inputTypes, + final boolean withOrdinality, + final int inputFieldCount, + final boolean outer) { + + // Effective output-column width contributed by each collection (-1 indicates + // a collection scalars, and is effectively converted to 1). + final int[] widths = new int[collectionIndices.length]; + int offset = 0; + for (int i = 0; i < collectionIndices.length; i++) { + widths[i] = fieldCounts[i] < 0 ? 1 : fieldCounts[i]; + offset += widths[i]; + } + final int totalElemWidth = offset; + final int expandedOutputFieldCount = totalElemWidth + (withOrdinality ? 1 : 0); + final int outputFieldCount = passthroughIndices.length + expandedOutputFieldCount; + + final Function1>> rows = + flatUncollectRows(passthroughIndices, collectionIndices, inputTypes, + withOrdinality, inputFieldCount, outer, widths, totalElemWidth, + expandedOutputFieldCount, outputFieldCount); + + if (outputFieldCount == 1) { + // A one-field output row type (no pass-through fields, one collection + // field expanding to a single column, no ordinality) makes + // PhysTypeImpl optimize the row format down to SCALAR, under which + // rows are bare values rather than singleton lists; unwrap to match. + // The unwrapped value need not itself be Comparable (e.g. a nested + // array), so operate on raw types to avoid an incorrect checkcast. + final Function1 scalarRows = inputRowObj -> { + final Enumerable rawRows = rows.apply(inputRowObj); + return rawRows.select(row -> requireNonNull((List) row, "row").get(0)); + }; + return (Function1) scalarRows; + } + return rows; } - /** - * Returns a function that, given a row containing one or more collection - * fields, produces an {@link Enumerable} of combined element rows using - * zip (positional pairing) semantics. - * - *

This is the standard semantics for SQL {@code UNNEST(a, b, ...)}: the - * i-th output row pairs element {@code a[i]} with element {@code b[i]}. - * Shorter collections are padded with {@code NULL}. - */ - public static Function1>> flatZip( - final int[] fieldCounts, final boolean withOrdinality, - final FlatProductInputType[] inputTypes) { - if (fieldCounts.length == 1) { - if (!withOrdinality && inputTypes[0] == FlatProductInputType.SCALAR) { - // Simple unnest without ordinality - //noinspection unchecked - return (Function1) LIST_AS_ENUMERABLE; - } else { - // unnest with ordinality for a single scalar column - return row -> z2(new Object[] { row }, fieldCounts, withOrdinality, inputTypes); + @SuppressWarnings({"rawtypes", "unchecked"}) + private static Function1>> + flatUncollectRows( + final int[] passthroughIndices, + final int[] collectionIndices, + final FlatProductInputType[] inputTypes, + final boolean withOrdinality, + final int inputFieldCount, + final boolean outer, + final int[] widths, + final int totalElemWidth, + final int expandedOutputFieldCount, + final int outputFieldCount) { + return inputRowObj -> { + // When there is exactly one input field, JavaRowFormat.optimize() always + // picks SCALAR format, meaning inputRowObj IS the field value, not a row. + final List inputRow = inputFieldCount == 1 + ? Collections.singletonList(inputRowObj) + : rowToList(inputRowObj); + + // For outer (LEFT JOIN) semantics: if every collection is null or empty, + // emit one row with pass-through fields preserved and NULL for all collection columns. + if (outer) { + boolean allEmpty = true; + for (int collectionIndex : collectionIndices) { + if (!collectionIsNullOrEmpty(inputRow.get(collectionIndex))) { + allEmpty = false; + break; + } + } + if (allEmpty) { + // Filled with nulls + final Object[] out = new Object[outputFieldCount]; + int outCol = 0; + for (int i : passthroughIndices) { + out[outCol++] = inputRow.get(i); + } + // Emit NULL for all collection columns and ordinality. + // FlatLists.of(List) (unlike copyOf(Object...), whose "not + // comparable" overload falls back to a plain Guava ImmutableList + // for more than 6 elements) always returns a proper ComparableList + // wrapper, regardless of element count or comparability. + return Linq4j.singletonEnumerable( + (FlatLists.ComparableList) FlatLists.of(Arrays.asList(out))); + } + } + + // Build one enumerator per collection in collectionIndices order. + final List>> enumerators = new ArrayList<>(); + for (int q = 0; q < collectionIndices.length; q++) { + Enumerator> enumerator = + collectionEnumerator(inputRow.get(collectionIndices[q]), inputTypes[q]); + enumerators.add(enumerator); } + + final Enumerable> elemRows = + new AbstractEnumerable>() { + @Override public Enumerator> enumerator() { + return new ZipPaddedEnumerator( + enumerators, widths, expandedOutputFieldCount, withOrdinality); + } + }; + + // For each element row from elemRows, splice in the pass-through input + // fields to produce the final output row. + // Fields in both sets emit first the raw value then the element columns. + return new AbstractEnumerable>() { + @Override public Enumerator> enumerator() { + final Enumerator> elemEnum = + elemRows.enumerator(); + return new Enumerator>() { + @Override public FlatLists.ComparableList current() { + final Object[] out = new Object[outputFieldCount]; + int outIdx = 0; + // Pass-through fields first (in input-index order). + for (int i : passthroughIndices) { + out[outIdx++] = inputRow.get(i); + } + // Expanded element columns second (already in collection-index order). + // Read as a raw List: element values (nested arrays, struct rows) + // need not actually implement Comparable, so avoid the checkcast + // to Comparable that a List-typed access would insert. + final List elemRow = elemEnum.current(); + for (int e = 0; e < totalElemWidth; e++) { + out[outIdx++] = elemRow.get(e); + } + if (withOrdinality) { + out[outIdx] = elemRow.get(totalElemWidth); + } + return (FlatLists.ComparableList) FlatLists.of(Arrays.asList(out)); + } + + @Override public boolean moveNext() { + return elemEnum.moveNext(); + } + @Override public void reset() { + elemEnum.reset(); + } + @Override public void close() { + elemEnum.close(); + } + }; + } + }; + }; + } + + /** Converts an input row (Object[], List, or scalar) to a {@link List}. */ + @SuppressWarnings("unchecked") + private static List rowToList(Object rowObj) { + if (rowObj instanceof List) { + return (List) rowObj; + } else if (rowObj instanceof Object[]) { + return Arrays.asList((Object[]) rowObj); + } else { + return Collections.singletonList(rowObj); } - return lists -> z2((Object[]) lists, fieldCounts, withOrdinality, inputTypes); } - /** - * Helper for {@link #flatZip}: unpacks each collection in {@code lists} - * into an enumerator and combines them using zip (positional) semantics, - * padding shorter collections with {@code NULL}. - * - * @param lists one element per collection (scalar list, struct list, or map) - * @param fieldCounts output column count for each collection (-1 for a collection of scalars) - * @param withOrdinality whether to append a 1-based ordinality column - * @param inputTypes type of elements in each collection (SCALAR, LIST, or MAP) - */ - @SuppressWarnings("rawtypes") - private static Enumerable> z2( - Object[] lists, int[] fieldCounts, boolean withOrdinality, - FlatProductInputType[] inputTypes) { - final List>> enumerators = new ArrayList<>(); - final int[] widths = new int[lists.length]; - int totalFieldCount = 0; - for (int i = 0; i < lists.length; i++) { - final int fieldCount = fieldCounts[i]; - final FlatProductInputType inputType = inputTypes[i]; - final Object inputObject = lists[i]; - switch (inputType) { - case SCALAR: - @SuppressWarnings("unchecked") List list = - (List) inputObject; - enumerators.add(Linq4j.transform(Linq4j.enumerator(list), FlatLists::of)); - widths[i] = 1; - break; - case LIST: - @SuppressWarnings("unchecked") List> listList = - (List>) inputObject; - enumerators.add(Linq4j.enumerator(listList)); - widths[i] = fieldCount; - break; - case MAP: - @SuppressWarnings("unchecked") Map map = - (Map) inputObject; - Enumerator> enumerator = - Linq4j.enumerator(map.entrySet()); - enumerators.add(Linq4j.transform(enumerator, e -> FlatLists.of(e.getKey(), e.getValue()))); - widths[i] = 2; - break; - default: - throw new IllegalArgumentException("Unknown input type: " + inputType); - } - totalFieldCount += (fieldCount < 0) ? 1 : fieldCount; + /** Returns {@code true} if {@code collection} is {@code null} or empty. */ + private static boolean collectionIsNullOrEmpty(@Nullable Object collection) { + if (collection == null) { + return true; } - if (withOrdinality) { - ++totalFieldCount; + if (collection instanceof Collection) { + return ((Collection) collection).isEmpty(); + } + if (collection instanceof Map) { + return ((Map) collection).isEmpty(); + } + if (collection instanceof Object[]) { + return ((Object[]) collection).length == 0; + } + return false; + } + + /** Creates an enumerator over element rows for a single collection field. + * Returns an empty enumerator for a {@code null} collection. */ + @SuppressWarnings({"unchecked", "rawtypes"}) + private static Enumerator> collectionEnumerator( + @Nullable Object collection, FlatProductInputType inputType) { + if (collection == null) { + return Linq4j.emptyEnumerator(); + } + switch (inputType) { + case SCALAR: + return (Enumerator) Linq4j.transform( + Linq4j.enumerator((List) collection), (Object e) -> FlatLists.of(e)); + case STRUCT: + return (Enumerator) Linq4j.transform( + Linq4j.enumerator((List) collection), + (Object e) -> FlatLists.of((Object) ((List) e).toArray())); + case LIST: + final List> listColl = + collection instanceof Object[] + ? (List) Arrays.asList((Object[]) collection) + : (List>) collection; + return Linq4j.enumerator(listColl); + case MAP: + final Map map = (Map) collection; + return Linq4j.transform( + Linq4j.enumerator(map.entrySet()), + e -> FlatLists.of(e.getKey(), e.getValue())); + default: + throw new IllegalArgumentException("Unexpected FlatProductInputType: " + inputType); } - final int fieldCount = totalFieldCount; - return new AbstractEnumerable>() { - @Override public Enumerator> enumerator() { - return new ZipPaddedEnumerator(enumerators, widths, fieldCount, withOrdinality); - } - }; } public static Object[] array(Object... args) { @@ -7753,9 +7895,9 @@ public enum JsonScope { JSON_KEYS, JSON_KEYS_AND_VALUES, JSON_VALUES } - /** Type of argument passed into {@link #flatZip}. */ + /** Type of argument passed into {@link #flatUncollect}. */ public enum FlatProductInputType { - SCALAR, LIST, MAP + SCALAR, LIST, MAP, STRUCT } /** Type of part to extract passed into {@link ParseUrlFunction#parseUrl}. */ diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java index af1753f8e8b..31c5f3dd9e6 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java @@ -108,9 +108,13 @@ public SqlUnnestOperator(boolean withOrdinality) { builder.add(field.getName(), fieldType); } } else { - RelDataType colType = padNullable - ? typeFactory.enforceTypeWithNullability(componentType, true) + RelDataType elementType = componentType.isStruct() + ? typeFactory.builder().kind(componentType.getStructKind()) + .addAll(componentType.getFieldList()).build() : componentType; + RelDataType colType = padNullable + ? typeFactory.enforceTypeWithNullability(elementType, true) + : elementType; builder.add(SqlUtil.deriveAliasFromOrdinal(operand), colType); } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 26faab77bc6..fcd089b686c 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -4117,6 +4117,20 @@ protected void validateJoin(SqlJoin join, SqlValidatorScope scope) { } } + // UNNEST on the right side is only meaningful with INNER, LEFT, CROSS, or COMMA. + if (isUnnestNode(right)) { + switch (joinType) { + case INNER: + case LEFT: + case CROSS: + case COMMA: + break; + default: + throw newValidationError(join.getJoinTypeNode(), + RESOURCE.unnestInvalidJoinType(joinType.name())); + } + } + // Which join types require/allow a ON/USING condition, or allow // a NATURAL keyword? switch (joinType) { @@ -4193,6 +4207,22 @@ protected void validateJoin(SqlJoin join, SqlValidatorScope scope) { } } + /** + * Returns whether {@code node} is (or wraps, via AS or LATERAL) an + * {@code UNNEST} call. + */ + private static boolean isUnnestNode(SqlNode node) { + switch (node.getKind()) { + case UNNEST: + return true; + case AS: + case LATERAL: + return isUnnestNode(((SqlCall) node).operand(0)); + default: + return false; + } + } + /** * Shuttle which determines whether all SqlCalls that are * comparisons are comparing columns from both namespaces. diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java b/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java index 6489edb05d7..f83c5ae0a18 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelStructuredTypeFlattener.java @@ -517,7 +517,29 @@ public void rewriteRel(Collect rel) { } public void rewriteRel(Uncollect rel) { - rewriteGeneric(rel); + // Remap passthrough and collection field indices through the flattened + // input: a struct passthrough field expands to the range of its flattened + // columns; collection fields keep width 1 (flattening never expands + // collection types) but their positions may shift. + final List oldFields = rel.getInput().getRowType().getFieldList(); + final ImmutableBitSet.Builder passthrough = ImmutableBitSet.builder(); + for (int pos : rel.getPassthroughFieldIndices()) { + final int newPos = getNewForOldInput(pos); + final int size = postFlattenSize(oldFields.get(pos).getType()); + for (int i = 0; i < size; i++) { + passthrough.set(newPos + i); + } + } + final ImmutableBitSet.Builder collections = ImmutableBitSet.builder(); + for (int pos : rel.getCollectionFieldIndices()) { + collections.set(getNewForOldInput(pos)); + } + final Uncollect newRel = + new Uncollect(rel.getCluster(), rel.getTraitSet(), + getNewForOldRel(rel.getInput()), rel.withOrdinality, + rel.getItemAliases(), passthrough.build(), collections.build(), + rel.isOuter, rel.expandStructFields); + setNewForOldRel(rel, newRel); } public void rewriteRel(LogicalIntersect rel) { diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index ed6d52491a4..d00fe90e53c 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -2861,7 +2861,8 @@ private void convertUnnest(Blackboard bb, SqlCall call, @Nullable List f // so Uncollect's row type stays aligned with the validator. List itemAliases; if (fieldNames != null) { - itemAliases = fieldNames; + // do not include the ordinality column name + itemAliases = fieldNames.subList(0, nodes.size()); } else { itemAliases = new ArrayList<>(nodes.size()); for (int i = 0; i < nodes.size(); i++) { @@ -2872,6 +2873,7 @@ private void convertUnnest(Blackboard bb, SqlCall call, @Nullable List f .push(child) .project(exprs) .uncollect(itemAliases, operator.withOrdinality) + .let(r -> fieldNames == null ? r : r.rename(fieldNames)) .build(); } else { // REVIEW danny 2020-04-26: should we unify the normal field aliases and diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index 54d662a3a13..d83c2c528c0 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -2395,6 +2395,46 @@ public RelBuilder uncollect(List itemAliases, boolean withOrdinality) { return this; } + /** + * Creates an {@link Uncollect} that unnests the collection-typed fields at + * the given 0-based indices of the top-of-stack relation, with explicit + * control over which other fields pass through to the output. + * + *

Output columns: all passthrough fields first (in ascending input index + * order), then all collection-element columns (in ascending input index + * order); all other input fields are dropped. + * + * @param passthroughFieldIndices 0-based indices of non-collection fields + * to include in the output unchanged + * @param collectionFieldIndices 0-based indices of the collection-typed + * fields to unnest + * @param withOrdinality whether to append an ORDINALITY column + * @param outer if true, preserves input rows with null/empty + * collections (LEFT JOIN); if false, drops them (INNER) + * @param expandStructFields if true, a collection whose element type is a struct + * produces one output column per struct field; if false, + * a single column typed as the whole element + */ + public RelBuilder uncollect( + ImmutableBitSet passthroughFieldIndices, + ImmutableBitSet collectionFieldIndices, + boolean withOrdinality, + boolean outer, + boolean expandStructFields) { + Frame frame = stack.pop(); + stack.push( + new Frame( + Uncollect.create( + cluster.traitSetOf(Convention.NONE), + frame.rel, + withOrdinality, + passthroughFieldIndices, + collectionFieldIndices, + outer, + expandStructFields))); + return this; + } + /** Ensures that the field names match those given. * *

If all fields have the same name, adds nothing; diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 295d7d662e6..a79e48b2839 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -327,9 +327,9 @@ public enum BuiltInMethod { @SuppressWarnings("deprecation") PAIR_LIST_COPY_OF(PairList.Helper.class, "copyOf", Object.class, Object.class, Object[].class), - FLAT_ZIP(SqlFunctions.class, "flatZip", int[].class, boolean.class, - FlatProductInputType[].class), - FLAT_LIST(SqlFunctions.class, "flatList"), + FLAT_UNCOLLECT(SqlFunctions.class, "flatUncollect", int[].class, + int[].class, int[].class, FlatProductInputType[].class, boolean.class, int.class, + boolean.class), LIST_N(FlatLists.class, "copyOf", Comparable[].class), LIST1(FlatLists.class, "ofSingle", Object.class), LIST2(FlatLists.class, "of", Object.class, Object.class), diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 49552b41985..88898b8611c 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -115,6 +115,7 @@ AliasListDuplicate=Duplicate name ''{0}'' in column alias list JoinRequiresCondition=INNER, LEFT, RIGHT, FULL, or ASOF join requires a condition (NATURAL keyword or ON or USING clause) DisallowsQualifyingCommonColumn=Cannot qualify common column ''{0}'' CrossJoinDisallowsCondition=Cannot specify condition (NATURAL keyword, or ON or USING clause) following CROSS JOIN +UnnestInvalidJoinType=UNNEST is only supported with INNER, LEFT, CROSS, or COMMA join, not ''{0}'' NaturalDisallowsOnOrUsing=Cannot specify NATURAL keyword with ON or USING clause ColumnInUsingNotUnique=Column name ''{0}'' in NATURAL join or USING clause is not unique on one side of join NaturalOrUsingColumnNotCompatible=Column ''{0}'' matched using NATURAL keyword or USING clause has incompatible types: cannot compare ''{1}'' to ''{2}'' diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 1d3dda5af67..b0cfa1a3749 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -8229,6 +8229,42 @@ private void checkLiteral2(String expression, String expected) { withHsqldb().ok(expectedHsqldb); } + /** Tests that a {@link org.apache.calcite.rel.core.Uncollect} + * converts to SQL via {@code LATERAL UNNEST}. */ + @Test void testGeneralizedUncollect() { + final Function relFn = b -> b + .scan("EMP") + .project(b.field("DEPTNO"), + b.call(SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, + b.field("ENAME"))) + .uncollect(ImmutableBitSet.of(0), ImmutableBitSet.of(1), false, false, true) + .build(); + final String expected = "SELECT \"$cor0\".\"DEPTNO\", \"t10\".\"$f1\"\n" + + "FROM (SELECT \"DEPTNO\", ARRAY[\"ENAME\"] AS \"$f1\"\n" + + "FROM \"scott\".\"EMP\") AS \"$cor0\",\n" + + "LATERAL UNNEST((SELECT \"$cor0\".\"$f1\"\n" + + "FROM (VALUES (0)) AS \"t\" (\"ZERO\"))) AS \"t10\" (\"$f1\")"; + relFn(relFn).ok(expected); + } + + /** As {@link #testGeneralizedUncollect()}, with outer (LEFT JOIN) + * semantics. */ + @Test void testGeneralizedUncollectOuter() { + final Function relFn = b -> b + .scan("EMP") + .project(b.field("DEPTNO"), + b.call(SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, + b.field("ENAME"))) + .uncollect(ImmutableBitSet.of(0), ImmutableBitSet.of(1), false, true, true) + .build(); + final String expected = "SELECT \"$cor0\".\"DEPTNO\", \"t10\".\"$f1\"\n" + + "FROM (SELECT \"DEPTNO\", ARRAY[\"ENAME\"] AS \"$f1\"\n" + + "FROM \"scott\".\"EMP\") AS \"$cor0\"\n" + + "LEFT JOIN LATERAL UNNEST((SELECT \"$cor0\".\"$f1\"\n" + + "FROM (VALUES (0)) AS \"t\" (\"ZERO\"))) AS \"t10\" (\"$f1\") ON TRUE"; + relFn(relFn).ok(expected); + } + @Test void testWithinGroup1() { final String query = "select \"product_class_id\", collect(\"net_weight\") " + "within group (order by \"net_weight\" desc) " diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java index b878bfb085e..bbb547b8583 100644 --- a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java +++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java @@ -185,6 +185,13 @@ protected Collection data() { .with(CalciteAssert.SchemaSpec.STEELWHEELS) .with(Lex.BIG_QUERY)) .connect(); + case "hr-presto": + // Same as "hr", but uses PRESTO conformance, under which + // UNNEST(array) AS alias does not expand struct elements. + return customize(CalciteAssert.hr() + .with(CalciteConnectionProperty.CONFORMANCE, + SqlConformanceEnum.PRESTO)) + .connect(); default: return super.connect(name, reference); } diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 0dee9a6b74c..98fce291c85 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -768,6 +768,153 @@ private HepProgram createHypergraphProgram() { .check(); } + /** Tests that {@link CoreRules#CORRELATE_UNCOLLECT_MERGE} and + * {@link CoreRules#PROJECT_UNCOLLECT_MERGE} together convert a Correlate + * over an Uncollect of a struct-element array into a single, generalized + * Uncollect with pruned passthrough fields. */ + @Test void testCorrelateUncollectMergeStructArray() { + final String sql = "select t2.ename\n" + + "from DEPT_NESTED as t1,\n" + + "unnest(t1.employees) as t2"; + sql(sql) + .withPreRule(CoreRules.PROJECT_REMOVE) + .withRule(CoreRules.CORRELATE_UNCOLLECT_MERGE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_UNCOLLECT_MERGE) + .check(); + } + + /** Tests that {@link CoreRules#CORRELATE_UNCOLLECT_MERGE} and + * {@link CoreRules#PROJECT_UNCOLLECT_MERGE} together convert a Correlate + * over a scalar-array Uncollect into a single, generalized Uncollect while + * preserving a non-collection left field referenced by the outer + * project. */ + @Test void testCorrelateUncollectMergeScalarArray() { + final String sql = "select t1.name, t2.admin\n" + + "from DEPT_NESTED_EXPANDED as t1,\n" + + "unnest(t1.admins) as t2(admin)"; + sql(sql) + .withPreRule(CoreRules.PROJECT_REMOVE) + .withRule(CoreRules.CORRELATE_UNCOLLECT_MERGE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_UNCOLLECT_MERGE) + .check(); + } + + /** Tests that {@link CoreRules#CORRELATE_UNCOLLECT_MERGE} preserves a + * referenced ORDINALITY column when {@code WITH ORDINALITY} is used. */ + @Test void testCorrelateUncollectMergeOrdinality() { + final String sql = "select t2.admin, t2.rn\n" + + "from DEPT_NESTED_EXPANDED as t1,\n" + + "unnest(t1.admins) WITH ORDINALITY as t2(admin, rn)"; + sql(sql) + .withPreRule(CoreRules.PROJECT_REMOVE) + .withRule(CoreRules.CORRELATE_UNCOLLECT_MERGE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_UNCOLLECT_MERGE) + .check(); + } + + /** Tests that {@link CoreRules#CORRELATE_UNCOLLECT_MERGE} fires + * when the outer project references the collection-typed field directly. + * The collection field appears in both {@code passthrough} and + * {@code collectionFields}, so both its raw value and its element columns + * are included in the output. */ + @Test void testCorrelateUncollectMergePassthroughCollectionRef() { + final String sql = "select t1.admins, t2.admin\n" + + "from DEPT_NESTED_EXPANDED as t1,\n" + + "unnest(t1.admins) as t2(admin)"; + sql(sql) + .withPreRule(CoreRules.PROJECT_REMOVE) + .withRule(CoreRules.CORRELATE_UNCOLLECT_MERGE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_UNCOLLECT_MERGE) + .check(); + } + + /** Tests that {@link CoreRules#CORRELATE_UNCOLLECT_MERGE} fires on a bare + * Correlate, with no Project above it; every left field passes through to + * the merged Uncollect's output. */ + @Test void testCorrelateUncollectMergeWithoutProject() { + final String sql = "select *\n" + + "from DEPT_NESTED_EXPANDED as t1,\n" + + "unnest(t1.admins) as t2(admin)"; + sql(sql) + .withPreRule(CoreRules.PROJECT_REMOVE) + .withRule(CoreRules.CORRELATE_UNCOLLECT_MERGE) + .check(); + } + + /** Tests that {@link CoreRules#CORRELATE_UNCOLLECT_MERGE} fires when the + * unnested collections are not in ascending field-index order: the merged + * Uncollect expands them in ascending order, and a projection on top + * restores the original column order. */ + @Test void testCorrelateUncollectMergeOutOfOrderCollections() { + final String sql = "select t1.name\n" + + "from DEPT_NESTED_EXPANDED as t1,\n" + + "unnest(t1.admins, t1.employees) as t2"; + sql(sql) + .withPreRule(CoreRules.PROJECT_REMOVE) + .withRule(CoreRules.CORRELATE_UNCOLLECT_MERGE) + .check(); + } + + /** As {@link #testCorrelateUncollectMergeOutOfOrderCollections()}, with two + * scalar arrays built by a subquery rather than table columns, and with the + * outer project selecting the element columns in unnest order. */ + @Test void testCorrelateUncollectMergeOutOfOrderScalarArrays() { + final String sql = "with t1 as (select array['a', 'b'] as sa, array[1, 2] as ia)\n" + + "select u.x, u.y\n" + + "from t1, unnest(t1.ia, t1.sa) as u(x, y)"; + sql(sql) + .withPreRule(CoreRules.PROJECT_REMOVE) + .withRule(CoreRules.CORRELATE_UNCOLLECT_MERGE) + .check(); + } + + /** Tests that {@link CoreRules#CORRELATE_UNCOLLECT_MERGE} fires when the + * same collection is unnested twice. The merged Uncollect unnests it once; + * the projection on top duplicates its element column, casting to nullable + * (zipping two collections makes the element columns nullable). */ + @Test void testCorrelateUncollectMergeRepeatedCollection() { + final String sql = "select t2.a, t2.b\n" + + "from DEPT_NESTED_EXPANDED as t1,\n" + + "unnest(t1.admins, t1.admins) as t2(a, b)"; + sql(sql) + .withPreRule(CoreRules.PROJECT_REMOVE) + .withRule(CoreRules.CORRELATE_UNCOLLECT_MERGE) + .check(); + } + + /** Tests that {@link CoreRules#CORRELATE_UNCOLLECT_MERGE} fires when the + * Uncollect does not expand struct elements: PRESTO conformance aliases + * the unnest item, so each element stays a single ROW-typed column. The + * merged Uncollect inherits {@code expandStructFields=false}. */ + @Test void testCorrelateUncollectMergeUnexpandedStruct() { + final String sql = "select d.deptno, e.empno\n" + + "from dept_nested_expanded as d,\n" + + "unnest(d.employees) as e"; + sql(sql) + .withConformance(SqlConformanceEnum.PRESTO) + .withPreRule(CoreRules.PROJECT_REMOVE) + .withRule(CoreRules.CORRELATE_UNCOLLECT_MERGE) + .check(); + } + + /** Tests that {@link CoreRules#PROJECT_UNCOLLECT_MERGE} drops the + * ORDINALITY column when the project does not reference it. */ + @Test void testProjectUncollectMergeDropOrdinality() { + final String sql = "select t1.name, t2.admin\n" + + "from DEPT_NESTED_EXPANDED as t1,\n" + + "unnest(t1.admins) WITH ORDINALITY as t2(admin, rn)"; + sql(sql) + .withPreRule(CoreRules.PROJECT_REMOVE) + .withRule(CoreRules.CORRELATE_UNCOLLECT_MERGE, + CoreRules.PROJECT_MERGE, + CoreRules.PROJECT_UNCOLLECT_MERGE) + .check(); + } + @Test void testFilterProjectTransposeRule3() { final String sql = "select * from (select deptno from emp) as d\n" + "where NOT EXISTS (\n" diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index 959e2fabc25..586e6060fc4 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -2118,20 +2118,32 @@ private long sqlTimestamp(String str) { "2024-01-01 00:00:00", "Asia/Sanghai")); } - // Tests for ZipPaddedEnumerator, accessed via the public SqlFunctions.flatZip API. + // A set of unit tests for SqlFunctions.flatUncollect, used by the + // Enumerable implementation of Uncollect's pass-through/outer semantics. + static final int FIELD_IS_SCALAR = -1; - /** Invokes {@link SqlFunctions#flatZip} over scalar collections and collects output rows. */ + // Tests for ZipPaddedEnumerator, accessed via the public + // SqlFunctions.flatUncollect API (no pass-through fields, all + // input fields are scalar collections). + + /** Invokes {@link SqlFunctions#flatUncollect} over scalar + * collections (no pass-through fields) and collects output rows. */ @SuppressWarnings({"rawtypes", "unchecked"}) private static List> zipScalars( boolean withOrdinality, List... inputs) { final int n = inputs.length; + final int[] collectionIndices = new int[n]; final int[] fieldCounts = new int[n]; - Arrays.fill(fieldCounts, 1); final SqlFunctions.FlatProductInputType[] types = new SqlFunctions.FlatProductInputType[n]; - Arrays.fill(types, SCALAR); + for (int i = 0; i < n; i++) { + collectionIndices[i] = i; + fieldCounts[i] = FIELD_IS_SCALAR; + types[i] = SCALAR; + } final Function1>> fn = - SqlFunctions.flatZip(fieldCounts, withOrdinality, types); + SqlFunctions.flatUncollect( + new int[]{}, collectionIndices, fieldCounts, types, withOrdinality, n, false); final Object arg = n == 1 ? inputs[0] : inputs; final List> rows = new ArrayList<>(); for (FlatLists.ComparableList row : fn.apply(arg)) { @@ -2141,8 +2153,8 @@ private static List> zipScalars( } @Test void testZipPaddedSingleCollectionWithOrdinality() { - // Single scalar collection with ordinality uses ZipPaddedEnumerator, not - // the LIST_AS_ENUMERABLE shortcut. + // Single scalar collection with ordinality: two output columns + // (element, ordinality), so no single-column scalar unwrapping applies. List> rows = zipScalars(true, Arrays.asList(10, 20, 30)); assertThat(rows, hasSize(3)); assertThat(rows.get(0), is(list(10, 1))); @@ -2199,13 +2211,211 @@ private static List> zipScalars( assertThat(rows.get(2), is(Arrays.asList(30, null, 3))); } + /** Applies {@code fn} to a single input row and collects all output rows. */ + private static List> collectRows( + Function1>> fn, + Object inputRow) { + final List> result = new ArrayList<>(); + for (FlatLists.ComparableList row : fn.apply(inputRow)) { + result.add(new ArrayList<>(row)); + } + return result; + } + + /** Like {@link #collectRows}, but for the single-output-column case, where + * {@link SqlFunctions#flatUncollect} returns bare scalars + * (matching {@link org.apache.calcite.adapter.enumerable.JavaRowFormat#SCALAR}) + * rather than singleton rows. */ + @SuppressWarnings("rawtypes") + private static List collectScalarRows( + Function1>> fn, + Object inputRow) { + final List result = new ArrayList<>(); + for (Object row : (Enumerable) fn.apply(inputRow)) { + result.add(row); + } + return result; + } + + @Test void testFlatUncollectInner() { + // Row schema: [name, arr] — passthrough=0, collection=1, INNER semantics. + final Function1>> fn = + SqlFunctions.flatUncollect( + new int[]{0}, new int[]{1}, + new int[]{FIELD_IS_SCALAR}, + new SqlFunctions.FlatProductInputType[]{SCALAR}, + false, + 2, + false); + + // Non-empty collection -> one output row per element. + List> rows = + collectRows(fn, new Object[]{"a", Arrays.asList(10, 20, 30)}); + assertThat(rows, hasSize(3)); + assertThat(rows.get(0), is(list("a", 10))); + assertThat(rows.get(1), is(list("a", 20))); + assertThat(rows.get(2), is(list("a", 30))); + // Empty collection -> row dropped. + assertThat(collectRows(fn, new Object[]{"b", Collections.emptyList()}), hasSize(0)); + // Null collection -> row dropped (null-safe). + assertThat(collectRows(fn, new Object[]{"c", null}), hasSize(0)); + } + + @Test void testFlatUncollectOuter() { + // Row schema: [name, arr] — passthrough=0, collection=1, OUTER (LEFT JOIN) semantics. + final Function1>> fn = + SqlFunctions.flatUncollect( + new int[]{0}, new int[]{1}, + new int[]{FIELD_IS_SCALAR}, + new SqlFunctions.FlatProductInputType[]{SCALAR}, + false, + 2, + true); + + // Non-empty collection -> same behaviour as INNER. + List> rows = + collectRows(fn, new Object[]{"a", Arrays.asList(1, 2)}); + assertThat(rows, hasSize(2)); + assertThat(rows.get(0), is(list("a", 1))); + assertThat(rows.get(1), is(list("a", 2))); + + // Empty collection -> one null row; passthrough field preserved. + List> emptyRows = + collectRows(fn, new Object[]{"b", Collections.emptyList()}); + assertThat(emptyRows, hasSize(1)); + assertThat(emptyRows.get(0), is(Arrays.asList("b", null))); + + // Null collection -> one null row; passthrough field preserved. + List> nullRows = + collectRows(fn, new Object[]{"c", null}); + assertThat(nullRows, hasSize(1)); + assertThat(nullRows.get(0), is(Arrays.asList("c", null))); + } + + @Test void testFlatUncollectOrdinality() { + // INNER + withOrdinality: ordinality column starts at 1. + final Function1>> innerFn = + SqlFunctions.flatUncollect( + new int[]{0}, new int[]{1}, + new int[]{FIELD_IS_SCALAR}, + new SqlFunctions.FlatProductInputType[]{SCALAR}, + true, + 2, + false); + + List> rows = + collectRows(innerFn, new Object[]{"a", Arrays.asList(10, 20)}); + assertThat(rows, hasSize(2)); + assertThat(rows.get(0), is(list("a", 10, 1))); + assertThat(rows.get(1), is(list("a", 20, 2))); + + // OUTER + withOrdinality + empty -> one null row; ordinality column also null. + final Function1>> outerFn = + SqlFunctions.flatUncollect( + new int[]{0}, new int[]{1}, + new int[]{FIELD_IS_SCALAR}, + new SqlFunctions.FlatProductInputType[]{SCALAR}, + true, + 2, + true); + + List> outerEmpty = + collectRows(outerFn, new Object[]{"b", Collections.emptyList()}); + assertThat(outerEmpty, hasSize(1)); + assertThat(outerEmpty.get(0), is(Arrays.asList("b", null, null))); + } + + @Test void testFlatUncollectMultiCollection() { + // Row schema: [a, b] — no passthrough, both fields are scalar collections. + // INNER: zip [1,2,3] with [10,20] -> 3 rows, shorter collection padded with null. + final Function1>> innerFn = + SqlFunctions.flatUncollect( + new int[]{}, new int[]{0, 1}, + new int[]{FIELD_IS_SCALAR, FIELD_IS_SCALAR}, + new SqlFunctions.FlatProductInputType[]{SCALAR, SCALAR}, + false, + 2, + false); + + List> rows = + collectRows(innerFn, new Object[]{Arrays.asList(1, 2, 3), Arrays.asList(10, 20)}); + assertThat(rows, hasSize(3)); + assertThat(rows.get(0), is(list(1, 10))); + assertThat(rows.get(1), is(list(2, 20))); + assertThat(rows.get(2), is(Arrays.asList(3, null))); + + // OUTER: one collection non-empty, the other null -> zip produces rows (not all-empty). + final Function1>> outerFn = + SqlFunctions.flatUncollect( + new int[]{}, new int[]{0, 1}, + new int[]{FIELD_IS_SCALAR, FIELD_IS_SCALAR}, + new SqlFunctions.FlatProductInputType[]{SCALAR, SCALAR}, + false, + 2, + true); + + List> partialRows = + collectRows(outerFn, new Object[]{Arrays.asList(1, 2), null}); + assertThat(partialRows, hasSize(2)); + assertThat(partialRows.get(0), is(Arrays.asList(1, null))); + assertThat(partialRows.get(1), is(Arrays.asList(2, null))); + + // OUTER: both collections null -> one null row. + List> bothNull = + collectRows(outerFn, new Object[]{null, null}); + assertThat(bothNull, hasSize(1)); + assertThat(bothNull.get(0), is(Arrays.asList(null, null))); + + // OUTER: both collections empty -> one null row. + List> bothEmpty = + collectRows(outerFn, new Object[]{Collections.emptyList(), Collections.emptyList()}); + assertThat(bothEmpty, hasSize(1)); + assertThat(bothEmpty.get(0), is(Arrays.asList(null, null))); + } + + @Test void testFlatUncollectScalarRowFormat() { + // When inputFieldCount==1, the input object IS the collection (JavaRowFormat.SCALAR). + // The output row type is also a single column (no pass-through, one + // scalar collection, no ordinality), so rows are bare scalars, not + // singleton lists, matching PhysTypeImpl's own SCALAR optimization. + final Function1>> fn = + SqlFunctions.flatUncollect( + new int[]{}, new int[]{0}, + new int[]{FIELD_IS_SCALAR}, + new SqlFunctions.FlatProductInputType[]{SCALAR}, + false, + 1, + false); + + List rows = collectScalarRows(fn, Arrays.asList(10, 20, 30)); + assertThat(rows, hasSize(3)); + assertThat(rows.get(0), is(10)); + assertThat(rows.get(1), is(20)); + assertThat(rows.get(2), is(30)); + + // Empty in OUTER scalar-format -> one bare null row. + final Function1>> outerFn = + SqlFunctions.flatUncollect( + new int[]{}, new int[]{0}, + new int[]{FIELD_IS_SCALAR}, + new SqlFunctions.FlatProductInputType[]{SCALAR}, + false, + 1, + true); + + List outerEmpty = collectScalarRows(outerFn, Collections.emptyList()); + assertThat(outerEmpty, hasSize(1)); + assertThat(outerEmpty.get(0), is(nullValue())); + } + @Test void testZipPaddedStructElements() { // Two LIST (struct) collections of width 2: [(1,2),(3,4)] and [(10,20)] - // → [1,2,10,20], [3,4,null,null] + // -> [1,2,10,20], [3,4,null,null] @SuppressWarnings({"rawtypes", "unchecked"}) final Function1>> fn = - SqlFunctions.flatZip(new int[]{2, 2}, false, - new SqlFunctions.FlatProductInputType[]{LIST, LIST}); + SqlFunctions.flatUncollect( + new int[]{}, new int[]{0, 1}, new int[]{2, 2}, + new SqlFunctions.FlatProductInputType[]{LIST, LIST}, false, 2, false); final List> col1 = Arrays.asList(FlatLists.of(1, 2), FlatLists.of(3, 4)); final List> col2 = diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 0fb4fa4055b..4dccb71e007 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -9541,6 +9541,30 @@ void testGroupExpressionEquivalenceParams() { .fails("Column 'ORDINALITY' not found in any table"); } + /** UNNEST is valid with INNER, LEFT, CROSS, and COMMA joins; + * all other join kinds must be rejected by the validator. */ + @Test void testUnnestJoinType() { + // Allowed join kinds — these must all validate without error. + sql("select * from dept inner join unnest(array[1, 2]) as u(x) on true").ok(); + sql("select * from dept left join unnest(array[1, 2]) as u(x) on true").ok(); + sql("select * from dept cross join unnest(array[1, 2]) as u(x)").ok(); + sql("select * from dept, unnest(array[1, 2]) as u(x)").ok(); + + // LATERAL wrapping must also be allowed for valid join kinds. + sql("select * from dept cross join lateral unnest(array[1, 2]) as u(x)").ok(); + sql("select * from dept left join lateral unnest(array[1, 2]) as u(x) on true").ok(); + + // Disallowed join kinds — validator must reject these. + sql("select * from dept right ^join^ unnest(array[1, 2]) as u(x) on true") + .fails("UNNEST is only supported with INNER, LEFT, CROSS, or COMMA join, not 'RIGHT'"); + sql("select * from dept full ^join^ unnest(array[1, 2]) as u(x) on true") + .fails("UNNEST is only supported with INNER, LEFT, CROSS, or COMMA join, not 'FULL'"); + + // LATERAL wrapping must also be rejected for invalid join kinds. + sql("select * from dept right ^join^ lateral unnest(array[1, 2]) as u(x) on true") + .fails("UNNEST is only supported with INNER, LEFT, CROSS, or COMMA join, not 'RIGHT'"); + } + @Test void unnestMapMustNameColumnsKeyAndValueWhenNotAliased() { sql("select * from unnest(map[1, 12, 2, 22])") .type("RecordType(INTEGER NOT NULL KEY, INTEGER NOT NULL VALUE) NOT NULL"); diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 4cb1da4092b..3fb1117dc76 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -2244,6 +2244,227 @@ MultiJoin(joinFilter=[true], isFullOuterJoin=[false], joinTypes=[[RIGHT, INNER]] LogicalTableScan(table=[[CATALOG, SALES, A]]) LogicalTableScan(table=[[CATALOG, SALES, B]]) LogicalTableScan(table=[[CATALOG, SALES, C]]) +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -12337,6 +12558,30 @@ LogicalProject(DEPTNO=[$7], F1=[$9], F2=[$10]) LogicalWindow(window#0=[window(order by [0] aggs [LAST_VALUE($7)])]) LogicalWindow(window#0=[window(order by [0] aggs [FIRST_VALUE($7)])]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index bd6a2246a14..96102ed7773 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -336,7 +336,7 @@ from UNNEST(ARRAY[1, 2, 3]) as t]]> @@ -348,7 +348,7 @@ LogicalProject(T=[$0]) LogicalProject(DEPTNO=[$0], E=[$5], EMPNO=[$6.EMPNO]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{2, 3}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(ADMINS=[$cor0.ADMINS], EMPLOYEES=[$cor0.EMPLOYEES]) LogicalValues(tuples=[[{ 0 }]]) ]]> @@ -365,7 +365,7 @@ from dept_nested_expanded as d CROSS JOIN LogicalProject(DEPTNO=[$0], E=[$5], EMPNO=[$6.EMPNO]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{2, 3}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(ADMINS=[$cor0.ADMINS], EMPLOYEES=[$cor0.EMPLOYEES]) LogicalValues(tuples=[[{ 0 }]]) ]]> @@ -382,7 +382,7 @@ from dept_nested_expanded as d CROSS JOIN LogicalProject(DEPTNO=[$0], EMPNO=[$5.EMPNO]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{2}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(EMPLOYEES=[$cor0.EMPLOYEES]) LogicalValues(tuples=[[{ 0 }]]) ]]> @@ -399,7 +399,7 @@ from dept_nested_expanded as d, LogicalProject(DEPTNO=[$0], EMPNO=[$5.EMPNO]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{2}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(EMPLOYEES=[$cor0.EMPLOYEES]) LogicalValues(tuples=[[{ 0 }]]) ]]> @@ -421,7 +421,7 @@ from dept_nested_expanded as d, LogicalProject(DEPTNO=[$0], EMPNO=[$5.EMPNO]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{2}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(EMPLOYEES=[$cor0.EMPLOYEES]) LogicalValues(tuples=[[{ 0 }]]) ]]> @@ -438,7 +438,7 @@ from dept_nested_expanded as d, LogicalProject(DEPTNO=[$0], A=[$5]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{3}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(ADMINS=[$cor0.ADMINS]) LogicalValues(tuples=[[{ 0 }]]) ]]> diff --git a/core/src/test/resources/sql/unnest.iq b/core/src/test/resources/sql/unnest.iq index 8defd3b01ce..7e9c3f862fa 100644 --- a/core/src/test/resources/sql/unnest.iq +++ b/core/src/test/resources/sql/unnest.iq @@ -626,4 +626,384 @@ WHERE ( !ok +# Tests for [CALCITE-7608] Enhance Uncollect +!use scott +!set hep-rules " ++CoreRules.CORRELATE_UNCOLLECT_MERGE ++CoreRules.PROJECT_MERGE ++CoreRules.PROJECT_UNCOLLECT_MERGE ++CoreRules.PROJECT_REMOVE" + +# Validated on Postgres by replacing COLLECT with ARRAY_AGG +SELECT u.x, z +FROM ( + SELECT x, COLLECT(y) as ys + FROM (VALUES (1, 1), (2, 2), (1, 3)) AS t (x, y) + GROUP BY x) AS u, + UNNEST(u.ys) AS z; ++---+---+ +| X | Z | ++---+---+ +| 1 | 1 | +| 1 | 3 | +| 2 | 2 | ++---+---+ +(3 rows) + +!ok +EnumerableUncollect(passthrough=[[X]], collectionFields=[[YS]]) + EnumerableAggregate(group=[{0}], YS=[COLLECT($1)]) + EnumerableValues(tuples=[[{ 1, 1 }, { 2, 2 }, { 1, 3 }]]) +!plan + +# Lateral UNNEST with ordinality +# Validated on Postgres by replacing COLLECT with ARRAY_AGG +SELECT u.x, z, o +FROM ( + SELECT x, collect(y) as ys + FROM (VALUES (10, 'a'), (10, 'b'), (20, 'c')) AS t (x, y) + GROUP BY x) AS u, + UNNEST(u.ys) WITH ORDINALITY AS t(z, o); ++----+---+---+ +| X | Z | O | ++----+---+---+ +| 10 | a | 1 | +| 10 | b | 2 | +| 20 | c | 1 | ++----+---+---+ +(3 rows) + +!ok +EnumerableUncollect(passthrough=[[X]], collectionFields=[[YS]], withOrdinality=[true]) + EnumerableAggregate(group=[{0}], YS=[COLLECT($1)]) + EnumerableValues(tuples=[[{ 10, 'a' }, { 10, 'b' }, { 20, 'c' }]]) +!plan + +# Two scalar arrays of different lengths: shorter is padded with NULL. +# Pass-through field n is carried unchanged. +# Validated on Postgres +SELECT n, a, b +FROM (SELECT 'test' AS n, ARRAY[1, 2, 3] AS arr1, ARRAY[10, 20] AS arr2) AS u, +UNNEST(u.arr1, u.arr2) AS t(a, b); ++------+---+----+ +| N | A | B | ++------+---+----+ +| test | 3 | | +| test | 1 | 10 | +| test | 2 | 20 | ++------+---+----+ +(3 rows) + +!ok +EnumerableUncollect(passthrough=[[N]], collectionFields=[[ARR1, ARR2]]) + EnumerableCalc(expr#0=[{inputs}], expr#1=['test'], expr#2=[1], expr#3=[2], expr#4=[3], expr#5=[ARRAY($t2, $t3, $t4)], expr#6=[10], expr#7=[20], expr#8=[ARRAY($t6, $t7)], N=[$t1], ARR1=[$t5], ARR2=[$t8]) + EnumerableValues(tuples=[[{ 0 }]]) +!plan + +# Validated on Postgres +SELECT n, b, a +FROM (SELECT 'test' AS n, ARRAY[1, 2, 3] AS arr1, ARRAY[10, 20] AS arr2) AS u, +UNNEST(u.arr2, u.arr1) AS t(b, a); ++------+----+---+ +| N | B | A | ++------+----+---+ +| test | | 3 | +| test | 10 | 1 | +| test | 20 | 2 | ++------+----+---+ +(3 rows) + +!ok + +# The same collection unnested twice zipped with itself. +# Validated on Postgres +SELECT a, b +FROM (SELECT ARRAY[1, 2] AS arr) AS u, +UNNEST(u.arr, u.arr) AS t(a, b); ++---+---+ +| A | B | ++---+---+ +| 1 | 1 | +| 2 | 2 | ++---+---+ +(2 rows) + +!ok + +# Validated on Postgres by replacing COLLECT with ARRAY_AGG +SELECT * +FROM ( + SELECT x, COLLECT(y) as ys + FROM (VALUES (1, 1), (2, 2), (1, 3)) AS t (x, y) + GROUP BY x) AS u, + UNNEST(u.ys) AS z; ++---+--------+---+ +| X | YS | Z | ++---+--------+---+ +| 1 | [1, 3] | 1 | +| 1 | [1, 3] | 3 | +| 2 | [2] | 2 | ++---+--------+---+ +(3 rows) + +!ok +EnumerableUncollect(passthrough=[[X, YS]], collectionFields=[[YS]]) + EnumerableAggregate(group=[{0}], YS=[COLLECT($1)]) + EnumerableValues(tuples=[[{ 1, 1 }, { 2, 2 }, { 1, 3 }]]) +!plan + +# MAP column: each map entry produces a (KEY, VALUE) pair. +# Pass-through field id is carried unchanged. +# Postgres does not support UNNEST(MAP), so this is manually validated +SELECT id, k, v +FROM (SELECT 42 AS id, MAP['a', 1, 'b', 2] AS m) AS u, +UNNEST(u.m) AS t(k, v); ++----+---+---+ +| ID | K | V | ++----+---+---+ +| 42 | a | 1 | +| 42 | b | 2 | ++----+---+---+ +(2 rows) + +!ok +EnumerableUncollect(passthrough=[[ID]], collectionFields=[[M]]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[42], expr#2=['a'], expr#3=[1], expr#4=['b'], expr#5=[2], expr#6=[MAP($t2, $t3, $t4, $t5)], ID=[$t1], M=[$t6]) + EnumerableValues(tuples=[[{ 0 }]]) +!plan + +# ARRAY of ROW: each element is a struct expanded into multiple columns. +# Pass-through field id is carried unchanged. +# validated on Postgres using a user-defined type for ROW +SELECT id, a, b +FROM (SELECT 99 AS id, ARRAY[ROW(1, 'x'), ROW(2, 'y')] AS arr) AS u, +UNNEST(u.arr) AS t(a, b); ++----+---+---+ +| ID | A | B | ++----+---+---+ +| 99 | 1 | x | +| 99 | 2 | y | ++----+---+---+ +(2 rows) + +!ok +EnumerableUncollect(passthrough=[[ID]], collectionFields=[[ARR]]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[99], expr#2=[1], expr#3=['x'], expr#4=[ROW($t2, $t3)], expr#5=[2], expr#6=['y'], expr#7=[ROW($t5, $t6)], expr#8=[ARRAY($t4, $t7)], ID=[$t1], ARR=[$t8]) + EnumerableValues(tuples=[[{ 0 }]]) +!plan + +# LEFT JOIN UNNEST tests +!use hr + +# depts LEFT JOIN UNNEST(employees). +# Marketing has 0 employees; the INNER comma-join drops Marketing; +# the LEFT JOIN preserves Marketing with NULL-padded employee columns. +# Validated on Postgres +select d."name", e.* +from "hr"."depts" as d +LEFT JOIN UNNEST(d."employees") as e ON TRUE; ++-----------+-------+--------+-----------+---------+------------+ +| name | empid | deptno | name0 | salary | commission | ++-----------+-------+--------+-----------+---------+------------+ +| HR | 200 | 20 | Eric | 8000.0 | 500 | +| Marketing | | | | | | +| Sales | 100 | 10 | Bill | 10000.0 | 1000 | +| Sales | 150 | 10 | Sebastian | 7000.0 | | ++-----------+-------+--------+-----------+---------+------------+ +(4 rows) + +!ok +EnumerableUncollect(passthrough=[[name]], collectionFields=[[employees]], isOuter=[true]) + EnumerableCalc(expr#0..3=[{inputs}], name=[$t1], employees=[$t2]) + EnumerableTableScan(table=[[hr, depts]]) +!plan + +# depts LEFT JOIN UNNEST(employees) WITH ORDINALITY. +# Marketing (empty employees) is preserved; its ORDINALITY is NULL. +# Validated on Postgres +select d."name", e.* +from "hr"."depts" as d +LEFT JOIN UNNEST(d."employees") WITH ORDINALITY as e ON TRUE; ++-----------+-------+--------+-----------+---------+------------+------------+ +| name | empid | deptno | name0 | salary | commission | ORDINALITY | ++-----------+-------+--------+-----------+---------+------------+------------+ +| HR | 200 | 20 | Eric | 8000.0 | 500 | 1 | +| Marketing | | | | | | | +| Sales | 100 | 10 | Bill | 10000.0 | 1000 | 1 | +| Sales | 150 | 10 | Sebastian | 7000.0 | | 2 | ++-----------+-------+--------+-----------+---------+------------+------------+ +(4 rows) + +!ok + +# Null array tests for Uncollect. +# LEFT JOIN preserves left rows with NULL for element columns. +# INNER: rows 1 and 3 yield elements; row 2 (null array) is dropped. +# Validated on Postgres +SELECT u.x, z +FROM (VALUES + (1, ARRAY[10, 20]), + (2, CAST(NULL AS INTEGER ARRAY)), + (3, ARRAY[30])) AS u(x, arr), +UNNEST(u.arr) AS z; ++---+----+ +| X | Z | ++---+----+ +| 1 | 10 | +| 1 | 20 | +| 3 | 30 | ++---+----+ +(3 rows) + +!ok + +# OUTER: rows 1 and 3 yield elements; row 2 (null array) yields one row with NULL z. +# Validated on Postgres +SELECT u.x, z +FROM (VALUES + (1, ARRAY[10, 20]), + (2, CAST(NULL AS INTEGER ARRAY)), + (3, ARRAY[30])) AS u(x, arr) +LEFT JOIN UNNEST(u.arr) AS z ON TRUE; ++---+----+ +| X | Z | ++---+----+ +| 1 | 10 | +| 1 | 20 | +| 2 | | +| 3 | 30 | ++---+----+ +(4 rows) + +!ok + +# OUTER + ordinality: null array row is preserved with NULL element and NULL ordinality. +# Validated on Postgres +SELECT u.x, z, o +FROM (VALUES + (1, ARRAY[10, 20]), + (2, CAST(NULL AS INTEGER ARRAY)), + (3, ARRAY[30])) AS u(x, arr) +LEFT JOIN UNNEST(u.arr) WITH ORDINALITY AS t(z, o) ON TRUE; ++---+----+---+ +| X | Z | O | ++---+----+---+ +| 1 | 10 | 1 | +| 1 | 20 | 2 | +| 2 | | | +| 3 | 30 | 1 | ++---+----+---+ +(4 rows) + +!ok + +# OUTER with no pass-through column +SELECT z +FROM (VALUES + (1, ARRAY[10, 20]), + (2, CAST(NULL AS INTEGER ARRAY)), + (3, ARRAY[30])) AS u(x, arr) +LEFT JOIN UNNEST(u.arr) AS z ON TRUE; ++----+ +| Z | ++----+ +| 10 | +| 20 | +| | +| 30 | ++----+ +(4 rows) + +!ok + +# PRESTO conformance: UNNEST(array) AS t(col) does not expand a struct +# element into its fields; col holds the whole struct, accessed by dot +# notation. +!use hr-presto + +# INNER comma-join: Marketing (0 employees) is dropped. +select d."name" as dept, e.emp."name" as ename, e.emp."empid" as empid +from "hr"."depts" as d, +UNNEST(d."employees") as e(emp); ++-------+-----------+-------+ +| DEPT | ENAME | EMPID | ++-------+-----------+-------+ +| HR | Eric | 200 | +| Sales | Bill | 100 | +| Sales | Sebastian | 150 | ++-------+-----------+-------+ +(3 rows) + +!ok +EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$t1.name], expr#3=[$t1.empid], DEPT=[$t0], ENAME=[$t2], EMPID=[$t3]) + EnumerableUncollect(passthrough=[[name]], collectionFields=[[employees]], expandStructFields=[false]) + EnumerableCalc(expr#0..3=[{inputs}], name=[$t1], employees=[$t2]) + EnumerableTableScan(table=[[hr, depts]]) +!plan + +# LEFT JOIN UNNEST with a NULL struct column. +select d."name" as dept, e.emp."name" as ename +from "hr"."depts" as d +LEFT JOIN UNNEST(d."employees") as e(emp) ON TRUE; ++-----------+-----------+ +| DEPT | ENAME | ++-----------+-----------+ +| HR | Eric | +| Marketing | | +| Sales | Bill | +| Sales | Sebastian | ++-----------+-----------+ +(4 rows) + +!ok +EnumerableCalc(expr#0..2=[{inputs}], expr#3=[$t2.name], DEPT=[$t0], ENAME=[$t3]) + EnumerableCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{1}]) + EnumerableCalc(expr#0..3=[{inputs}], name=[$t1], employees=[$t2]) + EnumerableTableScan(table=[[hr, depts]]) + EnumerableUncollect(expandStructFields=[false]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[$cor0], expr#2=[$t1.employees], employees=[$t2]) + EnumerableValues(tuples=[[{ 0 }]]) +!plan + +# WITH ORDINALITY: the struct column stays whole; ordinality still expands. +select e.emp."name" as ename, e.rn +from "hr"."depts" as d, +UNNEST(d."employees") WITH ORDINALITY as e(emp, rn); ++-----------+----+ +| ENAME | RN | ++-----------+----+ +| Eric | 1 | +| Bill | 1 | +| Sebastian | 2 | ++-----------+----+ +(3 rows) + +!ok +EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$t0.name], ENAME=[$t2], RN=[$t1]) + EnumerableUncollect(withOrdinality=[true], expandStructFields=[false]) + EnumerableCalc(expr#0..3=[{inputs}], employees=[$t2]) + EnumerableTableScan(table=[[hr, depts]]) +!plan + +# Output a whole struct column +select d."name" as dept, e.emp as emp +from "hr"."depts" as d, +UNNEST(d."employees") as e(emp); ++-------+------------------------------------+ +| DEPT | EMP | ++-------+------------------------------------+ +| HR | {200, 20, Eric, 8000.0, 500} | +| Sales | {100, 10, Bill, 10000.0, 1000} | +| Sales | {150, 10, Sebastian, 7000.0, null} | ++-------+------------------------------------+ +(3 rows) + +!ok +EnumerableUncollect(passthrough=[[name]], collectionFields=[[employees]], expandStructFields=[false]) + EnumerableCalc(expr#0..3=[{inputs}], name=[$t1], employees=[$t2]) + EnumerableTableScan(table=[[hr, depts]]) +!plan + +!set hep-rules original + # End unnest.iq