From d3450170f100686633b3a41ce7a4e6202c5228c6 Mon Sep 17 00:00:00 2001 From: Raorao Xiong Date: Fri, 12 Jun 2026 15:31:55 +0800 Subject: [PATCH 1/8] [FLINK-39986][table-planner][python] Support top-level projection deduplication for Python UDFs This commit implements Phase 1 of Python UDF Common Sub-expression Elimination (CSE). Identical deterministic Python UDF calls in the projection list are deduplicated at the ExecNode level, reducing redundant cross-process (JVM <-> Python Worker) communication. Key changes: - Modify CommonExecPythonCalc to deduplicate top-level calls and append a ref-reuse projection operator to restore output schema - Add ProjectionCodeGenerator.generateProjectionOperator for the expansion projection - Add unit tests and integration tests Generated-by: Claude-4.6-Opus --- .gitignore | 3 + flink-python/pyflink/table/tests/test_udf.py | 107 ++++++++ .../exec/common/CommonExecPythonCalc.java | 198 +++++++++++++-- .../codegen/ProjectionCodeGenerator.scala | 54 +++- .../CommonExecPythonCalcRefReuseTest.java | 238 ++++++++++++++++++ 5 files changed, 574 insertions(+), 26 deletions(-) create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalcRefReuseTest.java diff --git a/.gitignore b/.gitignore index cd54d67114c8da..e4fcada6d1b75b 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ scalastyle-output.xml .cursor .claude .worktrees +.codebuddy +.codegraph +openspec/ .metadata .settings .project diff --git a/flink-python/pyflink/table/tests/test_udf.py b/flink-python/pyflink/table/tests/test_udf.py index 182c46bbd9cf87..87d1cb986cd1b9 100644 --- a/flink-python/pyflink/table/tests/test_udf.py +++ b/flink-python/pyflink/table/tests/test_udf.py @@ -926,6 +926,113 @@ def test_create_and_drop_function(self): self.assertTrue('add_one_func' not in t_env.list_user_defined_functions()) self.assertTrue('subtract_one_func' not in t_env.list_user_defined_functions()) + def test_python_local_ref_reuse(self): + """ + Parameterized test for Python UDF local reference reuse. + Each case specifies a SQL query and a result verifier. + Similar to FunctionITCase.inputForTestCalcLocalRefReuse in Java. + + Deterministic UDF calls with same args are computed once and reused; + non-deterministic UDF calls are always executed independently. + The UUID suffix in UDF output proves whether calls were deduplicated. + """ + @udf(result_type=DataTypes.STRING()) + def Det(s): + import uuid + return (s.upper() + "_" + str(uuid.uuid4())[:8]) if s else None + + @udf(result_type=DataTypes.STRING(), deterministic=False) + def Nondet(s): + import uuid + return (s.upper() + "_" + str(uuid.uuid4())[:8]) if s else None + + self.t_env.create_temporary_system_function("Det", Det) + self.t_env.create_temporary_system_function("Nondet", Nondet) + + source_t = self.t_env.from_elements([("hello",)], ['s']) + self.t_env.create_temporary_view("SourceTable", source_t) + + def all_equal(vals): + """All values are identical (reused).""" + self.assertTrue(all(v == vals[0] for v in vals), + f"Expected all equal, got: {vals}") + self.assertTrue(vals[0].startswith("HELLO_")) + + def all_different(vals): + """All values are distinct (not reused).""" + self.assertEqual(len(set(vals)), len(vals), + f"Expected all different, got: {vals}") + for v in vals: + self.assertTrue(v.startswith("HELLO_")) + + # (description, sql, verify_fn) + cases = [ + # --- Basic ref reuse --- + ( + "identical Det calls are reused", + "SELECT Det(s), Det(s), Det(s) FROM SourceTable", + all_equal, + ), + ( + "Nondet calls are NOT reused", + "SELECT Nondet(s), Nondet(s), Nondet(s) FROM SourceTable", + all_different, + ), + ( + "mixed: Det reused, Nondet independent", + "SELECT Det(s), Det(s), Nondet(s) FROM SourceTable", + lambda vals: ( + self.assertEqual(vals[0], vals[1]), + self.assertNotEqual(vals[0], vals[2]), + self.assertTrue(vals[2].startswith("HELLO_")), + ), + ), + # --- Nested SQL with operators --- + ( + "Det(s) || Det(s) - concat with reused UDF", + "SELECT Det(s) || Det(s), Det(s) || Det(s), Det(s), Det(s) FROM SourceTable", + lambda vals: ( + self.assertEqual(len(vals), 4), + self.assertEqual(vals[2], vals[3]), + self.assertEqual(vals[0], vals[1]), + self.assertEqual(vals[0], vals[2] + vals[2]), + ), + ), + ( + "Nondet(Det(s)) - outer Nondet not reused, inner Det reused", + "SELECT Nondet(Det(s)), Nondet(Det(s)), Det(s) FROM SourceTable", + lambda vals: ( + self.assertEqual(len(vals), 3), + self.assertNotEqual(vals[0], vals[1]), + self.assertTrue(vals[2].startswith("HELLO_")), + ), + ), + ( + "Det(Nondet(s)) - nondet input disables reuse", + "SELECT Det(Nondet(s)), Det(Nondet(s)) FROM SourceTable", + lambda vals: ( + self.assertEqual(len(vals), 2), + self.assertNotEqual(vals[0], vals[1]), + ), + ), + ] + + for desc, sql, verify_fn in cases: + with self.subTest(desc): + result = self.t_env.sql_query(sql) + col_count = len(result.get_schema().get_field_names()) + cols = ", ".join([f"c{i} STRING" for i in range(col_count)]) + sink_table = generate_random_table_name() + self.t_env.execute_sql(f""" + CREATE TABLE {sink_table}({cols}) + WITH ('connector'='test-sink') + """) + result.execute_insert(sink_table).wait() + actual = source_sink_utils.results() + self.assertEqual(len(actual), 1) + vals = actual[0].replace("+I[", "").replace("]", "").split(", ") + verify_fn(vals) + # decide whether two floats are equal def float_equal(a, b, rel_tol=1e-09, abs_tol=0.0): diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java index 9c6013055cab6e..217771feca400b 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java @@ -18,6 +18,7 @@ package org.apache.flink.table.planner.plan.nodes.exec.common; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.dag.Transformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; @@ -43,7 +44,9 @@ import org.apache.flink.table.planner.plan.nodes.exec.utils.CommonPythonUtil; import org.apache.flink.table.planner.plan.nodes.exec.utils.ExecNodeUtil; import org.apache.flink.table.planner.plan.utils.PythonUtil; +import org.apache.flink.table.planner.utils.ShortcutUtils; import org.apache.flink.table.runtime.generated.GeneratedProjection; +import org.apache.flink.table.runtime.operators.CodeGenOperatorFactory; import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.RowType; @@ -62,6 +65,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.apache.flink.util.Preconditions.checkArgument; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -127,6 +131,136 @@ protected Transformation translateToPlanInternal( return ret; } + // ------------------------------------------------------------------------- + // Common Sub-expression Elimination for Python UDFs + // ------------------------------------------------------------------------- + + /** + * Deduplicates deterministic Python RexCalls by their string digest. Only deterministic calls + * can be safely reused; non-deterministic calls must be evaluated independently each time. + * + * @return a tuple of (unique calls list, mapping from original index to deduplicated index) + */ + @VisibleForTesting + Tuple2, int[]> deduplicatePythonCalls(List pythonRexCalls) { + LinkedHashMap digestToIndex = new LinkedHashMap<>(); + List uniqueCalls = new ArrayList<>(); + int[] originalToDedup = new int[pythonRexCalls.size()]; + + for (int i = 0; i < pythonRexCalls.size(); i++) { + RexCall call = pythonRexCalls.get(i); + String digest = call.toString(); + + boolean canReuse = ShortcutUtils.isDeterministicThroughProgram(call, null); + + Integer existingIndex = digestToIndex.get(digest); + if (canReuse && existingIndex != null) { + // Deterministic duplicate — reuse the existing call + originalToDedup[i] = existingIndex; + } else { + int newPos = uniqueCalls.size(); + if (canReuse) { + digestToIndex.put(digest, newPos); + } + uniqueCalls.add(call); + originalToDedup[i] = newPos; + } + } + return Tuple2.of(uniqueCalls, originalToDedup); + } + + /** + * Creates a projection operator that expands deduplicated results back to the expected output + * schema. The Python operator produces [forwarded_fields..., unique_calls...], and this + * projection maps duplicated call positions back to their shared unique result. + */ + private OneInputTransformation createRefReuseProjection( + OneInputTransformation pythonTransformation, + int[] originalToDedup, + int forwardedCount, + InternalTypeInfo pythonOperatorResultTypeInfo, + ExecNodeConfig config, + ClassLoader classLoader) { + int[] expansionMapping = + IntStream.concat( + IntStream.range(0, forwardedCount), + IntStream.of(originalToDedup).map(idx -> idx + forwardedCount)) + .toArray(); + + RowType pythonOutputRowType = pythonOperatorResultTypeInfo.toRowType(); + RowType finalOutputType = (RowType) getOutputType(); + + CodeGenOperatorFactory projectionOperator = + ProjectionCodeGenerator.generateProjectionOperator( + new CodeGeneratorContext(config, classLoader), + pythonOutputRowType, + finalOutputType, + expansionMapping, + "PythonCalcRefReuseProjection"); + + String refReuseDetail = buildRefReuseDetailName(originalToDedup, forwardedCount); + + return ExecNodeUtil.createOneInputTransformation( + pythonTransformation, + createTransformationMeta( + PYTHON_CALC_TRANSFORMATION + "-ref-reuse", + refReuseDetail, + "PythonCalcRefReuse", + config), + projectionOperator, + InternalTypeInfo.of(finalOutputType), + pythonTransformation.getParallelism(), + false); + } + + /** + * Builds a human-readable detail name describing which Python UDF calls reuse earlier + * deduplicated results. Uses output column names (e.g. EXPR$1, EXPR$2) to identify calls. For + * example, {@code PythonCalcRefReuse(EXPR$2=EXPR$1)} means EXPR$2 reuses the result of EXPR$1, + * since both represent the same deterministic expression. + */ + @VisibleForTesting + String buildRefReuseDetailName(int[] originalToDedup, int forwardedCount) { + List fieldNames = ((RowType) getOutputType()).getFieldNames(); + StringBuilder sb = new StringBuilder("PythonCalcRefReuse"); + List reuseDescs = new ArrayList<>(); + for (int i = 0; i < originalToDedup.length; i++) { + for (int j = 0; j < i; j++) { + if (originalToDedup[j] == originalToDedup[i]) { + String reusedName = fieldNames.get(forwardedCount + j); + String reuserName = fieldNames.get(forwardedCount + i); + reuseDescs.add(String.format("%s=%s", reuserName, reusedName)); + break; + } + } + } + if (!reuseDescs.isEmpty()) { + sb.append("("); + sb.append(String.join(", ", reuseDescs)); + sb.append(")"); + } + return sb.toString(); + } + + /** Builds the output type info for the Python operator after call deduplication. */ + private InternalTypeInfo buildDedupOutputTypeInfo( + List forwardedFields, + LogicalType[] inputLogicalTypes, + List uniquePythonRexCalls) { + List fieldTypes = new ArrayList<>(); + for (int idx : forwardedFields) { + fieldTypes.add(inputLogicalTypes[idx]); + } + for (RexCall call : uniquePythonRexCalls) { + fieldTypes.add(FlinkTypeFactory.toLogicalType(call.getType())); + } + return InternalTypeInfo.ofFields(fieldTypes.toArray(new LogicalType[0])); + } + + // ------------------------------------------------------------------------- + // Core translation logic + // ------------------------------------------------------------------------- + private OneInputTransformation createPythonOneInputTransformation( Transformation inputTransform, ExecNodeConfig config, @@ -144,51 +278,65 @@ private OneInputTransformation createPythonOneInputTransformat .map(x -> ((RexInputRef) x).getIndex()) .collect(Collectors.toList()); + LogicalType[] inputLogicalTypes = + ((InternalTypeInfo) inputTransform.getOutputType()).toRowFieldTypes(); + RowType inputType = + ((InternalTypeInfo) inputTransform.getOutputType()).toRowType(); + + // Deduplicate identical deterministic Python function calls + Tuple2, int[]> dedupResult = deduplicatePythonCalls(pythonRexCalls); + List uniquePythonRexCalls = dedupResult.f0; + int[] originalToDedup = dedupResult.f1; + boolean needsExpansionProjection = originalToDedup.length != uniquePythonRexCalls.size(); + Tuple2 extractResult = - extractPythonScalarFunctionInfos(pythonRexCalls, classLoader); + extractPythonScalarFunctionInfos(uniquePythonRexCalls, classLoader); int[] pythonUdfInputOffsets = extractResult.f0; PythonFunctionInfo[] pythonFunctionInfos = extractResult.f1; - LogicalType[] inputLogicalTypes = - ((InternalTypeInfo) inputTransform.getOutputType()).toRowFieldTypes(); InternalTypeInfo pythonOperatorInputTypeInfo = (InternalTypeInfo) inputTransform.getOutputType(); - List forwardedFieldsLogicalTypes = - forwardedFields.stream() - .map(i -> inputLogicalTypes[i]) - .collect(Collectors.toList()); - List pythonCallLogicalTypes = - pythonRexCalls.stream() - .map(node -> FlinkTypeFactory.toLogicalType(node.getType())) - .collect(Collectors.toList()); - List fieldsLogicalTypes = new ArrayList<>(); - fieldsLogicalTypes.addAll(forwardedFieldsLogicalTypes); - fieldsLogicalTypes.addAll(pythonCallLogicalTypes); - InternalTypeInfo pythonOperatorResultTyeInfo = - InternalTypeInfo.ofFields(fieldsLogicalTypes.toArray(new LogicalType[0])); + // Build output type using deduplicated unique calls + InternalTypeInfo pythonOperatorResultTypeInfo = + buildDedupOutputTypeInfo(forwardedFields, inputLogicalTypes, uniquePythonRexCalls); + OneInputStreamOperator pythonOperator = getPythonScalarFunctionOperator( config, classLoader, pythonConfig, pythonOperatorInputTypeInfo, - pythonOperatorResultTyeInfo, + pythonOperatorResultTypeInfo, pythonUdfInputOffsets, pythonFunctionInfos, forwardedFields.stream().mapToInt(x -> x).toArray(), - pythonRexCalls.stream() + uniquePythonRexCalls.stream() .anyMatch( x -> PythonUtil.containsPythonCall( x, PythonFunctionKind.PANDAS))); - return ExecNodeUtil.createOneInputTransformation( - inputTransform, - createTransformationMeta(PYTHON_CALC_TRANSFORMATION, config), - pythonOperator, - pythonOperatorResultTyeInfo, - inputTransform.getParallelism(), - false); + OneInputTransformation pythonTransformation = + ExecNodeUtil.createOneInputTransformation( + inputTransform, + createTransformationMeta(PYTHON_CALC_TRANSFORMATION, config), + pythonOperator, + pythonOperatorResultTypeInfo, + inputTransform.getParallelism(), + false); + + if (!needsExpansionProjection) { + return pythonTransformation; + } + + // Append a ref-reuse projection to map deduplicated results back to expected schema + return createRefReuseProjection( + pythonTransformation, + originalToDedup, + forwardedFields.size(), + pythonOperatorResultTypeInfo, + config, + classLoader); } private Tuple2 extractPythonScalarFunctionInfos( diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProjectionCodeGenerator.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProjectionCodeGenerator.scala index 3f89df2e1a6372..8398d27f0458e7 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProjectionCodeGenerator.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ProjectionCodeGenerator.scala @@ -17,7 +17,7 @@ */ package org.apache.flink.table.planner.codegen -import org.apache.flink.table.data.RowData +import org.apache.flink.table.data.{BoxedWrapperRowData, RowData} import org.apache.flink.table.data.binary.BinaryRowData import org.apache.flink.table.data.writer.BinaryRowWriter import org.apache.flink.table.planner.codegen.CodeGenUtils._ @@ -28,6 +28,7 @@ import org.apache.flink.table.planner.functions.aggfunctions._ import org.apache.flink.table.planner.plan.utils.{AggregateInfo, RexLiteralUtil} import org.apache.flink.table.planner.plan.utils.FunctionCallUtil.{Constant, FunctionParam} import org.apache.flink.table.runtime.generated.{GeneratedProjection, Projection} +import org.apache.flink.table.runtime.operators.CodeGenOperatorFactory import org.apache.flink.table.types.logical.{BigIntType, LogicalType, RowType} import scala.collection.mutable.ArrayBuffer @@ -399,4 +400,55 @@ object ProjectionCodeGenerator { inputTerm = DEFAULT_INPUT1_TERM, constantExpressions = constantExpressions) } + + /** + * Generates a [[CodeGenOperatorFactory]] that performs a simple column projection/reorder. This + * is used for local-ref reuse projection in Python Calc. + * + * @param ctx + * the code generator context. + * @param inputType + * the row type of input. + * @param outputType + * the row type of output. + * @param inputMapping + * the index array mapping output positions to input positions. + * @param opName + * the name of the generated operator. + * @return + * the CodeGenOperatorFactory for the projection operator. + */ + def generateProjectionOperator( + ctx: CodeGeneratorContext, + inputType: RowType, + outputType: RowType, + inputMapping: Array[Int], + opName: String): CodeGenOperatorFactory[RowData] = { + val inputTerm = DEFAULT_INPUT1_TERM + val exprGenerator = new ExprCodeGenerator(ctx, false) + .bindInput(inputType, inputTerm = inputTerm) + + val accessExprs = inputMapping.map { + inputIdx => GenerateUtils.generateFieldAccess(ctx, inputType, inputTerm, inputIdx) + } + + val resultExpr = + exprGenerator.generateResultExpression(accessExprs, outputType, classOf[BoxedWrapperRowData]) + + val processCode = + s""" + |${resultExpr.code} + |${OperatorCodeGenerator.generateCollect(resultExpr.resultTerm)} + |""".stripMargin + + val genOperator = + OperatorCodeGenerator.generateOneInputStreamOperator[RowData, RowData]( + ctx, + opName, + processCode, + inputType, + inputTerm = inputTerm) + + new CodeGenOperatorFactory(genOperator) + } } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalcRefReuseTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalcRefReuseTest.java new file mode 100644 index 00000000000000..3dcd292fe94a37 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalcRefReuseTest.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.common; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.planner.calcite.FlinkTypeFactory; +import org.apache.flink.table.planner.calcite.FlinkTypeSystem; +import org.apache.flink.table.planner.functions.sql.FlinkSqlOperatorTable; +import org.apache.flink.table.planner.plan.nodes.exec.ExecNodeContext; +import org.apache.flink.table.planner.plan.nodes.exec.InputProperty; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests for the reference reuse (deduplication) logic in {@link CommonExecPythonCalc}. + * + *

This test verifies that the {@code deduplicatePythonCalls} method correctly identifies and + * deduplicates structurally identical deterministic RexCalls, while preserving non-deterministic + * calls independently. + */ +class CommonExecPythonCalcRefReuseTest { + + // ------------------------------------------------------------------------- + // Parameterized tests for deduplicatePythonCalls + // ------------------------------------------------------------------------- + + @ParameterizedTest(name = "{0}") + @MethodSource("inputForDeduplicatePythonCalls") + void testDeduplicatePythonCalls( + String description, + List calls, + int expectedUniqueCount, + int[] expectedMapping) { + CommonExecPythonCalc calc = new TestPythonCalc(Collections.emptyList()); + Tuple2, int[]> result = calc.deduplicatePythonCalls(calls); + + assertThat(result.f0).as("unique call count").hasSize(expectedUniqueCount); + assertThat(result.f1).as("original-to-dedup mapping").containsExactly(expectedMapping); + } + + static Stream inputForDeduplicatePythonCalls() { + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + FlinkTypeFactory factory = new FlinkTypeFactory(classLoader, FlinkTypeSystem.INSTANCE); + RexBuilder rb = new RexBuilder(factory); + RelDataType intTp = + factory.createTypeWithNullability(factory.createSqlType(SqlTypeName.INTEGER), true); + RelDataType bigintTp = + factory.createTypeWithNullability(factory.createSqlType(SqlTypeName.BIGINT), true); + + RexNode ref0 = new RexInputRef(0, intTp); + RexNode ref1 = new RexInputRef(1, intTp); + RexNode ref2 = new RexInputRef(2, bigintTp); + + // Deterministic calls + RexCall plusAB1 = (RexCall) rb.makeCall(SqlStdOperatorTable.PLUS, ref0, ref1); + RexCall plusAB2 = (RexCall) rb.makeCall(SqlStdOperatorTable.PLUS, ref0, ref1); + RexCall plusAB3 = (RexCall) rb.makeCall(SqlStdOperatorTable.PLUS, ref0, ref1); + RexCall minusAB = (RexCall) rb.makeCall(SqlStdOperatorTable.MINUS, ref0, ref1); + RexCall plusAC = (RexCall) rb.makeCall(SqlStdOperatorTable.PLUS, ref0, ref2); + + // Non-deterministic calls + RexCall randA1 = (RexCall) rb.makeCall(FlinkSqlOperatorTable.RAND, ref0); + RexCall randA2 = (RexCall) rb.makeCall(FlinkSqlOperatorTable.RAND, ref0); + RexCall randB = (RexCall) rb.makeCall(FlinkSqlOperatorTable.RAND, ref1); + + // Nested deterministic calls + RexNode innerPlus = rb.makeCall(SqlStdOperatorTable.PLUS, ref0, ref1); + RexCall nestedCall1 = (RexCall) rb.makeCall(SqlStdOperatorTable.PLUS, innerPlus, innerPlus); + RexCall nestedCall2 = (RexCall) rb.makeCall(SqlStdOperatorTable.PLUS, innerPlus, innerPlus); + + return Stream.of( + Arguments.of( + "identical deterministic calls are deduped", + Arrays.asList(plusAB1, plusAB2, plusAB3), + 1, + new int[] {0, 0, 0}), + Arguments.of( + "different deterministic calls are not deduped", + Arrays.asList(plusAB1, minusAB), + 2, + new int[] {0, 1}), + Arguments.of( + "non-deterministic calls with different args are not deduped", + Arrays.asList(randA1, randB), + 2, + new int[] {0, 1}), + Arguments.of( + "identical non-deterministic calls are not deduped", + Arrays.asList(randA1, randA2), + 2, + new int[] {0, 1}), + Arguments.of( + "mixed deterministic and non-deterministic calls", + Arrays.asList(plusAB1, randA1, plusAB2), + 2, + new int[] {0, 1, 0}), + Arguments.of( + "nested deterministic calls are deduped", + Arrays.asList(nestedCall1, nestedCall2), + 1, + new int[] {0, 0}), + Arguments.of( + "single call - no deduplication needed", + Collections.singletonList(plusAB1), + 1, + new int[] {0}), + Arguments.of( + "partial duplicates with different args", + Arrays.asList(plusAB1, plusAB2, plusAC), + 2, + new int[] {0, 0, 1})); + } + + // ------------------------------------------------------------------------- + // Parameterized tests for buildRefReuseDetailName + // ------------------------------------------------------------------------- + + @ParameterizedTest(name = "{0}") + @MethodSource("inputForBuildRefReuseDetailName") + void testBuildRefReuseDetailName( + String description, + String[] fieldNames, + int[] originalToDedup, + int forwardedCount, + String expectedDetailName) { + IntType[] types = new IntType[fieldNames.length]; + Arrays.fill(types, new IntType()); + RowType outputType = RowType.of(types, fieldNames); + CommonExecPythonCalc calc = new TestPythonCalc(Collections.emptyList(), outputType); + + String result = calc.buildRefReuseDetailName(originalToDedup, forwardedCount); + + assertThat(result).isEqualTo(expectedDetailName); + } + + static Stream inputForBuildRefReuseDetailName() { + return Stream.of( + Arguments.of( + "single reuse: EXPR$2 reuses EXPR$1", + new String[] {"f1", "EXPR$1", "EXPR$2"}, + new int[] {0, 0}, + 1, + "PythonCalcRefReuse(EXPR$2=EXPR$1)"), + Arguments.of( + "multiple reuses: all reuse EXPR$0", + new String[] {"EXPR$0", "EXPR$1", "EXPR$2", "EXPR$3", "EXPR$4"}, + new int[] {0, 0, 0, 0, 0}, + 0, + "PythonCalcRefReuse(EXPR$1=EXPR$0, EXPR$2=EXPR$0, EXPR$3=EXPR$0, EXPR$4=EXPR$0)"), + Arguments.of( + "two groups of reuse", + new String[] {"EXPR$0", "EXPR$1", "EXPR$2", "EXPR$3", "EXPR$4"}, + new int[] {0, 1, 0, 1, 0}, + 0, + "PythonCalcRefReuse(EXPR$2=EXPR$0, EXPR$3=EXPR$1, EXPR$4=EXPR$0)"), + Arguments.of( + "no reuse: all calls are different", + new String[] {"EXPR$0", "EXPR$1", "EXPR$2"}, + new int[] {0, 1, 2}, + 0, + "PythonCalcRefReuse"), + Arguments.of( + "with forwarded fields: d reuses c", + new String[] {"a", "b", "c", "d"}, + new int[] {0, 0}, + 2, + "PythonCalcRefReuse(d=c)")); + } + + // ------------------------------------------------------------------------- + // Test helper: minimal concrete subclass of CommonExecPythonCalc + // ------------------------------------------------------------------------- + + /** + * A minimal concrete subclass of {@link CommonExecPythonCalc} used only for testing the + * deduplication logic. The abstract methods are not needed for ref-reuse unit tests. + */ + private static class TestPythonCalc extends CommonExecPythonCalc { + + TestPythonCalc(List projection) { + this(projection, RowType.of(new IntType())); + } + + TestPythonCalc(List projection, RowType outputType) { + super( + 0, + ExecNodeContext.newContext(CommonExecPythonCalc.class), + new Configuration(), + projection, + Collections.singletonList(InputProperty.DEFAULT), + outputType, + "TestPythonCalc"); + } + + @Override + protected org.apache.flink.api.dag.Transformation + translateToPlanInternal( + org.apache.flink.table.planner.delegation.PlannerBase planner, + org.apache.flink.table.planner.plan.nodes.exec.ExecNodeConfig config) { + throw new UnsupportedOperationException("Not needed for ref-reuse unit tests"); + } + } +} From 651ff98b5936ad1714209f1c1a0e653dd6077e9e Mon Sep 17 00:00:00 2001 From: Raorao Xiong Date: Thu, 18 Jun 2026 19:25:13 +0800 Subject: [PATCH 2/8] [FLINK-39986][table-planner][python] Support full-tree CSE and condition-projection deduplication for Python UDFs This commit implements Phase 2 of Python UDF CSE. It extends the deduplication to nested sub-expressions (full-tree CSE) and adds a new optimizer rule to deduplicate Python UDF calls shared between WHERE conditions and SELECT projections. Key changes: - Add PythonCallDeduplicator and PythonCallCseResult for structural deduplication with nested sub-expression reference maps - Add PythonFunctionInfo.ResultRef for referencing pre-computed results - Extend protobuf Input message with refIndex field - Modify Python worker (operations.py) to support sequential execution with result references - Add RemoteCalcConditionProjectionCseRule optimizer rule - Register the new rule in stream and batch rule sets - Add plan tests and integration tests Generated-by: Claude-4.6-Opus --- .../fn_execution/flink_fn_execution_pb2.py | 224 +++++++++--------- .../fn_execution/flink_fn_execution_pb2.pyi | 6 +- .../pyflink/fn_execution/table/operations.py | 59 +++-- .../tests/test_scalar_function_cse.py | 213 +++++++++++++++++ .../fn_execution/utils/operation_utils.py | 3 + .../pyflink/proto/flink-fn-execution.proto | 2 + flink-python/pyflink/table/tests/test_udf.py | 23 ++ .../apache/flink/python/util/ProtoUtils.java | 2 + .../functions/python/PythonFunctionInfo.java | 51 +++- .../exec/common/CommonExecPythonCalc.java | 125 ++++++---- .../exec/common/PythonCallCseResult.java | 90 +++++++ .../exec/common/PythonCallDeduplicator.java | 181 ++++++++++++++ .../nodes/exec/utils/CommonPythonUtil.java | 39 ++- .../plan/rules/FlinkBatchRuleSets.scala | 1 + .../plan/rules/FlinkStreamRuleSets.scala | 2 + .../rules/logical/PythonCalcSplitRule.scala | 6 +- ...RemoteCalcConditionProjectionCseRule.scala | 197 +++++++++++++++ .../CommonExecPythonCalcRefReuseTest.java | 95 +++++--- .../stream/sql/PythonCalcConditionCseTest.xml | 162 +++++++++++++ .../sql/PythonCalcConditionCseTest.scala | 114 +++++++++ 20 files changed, 1387 insertions(+), 208 deletions(-) create mode 100644 flink-python/pyflink/fn_execution/tests/test_scalar_function_cse.py create mode 100644 flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/PythonCallCseResult.java create mode 100644 flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/PythonCallDeduplicator.java create mode 100644 flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.scala create mode 100644 flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.xml create mode 100644 flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.scala diff --git a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py index 96984f80d5c777..083774738bcbf7 100644 --- a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py +++ b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py @@ -31,7 +31,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66link-fn-execution.proto\x12 org.apache.flink.fn_execution.v1\"*\n\x0cJobParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x86\x01\n\x05Input\x12\x44\n\x03udf\x18\x01 \x01(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunctionH\x00\x12\x15\n\x0binputOffset\x18\x02 \x01(\x05H\x00\x12\x17\n\rinputConstant\x18\x03 \x01(\x0cH\x00\x42\x07\n\x05input\"\xa8\x01\n\x13UserDefinedFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12\x14\n\x0cwindow_index\x18\x03 \x01(\x05\x12\x1a\n\x12takes_row_as_input\x18\x04 \x01(\x08\x12\x15\n\ris_pandas_udf\x18\x05 \x01(\x08\"\x90\x01\n\x0c\x41syncOptions\x12!\n\x19max_concurrent_operations\x18\x01 \x01(\x05\x12\x12\n\ntimeout_ms\x18\x02 \x01(\x03\x12\x15\n\rretry_enabled\x18\x03 \x01(\x08\x12\x1a\n\x12retry_max_attempts\x18\x04 \x01(\x05\x12\x16\n\x0eretry_delay_ms\x18\x05 \x01(\x03\"\xc3\x03\n\x14UserDefinedFunctions\x12\x43\n\x04udfs\x18\x01 \x03(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12=\n\x07windows\x18\x03 \x03(\x0b\x32,.org.apache.flink.fn_execution.v1.OverWindow\x12\x17\n\x0fprofile_enabled\x18\x04 \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x05 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x45\n\rasync_options\x18\x06 \x01(\x0b\x32..org.apache.flink.fn_execution.v1.AsyncOptions\x12g\n\x0fruntime_context\x18\x07 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xdd\x02\n\nOverWindow\x12L\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x37.org.apache.flink.fn_execution.v1.OverWindow.WindowType\x12\x16\n\x0elower_boundary\x18\x02 \x01(\x03\x12\x16\n\x0eupper_boundary\x18\x03 \x01(\x03\"\xd0\x01\n\nWindowType\x12\x13\n\x0fRANGE_UNBOUNDED\x10\x00\x12\x1d\n\x19RANGE_UNBOUNDED_PRECEDING\x10\x01\x12\x1d\n\x19RANGE_UNBOUNDED_FOLLOWING\x10\x02\x12\x11\n\rRANGE_SLIDING\x10\x03\x12\x11\n\rROW_UNBOUNDED\x10\x04\x12\x1b\n\x17ROW_UNBOUNDED_PRECEDING\x10\x05\x12\x1b\n\x17ROW_UNBOUNDED_FOLLOWING\x10\x06\x12\x0f\n\x0bROW_SLIDING\x10\x07\"\x8b\x06\n\x1cUserDefinedAggregateFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12Z\n\x05specs\x18\x03 \x03(\x0b\x32K.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec\x12\x12\n\nfilter_arg\x18\x04 \x01(\x05\x12\x10\n\x08\x64istinct\x18\x05 \x01(\x08\x12\x1a\n\x12takes_row_as_input\x18\x06 \x01(\x08\x1a\x82\x04\n\x0c\x44\x61taViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ield_index\x18\x02 \x01(\x05\x12i\n\tlist_view\x18\x03 \x01(\x0b\x32T.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.ListViewH\x00\x12g\n\x08map_view\x18\x04 \x01(\x0b\x32S.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.MapViewH\x00\x1aT\n\x08ListView\x12H\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x97\x01\n\x07MapView\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeB\x0b\n\tdata_view\"\xac\x04\n\x0bGroupWindow\x12M\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x38.org.apache.flink.fn_execution.v1.GroupWindow.WindowType\x12\x16\n\x0eis_time_window\x18\x02 \x01(\x08\x12\x14\n\x0cwindow_slide\x18\x03 \x01(\x03\x12\x13\n\x0bwindow_size\x18\x04 \x01(\x03\x12\x12\n\nwindow_gap\x18\x05 \x01(\x03\x12\x13\n\x0bis_row_time\x18\x06 \x01(\x08\x12\x18\n\x10time_field_index\x18\x07 \x01(\x05\x12\x17\n\x0f\x61llowedLateness\x18\x08 \x01(\x03\x12U\n\x0fnamedProperties\x18\t \x03(\x0e\x32<.org.apache.flink.fn_execution.v1.GroupWindow.WindowProperty\x12\x16\n\x0eshift_timezone\x18\n \x01(\t\"[\n\nWindowType\x12\x19\n\x15TUMBLING_GROUP_WINDOW\x10\x00\x12\x18\n\x14SLIDING_GROUP_WINDOW\x10\x01\x12\x18\n\x14SESSION_GROUP_WINDOW\x10\x02\"c\n\x0eWindowProperty\x12\x10\n\x0cWINDOW_START\x10\x00\x12\x0e\n\nWINDOW_END\x10\x01\x12\x16\n\x12ROW_TIME_ATTRIBUTE\x10\x02\x12\x17\n\x13PROC_TIME_ATTRIBUTE\x10\x03\"\xc7\x05\n\x1dUserDefinedAggregateFunctions\x12L\n\x04udfs\x18\x01 \x03(\x0b\x32>.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12\x10\n\x08grouping\x18\x03 \x03(\x05\x12\x1e\n\x16generate_update_before\x18\x04 \x01(\x08\x12\x44\n\x08key_type\x18\x05 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x1b\n\x13index_of_count_star\x18\x06 \x01(\x05\x12\x1e\n\x16state_cleaning_enabled\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x12\x1b\n\x13\x63ount_star_inserted\x18\x0b \x01(\x08\x12\x43\n\x0cgroup_window\x18\x0c \x01(\x0b\x32-.org.apache.flink.fn_execution.v1.GroupWindow\x12\x17\n\x0fprofile_enabled\x18\r \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x0e \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12g\n\x0fruntime_context\x18\x0f \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xf6\x0f\n\x06Schema\x12>\n\x06\x66ields\x18\x01 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.Schema.Field\x1a\x97\x01\n\x07MapInfo\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x1d\n\x08TimeInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\"\n\rTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a,\n\x17LocalZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\'\n\x12ZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a/\n\x0b\x44\x65\x63imalInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x12\r\n\x05scale\x18\x02 \x01(\x05\x1a\x1c\n\nBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1f\n\rVarBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1a\n\x08\x43harInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1d\n\x0bVarCharInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\xb0\x08\n\tFieldType\x12\x44\n\ttype_name\x18\x01 \x01(\x0e\x32\x31.org.apache.flink.fn_execution.v1.Schema.TypeName\x12\x10\n\x08nullable\x18\x02 \x01(\x08\x12U\n\x17\x63ollection_element_type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeH\x00\x12\x44\n\x08map_info\x18\x04 \x01(\x0b\x32\x30.org.apache.flink.fn_execution.v1.Schema.MapInfoH\x00\x12>\n\nrow_schema\x18\x05 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.SchemaH\x00\x12L\n\x0c\x64\x65\x63imal_info\x18\x06 \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.DecimalInfoH\x00\x12\x46\n\ttime_info\x18\x07 \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.TimeInfoH\x00\x12P\n\x0etimestamp_info\x18\x08 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.TimestampInfoH\x00\x12\x66\n\x1alocal_zoned_timestamp_info\x18\t \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.Schema.LocalZonedTimestampInfoH\x00\x12[\n\x14zoned_timestamp_info\x18\n \x01(\x0b\x32;.org.apache.flink.fn_execution.v1.Schema.ZonedTimestampInfoH\x00\x12J\n\x0b\x62inary_info\x18\x0b \x01(\x0b\x32\x33.org.apache.flink.fn_execution.v1.Schema.BinaryInfoH\x00\x12Q\n\x0fvar_binary_info\x18\x0c \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.VarBinaryInfoH\x00\x12\x46\n\tchar_info\x18\r \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.CharInfoH\x00\x12M\n\rvar_char_info\x18\x0e \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.VarCharInfoH\x00\x42\x0b\n\ttype_info\x1al\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12@\n\x04type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\"\xab\x02\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\x0b\n\x07TINYINT\x10\x01\x12\x0c\n\x08SMALLINT\x10\x02\x12\x07\n\x03INT\x10\x03\x12\n\n\x06\x42IGINT\x10\x04\x12\x0b\n\x07\x44\x45\x43IMAL\x10\x05\x12\t\n\x05\x46LOAT\x10\x06\x12\n\n\x06\x44OUBLE\x10\x07\x12\x08\n\x04\x44\x41TE\x10\x08\x12\x08\n\x04TIME\x10\t\x12\r\n\tTIMESTAMP\x10\n\x12\x0b\n\x07\x42OOLEAN\x10\x0b\x12\n\n\x06\x42INARY\x10\x0c\x12\r\n\tVARBINARY\x10\r\x12\x08\n\x04\x43HAR\x10\x0e\x12\x0b\n\x07VARCHAR\x10\x0f\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x10\x12\x07\n\x03MAP\x10\x11\x12\x0c\n\x08MULTISET\x10\x12\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x13\x12\x13\n\x0fZONED_TIMESTAMP\x10\x14\x12\x08\n\x04NULL\x10\x15\"\xc3\n\n\x08TypeInfo\x12\x46\n\ttype_name\x18\x01 \x01(\x0e\x32\x33.org.apache.flink.fn_execution.v1.TypeInfo.TypeName\x12M\n\x17\x63ollection_element_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfoH\x00\x12O\n\rrow_type_info\x18\x03 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfoH\x00\x12S\n\x0ftuple_type_info\x18\x04 \x01(\x0b\x32\x38.org.apache.flink.fn_execution.v1.TypeInfo.TupleTypeInfoH\x00\x12O\n\rmap_type_info\x18\x05 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.MapTypeInfoH\x00\x12Q\n\x0e\x61vro_type_info\x18\x06 \x01(\x0b\x32\x37.org.apache.flink.fn_execution.v1.TypeInfo.AvroTypeInfoH\x00\x1a\x8b\x01\n\x0bMapTypeInfo\x12<\n\x08key_type\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12>\n\nvalue_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\xb8\x01\n\x0bRowTypeInfo\x12L\n\x06\x66ields\x18\x01 \x03(\x0b\x32<.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfo.Field\x1a[\n\x05\x46ield\x12\x12\n\nfield_name\x18\x01 \x01(\t\x12>\n\nfield_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1aP\n\rTupleTypeInfo\x12?\n\x0b\x66ield_types\x18\x01 \x03(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\x1e\n\x0c\x41vroTypeInfo\x12\x0e\n\x06schema\x18\x01 \x01(\t\"\x8d\x03\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\n\n\x06STRING\x10\x01\x12\x08\n\x04\x42YTE\x10\x02\x12\x0b\n\x07\x42OOLEAN\x10\x03\x12\t\n\x05SHORT\x10\x04\x12\x07\n\x03INT\x10\x05\x12\x08\n\x04LONG\x10\x06\x12\t\n\x05\x46LOAT\x10\x07\x12\n\n\x06\x44OUBLE\x10\x08\x12\x08\n\x04\x43HAR\x10\t\x12\x0b\n\x07\x42IG_INT\x10\n\x12\x0b\n\x07\x42IG_DEC\x10\x0b\x12\x0c\n\x08SQL_DATE\x10\x0c\x12\x0c\n\x08SQL_TIME\x10\r\x12\x11\n\rSQL_TIMESTAMP\x10\x0e\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x0f\x12\x13\n\x0fPRIMITIVE_ARRAY\x10\x10\x12\t\n\x05TUPLE\x10\x11\x12\x08\n\x04LIST\x10\x12\x12\x07\n\x03MAP\x10\x13\x12\x11\n\rPICKLED_BYTES\x10\x14\x12\x10\n\x0cOBJECT_ARRAY\x10\x15\x12\x0b\n\x07INSTANT\x10\x16\x12\x08\n\x04\x41VRO\x10\x17\x12\x0e\n\nLOCAL_DATE\x10\x18\x12\x0e\n\nLOCAL_TIME\x10\x19\x12\x12\n\x0eLOCAL_DATETIME\x10\x1a\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x1b\x42\x0b\n\ttype_info\"\xd1\x07\n\x1dUserDefinedDataStreamFunction\x12\x63\n\rfunction_type\x18\x01 \x01(\x0e\x32L.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.FunctionType\x12g\n\x0fruntime_context\x18\x02 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x16\n\x0emetric_enabled\x18\x04 \x01(\x08\x12\x41\n\rkey_type_info\x18\x05 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12\x17\n\x0fprofile_enabled\x18\x06 \x01(\x08\x12\x17\n\x0fhas_side_output\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x1a\xb2\x02\n\x0eRuntimeContext\x12\x11\n\ttask_name\x18\x01 \x01(\t\x12\x1f\n\x17task_name_with_subtasks\x18\x02 \x01(\t\x12#\n\x1bnumber_of_parallel_subtasks\x18\x03 \x01(\x05\x12\'\n\x1fmax_number_of_parallel_subtasks\x18\x04 \x01(\x05\x12\x1d\n\x15index_of_this_subtask\x18\x05 \x01(\x05\x12\x16\n\x0e\x61ttempt_number\x18\x06 \x01(\x05\x12\x46\n\x0ejob_parameters\x18\x07 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x1f\n\x17in_batch_execution_mode\x18\x08 \x01(\x08\"\xad\x01\n\x0c\x46unctionType\x12\x0b\n\x07PROCESS\x10\x00\x12\x0e\n\nCO_PROCESS\x10\x01\x12\x11\n\rKEYED_PROCESS\x10\x02\x12\x14\n\x10KEYED_CO_PROCESS\x10\x03\x12\n\n\x06WINDOW\x10\x04\x12\x18\n\x14\x43O_BROADCAST_PROCESS\x10\x05\x12\x1e\n\x1aKEYED_CO_BROADCAST_PROCESS\x10\x06\x12\x11\n\rREVISE_OUTPUT\x10\x64\"\xe4\x0e\n\x0fStateDescriptor\x12\x12\n\nstate_name\x18\x01 \x01(\t\x12Z\n\x10state_ttl_config\x18\x02 \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig\x1a\xe0\r\n\x0eStateTTLConfig\x12`\n\x0bupdate_type\x18\x01 \x01(\x0e\x32K.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.UpdateType\x12j\n\x10state_visibility\x18\x02 \x01(\x0e\x32P.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.StateVisibility\x12w\n\x17ttl_time_characteristic\x18\x03 \x01(\x0e\x32V.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.TtlTimeCharacteristic\x12\x0b\n\x03ttl\x18\x04 \x01(\x03\x12n\n\x12\x63leanup_strategies\x18\x05 \x01(\x0b\x32R.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies\x1a\xca\x08\n\x11\x43leanupStrategies\x12 \n\x18is_cleanup_in_background\x18\x01 \x01(\x08\x12y\n\nstrategies\x18\x02 \x03(\x0b\x32\x65.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.MapStrategiesEntry\x1aX\n\x1aIncrementalCleanupStrategy\x12\x14\n\x0c\x63leanup_size\x18\x01 \x01(\x05\x12$\n\x1crun_cleanup_for_every_record\x18\x02 \x01(\x08\x1aK\n#RocksdbCompactFilterCleanupStrategy\x12$\n\x1cquery_time_after_num_entries\x18\x01 \x01(\x03\x1a\xe0\x04\n\x12MapStrategiesEntry\x12o\n\x08strategy\x18\x01 \x01(\x0e\x32].org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.Strategies\x12\x81\x01\n\x0e\x65mpty_strategy\x18\x02 \x01(\x0e\x32g.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.EmptyCleanupStrategyH\x00\x12\x95\x01\n\x1cincremental_cleanup_strategy\x18\x03 \x01(\x0b\x32m.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.IncrementalCleanupStrategyH\x00\x12\xa9\x01\n\'rocksdb_compact_filter_cleanup_strategy\x18\x04 \x01(\x0b\x32v.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.RocksdbCompactFilterCleanupStrategyH\x00\x42\x11\n\x0f\x43leanupStrategy\"b\n\nStrategies\x12\x1c\n\x18\x46ULL_STATE_SCAN_SNAPSHOT\x10\x00\x12\x17\n\x13INCREMENTAL_CLEANUP\x10\x01\x12\x1d\n\x19ROCKSDB_COMPACTION_FILTER\x10\x02\"*\n\x14\x45mptyCleanupStrategy\x12\x12\n\x0e\x45MPTY_STRATEGY\x10\x00\"D\n\nUpdateType\x12\x0c\n\x08\x44isabled\x10\x00\x12\x14\n\x10OnCreateAndWrite\x10\x01\x12\x12\n\x0eOnReadAndWrite\x10\x02\"J\n\x0fStateVisibility\x12\x1f\n\x1bReturnExpiredIfNotCleanedUp\x10\x00\x12\x16\n\x12NeverReturnExpired\x10\x01\"+\n\x15TtlTimeCharacteristic\x12\x12\n\x0eProcessingTime\x10\x00\"\xf1\x07\n\x13\x43oderInfoDescriptor\x12`\n\x10\x66latten_row_type\x18\x01 \x01(\x0b\x32\x44.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.FlattenRowTypeH\x00\x12Q\n\x08row_type\x18\x02 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RowTypeH\x00\x12U\n\narrow_type\x18\x03 \x01(\x0b\x32?.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.ArrowTypeH\x00\x12k\n\x16over_window_arrow_type\x18\x04 \x01(\x0b\x32I.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.OverWindowArrowTypeH\x00\x12Q\n\x08raw_type\x18\x05 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RawTypeH\x00\x12H\n\x04mode\x18\x06 \x01(\x0e\x32:.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.Mode\x12\"\n\x1aseparated_with_end_message\x18\x07 \x01(\x08\x1aJ\n\x0e\x46lattenRowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x43\n\x07RowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x45\n\tArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aO\n\x13OverWindowArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aH\n\x07RawType\x12=\n\ttype_info\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\" \n\x04Mode\x12\n\n\x06SINGLE\x10\x00\x12\x0c\n\x08MULTIPLE\x10\x01\x42\x0b\n\tdata_typeB-\n\x1forg.apache.flink.fnexecution.v1B\nFlinkFnApib\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66link-fn-execution.proto\x12 org.apache.flink.fn_execution.v1\"*\n\x0cJobParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x9a\x01\n\x05Input\x12\x44\n\x03udf\x18\x01 \x01(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunctionH\x00\x12\x15\n\x0binputOffset\x18\x02 \x01(\x05H\x00\x12\x17\n\rinputConstant\x18\x03 \x01(\x0cH\x00\x12\x12\n\x08refIndex\x18\x04 \x01(\x05H\x00\x42\x07\n\x05input\"\xa8\x01\n\x13UserDefinedFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12\x14\n\x0cwindow_index\x18\x03 \x01(\x05\x12\x1a\n\x12takes_row_as_input\x18\x04 \x01(\x08\x12\x15\n\ris_pandas_udf\x18\x05 \x01(\x08\"\x90\x01\n\x0c\x41syncOptions\x12!\n\x19max_concurrent_operations\x18\x01 \x01(\x05\x12\x12\n\ntimeout_ms\x18\x02 \x01(\x03\x12\x15\n\rretry_enabled\x18\x03 \x01(\x08\x12\x1a\n\x12retry_max_attempts\x18\x04 \x01(\x05\x12\x16\n\x0eretry_delay_ms\x18\x05 \x01(\x03\"\xc3\x03\n\x14UserDefinedFunctions\x12\x43\n\x04udfs\x18\x01 \x03(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12=\n\x07windows\x18\x03 \x03(\x0b\x32,.org.apache.flink.fn_execution.v1.OverWindow\x12\x17\n\x0fprofile_enabled\x18\x04 \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x05 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x45\n\rasync_options\x18\x06 \x01(\x0b\x32..org.apache.flink.fn_execution.v1.AsyncOptions\x12g\n\x0fruntime_context\x18\x07 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xdd\x02\n\nOverWindow\x12L\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x37.org.apache.flink.fn_execution.v1.OverWindow.WindowType\x12\x16\n\x0elower_boundary\x18\x02 \x01(\x03\x12\x16\n\x0eupper_boundary\x18\x03 \x01(\x03\"\xd0\x01\n\nWindowType\x12\x13\n\x0fRANGE_UNBOUNDED\x10\x00\x12\x1d\n\x19RANGE_UNBOUNDED_PRECEDING\x10\x01\x12\x1d\n\x19RANGE_UNBOUNDED_FOLLOWING\x10\x02\x12\x11\n\rRANGE_SLIDING\x10\x03\x12\x11\n\rROW_UNBOUNDED\x10\x04\x12\x1b\n\x17ROW_UNBOUNDED_PRECEDING\x10\x05\x12\x1b\n\x17ROW_UNBOUNDED_FOLLOWING\x10\x06\x12\x0f\n\x0bROW_SLIDING\x10\x07\"\x8b\x06\n\x1cUserDefinedAggregateFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12Z\n\x05specs\x18\x03 \x03(\x0b\x32K.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec\x12\x12\n\nfilter_arg\x18\x04 \x01(\x05\x12\x10\n\x08\x64istinct\x18\x05 \x01(\x08\x12\x1a\n\x12takes_row_as_input\x18\x06 \x01(\x08\x1a\x82\x04\n\x0c\x44\x61taViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ield_index\x18\x02 \x01(\x05\x12i\n\tlist_view\x18\x03 \x01(\x0b\x32T.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.ListViewH\x00\x12g\n\x08map_view\x18\x04 \x01(\x0b\x32S.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.MapViewH\x00\x1aT\n\x08ListView\x12H\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x97\x01\n\x07MapView\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeB\x0b\n\tdata_view\"\xac\x04\n\x0bGroupWindow\x12M\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x38.org.apache.flink.fn_execution.v1.GroupWindow.WindowType\x12\x16\n\x0eis_time_window\x18\x02 \x01(\x08\x12\x14\n\x0cwindow_slide\x18\x03 \x01(\x03\x12\x13\n\x0bwindow_size\x18\x04 \x01(\x03\x12\x12\n\nwindow_gap\x18\x05 \x01(\x03\x12\x13\n\x0bis_row_time\x18\x06 \x01(\x08\x12\x18\n\x10time_field_index\x18\x07 \x01(\x05\x12\x17\n\x0f\x61llowedLateness\x18\x08 \x01(\x03\x12U\n\x0fnamedProperties\x18\t \x03(\x0e\x32<.org.apache.flink.fn_execution.v1.GroupWindow.WindowProperty\x12\x16\n\x0eshift_timezone\x18\n \x01(\t\"[\n\nWindowType\x12\x19\n\x15TUMBLING_GROUP_WINDOW\x10\x00\x12\x18\n\x14SLIDING_GROUP_WINDOW\x10\x01\x12\x18\n\x14SESSION_GROUP_WINDOW\x10\x02\"c\n\x0eWindowProperty\x12\x10\n\x0cWINDOW_START\x10\x00\x12\x0e\n\nWINDOW_END\x10\x01\x12\x16\n\x12ROW_TIME_ATTRIBUTE\x10\x02\x12\x17\n\x13PROC_TIME_ATTRIBUTE\x10\x03\"\xc7\x05\n\x1dUserDefinedAggregateFunctions\x12L\n\x04udfs\x18\x01 \x03(\x0b\x32>.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12\x10\n\x08grouping\x18\x03 \x03(\x05\x12\x1e\n\x16generate_update_before\x18\x04 \x01(\x08\x12\x44\n\x08key_type\x18\x05 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x1b\n\x13index_of_count_star\x18\x06 \x01(\x05\x12\x1e\n\x16state_cleaning_enabled\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x12\x1b\n\x13\x63ount_star_inserted\x18\x0b \x01(\x08\x12\x43\n\x0cgroup_window\x18\x0c \x01(\x0b\x32-.org.apache.flink.fn_execution.v1.GroupWindow\x12\x17\n\x0fprofile_enabled\x18\r \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x0e \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12g\n\x0fruntime_context\x18\x0f \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xf6\x0f\n\x06Schema\x12>\n\x06\x66ields\x18\x01 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.Schema.Field\x1a\x97\x01\n\x07MapInfo\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x1d\n\x08TimeInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\"\n\rTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a,\n\x17LocalZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\'\n\x12ZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a/\n\x0b\x44\x65\x63imalInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x12\r\n\x05scale\x18\x02 \x01(\x05\x1a\x1c\n\nBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1f\n\rVarBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1a\n\x08\x43harInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1d\n\x0bVarCharInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\xb0\x08\n\tFieldType\x12\x44\n\ttype_name\x18\x01 \x01(\x0e\x32\x31.org.apache.flink.fn_execution.v1.Schema.TypeName\x12\x10\n\x08nullable\x18\x02 \x01(\x08\x12U\n\x17\x63ollection_element_type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeH\x00\x12\x44\n\x08map_info\x18\x04 \x01(\x0b\x32\x30.org.apache.flink.fn_execution.v1.Schema.MapInfoH\x00\x12>\n\nrow_schema\x18\x05 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.SchemaH\x00\x12L\n\x0c\x64\x65\x63imal_info\x18\x06 \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.DecimalInfoH\x00\x12\x46\n\ttime_info\x18\x07 \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.TimeInfoH\x00\x12P\n\x0etimestamp_info\x18\x08 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.TimestampInfoH\x00\x12\x66\n\x1alocal_zoned_timestamp_info\x18\t \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.Schema.LocalZonedTimestampInfoH\x00\x12[\n\x14zoned_timestamp_info\x18\n \x01(\x0b\x32;.org.apache.flink.fn_execution.v1.Schema.ZonedTimestampInfoH\x00\x12J\n\x0b\x62inary_info\x18\x0b \x01(\x0b\x32\x33.org.apache.flink.fn_execution.v1.Schema.BinaryInfoH\x00\x12Q\n\x0fvar_binary_info\x18\x0c \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.VarBinaryInfoH\x00\x12\x46\n\tchar_info\x18\r \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.CharInfoH\x00\x12M\n\rvar_char_info\x18\x0e \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.VarCharInfoH\x00\x42\x0b\n\ttype_info\x1al\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12@\n\x04type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\"\xab\x02\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\x0b\n\x07TINYINT\x10\x01\x12\x0c\n\x08SMALLINT\x10\x02\x12\x07\n\x03INT\x10\x03\x12\n\n\x06\x42IGINT\x10\x04\x12\x0b\n\x07\x44\x45\x43IMAL\x10\x05\x12\t\n\x05\x46LOAT\x10\x06\x12\n\n\x06\x44OUBLE\x10\x07\x12\x08\n\x04\x44\x41TE\x10\x08\x12\x08\n\x04TIME\x10\t\x12\r\n\tTIMESTAMP\x10\n\x12\x0b\n\x07\x42OOLEAN\x10\x0b\x12\n\n\x06\x42INARY\x10\x0c\x12\r\n\tVARBINARY\x10\r\x12\x08\n\x04\x43HAR\x10\x0e\x12\x0b\n\x07VARCHAR\x10\x0f\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x10\x12\x07\n\x03MAP\x10\x11\x12\x0c\n\x08MULTISET\x10\x12\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x13\x12\x13\n\x0fZONED_TIMESTAMP\x10\x14\x12\x08\n\x04NULL\x10\x15\"\xc3\n\n\x08TypeInfo\x12\x46\n\ttype_name\x18\x01 \x01(\x0e\x32\x33.org.apache.flink.fn_execution.v1.TypeInfo.TypeName\x12M\n\x17\x63ollection_element_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfoH\x00\x12O\n\rrow_type_info\x18\x03 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfoH\x00\x12S\n\x0ftuple_type_info\x18\x04 \x01(\x0b\x32\x38.org.apache.flink.fn_execution.v1.TypeInfo.TupleTypeInfoH\x00\x12O\n\rmap_type_info\x18\x05 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.MapTypeInfoH\x00\x12Q\n\x0e\x61vro_type_info\x18\x06 \x01(\x0b\x32\x37.org.apache.flink.fn_execution.v1.TypeInfo.AvroTypeInfoH\x00\x1a\x8b\x01\n\x0bMapTypeInfo\x12<\n\x08key_type\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12>\n\nvalue_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\xb8\x01\n\x0bRowTypeInfo\x12L\n\x06\x66ields\x18\x01 \x03(\x0b\x32<.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfo.Field\x1a[\n\x05\x46ield\x12\x12\n\nfield_name\x18\x01 \x01(\t\x12>\n\nfield_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1aP\n\rTupleTypeInfo\x12?\n\x0b\x66ield_types\x18\x01 \x03(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\x1e\n\x0c\x41vroTypeInfo\x12\x0e\n\x06schema\x18\x01 \x01(\t\"\x8d\x03\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\n\n\x06STRING\x10\x01\x12\x08\n\x04\x42YTE\x10\x02\x12\x0b\n\x07\x42OOLEAN\x10\x03\x12\t\n\x05SHORT\x10\x04\x12\x07\n\x03INT\x10\x05\x12\x08\n\x04LONG\x10\x06\x12\t\n\x05\x46LOAT\x10\x07\x12\n\n\x06\x44OUBLE\x10\x08\x12\x08\n\x04\x43HAR\x10\t\x12\x0b\n\x07\x42IG_INT\x10\n\x12\x0b\n\x07\x42IG_DEC\x10\x0b\x12\x0c\n\x08SQL_DATE\x10\x0c\x12\x0c\n\x08SQL_TIME\x10\r\x12\x11\n\rSQL_TIMESTAMP\x10\x0e\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x0f\x12\x13\n\x0fPRIMITIVE_ARRAY\x10\x10\x12\t\n\x05TUPLE\x10\x11\x12\x08\n\x04LIST\x10\x12\x12\x07\n\x03MAP\x10\x13\x12\x11\n\rPICKLED_BYTES\x10\x14\x12\x10\n\x0cOBJECT_ARRAY\x10\x15\x12\x0b\n\x07INSTANT\x10\x16\x12\x08\n\x04\x41VRO\x10\x17\x12\x0e\n\nLOCAL_DATE\x10\x18\x12\x0e\n\nLOCAL_TIME\x10\x19\x12\x12\n\x0eLOCAL_DATETIME\x10\x1a\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x1b\x42\x0b\n\ttype_info\"\xd1\x07\n\x1dUserDefinedDataStreamFunction\x12\x63\n\rfunction_type\x18\x01 \x01(\x0e\x32L.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.FunctionType\x12g\n\x0fruntime_context\x18\x02 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x16\n\x0emetric_enabled\x18\x04 \x01(\x08\x12\x41\n\rkey_type_info\x18\x05 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12\x17\n\x0fprofile_enabled\x18\x06 \x01(\x08\x12\x17\n\x0fhas_side_output\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x1a\xb2\x02\n\x0eRuntimeContext\x12\x11\n\ttask_name\x18\x01 \x01(\t\x12\x1f\n\x17task_name_with_subtasks\x18\x02 \x01(\t\x12#\n\x1bnumber_of_parallel_subtasks\x18\x03 \x01(\x05\x12\'\n\x1fmax_number_of_parallel_subtasks\x18\x04 \x01(\x05\x12\x1d\n\x15index_of_this_subtask\x18\x05 \x01(\x05\x12\x16\n\x0e\x61ttempt_number\x18\x06 \x01(\x05\x12\x46\n\x0ejob_parameters\x18\x07 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x1f\n\x17in_batch_execution_mode\x18\x08 \x01(\x08\"\xad\x01\n\x0c\x46unctionType\x12\x0b\n\x07PROCESS\x10\x00\x12\x0e\n\nCO_PROCESS\x10\x01\x12\x11\n\rKEYED_PROCESS\x10\x02\x12\x14\n\x10KEYED_CO_PROCESS\x10\x03\x12\n\n\x06WINDOW\x10\x04\x12\x18\n\x14\x43O_BROADCAST_PROCESS\x10\x05\x12\x1e\n\x1aKEYED_CO_BROADCAST_PROCESS\x10\x06\x12\x11\n\rREVISE_OUTPUT\x10\x64\"\xe4\x0e\n\x0fStateDescriptor\x12\x12\n\nstate_name\x18\x01 \x01(\t\x12Z\n\x10state_ttl_config\x18\x02 \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig\x1a\xe0\r\n\x0eStateTTLConfig\x12`\n\x0bupdate_type\x18\x01 \x01(\x0e\x32K.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.UpdateType\x12j\n\x10state_visibility\x18\x02 \x01(\x0e\x32P.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.StateVisibility\x12w\n\x17ttl_time_characteristic\x18\x03 \x01(\x0e\x32V.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.TtlTimeCharacteristic\x12\x0b\n\x03ttl\x18\x04 \x01(\x03\x12n\n\x12\x63leanup_strategies\x18\x05 \x01(\x0b\x32R.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies\x1a\xca\x08\n\x11\x43leanupStrategies\x12 \n\x18is_cleanup_in_background\x18\x01 \x01(\x08\x12y\n\nstrategies\x18\x02 \x03(\x0b\x32\x65.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.MapStrategiesEntry\x1aX\n\x1aIncrementalCleanupStrategy\x12\x14\n\x0c\x63leanup_size\x18\x01 \x01(\x05\x12$\n\x1crun_cleanup_for_every_record\x18\x02 \x01(\x08\x1aK\n#RocksdbCompactFilterCleanupStrategy\x12$\n\x1cquery_time_after_num_entries\x18\x01 \x01(\x03\x1a\xe0\x04\n\x12MapStrategiesEntry\x12o\n\x08strategy\x18\x01 \x01(\x0e\x32].org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.Strategies\x12\x81\x01\n\x0e\x65mpty_strategy\x18\x02 \x01(\x0e\x32g.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.EmptyCleanupStrategyH\x00\x12\x95\x01\n\x1cincremental_cleanup_strategy\x18\x03 \x01(\x0b\x32m.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.IncrementalCleanupStrategyH\x00\x12\xa9\x01\n\'rocksdb_compact_filter_cleanup_strategy\x18\x04 \x01(\x0b\x32v.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.RocksdbCompactFilterCleanupStrategyH\x00\x42\x11\n\x0f\x43leanupStrategy\"b\n\nStrategies\x12\x1c\n\x18\x46ULL_STATE_SCAN_SNAPSHOT\x10\x00\x12\x17\n\x13INCREMENTAL_CLEANUP\x10\x01\x12\x1d\n\x19ROCKSDB_COMPACTION_FILTER\x10\x02\"*\n\x14\x45mptyCleanupStrategy\x12\x12\n\x0e\x45MPTY_STRATEGY\x10\x00\"D\n\nUpdateType\x12\x0c\n\x08\x44isabled\x10\x00\x12\x14\n\x10OnCreateAndWrite\x10\x01\x12\x12\n\x0eOnReadAndWrite\x10\x02\"J\n\x0fStateVisibility\x12\x1f\n\x1bReturnExpiredIfNotCleanedUp\x10\x00\x12\x16\n\x12NeverReturnExpired\x10\x01\"+\n\x15TtlTimeCharacteristic\x12\x12\n\x0eProcessingTime\x10\x00\"\xf1\x07\n\x13\x43oderInfoDescriptor\x12`\n\x10\x66latten_row_type\x18\x01 \x01(\x0b\x32\x44.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.FlattenRowTypeH\x00\x12Q\n\x08row_type\x18\x02 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RowTypeH\x00\x12U\n\narrow_type\x18\x03 \x01(\x0b\x32?.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.ArrowTypeH\x00\x12k\n\x16over_window_arrow_type\x18\x04 \x01(\x0b\x32I.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.OverWindowArrowTypeH\x00\x12Q\n\x08raw_type\x18\x05 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RawTypeH\x00\x12H\n\x04mode\x18\x06 \x01(\x0e\x32:.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.Mode\x12\"\n\x1aseparated_with_end_message\x18\x07 \x01(\x08\x1aJ\n\x0e\x46lattenRowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x43\n\x07RowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x45\n\tArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aO\n\x13OverWindowArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aH\n\x07RawType\x12=\n\ttype_info\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\" \n\x04Mode\x12\n\n\x06SINGLE\x10\x00\x12\x0c\n\x08MULTIPLE\x10\x01\x42\x0b\n\tdata_typeB-\n\x1forg.apache.flink.fnexecution.v1B\nFlinkFnApib\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -42,115 +42,115 @@ _globals['_JOBPARAMETER']._serialized_start=62 _globals['_JOBPARAMETER']._serialized_end=104 _globals['_INPUT']._serialized_start=107 - _globals['_INPUT']._serialized_end=241 - _globals['_USERDEFINEDFUNCTION']._serialized_start=244 - _globals['_USERDEFINEDFUNCTION']._serialized_end=412 - _globals['_ASYNCOPTIONS']._serialized_start=415 - _globals['_ASYNCOPTIONS']._serialized_end=559 - _globals['_USERDEFINEDFUNCTIONS']._serialized_start=562 - _globals['_USERDEFINEDFUNCTIONS']._serialized_end=1013 - _globals['_OVERWINDOW']._serialized_start=1016 - _globals['_OVERWINDOW']._serialized_end=1365 - _globals['_OVERWINDOW_WINDOWTYPE']._serialized_start=1157 - _globals['_OVERWINDOW_WINDOWTYPE']._serialized_end=1365 - _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_start=1368 - _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_end=2147 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_start=1633 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_end=2147 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_start=1896 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_end=1980 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_start=1983 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_end=2134 - _globals['_GROUPWINDOW']._serialized_start=2150 - _globals['_GROUPWINDOW']._serialized_end=2706 - _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_start=2514 - _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_end=2605 - _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_start=2607 - _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_end=2706 - _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_start=2709 - _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_end=3420 - _globals['_SCHEMA']._serialized_start=3423 - _globals['_SCHEMA']._serialized_end=5461 - _globals['_SCHEMA_MAPINFO']._serialized_start=3498 - _globals['_SCHEMA_MAPINFO']._serialized_end=3649 - _globals['_SCHEMA_TIMEINFO']._serialized_start=3651 - _globals['_SCHEMA_TIMEINFO']._serialized_end=3680 - _globals['_SCHEMA_TIMESTAMPINFO']._serialized_start=3682 - _globals['_SCHEMA_TIMESTAMPINFO']._serialized_end=3716 - _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_start=3718 - _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_end=3762 - _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_start=3764 - _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_end=3803 - _globals['_SCHEMA_DECIMALINFO']._serialized_start=3805 - _globals['_SCHEMA_DECIMALINFO']._serialized_end=3852 - _globals['_SCHEMA_BINARYINFO']._serialized_start=3854 - _globals['_SCHEMA_BINARYINFO']._serialized_end=3882 - _globals['_SCHEMA_VARBINARYINFO']._serialized_start=3884 - _globals['_SCHEMA_VARBINARYINFO']._serialized_end=3915 - _globals['_SCHEMA_CHARINFO']._serialized_start=3917 - _globals['_SCHEMA_CHARINFO']._serialized_end=3943 - _globals['_SCHEMA_VARCHARINFO']._serialized_start=3945 - _globals['_SCHEMA_VARCHARINFO']._serialized_end=3974 - _globals['_SCHEMA_FIELDTYPE']._serialized_start=3977 - _globals['_SCHEMA_FIELDTYPE']._serialized_end=5049 - _globals['_SCHEMA_FIELD']._serialized_start=5051 - _globals['_SCHEMA_FIELD']._serialized_end=5159 - _globals['_SCHEMA_TYPENAME']._serialized_start=5162 - _globals['_SCHEMA_TYPENAME']._serialized_end=5461 - _globals['_TYPEINFO']._serialized_start=5464 - _globals['_TYPEINFO']._serialized_end=6811 - _globals['_TYPEINFO_MAPTYPEINFO']._serialized_start=5958 - _globals['_TYPEINFO_MAPTYPEINFO']._serialized_end=6097 - _globals['_TYPEINFO_ROWTYPEINFO']._serialized_start=6100 - _globals['_TYPEINFO_ROWTYPEINFO']._serialized_end=6284 - _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_start=6193 - _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_end=6284 - _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_start=6286 - _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_end=6366 - _globals['_TYPEINFO_AVROTYPEINFO']._serialized_start=6368 - _globals['_TYPEINFO_AVROTYPEINFO']._serialized_end=6398 - _globals['_TYPEINFO_TYPENAME']._serialized_start=6401 - _globals['_TYPEINFO_TYPENAME']._serialized_end=6798 - _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_start=6814 - _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_end=7791 - _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_start=7309 - _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_end=7615 - _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_start=7618 - _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_end=7791 - _globals['_STATEDESCRIPTOR']._serialized_start=7794 - _globals['_STATEDESCRIPTOR']._serialized_end=9686 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_start=7926 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_end=9686 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_start=8397 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_end=9495 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_start=8575 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_end=8663 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_start=8665 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_end=8740 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_start=8743 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_end=9351 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_start=9353 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_end=9451 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_start=9453 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_end=9495 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_start=9497 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_end=9565 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_start=9567 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_end=9641 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_start=9643 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_end=9686 - _globals['_CODERINFODESCRIPTOR']._serialized_start=9689 - _globals['_CODERINFODESCRIPTOR']._serialized_end=10698 - _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_start=10282 - _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_end=10356 - _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_start=10358 - _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_end=10425 - _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_start=10427 - _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_end=10496 - _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_start=10498 - _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_end=10577 - _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_start=10579 - _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_end=10651 - _globals['_CODERINFODESCRIPTOR_MODE']._serialized_start=10653 - _globals['_CODERINFODESCRIPTOR_MODE']._serialized_end=10685 + _globals['_INPUT']._serialized_end=261 + _globals['_USERDEFINEDFUNCTION']._serialized_start=264 + _globals['_USERDEFINEDFUNCTION']._serialized_end=432 + _globals['_ASYNCOPTIONS']._serialized_start=435 + _globals['_ASYNCOPTIONS']._serialized_end=579 + _globals['_USERDEFINEDFUNCTIONS']._serialized_start=582 + _globals['_USERDEFINEDFUNCTIONS']._serialized_end=1033 + _globals['_OVERWINDOW']._serialized_start=1036 + _globals['_OVERWINDOW']._serialized_end=1385 + _globals['_OVERWINDOW_WINDOWTYPE']._serialized_start=1177 + _globals['_OVERWINDOW_WINDOWTYPE']._serialized_end=1385 + _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_start=1388 + _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_end=2167 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_start=1653 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_end=2167 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_start=1916 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_end=2000 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_start=2003 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_end=2154 + _globals['_GROUPWINDOW']._serialized_start=2170 + _globals['_GROUPWINDOW']._serialized_end=2726 + _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_start=2534 + _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_end=2625 + _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_start=2627 + _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_end=2726 + _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_start=2729 + _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_end=3440 + _globals['_SCHEMA']._serialized_start=3443 + _globals['_SCHEMA']._serialized_end=5481 + _globals['_SCHEMA_MAPINFO']._serialized_start=3518 + _globals['_SCHEMA_MAPINFO']._serialized_end=3669 + _globals['_SCHEMA_TIMEINFO']._serialized_start=3671 + _globals['_SCHEMA_TIMEINFO']._serialized_end=3700 + _globals['_SCHEMA_TIMESTAMPINFO']._serialized_start=3702 + _globals['_SCHEMA_TIMESTAMPINFO']._serialized_end=3736 + _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_start=3738 + _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_end=3782 + _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_start=3784 + _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_end=3823 + _globals['_SCHEMA_DECIMALINFO']._serialized_start=3825 + _globals['_SCHEMA_DECIMALINFO']._serialized_end=3872 + _globals['_SCHEMA_BINARYINFO']._serialized_start=3874 + _globals['_SCHEMA_BINARYINFO']._serialized_end=3902 + _globals['_SCHEMA_VARBINARYINFO']._serialized_start=3904 + _globals['_SCHEMA_VARBINARYINFO']._serialized_end=3935 + _globals['_SCHEMA_CHARINFO']._serialized_start=3937 + _globals['_SCHEMA_CHARINFO']._serialized_end=3963 + _globals['_SCHEMA_VARCHARINFO']._serialized_start=3965 + _globals['_SCHEMA_VARCHARINFO']._serialized_end=3994 + _globals['_SCHEMA_FIELDTYPE']._serialized_start=3997 + _globals['_SCHEMA_FIELDTYPE']._serialized_end=5069 + _globals['_SCHEMA_FIELD']._serialized_start=5071 + _globals['_SCHEMA_FIELD']._serialized_end=5179 + _globals['_SCHEMA_TYPENAME']._serialized_start=5182 + _globals['_SCHEMA_TYPENAME']._serialized_end=5481 + _globals['_TYPEINFO']._serialized_start=5484 + _globals['_TYPEINFO']._serialized_end=6831 + _globals['_TYPEINFO_MAPTYPEINFO']._serialized_start=5978 + _globals['_TYPEINFO_MAPTYPEINFO']._serialized_end=6117 + _globals['_TYPEINFO_ROWTYPEINFO']._serialized_start=6120 + _globals['_TYPEINFO_ROWTYPEINFO']._serialized_end=6304 + _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_start=6213 + _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_end=6304 + _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_start=6306 + _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_end=6386 + _globals['_TYPEINFO_AVROTYPEINFO']._serialized_start=6388 + _globals['_TYPEINFO_AVROTYPEINFO']._serialized_end=6418 + _globals['_TYPEINFO_TYPENAME']._serialized_start=6421 + _globals['_TYPEINFO_TYPENAME']._serialized_end=6818 + _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_start=6834 + _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_end=7811 + _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_start=7329 + _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_end=7635 + _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_start=7638 + _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_end=7811 + _globals['_STATEDESCRIPTOR']._serialized_start=7814 + _globals['_STATEDESCRIPTOR']._serialized_end=9706 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_start=7946 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_end=9706 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_start=8417 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_end=9515 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_start=8595 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_end=8683 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_start=8685 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_end=8760 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_start=8763 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_end=9371 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_start=9373 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_end=9471 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_start=9473 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_end=9515 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_start=9517 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_end=9585 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_start=9587 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_end=9661 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_start=9663 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_end=9706 + _globals['_CODERINFODESCRIPTOR']._serialized_start=9709 + _globals['_CODERINFODESCRIPTOR']._serialized_end=10718 + _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_start=10302 + _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_end=10376 + _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_start=10378 + _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_end=10445 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_start=10447 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_end=10516 + _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_start=10518 + _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_end=10597 + _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_start=10599 + _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_end=10671 + _globals['_CODERINFODESCRIPTOR_MODE']._serialized_start=10673 + _globals['_CODERINFODESCRIPTOR_MODE']._serialized_end=10705 # @@protoc_insertion_point(module_scope) diff --git a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi index b6b77c6894609e..14f969c65e0e39 100644 --- a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi +++ b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi @@ -32,14 +32,16 @@ class JobParameter(_message.Message): def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... class Input(_message.Message): - __slots__ = ("udf", "inputOffset", "inputConstant") + __slots__ = ("udf", "inputOffset", "inputConstant", "refIndex") UDF_FIELD_NUMBER: _ClassVar[int] INPUTOFFSET_FIELD_NUMBER: _ClassVar[int] INPUTCONSTANT_FIELD_NUMBER: _ClassVar[int] + REFINDEX_FIELD_NUMBER: _ClassVar[int] udf: UserDefinedFunction inputOffset: int inputConstant: bytes - def __init__(self, udf: _Optional[_Union[UserDefinedFunction, _Mapping]] = ..., inputOffset: _Optional[int] = ..., inputConstant: _Optional[bytes] = ...) -> None: ... + refIndex: int + def __init__(self, udf: _Optional[_Union[UserDefinedFunction, _Mapping]] = ..., inputOffset: _Optional[int] = ..., inputConstant: _Optional[bytes] = ..., refIndex: _Optional[int] = ...) -> None: ... class UserDefinedFunction(_message.Message): __slots__ = ("payload", "inputs", "window_index", "takes_row_as_input", "is_pandas_udf") diff --git a/flink-python/pyflink/fn_execution/table/operations.py b/flink-python/pyflink/fn_execution/table/operations.py index 5fa132db849212..bf0cd479ee408d 100644 --- a/flink-python/pyflink/fn_execution/table/operations.py +++ b/flink-python/pyflink/fn_execution/table/operations.py @@ -138,24 +138,55 @@ def __init__(self, serialized_fn, one_arg_optimization=False, one_result_optimiz def generate_func(self, serialized_fn): """ - Generates a lambda function based on udfs. - :param serialized_fn: serialized function which contains a list of the proto - representation of the Python :class:`ScalarFunction` - :return: the generated lambda function + Generates a UDF execution function. Uses sequential execution with result references + when refIndex is present (CSE mode), otherwise uses lambda-based approach. """ - scalar_functions, variable_dict, user_defined_funcs = reduce( - lambda x, y: ( - ','.join([x[0], y[0]]), - dict(chain(x[1].items(), y[1].items())), - x[2] + y[2]), - [operation_utils.extract_user_defined_function( + udf_infos = [ + operation_utils.extract_user_defined_function( udf, one_arg_optimization=self._one_arg_optimization) - for udf in serialized_fn.udfs]) + for udf in serialized_fn.udfs] + + variable_dict = {} + user_defined_funcs = [] + func_strs = [] + for func_str, var_dict, funcs in udf_infos: + variable_dict.update(var_dict) + user_defined_funcs.extend(funcs) + func_strs.append(func_str) + + # Check if any UDF uses refIndex via the results[N] pattern + has_ref = any('results[' in fs for fs in func_strs) + + if not has_ref: + # Keep original lambda-based approach for backward compatibility + scalar_functions = ','.join(func_strs) + if self._one_result_optimization: + func_str = 'lambda value: %s' % scalar_functions + else: + func_str = 'lambda value: [%s]' % scalar_functions + generate_func = eval(func_str, variable_dict) + self._generated_code = func_str + return generate_func, user_defined_funcs + + # Sequential execution: each UDF stores its result in results[N] so that + # later UDFs can reference it via results[N]. This enables full-tree CSE. + # + # exec() is required because multi-statement logic cannot be expressed as a + # lambda. The generated code is deterministic and built from trusted internal + # UDF descriptors only. + code_lines = ['def _sequential_execute(value):'] + code_lines.append(' results = [None] * %d' % len(func_strs)) + for i, fn in enumerate(func_strs): + code_lines.append(' results[%d] = %s' % (i, fn)) if self._one_result_optimization: - func_str = 'lambda value: %s' % scalar_functions + code_lines.append(' return results[0]') else: - func_str = 'lambda value: [%s]' % scalar_functions - generate_func = eval(func_str, variable_dict) + code_lines.append(' return results') + code = '\n'.join(code_lines) + + exec(code, variable_dict) + generate_func = variable_dict['_sequential_execute'] + self._generated_code = code return generate_func, user_defined_funcs diff --git a/flink-python/pyflink/fn_execution/tests/test_scalar_function_cse.py b/flink-python/pyflink/fn_execution/tests/test_scalar_function_cse.py new file mode 100644 index 00000000000000..a409eadae3b10e --- /dev/null +++ b/flink-python/pyflink/fn_execution/tests/test_scalar_function_cse.py @@ -0,0 +1,213 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +""" +Tests for the CSE sequential execution logic in ScalarFunctionOperation. + +When Python UDFs use refIndex (full-tree CSE), the execution switches from a +lambda-based approach to a sequential model where each UDF result is stored +in a `results` list for later reference. +""" + +import unittest + +import cloudpickle + +from pyflink.fn_execution import flink_fn_execution_pb2 +from pyflink.fn_execution.table.operations import ScalarFunctionOperation +from pyflink.table.udf import DelegatingScalarFunction + + +def _make_udf_payload(func): + """Serialize a Python function into a UDF payload (cloudpickle of DelegatingScalarFunction).""" + return cloudpickle.dumps(DelegatingScalarFunction(func)) + + +def _build_serialized_fn(udf_protos): + """ + Build a UserDefinedFunctions protobuf message from a list of UserDefinedFunction protos. + """ + serialized_fn = flink_fn_execution_pb2.UserDefinedFunctions() + serialized_fn.metric_enabled = False + for udf_proto in udf_protos: + serialized_fn.udfs.append(udf_proto) + return serialized_fn + + +def _make_udf_proto(payload, inputs): + """ + Create a UserDefinedFunction proto. + + :param payload: serialized UDF bytes + :param inputs: list of Input protos + """ + udf = flink_fn_execution_pb2.UserDefinedFunction() + udf.payload = payload + for inp in inputs: + udf.inputs.append(inp) + return udf + + +def _input_offset(offset): + """Create an Input proto with inputOffset.""" + inp = flink_fn_execution_pb2.Input() + inp.inputOffset = offset + return inp + + +def _input_ref_index(index): + """Create an Input proto with refIndex (CSE reference).""" + inp = flink_fn_execution_pb2.Input() + inp.refIndex = index + return inp + + +def _input_udf(udf_proto): + """Create an Input proto with a nested UDF (traditional chaining).""" + inp = flink_fn_execution_pb2.Input() + inp.udf.CopyFrom(udf_proto) + return inp + + +class TestScalarFunctionCse(unittest.TestCase): + """ + Tests for CSE sequential execution in ScalarFunctionOperation. + + Java side deduplicates Python UDF calls and encodes shared sub-expressions + as refIndex=N. Python side detects this and generates a sequential function + that stores intermediate results. Without refIndex, lambda codegen is used. + """ + + def test_cse_sequential_execution(self): + """ + Verify refIndex triggers sequential codegen and produces correct results. + + Scenario: udf1(x,y), udf2(udf1(x,y)), udf3(udf1(x,y), udf2(udf1(x,y))) + Flattened: [udf1(v[0],v[1]), udf2(results[0]), udf3(results[0],results[1])] + """ + udf1_payload = _make_udf_payload(lambda x, y: x + y) + udf2_payload = _make_udf_payload(lambda x: x * 3) + udf3_payload = _make_udf_payload(lambda a, b: a + b) + + udf1_proto = _make_udf_proto(udf1_payload, [_input_offset(0), _input_offset(1)]) + udf2_proto = _make_udf_proto(udf2_payload, [_input_ref_index(0)]) + udf3_proto = _make_udf_proto(udf3_payload, [_input_ref_index(0), _input_ref_index(1)]) + + serialized_fn = _build_serialized_fn([udf1_proto, udf2_proto, udf3_proto]) + + op = ScalarFunctionOperation(serialized_fn) + + # Verify the function name confirms CSE codegen path + self.assertEqual(op.func.__name__, '_sequential_execute') + + # Verify the actual generated code content from ScalarFunctionOperation + generated_code = op._generated_code + self.assertIn('def _sequential_execute(value):', generated_code) + self.assertIn('results = [None] * 3', generated_code) + # udf1 reads from input columns + self.assertIn('results[0] = ', generated_code) + self.assertIn('value[0]', generated_code) + self.assertIn('value[1]', generated_code) + # udf2 references udf1's result via results[0] + self.assertIn('results[1] = ', generated_code) + self.assertIn('results[0]', generated_code) + # udf3 references both udf1 and udf2's results + self.assertIn('results[2] = ', generated_code) + self.assertIn('results[1]', generated_code) + self.assertIn('return results', generated_code) + + # Verify computation results + result = op.func([3, 4]) + # udf1(3, 4) = 3 + 4 = 7 + # udf2(results[0]) = udf2(7) = 7 * 3 = 21 + # udf3(results[0], results[1]) = udf3(7, 21) = 7 + 21 = 28 + self.assertEqual(result[0], 7) + self.assertEqual(result[1], 21) + self.assertEqual(result[2], 28) + + # Verify idempotency + result2 = op.func([10, 5]) + # udf1(10, 5) = 15, udf2(15) = 45, udf3(15, 45) = 60 + self.assertEqual(result2[0], 15) + self.assertEqual(result2[1], 45) + self.assertEqual(result2[2], 60) + + def test_no_cse_lambda_execution(self): + """ + Verify that without refIndex, traditional lambda codegen is used. + + Scenario 1: Independent UDFs with only inputOffset. + Scenario 2: Traditional nested UDF encoding (udf as input, not refIndex). + """ + # --- Scenario 1: Independent UDFs, no CSE --- + udf1_payload = _make_udf_payload(lambda x: x * 2) + udf2_payload = _make_udf_payload(lambda x: x + 10) + + udf1_proto = _make_udf_proto(udf1_payload, [_input_offset(0)]) + udf2_proto = _make_udf_proto(udf2_payload, [_input_offset(1)]) + + serialized_fn = _build_serialized_fn([udf1_proto, udf2_proto]) + + op = ScalarFunctionOperation(serialized_fn) + + # Verify the function name confirms lambda codegen path + self.assertEqual(op.func.__name__, '') + + # Verify the actual generated code is a lambda expression (no _sequential_execute) + generated_code = op._generated_code + self.assertTrue(generated_code.startswith('lambda value:')) + self.assertNotIn('results[', generated_code) + self.assertNotIn('_sequential_execute', generated_code) + # Should contain direct value[N] references + self.assertIn('value[0]', generated_code) + self.assertIn('value[1]', generated_code) + + # Verify computation results + result = op.func([5, 3]) + # udf1(5) = 10, udf2(3) = 13 + self.assertEqual(result[0], 10) + self.assertEqual(result[1], 13) + + # --- Scenario 2: Traditional nested UDF encoding, no refIndex --- + udf1_payload_nested = _make_udf_payload(lambda x: x * 2) + udf2_payload_nested = _make_udf_payload(lambda x: x + 10) + + inner_udf1_proto = _make_udf_proto(udf1_payload_nested, [_input_offset(0)]) + outer_udf2_proto = _make_udf_proto(udf2_payload_nested, [_input_udf(inner_udf1_proto)]) + + serialized_fn2 = _build_serialized_fn([outer_udf2_proto]) + + op2 = ScalarFunctionOperation(serialized_fn2) + + # Verify the function name confirms lambda codegen path + self.assertEqual(op2.func.__name__, '') + + # Verify the actual generated code is a lambda with nested function call + generated_code2 = op2._generated_code + self.assertTrue(generated_code2.startswith('lambda value:')) + self.assertNotIn('results[', generated_code2) + # Should contain nested function call pattern (f_outer(f_inner(value[0]))) + self.assertIn('value[0]', generated_code2) + + # Verify computation results + result2 = op2.func([5]) + # udf2(udf1(5)) = udf2(10) = 20 + self.assertEqual(result2[0], 20) + +if __name__ == '__main__': + unittest.main() diff --git a/flink-python/pyflink/fn_execution/utils/operation_utils.py b/flink-python/pyflink/fn_execution/utils/operation_utils.py index 5af5f6133a53fe..3ee382e93f1cca 100644 --- a/flink-python/pyflink/fn_execution/utils/operation_utils.py +++ b/flink-python/pyflink/fn_execution/utils/operation_utils.py @@ -134,6 +134,9 @@ def _extract_input(args) -> Tuple[str, Dict, List]: else: # the input argument is a column of the input row args_str.append("value[%s]" % arg.inputOffset) + elif arg.HasField("refIndex"): + # Reference to a previously computed UDF result (CSE). + args_str.append("results[%s]" % arg.refIndex) else: # the input argument is a constant value constant_value_name, parsed_constant_value = \ diff --git a/flink-python/pyflink/proto/flink-fn-execution.proto b/flink-python/pyflink/proto/flink-fn-execution.proto index 4ab5616b011b88..6373bfcb5d171f 100644 --- a/flink-python/pyflink/proto/flink-fn-execution.proto +++ b/flink-python/pyflink/proto/flink-fn-execution.proto @@ -39,6 +39,8 @@ message Input { UserDefinedFunction udf = 1; int32 inputOffset = 2; bytes inputConstant = 3; + // Reference to a previously computed UDF result (enables cross-subtree CSE). + int32 refIndex = 4; } } diff --git a/flink-python/pyflink/table/tests/test_udf.py b/flink-python/pyflink/table/tests/test_udf.py index 87d1cb986cd1b9..8838273d7b31a8 100644 --- a/flink-python/pyflink/table/tests/test_udf.py +++ b/flink-python/pyflink/table/tests/test_udf.py @@ -1015,6 +1015,29 @@ def all_different(vals): self.assertNotEqual(vals[0], vals[1]), ), ), + # --- Filter (WHERE) CSE cases --- + ( + "Det(s) in both WHERE and SELECT - CSE across condition and projection", + "SELECT Det(s), Det(s) FROM SourceTable WHERE Det(s) IS NOT NULL", + all_equal, + ), + ( + "Det(s) in WHERE and SELECT with expression - same UDF reused", + "SELECT Det(s), Det(s) || '_suffix' FROM SourceTable WHERE Det(s) IS NOT NULL", + lambda vals: ( + self.assertEqual(len(vals), 2), + self.assertTrue(vals[0].startswith("HELLO_")), + self.assertEqual(vals[1], vals[0] + "_suffix"), + ), + ), + ( + "Nondet(s) in WHERE and SELECT - merged by RexProgram CSE in split pipeline", + "SELECT Nondet(s), Nondet(s) FROM SourceTable WHERE Nondet(s) IS NOT NULL", + # Note: Ideally these should differ, but RexProgram.create merges + # structurally-equal expressions (including non-deterministic ones) when + # the split condition rule creates a two-level Calc structure. + all_equal, + ), ] for desc, sql, verify_fn in cases: diff --git a/flink-python/src/main/java/org/apache/flink/python/util/ProtoUtils.java b/flink-python/src/main/java/org/apache/flink/python/util/ProtoUtils.java index 2745b36cb56de8..5c8e3016a665af 100644 --- a/flink-python/src/main/java/org/apache/flink/python/util/ProtoUtils.java +++ b/flink-python/src/main/java/org/apache/flink/python/util/ProtoUtils.java @@ -180,6 +180,8 @@ public static FlinkFnApi.UserDefinedFunction createUserDefinedFunctionProto( FlinkFnApi.Input.Builder inputProto = FlinkFnApi.Input.newBuilder(); if (input instanceof PythonFunctionInfo) { inputProto.setUdf(createUserDefinedFunctionProto((PythonFunctionInfo) input)); + } else if (input instanceof PythonFunctionInfo.ResultRef) { + inputProto.setRefIndex(((PythonFunctionInfo.ResultRef) input).index); } else if (input instanceof Integer) { inputProto.setInputOffset((Integer) input); } else { diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java index c82bea6c219aad..69fb020986a9a2 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java @@ -36,11 +36,58 @@ public class PythonFunctionInfo implements Serializable { private final PythonFunction pythonFunction; /** - * The input arguments, it could be an input offset of the input row or the execution result of - * another python function described as PythonFunctionInfo. + * The input arguments. It could be one of the following: + * + *

    + *
  • {@link Integer} – an input offset of the input row + *
  • {@link PythonFunctionInfo} – the execution result of another python function (nested + * call) + *
  • {@code byte[]} – a constant value + *
  • {@link ResultRef} – a reference to the result of a previously computed function in the + * flattened UDF list (used for cross-subtree CSE) + *
*/ private Object[] inputs; + /** + * Represents a reference to the result of a previously computed function in a flattened UDF + * list. This enables cross-subtree Common Subexpression Elimination (CSE) where a nested UDF + * call that appears in multiple positions can be computed once and its result referenced by + * index. + */ + public static class ResultRef implements Serializable { + private static final long serialVersionUID = 1L; + + /** The index of the referenced function in the flattened UDF list. */ + public final int index; + + public ResultRef(int index) { + this.index = index; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ResultRef resultRef = (ResultRef) o; + return index == resultRef.index; + } + + @Override + public int hashCode() { + return Integer.hashCode(index); + } + + @Override + public String toString() { + return "ResultRef(" + index + ")"; + } + } + public PythonFunctionInfo(PythonFunction pythonFunction, Object[] inputs) { this.pythonFunction = Preconditions.checkNotNull(pythonFunction); this.inputs = Preconditions.checkNotNull(inputs); diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java index 217771feca400b..ed00e10717ab77 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java @@ -44,7 +44,6 @@ import org.apache.flink.table.planner.plan.nodes.exec.utils.CommonPythonUtil; import org.apache.flink.table.planner.plan.nodes.exec.utils.ExecNodeUtil; import org.apache.flink.table.planner.plan.utils.PythonUtil; -import org.apache.flink.table.planner.utils.ShortcutUtils; import org.apache.flink.table.runtime.generated.GeneratedProjection; import org.apache.flink.table.runtime.operators.CodeGenOperatorFactory; import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; @@ -64,6 +63,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -95,6 +95,18 @@ public abstract class CommonExecPythonCalc extends ExecNodeBase @JsonProperty(FIELD_NAME_PROJECTION) private final List projection; + /** + * Total number of flattened Python UDF calls across all projection trees (-1 = not yet + * computed). + */ + private int cseFlattenedCount = -1; + + /** Number of unique Python UDF calls after deduplication (-1 = not yet computed). */ + private int cseDedupedCount = -1; + + /** CSE annotation for plan description (null = not yet computed). */ + private String cseAnnotation = null; + public CommonExecPythonCalc( int id, ExecNodeContext context, @@ -135,40 +147,6 @@ protected Transformation translateToPlanInternal( // Common Sub-expression Elimination for Python UDFs // ------------------------------------------------------------------------- - /** - * Deduplicates deterministic Python RexCalls by their string digest. Only deterministic calls - * can be safely reused; non-deterministic calls must be evaluated independently each time. - * - * @return a tuple of (unique calls list, mapping from original index to deduplicated index) - */ - @VisibleForTesting - Tuple2, int[]> deduplicatePythonCalls(List pythonRexCalls) { - LinkedHashMap digestToIndex = new LinkedHashMap<>(); - List uniqueCalls = new ArrayList<>(); - int[] originalToDedup = new int[pythonRexCalls.size()]; - - for (int i = 0; i < pythonRexCalls.size(); i++) { - RexCall call = pythonRexCalls.get(i); - String digest = call.toString(); - - boolean canReuse = ShortcutUtils.isDeterministicThroughProgram(call, null); - - Integer existingIndex = digestToIndex.get(digest); - if (canReuse && existingIndex != null) { - // Deterministic duplicate — reuse the existing call - originalToDedup[i] = existingIndex; - } else { - int newPos = uniqueCalls.size(); - if (canReuse) { - digestToIndex.put(digest, newPos); - } - uniqueCalls.add(call); - originalToDedup[i] = newPos; - } - } - return Tuple2.of(uniqueCalls, originalToDedup); - } - /** * Creates a projection operator that expands deduplicated results back to the expected output * schema. The Python operator produces [forwarded_fields..., unique_calls...], and this @@ -283,14 +261,29 @@ private OneInputTransformation createPythonOneInputTransformat RowType inputType = ((InternalTypeInfo) inputTransform.getOutputType()).toRowType(); - // Deduplicate identical deterministic Python function calls - Tuple2, int[]> dedupResult = deduplicatePythonCalls(pythonRexCalls); - List uniquePythonRexCalls = dedupResult.f0; - int[] originalToDedup = dedupResult.f1; - boolean needsExpansionProjection = originalToDedup.length != uniquePythonRexCalls.size(); + // Deduplicate identical deterministic Python function calls (full-tree CSE) + PythonCallCseResult cseResult = PythonCallDeduplicator.deduplicate(pythonRexCalls); + List uniquePythonRexCalls = cseResult.getUniqueCalls(); + int[] originalToDedup = cseResult.getOriginalToDedupMapping(); + Map refMap = cseResult.getRefMap(); + boolean needsExpansionProjection = cseResult.needsExpansionProjection(); + + // Compute and log CSE statistics + cseFlattenedCount = PythonCallDeduplicator.countFlattenedCalls(pythonRexCalls); + cseDedupedCount = uniquePythonRexCalls.size(); + cseAnnotation = buildCseTopLevelAnnotation(); + if (cseFlattenedCount > cseDedupedCount && LOG.isDebugEnabled()) { + LOG.debug( + "Python CSE in {}: {} UDF calls flattened to {} unique ({} saved, {}% reduction)", + getDescription(), + cseFlattenedCount, + cseDedupedCount, + cseFlattenedCount - cseDedupedCount, + (cseFlattenedCount - cseDedupedCount) * 100 / cseFlattenedCount); + } Tuple2 extractResult = - extractPythonScalarFunctionInfos(uniquePythonRexCalls, classLoader); + extractPythonScalarFunctionInfos(uniquePythonRexCalls, classLoader, refMap); int[] pythonUdfInputOffsets = extractResult.f0; PythonFunctionInfo[] pythonFunctionInfos = extractResult.f1; InternalTypeInfo pythonOperatorInputTypeInfo = @@ -340,14 +333,14 @@ private OneInputTransformation createPythonOneInputTransformat } private Tuple2 extractPythonScalarFunctionInfos( - List rexCalls, ClassLoader classLoader) { + List rexCalls, ClassLoader classLoader, Map refMap) { LinkedHashMap inputNodes = new LinkedHashMap<>(); PythonFunctionInfo[] pythonFunctionInfos = rexCalls.stream() .map( x -> CommonPythonUtil.createPythonFunctionInfo( - x, inputNodes, classLoader)) + x, inputNodes, classLoader, refMap)) .collect(Collectors.toList()) .toArray(new PythonFunctionInfo[rexCalls.size()]); @@ -476,4 +469,50 @@ private OneInputStreamOperator getPythonScalarFunctionOperator throw new TableException("Python Scalar Function Operator constructed failed.", e); } } + + @Override + public String getDescription() { + ensureCseStatsComputed(); + if (cseAnnotation != null && !cseAnnotation.isEmpty()) { + return super.getDescription() + cseAnnotation; + } + return super.getDescription(); + } + + /** + * Lazily computes CSE statistics for plan visualization. Called from {@link #getDescription()} + * which may be invoked before {@link #translateToPlanInternal}. + */ + private void ensureCseStatsComputed() { + if (cseFlattenedCount >= 0) { + return; + } + + List pythonCalls = + projection.stream() + .filter(x -> x instanceof RexCall) + .map(x -> (RexCall) x) + .filter(PythonUtil::isPythonCall) + .collect(Collectors.toList()); + + if (pythonCalls.isEmpty()) { + cseFlattenedCount = 0; + cseDedupedCount = 0; + cseAnnotation = ""; + return; + } + + cseFlattenedCount = PythonCallDeduplicator.countFlattenedCalls(pythonCalls); + cseDedupedCount = PythonCallDeduplicator.deduplicate(pythonCalls).getUniqueCount(); + cseAnnotation = buildCseTopLevelAnnotation(); + } + + /** + * Builds a CSE annotation showing top-level reuse relationships. Example: {@code (CSE: $2->$1, + * $3->$2)} indicates $2 reuses $1 as a sub-expression. + */ + private String buildCseTopLevelAnnotation() { + List outputFieldNames = ((RowType) getOutputType()).getFieldNames(); + return PythonCallDeduplicator.buildCseTopLevelAnnotation(projection, outputFieldNames); + } } diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/PythonCallCseResult.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/PythonCallCseResult.java new file mode 100644 index 00000000000000..ed9b4a5d045a68 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/PythonCallCseResult.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.common; + +import org.apache.flink.annotation.Internal; + +import org.apache.calcite.rex.RexCall; + +import java.util.List; +import java.util.Map; + +/** Encapsulates the result of Python UDF call Common Sub-expression Elimination (CSE). */ +@Internal +public class PythonCallCseResult { + + /** Unique Python UDF call list after deduplication (flattened and deduplicated). */ + private final List uniqueCalls; + + /** + * Mapping from original projection index to deduplicated unique call index. + * + *

For example, given {@code SELECT udf1(x), udf1(x)}, the mapping is {@code [0, 0]}, + * indicating both projections point to the 0th call in uniqueCalls. + */ + private final int[] originalToDedupMapping; + + /** + * Sub-expression cross-reference map from unique RexCall to its index in uniqueCalls. + * + *

Used when building {@code PythonFunctionInfo} to allow nested Python UDF calls to reuse + * already-computed sub-expression results via reference instead of recomputation. + */ + private final Map refMap; + + public PythonCallCseResult( + List uniqueCalls, int[] originalToDedupMapping, Map refMap) { + this.uniqueCalls = uniqueCalls; + this.originalToDedupMapping = originalToDedupMapping; + this.refMap = refMap; + } + + /** Returns the unique Python UDF call list after deduplication. */ + public List getUniqueCalls() { + return uniqueCalls; + } + + /** Returns the mapping array from original projection index to deduplicated index. */ + public int[] getOriginalToDedupMapping() { + return originalToDedupMapping; + } + + /** Returns the sub-expression cross-reference map. */ + public Map getRefMap() { + return refMap; + } + + /** Returns whether an expansion projection is needed to restore the original output schema. */ + public boolean needsExpansionProjection() { + return originalToDedupMapping.length != uniqueCalls.size(); + } + + /** + * Returns the number of top-level Python UDF calls in the original projection (before + * deduplication). This equals the length of the originalToDedupMapping array. + */ + public int getOriginalCount() { + return originalToDedupMapping.length; + } + + /** Returns the number of unique calls after deduplication. */ + public int getUniqueCount() { + return uniqueCalls.size(); + } +} diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/PythonCallDeduplicator.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/PythonCallDeduplicator.java new file mode 100644 index 00000000000000..1544d1590e8861 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/PythonCallDeduplicator.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.common; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.table.planner.plan.utils.PythonUtil; +import org.apache.flink.table.planner.utils.ShortcutUtils; + +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; + +/** + * Utility for Python UDF Common Sub-expression Elimination (CSE). + * + *

Deduplicates Python UDF calls to reduce cross-process (JVM ↔ Python Worker) overhead. + * Workflow: flatten nested call trees → deduplicate by structural equivalence → build index + * mappings and cross-reference maps. + */ +@Internal +public class PythonCallDeduplicator { + + /** + * Recursively collects all deterministic Python UDF calls from a call tree in DFS post-order. + * + *

Post-order ensures child results are computed before parents that reference them via + * refIndex. Non-deterministic children are NOT flattened to prevent incorrect sharing. + */ + @VisibleForTesting + static List collectAllPythonUdfCalls(RexCall root) { + List result = new ArrayList<>(); + for (RexNode operand : root.getOperands()) { + if (operand instanceof RexCall && PythonUtil.isPythonCall((RexCall) operand)) { + RexCall childCall = (RexCall) operand; + // Only flatten deterministic child calls for CSE. + // Non-deterministic calls must remain nested to avoid incorrect sharing. + if (ShortcutUtils.isDeterministicThroughProgram(childCall, null)) { + result.addAll(collectAllPythonUdfCalls(childCall)); + } + } + } + result.add(root); + return result; + } + + /** + * Deduplicates Python UDF calls including nested sub-expressions (full-tree CSE). + * + *

All calls from projection trees are flattened into a single list, then deduplicated by + * structural equivalence. This enables cross-subtree reuse: e.g., in {@code SELECT udf1(x), + * udf2(udf1(x))}, the inner {@code udf1(x)} is computed only once. + */ + public static PythonCallCseResult deduplicate(List pythonRexCalls) { + // Step 1: Flatten — collect all Python UDF calls from all projection trees (post-order) + int[] projectionRootPositions = new int[pythonRexCalls.size()]; + List allCalls = new ArrayList<>(); + for (int i = 0; i < pythonRexCalls.size(); i++) { + List subtreeCalls = collectAllPythonUdfCalls(pythonRexCalls.get(i)); + // In post-order, the root call is always the last element in the subtree list + projectionRootPositions[i] = allCalls.size() + subtreeCalls.size() - 1; + allCalls.addAll(subtreeCalls); + } + + // Step 2: Deduplicate the flattened list + LinkedHashMap callToIndex = new LinkedHashMap<>(); + List uniqueCalls = new ArrayList<>(); + int[] allToUnique = new int[allCalls.size()]; + + for (int i = 0; i < allCalls.size(); i++) { + RexCall call = allCalls.get(i); + boolean canReuse = ShortcutUtils.isDeterministicThroughProgram(call, null); + Integer existingIndex = callToIndex.get(call); + if (canReuse && existingIndex != null) { + allToUnique[i] = existingIndex; + } else { + int newPos = uniqueCalls.size(); + if (canReuse) { + callToIndex.put(call, newPos); + } + uniqueCalls.add(call); + allToUnique[i] = newPos; + } + } + + // Step 3: Build mapping from original projection positions to flattened indices + int[] originalToDedup = new int[pythonRexCalls.size()]; + for (int i = 0; i < pythonRexCalls.size(); i++) { + originalToDedup[i] = allToUnique[projectionRootPositions[i]]; + } + + // Step 4: Build refMap for sub-expression cross-referencing. + // putIfAbsent preserves the first occurrence index, ensuring a parent references + // its own child rather than a later structurally-equal duplicate. + LinkedHashMap refMap = new LinkedHashMap<>(); + for (int i = 0; i < uniqueCalls.size(); i++) { + refMap.putIfAbsent(uniqueCalls.get(i), i); + } + + return new PythonCallCseResult(uniqueCalls, originalToDedup, refMap); + } + + /** Counts the total number of flattened Python UDF calls. */ + public static int countFlattenedCalls(List pythonRexCalls) { + int total = 0; + for (RexCall call : pythonRexCalls) { + total += collectAllPythonUdfCalls(call).size(); + } + return total; + } + + /** + * Builds a CSE annotation showing top-level reuse relationships. + * + *

Example: given {@code SumFun(f1,f2) AS $1, SumFun(SumFun(f1,f2),f1) AS $2}, produces + * {@code (CSE: $2->$1)} indicating $2's inner expression reuses $1. + */ + public static String buildCseTopLevelAnnotation( + List projection, List outputFieldNames) { + // Find all Python UDF RexCall indices in projection + List callIndices = new ArrayList<>(); + for (int i = 0; i < projection.size(); i++) { + if (projection.get(i) instanceof RexCall + && PythonUtil.isPythonCall((RexCall) projection.get(i))) { + callIndices.add(i); + } + } + + if (callIndices.size() <= 1) { + return ""; + } + + List parts = new ArrayList<>(); + for (int i = 1; i < callIndices.size(); i++) { + int idx = callIndices.get(i); + RexCall call = (RexCall) projection.get(idx); + for (RexNode operand : call.getOperands()) { + if (operand instanceof RexCall && PythonUtil.isPythonCall((RexCall) operand)) { + for (int j = 0; j < i; j++) { + int prevIdx = callIndices.get(j); + if (operand.equals(projection.get(prevIdx))) { + parts.add( + outputFieldNames.get(idx) + + "->" + + outputFieldNames.get(prevIdx)); + break; + } + } + } + } + } + + if (parts.isEmpty()) { + return ""; + } + return " (CSE: " + String.join(", ", parts) + ")"; + } + + private PythonCallDeduplicator() { + // Utility class, no instantiation + } +} diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java index d82a37f5e7c8a0..686e18f3e37c9c 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java @@ -49,6 +49,7 @@ import org.apache.flink.table.planner.plan.schema.TimeIndicatorRelDataType; import org.apache.flink.table.planner.plan.utils.AggregateInfo; import org.apache.flink.table.planner.plan.utils.AggregateInfoList; +import org.apache.flink.table.planner.plan.utils.PythonUtil; import org.apache.flink.table.runtime.dataview.DataViewSpec; import org.apache.flink.table.runtime.dataview.ListViewSpec; import org.apache.flink.table.runtime.dataview.MapViewSpec; @@ -83,6 +84,7 @@ import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -122,6 +124,20 @@ public static Configuration extractPythonConfiguration( public static PythonFunctionInfo createPythonFunctionInfo( RexCall pythonRexCall, Map inputNodes, ClassLoader classLoader) { + return createPythonFunctionInfo( + pythonRexCall, inputNodes, classLoader, Collections.emptyMap()); + } + + /** + * Creates a {@link PythonFunctionInfo} with optional refMap for cross-subtree CSE. When a + * nested operand is found in the refMap, a {@link PythonFunctionInfo.ResultRef} is emitted + * instead of a nested {@link PythonFunctionInfo}. + */ + public static PythonFunctionInfo createPythonFunctionInfo( + RexCall pythonRexCall, + Map inputNodes, + ClassLoader classLoader, + Map refMap) { SqlOperator operator = pythonRexCall.getOperator(); try { if (operator instanceof ScalarSqlFunction) { @@ -129,19 +145,22 @@ public static PythonFunctionInfo createPythonFunctionInfo( pythonRexCall, inputNodes, ((ScalarSqlFunction) operator).scalarFunction(), - classLoader); + classLoader, + refMap); } else if (operator instanceof TableSqlFunction) { return createPythonFunctionInfo( pythonRexCall, inputNodes, ((TableSqlFunction) operator).udtf(), - classLoader); + classLoader, + refMap); } else if (operator instanceof BridgingSqlFunction) { return createPythonFunctionInfo( pythonRexCall, inputNodes, ((BridgingSqlFunction) operator).getDefinition(), - classLoader); + classLoader, + refMap); } } catch (InvocationTargetException | IllegalAccessException e) { throw new TableException("Method pickleValue accessed failed. ", e); @@ -428,12 +447,21 @@ private static PythonFunctionInfo createPythonFunctionInfo( RexCall pythonRexCall, Map inputNodes, FunctionDefinition functionDefinition, - ClassLoader classLoader) + ClassLoader classLoader, + Map refMap) throws InvocationTargetException, IllegalAccessException { ArrayList inputs = new ArrayList<>(); for (RexNode operand : pythonRexCall.getOperands()) { if (operand instanceof RexCall) { RexCall childPythonRexCall = (RexCall) operand; + // CSE: reference pre-computed result instead of creating nested info. + if (!refMap.isEmpty() && PythonUtil.isPythonCall(childPythonRexCall)) { + Integer refIndex = refMap.get(childPythonRexCall); + if (refIndex != null) { + inputs.add(new PythonFunctionInfo.ResultRef(refIndex)); + continue; + } + } if (childPythonRexCall.getOperator() instanceof SqlCastFunction && childPythonRexCall.getOperands().get(0) instanceof RexInputRef && childPythonRexCall.getOperands().get(0).getType() @@ -441,7 +469,8 @@ private static PythonFunctionInfo createPythonFunctionInfo( operand = childPythonRexCall.getOperands().get(0); } else { PythonFunctionInfo argPythonInfo = - createPythonFunctionInfo(childPythonRexCall, inputNodes, classLoader); + createPythonFunctionInfo( + childPythonRexCall, inputNodes, classLoader, refMap); inputs.add(argPythonInfo); continue; } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala index e840808b5e1c24..b14b10e02d5b83 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala @@ -391,6 +391,7 @@ object FlinkBatchRuleSets { PythonCalcSplitRule.SPLIT_CONDITION_REX_FIELD, PythonCalcSplitRule.SPLIT_PROJECTION_REX_FIELD, PythonCalcSplitRule.SPLIT_CONDITION, + PythonCalcSplitRule.CONDITION_PROJECTION_CSE, PythonCalcSplitRule.SPLIT_PROJECT, PythonCalcSplitRule.SPLIT_PANDAS_IN_PROJECT, PythonCalcSplitRule.EXPAND_PROJECT, diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala index bc06d2d688baac..3a8d0e41402d67 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala @@ -408,6 +408,8 @@ object FlinkStreamRuleSets { PythonCalcSplitRule.SPLIT_PROJECTION_REX_FIELD, // Avoids dealing with a python call in the condition. PythonCalcSplitRule.SPLIT_CONDITION, + // Deduplicates Python UDF calls shared between condition and projection. + PythonCalcSplitRule.CONDITION_PROJECTION_CSE, // Avoids dealing with Java calls in the same Calc as python calls. PythonCalcSplitRule.SPLIT_PROJECT, // Splits calcs which contain both general Python functions and pandas Python functions diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRule.scala index 7e54ffbca03c9c..15850466bcdf58 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRule.scala @@ -96,11 +96,13 @@ class PythonRemoteCallFinder extends RemoteCallFinder { object PythonCalcSplitRule { /** - * These rules should be applied sequentially in the order of SPLIT_CONDITION, SPLIT_PROJECT, - * SPLIT_PANDAS_IN_PROJECT, EXPAND_PROJECT, PUSH_CONDITION and REWRITE_PROJECT. + * These rules should be applied sequentially in the order of SPLIT_CONDITION, + * CONDITION_PROJECTION_CSE, SPLIT_PROJECT, SPLIT_PANDAS_IN_PROJECT, EXPAND_PROJECT, + * PUSH_CONDITION and REWRITE_PROJECT. */ private val callFinder = new PythonRemoteCallFinder() val SPLIT_CONDITION: RelOptRule = new RemoteCalcSplitConditionRule(callFinder) + val CONDITION_PROJECTION_CSE: RelOptRule = new RemoteCalcConditionProjectionCseRule(callFinder) val SPLIT_PROJECT: RelOptRule = new RemoteCalcSplitProjectionRule(callFinder) val SPLIT_PANDAS_IN_PROJECT: RelOptRule = new PythonCalcSplitPandasInProjectionRule(callFinder) val SPLIT_PROJECTION_REX_FIELD: RelOptRule = new RemoteCalcSplitProjectionRexFieldRule(callFinder) diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.scala new file mode 100644 index 00000000000000..c2b42e000fa806 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.scala @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.table.planner.plan.rules.logical + +import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCalc +import org.apache.flink.table.planner.utils.ShortcutUtils + +import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall} +import org.apache.calcite.plan.RelOptRule.{any, operand} +import org.apache.calcite.rex.{RexBuilder, RexCall, RexInputRef, RexNode, RexShuttle} + +import scala.collection.JavaConversions._ + +/** + * Rule that eliminates common Python UDF sub-expressions between the condition and projection of a + * Calc node. + * + *

After [[RemoteCalcSplitConditionRule]] splits a Calc with Python UDFs in its condition, the + * result is a two-level Calc structure: + * + * 

 TopCalc(projection=[pyFunc(a, b) + 1, pyFunc(a, b) + 2], condition=[$2 > 0])
+ * BottomCalc(projection=[a, b, pyFunc(a, b) AS f0])
+ * + *

The TopCalc's projection still contains `pyFunc(a, b)` which is structurally identical to the + * already-computed `f0` in the BottomCalc. This rule detects such duplicates and rewrites the + * TopCalc's projection to reference the BottomCalc's output directly: + * + * 

 TopCalc(projection=[$2 + 1, $2 + 2], condition=[$2 > 0]) BottomCalc(projection=[a, b,
+ * pyFunc(a, b) AS f0])
+ * + *

This ensures that when the plan is later split into separate PythonCalc nodes, the same UDF + * call is not computed twice across condition and projection. + * + * @param callFinder + * the finder used to identify remote (Python) function calls + */ +class RemoteCalcConditionProjectionCseRule(private val callFinder: RemoteCalcCallFinder) + extends RelOptRule( + operand(classOf[FlinkLogicalCalc], operand(classOf[FlinkLogicalCalc], any)), + "RemoteCalcConditionProjectionCseRule") { + + override def matches(call: RelOptRuleCall): Boolean = { + val topCalc: FlinkLogicalCalc = call.rel(0) + val bottomCalc: FlinkLogicalCalc = call.rel(1) + + // This rule only applies when the top calc has a condition — it deduplicates Python UDF + // calls shared between condition and projection. Without a condition, there is nothing + // to deduplicate, and firing would interfere with other rules (e.g. PythonMapMergeRule). + if (topCalc.getProgram.getCondition == null) { + return false + } + + val topProjects = topCalc.getProgram.getProjectList.map(topCalc.getProgram.expandLocalRef) + + // The bottom calc must have deterministic Python UDF calls in its projection + // (from condition extraction). Non-deterministic calls must NOT be shared. + val bottomProjects = + bottomCalc.getProgram.getProjectList.map(bottomCalc.getProgram.expandLocalRef) + val bottomPythonCalls: Map[RexNode, Int] = bottomProjects.zipWithIndex + .filter { + case (node, _) => + callFinder.containsRemoteCall(node) && + ShortcutUtils.isDeterministicThroughProgram(node, null) + } + .map { case (node, idx) => (node, idx) } + .toMap + + if (bottomPythonCalls.isEmpty) { + return false + } + + // The top calc's projection must contain Python UDF calls that are structurally equal + // to calls already computed in the bottom calc's projection + topProjects.exists(containsCallMatchingBottom(_, bottomPythonCalls)) + } + + override def onMatch(call: RelOptRuleCall): Unit = { + val topCalc: FlinkLogicalCalc = call.rel(0) + val bottomCalc: FlinkLogicalCalc = call.rel(1) + val rexBuilder: RexBuilder = call.builder().getRexBuilder + + val topProjects = topCalc.getProgram.getProjectList.map(topCalc.getProgram.expandLocalRef) + val topCondition = Option(topCalc.getProgram.getCondition) + .map(topCalc.getProgram.expandLocalRef) + + val bottomProjects = + bottomCalc.getProgram.getProjectList.map(bottomCalc.getProgram.expandLocalRef) + + // Build a map from deterministic Python UDF call (in bottom calc's projection) to its + // output index. Non-deterministic calls are excluded to prevent incorrect sharing. + val bottomPythonCalls: Map[RexNode, Int] = bottomProjects.zipWithIndex + .filter { + case (node, _) => + callFinder.containsRemoteCall(node) && + ShortcutUtils.isDeterministicThroughProgram(node, null) + } + .map { case (node, idx) => (node, idx) } + .toMap + + // Rewrite the top calc's projection: replace Python UDF calls that match bottom calc's + // projection with RexInputRef pointing to the bottom calc's output position + val rewriter = new CseRewriteShuttle(bottomPythonCalls, bottomCalc.getRowType) + val newTopProjects: Seq[RexNode] = topProjects.map(_.accept(rewriter)) + + // Check if any rewriting actually happened + if (!rewriter.hasRewritten) { + return + } + + // Build the new top calc with rewritten projections + val newTopCalc = topCalc.copy( + topCalc.getTraitSet, + bottomCalc, + org.apache.calcite.rex.RexProgram.create( + bottomCalc.getRowType, + newTopProjects.toList, + topCondition.orNull, + topCalc.getRowType, + rexBuilder + ) + ) + + call.transformTo(newTopCalc) + } + + /** + * Checks if a RexNode contains a Python UDF call that is structurally equal to one of the calls + * in the bottom calc's projection. + */ + private def containsCallMatchingBottom( + node: RexNode, + bottomPythonCalls: Map[RexNode, Int]): Boolean = { + node match { + case call: RexCall => + if (callFinder.isRemoteCall(call) && bottomPythonCalls.contains(call)) { + true + } else { + call.getOperands.exists(containsCallMatchingBottom(_, bottomPythonCalls)) + } + case _ => false + } + } + + // Consider the rules to be equal if they are the same class and their call finders are the same + override def equals(obj: Any): Boolean = { + obj match { + case other: RemoteCalcConditionProjectionCseRule => + super.equals(other) && callFinder.getClass.equals(other.callFinder.getClass) + case _ => false + } + } + + override def hashCode(): Int = { + super.hashCode() * 31 + callFinder.getClass.hashCode() + } +} + +/** + * A RexShuttle that replaces Python UDF calls in the top calc's projection with RexInputRef + * pointing to the bottom calc's output position where the same call was already computed. + */ +private class CseRewriteShuttle( + bottomPythonCalls: Map[RexNode, Int], + bottomRowType: org.apache.calcite.rel.`type`.RelDataType) + extends RexShuttle { + + private var _hasRewritten: Boolean = false + + def hasRewritten: Boolean = _hasRewritten + + override def visitCall(call: RexCall): RexNode = { + // If this call matches a bottom calc's Python UDF output, replace with InputRef + bottomPythonCalls.get(call) match { + case Some(idx) => + _hasRewritten = true + new RexInputRef(idx, bottomRowType.getFieldList.get(idx).getType) + case None => + // Continue rewriting operands + super.visitCall(call) + } + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalcRefReuseTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalcRefReuseTest.java index 3dcd292fe94a37..bcf0ff55553d76 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalcRefReuseTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalcRefReuseTest.java @@ -18,7 +18,6 @@ package org.apache.flink.table.planner.plan.nodes.exec.common; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.table.planner.calcite.FlinkTypeFactory; import org.apache.flink.table.planner.calcite.FlinkTypeSystem; @@ -35,6 +34,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -46,19 +46,9 @@ import static org.assertj.core.api.Assertions.assertThat; -/** - * Tests for the reference reuse (deduplication) logic in {@link CommonExecPythonCalc}. - * - *

This test verifies that the {@code deduplicatePythonCalls} method correctly identifies and - * deduplicates structurally identical deterministic RexCalls, while preserving non-deterministic - * calls independently. - */ +/** Tests for {@link PythonCallDeduplicator} and ref-reuse logic in {@link CommonExecPythonCalc}. */ class CommonExecPythonCalcRefReuseTest { - // ------------------------------------------------------------------------- - // Parameterized tests for deduplicatePythonCalls - // ------------------------------------------------------------------------- - @ParameterizedTest(name = "{0}") @MethodSource("inputForDeduplicatePythonCalls") void testDeduplicatePythonCalls( @@ -66,11 +56,14 @@ void testDeduplicatePythonCalls( List calls, int expectedUniqueCount, int[] expectedMapping) { - CommonExecPythonCalc calc = new TestPythonCalc(Collections.emptyList()); - Tuple2, int[]> result = calc.deduplicatePythonCalls(calls); + PythonCallCseResult result = PythonCallDeduplicator.deduplicate(calls); - assertThat(result.f0).as("unique call count").hasSize(expectedUniqueCount); - assertThat(result.f1).as("original-to-dedup mapping").containsExactly(expectedMapping); + assertThat(result.getUniqueCalls()).as("unique call count").hasSize(expectedUniqueCount); + assertThat(result.getOriginalToDedupMapping()) + .as("original-to-dedup mapping") + .containsExactly(expectedMapping); + assertThat(result.getRefMap()).as("ref map").isNotNull(); + assertThat(result.getRefMap()).as("ref map").isNotNull().isNotEmpty(); } static Stream inputForDeduplicatePythonCalls() { @@ -102,6 +95,17 @@ static Stream inputForDeduplicatePythonCalls() { RexNode innerPlus = rb.makeCall(SqlStdOperatorTable.PLUS, ref0, ref1); RexCall nestedCall1 = (RexCall) rb.makeCall(SqlStdOperatorTable.PLUS, innerPlus, innerPlus); RexCall nestedCall2 = (RexCall) rb.makeCall(SqlStdOperatorTable.PLUS, innerPlus, innerPlus); + RexCall innerPlusCall = (RexCall) rb.makeCall(SqlStdOperatorTable.PLUS, ref0, ref1); + + // Calls that differ only in return type (same operator, same operands). + // For non-CAST operators, RexCall.toString() omits the type suffix (digestWithType() + // returns false), so toString()-based dedup would incorrectly merge these. + // RexCall.equals() includes the return type and correctly keeps them separate. + RexCall plusIntType = + (RexCall) rb.makeCall(intTp, SqlStdOperatorTable.PLUS, Arrays.asList(ref0, ref1)); + RexCall plusBigintType = + (RexCall) + rb.makeCall(bigintTp, SqlStdOperatorTable.PLUS, Arrays.asList(ref0, ref1)); return Stream.of( Arguments.of( @@ -143,12 +147,54 @@ static Stream inputForDeduplicatePythonCalls() { "partial duplicates with different args", Arrays.asList(plusAB1, plusAB2, plusAC), 2, - new int[] {0, 0, 1})); + new int[] {0, 0, 1}), + Arguments.of( + "calls with identical structure but different return types are not deduplicated", + Arrays.asList(plusIntType, plusBigintType), + 2, + new int[] {0, 1}), + Arguments.of( + "full-tree CSE: nested call tree is flattened; inner call deduplicated when shared", + Arrays.asList(nestedCall1, innerPlusCall), + 2, + new int[] {0, 1})); } - // ------------------------------------------------------------------------- - // Parameterized tests for buildRefReuseDetailName - // ------------------------------------------------------------------------- + @Test + void testRefMapResolvesStructurallyEqualSubExpressions() { + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + FlinkTypeFactory factory = new FlinkTypeFactory(classLoader, FlinkTypeSystem.INSTANCE); + RexBuilder rb = new RexBuilder(factory); + RelDataType intTp = + factory.createTypeWithNullability(factory.createSqlType(SqlTypeName.INTEGER), true); + + RexNode ref0 = new RexInputRef(0, intTp); + RexNode ref1 = new RexInputRef(1, intTp); + RexNode ref2 = new RexInputRef(2, intTp); + + // outerCall = PLUS(PLUS(ref0, ref1), ref2) + RexCall innerCall = (RexCall) rb.makeCall(SqlStdOperatorTable.PLUS, ref0, ref1); + RexCall outerCall = (RexCall) rb.makeCall(SqlStdOperatorTable.PLUS, innerCall, ref2); + // standaloneCall is structurally equal to innerCall + RexCall standaloneCall = (RexCall) rb.makeCall(SqlStdOperatorTable.PLUS, ref0, ref1); + + assertThat(innerCall).isEqualTo(standaloneCall); + + PythonCallCseResult result = + PythonCallDeduplicator.deduplicate(Arrays.asList(outerCall, standaloneCall)); + + // outerCall and standaloneCall are different top-level calls → 2 unique + assertThat(result.getUniqueCalls()).hasSize(2); + assertThat(result.getOriginalToDedupMapping()).containsExactly(0, 1); + + // refMap maps structurally-equal expressions to the same index + java.util.Map refMap = result.getRefMap(); + assertThat(refMap).hasSize(2); + assertThat(refMap.get(outerCall)).isEqualTo(0); + assertThat(refMap.get(standaloneCall)).isEqualTo(1); + // innerCall resolves to same entry as standaloneCall (structural equality) + assertThat(refMap.get(innerCall)).isEqualTo(1); + } @ParameterizedTest(name = "{0}") @MethodSource("inputForBuildRefReuseDetailName") @@ -202,14 +248,7 @@ static Stream inputForBuildRefReuseDetailName() { "PythonCalcRefReuse(d=c)")); } - // ------------------------------------------------------------------------- - // Test helper: minimal concrete subclass of CommonExecPythonCalc - // ------------------------------------------------------------------------- - - /** - * A minimal concrete subclass of {@link CommonExecPythonCalc} used only for testing the - * deduplication logic. The abstract methods are not needed for ref-reuse unit tests. - */ + /** Minimal concrete subclass for testing deduplication logic. */ private static class TestPythonCalc extends CommonExecPythonCalc { TestPythonCalc(List projection) { diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.xml new file mode 100644 index 00000000000000..8715e60bd46d0f --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.xml @@ -0,0 +1,162 @@ + + + + + + + + + + + + + + + + + 0]]> + + + (pyFunc2($0, $2), 0)]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) +]]> + + + 0)]) + +- PythonCalc(select=[a, b, pyFunc2(a, c) AS f0]) + +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c]) +]]> + + + + + 0 AND pyFunc2(b, c) > 0]]> + + + (pyFunc1($0, $1), 0), >(pyFunc2($1, $2), 0))]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) +]]> + + + 0) AND (f1 > 0))]) ++- PythonCalc(select=[a, b, c, pyFunc1(a, b) AS f0, pyFunc2(b, c) AS f1]) + +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c]) +]]> + + + + + 0]]> + + + (pyFunc1($0, $1), 0)]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) +]]> + + + 0)]) ++- PythonCalc(select=[a, b, pyFunc1(a, b) AS f0]) + +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c]) +]]> + + + + + 0]]> + + + (pyFunc1($0, $1), 0)]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) +]]> + + + 0)]) + +- PythonCalc(select=[a, b, c, pyFunc1(a, b) AS f0]) + +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c]) +]]> + + + + + 0]]> + + + (pyFunc1($0, $1), 0)]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) +]]> + + + 0)]) ++- PythonCalc(select=[a, b, pyFunc1(a, b) AS f0]) + +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c]) +]]> + + + + + 0]]> + + + (pyFunc1($0, $1), 0)]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) +]]> + + + EXPR$1) ++- Calc(select=[f0, c, a], where=[(f0 > 0)]) + +- PythonCalc(select=[a, b, c, pyFunc1(a, b) AS f0]) + +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c]) +]]> + + + diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.scala new file mode 100644 index 00000000000000..7ce5603ece8b48 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.scala @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.table.planner.plan.stream.sql + +import org.apache.flink.table.api._ +import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions._ +import org.apache.flink.table.planner.utils.TableTestBase + +import org.junit.jupiter.api.{BeforeEach, Test} + +/** + * Tests for + * [[org.apache.flink.table.planner.plan.rules.logical.RemoteCalcConditionProjectionCseRule]]. + * + * Verifies that Python UDF calls shared between condition and projection are deduplicated so that + * the same UDF is not computed in two separate PythonCalc nodes. Tests cover: + * - UDF nested calls + * - UDF repeated calls in projection + * - UDF in condition (WHERE clause) + * + * Each test verifies the final physical execution plan (ExecNode level) to confirm the number of + * PythonCalc nodes and UDF call count is minimized. + */ +class PythonCalcConditionCseTest extends TableTestBase { + + private val util = streamTestUtil() + + @BeforeEach + def setup(): Unit = { + util.addTableSource[(Int, Int, Int)]("MyTable", 'a, 'b, 'c) + util.addTemporarySystemFunction("pyFunc1", new PythonScalarFunction("pyFunc1")) + util.addTemporarySystemFunction("pyFunc2", new PythonScalarFunction("pyFunc2")) + util.addTemporarySystemFunction("pyFunc3", new PythonScalarFunction("pyFunc3")) + util.addTemporarySystemFunction("pyFunc4", new BooleanPythonScalarFunction("pyFunc4")) + } + + @Test + def testSameUdfInConditionAndProjection(): Unit = { + // pyFunc1(a, b) appears in both WHERE and SELECT — should result in only one PythonCalc node + val sqlQuery = + "SELECT pyFunc1(a, b) + 1, pyFunc1(a, b) + 2 FROM MyTable WHERE pyFunc1(a, b) > 0" + util.verifyExecPlan(sqlQuery) + } + + @Test + def testDifferentUdfInConditionAndProjection(): Unit = { + // Different UDFs in condition and projection — two PythonCalc nodes expected + val sqlQuery = "SELECT pyFunc1(a, b) FROM MyTable WHERE pyFunc2(a, c) > 0" + util.verifyExecPlan(sqlQuery) + } + + @Test + def testNestedUdfInProjectionWithSameInCondition(): Unit = { + // pyFunc1(a, b) is used in condition and also nested inside pyFunc2 in projection + val sqlQuery = + "SELECT pyFunc2(pyFunc1(a, b), c), pyFunc1(a, b) FROM MyTable WHERE pyFunc1(a, b) > 0" + util.verifyExecPlan(sqlQuery) + } + + @Test + def testRepeatedUdfCallsInProjectionWithCondition(): Unit = { + // pyFunc1(a, b) repeated 3 times in projection and once in condition — single PythonCalc + val sqlQuery = + """SELECT pyFunc1(a, b) + 1, pyFunc1(a, b) + 2, pyFunc1(a, b) + 3 + |FROM MyTable + |WHERE pyFunc1(a, b) > 0""".stripMargin + util.verifyExecPlan(sqlQuery) + } + + @Test + def testConditionOnlyUdf(): Unit = { + // UDF only in condition, not in projection — standard split behavior + val sqlQuery = "SELECT a, b FROM MyTable WHERE pyFunc4(a, c)" + util.verifyExecPlan(sqlQuery) + } + + @Test + def testMultipleUdfsSharedBetweenConditionAndProjection(): Unit = { + // Two different UDFs both appear in condition and projection — single PythonCalc + val sqlQuery = + """SELECT pyFunc1(a, b) + pyFunc2(b, c) + |FROM MyTable + |WHERE pyFunc1(a, b) > 0 AND pyFunc2(b, c) > 0""".stripMargin + util.verifyExecPlan(sqlQuery) + } + + @Test + def testNestedUdfWithCseAnnotation(): Unit = { + // Deeply nested UDF calls: pyFunc1(a, b) is shared across multiple levels. + // After condition-projection CSE rule extracts pyFunc1(a, b) as f0, + // the second PythonCalc will have multiple UDF calls with nested references, + // triggering the CSE annotation in getDescription(). + val sqlQuery = + """SELECT pyFunc1(a, b), pyFunc1(pyFunc1(a, b), c), pyFunc1(pyFunc1(pyFunc1(a, b), c), a) + |FROM MyTable + |WHERE pyFunc1(a, b) > 0""".stripMargin + util.verifyExecPlan(sqlQuery) + } +} From b46fc72c3095c5920b016ef18acfe40a9fef06fb Mon Sep 17 00:00:00 2001 From: Raorao Xiong Date: Sun, 5 Jul 2026 20:10:02 +0800 Subject: [PATCH 3/8] [FLINK-39986][table-planner][python] Fix compilation error: rename RemoteCalcCallFinder to RemoteCallFinder Fix typo in RemoteCalcConditionProjectionCseRule constructor parameter type. The correct interface name is RemoteCallFinder, not RemoteCalcCallFinder. Generated-by: Claude-4.6-Opus --- .../rules/logical/RemoteCalcConditionProjectionCseRule.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.scala index c2b42e000fa806..f37681751251dc 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.scala @@ -49,7 +49,7 @@ import scala.collection.JavaConversions._ * @param callFinder * the finder used to identify remote (Python) function calls */ -class RemoteCalcConditionProjectionCseRule(private val callFinder: RemoteCalcCallFinder) +class RemoteCalcConditionProjectionCseRule(private val callFinder: RemoteCallFinder) extends RelOptRule( operand(classOf[FlinkLogicalCalc], operand(classOf[FlinkLogicalCalc], any)), "RemoteCalcConditionProjectionCseRule") { From 647f4a6265775e24d71d6e2697c84d08bf624697 Mon Sep 17 00:00:00 2001 From: Raorao Xiong Date: Sun, 5 Jul 2026 20:51:13 +0800 Subject: [PATCH 4/8] [FLINK-39986][table-planner][python] Convert Scala classes to Java and use RelRule API - Convert RemoteCalcConditionProjectionCseRule from Scala to Java using the modern RelRule pattern instead of deprecated RelOptRule - Convert PythonCalcConditionCseTest from Scala to Java - Update PythonCalcSplitRule to use Config-based rule instantiation Generated-by: Claude-4.6-Opus --- .../RemoteCalcConditionProjectionCseRule.java | 255 ++++++++++++++++++ .../rules/logical/PythonCalcSplitRule.scala | 3 +- ...RemoteCalcConditionProjectionCseRule.scala | 197 -------------- .../sql/PythonCalcConditionCseTest.java | 98 +++++++ .../sql/PythonCalcConditionCseTest.scala | 114 -------- 5 files changed, 355 insertions(+), 312 deletions(-) create mode 100644 flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.java delete mode 100644 flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.scala create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.java delete mode 100644 flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.scala diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.java new file mode 100644 index 00000000000000..9926dc39f6d132 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.rules.logical; + +import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCalc; +import org.apache.flink.table.planner.utils.ShortcutUtils; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexProgram; +import org.apache.calcite.rex.RexShuttle; +import org.immutables.value.Value; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * Rule that eliminates common Python UDF sub-expressions between the condition and projection of a + * Calc node. + * + *

After {@link RemoteCalcSplitConditionRule} splits a Calc with Python UDFs in its condition, + * the result is a two-level Calc structure: + * + * 

+ * TopCalc(projection=[pyFunc(a, b) + 1, pyFunc(a, b) + 2], condition=[$2 > 0])
+ *   BottomCalc(projection=[a, b, pyFunc(a, b) AS f0])
+ *
+ * + *

The TopCalc's projection still contains {@code pyFunc(a, b)} which is structurally identical + * to the already-computed {@code f0} in the BottomCalc. This rule detects such duplicates and + * rewrites the TopCalc's projection to reference the BottomCalc's output directly: + * + * 

+ * TopCalc(projection=[$2 + 1, $2 + 2], condition=[$2 > 0])
+ *   BottomCalc(projection=[a, b, pyFunc(a, b) AS f0])
+ *
+ */ +@Value.Enclosing +public class RemoteCalcConditionProjectionCseRule + extends RelRule { + + protected RemoteCalcConditionProjectionCseRule(Config config) { + super(config); + } + + @Override + public boolean matches(RelOptRuleCall call) { + FlinkLogicalCalc topCalc = call.rel(0); + FlinkLogicalCalc bottomCalc = call.rel(1); + RemoteCallFinder callFinder = config.remoteCallFinder(); + + // Only applies when the top calc has a condition. + if (topCalc.getProgram().getCondition() == null) { + return false; + } + + List topProjects = expandProjects(topCalc); + Map bottomPythonCalls = buildBottomPythonCallMap(bottomCalc, callFinder); + + if (bottomPythonCalls.isEmpty()) { + return false; + } + + // Check if any top projection contains a call matching the bottom calc's output. + return topProjects.stream() + .anyMatch(node -> containsCallMatchingBottom(node, bottomPythonCalls, callFinder)); + } + + @Override + public void onMatch(RelOptRuleCall call) { + FlinkLogicalCalc topCalc = call.rel(0); + FlinkLogicalCalc bottomCalc = call.rel(1); + RemoteCallFinder callFinder = config.remoteCallFinder(); + RexBuilder rexBuilder = call.builder().getRexBuilder(); + + List topProjects = expandProjects(topCalc); + RexNode topCondition = + topCalc.getProgram().getCondition() != null + ? topCalc.getProgram().expandLocalRef(topCalc.getProgram().getCondition()) + : null; + + Map bottomPythonCalls = buildBottomPythonCallMap(bottomCalc, callFinder); + RelDataType bottomRowType = bottomCalc.getRowType(); + + // Rewrite top projections: replace matching calls with RexInputRef. + CseRewriteShuttle rewriter = new CseRewriteShuttle(bottomPythonCalls, bottomRowType); + List newTopProjects = + topProjects.stream().map(p -> p.accept(rewriter)).collect(Collectors.toList()); + + if (!rewriter.hasRewritten()) { + return; + } + + // Build the new top calc with rewritten projections. + call.transformTo( + topCalc.copy( + topCalc.getTraitSet(), + bottomCalc, + RexProgram.create( + bottomRowType, + newTopProjects, + topCondition, + topCalc.getRowType(), + rexBuilder))); + } + + private List expandProjects(FlinkLogicalCalc calc) { + RexProgram program = calc.getProgram(); + return program.getProjectList().stream() + .map(program::expandLocalRef) + .collect(Collectors.toList()); + } + + /** + * Builds a map from deterministic Python UDF calls in the bottom calc's projection to their + * output index. + */ + private Map buildBottomPythonCallMap( + FlinkLogicalCalc bottomCalc, RemoteCallFinder callFinder) { + RexProgram program = bottomCalc.getProgram(); + List bottomProjects = + program.getProjectList().stream() + .map(program::expandLocalRef) + .collect(Collectors.toList()); + + Map result = new HashMap<>(); + IntStream.range(0, bottomProjects.size()) + .filter( + i -> { + RexNode node = bottomProjects.get(i); + return callFinder.containsRemoteCall(node) + && ShortcutUtils.isDeterministicThroughProgram(node, null); + }) + .forEach(i -> result.put(bottomProjects.get(i), i)); + return result; + } + + private boolean containsCallMatchingBottom( + RexNode node, Map bottomPythonCalls, RemoteCallFinder callFinder) { + if (node instanceof RexCall) { + RexCall rexCall = (RexCall) node; + if (callFinder.isRemoteCall(rexCall) && bottomPythonCalls.containsKey(rexCall)) { + return true; + } + return rexCall.getOperands().stream() + .anyMatch(op -> containsCallMatchingBottom(op, bottomPythonCalls, callFinder)); + } + return false; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof RemoteCalcConditionProjectionCseRule)) { + return false; + } + RemoteCalcConditionProjectionCseRule other = (RemoteCalcConditionProjectionCseRule) obj; + return super.equals(other) + && config.remoteCallFinder() + .getClass() + .equals(other.config.remoteCallFinder().getClass()); + } + + @Override + public int hashCode() { + return super.hashCode() * 31 + config.remoteCallFinder().getClass().hashCode(); + } + + // ------------------------------------------------------------------------- + + /** + * Replaces Python UDF calls in the top calc's projection with RexInputRef pointing to the + * bottom calc's output position where the same call was already computed. + */ + private static class CseRewriteShuttle extends RexShuttle { + private final Map bottomPythonCalls; + private final RelDataType bottomRowType; + private boolean rewritten = false; + + CseRewriteShuttle(Map bottomPythonCalls, RelDataType bottomRowType) { + this.bottomPythonCalls = bottomPythonCalls; + this.bottomRowType = bottomRowType; + } + + boolean hasRewritten() { + return rewritten; + } + + @Override + public RexNode visitCall(RexCall call) { + Integer idx = bottomPythonCalls.get(call); + if (idx != null) { + rewritten = true; + return new RexInputRef(idx, bottomRowType.getFieldList().get(idx).getType()); + } + return super.visitCall(call); + } + } + + // ------------------------------------------------------------------------- + + /** Rule configuration. */ + @Value.Immutable(singleton = false) + public interface Config extends RelRule.Config { + Config DEFAULT = + ImmutableRemoteCalcConditionProjectionCseRule.Config.builder() + .operandSupplier( + b0 -> + b0.operand(FlinkLogicalCalc.class) + .oneInput( + b1 -> + b1.operand(FlinkLogicalCalc.class) + .anyInputs())) + .description("RemoteCalcConditionProjectionCseRule") + .build(); + + @Value.Default + default RemoteCallFinder remoteCallFinder() { + return new PythonRemoteCallFinder(); + } + + /** Sets {@link #remoteCallFinder()}. */ + Config withRemoteCallFinder(RemoteCallFinder callFinder); + + @Override + default RemoteCalcConditionProjectionCseRule toRule() { + return new RemoteCalcConditionProjectionCseRule(this); + } + } +} diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRule.scala index 15850466bcdf58..ae54a31b49af7a 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRule.scala @@ -102,7 +102,8 @@ object PythonCalcSplitRule { */ private val callFinder = new PythonRemoteCallFinder() val SPLIT_CONDITION: RelOptRule = new RemoteCalcSplitConditionRule(callFinder) - val CONDITION_PROJECTION_CSE: RelOptRule = new RemoteCalcConditionProjectionCseRule(callFinder) + val CONDITION_PROJECTION_CSE: RelOptRule = + RemoteCalcConditionProjectionCseRule.Config.DEFAULT.withRemoteCallFinder(callFinder).toRule() val SPLIT_PROJECT: RelOptRule = new RemoteCalcSplitProjectionRule(callFinder) val SPLIT_PANDAS_IN_PROJECT: RelOptRule = new PythonCalcSplitPandasInProjectionRule(callFinder) val SPLIT_PROJECTION_REX_FIELD: RelOptRule = new RemoteCalcSplitProjectionRexFieldRule(callFinder) diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.scala deleted file mode 100644 index f37681751251dc..00000000000000 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcConditionProjectionCseRule.scala +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.flink.table.planner.plan.rules.logical - -import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCalc -import org.apache.flink.table.planner.utils.ShortcutUtils - -import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall} -import org.apache.calcite.plan.RelOptRule.{any, operand} -import org.apache.calcite.rex.{RexBuilder, RexCall, RexInputRef, RexNode, RexShuttle} - -import scala.collection.JavaConversions._ - -/** - * Rule that eliminates common Python UDF sub-expressions between the condition and projection of a - * Calc node. - * - *

After [[RemoteCalcSplitConditionRule]] splits a Calc with Python UDFs in its condition, the - * result is a two-level Calc structure: - * - * 

 TopCalc(projection=[pyFunc(a, b) + 1, pyFunc(a, b) + 2], condition=[$2 > 0])
- * BottomCalc(projection=[a, b, pyFunc(a, b) AS f0])
- * - *

The TopCalc's projection still contains `pyFunc(a, b)` which is structurally identical to the - * already-computed `f0` in the BottomCalc. This rule detects such duplicates and rewrites the - * TopCalc's projection to reference the BottomCalc's output directly: - * - * 

 TopCalc(projection=[$2 + 1, $2 + 2], condition=[$2 > 0]) BottomCalc(projection=[a, b,
- * pyFunc(a, b) AS f0])
- * - *

This ensures that when the plan is later split into separate PythonCalc nodes, the same UDF - * call is not computed twice across condition and projection. - * - * @param callFinder - * the finder used to identify remote (Python) function calls - */ -class RemoteCalcConditionProjectionCseRule(private val callFinder: RemoteCallFinder) - extends RelOptRule( - operand(classOf[FlinkLogicalCalc], operand(classOf[FlinkLogicalCalc], any)), - "RemoteCalcConditionProjectionCseRule") { - - override def matches(call: RelOptRuleCall): Boolean = { - val topCalc: FlinkLogicalCalc = call.rel(0) - val bottomCalc: FlinkLogicalCalc = call.rel(1) - - // This rule only applies when the top calc has a condition — it deduplicates Python UDF - // calls shared between condition and projection. Without a condition, there is nothing - // to deduplicate, and firing would interfere with other rules (e.g. PythonMapMergeRule). - if (topCalc.getProgram.getCondition == null) { - return false - } - - val topProjects = topCalc.getProgram.getProjectList.map(topCalc.getProgram.expandLocalRef) - - // The bottom calc must have deterministic Python UDF calls in its projection - // (from condition extraction). Non-deterministic calls must NOT be shared. - val bottomProjects = - bottomCalc.getProgram.getProjectList.map(bottomCalc.getProgram.expandLocalRef) - val bottomPythonCalls: Map[RexNode, Int] = bottomProjects.zipWithIndex - .filter { - case (node, _) => - callFinder.containsRemoteCall(node) && - ShortcutUtils.isDeterministicThroughProgram(node, null) - } - .map { case (node, idx) => (node, idx) } - .toMap - - if (bottomPythonCalls.isEmpty) { - return false - } - - // The top calc's projection must contain Python UDF calls that are structurally equal - // to calls already computed in the bottom calc's projection - topProjects.exists(containsCallMatchingBottom(_, bottomPythonCalls)) - } - - override def onMatch(call: RelOptRuleCall): Unit = { - val topCalc: FlinkLogicalCalc = call.rel(0) - val bottomCalc: FlinkLogicalCalc = call.rel(1) - val rexBuilder: RexBuilder = call.builder().getRexBuilder - - val topProjects = topCalc.getProgram.getProjectList.map(topCalc.getProgram.expandLocalRef) - val topCondition = Option(topCalc.getProgram.getCondition) - .map(topCalc.getProgram.expandLocalRef) - - val bottomProjects = - bottomCalc.getProgram.getProjectList.map(bottomCalc.getProgram.expandLocalRef) - - // Build a map from deterministic Python UDF call (in bottom calc's projection) to its - // output index. Non-deterministic calls are excluded to prevent incorrect sharing. - val bottomPythonCalls: Map[RexNode, Int] = bottomProjects.zipWithIndex - .filter { - case (node, _) => - callFinder.containsRemoteCall(node) && - ShortcutUtils.isDeterministicThroughProgram(node, null) - } - .map { case (node, idx) => (node, idx) } - .toMap - - // Rewrite the top calc's projection: replace Python UDF calls that match bottom calc's - // projection with RexInputRef pointing to the bottom calc's output position - val rewriter = new CseRewriteShuttle(bottomPythonCalls, bottomCalc.getRowType) - val newTopProjects: Seq[RexNode] = topProjects.map(_.accept(rewriter)) - - // Check if any rewriting actually happened - if (!rewriter.hasRewritten) { - return - } - - // Build the new top calc with rewritten projections - val newTopCalc = topCalc.copy( - topCalc.getTraitSet, - bottomCalc, - org.apache.calcite.rex.RexProgram.create( - bottomCalc.getRowType, - newTopProjects.toList, - topCondition.orNull, - topCalc.getRowType, - rexBuilder - ) - ) - - call.transformTo(newTopCalc) - } - - /** - * Checks if a RexNode contains a Python UDF call that is structurally equal to one of the calls - * in the bottom calc's projection. - */ - private def containsCallMatchingBottom( - node: RexNode, - bottomPythonCalls: Map[RexNode, Int]): Boolean = { - node match { - case call: RexCall => - if (callFinder.isRemoteCall(call) && bottomPythonCalls.contains(call)) { - true - } else { - call.getOperands.exists(containsCallMatchingBottom(_, bottomPythonCalls)) - } - case _ => false - } - } - - // Consider the rules to be equal if they are the same class and their call finders are the same - override def equals(obj: Any): Boolean = { - obj match { - case other: RemoteCalcConditionProjectionCseRule => - super.equals(other) && callFinder.getClass.equals(other.callFinder.getClass) - case _ => false - } - } - - override def hashCode(): Int = { - super.hashCode() * 31 + callFinder.getClass.hashCode() - } -} - -/** - * A RexShuttle that replaces Python UDF calls in the top calc's projection with RexInputRef - * pointing to the bottom calc's output position where the same call was already computed. - */ -private class CseRewriteShuttle( - bottomPythonCalls: Map[RexNode, Int], - bottomRowType: org.apache.calcite.rel.`type`.RelDataType) - extends RexShuttle { - - private var _hasRewritten: Boolean = false - - def hasRewritten: Boolean = _hasRewritten - - override def visitCall(call: RexCall): RexNode = { - // If this call matches a bottom calc's Python UDF output, replace with InputRef - bottomPythonCalls.get(call) match { - case Some(idx) => - _hasRewritten = true - new RexInputRef(idx, bottomRowType.getFieldList.get(idx).getType) - case None => - // Continue rewriting operands - super.visitCall(call) - } - } -} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.java new file mode 100644 index 00000000000000..37a0394651537e --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.stream.sql; + +import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.BooleanPythonScalarFunction; +import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction; +import org.apache.flink.table.planner.utils.JavaStreamTableTestUtil; +import org.apache.flink.table.planner.utils.TableTestBase; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link + * org.apache.flink.table.planner.plan.rules.logical.RemoteCalcConditionProjectionCseRule}. + * + *

Verifies that Python UDF calls shared between condition and projection are deduplicated. + */ +class PythonCalcConditionCseTest extends TableTestBase { + + private final JavaStreamTableTestUtil util = javaStreamTestUtil(); + + @BeforeEach + void setup() { + util.tableEnv() + .executeSql( + "CREATE TABLE MyTable (a INT, b INT, c INT) WITH ('connector' = 'values')"); + util.tableEnv() + .createTemporarySystemFunction("pyFunc1", new PythonScalarFunction("pyFunc1")); + util.tableEnv() + .createTemporarySystemFunction("pyFunc2", new PythonScalarFunction("pyFunc2")); + util.tableEnv() + .createTemporarySystemFunction("pyFunc3", new PythonScalarFunction("pyFunc3")); + util.tableEnv() + .createTemporarySystemFunction( + "pyFunc4", new BooleanPythonScalarFunction("pyFunc4")); + } + + @Test + void testSameUdfInConditionAndProjection() { + util.verifyExecPlan( + "SELECT pyFunc1(a, b) + 1, pyFunc1(a, b) + 2 FROM MyTable WHERE pyFunc1(a, b) > 0"); + } + + @Test + void testDifferentUdfInConditionAndProjection() { + util.verifyExecPlan("SELECT pyFunc1(a, b) FROM MyTable WHERE pyFunc2(a, c) > 0"); + } + + @Test + void testNestedUdfInProjectionWithSameInCondition() { + util.verifyExecPlan( + "SELECT pyFunc2(pyFunc1(a, b), c), pyFunc1(a, b) FROM MyTable WHERE pyFunc1(a, b) > 0"); + } + + @Test + void testRepeatedUdfCallsInProjectionWithCondition() { + util.verifyExecPlan( + "SELECT pyFunc1(a, b) + 1, pyFunc1(a, b) + 2, pyFunc1(a, b) + 3 " + + "FROM MyTable WHERE pyFunc1(a, b) > 0"); + } + + @Test + void testConditionOnlyUdf() { + util.verifyExecPlan("SELECT a, b FROM MyTable WHERE pyFunc4(a, c)"); + } + + @Test + void testMultipleUdfsSharedBetweenConditionAndProjection() { + util.verifyExecPlan( + "SELECT pyFunc1(a, b) + pyFunc2(b, c) " + + "FROM MyTable WHERE pyFunc1(a, b) > 0 AND pyFunc2(b, c) > 0"); + } + + @Test + void testNestedUdfWithCseAnnotation() { + util.verifyExecPlan( + "SELECT pyFunc1(a, b), pyFunc1(pyFunc1(a, b), c), " + + "pyFunc1(pyFunc1(pyFunc1(a, b), c), a) " + + "FROM MyTable WHERE pyFunc1(a, b) > 0"); + } +} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.scala deleted file mode 100644 index 7ce5603ece8b48..00000000000000 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.scala +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.flink.table.planner.plan.stream.sql - -import org.apache.flink.table.api._ -import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions._ -import org.apache.flink.table.planner.utils.TableTestBase - -import org.junit.jupiter.api.{BeforeEach, Test} - -/** - * Tests for - * [[org.apache.flink.table.planner.plan.rules.logical.RemoteCalcConditionProjectionCseRule]]. - * - * Verifies that Python UDF calls shared between condition and projection are deduplicated so that - * the same UDF is not computed in two separate PythonCalc nodes. Tests cover: - * - UDF nested calls - * - UDF repeated calls in projection - * - UDF in condition (WHERE clause) - * - * Each test verifies the final physical execution plan (ExecNode level) to confirm the number of - * PythonCalc nodes and UDF call count is minimized. - */ -class PythonCalcConditionCseTest extends TableTestBase { - - private val util = streamTestUtil() - - @BeforeEach - def setup(): Unit = { - util.addTableSource[(Int, Int, Int)]("MyTable", 'a, 'b, 'c) - util.addTemporarySystemFunction("pyFunc1", new PythonScalarFunction("pyFunc1")) - util.addTemporarySystemFunction("pyFunc2", new PythonScalarFunction("pyFunc2")) - util.addTemporarySystemFunction("pyFunc3", new PythonScalarFunction("pyFunc3")) - util.addTemporarySystemFunction("pyFunc4", new BooleanPythonScalarFunction("pyFunc4")) - } - - @Test - def testSameUdfInConditionAndProjection(): Unit = { - // pyFunc1(a, b) appears in both WHERE and SELECT — should result in only one PythonCalc node - val sqlQuery = - "SELECT pyFunc1(a, b) + 1, pyFunc1(a, b) + 2 FROM MyTable WHERE pyFunc1(a, b) > 0" - util.verifyExecPlan(sqlQuery) - } - - @Test - def testDifferentUdfInConditionAndProjection(): Unit = { - // Different UDFs in condition and projection — two PythonCalc nodes expected - val sqlQuery = "SELECT pyFunc1(a, b) FROM MyTable WHERE pyFunc2(a, c) > 0" - util.verifyExecPlan(sqlQuery) - } - - @Test - def testNestedUdfInProjectionWithSameInCondition(): Unit = { - // pyFunc1(a, b) is used in condition and also nested inside pyFunc2 in projection - val sqlQuery = - "SELECT pyFunc2(pyFunc1(a, b), c), pyFunc1(a, b) FROM MyTable WHERE pyFunc1(a, b) > 0" - util.verifyExecPlan(sqlQuery) - } - - @Test - def testRepeatedUdfCallsInProjectionWithCondition(): Unit = { - // pyFunc1(a, b) repeated 3 times in projection and once in condition — single PythonCalc - val sqlQuery = - """SELECT pyFunc1(a, b) + 1, pyFunc1(a, b) + 2, pyFunc1(a, b) + 3 - |FROM MyTable - |WHERE pyFunc1(a, b) > 0""".stripMargin - util.verifyExecPlan(sqlQuery) - } - - @Test - def testConditionOnlyUdf(): Unit = { - // UDF only in condition, not in projection — standard split behavior - val sqlQuery = "SELECT a, b FROM MyTable WHERE pyFunc4(a, c)" - util.verifyExecPlan(sqlQuery) - } - - @Test - def testMultipleUdfsSharedBetweenConditionAndProjection(): Unit = { - // Two different UDFs both appear in condition and projection — single PythonCalc - val sqlQuery = - """SELECT pyFunc1(a, b) + pyFunc2(b, c) - |FROM MyTable - |WHERE pyFunc1(a, b) > 0 AND pyFunc2(b, c) > 0""".stripMargin - util.verifyExecPlan(sqlQuery) - } - - @Test - def testNestedUdfWithCseAnnotation(): Unit = { - // Deeply nested UDF calls: pyFunc1(a, b) is shared across multiple levels. - // After condition-projection CSE rule extracts pyFunc1(a, b) as f0, - // the second PythonCalc will have multiple UDF calls with nested references, - // triggering the CSE annotation in getDescription(). - val sqlQuery = - """SELECT pyFunc1(a, b), pyFunc1(pyFunc1(a, b), c), pyFunc1(pyFunc1(pyFunc1(a, b), c), a) - |FROM MyTable - |WHERE pyFunc1(a, b) > 0""".stripMargin - util.verifyExecPlan(sqlQuery) - } -} From 219c281d328f5626274765da2ce45afc2e66c75d Mon Sep 17 00:00:00 2001 From: Raorao Xiong Date: Mon, 6 Jul 2026 10:36:12 +0800 Subject: [PATCH 5/8] [FLINK-39986][table-planner][python] Fix test: use addTableSource and fix XML order Use addTableSource with Schema instead of CREATE TABLE with values connector to prevent projection pushdown from changing the plan output. Reorder TestCase elements in XML alphabetically to match DiffRepository validation requirements. Generated-by: Claude-4.6-Opus --- .../sql/PythonCalcConditionCseTest.java | 12 +++-- .../stream/sql/PythonCalcConditionCseTest.xml | 46 +++++++++---------- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.java index 37a0394651537e..90ca44f07ced26 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.java @@ -18,6 +18,8 @@ package org.apache.flink.table.planner.plan.stream.sql; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.Schema; import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.BooleanPythonScalarFunction; import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction; import org.apache.flink.table.planner.utils.JavaStreamTableTestUtil; @@ -38,9 +40,13 @@ class PythonCalcConditionCseTest extends TableTestBase { @BeforeEach void setup() { - util.tableEnv() - .executeSql( - "CREATE TABLE MyTable (a INT, b INT, c INT) WITH ('connector' = 'values')"); + util.addTableSource( + "MyTable", + Schema.newBuilder() + .column("a", DataTypes.INT()) + .column("b", DataTypes.INT()) + .column("c", DataTypes.INT()) + .build()); util.tableEnv() .createTemporarySystemFunction("pyFunc1", new PythonScalarFunction("pyFunc1")); util.tableEnv() diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.xml index 8715e60bd46d0f..8de30df5181185 100644 --- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.xml +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.xml @@ -76,43 +76,46 @@ Calc(select=[(f0 + f1) AS EXPR$0], where=[((f0 > 0) AND (f1 > 0))]) ]]> - + - 0]]> + 0]]> (pyFunc1($0, $1), 0)]) +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) ]]> 0)]) -+- PythonCalc(select=[a, b, pyFunc1(a, b) AS f0]) - +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c]) +Calc(select=[f00 AS EXPR$0, f0 AS EXPR$1]) ++- PythonCalc(select=[f0, pyFunc2(f0, c) AS f00]) + +- Calc(select=[f0, c], where=[(f0 > 0)]) + +- PythonCalc(select=[a, b, c, pyFunc1(a, b) AS f0]) + +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c]) ]]> - + - 0]]> + 0]]> (pyFunc1($0, $1), 0)]) +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) ]]> 0)]) - +- PythonCalc(select=[a, b, c, pyFunc1(a, b) AS f0]) - +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c]) +PythonCalc(select=[f0 AS EXPR$0, pyFunc1(f0, c) AS EXPR$1, pyFunc1(pyFunc1(f0, c), a) AS EXPR$2]) (CSE: EXPR$2->EXPR$1) ++- Calc(select=[f0, c, a], where=[(f0 > 0)]) + +- PythonCalc(select=[a, b, c, pyFunc1(a, b) AS f0]) + +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c]) ]]> @@ -137,25 +140,22 @@ Calc(select=[(f0 + 1) AS EXPR$0, (f0 + 2) AS EXPR$1, (f0 + 3) AS EXPR$2], where= ]]> - + - 0]]> + 0]]> (pyFunc1($0, $1), 0)]) +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) ]]> EXPR$1) -+- Calc(select=[f0, c, a], where=[(f0 > 0)]) - +- PythonCalc(select=[a, b, c, pyFunc1(a, b) AS f0]) - +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c]) +Calc(select=[(f0 + 1) AS EXPR$0, (f0 + 2) AS EXPR$1], where=[(f0 > 0)]) ++- PythonCalc(select=[a, b, pyFunc1(a, b) AS f0]) + +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c]) ]]> From b9b3ed4f4f158cc19485c954b22e52e9fcc61f05 Mon Sep 17 00:00:00 2001 From: rockyxiong Date: Mon, 6 Jul 2026 14:44:53 +0800 Subject: [PATCH 6/8] fix xml ut test --- .../table/functions/python/PythonFunctionInfo.java | 1 + .../plan/stream/sql/PythonCalcConditionCseTest.xml | 12 +++--------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java index 69fb020986a9a2..e17d2bb9d07f87 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java @@ -55,6 +55,7 @@ public class PythonFunctionInfo implements Serializable { * call that appears in multiple positions can be computed once and its result referenced by * index. */ + @Internal public static class ResultRef implements Serializable { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.xml index 8de30df5181185..013a1649b840b0 100644 --- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.xml +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/PythonCalcConditionCseTest.xml @@ -57,9 +57,7 @@ PythonCalc(select=[pyFunc1(a, b) AS EXPR$0]) - 0 AND pyFunc2(b, c) > 0]]> + 0 AND pyFunc2(b, c) > 0]]> - 0]]> + 0]]> - 0]]> + 0]]> Date: Mon, 6 Jul 2026 16:14:58 +0800 Subject: [PATCH 7/8] fix python unit test --- .../tests/test_scalar_function_cse.py | 65 +++++++++++++++++++ .../fn_execution/utils/operation_utils.py | 5 ++ .../exec/common/CommonExecPythonCalc.java | 30 ++++++++- 3 files changed, 97 insertions(+), 3 deletions(-) diff --git a/flink-python/pyflink/fn_execution/tests/test_scalar_function_cse.py b/flink-python/pyflink/fn_execution/tests/test_scalar_function_cse.py index a409eadae3b10e..9a6a354ce4261d 100644 --- a/flink-python/pyflink/fn_execution/tests/test_scalar_function_cse.py +++ b/flink-python/pyflink/fn_execution/tests/test_scalar_function_cse.py @@ -209,5 +209,70 @@ def test_no_cse_lambda_execution(self): # udf2(udf1(5)) = udf2(10) = 20 self.assertEqual(result2[0], 20) + def test_cse_with_takes_row_as_input(self): + """ + Verify that when takesRowAsInput=True but the UDF receives its input + via refIndex (CSE), the generated code uses results[N] instead of + overriding with `value`. + + Scenario: func(value[0]) returns Row, func2 consumes func's result via refIndex. + func2 has takesRowAsInput=True because in the original plan it received a Row. + After CSE flattening, it should receive results[0] (func's output), not `value` + (the original input row). + + This simulates the PythonMapMergeRule + CSE flattening for: + .map(func(t.a)).map(func2) + where func2 takes Row as input. + """ + # func: returns Row(a, b) + func_payload = _make_udf_payload(lambda x: type('Row', (), {'a': x + 1, 'b': x * x})()) + # Better: use actual Row + from pyflink.common import Row + func_payload = _make_udf_payload(lambda x: Row(a=x + 1, b=x * x)) + + # func2: takes Row, returns Row + func2_payload = _make_udf_payload(lambda x: Row(a=x.a + 1, b=x.b * 2)) + + # func: takesRowAsInput=False, reads from value[0] + func_proto = _make_udf_proto(func_payload, [_input_offset(0)]) + func_proto.takes_row_as_input = False + + # func2: takesRowAsInput=True, references func's result via refIndex=0 + func2_proto = _make_udf_proto(func2_payload, [_input_ref_index(0)]) + func2_proto.takes_row_as_input = True + + serialized_fn = _build_serialized_fn([func_proto, func2_proto]) + + op = ScalarFunctionOperation(serialized_fn) + + # Verify the function name confirms CSE codegen path + self.assertEqual(op.func.__name__, '_sequential_execute') + + # Verify the generated code + generated_code = op._generated_code + self.assertIn('def _sequential_execute(value):', generated_code) + # func2 (results[1]) should receive results[0], NOT `value` + line2 = [l for l in generated_code.split('\n') if 'results[1]' in l][0] + self.assertIn('results[0]', line2, "func2 should receive results[0], not value") + + # Verify computation + result = op.func([3]) + # func(3) = Row(a=4, b=9) + # func2(Row(4, 9)) = Row(a=5, b=18) + self.assertEqual(result[0].a, 4) + self.assertEqual(result[0].b, 9) + self.assertEqual(result[1].a, 5) + self.assertEqual(result[1].b, 18) + + # Verify idempotency + result2 = op.func([5]) + # func(5) = Row(a=6, b=25) + # func2(Row(6, 25)) = Row(a=7, b=50) + self.assertEqual(result2[0].a, 6) + self.assertEqual(result2[0].b, 25) + self.assertEqual(result2[1].a, 7) + self.assertEqual(result2[1].b, 50) + + if __name__ == '__main__': unittest.main() diff --git a/flink-python/pyflink/fn_execution/utils/operation_utils.py b/flink-python/pyflink/fn_execution/utils/operation_utils.py index 3ee382e93f1cca..ff96585e4c2513 100644 --- a/flink-python/pyflink/fn_execution/utils/operation_utils.py +++ b/flink-python/pyflink/fn_execution/utils/operation_utils.py @@ -174,6 +174,11 @@ def _extract_input(args) -> Tuple[str, Dict, List]: # we need to merge these Pandas.Series into a Pandas.DataFrame variable_dict['wrap_input_series_as_dataframe'] = wrap_input_series_as_dataframe func_str = "%s(wrap_input_series_as_dataframe(%s))" % (func_name, func_args) + elif 'results[' in func_args: + # CSE: when args contain refIndex references (results[N]), the UDF + # receives a previously computed intermediate result rather than + # the original input row. We must use func_args instead of `value`. + func_str = "%s(%s)" % (func_name, func_args) else: # directly use `value` as input argument # e.g. diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java index ed00e10717ab77..0bde863ce7219c 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java @@ -224,15 +224,34 @@ String buildRefReuseDetailName(int[] originalToDedup, int forwardedCount) { private InternalTypeInfo buildDedupOutputTypeInfo( List forwardedFields, LogicalType[] inputLogicalTypes, - List uniquePythonRexCalls) { + List uniquePythonRexCalls, + List outputFieldNames) { List fieldTypes = new ArrayList<>(); + List fieldNames = new ArrayList<>(); for (int idx : forwardedFields) { fieldTypes.add(inputLogicalTypes[idx]); } for (RexCall call : uniquePythonRexCalls) { fieldTypes.add(FlinkTypeFactory.toLogicalType(call.getType())); } - return InternalTypeInfo.ofFields(fieldTypes.toArray(new LogicalType[0])); + // Assign field names: use output field names for forwarded fields first, + // then for UDF results. When needsExpansionProjection is true, the unique + // call count may differ from the output field count; in that case the + // expansion projection handles the mapping, and the intermediate field + // names are derived from the output field names where possible. + int totalFields = fieldTypes.size(); + int forwardedCount = forwardedFields.size(); + for (int i = 0; i < totalFields && i < outputFieldNames.size(); i++) { + fieldNames.add(outputFieldNames.get(i)); + } + // Fill remaining names (if unique calls count exceeds output names, e.g. + // flattened nested calls produce more fields than the output schema) + for (int i = fieldNames.size(); i < totalFields; i++) { + fieldNames.add("f" + i); + } + return InternalTypeInfo.ofFields( + fieldTypes.toArray(new LogicalType[0]), + fieldNames.toArray(new String[0])); } // ------------------------------------------------------------------------- @@ -290,8 +309,13 @@ private OneInputTransformation createPythonOneInputTransformat (InternalTypeInfo) inputTransform.getOutputType(); // Build output type using deduplicated unique calls + RowType outputType = (RowType) getOutputType(); InternalTypeInfo pythonOperatorResultTypeInfo = - buildDedupOutputTypeInfo(forwardedFields, inputLogicalTypes, uniquePythonRexCalls); + buildDedupOutputTypeInfo( + forwardedFields, + inputLogicalTypes, + uniquePythonRexCalls, + outputType.getFieldNames()); OneInputStreamOperator pythonOperator = getPythonScalarFunctionOperator( From a9c8d544dda29b517c66874e560bf332d9619c15 Mon Sep 17 00:00:00 2001 From: Raorao Xiong Date: Mon, 6 Jul 2026 19:24:30 +0800 Subject: [PATCH 8/8] fix style --- .../planner/plan/nodes/exec/common/CommonExecPythonCalc.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java index 0bde863ce7219c..7a5cc3cb200635 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java @@ -250,8 +250,7 @@ private InternalTypeInfo buildDedupOutputTypeInfo( fieldNames.add("f" + i); } return InternalTypeInfo.ofFields( - fieldTypes.toArray(new LogicalType[0]), - fieldNames.toArray(new String[0])); + fieldTypes.toArray(new LogicalType[0]), fieldNames.toArray(new String[0])); } // -------------------------------------------------------------------------