diff --git a/build.sh b/build.sh index e0c327b6d0c1aa..052b34668848a4 100755 --- a/build.sh +++ b/build.sh @@ -588,6 +588,7 @@ FE_MODULES='' modules=("") if [[ "${BUILD_FE}" -eq 1 ]]; then modules+=("fe-common") + modules+=("horn4j") modules+=("fe-core") if [[ "${WITH_TDE_DIR}" != "" ]]; then modules+=("fe-${WITH_TDE_DIR}") diff --git a/fe/check/checkstyle/suppressions.xml b/fe/check/checkstyle/suppressions.xml index 8e1c40e31620fe..35f73fd7f45aa3 100644 --- a/fe/check/checkstyle/suppressions.xml +++ b/fe/check/checkstyle/suppressions.xml @@ -75,4 +75,8 @@ under the License. + + + + diff --git a/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-00 b/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-00 new file mode 100644 index 00000000000000..58d26b41823093 Binary files /dev/null and b/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-00 differ diff --git a/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-01 b/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-01 new file mode 100644 index 00000000000000..e8174e0effb102 Binary files /dev/null and b/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-01 differ diff --git a/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-02 b/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-02 new file mode 100644 index 00000000000000..1aa0a914996f67 Binary files /dev/null and b/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-02 differ diff --git a/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-03 b/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-03 new file mode 100644 index 00000000000000..a3afd87451ccd6 Binary files /dev/null and b/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-03 differ diff --git a/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-04 b/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-04 new file mode 100644 index 00000000000000..4cc3a7d01f0f18 Binary files /dev/null and b/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-04 differ diff --git a/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-05 b/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-05 new file mode 100644 index 00000000000000..28db5306d7d7ea Binary files /dev/null and b/fe/custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts/part-05 differ diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 57a4eb53523d67..6767ec9f642632 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -138,6 +138,11 @@ under the License. fe-authentication-role-mapping ${project.version} + + org.apache.horn4j + horn4j + 1.0.0-SNAPSHOT + org.springframework.boot spring-boot-devtools diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index e61155ed7ff33a..46f1aa6b936c39 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -285,6 +285,7 @@ HELP: 'HELP'; HISTOGRAM: 'HISTOGRAM'; HLL: 'HLL'; HLL_UNION: 'HLL_UNION'; +HORN: 'HORN'; HOSTNAME: 'HOSTNAME'; HOTSPOT: 'HOTSPOT'; HOUR: 'HOUR'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index a710576d1ddf24..e4137041d40d4b 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1177,6 +1177,7 @@ planType | SHAPE | MEMO | DISTRIBUTED + | HORN | ALL // default type ; @@ -2160,6 +2161,7 @@ nonReserved | HINT_START | HISTOGRAM | HLL_UNION + | HORN | HOSTNAME | HOTSPOT | HOUR diff --git a/fe/fe-core/src/main/java/org/apache/doris/horn/HornOptimizationContext.java b/fe/fe-core/src/main/java/org/apache/doris/horn/HornOptimizationContext.java new file mode 100644 index 00000000000000..f8fed6ead57c42 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/horn/HornOptimizationContext.java @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.horn; + +import org.apache.horn4j.thrift.TEngineType; +import org.apache.horn4j.thrift.TExplainLevel; +import org.apache.horn4j.thrift.TQueryOptionConfig; +import org.apache.horn4j.thrift.TTable; +import org.apache.horn4j.thrift.TTableBatch; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.hint.LeadingHint; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Horn optimization context shared across the input-translation, JNI, and */ +public class HornOptimizationContext { + + private final CascadesContext cascadesContext; + + /** Table metadata + statistics */ + private TTableBatch tableBatch = new TTableBatch(); + + /** Horn optimizer configuration */ + private TQueryOptionConfig queryOptionConfig; + + /** forward 每个 LogicalOlapScan 的裁剪后分区集,按 relationId(= thrift */ + private final Map> scanPrunedPartitions = new HashMap<>(); + + /** backward 阶段 horn 端 scalar_unique_id → 反译产生的 Doris Slot 的全局映射， */ + private final Map scalarUidToSlot = new HashMap<>(); + + /** backward 阶段 horn 端 agg fn 的 scalar_unique_id → 反译出的 Doris */ + private final Map scalarUidToAggFunc = new HashMap<>(); + + public HornOptimizationContext(CascadesContext cascadesContext) { + this.cascadesContext = cascadesContext; + this.queryOptionConfig = buildDefaultQueryOptionConfig(); + applyLeadingHint(this.queryOptionConfig); + } + + /** 把 SQL 中 {@code /*+ LEADING(t1 t2 t3) *​/} hint 转发给 horn —— 让 horn 内部 */ + private void applyLeadingHint(TQueryOptionConfig config) { + if (!cascadesContext.isLeadingJoin()) { + return; + } + LeadingHint leading = (LeadingHint) cascadesContext.getHintMap().get("Leading"); + if (leading != null && !leading.getTablelist().isEmpty()) { + + config.setJoin_leading_string(new ArrayList<>(leading.getTablelist())); + } + } + + public CascadesContext getCascadesContext() { + return cascadesContext; + } + + public TTableBatch getTableBatch() { + return tableBatch; + } + + public void setTableBatch(TTableBatch tableBatch) { + this.tableBatch = tableBatch; + } + + public void putScanPrunedPartitions(long relationId, List prunedPartitionIds) { + scanPrunedPartitions.put(relationId, prunedPartitionIds); + } + + public List getScanPrunedPartitions(long relationId) { + return scanPrunedPartitions.get(relationId); + } + + public void addTable(TTable ttable) { + if (tableBatch.getTables() == null) { + tableBatch.setTables(new ArrayList<>()); + } + tableBatch.getTables().add(ttable); + } + + public Map getScalarUidToSlot() { + return scalarUidToSlot; + } + + public Map getScalarUidToAggFunc() { + return scalarUidToAggFunc; + } + + public TQueryOptionConfig getQueryOptionConfig() { + return queryOptionConfig; + } + + public TExplainLevel getExplainLevel() { + return TExplainLevel.EXTRACTED_PLAN; + } + + private TQueryOptionConfig buildDefaultQueryOptionConfig() { + TQueryOptionConfig config = new TQueryOptionConfig(); + config.setHorn_dphyper_enable(true); + config.setHorn_statistics_default_selectivity(0.4); + config.setHorn_statistics_damping_factor_filter(0.75); + config.setHorn_statistics_damping_factor_groupby(0.75); + config.setHorn_statistics_damping_factor_join(0.1); + // 默认允许无统计的表也走 horn（用 default selectivity 等估算） + config.setEnable_horn_no_statistics_support(true); + config.setLocal_time_zone(cascadesContext.getConnectContext() + .getSessionVariable().getTimeZone()); + config.setMt_dop(1); + config.setEngine_type(TEngineType.kDoris); + boolean detailInfo = cascadesContext.getConnectContext() + .getSessionVariable().enableHornDetailInfo; + config.setEnable_horn_cascade_log(detailInfo); + config.setEnable_horn_cascade_detail_log(detailInfo); + config.setEnable_horn_stats_log(detailInfo); + config.setHorn_print_memo_after_optimization(detailInfo); // detail 模式下 dump memo（调试 cascade 用） + config.setEnable_horn_profile(cascadesContext.getConnectContext() + .getSessionVariable().enableHornProfile); + config.setExplain_level(getExplainLevel()); + return config; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/horn/HornOptimizer.java b/fe/fe-core/src/main/java/org/apache/doris/horn/HornOptimizer.java new file mode 100644 index 00000000000000..6c3fabb7554997 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/horn/HornOptimizer.java @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.horn; + +import org.apache.doris.common.UserException; +import org.apache.doris.horn.doris2horn.HornLogicalPlanThriftBuilder; +import org.apache.doris.horn.horn2doris.DorisPhysicalPlanBuilder; +import org.apache.horn4j.thrift.TFlattenedExpression; +import org.apache.horn4j.thrift.THornOptimizeProfile; +import org.apache.horn4j.thrift.THornOptimizeResult; +import org.apache.horn4j.HornNative; +import org.apache.doris.metric.HornMetric; +import org.apache.doris.metric.MetricRepo; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TSerializer; + +/** Doris-side entry to the Horn CBO optimizer */ +public class HornOptimizer { + + private static final Logger LOG = LogManager.getLogger(HornOptimizer.class); + + /** Horn 优化失败的分类，对齐 Horn 三段式流水线（正向翻译 → Horn 优化 → 反向翻译） */ + public enum HornFailureKind { + // A: Doris→Horn 正向翻译失败 → 计 translate_plan_error（Doris 集成翻译层缺口） + HORN_FORWARD_ERROR, + // B: Horn 声明不处理（HornNotHandleError/External） → 计 fallback（对齐 kernel countHornFallBack） + HORN_NOT_HANDLE, + // C: Horn 优化器内部错误（HornOptimizerError） → 计 error（对齐 kernel countHornError） + HORN_OPTIMIZE_ERROR, + // D: Horn→Doris 反向翻译失败 → 计 translate_plan_error（Doris 集成翻译层缺口） + HORN_BACKWARD_ERROR, + // 兜底：native 抛异常但分类前缀识别不出（JNI 胶水错 / 未知 std 异常 / 契约漂移） → 计 error + UNKNOWN + } + + private final CascadesContext cascadesContext; + private String hornExplainString; + private String fallbackReason; + // 仅在 optimize() 失败（返回 null）时赋值并被 NereidsPlanner 读取；成功路径不读它。 + private HornFailureKind failureKind; + + public HornOptimizer(CascadesContext cascadesContext) { + this.cascadesContext = cascadesContext; + } + + public String getHornExplainString() { + return hornExplainString; + } + + public String getFallbackReason() { + return fallbackReason; + } + + public HornFailureKind getFailureKind() { + return failureKind; + } + + public PhysicalPlan optimize(LogicalPlan logicalPlan) throws UserException { + HornOptimizationContext hornCtx = new HornOptimizationContext(cascadesContext); + + // 阶段 A：Doris LogicalPlan → TFlattenedExpression → 序列化。 + byte[] planBytes; + byte[] tableBatchBytes; + byte[] queryOptionBytes; + try { + LOG.info("Horn CBO: translating Doris plan to Horn input"); + HornLogicalPlanThriftBuilder planVisitor = new HornLogicalPlanThriftBuilder(hornCtx); + TFlattenedExpression plan = planVisitor.translate(logicalPlan); + + TSerializer serializer = new TSerializer(); + planBytes = serializer.serialize(plan); + tableBatchBytes = serializer.serialize(hornCtx.getTableBatch()); + queryOptionBytes = serializer.serialize(hornCtx.getQueryOptionConfig()); + } catch (Exception e) { + failureKind = HornFailureKind.HORN_FORWARD_ERROR; + fallbackReason = e.getMessage(); + LOG.info("Horn CBO: forward-translation failed, fallback: {}", fallbackReason); + return null; + } + + // 阶段 B/C：JNI 调 Horn native（in-process horn4j）。 + byte[] resultBytes; + try { + LOG.info("Horn CBO: calling Horn via horn4j JNI"); + resultBytes = new HornNative().optimize(planBytes, tableBatchBytes, queryOptionBytes); + } catch (Exception e) { + // 依据 kernel HornError 的分类前缀（既定契约：horn_error.h GetErrorTypeString 把类型拼进 message） + String msg = e.getMessage(); + if (msg != null && msg.contains("HornNotHandle")) { + failureKind = HornFailureKind.HORN_NOT_HANDLE; + } else if (msg != null && msg.contains("HornOptimizerError")) { + failureKind = HornFailureKind.HORN_OPTIMIZE_ERROR; + } else { + failureKind = HornFailureKind.UNKNOWN; + } + fallbackReason = msg; + LOG.warn("Horn CBO: native failed ({}), fallback: {}", failureKind, fallbackReason); + return null; + } + + // 阶段 D：反序列化 + 消费 profile + Horn output → Doris PhysicalPlan。 + try { + LOG.info("Horn CBO: rebuilding Doris plan from Horn output"); + THornOptimizeResult hornResult = new THornOptimizeResult(); + TDeserializer deserializer = new TDeserializer(); + deserializer.deserialize(hornResult, resultBytes); + + hornExplainString = hornResult.getExplain_string(); + if (hornExplainString != null) { + LOG.info("Horn CBO explain:\n{}", hornExplainString); + } + + consumeProfile(hornResult); + + DorisPhysicalPlanBuilder builder = new DorisPhysicalPlanBuilder(hornCtx); + PhysicalPlan physicalPlan = builder.build(hornResult.getTfexpr()); + + LOG.info("Horn CBO: optimization complete"); + return physicalPlan; + } catch (Exception e) { + failureKind = HornFailureKind.HORN_BACKWARD_ERROR; + // 反译失败时 horn C++ 已产出计划文本，getHornExplainString() 已被上面赋值， + fallbackReason = e.getMessage(); + LOG.warn("Horn CBO: backward-translation failed, fallback: {}", fallbackReason, e); + return null; + } + } + + /** 消费 Horn profile → 更新 FE 量表指标（profile 无条件产出，无需任何 session 开关） */ + private static void consumeProfile(THornOptimizeResult hornResult) { + if (hornResult.isSetProfile() && MetricRepo.isInit) { + THornOptimizeProfile p = hornResult.getProfile(); + HornMetric.GAUGE_HORN_OPTIMIZE_LATENCY_NS.setValue(p.getHorn_optimize_latency()); // #6 + HornMetric.GAUGE_HORN_CASCADE_GROUP_COUNT.setValue(p.getCascade_group_count()); // #7 + HornMetric.GAUGE_HORN_CASCADE_GROUP_EXPR_COUNT.setValue( + p.getCascade_group_expression_count()); // #8 + HornMetric.GAUGE_HORN_CASCADE_SAFE_TO_PRUNE_RATIO.setValue( + p.getCascade_safe_to_prune_ratio()); // #9 + HornMetric.GAUGE_HORN_SCHEDULER_JOB_COUNT.setValue(p.getScheduler_job_count()); // #10 + HornMetric.GAUGE_HORN_DPHYPER_CSG_CMP_COUNT.setValue(p.getDphyper_csg_cmp_count()); // #11 + HornMetric.GAUGE_HORN_DPHYPER_ENUM_SUCCESS_RATIO.setValue( + p.getDphyper_enumeration_successful_ratio()); // #12 + } + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/horn/doris2horn/Doris2HornUtils.java b/fe/fe-core/src/main/java/org/apache/doris/horn/doris2horn/Doris2HornUtils.java new file mode 100644 index 00000000000000..7cf0c95d93fc24 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/horn/doris2horn/Doris2HornUtils.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.horn.doris2horn; + +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Expression; + +/** doris2horn 正向翻译路径下的纯静态辅助方法集中点（对称 horn2doris 侧的 */ +public final class Doris2HornUtils { + + private Doris2HornUtils() { + } + + /** 剥掉一层 {@link Alias} 取内部表达式：Alias 在 horn wire 上是透明的（exprId/name */ + public static Expression unwrapExpr(Expression expr) { + return (expr instanceof Alias) ? ((Alias) expr).child() : expr; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/horn/doris2horn/DorisTypeToHornConverter.java b/fe/fe-core/src/main/java/org/apache/doris/horn/doris2horn/DorisTypeToHornConverter.java new file mode 100644 index 00000000000000..a60fd3de4717e1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/horn/doris2horn/DorisTypeToHornConverter.java @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.horn.doris2horn; + +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.horn4j.thrift.TColumnType; +import org.apache.horn4j.thrift.TPrimitiveType; +import org.apache.horn4j.thrift.TScalarType; +import org.apache.horn4j.thrift.TTypeNode; +import org.apache.horn4j.thrift.TTypeNodeType; +import org.apache.doris.nereids.types.DataType; + +import java.util.Collections; + +/** Converts Doris Nereids/catalog types to Horn TColumnType (Thrift) */ +public class DorisTypeToHornConverter { + + /** Convert a Nereids DataType to Horn TColumnType */ + public static TColumnType convert(DataType dataType) { + return wrap(mapPrimitive(dataType.toCatalogDataType())); + } + + /** Convert a legacy Doris catalog Type to Horn TColumnType (used for table metadata) */ + public static TColumnType convertCatalogType(Type catalogType) { + return wrap(mapPrimitive(catalogType)); + } + + /** Map a Doris {@link Type} to a Horn {@link TScalarType} */ + private static TScalarType mapPrimitive(Type catalogType) { + TScalarType scalarType = new TScalarType(); + switch (catalogType.getPrimitiveType()) { + case BOOLEAN: + scalarType.setType(TPrimitiveType.BOOLEAN); + return scalarType; + case TINYINT: + scalarType.setType(TPrimitiveType.TINYINT); + return scalarType; + case SMALLINT: + scalarType.setType(TPrimitiveType.SMALLINT); + return scalarType; + case INT: + scalarType.setType(TPrimitiveType.INT); + return scalarType; + case BIGINT: + scalarType.setType(TPrimitiveType.BIGINT); + return scalarType; + case FLOAT: + scalarType.setType(TPrimitiveType.FLOAT); + return scalarType; + case DOUBLE: + scalarType.setType(TPrimitiveType.DOUBLE); + return scalarType; + case DATE: + case DATEV2: + scalarType.setType(TPrimitiveType.DATE); + return scalarType; + case NULL_TYPE: + scalarType.setType(TPrimitiveType.NULL_TYPE); + return scalarType; + case INVALID_TYPE: + // 无值表达式（WindowFrame / FrameBoundary 等）的 placeholder： + scalarType.setType(TPrimitiveType.INVALID_TYPE); + return scalarType; + case STRING: + scalarType.setType(TPrimitiveType.STRING); + return scalarType; + case CHAR: + scalarType.setType(TPrimitiveType.CHAR); + scalarType.setLen(((ScalarType) catalogType).getLength()); + return scalarType; + case VARCHAR: + scalarType.setType(TPrimitiveType.VARCHAR); + scalarType.setLen(((ScalarType) catalogType).getLength()); + return scalarType; + case DECIMALV2: + case DECIMAL32: + case DECIMAL64: + case DECIMAL128: + scalarType.setType(TPrimitiveType.DECIMAL); + scalarType.setPrecision(((ScalarType) catalogType).getPrecision()); + scalarType.setScale(((ScalarType) catalogType).getScalarScale()); + return scalarType; + default: + // 暂不支持 LARGEINT、DECIMAL256、TIME、TIMESTAMP 等类型 + throw new UnsupportedOperationException( + "Horn: unsupported Doris type: " + catalogType); + } + } + + private static TColumnType wrap(TScalarType scalarType) { + TTypeNode node = new TTypeNode(); + node.setType(TTypeNodeType.SCALAR); + node.setScalar_type(scalarType); + TColumnType result = new TColumnType(); + result.setTypes(Collections.singletonList(node)); + return result; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/horn/doris2horn/HornLogicalPlanThriftBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/horn/doris2horn/HornLogicalPlanThriftBuilder.java new file mode 100644 index 00000000000000..03c84e38e9daf1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/horn/doris2horn/HornLogicalPlanThriftBuilder.java @@ -0,0 +1,798 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.horn.doris2horn; + +import org.apache.doris.horn.HornOptimizationContext; +import org.apache.doris.catalog.ColocateTableIndex; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.OlapTable; +import org.apache.horn4j.thrift.TColumn; +import org.apache.horn4j.thrift.TColumnStats; +import org.apache.horn4j.thrift.TColumnValue; +import org.apache.horn4j.thrift.TDistributionSpec; +import org.apache.horn4j.thrift.TDistributionType; +import org.apache.horn4j.thrift.TDorisHashSpec; +import org.apache.horn4j.thrift.TSpecialHashSpec; +import org.apache.horn4j.thrift.TSpecialHashSpecKind; +import org.apache.horn4j.thrift.TExpression; +import org.apache.horn4j.thrift.TFlattenedExpression; +import org.apache.horn4j.thrift.TLogicalAgg; +import org.apache.horn4j.thrift.TLogicalFilter; +import org.apache.horn4j.thrift.TLogicalGrouping; +import org.apache.horn4j.thrift.TLogicalJoin; +import org.apache.horn4j.thrift.TLogicalLimit; +import org.apache.horn4j.thrift.TLogicalProject; +import org.apache.horn4j.thrift.TLogicalSetOp; +import org.apache.horn4j.thrift.TLogicalWindow; +import org.apache.horn4j.thrift.TOrderSpec; +import org.apache.horn4j.thrift.TLogicalScan; +import org.apache.horn4j.thrift.TLogicalTableScan; +import org.apache.horn4j.thrift.TLogicalValueScan; +import org.apache.horn4j.thrift.TOperator; +import org.apache.horn4j.thrift.TOperatorType; +import org.apache.horn4j.thrift.TOperatorUnion; +import org.apache.horn4j.thrift.TScanTableType; +import org.apache.horn4j.thrift.TTable; +import org.apache.horn4j.thrift.TTableStats; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.WindowFrame; +import org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow; +import org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow.WindowFrameGroup; +import org.apache.doris.nereids.rules.rewrite.MultiDistinctFunctionStrategy; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; +import org.apache.doris.nereids.trees.plans.LimitPhase; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.algebra.Repeat; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; +import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; +import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.properties.DistributionSpec; +import org.apache.doris.nereids.properties.DistributionSpecHash; +import org.apache.doris.horn.horn2doris.Horn2DorisUtils; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.statistics.ColumnStatistic; +import org.apache.doris.statistics.StatisticsCache; + +import com.google.common.collect.Lists; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** Visitor-based Doris LogicalPlan → Horn TFlattenedExpression translator */ +public class HornLogicalPlanThriftBuilder extends DefaultPlanVisitor> { + + private static final org.apache.logging.log4j.Logger LOG = + org.apache.logging.log4j.LogManager.getLogger(HornLogicalPlanThriftBuilder.class); + + private final HornOptimizationContext hornCtx; + private final HornScalarThriftBuilder scalarTranslator; + + public HornLogicalPlanThriftBuilder(HornOptimizationContext hornCtx) { + this.hornCtx = hornCtx; + this.scalarTranslator = new HornScalarThriftBuilder(); + } + + /** Translate the whole LogicalPlan tree to TFlattenedExpression */ + public TFlattenedExpression translate(LogicalPlan logicalPlan) { + TFlattenedExpression result = new TFlattenedExpression(); + List texprs = new ArrayList<>(); + // sink 语义 = identity Project，承载 SELECT 列序 + logicalPlan.accept(this, texprs); + result.setTexprs(texprs); + return result; + } + + @Override + public Void visit(Plan plan, List texprs) { + throw new UnsupportedOperationException( + "Horn: unsupported Nereids node: " + plan.getClass().getSimpleName()); + } + + /** Append a TExpression for {@code plan}, then recurse into children pre-order */ + private void emit(Plan plan, TOperatorType opType, TOperatorUnion opUnion, + List texprs) { + TExpression texpr = new TExpression(); + TOperator top = new TOperator(); + top.setOp_type(opType); + top.setOp_union(opUnion); + texpr.setOp(top); + texpr.setArity(plan.children().size()); + + // FE-declared output column refs: each entry is a kColumnRef-shaped TOperator + List outputList = new ArrayList<>(); + for (Slot slot : plan.getOutput()) { + if (slot instanceof SlotReference) { + outputList.add(scalarTranslator.visitSlotReference((SlotReference) slot, null)); + } + } + if (!outputList.isEmpty()) { + texpr.setOutput_list(outputList); + } + + texprs.add(texpr); + + for (Plan child : plan.children()) { + child.accept(this, texprs); + } + } + + @Override + public Void visitLogicalOlapScan(LogicalOlapScan scan, List texprs) { + TOperatorUnion opUnion = new TOperatorUnion(); + TLogicalTableScan tscan = new TLogicalTableScan(); + OlapTable table = scan.getTable(); + + // ====== DEBUG: LogicalOlapScan 此时是否已 partition pruning ====== + LOG.warn("HORN_FWD_SCAN table={} totalPartitions={} selectedPartitionIds={} selectedTabletIds.size={}", + table.getName(), table.getPartitionIds().size(), + scan.getSelectedPartitionIds(), scan.getSelectedTabletIds().size()); + // ====== END DEBUG ====== + + tscan.setDatabase_name(scan.getQualifier().get(1)); + tscan.setTable_name(table.getName()); + tscan.setTable_alias(table.getName()); + tscan.setTable_type(TScanTableType.kTciTable); + // schema_id 不设：BE expression_builder.cc:40 默认读 -1，FindTablePair 用 + + // scan_list：scan 自有输出列(含列裁剪)的 ColumnRef(带 external_slot_id)， + List scanList = new ArrayList<>(); + for (Slot slot : scan.getOutput()) { + if (slot instanceof SlotReference) { + scanList.add(scalarTranslator.visitSlotReference((SlotReference) slot, null)); + } + } + tscan.setScan_list(scanList); + + // external_relation_id: FE 自有 per-scan opaque id（relationId） + long relationId = scan.getRelationId().asInt(); + tscan.setExternal_relation_id(relationId); + hornCtx.putScanPrunedPartitions(relationId, new ArrayList<>(scan.getSelectedPartitionIds())); + + // task #227 sort elimination: emit base table 物理排序键 (OLAP tablet 内部 + KeysType keysType = table.getKeysType(); + if (keysType == KeysType.DUP_KEYS || keysType == KeysType.UNIQUE_KEYS + || keysType == KeysType.AGG_KEYS) { + // Doris OLAP 的 key 列物理上恒为「升序存储、NULL 作最小值（nulls first）」，据此声明 scan + List keyOrderKeys = new ArrayList<>(); + for (Column keyCol : table.getBaseSchemaKeyColumns()) { + SlotReference keySlot = findOutputSlotByColumn(scan.getOutput(), keyCol.getName()); + if (keySlot == null) { + break; + } + keyOrderKeys.add(new OrderKey(keySlot, /*isAsc=*/true, /*nullFirst=*/true)); + } + if (!keyOrderKeys.isEmpty()) { + tscan.setBase_table_order_spec(buildOrderSpec(keyOrderKeys)); + } + } + + // scan 自有 distribution_spec：HASH 表 → kHashed + 分桶列 ColumnRef(getDistributionColumns + DistributionInfo distInfo = table.getDefaultDistributionInfo(); + TDistributionSpec distSpec = new TDistributionSpec(); + if (distInfo instanceof HashDistributionInfo) { + distSpec.setDistribution_type(TDistributionType.kHashed); + distSpec.setBucket_num(distInfo.getBucketNum()); + List hashColumns = new ArrayList<>(); + for (Column distCol : ((HashDistributionInfo) distInfo).getDistributionColumns()) { + SlotReference distSlot = findOutputSlotByColumn(scan.getOutput(), distCol.getName()); + if (distSlot != null) { + hashColumns.add(scalarTranslator.visitSlotReference(distSlot, null)); + } + } + distSpec.setHash_columns(hashColumns); + // colocate 物理身份：属同一 stable colocate group 的所有 scan 拿同一个 grpId + ColocateTableIndex colocateIndex = Env.getCurrentColocateIndex(); + long colocateGroupId = -1; + if (colocateIndex.isColocateTable(table.getId()) + && !colocateIndex.isGroupUnstable(colocateIndex.getGroup(table.getId()))) { + colocateGroupId = colocateIndex.getGroup(table.getId()).grpId; + } + // FE 直接把整个 colocate 物理身份组装成 TDorisHashSpec 灌进 special_hash_spec + TDorisHashSpec dorisHashSpec = new TDorisHashSpec(); + dorisHashSpec.setColocate_group_id(colocateGroupId); + dorisHashSpec.setTable_id(table.getId()); + dorisHashSpec.setSelected_partition_ids(new ArrayList<>(scan.getSelectedPartitionIds())); + TSpecialHashSpec specialHashSpec = new TSpecialHashSpec(); + specialHashSpec.setKind(TSpecialHashSpecKind.kDoris); + specialHashSpec.setDoris_hash_spec(dorisHashSpec); + distSpec.setSpecial_hash_spec(specialHashSpec); + } else { + distSpec.setDistribution_type(TDistributionType.kRandom); + } + tscan.setDistribution_spec(distSpec); + + buildCatalog(table, tscan.getDatabase_name(), tscan.getTable_name()); + + TLogicalScan tLogicalScan = new TLogicalScan(); + tLogicalScan.setLogical_table_scan(tscan); + opUnion.setLogical_scan(tLogicalScan); + emit(scan, TOperatorType.kLogicalTableGet, opUnion, texprs); + return null; + } + + @Override + public Void visitLogicalOneRowRelation(LogicalOneRowRelation oneRow, List texprs) { + // SELECT 1, 'x' → LogicalValueGet, value_lists = [[lit, ...]] + buildValueGet(oneRow, java.util.Collections.singletonList(oneRow.getProjects()), texprs); + return null; + } + + @Override + public Void visitLogicalEmptyRelation(LogicalEmptyRelation emptyRelation, List texprs) { + // SELECT * FROM t LIMIT 0 / WHERE false → LogicalValueGet, value_lists = [] + buildValueGet(emptyRelation, java.util.Collections.emptyList(), texprs); + return null; + } + + /** 两个 Doris constants-source 节点（OneRow / Empty）→ horn */ + private void buildValueGet(Plan plan, List> rows, List texprs) { + TLogicalValueScan valueScan = new TLogicalValueScan(); + if (!rows.isEmpty()) { + List> valueLists = new ArrayList<>(rows.size()); + for (List row : rows) { + List rowCells = new ArrayList<>(row.size()); + for (NamedExpression cell : row) { + Expression literalExpr = Doris2HornUtils.unwrapExpr(cell); + rowCells.add(scalarTranslator.translate(literalExpr)); + } + valueLists.add(rowCells); + } + valueScan.setValue_lists(valueLists); + } + List outputSchema = new ArrayList<>(); + for (Slot slot : plan.getOutput()) { + if (slot instanceof SlotReference) { + outputSchema.add(scalarTranslator.visitSlotReference((SlotReference) slot, null)); + } + } + valueScan.setOutput_list(outputSchema); + + TLogicalScan scanWrapper = new TLogicalScan(); + scanWrapper.setLogical_value_scan(valueScan); + TOperatorUnion opUnion = new TOperatorUnion(); + opUnion.setLogical_scan(scanWrapper); + emit(plan, TOperatorType.kLogicalValueGet, opUnion, texprs); + } + + /** Build TTable (schema + col stats + table stats) for horn's StaticMataProvider */ + private void buildCatalog(OlapTable table, String dbName, String tblName) { + TTable ttable = new TTable(); + ttable.setDb_name(dbName); + ttable.setTbl_name(tblName); + + // 以下 3 个 thrift 字段 horn BE 接到后 ParseTableDescriptor 解析存进 + + ConnectContext ctx = hornCtx.getCascadesContext().getConnectContext(); + StatisticsCache statsCache = Env.getCurrentEnv().getStatisticsCache(); + long catalogId = table.getDatabase().getCatalog().getId(); + long dbId = table.getDatabase().getId(); + long tblId = table.getId(); + long idxId = table.getBaseIndexId(); + + List baseSchema = table.getBaseSchema(); + List columns = new ArrayList<>(); + boolean allColumnStatsAvailable = true; + long tableRowCount = -1; + for (int i = 0; i < baseSchema.size(); i++) { + Column col = baseSchema.get(i); + ColumnStatistic colStat = statsCache.getColumnStatistics( + catalogId, dbId, tblId, idxId, col.getName(), ctx); + boolean columnStatsAvailable = colStat != null && colStat != ColumnStatistic.UNKNOWN; + if (columnStatsAvailable && tableRowCount < 0) { + tableRowCount = (long) colStat.count; + } else if (!columnStatsAvailable) { + allColumnStatsAvailable = false; + } + + TColumn tcol = new TColumn(); + tcol.setColumnName(col.getName()); + tcol.setColumnType(DorisTypeToHornConverter.convertCatalogType(col.getType())); + tcol.setPosition(i); + if (columnStatsAvailable) { + TColumnStats tcolStats = new TColumnStats(); + tcolStats.setAvg_size(colStat.avgSizeByte); + tcolStats.setMax_size(col.getType().getLength()); + tcolStats.setNum_distinct_values((long) colStat.ndv); + tcolStats.setNum_nulls((long) colStat.numNulls); + TColumnValue low = new TColumnValue(); + low.setDouble_val(colStat.minValue); + tcolStats.setLow_value(low); + TColumnValue high = new TColumnValue(); + high.setDouble_val(colStat.maxValue); + tcolStats.setHigh_value(high); + tcol.setCol_stats(tcolStats); + } + columns.add(tcol); + } + ttable.setColumns(columns); + + TTableStats tableStats = new TTableStats(); + tableStats.setNum_rows(allColumnStatsAvailable ? tableRowCount : -1); + ttable.setTable_stats(tableStats); + + // table_id / partition_num 不再传:都是 horn C++ 从不读的死字段 + hornCtx.addTable(ttable); + } + + @Override + public Void visitLogicalFilter(LogicalFilter filter, List texprs) { + TOperatorUnion opUnion = new TOperatorUnion(); + TLogicalFilter tfilter = new TLogicalFilter(); + tfilter.setPredicate_list( + scalarTranslator.translateList(new ArrayList<>(filter.getConjuncts()))); + opUnion.setLogical_filter(tfilter); + emit(filter, TOperatorType.kLogicalSelect, opUnion, texprs); + return null; + } + + @Override + public Void visitLogicalProject(LogicalProject project, List texprs) { + buildProject(project, project.getProjects(), project.getOutput(), texprs); + return null; + } + + /** identity Project，sink.outputExprs 即 SELECT 列序、对应 Project 的 */ + @Override + public Void visitLogicalResultSink(LogicalResultSink sink, List texprs) { + buildProject(sink, sink.getOutputExprs(), sink.getOutput(), texprs); + return null; + } + + /** 把 plan 当成 LogicalProject emit：projects 写 project_list、output 写 */ + private void buildProject(Plan plan, List projects, + List output, List texprs) { + TLogicalProject projectOp = new TLogicalProject(); + projectOp.setProject_list(scalarTranslator.translateList(projects)); + List rewriteList = new ArrayList<>(); + for (Slot slot : output) { + if (slot instanceof SlotReference) { + rewriteList.add(scalarTranslator.visitSlotReference((SlotReference) slot, null)); + } + } + projectOp.setRewrite_list(rewriteList); + TOperatorUnion opUnion = new TOperatorUnion(); + opUnion.setLogical_project(projectOp); + emit(plan, TOperatorType.kLogicalProject, opUnion, texprs); + } + + /** Doris LogicalAggregate → horn TLogicalAgg（单层 forward，horn cascades */ + @Override + public Void visitLogicalAggregate(LogicalAggregate agg, List texprs) { + // 由 visitLogicalRepeat 单独 emit TLogicalGrouping，所以这里不再 fallback + if (hasDistinctAggFn(agg)) { + LogicalAggregate converted = MultiDistinctFunctionStrategy.rewrite(agg); + if (hasDistinctAggFn(converted)) { + throw new UnsupportedOperationException( + "Horn: distinct agg fn not convertible to multi_distinct"); + } + return converted.accept(this, texprs); + } + TLogicalAgg tagg = new TLogicalAgg(); + tagg.setGroup_by_exprs(scalarTranslator.translateList(agg.getGroupByExpressions())); + // 从 outputExpressions 抽出 AggregateFunction（通常包在 Alias 里），逐个 + List aggFuncs = new ArrayList<>(); + List aggComputedCols = new ArrayList<>(); + for (NamedExpression out : agg.getOutputExpressions()) { + Expression inner = Doris2HornUtils.unwrapExpr(out); + if (inner instanceof AggregateFunction) { + aggFuncs.add(scalarTranslator.translate(inner)); + aggComputedCols.add(scalarTranslator.visitSlotReference( + (SlotReference) out.toSlot(), null)); + } + } + tagg.setAggregate_functions(aggFuncs); + tagg.setAgg_computed_cols(aggComputedCols); + tagg.setIs_split(false); + TOperatorUnion opUnion = new TOperatorUnion(); + opUnion.setLogical_agg(tagg); + emit(agg, TOperatorType.kLogicalAgg, opUnion, texprs); + return null; + } + + /** 降级成 {@code Agg(group by 全列) → UnionAll}，故到此 qualifier 恒为 ALL；且经 */ + @Override + public Void visitLogicalUnion(LogicalUnion union, List texprs) { + if (union.getQualifier() != Qualifier.ALL) { + throw new UnsupportedOperationException( + "Horn: non-ALL union should have been lowered by BuildAggForUnion"); + } + // 纯常量 union（0 孩子、仅 constantExprsList，如 SELECT 1 UNION ALL SELECT 2）暂不支持： + if (union.children().isEmpty()) { + throw new UnsupportedOperationException( + "Horn: constant-only union (no child) not supported (cc StatisticsDerive SIGSEGV)"); + } + emitSetOp(union, TOperatorType.kLogicalNarrayUnionAll, texprs); + return null; + } + + /** MergeSetOperations 可把同型 intersect 合并成 N 元单节点；cc 的 implement xform 已改 PatternMultiLeaf */ + @Override + public Void visitLogicalIntersect(LogicalIntersect intersect, List texprs) { + // intersect 恒 DISTINCT（Doris 绑定期拒 ALL）；防御性断言，非 DISTINCT → fallback。 + if (intersect.getQualifier() != Qualifier.DISTINCT) { + throw new UnsupportedOperationException( + "Horn: intersect must be DISTINCT (Doris rejects ALL at binding)"); + } + emitSetOp(intersect, TOperatorType.kLogicalIntersect, texprs); + return null; + } + + /** Doris LogicalExcept → horn TLogicalSetOp（op_type=kLogicalExcept） */ + @Override + public Void visitLogicalExcept(LogicalExcept except, List texprs) { + // except 恒 DISTINCT（Doris 绑定期拒 ALL）；防御性断言，非 DISTINCT → fallback。 + if (except.getQualifier() != Qualifier.DISTINCT) { + throw new UnsupportedOperationException( + "Horn: except must be DISTINCT (Doris rejects ALL at binding)"); + } + emitSetOp(except, TOperatorType.kLogicalExcept, texprs); + return null; + } + + /** 所有 N 元 set op（UNION ALL / INTERSECT / EXCEPT）共用：建 TLogicalSetOp（children_output_list） */ + private void emitSetOp(LogicalSetOperation setOp, TOperatorType opType, + List texprs) { + TLogicalSetOp tsetop = new TLogicalSetOp(); + List> childrenOut = new ArrayList<>(); + for (List oneChild : setOp.getRegularChildrenOutputs()) { + childrenOut.add(scalarTranslator.translateList(oneChild)); + } + tsetop.setChildren_output_list(childrenOut); + if (setOp instanceof LogicalUnion) { + List> constExprsList = ((LogicalUnion) setOp).getConstantExprsList(); + if (!constExprsList.isEmpty()) { + List> constRows = new ArrayList<>(); + for (List row : constExprsList) { + constRows.add(scalarTranslator.translateList(row)); + } + tsetop.setChildren_const_expr_list(constRows); + } + } + TOperatorUnion opUnion = new TOperatorUnion(); + opUnion.setLogical_set_op(tsetop); + emit(setOp, opType, opUnion, texprs); + } + + /** NormalizeRepeat 已 desugar： */ + @Override + public Void visitLogicalRepeat(LogicalRepeat repeat, List texprs) { + // 1. grouping_set_list + List> tGroupingSets = new ArrayList<>(repeat.getGroupingSets().size()); + for (List set : repeat.getGroupingSets()) { + tGroupingSets.add(scalarTranslator.translateList(set)); + } + + // 2. grouping_expr_list = flatten distinct + Set groupingExprSet = new LinkedHashSet<>(); + for (List set : repeat.getGroupingSets()) { + groupingExprSet.addAll(set); + } + List tGroupingExprs = new ArrayList<>(groupingExprSet.size()); + for (Expression e : groupingExprSet) { + tGroupingExprs.add(scalarTranslator.translate(e)); + } + + // 3 + SlotReference groupingIdSlot = repeat.getGroupingId().get(); + TOperator tGroupingIdFunc = scalarTranslator.visitSlotReference(groupingIdSlot, null); + + // agg_fun_input_cols：repeat.outputExpressions 里「裸 SlotReference 且不是 grouping key」的列， + boolean hasGroupingPrefix = false; + List tAggFunInputCols = new ArrayList<>(); + for (NamedExpression out : repeat.getOutputExpressions()) { + Expression inner = Doris2HornUtils.unwrapExpr(out); + if (inner instanceof SlotReference && !groupingExprSet.contains(inner)) { + tAggFunInputCols.add(scalarTranslator.visitSlotReference((SlotReference) inner, null)); + } else if (inner instanceof GroupingScalarFunction) { + hasGroupingPrefix = true; + } + } + + TLogicalGrouping tlg = new TLogicalGrouping(); + tlg.setGrouping_set_list(tGroupingSets); + tlg.setGrouping_expr_list(tGroupingExprs); + tlg.setGrouping_id_func(Lists.newArrayList(tGroupingIdFunc)); + if (!tAggFunInputCols.isEmpty()) { + tlg.setAgg_fun_input_cols(tAggFunInputCols); + } + TOperatorUnion opUnion = new TOperatorUnion(); + opUnion.setLogical_grouping(tlg); + + // 装 outputExpressions（含 `Grouping(..) AS PREFIX`，visitAlias 剥到函数）+ GROUPING_ID 透传，rewrite_list 是其 + if (!hasGroupingPrefix) { + emit(repeat, TOperatorType.kLogicalGrouping, opUnion, texprs); + return null; + } + List projects = new ArrayList<>(repeat.getOutputExpressions()); + projects.add(groupingIdSlot); + List projectOutput = new ArrayList<>(); + for (NamedExpression project : projects) { + projectOutput.add(scalarTranslator.visitSlotReference((SlotReference) project.toSlot(), null)); + } + TLogicalProject projectOp = new TLogicalProject(); + projectOp.setProject_list(scalarTranslator.translateList(projects)); + projectOp.setRewrite_list(projectOutput); + TOperatorUnion projectUnion = new TOperatorUnion(); + projectUnion.setLogical_project(projectOp); + + // pre-order：先手 add Project texpr（arity=1，不递归），再 emit grouping（emit 自带递归到 child），自然形成 + TExpression projectTexpr = new TExpression(); + TOperator projectTop = new TOperator(); + projectTop.setOp_type(TOperatorType.kLogicalProject); + projectTop.setOp_union(projectUnion); + projectTexpr.setOp(projectTop); + projectTexpr.setArity(1); + projectTexpr.setOutput_list(projectOutput); + texprs.add(projectTexpr); + emit(repeat, TOperatorType.kLogicalGrouping, opUnion, texprs); + return null; + } + + /** agg 的 outputExpressions 中是否含带 distinct 标志的 AggregateFunction */ + private boolean hasDistinctAggFn(LogicalAggregate agg) { + for (NamedExpression out : agg.getOutputExpressions()) { + Expression inner = Doris2HornUtils.unwrapExpr(out); + if (inner instanceof AggregateFunction && ((AggregateFunction) inner).isDistinct()) { + return true; + } + } + return false; + } + + /**

单层 forward 模板对齐 visitLogicalAggregate： */ + @Override + public Void visitLogicalWindow(LogicalWindow logicalWindow, List texprs) { + List windowExpressions = logicalWindow.getWindowExpressions(); + if (windowExpressions.isEmpty()) { + throw new UnsupportedOperationException( + "Horn: LogicalWindow with empty windowExpressions"); + } + + // LogicalWindowToPhysicalWindow.createWindowFrameGroups（partition 比较忽略 key 顺序） + List sameSpecGroups = + LogicalWindowToPhysicalWindow.createWindowFrameGroups(windowExpressions); + if (sameSpecGroups.size() > 1) { + Plan windowChain = logicalWindow.child(); + for (WindowFrameGroup sameSpecGroup : sameSpecGroups) { + windowChain = new LogicalWindow<>(sameSpecGroup.getGroups(), windowChain); + } + windowChain.accept(this, texprs); + return null; + } + + // 走到这里 windowExpressions 必为同一 spec：createWindowFrameGroups 只返回 1 组，或多 spec + WindowExpression leaderWindow = + (WindowExpression) Doris2HornUtils.unwrapExpr(windowExpressions.get(0)); + List partitionKeys = leaderWindow.getPartitionKeys(); + List orderKeys = leaderWindow.getOrderKeys(); + Optional windowFrame = leaderWindow.getWindowFrame(); + + List analyticFunctions = new ArrayList<>(windowExpressions.size()); + List analyticComputedColumns = new ArrayList<>(windowExpressions.size()); + for (NamedExpression windowExpression : windowExpressions) { + WindowExpression analyticFunction = + (WindowExpression) Doris2HornUtils.unwrapExpr(windowExpression); + analyticFunctions.add(scalarTranslator.translate(analyticFunction.getFunction())); + // Alias.toSlot() 携带 ExprId + name + type —— 跟 agg_computed_cols 同样路径。 + analyticComputedColumns.add(scalarTranslator.visitSlotReference( + (SlotReference) windowExpression.toSlot(), null)); + } + + TLogicalWindow logicalWindowThrift = new TLogicalWindow(); + logicalWindowThrift.setAnalytic_functions(analyticFunctions); + logicalWindowThrift.setAnalytic_computed_cols(analyticComputedColumns); + if (!partitionKeys.isEmpty()) { + logicalWindowThrift.setPartition_exprs(scalarTranslator.translateList(partitionKeys)); + } + if (!orderKeys.isEmpty()) { + List windowOrderKeys = new ArrayList<>(orderKeys.size()); + for (OrderExpression orderExpr : orderKeys) { + windowOrderKeys.add(orderExpr.getOrderKey()); + } + logicalWindowThrift.setOrder_by_exprs(buildOrderSpec(windowOrderKeys)); + } + if (windowFrame.isPresent()) { + // 直接调 visitWindowFrame 得到 TOperator(op_type=kWindowFrame, scalar union)， + logicalWindowThrift.setWindow_frame(Collections.singletonList( + scalarTranslator.visitWindowFrame(windowFrame.get(), null))); + } + + TOperatorUnion operatorUnion = new TOperatorUnion(); + operatorUnion.setLogical_window(logicalWindowThrift); + emit(logicalWindow, TOperatorType.kLogicalWindow, operatorUnion, texprs); + return null; + } + + // v7 (=v3) 路径：sort / limit / topn 全 emit 成 TLogicalLimit (带 order_spec / limit / offset) + + @Override + public Void visitLogicalSort(LogicalSort sort, List texprs) { + // sort wrapper：limit=-1 表示纯 ORDER BY，没有 split 概念 + buildLimit(sort, -1L, 0L, sort.getOrderKeys(), true, false, texprs); + return null; + } + + @Override + public Void visitLogicalLimit(LogicalLimit limit, List texprs) { + // Doris Nereids LogicalLimit 的 phase != ORIGIN 表明已经被 Nereids 端拆出 + boolean isSplit = limit.getPhase() != LimitPhase.ORIGIN; + boolean isGlobal = limit.getPhase() != LimitPhase.LOCAL; + buildLimit(limit, limit.getLimit(), limit.getOffset(), null, isGlobal, isSplit, texprs); + return null; + } + + @Override + public Void visitLogicalTopN(LogicalTopN topn, List texprs) { + // Doris LogicalTopN 没有 phase 字段 — 单一形态，相当于 ORIGIN，is_split=false + buildLimit(topn, topn.getLimit(), topn.getOffset(), topn.getOrderKeys(), true, false, texprs); + return null; + } + + /** 公共 emit helper：visitLogicalSort/Limit/TopN 三个变体共用同一份 TLogicalLimit wire */ + private void buildLimit(Plan plan, long limit, long offset, + List orderKeys, boolean isGlobal, boolean isSplit, + List texprs) { + TLogicalLimit tlimit = new TLogicalLimit(); + tlimit.setLimit(limit); + tlimit.setOffset(offset); + tlimit.setIs_global_limit(isGlobal); + tlimit.setIs_split(isSplit); + if (orderKeys != null && !orderKeys.isEmpty()) { + tlimit.setOrder_spec(buildOrderSpec(orderKeys)); + } + TOperatorUnion opUnion = new TOperatorUnion(); + opUnion.setLogical_limit(tlimit); + emit(plan, TOperatorType.kLogicalLimit, opUnion, texprs); + } + + /** 在 scan 输出里按列名（忽略大小写）找对应 SlotReference；找不到返回 null */ + private SlotReference findOutputSlotByColumn(List outputSlots, String columnName) { + for (Slot slot : outputSlots) { + if (slot instanceof SlotReference) { + Optional originalColumn = ((SlotReference) slot).getOriginalColumn(); + if (originalColumn.isPresent() + && originalColumn.get().getName().equalsIgnoreCase(columnName)) { + return (SlotReference) slot; + } + } + } + return null; + } + + /** 把 Doris OrderKey 列表翻成 horn TOrderSpec (parallel lists 形态) */ + private TOrderSpec buildOrderSpec(List orderKeys) { + TOrderSpec orderSpec = new TOrderSpec(); + List exprList = new ArrayList<>(orderKeys.size()); + List ascList = new ArrayList<>(orderKeys.size()); + List nullsFirstList = new ArrayList<>(orderKeys.size()); + for (OrderKey orderKey : orderKeys) { + exprList.add(scalarTranslator.translate(orderKey.getExpr())); + ascList.add(orderKey.isAsc()); + nullsFirstList.add(orderKey.isNullFirst()); + } + orderSpec.setOrder_by_exprs(exprList); + orderSpec.setIs_asc_order(ascList); + orderSpec.setNulls_first(nullsFirstList); + return orderSpec; + } + + @Override + public Void visitLogicalJoin(LogicalJoin join, List texprs) { + // 全部 10 种 Doris JoinType 映射到对应 horn logical join op_type + TOperatorType opType; + switch (join.getJoinType()) { + case INNER_JOIN: + opType = TOperatorType.kLogicalInnerJoin; + break; + case CROSS_JOIN: + opType = TOperatorType.kLogicalCrossJoin; + break; + case LEFT_OUTER_JOIN: + opType = TOperatorType.kLogicalLeftOuterJoin; + break; + case RIGHT_OUTER_JOIN: + opType = TOperatorType.kLogicalRightOuterJoin; + break; + case FULL_OUTER_JOIN: + opType = TOperatorType.kLogicalFullOuterJoin; + break; + case LEFT_SEMI_JOIN: + opType = TOperatorType.kLogicalLeftSemiJoin; + break; + case RIGHT_SEMI_JOIN: + opType = TOperatorType.kLogicalRightSemiJoin; + break; + case LEFT_ANTI_JOIN: + opType = TOperatorType.kLogicalLeftAntiSemiJoin; + break; + case RIGHT_ANTI_JOIN: + opType = TOperatorType.kLogicalRightAntiSemiJoin; + break; + case NULL_AWARE_LEFT_ANTI_JOIN: + // null-aware mark join（NOT IN）的 null 传播语义 horn 不实现 → fallback Nereids。 + if (join.isMarkJoin()) { + throw new UnsupportedOperationException( + "Horn: null-aware mark join (NOT IN) not supported"); + } + opType = TOperatorType.kLogicalNullAwareLeftAntiSemiJoin; + break; + default: + throw new UnsupportedOperationException( + "Horn: unsupported join type: " + join.getJoinType()); + } + TOperatorUnion opUnion = new TOperatorUnion(); + TLogicalJoin tjoin = new TLogicalJoin(); + if (!join.getHashJoinConjuncts().isEmpty()) { + // 等值 join 条件 (hash join 用) + tjoin.setEqual_join_predicate_list( + scalarTranslator.translateList(new ArrayList<>(join.getHashJoinConjuncts()))); + } + if (!join.getOtherJoinConjuncts().isEmpty()) { + // 非等值 / 后置 join 条件 (post-match filter) + tjoin.setOther_join_predicate_list( + scalarTranslator.translateList(new ArrayList<>(join.getOtherJoinConjuncts()))); + } + // mark join：emit mark_slot（MarkJoinSlotReference 是 SlotReference 子类，visitSlotReference + if (join.isMarkJoin()) { + // mark_slot 用单元素 list（同 grouping_id_func）规避 thrift TOperator 递归。 + tjoin.setMark_slot(Lists.newArrayList(scalarTranslator.visitSlotReference( + join.getMarkJoinSlotReference().get(), null))); + if (!join.getMarkJoinConjuncts().isEmpty()) { + tjoin.setMark_join_predicate_list( + scalarTranslator.translateList(new ArrayList<>(join.getMarkJoinConjuncts()))); + } + } + opUnion.setLogical_join(tjoin); + // emit 内 plan.children() = [left, right] 顺序 (BinaryPlan) + emit(join, opType, opUnion, texprs); + return null; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/horn/doris2horn/HornScalarThriftBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/horn/doris2horn/HornScalarThriftBuilder.java new file mode 100644 index 00000000000000..2b3f7be889a62e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/horn/doris2horn/HornScalarThriftBuilder.java @@ -0,0 +1,602 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.horn.doris2horn; + +import org.apache.doris.catalog.TableIf; +import org.apache.horn4j.thrift.TArithmeticOperatorType; +import org.apache.horn4j.thrift.TBinaryOperatorType; +import org.apache.horn4j.thrift.TBoolLiteral; +import org.apache.horn4j.thrift.TDateLiteral; +import org.apache.horn4j.thrift.TDecimalLiteral; +import org.apache.horn4j.thrift.TFlattenedScalar; +import org.apache.horn4j.thrift.TFloatLiteral; +import org.apache.horn4j.thrift.TIntLiteral; +import org.apache.horn4j.thrift.TLiteralType; +import org.apache.horn4j.thrift.TLiteralValue; +import org.apache.horn4j.thrift.TOperator; +import org.apache.horn4j.thrift.TOperatorType; +import org.apache.horn4j.thrift.TOperatorUnion; +import org.apache.horn4j.thrift.TScalar; +import org.apache.horn4j.thrift.TScalarAggFn; +import org.apache.horn4j.thrift.TScalarArithmetic; +import org.apache.horn4j.thrift.TScalarBinaryPredicate; +import org.apache.horn4j.thrift.TScalarFnCall; +import org.apache.horn4j.thrift.TScalarColumnRef; +import org.apache.horn4j.thrift.TScalarLiteral; +import org.apache.horn4j.thrift.TScalarPredicate; +import org.apache.horn4j.thrift.TScalarUnion; +import org.apache.horn4j.thrift.TScalarAnalyticWindowBoundary; +import org.apache.horn4j.thrift.TScalarUniqueId; +import org.apache.horn4j.thrift.TScalarWindowFrame; +import org.apache.horn4j.thrift.TStringLiteral; +import org.apache.horn4j.thrift.TColumnType; +import org.apache.horn4j.thrift.TWindowBoundaryType; +import org.apache.horn4j.thrift.TWindowType; +import org.apache.doris.catalog.Type; +import org.apache.doris.horn.horn2doris.DorisFunctionBuilder; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.And; +import org.apache.doris.nereids.trees.expressions.BinaryArithmetic; +import org.apache.doris.nereids.trees.expressions.TimestampArithmetic; +import org.apache.doris.nereids.trees.expressions.BitAnd; +import org.apache.doris.nereids.trees.expressions.BitNot; +import org.apache.doris.nereids.trees.expressions.BitOr; +import org.apache.doris.nereids.trees.expressions.BitXor; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.CompoundPredicate; +import org.apache.doris.nereids.trees.expressions.Between; +import org.apache.doris.nereids.trees.expressions.Divide; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; +import org.apache.doris.nereids.trees.expressions.InPredicate; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.IntegralDivide; +import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.MatchAll; +import org.apache.doris.nereids.trees.expressions.MatchAny; +import org.apache.doris.nereids.trees.expressions.MatchPhrase; +import org.apache.doris.nereids.trees.expressions.MatchPhraseEdge; +import org.apache.doris.nereids.trees.expressions.MatchPhrasePrefix; +import org.apache.doris.nereids.trees.expressions.MatchRegexp; +import org.apache.doris.nereids.trees.expressions.Mod; +import org.apache.doris.nereids.trees.expressions.Multiply; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.Subtract; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.WindowFrame; +import org.apache.doris.nereids.trees.expressions.WindowFrame.FrameBoundType; +import org.apache.doris.nereids.trees.expressions.WindowFrame.FrameBoundary; +import org.apache.doris.nereids.trees.expressions.WindowFrame.FrameUnitsType; +import org.apache.doris.nereids.trees.expressions.functions.window.CumeDist; +import org.apache.doris.nereids.trees.expressions.functions.window.DenseRank; +import org.apache.doris.nereids.trees.expressions.functions.window.FirstValue; +import org.apache.doris.nereids.trees.expressions.functions.window.Lag; +import org.apache.doris.nereids.trees.expressions.functions.window.LastValue; +import org.apache.doris.nereids.trees.expressions.functions.window.Lead; +import org.apache.doris.nereids.trees.expressions.functions.window.NthValue; +import org.apache.doris.nereids.trees.expressions.functions.window.Ntile; +import org.apache.doris.nereids.trees.expressions.functions.window.PercentRank; +import org.apache.doris.nereids.trees.expressions.functions.window.Rank; +import org.apache.doris.nereids.trees.expressions.functions.window.RowNumber; +import org.apache.doris.nereids.trees.expressions.functions.window.WindowFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.Avg; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.Min; +import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctCount; +import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctSum; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.Like; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Abs; +import org.apache.doris.nereids.trees.expressions.functions.scalar.If; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Round; +import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Substring; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Year; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal; +import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; +import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; +import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** ExpressionVisitor: Doris Nereids Expression → Horn TOperator (TFlattenedScalar) */ +public class HornScalarThriftBuilder extends ExpressionVisitor { + + /** Serialize an Expression as a TOperator (fscalar slot, pre-order flattened) */ + public TOperator translate(Expression expr) { + TOperator top = new TOperator(); + TOperatorUnion opUnion = new TOperatorUnion(); + TFlattenedScalar fscalar = new TFlattenedScalar(); + List scalars = new ArrayList<>(); + flattenPreOrder(expr, scalars); + fscalar.setScalars(scalars); + opUnion.setFscalar(fscalar); + top.setOp_union(opUnion); + top.setOp_type(TOperatorType.kInvalid); + return top; + } + + public List translateList(List exprs) { + List result = new ArrayList<>(); + for (Expression expr : exprs) { + result.add(translate(expr)); + } + return result; + } + + /** Pre-order traversal: visit, then recurse children */ + private void flattenPreOrder(Expression expr, List out) { + // Alias is transparent: emit only its inner expression's subtree, + if (expr instanceof Alias) { + flattenPreOrder(expr.child(0), out); + return; + } + out.add(expr.accept(this, null)); + for (Expression child : expr.children()) { + flattenPreOrder(child, out); + } + } + + /** 把已经算好的 (data_type, arity, op_type, scalar_union) 打包成 TOperator */ + private TOperator buildScalar(TColumnType dataType, int arity, + TOperatorType opType, TScalarUnion scalarUnion) { + TScalar tscalar = new TScalar(); + tscalar.setData_type(dataType); + // Set uid to -1 (empty): BE's CreateUniqueId will assign a proper counter-based uid + tscalar.setScalar_unique_id(new TScalarUniqueId(-1)); + tscalar.setArity(arity); + tscalar.setScalar_union(scalarUnion); + + TOperatorUnion opUnion = new TOperatorUnion(); + opUnion.setScalar(tscalar); + TOperator top = new TOperator(); + top.setOp_union(opUnion); + top.setOp_type(opType); + return top; + } + + @Override + public TOperator visit(Expression expr, Void ctx) { + throw new UnsupportedOperationException( + "Horn: unsupported expression type: " + expr.getClass().getSimpleName()); + } + + @Override + public TOperator visitSlotReference(SlotReference slot, Void ctx) { + TScalarUnion scalarUnion = new TScalarUnion(); + TScalarColumnRef columnRef = new TScalarColumnRef(); + columnRef.setColumn_name(slot.getName()); + // 用 oneLevelTable（不穿透 view）而不是 originalTable（穿透到 base table） + if (slot.getOneLevelTable().isPresent()) { + TableIf table = slot.getOneLevelTable().get(); + columnRef.setTable_name(table.getName()); + if (table.getDatabase() != null) { + columnRef.setDatabase_name(table.getDatabase().getFullName()); + } + } + columnRef.setExternal_slot_id((int) slot.getExprId().asInt()); + scalarUnion.setScalar_column_ref(columnRef); + return buildScalar(DorisTypeToHornConverter.convert(slot.getDataType()), + 0, TOperatorType.kColumnRef, scalarUnion); + } + + @Override + public TOperator visitLiteral(Literal literal, Void ctx) { + TScalarLiteral scalarLiteral = new TScalarLiteral(); + TStringLiteral displayLiteral = new TStringLiteral(); + displayLiteral.setValue(literal.toString()); + scalarLiteral.setDisplay_literal(displayLiteral); + if (literal instanceof NullLiteral) { + // Null literal: no literal_value set, only is_null_type=true. + scalarLiteral.setIs_null_type(true); + } else { + TLiteralValue literalValue = new TLiteralValue(); + TLiteralType literalType; + if (literal instanceof BooleanLiteral) { + literalValue.setBool_literal(new TBoolLiteral(((BooleanLiteral) literal).getValue())); + literalType = TLiteralType.kBoolLiteral; + } else if (literal instanceof TinyIntLiteral || literal instanceof SmallIntLiteral + || literal instanceof IntegerLiteral || literal instanceof BigIntLiteral) { + // LargeIntLiteral 暂不支持 + literalValue.setInt_literal( + new TIntLiteral(((IntegerLikeLiteral) literal).getLongValue())); + literalType = TLiteralType.kIntLiteral; + } else if (literal instanceof DecimalLiteral || literal instanceof DecimalV3Literal) { + String decimalString = literal.getValue().toString(); + TDecimalLiteral decimalLiteral = new TDecimalLiteral(); + decimalLiteral.setValue(decimalString.getBytes(StandardCharsets.UTF_8)); + literalValue.setDecimal_literal(decimalLiteral); + literalType = TLiteralType.kDecimalLiteral; + } else if (literal instanceof FloatLiteral || literal instanceof DoubleLiteral) { + // Float / Double + double doubleValue = ((Number) literal.getValue()).doubleValue(); + literalValue.setFloat_literal(new TFloatLiteral(doubleValue)); + literalType = TLiteralType.kFloatLiteral; + } else if (literal instanceof StringLikeLiteral) { + // Varchar/Char/String: use getStringValue() to avoid quotes. + TStringLiteral stringLiteral = new TStringLiteral(); + stringLiteral.setValue(((StringLikeLiteral) literal).getStringValue()); + literalValue.setString_literal(stringLiteral); + literalType = TLiteralType.kStringLiteral; + } else if (literal instanceof DateLiteral || literal instanceof DateV2Literal) { + // 统一先转化为 DateV2Literal + DateLiteral src = (DateLiteral) literal; + DateV2Literal dateLiteral = (literal instanceof DateV2Literal) + ? (DateV2Literal) literal + : new DateV2Literal(src.getYear(), src.getMonth(), src.getDay()); + long daysSinceEpoch = dateLiteral.toJavaDateType().toLocalDate().toEpochDay(); + TDateLiteral tDate = new TDateLiteral(); + tDate.setDays_since_epoch((int) daysSinceEpoch); + tDate.setDate_string(dateLiteral.toString()); + literalValue.setDate_literal(tDate); + literalType = TLiteralType.kDateLiteral; + } else { + // fallback + return visit(literal, ctx); + } + scalarLiteral.setLiteral_value(literalValue); + scalarLiteral.setLiteral_type(literalType); + scalarLiteral.setIs_null_type(false); + } + TScalarUnion scalarUnion = new TScalarUnion(); + scalarUnion.setScalar_literal(scalarLiteral); + return buildScalar(DorisTypeToHornConverter.convert(literal.getDataType()), + 0, TOperatorType.kLiteral, scalarUnion); + } + + @Override + public TOperator visitComparisonPredicate(ComparisonPredicate cp, Void ctx) { + TBinaryOperatorType type; + if (cp instanceof EqualTo) { + type = TBinaryOperatorType.kEqual; + } else if (cp instanceof GreaterThan) { + type = TBinaryOperatorType.kGatherThan; + } else if (cp instanceof GreaterThanEqual) { + type = TBinaryOperatorType.kGatherEqual; + } else if (cp instanceof LessThan) { + type = TBinaryOperatorType.kLowerThan; + } else if (cp instanceof LessThanEqual) { + type = TBinaryOperatorType.kLowerEqual; + } else { + // NullSafeEqual / DistinctFrom 走 horn 的特殊 binary type，独立任务。 + return visit(cp, ctx); + } + TScalarUnion scalarUnion = new TScalarUnion(); + TScalarBinaryPredicate binaryPredicate = new TScalarBinaryPredicate(); + binaryPredicate.setBinary_type(type); + TScalarPredicate scalarPredicate = new TScalarPredicate(); + scalarPredicate.setHas_always_true_hint(false); + scalarPredicate.setScalar_binary_predicate(binaryPredicate); + scalarUnion.setScalar_predicate(scalarPredicate); + return buildScalar(DorisTypeToHornConverter.convert(cp.getDataType()), + cp.children().size(), TOperatorType.kBinaryPredicate, scalarUnion); + } + + @Override + public TOperator visitCompoundPredicate(CompoundPredicate cp, Void ctx) { + TOperatorType opType; + if (cp instanceof And) { + opType = TOperatorType.kAndPredicate; + } else if (cp instanceof Or) { + opType = TOperatorType.kOrPredicate; + } else { + return visit(cp, ctx); + } + TScalarUnion scalarUnion = new TScalarUnion(); + scalarUnion.setScalar_predicate(new TScalarPredicate(false)); + return buildScalar(DorisTypeToHornConverter.convert(cp.getDataType()), + cp.children().size(), opType, scalarUnion); + } + + /** Alias is "transparent" in horn's flattened scalar model: the Alias's */ + @Override + public TOperator visitAlias(Alias alias, Void ctx) { + return alias.child().accept(this, ctx); + } + + /** P0 聚合函数（count / sum / min / max / avg）+ multi_distinct_count / multi_distinct_sum */ + @Override + public TOperator visitAggregateFunction(AggregateFunction agg, Void ctx) { + if (agg.isDistinct()) { + throw new UnsupportedOperationException( + "Horn: unsupported expression type: distinct " + agg.getName().toLowerCase()); + } + // 共享白名单（DorisFunctionBuilder.KNOWN_AGG_FNS）：跟 scalar / window 同模式， + String name = agg.getName().toLowerCase(); + if (!DorisFunctionBuilder.KNOWN_AGG_FNS.contains(name)) { + throw new UnsupportedOperationException( + "Horn: unsupported expression type: " + agg.getClass().getSimpleName()); + } + TScalarAggFn aggFn = new TScalarAggFn(); + aggFn.setFn_name(name); + TScalarUnion scalarUnion = new TScalarUnion(); + scalarUnion.setScalar_agg_fn(aggFn); + return buildScalar(DorisTypeToHornConverter.convert(agg.getDataType()), + agg.children().size(), TOperatorType.kAggFn, scalarUnion); + } + + /** Cast → horn ScalarFunctionCall with fn_name="castto" */ + @Override + public TOperator visitCast(Cast cast, Void ctx) { + String typeName = cast.getDataType().simpleString().toLowerCase(); + TScalarFnCall fnCall = new TScalarFnCall(); + fnCall.setFn_name("castto" + typeName); + fnCall.setIs_implicit(!cast.isExplicitType()); + TScalarUnion scalarUnion = new TScalarUnion(); + scalarUnion.setScalar_fn_call(fnCall); + return buildScalar(DorisTypeToHornConverter.convert(cast.getDataType()), + cast.children().size(), TOperatorType.kScalarFnCall, scalarUnion); + } + + /** TScalarFnCall(fn_name=小写函数名)，复用 cast 同一条 wire 通道 */ + /** Not（NOT LIKE / != 的 Not(EqualTo) 形态等）走 fn 通道黑盒 */ + @Override + public TOperator visitNot(Not not, Void ctx) { + TScalarFnCall fnCall = new TScalarFnCall(); + fnCall.setFn_name("not"); + TScalarUnion scalarUnion = new TScalarUnion(); + scalarUnion.setScalar_fn_call(fnCall); + return buildScalar(DorisTypeToHornConverter.convert(not.getDataType()), + not.children().size(), TOperatorType.kScalarFnCall, scalarUnion); + } + + /** Impala/kernel 的做法 —— `InPredicate.toThrift` 序列化为 FUNCTION_CALL */ + @Override + public TOperator visitInPredicate(InPredicate in, Void ctx) { + TScalarFnCall fnCall = new TScalarFnCall(); + fnCall.setFn_name("in"); + TScalarUnion scalarUnion = new TScalarUnion(); + scalarUnion.setScalar_fn_call(fnCall); + return buildScalar(DorisTypeToHornConverter.convert(in.getDataType()), + in.children().size(), TOperatorType.kScalarFnCall, scalarUnion); + } + + /** BitNot(x)(SQL `~x`)走 fn 通道黑盒,fn_name="bitnot",children=[x] */ + @Override + public TOperator visitBitNot(BitNot bitNot, Void ctx) { + TScalarFnCall fnCall = new TScalarFnCall(); + fnCall.setFn_name("bitnot"); + TScalarUnion scalarUnion = new TScalarUnion(); + scalarUnion.setScalar_fn_call(fnCall); + return buildScalar(DorisTypeToHornConverter.convert(bitNot.getDataType()), + bitNot.children().size(), TOperatorType.kScalarFnCall, scalarUnion); + } + + /** Between(x, lo, hi)(即 SQL `x BETWEEN lo AND hi`)走 fn 通道黑盒, */ + @Override + public TOperator visitBetween(Between between, Void ctx) { + TScalarFnCall fnCall = new TScalarFnCall(); + fnCall.setFn_name("between"); + TScalarUnion scalarUnion = new TScalarUnion(); + scalarUnion.setScalar_fn_call(fnCall); + return buildScalar(DorisTypeToHornConverter.convert(between.getDataType()), + between.children().size(), TOperatorType.kScalarFnCall, scalarUnion); + } + + /** Match 家族(MatchAny / MatchAll / MatchPhrase / MatchPhrasePrefix / */ + @Override + public TOperator visitMatch(Match match, Void ctx) { + String fnName; + if (match instanceof MatchAny) { + fnName = "match_any"; + } else if (match instanceof MatchAll) { + fnName = "match_all"; + } else if (match instanceof MatchPhrasePrefix) { + fnName = "match_phrase_prefix"; + } else if (match instanceof MatchPhraseEdge) { + fnName = "match_phrase_edge"; + } else if (match instanceof MatchPhrase) { + fnName = "match_phrase"; + } else if (match instanceof MatchRegexp) { + fnName = "match_regexp"; + } else { + throw new UnsupportedOperationException( + "Horn: unsupported Match subclass: " + match.getClass().getSimpleName()); + } + TScalarFnCall fnCall = new TScalarFnCall(); + fnCall.setFn_name(fnName); + TScalarUnion scalarUnion = new TScalarUnion(); + scalarUnion.setScalar_fn_call(fnCall); + return buildScalar(DorisTypeToHornConverter.convert(match.getDataType()), + match.children().size(), TOperatorType.kScalarFnCall, scalarUnion); + } + + /** IsNull(x)（x IS NULL）走 fn 通道黑盒，fn_name="is_null"，child=[x] */ + @Override + public TOperator visitIsNull(IsNull isNull, Void ctx) { + TScalarFnCall fnCall = new TScalarFnCall(); + fnCall.setFn_name("is_null"); + TScalarUnion scalarUnion = new TScalarUnion(); + scalarUnion.setScalar_fn_call(fnCall); + return buildScalar(DorisTypeToHornConverter.convert(isNull.getDataType()), + isNull.children().size(), TOperatorType.kScalarFnCall, scalarUnion); + } + + /** GROUPING(col) / GROUPING_ID(c1,...,cn) 走 ScalarFunc("grouping"/"grouping_id") 黑盒通道 */ + @Override + public TOperator visitGroupingScalarFunction(GroupingScalarFunction fn, Void ctx) { + TScalarFnCall fnCall = new TScalarFnCall(); + fnCall.setFn_name(fn.getName().toLowerCase()); + TScalarUnion scalarUnion = new TScalarUnion(); + scalarUnion.setScalar_fn_call(fnCall); + return buildScalar(DorisTypeToHornConverter.convert(fn.getDataType()), + fn.children().size(), TOperatorType.kScalarFnCall, scalarUnion); + } + + @Override + public TOperator visitScalarFunction(ScalarFunction fn, Void ctx) { + // 共享白名单（DorisFunctionBuilder.KNOWN_SCALAR_FNS）：backward 端反译时 + String name = fn.getName().toLowerCase(); + if (!DorisFunctionBuilder.KNOWN_SCALAR_FNS.contains(name)) { + return visit(fn, ctx); + } + TScalarFnCall fnCall = new TScalarFnCall(); + fnCall.setFn_name(name); + TScalarUnion scalarUnion = new TScalarUnion(); + scalarUnion.setScalar_fn_call(fnCall); + return buildScalar(DorisTypeToHornConverter.convert(fn.getDataType()), + fn.children().size(), TOperatorType.kScalarFnCall, scalarUnion); + } + + /** TimestampArithmetic（{@code date +/- INTERVAL n UNIT}）：它 extends Expression（非 */ + @Override + public TOperator visitTimestampArithmetic(TimestampArithmetic arith, Void ctx) { + String name = arith.getFuncName() == null ? null : arith.getFuncName().toLowerCase(); + if (name == null || !DorisFunctionBuilder.KNOWN_SCALAR_FNS.contains(name)) { + return visit(arith, ctx); + } + TScalarFnCall fnCall = new TScalarFnCall(); + fnCall.setFn_name(name); + TScalarUnion scalarUnion = new TScalarUnion(); + scalarUnion.setScalar_fn_call(fnCall); + return buildScalar(DorisTypeToHornConverter.convert(arith.getDataType()), + arith.children().size(), TOperatorType.kScalarFnCall, scalarUnion); + } + + /** ScalarFunction 共用 wire 通道 —— 都 emit kScalarFnCall + fn_name=lower(getName()) */ + @Override + public TOperator visitWindowFunction(WindowFunction fn, Void ctx) { + // 共享白名单（DorisFunctionBuilder.KNOWN_WINDOW_FNS）：跟 scalar fn 同模式， + String name = fn.getName().toLowerCase(); + if (!DorisFunctionBuilder.KNOWN_WINDOW_FNS.contains(name)) { + return visit(fn, ctx); + } + TScalarFnCall fnCall = new TScalarFnCall(); + fnCall.setFn_name(name); + TScalarUnion scalarUnion = new TScalarUnion(); + scalarUnion.setScalar_fn_call(fnCall); + return buildScalar(DorisTypeToHornConverter.convert(fn.getDataType()), + fn.children().size(), TOperatorType.kScalarFnCall, scalarUnion); + } + + /** WindowFrame → horn TScalarWindowFrame（kWindowFrame） */ + /** WindowFrame → TScalarWindowFrame（kWindowFrame） */ + @Override + public TOperator visitWindowFrame(WindowFrame frame, Void ctx) { + TScalarWindowFrame twf = new TScalarWindowFrame(); + twf.setWindow_type(frame.getFrameUnits() == FrameUnitsType.ROWS + ? TWindowType.kRows : TWindowType.kRange); + if (!frame.getLeftBoundary().isNull()) { + twf.setWindow_start(Collections.singletonList( + buildFrameBoundary(frame.getLeftBoundary(), true))); + } + if (!frame.getRightBoundary().isNull()) { + twf.setWindow_end(Collections.singletonList( + buildFrameBoundary(frame.getRightBoundary(), false))); + } + TScalarUnion union = new TScalarUnion(); + union.setScalar_window_frame(twf); + return buildScalar(DorisTypeToHornConverter.convertCatalogType(Type.INVALID), + 0, TOperatorType.kWindowFrame, union); + } + + /** FrameBoundary 不是 Expression（是 WindowFrame 的静态内部类，根本没 */ + private TOperator buildFrameBoundary(FrameBoundary frameBoundary, boolean isLeft) { + TScalarAnalyticWindowBoundary tBoundary = new TScalarAnalyticWindowBoundary(); + FrameBoundType frameBoundType = frameBoundary.getFrameBoundType(); + switch (frameBoundType) { + case UNBOUNDED_PRECEDING: + case UNBOUNDED_FOLLOWING: + tBoundary.setBoundary_type(TWindowBoundaryType.kUnbounded); + break; + case CURRENT_ROW: + tBoundary.setBoundary_type(TWindowBoundaryType.kCurrentRow); + break; + case PRECEDING: + tBoundary.setBoundary_type(TWindowBoundaryType.kPreceding); + break; + case FOLLOWING: + tBoundary.setBoundary_type(TWindowBoundaryType.kFollowing); + break; + default: + throw new UnsupportedOperationException( + "Horn: unsupported FrameBoundType: " + frameBoundType); + } + if (frameBoundary.hasOffset() && frameBoundary.getBoundOffset().isPresent()) { + Expression offset = frameBoundary.getBoundOffset().get(); + if (offset instanceof IntegerLikeLiteral) { + tBoundary.setRows_offset_value(((IntegerLikeLiteral) offset).getLongValue()); + } else { + tBoundary.setRange_offset_predicate(Collections.singletonList( + offset.accept(this, null))); + } + } + TScalarUnion union = new TScalarUnion(); + union.setScalar_analytic_window_boundary(tBoundary); + return buildScalar(DorisTypeToHornConverter.convertCatalogType(Type.INVALID), + 0, TOperatorType.kAnalyticWindowBoundary, union); + } + + @Override + public TOperator visitBinaryArithmetic(BinaryArithmetic arith, Void ctx) { + TArithmeticOperatorType opType; + if (arith instanceof Add) { + opType = TArithmeticOperatorType.kAdd; + } else if (arith instanceof Subtract) { + opType = TArithmeticOperatorType.kSubtract; + } else if (arith instanceof Multiply) { + opType = TArithmeticOperatorType.kMultiply; + } else if (arith instanceof Divide) { + opType = TArithmeticOperatorType.kDivide; + } else if (arith instanceof Mod) { + opType = TArithmeticOperatorType.kMod; + } else if (arith instanceof IntegralDivide) { + opType = TArithmeticOperatorType.kIntDivide; + } else if (arith instanceof BitAnd) { + opType = TArithmeticOperatorType.kBitAnd; + } else if (arith instanceof BitOr) { + opType = TArithmeticOperatorType.kBitOr; + } else if (arith instanceof BitXor) { + opType = TArithmeticOperatorType.kBitXor; + } else { + return visit(arith, ctx); + } + TScalarUnion scalarUnion = new TScalarUnion(); + TScalarArithmetic arithmetic = new TScalarArithmetic(); + arithmetic.setArithmetic_op_type(opType); + scalarUnion.setScalar_arithmetic(arithmetic); + return buildScalar(DorisTypeToHornConverter.convert(arith.getDataType()), + arith.children().size(), TOperatorType.kArithmetic, scalarUnion); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/horn/horn2doris/DorisExpressionBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/horn/horn2doris/DorisExpressionBuilder.java new file mode 100644 index 00000000000000..5b8d07c63e3ce7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/horn/horn2doris/DorisExpressionBuilder.java @@ -0,0 +1,501 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.horn.horn2doris; + +import org.apache.horn4j.thrift.TArithmeticOperatorType; +import org.apache.horn4j.thrift.TLiteralValue; +import org.apache.horn4j.thrift.TOperator; +import org.apache.horn4j.thrift.TOperatorType; +import org.apache.horn4j.thrift.TScalar; +import org.apache.horn4j.thrift.TScalarAggFn; +import org.apache.horn4j.thrift.TScalarArithmetic; +import org.apache.horn4j.thrift.TScalarBinaryPredicate; +import org.apache.horn4j.thrift.TScalarColumnRef; +import org.apache.horn4j.thrift.TScalarAnalyticWindowBoundary; +import org.apache.horn4j.thrift.TScalarFnCall; +import org.apache.horn4j.thrift.TScalarLiteral; +import org.apache.horn4j.thrift.TScalarWindowFrame; +import org.apache.horn4j.thrift.TWindowType; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.And; +import org.apache.doris.nereids.trees.expressions.BitAnd; +import org.apache.doris.nereids.trees.expressions.BitNot; +import org.apache.doris.nereids.trees.expressions.BitOr; +import org.apache.doris.nereids.trees.expressions.BitXor; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Divide; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; +import org.apache.doris.nereids.trees.expressions.Between; +import org.apache.doris.nereids.trees.expressions.IntegralDivide; +import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.InPredicate; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.Like; +import org.apache.doris.nereids.trees.expressions.MatchAll; +import org.apache.doris.nereids.trees.expressions.MatchAny; +import org.apache.doris.nereids.trees.expressions.MatchPhrase; +import org.apache.doris.nereids.trees.expressions.MatchPhraseEdge; +import org.apache.doris.nereids.trees.expressions.MatchPhrasePrefix; +import org.apache.doris.nereids.trees.expressions.MatchRegexp; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.Mod; +import org.apache.doris.nereids.trees.expressions.Multiply; +import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.Subtract; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.Avg; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.Min; +import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctCount; +import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctSum; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Abs; +import org.apache.doris.nereids.trees.expressions.functions.scalar.If; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Round; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Substring; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Year; +import org.apache.doris.nereids.trees.expressions.functions.window.CumeDist; +import org.apache.doris.nereids.trees.expressions.functions.window.DenseRank; +import org.apache.doris.nereids.trees.expressions.functions.window.FirstValue; +import org.apache.doris.nereids.trees.expressions.functions.window.Lag; +import org.apache.doris.nereids.trees.expressions.functions.window.LastValue; +import org.apache.doris.nereids.trees.expressions.functions.window.Lead; +import org.apache.doris.nereids.trees.expressions.functions.window.NthValue; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Grouping; +import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingId; +import org.apache.doris.nereids.trees.expressions.functions.window.Ntile; +import org.apache.doris.nereids.trees.expressions.functions.window.PercentRank; +import org.apache.doris.nereids.trees.expressions.functions.window.Rank; +import org.apache.doris.nereids.trees.expressions.functions.window.RowNumber; +import org.apache.doris.nereids.trees.expressions.WindowFrame; +import org.apache.doris.nereids.trees.expressions.WindowFrame.FrameBoundType; +import org.apache.doris.nereids.trees.expressions.WindowFrame.FrameBoundary; +import org.apache.doris.nereids.trees.expressions.WindowFrame.FrameUnitsType; +import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal; +import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; +import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DecimalV3Type; +import org.apache.doris.nereids.types.FloatType; + +import com.google.common.collect.ImmutableList; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import org.apache.doris.horn.HornOptimizationContext; +import org.apache.doris.horn.doris2horn.HornScalarThriftBuilder; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +/** Translate Horn TOperator (TScalar expressions) back to Doris Nereids Expressions */ +public class DorisExpressionBuilder { + + private static final Logger LOG = LogManager.getLogger(DorisExpressionBuilder.class); + + /** 反译 ctx：scalarUidToSlot 等 backward 共享状态都挂在这里 */ + private final HornOptimizationContext hornCtx; + + public DorisExpressionBuilder(HornOptimizationContext hornCtx) { + this.hornCtx = hornCtx; + } + + public Expression translate(TOperator op) { + return translateHelper(op); + } + + public List translateList(List ops) { + List result = new ArrayList<>(); + if (ops == null) { + return result; + } + for (TOperator op : ops) { + result.add(translate(op)); + } + return result; + } + + + private Expression translateHelper(TOperator op) { + // BE output can be either fscalar (pre-order flattened) or scalar (single node); + if (op.getOp_union().isSetFscalar()) { + List scalars = op.getOp_union().getFscalar().getScalars(); + if (scalars != null && !scalars.isEmpty()) { + return translateFlattenedScalar(scalars, new AtomicInteger(0)); + } + } + + TOperatorType opType = op.getOp_type(); + if (!op.getOp_union().isSetScalar()) { + throw new IllegalArgumentException( + "Horn expr translator: TOperator has no scalar data, type=" + opType); + } + + TScalar scalar = op.getOp_union().getScalar(); + + List children = new ArrayList<>(); + if (scalar.isSetChildren()) { + for (TOperator childOp : scalar.getChildren()) { + children.add(translateHelper(childOp)); + } + } + + return translateScalarNode(opType, scalar, children); + } + + /** Rebuild an expression tree from a pre-order flattened TScalar list */ + private Expression translateFlattenedScalar(List scalars, AtomicInteger idx) { + TOperator op = scalars.get(idx.getAndIncrement()); + TScalar scalar = op.getOp_union().getScalar(); + int arity = (int) scalar.getArity(); + + List children = new ArrayList<>(); + for (int i = 0; i < arity; i++) { + children.add(translateFlattenedScalar(scalars, idx)); + } + return translateScalarNode(op.getOp_type(), scalar, children); + } + + private Expression translateScalarNode(TOperatorType opType, TScalar scalar, + List children) { + // resultType lazy：只有 kLiteral / kScalarFnCall 真正需要 evaluated 类型 + switch (opType) { + case kColumnRef: + return translateColumnRef(scalar); + case kLiteral: + return translateLiteral(scalar, + HornTypeToDorisConverter.convert(scalar.getData_type())); + case kBinaryPredicate: + return translateBinaryPredicate(scalar, children); + case kAndPredicate: + return new And(children); + case kOrPredicate: + return new Or(children); + case kArithmetic: + return translateArithmetic(scalar, children); + case kScalarFnCall: + return translateScalarFnCall(scalar, children, + HornTypeToDorisConverter.convert(scalar.getData_type())); + case kAggFn: + return translateAggFn(scalar, children); + case kWindowFrame: + // WindowFrame is-a Expression（未 override getDataType），与正向 + return buildWindowFrame(scalar); + default: + throw new UnsupportedOperationException( + "Horn expr translator: unsupported operator type " + opType); + } + } + + private Expression translateArithmetic(TScalar scalar, List children) { + TScalarArithmetic arith = scalar.getScalar_union().getScalar_arithmetic(); + Expression left = children.get(0); + Expression right = children.get(1); + TArithmeticOperatorType opType = arith.getArithmetic_op_type(); + switch (opType) { + case kAdd: + return new Add(left, right); + case kSubtract: + return new Subtract(left, right); + case kMultiply: + return new Multiply(left, right); + case kDivide: + return new Divide(left, right); + case kMod: + return new Mod(left, right); + case kIntDivide: + return new IntegralDivide(left, right); + case kBitAnd: + return new BitAnd(left, right); + case kBitOr: + return new BitOr(left, right); + case kBitXor: + return new BitXor(left, right); + default: + throw new UnsupportedOperationException( + "Horn expr translator: unsupported arithmetic op " + opType); + } + } + + /** Reverse of {@link HornScalarThriftBuilder#visitCast} */ + private Expression translateScalarFnCall(TScalar scalar, List children, + DataType resultType) { + TScalarFnCall fn = scalar.getScalar_union().getScalar_fn_call(); + String fnName = fn.getFn_name(); + if (fnName == null) { + throw new UnsupportedOperationException( + "Horn expr translator: ScalarFnCall without fn_name"); + } + if (fnName.startsWith("castto") && children.size() == 1) { + return new Cast(children.get(0), resultType, !fn.isIs_implicit()); + } + // 黑盒标记函数 alias()（复合根→独立 uid，与被掩的 grouping 键分家） + if ("alias".equals(fnName)) { + return children.get(0); + } + // 特例：not / in / is_null 不是 BoundFunction 子类（属于 predicate 类）， + switch (fnName) { + case "not": + requireArity(fnName, children, 1); + return new Not(children.get(0)); + case "in": + // kernel 模式：children = [compareExpr, opt1, ..., optN]， + if (children.size() < 2) { + throw new UnsupportedOperationException( + "Horn expr translator: in needs ≥2 children, got " + + children.size()); + } + return new InPredicate(children.get(0), + children.subList(1, children.size())); + case "between": + // Between(compareExpr, lowerBound, upperBound),3 children,与 forward + requireArity(fnName, children, 3); + return new Between(children.get(0), children.get(1), children.get(2)); + case "bitnot": + // BitNot(x)(SQL `~x`),与 forward visitBitNot 对称 + requireArity(fnName, children, 1); + return new BitNot(children.get(0)); + case "match_any": + // Match 家族 fn 通道反译,与 HornScalarThriftBuilder.visitMatch 对偶 + requireArity(fnName, children, 2); + return new MatchAny(children.get(0), children.get(1)); + case "match_all": + requireArity(fnName, children, 2); + return new MatchAll(children.get(0), children.get(1)); + case "match_phrase": + requireArity(fnName, children, 2); + return new MatchPhrase(children.get(0), children.get(1)); + case "match_phrase_prefix": + requireArity(fnName, children, 2); + return new MatchPhrasePrefix(children.get(0), children.get(1)); + case "match_phrase_edge": + requireArity(fnName, children, 2); + return new MatchPhraseEdge(children.get(0), children.get(1)); + case "match_regexp": + requireArity(fnName, children, 2); + return new MatchRegexp(children.get(0), children.get(1)); + case "is_null": + // x IS NULL，与正向 visitIsNull 的 fn_name="is_null" 同步。 + requireArity(fnName, children, 1); + return new IsNull(children.get(0)); + case "grouping": + // 反向 HornScalarThriftBuilder.visitGroupingScalarFunction（fn_name="grouping"） + requireArity(fnName, children, 1); + return new Grouping(children.get(0)); + case "grouping_id": + if (children.isEmpty()) { + throw new UnsupportedOperationException( + "Horn expr translator: grouping_id needs ≥1 args"); + } + // GroupingId(Expression firstArg, Expression... varArgs) varargs ctor + Expression[] rest = children.subList(1, children.size()) + .toArray(new Expression[0]); + return new GroupingId(children.get(0), rest); + default: + // 普通 scalar / window fn：白名单内的走 Doris FunctionRegistry 通用工厂 + if (DorisFunctionBuilder.KNOWN_SCALAR_FNS.contains(fnName) + || DorisFunctionBuilder.KNOWN_WINDOW_FNS.contains(fnName)) { + return DorisFunctionBuilder.build(fnName, children); + } + throw new UnsupportedOperationException( + "Horn expr translator: unsupported ScalarFnCall fn_name=" + fnName + + " arity=" + children.size()); + } + } + + /** horn ScalarAggFn → Doris AggregateFunction */ + private Expression translateAggFn(TScalar scalar, List children) { + TScalarAggFn fn = scalar.getScalar_union().getScalar_agg_fn(); + String fnName = fn.getFn_name(); + if (fnName == null) { + throw new UnsupportedOperationException( + "Horn expr translator: ScalarAggFn missing fn_name"); + } + if (DorisFunctionBuilder.KNOWN_AGG_FNS.contains(fnName)) { + return DorisFunctionBuilder.build(fnName, children); + } + throw new UnsupportedOperationException( + "Horn expr translator: unsupported ScalarAggFn fn_name=" + fnName); + } + + private static void requireArity(String fnName, List children, int expected) { + if (children.size() != expected) { + throw new UnsupportedOperationException( + "Horn expr translator: " + fnName + " expected arity=" + expected + + " got=" + children.size()); + } + } + + /** Reverse of HornScalarThriftBuilder.visitWindowFrame */ + private WindowFrame buildWindowFrame(TScalar scalar) { + TScalarWindowFrame twf = scalar.getScalar_union().getScalar_window_frame(); + FrameUnitsType units = twf.getWindow_type() == TWindowType.kRows + ? FrameUnitsType.ROWS : FrameUnitsType.RANGE; + FrameBoundary left = twf.isSetWindow_start() && !twf.getWindow_start().isEmpty() + ? buildFrameBoundary(twf.getWindow_start().get(0), true) + : new FrameBoundary(FrameBoundType.EMPTY_BOUNDARY); + FrameBoundary right = twf.isSetWindow_end() && !twf.getWindow_end().isEmpty() + ? buildFrameBoundary(twf.getWindow_end().get(0), false) + : new FrameBoundary(FrameBoundType.EMPTY_BOUNDARY); + return new WindowFrame(units, left, right); + } + + /** Reverse of HornScalarThriftBuilder.buildFrameBoundary */ + private FrameBoundary buildFrameBoundary(TOperator op, boolean isLeft) { + // boundary 子树（cc ScalarWindowFrame::ToThrift 也走 ToThriftList 拍平进 + TScalar scalar = Horn2DorisUtils.getRootScalar(op); + TScalarAnalyticWindowBoundary tb = scalar + .getScalar_union().getScalar_analytic_window_boundary(); + switch (tb.getBoundary_type()) { + case kCurrentRow: + return new FrameBoundary(FrameBoundType.CURRENT_ROW); + case kUnbounded: + return new FrameBoundary(isLeft + ? FrameBoundType.UNBOUNDED_PRECEDING + : FrameBoundType.UNBOUNDED_FOLLOWING); + case kPreceding: + return new FrameBoundary( + Optional.of(buildBoundaryOffset(tb)), + FrameBoundType.PRECEDING); + case kFollowing: + return new FrameBoundary( + Optional.of(buildBoundaryOffset(tb)), + FrameBoundType.FOLLOWING); + default: + throw new UnsupportedOperationException( + "Horn expr translator: unsupported boundary type " + + tb.getBoundary_type()); + } + } + + /** Extract offset for PRECEDING / FOLLOWING boundary */ + private Expression buildBoundaryOffset(TScalarAnalyticWindowBoundary tb) { + if (tb.isSetRange_offset_predicate() && !tb.getRange_offset_predicate().isEmpty()) { + return translate(tb.getRange_offset_predicate().get(0)); + } + if (tb.isSetRows_offset_value()) { + return new BigIntLiteral(tb.getRows_offset_value()); + } + throw new UnsupportedOperationException( + "Horn expr translator: FrameBoundary PRECEDING/FOLLOWING missing offset"); + } + + private Expression translateColumnRef(TScalar scalar) { + long hornScalarUid = scalar.getScalar_unique_id().getUnique_id(); + Slot existing = hornCtx.getScalarUidToSlot().get(hornScalarUid); + if (existing != null) { + return existing; + } + TScalarColumnRef colRef = scalar.getScalar_union().getScalar_column_ref(); + DataType type = HornTypeToDorisConverter.convert(scalar.getData_type()); + List qualifier = ImmutableList.of(colRef.getDatabase_name(), colRef.getTable_name()); + SlotReference newSlot = new SlotReference(colRef.getColumn_name(), type, true, qualifier); + hornCtx.getScalarUidToSlot().put(hornScalarUid, newSlot); + return newSlot; + } + + private Expression translateLiteral(TScalar scalar, DataType resultType) { + TScalarLiteral lit = scalar.getScalar_union().getScalar_literal(); + + if (lit.isIs_null_type()) { + return new NullLiteral(resultType); + } + + if (lit.isSetLiteral_value()) { + TLiteralValue literalValue = lit.getLiteral_value(); + if (literalValue.isSetBool_literal()) { + return BooleanLiteral.of(literalValue.getBool_literal().isValue()); + } + if (literalValue.isSetInt_literal()) { + Literal typed = Literal.convertToTypedLiteral( + literalValue.getInt_literal().getValue(), resultType); + if (typed != null) { + return typed; + } + } + if (literalValue.isSetString_literal()) { + return new StringLiteral(literalValue.getString_literal().getValue()); + } + if (literalValue.isSetFloat_literal()) { + double val = literalValue.getFloat_literal().getValue(); + if (resultType instanceof FloatType) { + return new FloatLiteral((float) val); + } + return new DoubleLiteral(val); + } + if (literalValue.isSetDecimal_literal()) { + // 统一转化使用DecimalV3Literal + String decimalString = new String( + literalValue.getDecimal_literal().getValue(), StandardCharsets.UTF_8); + return new DecimalV3Literal((DecimalV3Type) resultType, new BigDecimal(decimalString)); + } + if (literalValue.isSetDate_literal()) { + // 统一转化使用DateV2Literal + String dateString = literalValue.getDate_literal().getDate_string(); + return new DateV2Literal(dateString); + } + } + + throw new UnsupportedOperationException( + "Horn expr translator: unrecognized literal type for result " + resultType); + } + + private Expression translateBinaryPredicate(TScalar scalar, List children) { + TScalarBinaryPredicate pred = scalar.getScalar_union() + .getScalar_predicate().getScalar_binary_predicate(); + Expression left = children.get(0); + Expression right = children.get(1); + + switch (pred.getBinary_type()) { + case kEqual: + return new EqualTo(left, right); + case kNotEqual: + // Nereids 没有 NotEqual 类，用 Not(EqualTo) 表达 + return new Not(new EqualTo(left, right)); + case kLowerThan: + return new LessThan(left, right); + case kLowerEqual: + return new LessThanEqual(left, right); + case kGatherThan: + return new GreaterThan(left, right); + case kGatherEqual: + return new GreaterThanEqual(left, right); + default: + throw new UnsupportedOperationException( + "Horn expr translator: unsupported binary predicate type " + + pred.getBinary_type()); + } + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/horn/horn2doris/DorisFunctionBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/horn/horn2doris/DorisFunctionBuilder.java new file mode 100644 index 00000000000000..3b9162227e10bb --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/horn/horn2doris/DorisFunctionBuilder.java @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.horn.horn2doris; + +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.FunctionRegistry; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder; +import org.apache.doris.nereids.util.TypeCoercionUtils; + +import com.google.common.collect.ImmutableSet; + +import java.util.List; +import java.util.Optional; + +/** horn wire 上的 fn_name 跟 Doris BoundFunction 的双向桥接 */ +public final class DorisFunctionBuilder { + + /** forward + backward 共享的 scalar fn 白名单 */ + public static final ImmutableSet KNOWN_SCALAR_FNS = ImmutableSet.of( + "abs", "ceil", "coalesce", "concat", + "datediff", "date_add", "date_format", "date_sub", + "day", "floor", "from_unixtime", "greatest", + "if", "ifnull", "least", "length", + "like", "lower", "ltrim", "mod", + "month", "nullif", "power", "quarter", + "regexp", "regexp_like", "rlike", + "round", "rtrim", "sqrt", "substring", + "to_date", "trim", "unix_timestamp", "upper", + "year", + // 日期算术（TimestampArithmetic.getFuncName() 形态 + DaysAdd/DaysSub ScalarFunction 形态） + "days_add", "days_sub", "weeks_add", "weeks_sub", + "months_add", "months_sub", "quarters_add", "quarters_sub", + "years_add", "years_sub", + // Math 高阶 + "sin", "cos", "tan", "asin", "acos", "atan", "atan2", + "sinh", "cosh", "tanh", "asinh", "acosh", "atanh", + "log", "log2", "log10", "ln", "exp", "pi", "e", + "sign", "pow", "radians", "degrees", "cbrt", "cot", + "bin", "conv", "hex", "unhex", + // Random (side-effectful, but Doris allows) + "rand", "random", + // 字符串扩展 + "split_part", "replace", "reverse", "repeat", + "lpad", "rpad", "locate", "find_in_set", + "strleft", "strright", "ascii", "char_length", + "concat_ws", "instr", "space", "left", "right", + // Regexp 家族 + "regexp_extract", "regexp_extract_all", + "regexp_replace", "regexp_replace_one", "regexp_count", + // Hash / 编码 + "md5", "sha1", "sha2", "crc32", + "to_base64", "from_base64", "aes_encrypt", "aes_decrypt", + // 日期高阶 + "date_trunc", "week", "weekofyear", "yearweek", + "dayofweek", "dayofyear", "dayofmonth", "dayname", + "hour", "minute", "second", "microsecond", + "curdate", "curtime", "now", "current_timestamp", + "add_time", "convert_tz", + // 类型转换 / 元信息 + "uuid", "version", "connection_id", "database", + "current_catalog", "current_user"); + + /** forward + backward 共享的 window fn 白名单 */ + public static final ImmutableSet KNOWN_WINDOW_FNS = ImmutableSet.of( + "cume_dist", "dense_rank", "first_value", "lag", "last_value", + "lead", "nth_value", "ntile", "percent_rank", "rank", "row_number"); + + /** forward + backward 共享的 agg fn 白名单 */ + public static final ImmutableSet KNOWN_AGG_FNS = ImmutableSet.of( + "any_value", "avg", "count", "max", "min", "multi_distinct_count", + "multi_distinct_sum", "stddev_samp", "sum", + // 分位 / 统计 + "stddev", "stddev_pop", "variance", "variance_pop", + "variance_samp", "var_samp", "var_pop", + "covar", "covar_samp", + "corr", "corr_welford", + "median", "percentile", "percentile_approx", + // 聚合位运算 / 布尔 + "group_bit_and", "group_bit_or", "group_bit_xor", + "bool_and", "bool_or", "bool_xor", + // 位置 / 分组 + "max_by", "min_by", "avg_weighted", + // 分组拼接 + "group_concat", "multi_distinct_group_concat", + // 特殊聚合 + "sum0", "multi_distinct_sum0", "count_by_enum", + "topn", "topn_array", "topn_weighted", + "retention", "sequence_count", "sequence_match", + "window_funnel", + "kurt", "skew"); + + /** 反译：fn_name + children → BoundFunction（含 implicit cast） */ + public static Expression build(String fnName, List children) { + FunctionRegistry registry = Env.getCurrentEnv().getFunctionRegistry(); + Optional> builders = registry.tryGetBuiltinBuilders(fnName); + if (!builders.isPresent() || builders.get().isEmpty()) { + throw new UnsupportedOperationException( + "DorisFunctionBuilder: no builtin builder for fn_name=" + fnName); + } + FunctionBuilder picked = null; + for (FunctionBuilder fb : builders.get()) { + if (fb.canApply(children)) { + picked = fb; + break; + } + } + if (picked == null) { + throw new UnsupportedOperationException( + "DorisFunctionBuilder: no ctor for fn=" + fnName + + " arity=" + children.size()); + } + BoundFunction fn = (BoundFunction) picked.build(fnName, children).second; + return (Expression) TypeCoercionUtils.processBoundFunction(fn); + } + + private DorisFunctionBuilder() { + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/horn/horn2doris/DorisPhysicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/horn/horn2doris/DorisPhysicalPlanBuilder.java new file mode 100644 index 00000000000000..567b092e47ae7b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/horn/horn2doris/DorisPhysicalPlanBuilder.java @@ -0,0 +1,890 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.horn.horn2doris; + +import org.apache.doris.horn.HornOptimizationContext; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.common.UserException; +import org.apache.horn4j.thrift.TDistributionSpec; +import org.apache.horn4j.thrift.TDorisHashSpec; +import org.apache.horn4j.thrift.TSpecialHashSpecKind; +import org.apache.horn4j.thrift.TExpression; +import org.apache.horn4j.thrift.TPhysicalMotion; +import org.apache.horn4j.thrift.TFlattenedExpression; +import org.apache.horn4j.thrift.TOperator; +import org.apache.horn4j.thrift.TOperatorType; +import org.apache.horn4j.thrift.TPhysicalFilter; +import org.apache.horn4j.thrift.TPhysicalJoin; +import org.apache.horn4j.thrift.TPhysicalAgg; +import org.apache.horn4j.thrift.TPhysicalLimit; +import org.apache.horn4j.thrift.TPhysicalProject; +import org.apache.horn4j.thrift.TPhysicalScan; +import org.apache.horn4j.thrift.TPhysicalSetOp; +import org.apache.horn4j.thrift.TPhysicalSort; +import org.apache.horn4j.thrift.TPhysicalValueScan; +import org.apache.horn4j.thrift.TPhysicalTableScan; +import org.apache.horn4j.thrift.TPhysicalWindow; +import org.apache.horn4j.thrift.TPhysicalGrouping; +import org.apache.horn4j.thrift.TOrderSpec; +import org.apache.horn4j.thrift.TScalar; +import org.apache.doris.nereids.properties.DataTrait; +import org.apache.doris.nereids.properties.DistributionSpec; +import org.apache.doris.nereids.properties.DistributionSpecExecutionAny; +import org.apache.doris.nereids.properties.DistributionSpecGather; +import org.apache.doris.nereids.properties.DistributionSpecHash; +import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; +import org.apache.doris.nereids.properties.DistributionSpecReplicated; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.hint.DistributeHint; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.properties.RequireProperties; +import org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow.WindowFrameGroup; +import org.apache.doris.nereids.trees.expressions.AggregateExpression; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.WindowFrame; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateParam; +import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.DistributeType; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.PreAggStatus; +import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; +import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit; +import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; +import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort; +import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN; +import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion; +import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect; +import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalWindow; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.plans.AggMode; +import org.apache.doris.nereids.trees.plans.AggPhase; +import org.apache.doris.nereids.trees.plans.LimitPhase; +import org.apache.doris.nereids.trees.plans.SortPhase; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.JoinUtils; +import org.apache.doris.statistics.Statistics; + +import com.google.common.collect.ImmutableList; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +/** Rebuild a Doris PhysicalPlan from a Horn TFlattenedExpression */ +public class DorisPhysicalPlanBuilder { + + private static final Logger LOG = LogManager.getLogger(DorisPhysicalPlanBuilder.class); + + private final HornOptimizationContext hornCtx; + private final DorisExpressionBuilder exprTranslator; + + public DorisPhysicalPlanBuilder(HornOptimizationContext hornCtx) { + this.hornCtx = hornCtx; + this.exprTranslator = new DorisExpressionBuilder(hornCtx); + } + + public PhysicalPlan build(TFlattenedExpression flatPlan) throws UserException { + if (flatPlan == null || flatPlan.getTexprs() == null || flatPlan.getTexprs().isEmpty()) { + throw new UserException("Horn CBO: empty optimized plan"); + } + + TExpression root = restoreTree(flatPlan, new AtomicInteger(0)); + + // 默认 need_singleton=false + PhysicalPlan hornPhysicalPlan = buildHelper(root); + + // forward 端 visitLogicalResultSink 把 sink emit 成 root LogicalProject + List sinkOutputs = hornPhysicalPlan.getOutput().stream() + .map(s -> (NamedExpression) s) + .collect(Collectors.toList()); + return new PhysicalResultSink<>( + sinkOutputs, + Optional.empty(), + hornPhysicalPlan.getLogicalProperties(), + hornPhysicalPlan); + } + + private TExpression restoreTree(TFlattenedExpression flat, AtomicInteger idx) { + TExpression expr = flat.getTexprs().get(idx.getAndIncrement()); + long arity = expr.getArity(); + List children = new ArrayList<>(); + for (int i = 0; i < arity; i++) { + children.add(restoreTree(flat, idx)); + } + expr.setChildren(children); + return expr; + } + + private PhysicalPlan buildHelper(TExpression texpr) throws UserException { + // Build children first + List children = new ArrayList<>(); + if (texpr.getChildren() != null) { + for (TExpression child : texpr.getChildren()) { + children.add(buildHelper(child)); + } + } + + TOperatorType opType = texpr.getOp().getOp_type(); + PhysicalPlan plan; + + switch (opType) { + case kPhysicalTableScan: + plan = buildScan(texpr); + break; + + case kPhysicalValueScan: + plan = buildValueScan(texpr); + break; + + case kPhysicalFilter: + plan = buildFilter(texpr, children.get(0)); + break; + + case kPhysicalProject: + plan = buildProject(texpr, children.get(0)); + break; + + case kPhysicalSort: + plan = buildSort(texpr, children.get(0)); + break; + + case kPhysicalLimit: + plan = buildLimit(texpr, children.get(0)); + break; + + // group_by 非空 → kPhysicalHashAgg；空 → kPhysicalScalarAgg。同一套 buildAgg。 + case kPhysicalHashAgg: + case kPhysicalScalarAgg: + plan = buildAgg(texpr, children.get(0)); + break; + + // Doris Nereids PhysicalWindow 走外部排序模式：cascades 通过 + case kPhysicalWindow: + plan = buildWindow(texpr, children.get(0)); + break; + + case kPhysicalGrouping: + plan = buildRepeat(texpr, children.get(0)); + break; + + // Motion → PhysicalDistribute (Fragment boundary) + case kPhysicalMotionGather: + case kPhysicalMotionBroadcast: + case kPhysicalMotionHashDistribute: + case kPhysicalMotionRandom: + plan = buildMotion(texpr, children.get(0)); + break; + + case kPhysicalInnerHashJoin: + case kPhysicalInnerNestedLoopJoin: + case kPhysicalLeftOuterHashJoin: + case kPhysicalLeftOuterNestedLoopJoin: + case kPhysicalRightOuterHashJoin: + case kPhysicalRightOuterNestedLoopJoin: + case kPhysicalFullOuterHashJoin: + case kPhysicalFullOuterNestedLoopJoin: + case kPhysicalLeftSemiHashJoin: + case kPhysicalLeftSemiNestedLoopJoin: + case kPhysicalRightSemiHashJoin: + case kPhysicalRightSemiNestedLoopJoin: + case kPhysicalLeftAntiSemiHashJoin: + case kPhysicalLeftAntiSemiNestedLoopJoin: + case kPhysicalRightAntiSemiHashJoin: + case kPhysicalRightAntiSemiNestedLoopJoin: + case kPhysicalNullAwareLeftAntiSemiHashJoin: + case kPhysicalNullAwareLeftAntiSemiNestedLoopJoin: + case kPhysicalCrossJoin: + plan = buildJoin(texpr, children.get(0), children.get(1)); + break; + + // distinct 已被 Doris BuildAggForUnion 降级成 Agg+UnionAll，故只需 ALL 路径； + case kPhysicalNarrayUnionAll: + case kPhysicalUnionAll: + case kPhysicalIntersect: + case kPhysicalExcept: + plan = buildSetOp(texpr, children); + break; + + default: + throw new UserException("Horn output: unsupported operator " + opType); + } + + // 统一在末尾给 plan 设 PhysicalProperties + Statistics + PhysicalPlan finalPlan = Horn2DorisUtils.setPhysicalProperties(texpr, plan, hornCtx); + // 统一同步：把 scalarUidToSlot 中 ExprId 与 finalPlan.getOutput() 重合的 + syncMapWithPlanOutput(finalPlan); + return finalPlan; + } + + /** 按 ExprId 把 finalPlan.getOutput() 的 slot 真值同步进 scalarUidToSlot */ + private void syncMapWithPlanOutput(PhysicalPlan finalPlan) { + if (hornCtx.getScalarUidToSlot().isEmpty()) { + return; + } + List output = finalPlan.getOutput(); + if (output.isEmpty()) { + return; + } + Map byId = new HashMap<>(output.size()); + for (Slot s : output) { + byId.put(s.getExprId(), s); + } + hornCtx.getScalarUidToSlot().replaceAll((uid, slot) -> { + Slot fresh = byId.get(slot.getExprId()); + return fresh != null ? fresh : slot; + }); + } + + /** Horn TPhysicalSetOp（op_type=kPhysicalNarrayUnionAll / kPhysicalUnionAll）→ Doris */ + private PhysicalPlan buildSetOp(TExpression texpr, List children) throws UserException { + TPhysicalSetOp tsetop = texpr.getOp().getOp_union().getPhysical_set_op(); + + // outputs：union 自身输出列（plain column ref → SlotReference，translate 内部注册 uid→slot）。 + List outputs = new ArrayList<>(); + for (Expression e : exprTranslator.translateList(tsetop.getOutput_list())) { + outputs.add((NamedExpression) e); + } + + // childrenOutputs：每孩子一组 SlotReference（按列对齐 output，位置映射契约）。 + List> childrenOutputs = new ArrayList<>(); + for (List oneChild : tsetop.getChildren_output_list()) { + List cols = new ArrayList<>(); + for (Expression e : exprTranslator.translateList(oneChild)) { + cols.add((SlotReference) e); + } + childrenOutputs.add(cols); + } + + List planChildren = new ArrayList<>(children); + + TOperatorType opType = texpr.getOp().getOp_type(); + if (opType == TOperatorType.kPhysicalIntersect || opType == TOperatorType.kPhysicalExcept) { + List aligned = Horn2DorisUtils.alignSetOpOutputNullable(outputs, childrenOutputs); + if (opType == TOperatorType.kPhysicalIntersect) { + return new PhysicalIntersect(Qualifier.DISTINCT, aligned, childrenOutputs, + null /* logicalProperties → lazy */, planChildren); + } + return new PhysicalExcept(Qualifier.DISTINCT, aligned, childrenOutputs, + null /* logicalProperties → lazy */, planChildren); + } + + // union all：constantExprsList（可空，透传） + List> constExprs = new ArrayList<>(); + if (tsetop.isSetChildren_const_expr_list()) { + for (List row : tsetop.getChildren_const_expr_list()) { + List r = new ArrayList<>(); + for (Expression e : exprTranslator.translateList(row)) { + r.add(new Alias(e)); + } + constExprs.add(r); + } + } + return new PhysicalUnion(Qualifier.ALL, outputs, childrenOutputs, constExprs, + null /* logicalProperties → lazy computeOutput */, planChildren); + } + + private PhysicalPlan buildScan(TExpression texpr) throws UserException { + TPhysicalScan physicalScan = texpr.getOp().getOp_union().getPhysical_scan(); + TPhysicalTableScan tableScan = physicalScan.getPhysical_table_scan(); + String tableName = tableScan.getTable_name(); + String dbName = tableScan.getDatabase_name(); + + // horn 只处理 OlapTable + OlapTable olapTable = (OlapTable) hornCtx.getCascadesContext().getConnectContext() + .getCurrentCatalog() + .getDbOrAnalysisException(dbName) + .getTableOrAnalysisException(tableName); + + // 从 Horn output_list 构造 scan slots，并添加到 scalarUidToSlot。 + StatementContext stmtCtx = hornCtx.getCascadesContext().getStatementContext(); + ImmutableList qualifier = ImmutableList.of(dbName, tableName); + List scanSlots = new ArrayList<>(); + if (texpr.getOutput_list() != null) { + for (TOperator outOp : texpr.getOutput_list()) { + TScalar scalar = Horn2DorisUtils.getRootScalar(outOp); + String colName = scalar.getScalar_union().getScalar_column_ref().getColumn_name(); + SlotReference slotRef = SlotReference.fromColumn( + stmtCtx.getNextExprId(), olapTable, olapTable.getColumn(colName), qualifier); + hornCtx.getScalarUidToSlot().put(scalar.getScalar_unique_id().getUnique_id(), slotRef); + scanSlots.add(slotRef); + } + } + // 分布列被裁剪时（如 count(*) / GROUP BY 非分布键）补回 scanSlots —— 对齐 + DistributionInfo scanDistInfo = olapTable.getDefaultDistributionInfo(); + if (scanDistInfo instanceof HashDistributionInfo) { + for (Column distCol : ((HashDistributionInfo) scanDistInfo).getDistributionColumns()) { + if (scanSlots.stream().noneMatch( + s -> s.getName().equalsIgnoreCase(distCol.getName()))) { + scanSlots.add(SlotReference.fromColumn( + stmtCtx.getNextExprId(), olapTable, distCol, qualifier)); + } + } + } + // RANDOM 分布表（无分布列可补）的 count(*) 场景 scanSlots 仍空：实测删掉本 + if (scanSlots.isEmpty()) { + List candidates = new ArrayList<>(); + for (Column col : olapTable.getBaseSchema()) { + candidates.add(SlotReference.fromColumn( + stmtCtx.getNextExprId(), olapTable, col, qualifier)); + } + scanSlots.add(ExpressionUtils.selectMinimumColumn(candidates)); + } + // 分区集走 external_relation_id 查 OptimizeContext 里 forward 登记的 prunedPartitionIds, + if (!tableScan.isSetExternal_relation_id()) { + throw new UserException("horn buildScan: TPhysicalTableScan missing external_relation_id (table " + + tableName + ")"); + } + long extRelationId = tableScan.getExternal_relation_id(); + List partitionIds = hornCtx.getScanPrunedPartitions(extRelationId); + if (partitionIds == null) { + throw new UserException("horn buildScan: no pruned-partition binding for external_relation_id=" + + extRelationId + " (table " + tableName + ")"); + } + List tabletIds = new ArrayList<>(); + for (Long partId : partitionIds) { + olapTable.getPartition(partId).getBaseIndex().getTablets() + .forEach(t -> tabletIds.add(t.getId())); + } + + // PhysicalCatalogRelation.computeOutput() 用 table.getBaseSchema() 重 ExprId， + ImmutableList immutableScanSlots = ImmutableList.copyOf(scanSlots); + LogicalProperties scanProps = new LogicalProperties( + () -> immutableScanSlots, + () -> DataTrait.EMPTY_TRAIT); + + DistributionSpec scanDistSpec = Horn2DorisUtils.buildScanDistributionSpec( + olapTable, scanSlots, partitionIds); + + PhysicalOlapScan scan = new PhysicalOlapScan( + hornCtx.getCascadesContext().getStatementContext().getNextRelationId(), + olapTable, + ImmutableList.of(dbName, tableName), + olapTable.getBaseIndexId(), + tabletIds, + partitionIds, + scanDistSpec, + PreAggStatus.on(), + immutableScanSlots, + Optional.empty(), + scanProps, + Optional.empty(), + immutableScanSlots, + Collections.emptyList(), + Collections.emptyList(), + Optional.empty(), + Optional.empty(), + Collections.emptyList(), + Optional.empty() + ); + + return scan; + } + + /** horn PhysicalValueScan 反译为 Doris 物理叶子： */ + private PhysicalPlan buildValueScan(TExpression texpr) throws UserException { + // horn PhysicalValueScan::ToThrift 走 TPhysicalScan wrapper sub-struct + TPhysicalValueScan valueScan = texpr.getOp().getOp_union() + .getPhysical_scan().getPhysical_value_scan(); + StatementContext stmtCtx = hornCtx.getCascadesContext().getStatementContext(); + + List schemaColumns = valueScan.isSetOutput_list() + ? valueScan.getOutput_list() : Collections.emptyList(); + if (schemaColumns.isEmpty()) { + throw new UserException( + "Horn output: PhysicalValueScan missing output_list (schema)"); + } + + List schemaExprs = exprTranslator.translateList(schemaColumns); + List schemaSlots = new ArrayList<>(schemaExprs.size()); + for (Expression e : schemaExprs) { + schemaSlots.add((Slot) e); + } + + List> rows = valueScan.isSetValue_lists() + ? valueScan.getValue_lists() : Collections.emptyList(); + + // PhysicalOneRowRelation 在主线没 override computeOutput → 传 null logicalProperties + ImmutableList immutableSchema = ImmutableList.copyOf(schemaSlots); + LogicalProperties relationProps = new LogicalProperties( + () -> immutableSchema, + () -> DataTrait.EMPTY_TRAIT); + + if (rows.isEmpty()) { + return new PhysicalEmptyRelation(stmtCtx.getNextRelationId(), schemaSlots, relationProps); + } + if (rows.size() > 1) { + throw new UserException("Horn output: PhysicalValueScan with " + + rows.size() + " rows not supported"); + } + // 1 行：PhysicalOneRowRelation.output 直接从 projects.toSlot 派生， + List singleRow = rows.get(0); + if (singleRow.size() != schemaSlots.size()) { + throw new UserException("Horn output: PhysicalValueScan row arity " + + singleRow.size() + " != schema arity " + schemaSlots.size()); + } + List projects = new ArrayList<>(schemaSlots.size()); + for (int i = 0; i < schemaSlots.size(); i++) { + Slot slot = schemaSlots.get(i); + Expression cellExpr = exprTranslator.translate(singleRow.get(i)); + projects.add(new Alias(slot.getExprId(), cellExpr, slot.getName())); + } + return new PhysicalOneRowRelation(stmtCtx.getNextRelationId(), projects, relationProps); + } + + private PhysicalPlan buildFilter(TExpression texpr, PhysicalPlan child) throws UserException { + TPhysicalFilter tfilter = texpr.getOp().getOp_union().getPhysical_filter(); + + Set conjuncts = new HashSet<>(); + for (TOperator predOp : tfilter.getFilter_list()) { + Expression expr = exprTranslator.translate(predOp); + LOG.debug("HORN_FILTER predicate: {}", expr); + conjuncts.add(expr); + } + return new PhysicalFilter<>(conjuncts, null, child); + } + + private PhysicalPlan buildProject(TExpression texpr, PhysicalPlan child) throws UserException { + TPhysicalProject tproj = texpr.getOp().getOp_union().getPhysical_project(); + + List projectList = tproj.getProject_list(); + List rewriteList = tproj.getRewrite_list(); + List projects = new ArrayList<>(); + for (int i = 0; i < projectList.size(); i++) { + TOperator projOp = projectList.get(i); + long projUid = Horn2DorisUtils.getRootScalar(projOp).getScalar_unique_id().getUnique_id(); + Slot existingSlot = hornCtx.getScalarUidToSlot().get(projUid); + if (existingSlot != null) { + // 已注册 slot 直接透传 + if (existingSlot instanceof MarkJoinSlotReference) { + String aliasName = Horn2DorisUtils.getRootScalar(rewriteList.get(i)) + .getScalar_union().getScalar_column_ref().getColumn_name(); + if (aliasName != null && !aliasName.equalsIgnoreCase(existingSlot.getName())) { + NamedExpression aliased = new Alias(existingSlot, aliasName); + projects.add(aliased); + hornCtx.getScalarUidToSlot().put(projUid, aliased.toSlot()); + continue; + } + } + projects.add(existingSlot); + continue; + } + Expression projExpr = exprTranslator.translate(projOp); + String aliasName = Horn2DorisUtils.getRootScalar(rewriteList.get(i)) + .getScalar_union().getScalar_column_ref().getColumn_name(); + NamedExpression namedProj = new Alias(projExpr, aliasName); + projects.add(namedProj); + hornCtx.getScalarUidToSlot().put(projUid, namedProj.toSlot()); + } + + // identity，且 PhysicalProject 里的 Grouping 翻译到 BE 会 "root is null"），故把含 GroupingScalarFunction 的 + if (child instanceof PhysicalRepeat + && projects.stream().anyMatch(p -> p.containsType(GroupingScalarFunction.class))) { + PhysicalRepeat repeat = (PhysicalRepeat) child; + List repeatOutput = new ArrayList<>(repeat.getOutputExpressions()); + List newProjects = new ArrayList<>(projects.size()); + for (NamedExpression p : projects) { + if (p.containsType(GroupingScalarFunction.class)) { + repeatOutput.add(p); // Grouping(..) AS GROUPING_PREFIX_* 还原进 Repeat + newProjects.add(p.toSlot()); // Project 退化成透传该 slot + } else { + newProjects.add(p); + } + } + // Doris 计划不可变：withAggOutput 重建带新 output 的 Repeat，resetLogicalProperties 让 output 随新增列重算 + child = repeat.withAggOutput(repeatOutput).resetLogicalProperties(); + projects = newProjects; + } + + // logicalProperties 传 null → AbstractPhysicalPlan ctor 转 Optional.empty() + return new PhysicalProject<>(projects, null, child); + } + + /** Translate a Horn Motion node to a Doris PhysicalDistribute (Fragment boundary) */ + private PhysicalPlan buildMotion(TExpression texpr, PhysicalPlan child) throws UserException { + TPhysicalMotion tmotion = texpr.getOp().getOp_union().getPhysical_motion(); + TDistributionSpec tspec = tmotion.getDistribution_spec(); + // Merge gather: horn PhysicalMotionGather 自带 sort_info 时表示跨 fragment + if (tmotion.isSetSort_info() + && tmotion.getSort_info().getOrder_by_exprs() != null + && !tmotion.getSort_info().getOrder_by_exprs().isEmpty()) { + List orderKeys = buildOrderKeys(tmotion.getSort_info()); + PhysicalPlan gather = new PhysicalDistribute<>(toDistributionSpec(tspec), child); + return new PhysicalQuickSort<>(orderKeys, SortPhase.MERGE_SORT, null, gather); + } + // 普通 Motion (无 merge gather)：单纯 PhysicalDistribute。 + return new PhysicalDistribute<>(toDistributionSpec(tspec), child); + } + + private PhysicalPlan buildJoin(TExpression texpr, PhysicalPlan left, PhysicalPlan right) + throws UserException { + TOperatorType opType = texpr.getOp().getOp_type(); + JoinType joinType = Horn2DorisUtils.getJoinType(opType); + boolean useHash = Horn2DorisUtils.isHashJoin(opType); + + TPhysicalJoin tjoin = texpr.getOp().getOp_union().getPhysical_join(); + List hashConjuncts = exprTranslator.translateList(tjoin.getEqual_join_predicate_list()); + List otherConjuncts = exprTranslator.translateList(tjoin.getOther_join_conjuncts()); + + // mark join（IN / EXISTS-OR 子查询）：重建 MarkJoinSlotReference + mark 谓词 + Optional markSlotRef = Optional.empty(); + List markConjuncts = ExpressionUtils.EMPTY_CONDITION; + if (tjoin.isSetMark_slot() && !tjoin.getMark_slot().isEmpty()) { + // mark_slot 是单元素 list（同 grouping_id_func）。 + TScalar markScalar = Horn2DorisUtils.getRootScalar(tjoin.getMark_slot().get(0)); + long markUid = markScalar.getScalar_unique_id().getUnique_id(); + Slot existing = hornCtx.getScalarUidToSlot().get(markUid); + MarkJoinSlotReference mjsr; + if (existing instanceof MarkJoinSlotReference) { + mjsr = (MarkJoinSlotReference) existing; + } else { + String markName = markScalar.getScalar_union().getScalar_column_ref().getColumn_name(); + mjsr = new MarkJoinSlotReference( + hornCtx.getCascadesContext().getStatementContext().getNextExprId(), + markName, false); + hornCtx.getScalarUidToSlot().put(markUid, mjsr); + } + markSlotRef = Optional.of(mjsr); + if (tjoin.isSetMark_join_predicate_list() + && tjoin.getMark_join_predicate_list() != null + && !tjoin.getMark_join_predicate_list().isEmpty()) { + markConjuncts = exprTranslator.translateList(tjoin.getMark_join_predicate_list()); + } + } + + // mark join：传 null logicalProperties → AbstractPhysicalPlan 惰性走 computeOutput()， + LogicalProperties props = markSlotRef.isPresent() ? null : new LogicalProperties( + () -> JoinUtils.getJoinOutput(joinType, left, right), + () -> DataTrait.EMPTY_TRAIT); + // PhysicalProperties 由 buildHelper 末尾 attachProperties 统一处理 + PhysicalPlan join; + if (useHash) { + join = new PhysicalHashJoin<>(joinType, hashConjuncts, otherConjuncts, + markConjuncts, new DistributeHint(DistributeType.NONE), markSlotRef, + props, left, right); + } else { + join = new PhysicalNestedLoopJoin<>(joinType, hashConjuncts, otherConjuncts, + markConjuncts, markSlotRef, props, left, right); + } + // outer join 输出侧 nullable 升格的回写改由 buildHelper 末尾统一 + return join; + } + + /** Convert a Horn TDistributionSpec to a Doris DistributionSpec */ + private DistributionSpec toDistributionSpec(TDistributionSpec tspec) { + switch (tspec.getDistribution_type()) { + case kSingleton: + // motion gather 之后单点 → DistributionSpecGather。 + return DistributionSpecGather.INSTANCE; + case kReplicated: + case kUniversal: + // kReplicated: PhysicalMotionBroadcast 的 target，每个 instance 都 + return DistributionSpecReplicated.INSTANCE; + case kHashed: { + // kHashed 一定带 hash_columns；每个 hash 列的 unique_id 一定已被 + List hashExprIds = new ArrayList<>(); + for (TOperator hashCol : tspec.getHash_columns()) { + long hornScalarUid = Horn2DorisUtils.getRootScalar(hashCol).getScalar_unique_id().getUnique_id(); + hashExprIds.add(((SlotReference) hornCtx.getScalarUidToSlot().get(hornScalarUid)).getExprId()); + } + // 带 DorisHashSpec 物理身份 → 绑定到目标表 storage 桶布局 + if (tspec.isSetSpecial_hash_spec() + && tspec.getSpecial_hash_spec().getKind() == TSpecialHashSpecKind.kDoris) { + TDorisHashSpec storageBucketSpec = tspec.getSpecial_hash_spec().getDoris_hash_spec(); + long storageBucketTableId = storageBucketSpec.getTable_id(); + OlapTable storageBucketTable = (OlapTable) Env.getCurrentInternalCatalog() + .getTableByTableId(storageBucketTableId); + return new DistributionSpecHash( + hashExprIds, ShuffleType.STORAGE_BUCKETED, + storageBucketTableId, + storageBucketTable.getBaseIndexId(), + new LinkedHashSet<>(storageBucketSpec.getSelected_partition_ids())); + } + // 无 special(Empty)→ 普通运行时 hash 重分布 → EXECUTION_BUCKETED + return new DistributionSpecHash(hashExprIds, ShuffleType.EXECUTION_BUCKETED); + } + case kRandom: + // PhysicalMotionRandom 主动 scramble 到运行时任意节点 + return DistributionSpecExecutionAny.INSTANCE; + default: + // kAny / kNonSingleton: horn 端 require-only 标签，motion derive + throw new IllegalStateException( + "Unexpected horn motion distribution type on derive path: " + + tspec.getDistribution_type()); + } + } + + // v7 PhysicalSort 反译：OrderEnforcer 兜底产的 sort enforcer，反译成 Doris + private PhysicalPlan buildSort(TExpression texpr, PhysicalPlan child) throws UserException { + TPhysicalSort tsort = texpr.getOp().getOp_union().getPhysical_sort(); + TOrderSpec sortInfo = tsort.getSort_info(); + if (sortInfo == null) { + throw new UserException("Horn output: PhysicalSort missing sort_info"); + } + List orderKeys = buildOrderKeys(sortInfo); + return new PhysicalQuickSort<>(orderKeys, SortPhase.LOCAL_SORT, null, child); + } + + // v7 PhysicalLimit 反译: + private PhysicalPlan buildLimit(TExpression texpr, PhysicalPlan child) { + TPhysicalLimit tlimit = texpr.getOp().getOp_union().getPhysical_limit(); + long limit = tlimit.isSetLimit() ? tlimit.getLimit() : -1; + long offset = tlimit.isSetOffset() ? tlimit.getOffset() : 0; + boolean isGlobal = tlimit.isSetIs_global_limit() ? tlimit.isIs_global_limit() : true; + if (tlimit.isSetOrder_spec() + && tlimit.getOrder_spec().getOrder_by_exprs() != null + && !tlimit.getOrder_spec().getOrder_by_exprs().isEmpty()) { + List orderKeys = buildOrderKeys(tlimit.getOrder_spec()); + SortPhase sortPhase = isGlobal ? SortPhase.MERGE_SORT : SortPhase.LOCAL_SORT; + PhysicalPlan topnChild = child instanceof PhysicalQuickSort + ? (PhysicalPlan) ((PhysicalQuickSort) child).child(0) + : child; + return new PhysicalTopN<>(orderKeys, limit, offset, sortPhase, null, topnChild); + } + LimitPhase phase = isGlobal ? LimitPhase.GLOBAL : LimitPhase.LOCAL; + return new PhysicalLimit<>(limit, offset, phase, null, child); + } + + /** 反译 horn PhysicalHashAgg / PhysicalScalarAgg → Doris PhysicalHashAggregate */ + private PhysicalPlan buildAgg(TExpression texpr, PhysicalPlan child) throws UserException { + TPhysicalAgg tagg = texpr.getOp().getOp_union().getPhysical_agg(); + boolean isSplit = tagg.isIs_split(); + boolean isLocal = tagg.isIs_local_agg(); + + // group_by：列引用（极少表达式），反译后透传进 output（两阶段 group key slot 不变） + List groupByExprs = new ArrayList<>(); + List output = new ArrayList<>(); + if (tagg.getGroup_by_exprs() != null) { + for (TOperator gbOp : tagg.getGroup_by_exprs()) { + Expression gb = exprTranslator.translate(gbOp); + groupByExprs.add(gb); + if (gb instanceof NamedExpression) { + output.add((NamedExpression) gb); + } else { + output.add(new Alias(gb)); + } + } + } + + // 注意：分支循环里只拼 output，不注册 map —— 注册统一在构造之后（见方法尾）， + AggregateParam param; + if (isSplit && isLocal) { + // ---- local: 原函数 → AggregateExpression(INPUT_TO_BUFFER) 包 Alias 出 buffer slot ---- + param = new AggregateParam(AggPhase.LOCAL, AggMode.INPUT_TO_BUFFER); + if (tagg.getAggregate_functions() != null) { + for (TOperator fnOp : tagg.getAggregate_functions()) { + long uid = Horn2DorisUtils.getRootScalar(fnOp).getScalar_unique_id().getUnique_id(); + Expression fnExpr = exprTranslator.translate(fnOp); + if (!(fnExpr instanceof AggregateFunction)) { + throw new UserException("Horn: local agg expected AggregateFunction, got " + + fnExpr.getClass().getSimpleName()); + } + AggregateFunction fn = (AggregateFunction) fnExpr; + Alias buffer = new Alias(new AggregateExpression(fn, param)); + output.add(buffer); + hornCtx.getScalarUidToSlot().put(uid, buffer.toSlot()); + hornCtx.getScalarUidToAggFunc().put(uid, fn); + } + } + } else if (isSplit) { + // ---- global: thrift 上是 ColumnRef，按 uid lookup 原函数 + buffer slot + param = new AggregateParam(AggPhase.GLOBAL, AggMode.BUFFER_TO_RESULT); + if (tagg.getAggregate_functions() != null) { + for (TOperator fnOp : tagg.getAggregate_functions()) { + long uid = Horn2DorisUtils.getRootScalar(fnOp).getScalar_unique_id().getUnique_id(); + AggregateFunction fn = hornCtx.getScalarUidToAggFunc().get(uid); + Slot buffer = hornCtx.getScalarUidToSlot().get(uid); + if (fn == null || buffer == null) { + throw new UserException( + "Horn: global agg cannot find local fn/buffer for uid=" + uid); + } + Alias result = new Alias(new AggregateExpression(fn, param, buffer)); + output.add(result); + hornCtx.getScalarUidToSlot().put(uid, result.toSlot()); + } + } + } else { + // ---- 单层 (CombineTwoAdjacentAggs 撤销 split 或未拆): INPUT_TO_RESULT ---- + param = new AggregateParam(AggPhase.GLOBAL, AggMode.INPUT_TO_RESULT); + if (tagg.getAggregate_functions() != null) { + for (TOperator fnOp : tagg.getAggregate_functions()) { + long uid = Horn2DorisUtils.getRootScalar(fnOp).getScalar_unique_id().getUnique_id(); + Expression fnExpr = exprTranslator.translate(fnOp); + if (!(fnExpr instanceof AggregateFunction)) { + throw new UserException("Horn: single agg expected AggregateFunction, got " + + fnExpr.getClass().getSimpleName()); + } + AggregateFunction fn = (AggregateFunction) fnExpr; + Alias result = new Alias(new AggregateExpression(fn, param)); + output.add(result); + hornCtx.getScalarUidToSlot().put(uid, result.toSlot()); + } + } + } + + PhysicalHashAggregate aggPlan = new PhysicalHashAggregate<>( + groupByExprs, output, Optional.empty(), + param, false /* maybeUsingStream */, null /* logicalProperties */, + false /* hasSourceRepeat */, child); + + // scalarUidToSlot 的 nullable 升格回写改由 buildHelper 末尾统一 + return aggPlan; + } + + /** 反译 horn PhysicalWindow → Doris PhysicalWindow */ + private PhysicalPlan buildWindow(TExpression texpr, PhysicalPlan child) throws UserException { + TPhysicalWindow twindow = texpr.getOp().getOp_union().getPhysical_window(); + + List partitionKeys = twindow.isSetPartition_exprs() + ? exprTranslator.translateList(twindow.getPartition_exprs()) + : ImmutableList.of(); + + List orderKeys = new ArrayList<>(); + if (twindow.isSetOrder_by_exprs()) { + for (OrderKey orderKey : buildOrderKeys(twindow.getOrder_by_exprs())) { + orderKeys.add(new OrderExpression(orderKey)); + } + } + + // frame 经 translateScalarNode 的 kWindowFrame case 统一分发（WindowFrame is-a + WindowFrame frame = (WindowFrame) exprTranslator.translate(twindow.getWindow_frame().get(0)); + + List windowExpressions = new ArrayList<>(); + for (TOperator fnOp : twindow.getAnalytic_functions()) { + TScalar rootScalar = Horn2DorisUtils.getRootScalar(fnOp); + long uid = rootScalar.getScalar_unique_id().getUnique_id(); + Expression fnExpr = exprTranslator.translate(fnOp); + WindowExpression wExpr = new WindowExpression( + fnExpr, partitionKeys, orderKeys, frame, false /* isSkew */); + Alias windowAlias = new Alias(wExpr, fnExpr.toSql()); + windowExpressions.add(windowAlias); + hornCtx.getScalarUidToSlot().put(uid, windowAlias.toSlot()); + } + + WindowFrameGroup wfg = new WindowFrameGroup(windowExpressions.get(0)); + for (int i = 1; i < windowExpressions.size(); i++) { + wfg.addGroup(windowExpressions.get(i)); + } + + // 跟 Doris 主线 LogicalWindowToPhysicalWindow.java:133 一致： + RequireProperties requireProperties = RequireProperties.followParent(); + + return new PhysicalWindow<>( + wfg, requireProperties, windowExpressions, false /* isSkew */, + null /* logicalProperties */, child); + } + + /**

GROUPING_ID slot constructed nullable=false to match Doris NormalizeRepeat default */ + private PhysicalPlan buildRepeat(TExpression texpr, PhysicalPlan child) throws UserException { + TPhysicalGrouping tg = texpr.getOp().getOp_union().getPhysical_grouping(); + + // 1. grouping_set_list → List> + List> groupingSets = new ArrayList<>(tg.getGrouping_set_list().size()); + for (List tset : tg.getGrouping_set_list()) { + groupingSets.add(exprTranslator.translateList(tset)); + } + + // 2 + TOperator tGroupingIdFn = tg.getGrouping_id_func().get(0); + SlotReference groupingId = (SlotReference) exprTranslator.translate(tGroupingIdFn); + groupingId = groupingId.withNullable(false); + long groupingIdUid = Horn2DorisUtils.getRootScalar(tGroupingIdFn) + .getScalar_unique_id().getUnique_id(); + hornCtx.getScalarUidToSlot().put(groupingIdUid, groupingId); + + // 3 + List outputExpressions = new ArrayList<>(); + for (TOperator colOp : tg.getGrouping_expr_list()) { + SlotReference slot = (SlotReference) exprTranslator.translate(colOp); + outputExpressions.add(slot); + long uid = Horn2DorisUtils.getRootScalar(colOp).getScalar_unique_id().getUnique_id(); + hornCtx.getScalarUidToSlot().put(uid, slot); + } + if (tg.isSetAggregate_function_inputs()) { + for (TOperator colOp : tg.getAggregate_function_inputs()) { + SlotReference slot = (SlotReference) exprTranslator.translate(colOp); + outputExpressions.add(slot); + long uid = Horn2DorisUtils.getRootScalar(colOp).getScalar_unique_id().getUnique_id(); + hornCtx.getScalarUidToSlot().put(uid, slot); + } + } + + // 4 + return new PhysicalRepeat<>( + groupingSets, outputExpressions, groupingId, + null /* logicalProperties */, child); + } + + /** 把 horn TOrderSpec.order_by_exprs 翻译成 Doris OrderKey 列表 */ + private List buildOrderKeys(TOrderSpec sortInfo) { + List orderKeys = new ArrayList<>(); + List exprList = sortInfo.getOrder_by_exprs(); + List ascList = sortInfo.getIs_asc_order(); + List nullsFirstList = sortInfo.getNulls_first(); + if (exprList == null) { + return orderKeys; + } + for (int i = 0; i < exprList.size(); ++i) { + Expression expr = exprTranslator.translate(exprList.get(i)); + boolean asc = i < ascList.size() ? ascList.get(i) : true; + boolean nullsFirst = i < nullsFirstList.size() ? nullsFirstList.get(i) : !asc; + orderKeys.add(new OrderKey(expr, asc, nullsFirst)); + } + return orderKeys; + } + + // getRootScalar 已移到 Horn2DorisUtils（跨 builder 复用：DorisExpressionBuilder.buildFrameBoundary 也需要） + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/horn/horn2doris/Horn2DorisUtils.java b/fe/fe-core/src/main/java/org/apache/doris/horn/horn2doris/Horn2DorisUtils.java new file mode 100644 index 00000000000000..4ccd5ad0848761 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/horn/horn2doris/Horn2DorisUtils.java @@ -0,0 +1,305 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.horn.horn2doris; + +import org.apache.doris.catalog.ColocateTableIndex; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.horn.HornOptimizationContext; +import org.apache.doris.nereids.properties.DistributionSpec; +import org.apache.doris.nereids.properties.DistributionSpecAny; +import org.apache.doris.nereids.properties.DistributionSpecExecutionAny; +import org.apache.doris.nereids.properties.DistributionSpecGather; +import org.apache.doris.nereids.properties.DistributionSpecHash; +import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; +import org.apache.doris.nereids.properties.DistributionSpecReplicated; +import org.apache.doris.nereids.properties.DistributionSpecStorageAny; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.horn4j.thrift.TDistributionSpec; +import org.apache.horn4j.thrift.TSpecialHashSpecKind; +import org.apache.horn4j.thrift.TDorisHashSpec; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; +import org.apache.doris.nereids.util.JoinUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.statistics.ColumnStatistic; +import org.apache.doris.statistics.Statistics; +import org.apache.doris.statistics.StatisticsCache; +import org.apache.horn4j.thrift.TExpression; +import org.apache.horn4j.thrift.TOperator; +import org.apache.horn4j.thrift.TOperatorType; +import org.apache.horn4j.thrift.TOperatorUnion; +import org.apache.horn4j.thrift.TScalar; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; + +/** horn2doris 反译路径下的纯静态辅助方法集中点 —— 跟 {@link DorisPhysicalPlanBuilder} */ +public final class Horn2DorisUtils { + + private static final Logger LOG = LogManager.getLogger(Horn2DorisUtils.class); + + private Horn2DorisUtils() { + } + + /** 从 TOperator 取 root TScalar，兼容 scalar 单节点 / fscalar 扁平树两种 union 形态 */ + public static TScalar getRootScalar(TOperator op) { + TOperatorUnion union = op.getOp_union(); + if (union.isSetFscalar()) { + List scalars = union.getFscalar().getScalars(); + if (scalars == null || scalars.isEmpty()) { + throw new IllegalArgumentException( + "Horn: fscalar with empty scalars list, op_type=" + op.getOp_type()); + } + return scalars.get(0).getOp_union().getScalar(); + } + if (!union.isSetScalar()) { + throw new IllegalArgumentException( + "Horn: TOperator has neither scalar nor fscalar payload, op_type=" + + op.getOp_type()); + } + return union.getScalar(); + } + + public static boolean isHashJoin(TOperatorType opType) { + switch (opType) { + case kPhysicalInnerHashJoin: + case kPhysicalLeftOuterHashJoin: + case kPhysicalRightOuterHashJoin: + case kPhysicalFullOuterHashJoin: + case kPhysicalLeftSemiHashJoin: + case kPhysicalRightSemiHashJoin: + case kPhysicalLeftAntiSemiHashJoin: + case kPhysicalRightAntiSemiHashJoin: + case kPhysicalNullAwareLeftAntiSemiHashJoin: + return true; + default: + return false; + } + } + + public static JoinType getJoinType(TOperatorType opType) { + switch (opType) { + case kPhysicalInnerHashJoin: + case kPhysicalInnerNestedLoopJoin: + return JoinType.INNER_JOIN; + case kPhysicalLeftOuterHashJoin: + case kPhysicalLeftOuterNestedLoopJoin: + return JoinType.LEFT_OUTER_JOIN; + case kPhysicalRightOuterHashJoin: + case kPhysicalRightOuterNestedLoopJoin: + return JoinType.RIGHT_OUTER_JOIN; + case kPhysicalFullOuterHashJoin: + case kPhysicalFullOuterNestedLoopJoin: + return JoinType.FULL_OUTER_JOIN; + case kPhysicalLeftSemiHashJoin: + case kPhysicalLeftSemiNestedLoopJoin: + return JoinType.LEFT_SEMI_JOIN; + case kPhysicalRightSemiHashJoin: + case kPhysicalRightSemiNestedLoopJoin: + return JoinType.RIGHT_SEMI_JOIN; + case kPhysicalLeftAntiSemiHashJoin: + case kPhysicalLeftAntiSemiNestedLoopJoin: + return JoinType.LEFT_ANTI_JOIN; + case kPhysicalRightAntiSemiHashJoin: + case kPhysicalRightAntiSemiNestedLoopJoin: + return JoinType.RIGHT_ANTI_JOIN; + case kPhysicalNullAwareLeftAntiSemiHashJoin: + case kPhysicalNullAwareLeftAntiSemiNestedLoopJoin: + return JoinType.NULL_AWARE_LEFT_ANTI_JOIN; + case kPhysicalCrossJoin: + return JoinType.CROSS_JOIN; + default: + throw new IllegalStateException("Horn: unsupported join op_type: " + opType); + } + } + + /** Build Doris Statistics from Horn TExpression.cost_info and plan output */ + public static Statistics buildStatistics(TExpression texpr, PhysicalPlan plan) { + double rowCount = 1.0; + if (texpr.isSetCost_info() && texpr.getCost_info().isSetRow_numbers()) { + rowCount = Math.max(texpr.getCost_info().getRow_numbers(), 1.0); + } + Map colStats = new HashMap<>(); + ConnectContext ctx = ConnectContext.get(); + StatisticsCache statsCache = ctx != null ? Env.getCurrentEnv().getStatisticsCache() : null; + for (Slot slot : plan.getOutput()) { + colStats.put(slot, resolveColumnStatistic(slot, statsCache, ctx)); + } + return new Statistics(rowCount, colStats); + } + + private static ColumnStatistic resolveColumnStatistic( + Slot slot, StatisticsCache statsCache, ConnectContext ctx) { + if (statsCache == null || !(slot instanceof SlotReference)) { + return ColumnStatistic.UNKNOWN; + } + SlotReference sr = (SlotReference) slot; + if (!sr.getOriginalTable().isPresent() || !sr.getOriginalColumn().isPresent()) { + return ColumnStatistic.UNKNOWN; + } + if (!(sr.getOriginalTable().get() instanceof OlapTable)) { + return ColumnStatistic.UNKNOWN; + } + OlapTable table = (OlapTable) sr.getOriginalTable().get(); + ColumnStatistic stat = statsCache.getColumnStatistics( + table.getDatabase().getCatalog().getId(), + table.getDatabase().getId(), + table.getId(), + table.getBaseIndexId(), + sr.getOriginalColumn().get().getName(), + ctx); + return stat != null ? stat : ColumnStatistic.UNKNOWN; + } + + /** 给 plan 设 PhysicalProperties + Statistics */ + public static PhysicalPlan setPhysicalProperties(TExpression texpr, PhysicalPlan plan, + HornOptimizationContext hornCtx) { + PhysicalProperties props; + if (plan instanceof PhysicalOlapScan) { + props = new PhysicalProperties(((PhysicalOlapScan) plan).getDistributionSpec()); + } else if (plan instanceof PhysicalDistribute) { + props = new PhysicalProperties(((PhysicalDistribute) plan).getDistributionSpec()); + } else if (texpr.isSetDistribution_spec()) { + props = new PhysicalProperties( + buildDistributionSpec(texpr.getDistribution_spec(), hornCtx)); + } else { + // 边界:texpr 未带 distribution_spec(极少数场景如 ResultSink),兜底 ANY。 + props = PhysicalProperties.ANY; + } + return (PhysicalPlan) plan.withPhysicalPropertiesAndStats( + props, buildStatistics(texpr, plan)); + } + + /** 反译 Horn 侧 TDistributionSpec 到 Doris DistributionSpec */ + private static DistributionSpec buildDistributionSpec(TDistributionSpec tspec, + HornOptimizationContext hornCtx) { + switch (tspec.getDistribution_type()) { + case kSingleton: + return DistributionSpecGather.INSTANCE; + case kReplicated: + case kUniversal: + return DistributionSpecReplicated.INSTANCE; + case kRandom: + return DistributionSpecExecutionAny.INSTANCE; + case kAny: + case kNonSingleton: + return DistributionSpecAny.INSTANCE; + case kHashed: { + List hashExprIds = new ArrayList<>(); + if (tspec.isSetHash_columns()) { + for (TOperator hashCol : tspec.getHash_columns()) { + long uid = getRootScalar(hashCol).getScalar_unique_id().getUnique_id(); + Slot slot = hornCtx.getScalarUidToSlot().get(uid); + if (slot instanceof SlotReference) { + hashExprIds.add(slot.getExprId()); + } + } + } + if (tspec.isSetSpecial_hash_spec() + && tspec.getSpecial_hash_spec().getKind() == TSpecialHashSpecKind.kDoris) { + TDorisHashSpec dorisHashSpec = tspec.getSpecial_hash_spec().getDoris_hash_spec(); + long tableId = dorisHashSpec.getTable_id(); + OlapTable olapTable = (OlapTable) Env.getCurrentInternalCatalog() + .getTableByTableId(tableId); + if (olapTable == null) { + return new DistributionSpecHash(hashExprIds, ShuffleType.EXECUTION_BUCKETED); + } + return new DistributionSpecHash(hashExprIds, ShuffleType.STORAGE_BUCKETED, + tableId, olapTable.getBaseIndexId(), + new LinkedHashSet<>(dorisHashSpec.getSelected_partition_ids())); + } + // 无 special → 纯运行时 hash 重分布,无 tablet 身份 + return new DistributionSpecHash(hashExprIds, ShuffleType.EXECUTION_BUCKETED); + } + default: + return DistributionSpecAny.INSTANCE; + } + } + + /** intersect/except 输出列 nullable 对齐——复刻 Doris {@code LogicalSetOperation.buildNewOutputs} */ + public static List alignSetOpOutputNullable( + List outputs, List> childrenOutputs) { + List aligned = new ArrayList<>(outputs.size()); + for (int i = 0; i < outputs.size(); i++) { + boolean childOrNullable = false; + for (List childCols : childrenOutputs) { + if (i < childCols.size() && childCols.get(i).nullable()) { + childOrNullable = true; + break; + } + } + NamedExpression out = outputs.get(i); + if (out instanceof SlotReference && ((SlotReference) out).nullable() != childOrNullable) { + aligned.add(((SlotReference) out).withNullable(childOrNullable)); + } else { + aligned.add(out); + } + } + return aligned; + } + + /** 给 scan 算子构造 colocate-capable DistributionSpec —— 仿主线 */ + public static DistributionSpec buildScanDistributionSpec( + OlapTable olapTable, List scanSlots, List partitionIds) { + DistributionInfo distInfo = olapTable.getDefaultDistributionInfo(); + ColocateTableIndex colocateIndex = Env.getCurrentColocateIndex(); + boolean isBelongStableCG = colocateIndex.isColocateTable(olapTable.getId()) + && !colocateIndex.isGroupUnstable(colocateIndex.getGroup(olapTable.getId())) + && olapTable.getCatalogId() == Env.getCurrentInternalCatalog().getId(); + boolean isSelectUnpartition = olapTable.getPartitionInfo().getType() == PartitionType.UNPARTITIONED + || partitionIds.size() == 1; + if (!(distInfo instanceof HashDistributionInfo) || (!isBelongStableCG && !isSelectUnpartition)) { + return DistributionSpecStorageAny.INSTANCE; + } + HashDistributionInfo hashDist = (HashDistributionInfo) distInfo; + List hashExprIds = new ArrayList<>(); + for (Column distCol : hashDist.getDistributionColumns()) { + for (Slot slot : scanSlots) { + if (slot instanceof SlotReference + && slot.getName().equalsIgnoreCase(distCol.getName())) { + hashExprIds.add(slot.getExprId()); + break; + } + } + } + return new DistributionSpecHash(hashExprIds, ShuffleType.NATURAL, + olapTable.getId(), olapTable.getBaseIndexId(), + new LinkedHashSet<>(partitionIds)); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/horn/horn2doris/HornTypeToDorisConverter.java b/fe/fe-core/src/main/java/org/apache/doris/horn/horn2doris/HornTypeToDorisConverter.java new file mode 100644 index 00000000000000..1f43b47e124edc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/horn/horn2doris/HornTypeToDorisConverter.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.horn.horn2doris; + +import org.apache.horn4j.thrift.TColumnType; +import org.apache.horn4j.thrift.TPrimitiveType; +import org.apache.horn4j.thrift.TScalarType; +import org.apache.horn4j.thrift.TTypeNode; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.CharType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.DecimalV3Type; +import org.apache.doris.nereids.types.DoubleType; +import org.apache.doris.nereids.types.FloatType; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.NullType; +import org.apache.doris.nereids.types.SmallIntType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TinyIntType; +import org.apache.doris.nereids.types.VarcharType; + +/** Converts Horn/Impala TColumnType (Thrift) back to Doris Nereids DataType */ +public class HornTypeToDorisConverter { + + /** Convert Horn TColumnType to Doris DataType */ + public static DataType convert(TColumnType hornType) { + if (hornType == null || hornType.getTypes() == null || hornType.getTypes().isEmpty()) { + throw new UnsupportedOperationException("Horn: empty TColumnType"); + } + + TTypeNode typeNode = hornType.getTypes().get(0); + if (!typeNode.isSetScalar_type()) { + throw new UnsupportedOperationException( + "Horn: non-scalar TColumnType not supported: " + hornType); + } + + TScalarType scalarType = typeNode.getScalar_type(); + TPrimitiveType primType = scalarType.getType(); + + switch (primType) { + case BOOLEAN: + return BooleanType.INSTANCE; + case TINYINT: + return TinyIntType.INSTANCE; + case SMALLINT: + return SmallIntType.INSTANCE; + case INT: + return IntegerType.INSTANCE; + case BIGINT: + return BigIntType.INSTANCE; + case FLOAT: + return FloatType.INSTANCE; + case DOUBLE: + return DoubleType.INSTANCE; + case DECIMAL: + // 反译统一回 DecimalV3 + return DecimalV3Type.createDecimalV3Type( + scalarType.getPrecision(), scalarType.getScale()); + case CHAR: + return CharType.createCharType(scalarType.getLen()); + case VARCHAR: + return VarcharType.createVarcharType(scalarType.getLen()); + case STRING: + return StringType.INSTANCE; + case DATE: + // 反译统一回 DateV2 + return DateV2Type.INSTANCE; + case NULL_TYPE: + return NullType.INSTANCE; + default: + // 暂不支持 TIMESTAMP + throw new UnsupportedOperationException( + "Horn: unsupported TPrimitiveType: " + primType); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/horn/package-info.java b/fe/fe-core/src/main/java/org/apache/doris/horn/package-info.java new file mode 100644 index 00000000000000..d8250148f79896 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/horn/package-info.java @@ -0,0 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/** Horn CBO Optimizer integration for Apache Doris */ +package org.apache.doris.horn; diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/HornMetric.java b/fe/fe-core/src/main/java/org/apache/doris/metric/HornMetric.java new file mode 100644 index 00000000000000..63dbf7f6c6c601 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/HornMetric.java @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.metric; + +import org.apache.doris.metric.Metric.MetricUnit; + +/** + * Horn CBO 优化器监控指标（对齐 kernel 的 HornMetrics.java）。 + * 独立成类、统一 HORN_PREFIX 前缀，注册到全局 MetricRepo.DORIS_METRIC_REGISTER。 + * init() 由 MetricRepo.init() 调用一次。 + */ +public class HornMetric { + // Horn 指标统一业务前缀（框架还会自动再加 doris_fe_）。 + // 改前缀只改这一处，例如想要 horn_cbo_ 就把值换成 "horn_cbo_"。 + private static final String HORN_PREFIX = "horn_"; + + // 计数器（累计） + public static LongCounterMetric COUNTER_HORN_QUERY; // doris_fe_horn_query_total + public static LongCounterMetric COUNTER_HORN_QUERY_SUPPORT; // ..._query_support_total + public static LongCounterMetric COUNTER_HORN_QUERY_SUCCESS; // ..._query_success_total + public static LongCounterMetric COUNTER_HORN_QUERY_ERROR; // ..._query_error_total + public static LongCounterMetric COUNTER_HORN_QUERY_FALLBACK; // ..._query_fallback_total（算 ratio 用） + // Doris 集成专属：forward/backward 计划翻译失败合并桶（不算 kernel 语义的 error/fallback） + public static LongCounterMetric COUNTER_HORN_QUERY_TRANSLATE_PLAN_ERROR; // ..._query_translate_plan_error_total + + // 量表（反映最近一次查询；并发弱可见性见 porting-plan §2） + public static GaugeMetricImpl GAUGE_HORN_FALLBACK_RATIO; + public static GaugeMetricImpl GAUGE_HORN_OPTIMIZE_LATENCY_NS; + public static GaugeMetricImpl GAUGE_HORN_CASCADE_GROUP_COUNT; + public static GaugeMetricImpl GAUGE_HORN_CASCADE_GROUP_EXPR_COUNT; + public static GaugeMetricImpl GAUGE_HORN_CASCADE_SAFE_TO_PRUNE_RATIO; + public static GaugeMetricImpl GAUGE_HORN_SCHEDULER_JOB_COUNT; + public static GaugeMetricImpl GAUGE_HORN_DPHYPER_CSG_CMP_COUNT; + public static GaugeMetricImpl GAUGE_HORN_DPHYPER_ENUM_SUCCESS_RATIO; + + /** 拼上统一前缀，指标名主体不再各自硬编码 horn_。 */ + private static String name(String suffix) { + return HORN_PREFIX + suffix; + } + + public static void init() { + COUNTER_HORN_QUERY = register(new LongCounterMetric(name("query_total"), + MetricUnit.REQUESTS, "total queries entering horn optimize path")); + COUNTER_HORN_QUERY_SUPPORT = register(new LongCounterMetric(name("query_support_total"), + MetricUnit.REQUESTS, "queries that attempted horn optimization")); + COUNTER_HORN_QUERY_SUCCESS = register(new LongCounterMetric(name("query_success_total"), + MetricUnit.REQUESTS, "queries optimized by horn successfully")); + COUNTER_HORN_QUERY_ERROR = register(new LongCounterMetric(name("query_error_total"), + MetricUnit.REQUESTS, "queries where horn threw an error")); + COUNTER_HORN_QUERY_FALLBACK = register(new LongCounterMetric(name("query_fallback_total"), + MetricUnit.REQUESTS, "queries that fell back from horn to nereids")); + COUNTER_HORN_QUERY_TRANSLATE_PLAN_ERROR = register(new LongCounterMetric( + name("query_translate_plan_error_total"), + MetricUnit.REQUESTS, + "queries where doris<->horn plan translation (forward or backward) failed")); + + // 注意：GaugeMetricImpl 构造函数是 4 参数 (name, unit, description, defaultValue)。 + GAUGE_HORN_FALLBACK_RATIO = register(new GaugeMetricImpl<>(name("fallback_ratio"), + MetricUnit.PERCENT, "fraction of supported queries that fell back to nereids", 0.0)); + GAUGE_HORN_OPTIMIZE_LATENCY_NS = register(new GaugeMetricImpl<>(name("optimize_latency_ns"), + MetricUnit.NANOSECONDS, "horn optimize wall time of last query (ns)", 0L)); + GAUGE_HORN_CASCADE_GROUP_COUNT = register(new GaugeMetricImpl<>(name("cascade_group_count"), + MetricUnit.NOUNIT, "cascades memo group count of last query", 0L)); + GAUGE_HORN_CASCADE_GROUP_EXPR_COUNT = register(new GaugeMetricImpl<>(name("cascade_group_expression_count"), + MetricUnit.NOUNIT, "cascades memo group expression count of last query", 0L)); + GAUGE_HORN_CASCADE_SAFE_TO_PRUNE_RATIO = register(new GaugeMetricImpl<>(name("cascade_safe_to_prune_ratio"), + MetricUnit.PERCENT, "safe-to-prune ratio of last query", 0.0)); + GAUGE_HORN_SCHEDULER_JOB_COUNT = register(new GaugeMetricImpl<>(name("scheduler_job_count"), + MetricUnit.NOUNIT, "optimizer scheduler job count of last query", 0L)); + GAUGE_HORN_DPHYPER_CSG_CMP_COUNT = register(new GaugeMetricImpl<>(name("dphyper_csg_cmp_count"), + MetricUnit.NOUNIT, "dphyper csg-cmp emit count of last query", 0L)); + GAUGE_HORN_DPHYPER_ENUM_SUCCESS_RATIO = register(new GaugeMetricImpl<>( + name("dphyper_enumeration_successful_ratio"), + MetricUnit.PERCENT, "dphyper enumeration successful ratio of last query", 0.0)); + } + + /** 统一注册到全局 registry 并返回自身，便于链式赋值。 */ + private static M register(M metric) { + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(metric); + return metric; + } + + /** fallback ratio = fallback / support，带除零保护；由 MetricCalculator 周期调用。 */ + public static void updateFallbackRatio() { + long support = COUNTER_HORN_QUERY_SUPPORT.getValue(); + long fallback = COUNTER_HORN_QUERY_FALLBACK.getValue(); + GAUGE_HORN_FALLBACK_RATIO.setValue(support == 0 ? 0.0 : (double) fallback / (double) support); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricCalculator.java index d15b3296efd06b..cdec7fdefc22a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricCalculator.java @@ -95,6 +95,9 @@ private void update() { lastMetaServiceRpcCounter = currentMetaServiceCounter; } + // horn fallback ratio = fallback / support + HornMetric.updateFallbackRatio(); + updateCloudMetrics(interval); lastTs = currentTs; diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java index 1d6e982861fbc6..a02ee1bdf4aadb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java @@ -1129,6 +1129,7 @@ public Integer getValue() { // init system metrics initSystemMetrics(); CloudMetrics.init(); + HornMetric.init(); updateMetrics(); isInit = true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 4d3c5cf9c59d1c..448b2f923a4439 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -33,6 +33,9 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; import org.apache.doris.foundation.format.FormatOptions; +import org.apache.doris.horn.HornOptimizer; +import org.apache.doris.metric.HornMetric; +import org.apache.doris.metric.MetricRepo; import org.apache.doris.mysql.FieldInfo; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; @@ -118,6 +121,11 @@ public class NereidsPlanner extends Planner { protected Plan rewrittenPlan; protected Plan optimizedPlan; protected PhysicalPlan physicalPlan; + // Horn CBO result: set by optimize(), consumed by planWithoutLock(). + // null means Horn was not used or failed (fallback to Nereids). + private PhysicalPlan hornPhysicalPlan; + private String hornExplainString; + private String hornFallbackReason; private CascadesContext cascadesContext; private final StatementContext statementContext; @@ -294,7 +302,21 @@ private Plan planWithoutLock( } // rule-based optimize - rewrite(showRewriteProcess(explainLevel, showPlanProcess)); + // Horn's tree IR cannot model a materialized CTE (a DAG: one producer, N consumers). When Horn is + // enabled, force CTE inline so the rewrite phase (CTEInline) desugars every CTE into plain operators + // Horn already supports; otherwise materialized CTEs leave LogicalCTEAnchor nodes that make Horn fall + // back. Restored right after rewrite (only the rewrite phase reads enableCTEMaterialize) so the + // session variable is not polluted. See horn/docs/extended5/06-cte-inline-bypass-design.md. + SessionVariable cteSv = cascadesContext.getConnectContext().getSessionVariable(); + boolean savedCteMaterialize = cteSv.enableCTEMaterialize; + if (cteSv.enableHornOptimizer) { + cteSv.enableCTEMaterialize = false; + } + try { + rewrite(showRewriteProcess(explainLevel, showPlanProcess)); + } finally { + cteSv.enableCTEMaterialize = savedCteMaterialize; + } // try to pre mv rewrite preMaterializedViewRewrite(); if (explainLevel == ExplainLevel.REWRITTEN_PLAN || explainLevel == ExplainLevel.ALL_PLAN) { @@ -305,27 +327,54 @@ private Plan planWithoutLock( } optimize(showPlanProcess); - // print memo before choose plan. - // if chooseNthPlan failed, we could get memo to debug - if (cascadesContext.getConnectContext().getSessionVariable().dumpNereidsMemo) { - Memo memo = cascadesContext.getMemo(); - if (memo != null) { - LOG.info("{}\n{}", ConnectContext.get().getQueryIdentifier(), memo.toString()); - } else { - LOG.info("{}\nMemo is null", ConnectContext.get().getQueryIdentifier()); + + PhysicalPlan physicalPlan; + if (hornPhysicalPlan != null) { + // postProcess（RuntimeFilter / Fragment assignment 等）。 + // Horn does not support lazy materialization yet; disable LazyMaterializeTopN. + SessionVariable sv = cascadesContext.getConnectContext().getSessionVariable(); + int savedThreshold = sv.topNLazyMaterializationThreshold; + sv.topNLazyMaterializationThreshold = -1; + // 关闭 BE 的 LEFT_SEMI_DIRECT_RETURN 优化（docs/extended4/11）。 + // 该 BE 优化在 LEFT_SEMI + 单 eq + 无 other + 单 BE instance IN + // filter ready 时让 probe 跳过 hash 比对直接 forward。其 multi-instance + // 决策不一致：local producer ready 即触发，但 consumer 拿到的是全局 + // merged 后的 filter，可能因部分 instance 超 max_in_num 而 DISABLED + // → probe 未过滤、但走 direct_return（tpch q20：horn LEFT_SEMI(BROADCAST) + // 偶然触发，native 选 RIGHT_SEMI 不触发）。此变量是 BE-side query option， + // 由 SessionVariable.toThrift() 在 CoordinatorContext 创建时传给 BE + // —— postProcess 之后再才发生，故修改必须 sticky 到 plan() 返回之后。 + // 不恢复（sticky 至 session 结束或显式 SET）；不动 plan 形态、不关 RF， + // 仅关一个 hash join 性能优化。根治待 horn cascades 增 LEFT_SEMI↔RIGHT_SEMI + // commute 规则（v9 候选）或 Doris 社区修 BE multi-instance 一致性。 + sv.enableLeftSemiDirectReturnOpt = false; + try { + physicalPlan = postProcess(hornPhysicalPlan); + } finally { + sv.topNLazyMaterializationThreshold = savedThreshold; } + } else { + // Standard Nereids path: extract best plan from Memo. + if (cascadesContext.getConnectContext().getSessionVariable().dumpNereidsMemo) { + Memo memo = cascadesContext.getMemo(); + if (memo != null) { + LOG.info("{}\n{}", ConnectContext.get().getQueryIdentifier(), memo.toString()); + } else { + LOG.info("{}\nMemo is null", ConnectContext.get().getQueryIdentifier()); + } + } + int nth = cascadesContext.getConnectContext().getSessionVariable().getNthOptimizedPlan(); + physicalPlan = chooseNthPlan(getRoot(), requireProperties, nth); + physicalPlan = postProcess(physicalPlan); } - int nth = cascadesContext.getConnectContext().getSessionVariable().getNthOptimizedPlan(); - PhysicalPlan physicalPlan = chooseNthPlan(getRoot(), requireProperties, nth); - - physicalPlan = postProcess(physicalPlan); if (cascadesContext.getConnectContext().getSessionVariable().dumpNereidsMemo) { String tree = physicalPlan.treeString(); LOG.info("{}\n{}", ConnectContext.get().getQueryIdentifier(), tree); } if (explainLevel == ExplainLevel.OPTIMIZED_PLAN || explainLevel == ExplainLevel.ALL_PLAN - || explainLevel == ExplainLevel.SHAPE_PLAN) { + || explainLevel == ExplainLevel.SHAPE_PLAN + || explainLevel == ExplainLevel.HORN_PLAN) { optimizedPlan = physicalPlan; } // serialize optimized plan to dumpfile, dumpfile do not have this part means optimize failed @@ -559,9 +608,77 @@ protected void optimize(boolean showPlanProcess) { if (LOG.isDebugEnabled()) { LOG.debug("Start optimize plan"); } - keepOrShowPlanProcess(showPlanProcess, () -> { - new Optimizer(cascadesContext).execute(); - }); + + // Horn CBO optimizer: try Horn first, fall back to Nereids on any failure. + boolean useHorn = cascadesContext.getConnectContext().getSessionVariable().enableHornOptimizer; + if (useHorn) { + if (MetricRepo.isInit) { + HornMetric.COUNTER_HORN_QUERY.increase(1L); // #1 进入 Horn 判断点 + HornMetric.COUNTER_HORN_QUERY_SUPPORT.increase(1L); // #2 support + } + long hornStart = System.currentTimeMillis(); + try { + HornOptimizer hornOptimizer = new HornOptimizer(cascadesContext); + Plan rewrittenPlan = cascadesContext.getRewritePlan(); + if (rewrittenPlan instanceof LogicalPlan) { + hornPhysicalPlan = hornOptimizer.optimize((LogicalPlan) rewrittenPlan); + } + long hornElapsed = System.currentTimeMillis() - hornStart; + if (hornPhysicalPlan != null) { + hornExplainString = hornOptimizer.getHornExplainString(); + if (MetricRepo.isInit) { + HornMetric.COUNTER_HORN_QUERY_SUCCESS.increase(1L); // #3 success + } + LOG.info("Horn CBO: succeeded in {} ms", hornElapsed); + } else { + if (MetricRepo.isInit) { + // 按 HornOptimizer 的失败分类分流三桶： + // HORN_NOT_HANDLE → fallback（对齐 kernel 预期回退） + // HORN_FORWARD/BACKWARD_ERROR → translate_plan_error（Doris 翻译层缺口） + // HORN_OPTIMIZE_ERROR/UNKNOWN → error（Horn 真错误 / 兜底） + switch (hornOptimizer.getFailureKind()) { + case HORN_NOT_HANDLE: + HornMetric.COUNTER_HORN_QUERY_FALLBACK.increase(1L); + break; + case HORN_FORWARD_ERROR: + case HORN_BACKWARD_ERROR: + HornMetric.COUNTER_HORN_QUERY_TRANSLATE_PLAN_ERROR.increase(1L); + break; + case HORN_OPTIMIZE_ERROR: + case UNKNOWN: + default: + HornMetric.COUNTER_HORN_QUERY_ERROR.increase(1L); + break; + } + } + String reason = hornOptimizer.getFallbackReason(); + hornFallbackReason = (reason != null ? reason : "returned null") + + " (" + hornElapsed + " ms)"; + // 即使反译失败（如 Doris ChildOutputPropertyDeriver 抛错），horn C++ 侧已产出计划文本 + // （HornOptimizer 在 build() 之前就设了 explain string）。捕获它，让 EXPLAIN HORN PLAN + // 能打印「horn 生成了什么 + Doris 为何拒绝」，而非只显示 fallback 原因。 + hornExplainString = hornOptimizer.getHornExplainString(); + LOG.info("Horn CBO: fallback to Nereids: {}", hornFallbackReason); + } + } catch (Exception e) { + if (MetricRepo.isInit) { + // HornOptimizer 内部已分段 catch 干净、正常不再抛异常到这里； + // 此处是防御性保险丝：未预料的异常逃逸一律记 error。 + HornMetric.COUNTER_HORN_QUERY_ERROR.increase(1L); // #4 error + } + long hornElapsed = System.currentTimeMillis() - hornStart; + hornFallbackReason = e.getMessage() + " (" + hornElapsed + " ms)"; + LOG.warn("Horn CBO: fallback to Nereids: {}", hornFallbackReason); + hornPhysicalPlan = null; + } + } + + // If Horn did not produce a usable plan, run the standard Nereids CBO. + if (hornPhysicalPlan == null) { + keepOrShowPlanProcess(showPlanProcess, () -> { + new Optimizer(cascadesContext).execute(); + }); + } NereidsTracer.logImportantTime("EndOptimizePlan"); if (LOG.isDebugEnabled()) { LOG.debug("End optimize plan"); @@ -898,6 +1015,27 @@ public String getExplainString(ExplainOptions explainOptions) { case OPTIMIZED_PLAN: plan = "cost = " + cost + "\n" + optimizedPlan.treeString() + mvSummary; break; + case HORN_PLAN: + if (hornExplainString != null && !hornExplainString.isEmpty()) { + plan = hornExplainString; + // 若 horn 出了计划但 Doris 反译失败：既打印 horn 计划，也附上失败原因，便于定位 + // 「horn 生成了什么、Doris 在哪一步拒绝」。执行仍走 Nereids fallback。多行排版便于阅读。 + if (hornFallbackReason != null) { + plan += "\n\n" + + "==================== HORN → DORIS BACKWARD FAILED ====================\n" + + "Horn produced the plan above, but Doris backward translation failed.\n" + + "Execution fell back to Nereids.\n" + + "Reason:\n" + + " " + hornFallbackReason.replace(". ", ".\n ") + "\n" + + "====================================================================="; + } + } else if (hornFallbackReason != null) { + plan = "Horn CBO fallback to Nereids: " + hornFallbackReason; + } else { + plan = "Horn CBO was not used for this query. " + + "Check enable_horn_optimizer=true."; + } + break; case SHAPE_PLAN: plan = optimizedPlan.shape(""); break; @@ -955,7 +1093,10 @@ public String getExplainString(ExplainOptions explainOptions) { default: plan = super.getExplainString(explainOptions); plan += mvSummary; - plan += "\n\n\n========== STATISTICS ==========\n"; + if (hornFallbackReason != null) { + plan += "\n\nHorn CBO fallback to Nereids: " + hornFallbackReason + "\n"; + } + plan += "\n\n========== STATISTICS ==========\n"; if (statementContext != null) { if (statementContext.isHasUnknownColStats()) { plan += "planned with unknown column statistics\n"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 1755ee061b52ea..51cfbfefe870f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -2611,13 +2611,41 @@ public PlanFragment visitPhysicalQuickSort(PhysicalQuickSort sor // It means the local has satisfied the Gather property. We can just ignore mergeSort return inputFragment; } - SortNode sortNode = (SortNode) inputFragment.getPlanRoot().getChild(0); - ((ExchangeNode) inputFragment.getPlanRoot()).setMergeInfo(sortNode.getSortInfo()); + ExchangeNode exchangeNode = (ExchangeNode) inputFragment.getPlanRoot(); + // 主路径 (Nereids 标准 ORDER BY plan)：Exchange child 是 LOCAL_SORT， + // mergeInfo 从 child SortNode 取，同时标记 mergeByExchange。 + // 兜底路径 (horn sort_elimination)：child 已天然有序，反译时 Exchange + // 下没有 LOCAL_SORT 占位；这时直接从 PhysicalQuickSort 自身 orderKeys + // 构造 SortInfo。注意 sortTuple 必须复用 child 已注册的 tuple + // (exchangeNode 的 input tuple)，不能 generateTupleDesc 新建——因为没 + // SortNode 挂进 fragment，新 tuple 不会进 fragment.tupleIds，BE 找不到。 + if (exchangeNode.getChild(0) instanceof SortNode) { + SortNode childSort = (SortNode) exchangeNode.getChild(0); + exchangeNode.setMergeInfo(childSort.getSortInfo()); + childSort.setMergeByExchange(); + // distributeExprLists 注意必须设到下游 SortNode（不是 exchange 本身）— + // BE 端 mergeInfo 路径上 distribute 信息要从 SortNode 取 + childSort.setChildrenDistributeExprLists(distributeExprLists); + } else { + TupleDescriptor childTuple = context.getDescTable() + .getTupleDesc(exchangeNode.getTupleIds().get(0)); + List orderingExprs = Lists.newArrayList(); + List ascOrders = Lists.newArrayList(); + List nullsFirstParams = Lists.newArrayList(); + sort.getOrderKeys().forEach(k -> { + orderingExprs.add(ExpressionTranslator.translate(k.getExpr(), context)); + ascOrders.add(k.isAsc()); + nullsFirstParams.add(k.isNullFirst()); + }); + exchangeNode.setMergeInfo( + new SortInfo(orderingExprs, ascOrders, nullsFirstParams, childTuple)); + // 兜底路径 child 不是 SortNode，没有 sortNode 持 distributeExprLists； + // exchangeNode 自己持作 fallback (无 child sortNode 时的安放点)。 + exchangeNode.setChildrenDistributeExprLists(distributeExprLists); + } if (inputFragment.hasChild(0) && inputFragment.getChild(0).getSink() != null) { inputFragment.getChild(0).getSink().setMerge(true); } - sortNode.setMergeByExchange(); - sortNode.setChildrenDistributeExprLists(distributeExprLists); } return inputFragment; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index f44fe31afde594..2b71ccc5d467ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -5192,6 +5192,9 @@ private ExplainLevel parseExplainPlanType(PlanTypeContext planTypeContext) { if (planTypeContext.DISTRIBUTED() != null) { return ExplainLevel.DISTRIBUTED_PLAN; } + if (planTypeContext.HORN() != null) { + return ExplainLevel.HORN_PLAN; + } return ExplainLevel.ALL_PLAN; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index 37c7f9351908f7..1ab6730c74d982 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -90,6 +90,7 @@ public class NereidsParser { EXPLAIN_TOKENS.set(DorisLexer.OPTIMIZED); EXPLAIN_TOKENS.set(DorisLexer.PLAN); EXPLAIN_TOKENS.set(DorisLexer.PROCESS); + EXPLAIN_TOKENS.set(DorisLexer.HORN); ImmutableSet.Builder nonReserveds = ImmutableSet.builder(); for (Method declaredMethod : NonReservedContext.class.getDeclaredMethods()) { @@ -178,6 +179,9 @@ public static Optional> tryParseExplainPlan(String } else if (tokenType == DorisLexer.PHYSICAL || tokenType == DorisLexer.OPTIMIZED) { explainLevel = ExplainLevel.OPTIMIZED_PLAN; token = readUntilNonComment(tokenSource); + } else if (tokenType == DorisLexer.HORN) { + explainLevel = ExplainLevel.HORN_PLAN; + token = readUntilNonComment(tokenSource); } if (token == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalWindowToPhysicalWindow.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalWindowToPhysicalWindow.java index 2581a87cdfc421..6499812911c824 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalWindowToPhysicalWindow.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalWindowToPhysicalWindow.java @@ -142,8 +142,11 @@ private PhysicalWindow createPhysicalWindow(Plan root, WindowFrameGroup wi * WindowFunctionRelatedGroups * ******************************************************************************************** */ - // todo: can we simplify the following three algorithms? - private List createWindowFrameGroups(List windowList) { + /** + * public static: 纯局部逻辑（不依赖实例状态）。暴露给 horn 正向反译（doris2horn）复用同一套 + * (partition, order, frame) 分组，避免在 horn 侧重复实现——多 spec window 拆成单 spec 链时用。 + */ + public static List createWindowFrameGroups(List windowList) { List windowFrameGroupList = Lists.newArrayList(); for (int i = 0; i < windowList.size(); i++) { NamedExpression windowAlias = windowList.get(i); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java index e2194048e4859e..0abc25b0fb7b23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java @@ -56,7 +56,8 @@ public enum ExplainLevel { SHAPE_PLAN(true), MEMO_PLAN(true), DISTRIBUTED_PLAN(true), - ALL_PLAN(true) + ALL_PLAN(true), + HORN_PLAN(true) ; public final boolean isPlanLevel; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index df5ac1ec75b9d5..e814f6af25177b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -408,6 +408,9 @@ public String toString() { public static final String ENABLE_DPHYP_OPTIMIZER = "enable_dphyp_optimizer"; public static final String DPHYPER_LIMIT = "dphyper_limit"; + public static final String ENABLE_HORN_OPTIMIZER = "enable_horn_optimizer"; + public static final String ENABLE_HORN_DETAIL_INFO = "enable_horn_detail_info"; + public static final String ENABLE_HORN_PROFILE = "enable_horn_profile"; public static final String ENABLE_LEFT_ZIG_ZAG = "enable_left_zig_zag"; public static final String ENABLE_HBO_OPTIMIZATION = "enable_hbo_optimization"; public static final String ENABLE_HBO_INFO_COLLECTION = "enable_hbo_info_collection"; @@ -1959,6 +1962,21 @@ public void setMaxJoinNumberOfReorder(int maxJoinNumberOfReorder) { @VariableMgr.VarAttr(name = ENABLE_DPHYP_OPTIMIZER) public boolean enableDPHypOptimizer = false; + @VariableMgr.VarAttr(name = ENABLE_HORN_OPTIMIZER, description = { + "启用 Horn CBO 优化器替代 Nereids 内置优化器进行查询优化", + "Enable Horn CBO optimizer to replace the built-in Nereids optimizer for query optimization"}) + public boolean enableHornOptimizer = false; + + @VariableMgr.VarAttr(name = ENABLE_HORN_DETAIL_INFO, description = { + "启用 Horn 优化器详细日志（cascade log、detail log、stats log）", + "Enable Horn optimizer detailed logging (cascade log, detail log, stats log)"}) + public boolean enableHornDetailInfo = false; + + @VariableMgr.VarAttr(name = ENABLE_HORN_PROFILE, description = { + "EXPLAIN HORN PLAN 输出附带 cascades 各阶段耗时 timing 列表", + "Append cascades stage timing (Total Time + per-stage) to EXPLAIN HORN PLAN output"}) + public boolean enableHornProfile = false; + @VariableMgr.VarAttr(name = SHORT_CIRCUIT_EVALUATION, fuzzy = true, description = { "是否启用短路求值", "Whether to enable short-circuit evaluation" }) public boolean shortCircuitEvaluation = false; diff --git a/fe/horn4j/pom.xml b/fe/horn4j/pom.xml new file mode 100644 index 00000000000000..c2b0e4d27103a3 --- /dev/null +++ b/fe/horn4j/pom.xml @@ -0,0 +1,82 @@ + + + + 4.0.0 + + org.apache.doris + fe + ${revision} + ../pom.xml + + org.apache.horn4j + horn4j + 1.0.0-SNAPSHOT + jar + + + ${project.basedir}/../custom_libs/horn4j-1.0.0-SNAPSHOT.jar.parts + ${project.build.directory}/horn4j-1.0.0-SNAPSHOT.jar + fc1ab527c057197f738fe763a6dd2e12d05815051b4adfe35766fac32de1ea40 + + + + + org.apache.thrift + libthrift + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + unpack-vendored-horn4j + generate-resources + + run + + + + + + + + + + + + + + + + + + + + + diff --git a/fe/pom.xml b/fe/pom.xml index 0f6b3cd7fef538..5b5f722445d749 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -218,6 +218,7 @@ under the License. fe-foundation fe-common + horn4j fe-core hive-udf be-java-extensions