From 75d266e582ad9161b313935abbab12ae65b86197 Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 12 Nov 2025 08:50:16 +0800 Subject: [PATCH 01/20] simple Leader & Follower hint # Conflicts: # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java --- .../queryengine/common/MPPQueryContext.java | 11 ++ .../plan/AbstractFragmentParallelPlanner.java | 5 + .../analyzer/StatementAnalyzer.java | 14 +++ .../sql/ast/QuerySpecification.java | 27 ++++- .../plan/relational/sql/ast/SelectHint.java | 93 +++++++++++++++ .../relational/sql/parser/AstBuilder.java | 65 ++++++++++- .../plan/relational/sql/util/QueryUtil.java | 7 +- .../relational/utils/hint/FollowerHint.java | 30 +++++ .../plan/relational/utils/hint/Hint.java | 45 ++++++++ .../relational/utils/hint/HintDefinition.java | 107 ++++++++++++++++++ .../relational/utils/hint/LeaderHint.java | 30 +++++ .../relational/grammar/sql/RelationalSql.g4 | 19 +++- 12 files changed, 444 insertions(+), 9 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index fac3afff8b0b0..bdeae41060324 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -37,6 +37,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.statistics.QueryPlanStatistics; import org.apache.iotdb.db.utils.cte.CteDataStore; @@ -125,6 +126,8 @@ public enum ExplainType { private boolean userQuery = false; + private Map hintMap = new HashMap<>(); + private Map, Query> cteQueries = new HashMap<>(); // Stores the EXPLAIN/EXPLAIN ANALYZE results for Common Table Expressions (CTEs) @@ -183,6 +186,14 @@ public MPPQueryContext( this.initResultNodeContext(); } + public void setHintMap(Map hintMap) { + this.hintMap = hintMap; + } + + public Map getHintMap() { + return hintMap; + } + public void setReserveMemoryForSchemaTreeFunc(LongConsumer reserveMemoryForSchemaTreeFunc) { this.reserveMemoryForSchemaTreeFunc = reserveMemoryForSchemaTreeFunc; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java index d7415320e048a..e4b6b12671083 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java @@ -118,6 +118,11 @@ protected TDataNodeLocation selectTargetDataNode(TRegionReplicaSet regionReplica } boolean selectRandomDataNode = ReadConsistencyLevel.WEAK == this.readConsistencyLevel; + if (ReadConsistencyLevel.STRONG == this.readConsistencyLevel + && queryContext.getHintMap().containsKey("Follower")) { + selectRandomDataNode = true; + } + // When planning fragment onto specific DataNode, the DataNode whose endPoint is in // black list won't be considered because it may have connection issue now. List availableDataNodes = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index bc6d54c37e7fa..133f9b677360a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -146,6 +146,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Select; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SelectHint; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SelectItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetOperation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; @@ -193,6 +194,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedInsertStatement; import org.apache.iotdb.db.queryengine.plan.relational.type.CompatibleResolver; import org.apache.iotdb.db.queryengine.plan.relational.type.TypeManager; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -1153,6 +1155,10 @@ protected Scope visitQuerySpecification(QuerySpecification node, Optional node.getWhere().ifPresent(where -> analyzeWhere(node, sourceScope, where)); List outputExpressions = analyzeSelect(node, sourceScope); + + Map hintMap = analyzeHint(node); + queryContext.setHintMap(hintMap); + Analysis.GroupingSetAnalysis groupByAnalysis = analyzeGroupBy(node, sourceScope, outputExpressions); analyzeHaving(node, sourceScope); @@ -1491,6 +1497,14 @@ private void analyzeWhere(Node node, Scope scope, Expression predicate) { analysis.setWhere(node, predicate); } + private Map analyzeHint(QuerySpecification node) { + Optional selectHint = node.getHintMap(); + if (selectHint.isPresent()) { + return selectHint.get().getHintMap(); + } + return ImmutableMap.of(); + } + private List analyzeSelect(QuerySpecification node, Scope scope) { ImmutableList.Builder outputExpressionBuilder = ImmutableList.builder(); ImmutableList.Builder selectExpressionBuilder = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java index 5de61a2dde2e2..3edf33320d37b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java @@ -45,6 +45,8 @@ public class QuerySpecification extends QueryBody { private final Optional offset; private final Optional limit; + private final Optional selectHint; + public QuerySpecification( Select select, Optional from, @@ -55,8 +57,21 @@ public QuerySpecification( List windows, Optional orderBy, Optional offset, - Optional limit) { - this(null, select, from, where, groupBy, having, fill, windows, orderBy, offset, limit); + Optional limit, + Optional selectHint) { + this( + null, + select, + from, + where, + groupBy, + having, + fill, + windows, + orderBy, + offset, + limit, + selectHint); } public QuerySpecification( @@ -70,7 +85,8 @@ public QuerySpecification( List windows, Optional orderBy, Optional offset, - Optional limit) { + Optional limit, + Optional selectHint) { super(location); this.select = requireNonNull(select, "select is null"); @@ -83,6 +99,7 @@ public QuerySpecification( this.orderBy = requireNonNull(orderBy, "orderBy is null"); this.offset = requireNonNull(offset, "offset is null"); this.limit = requireNonNull(limit, "limit is null"); + this.selectHint = requireNonNull(selectHint, "hintMap is null"); } public Select getSelect() { @@ -125,6 +142,10 @@ public Optional getLimit() { return limit; } + public Optional getHintMap() { + return selectHint; + } + @Override public R accept(AstVisitor visitor, C context) { return visitor.visitQuerySpecification(this, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java new file mode 100644 index 0000000000000..6d9ee4b6ab1f3 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java @@ -0,0 +1,93 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; + +import com.google.common.collect.ImmutableList; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class SelectHint extends Node { + Map hintMap; + + public SelectHint() { + super(null); + this.hintMap = new HashMap<>(); + } + + public SelectHint(Map hintMap) { + super(null); + this.hintMap = hintMap; + } + + public Map getHintMap() { + return hintMap; + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public int hashCode() { + return Objects.hash(hintMap); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + SelectHint other = (SelectHint) obj; + return Objects.equals(this.hintMap, other.hintMap); + } + + @Override + public String toString() { + if (hintMap == null || hintMap.isEmpty()) { + return ""; + } + + StringBuilder sb = new StringBuilder(); + sb.append("/*+ "); + + boolean first = true; + for (Map.Entry entry : hintMap.entrySet()) { + if (!first) { + sb.append(" "); + } + sb.append(entry.getValue().toString()); + first = false; + } + + sb.append(" */"); + return sb.toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 83105468a0204..1ac635a4c9206 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -171,6 +171,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowPattern; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Select; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SelectHint; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SelectItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetColumnComment; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetConfiguration; @@ -248,6 +249,10 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ZeroOrOneQuantifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.util.AstUtil; import org.apache.iotdb.db.queryengine.plan.relational.type.AuthorRType; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.FollowerHint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.HintDefinition; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeaderHint; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; @@ -267,6 +272,8 @@ import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.tree.ParseTree; @@ -2234,7 +2241,8 @@ public Node visitQueryNoWith(RelationalSqlParser.QueryNoWithContext ctx) { query.getWindows(), orderBy, offset, - limit), + limit, + query.getHintMap()), Optional.empty(), Optional.empty(), Optional.empty(), @@ -2350,6 +2358,12 @@ public Node visitQuerySpecification(RelationalSqlParser.QuerySpecificationContex from = Optional.of(relation); } + // Hint Map + Optional selectHint = + ctx.selectHint() != null + ? Optional.of((SelectHint) visitSelectHint(ctx.selectHint())) + : Optional.empty(); + return new QuerySpecification( getLocation(ctx), new Select(getLocation(ctx.SELECT()), isDistinct(ctx.setQuantifier()), selectItems), @@ -2361,7 +2375,8 @@ public Node visitQuerySpecification(RelationalSqlParser.QuerySpecificationContex visit(ctx.windowDefinition(), WindowDefinition.class), Optional.empty(), Optional.empty(), - Optional.empty()); + Optional.empty(), + selectHint); } @Override @@ -2390,6 +2405,52 @@ public Node visitSelectAll(RelationalSqlParser.SelectAllContext ctx) { } } + @Override + public Node visitSelectHint(RelationalSqlParser.SelectHintContext ctx) { + Map hintMap = new HashMap<>(); + for (RelationalSqlParser.HintItemContext hiCtx : ctx.hintItem()) { + if (hiCtx instanceof RelationalSqlParser.ParameterizedHintContext) { + // RelationalSqlParser.ParameterizedHintContext paramHint = + // (RelationalSqlParser.ParameterizedHintContext) hiCtx; + // List identifiers = paramHint.identifier(); + // String hintName = identifiers.get(0).getText(); + // List params = new ArrayList<>(); + // for (int i = 1; i < identifiers.size(); i++) { + // params.add(identifiers.get(i).getText()); + // } + // Hint hint = new LeadingHint(params); + // hintMap.put(hintName, hint); + } else if (hiCtx instanceof RelationalSqlParser.SimpleHintContext) { + RelationalSqlParser.SimpleHintContext simpleHint = + (RelationalSqlParser.SimpleHintContext) hiCtx; + String hintName = simpleHint.identifier().getText(); + addSimpleHint(hintName.toUpperCase(), hintMap); + } + } + + return new SelectHint(hintMap); + } + + private static final Map HINT_DEFINITIONS = + ImmutableMap.of( + "LEADER", + new HintDefinition(LeaderHint.hintName, LeaderHint::new, ImmutableSet.of("Follower")), + "FOLLOWER", + new HintDefinition( + FollowerHint.hintName, FollowerHint::new, ImmutableSet.of("Leader")) + // "MERGE_JOIN", new HintDefinition("MergeJoin", MergeJoinHint::new, + // ImmutableSet.of("NestLoopJoin", "HashJoin")), + // "NL_JOIN", new HintDefinition("NestLoopJoin", NestLoopJoinHint::new, + // ImmutableSet.of("MergeJoin", "HashJoin")) + ); + + private void addSimpleHint(String hintName, Map hintMap) { + HintDefinition definition = HINT_DEFINITIONS.get(hintName); + if (definition != null) { + definition.addTo(hintMap); + } + } + @Override public Node visitGroupBy(RelationalSqlParser.GroupByContext ctx) { return new GroupBy( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/QueryUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/QueryUtil.java index f0bd840453e2e..c04b0df5ac630 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/QueryUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/QueryUtil.java @@ -160,6 +160,7 @@ public static Query simpleQuery(Select select) { ImmutableList.of(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty())); } @@ -214,7 +215,8 @@ public static Query simpleQuery( ImmutableList.of(), orderBy, offset, - limit)); + limit, + Optional.empty())); } public static Query simpleQuery( @@ -239,7 +241,8 @@ public static Query simpleQuery( windows, orderBy, offset, - limit)); + limit, + Optional.empty())); } public static Query query(QueryBody body) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java new file mode 100644 index 0000000000000..125e801176a4d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java @@ -0,0 +1,30 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +public class FollowerHint extends Hint { + public static String hintName = "Follower"; + + public FollowerHint() { + super(hintName); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java new file mode 100644 index 0000000000000..8bc6903320524 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java @@ -0,0 +1,45 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +import java.util.Objects; + +public abstract class Hint { + protected String hintName; + + protected Hint(String hintName) { + this.hintName = Objects.requireNonNull(hintName, "hintName can not be null"); + } + + protected void setHintName(String hintName) { + this.hintName = hintName; + } + + protected String getHintName() { + return hintName; + } + + @Override + public String toString() { + return hintName; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java new file mode 100644 index 0000000000000..ac0bd98f7584f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java @@ -0,0 +1,107 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +/** + * Defines the properties and creation logic for a SQL hint. This class encapsulates the hint's key, + * creation strategy, and which other hints it is mutually exclusive with. + */ +public final class HintDefinition { + + private final String key; + private final Supplier supplier; + private final Set mutuallyExclusive; + + /** + * Creates a new hint definition. + * + * @param key the key to use when storing the hint in the hint map + * @param supplier factory method to create the hint instance, receives the key as parameter + * @param mutuallyExclusive set of hint keys that are mutually exclusive with this hint + */ + public HintDefinition(String key, Supplier supplier, Set mutuallyExclusive) { + this.key = key; + this.supplier = supplier; + this.mutuallyExclusive = mutuallyExclusive; + } + + /** + * Gets the hint key used in the hint map. + * + * @return the hint key + */ + public String getKey() { + return key; + } + + /** + * Gets the supplier that creates the hint instance. + * + * @return the hint supplier + */ + public Supplier getSupplier() { + return supplier; + } + + /** + * Gets the set of hint keys that are mutually exclusive with this hint. + * + * @return the set of mutually exclusive hint keys + */ + public Set getMutuallyExclusive() { + return mutuallyExclusive; + } + + /** + * Creates the hint instance. + * + * @return the created hint + */ + public Hint createHint() { + return supplier.get(); + } + + /** + * Checks if this hint is mutually exclusive with the given hint map. + * + * @param hintMap the current hint map to check against + * @return true if this hint can be added (no conflicts), false otherwise + */ + public boolean canAddTo(Map hintMap) { + return mutuallyExclusive.stream().noneMatch(hintMap::containsKey); + } + + /** + * Adds this hint to the hint map if no mutually exclusive hints exist. + * + * @param hintMap the hint map to add to + */ + public void addTo(Map hintMap) { + if (canAddTo(hintMap)) { + hintMap.put(key, createHint()); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java new file mode 100644 index 0000000000000..b9c7f7fd82ec5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java @@ -0,0 +1,30 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +public class LeaderHint extends Hint { + public static String hintName = "Leader"; + + public LeaderHint() { + super(hintName); + } +} diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 07186fd422219..577ac1e739eff 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -1004,7 +1004,7 @@ sortItem ; querySpecification - : SELECT setQuantifier? selectItem (',' selectItem)* + : SELECT selectHint? setQuantifier? selectItem (',' selectItem)* (FROM relation (',' relation)*)? (WHERE where=booleanExpression)? (GROUP BY groupBy)? @@ -1085,6 +1085,15 @@ joinCriteria | USING '(' identifier (',' identifier)* ')' ; +selectHint + : HINT_START hintItem (',' hintItem)* HINT_END + ; + +hintItem + : identifier '(' ( identifier (',' identifier)* )? ')' #parameterizedHint + | identifier #simpleHint + ; + patternRecognition : aliasedRelation ( MATCH_RECOGNIZE '(' @@ -1477,6 +1486,7 @@ nonReserved | WEEK | WHILE | WINDOW | WITHIN | WITHOUT | WORK | WRAPPER | WRITE | YEAR | ZONE + | HINT_START | HINT_END ; ABSENT: 'ABSENT'; @@ -1902,6 +1912,8 @@ CONCAT: '||'; QUESTION_MARK: '?'; SEMICOLON: ';'; +HINT_START: '/*+'; +HINT_END: '*/'; STRING : '\'' ( ~'\'' | '\'\'' )* '\'' @@ -1998,9 +2010,12 @@ SIMPLE_COMMENT ; BRACKETED_COMMENT - : '/*' .*? '*/' -> channel(HIDDEN) + : '/*' (~[+] .*?)? '*/' -> channel(HIDDEN) ; +EMPTY_HINT + : '/*+' [ \r\n\t]* '*/' -> channel(HIDDEN) + ; WS : [ \r\n\t]+ -> channel(HIDDEN) ; From be85803792e31542eb2f58f84c544b377176f15d Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 12 Nov 2025 16:29:18 +0800 Subject: [PATCH 02/20] Leader & Follower hint improvement --- .../plan/AbstractFragmentParallelPlanner.java | 8 ++-- .../relational/sql/parser/AstBuilder.java | 41 +++++++++++++------ .../relational/utils/hint/FollowerHint.java | 16 +++++++- .../plan/relational/utils/hint/Hint.java | 8 +--- .../relational/utils/hint/HintDefinition.java | 37 +++++------------ .../relational/utils/hint/LeaderHint.java | 16 +++++++- .../relational/grammar/sql/RelationalSql.g4 | 2 +- 7 files changed, 76 insertions(+), 52 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java index e4b6b12671083..caf8128009e1e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java @@ -118,10 +118,10 @@ protected TDataNodeLocation selectTargetDataNode(TRegionReplicaSet regionReplica } boolean selectRandomDataNode = ReadConsistencyLevel.WEAK == this.readConsistencyLevel; - if (ReadConsistencyLevel.STRONG == this.readConsistencyLevel - && queryContext.getHintMap().containsKey("Follower")) { - selectRandomDataNode = true; - } + // if (ReadConsistencyLevel.STRONG == this.readConsistencyLevel + // && queryContext.getHintMap().containsKey("Follower")) { + // selectRandomDataNode = true; + // } // When planning fragment onto specific DataNode, the DataNode whose endPoint is in // black list won't be considered because it may have connection issue now. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 1ac635a4c9206..36fb12aaa5743 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -2410,16 +2410,16 @@ public Node visitSelectHint(RelationalSqlParser.SelectHintContext ctx) { Map hintMap = new HashMap<>(); for (RelationalSqlParser.HintItemContext hiCtx : ctx.hintItem()) { if (hiCtx instanceof RelationalSqlParser.ParameterizedHintContext) { - // RelationalSqlParser.ParameterizedHintContext paramHint = - // (RelationalSqlParser.ParameterizedHintContext) hiCtx; - // List identifiers = paramHint.identifier(); - // String hintName = identifiers.get(0).getText(); - // List params = new ArrayList<>(); - // for (int i = 1; i < identifiers.size(); i++) { - // params.add(identifiers.get(i).getText()); - // } - // Hint hint = new LeadingHint(params); - // hintMap.put(hintName, hint); + RelationalSqlParser.ParameterizedHintContext paramHint = + (RelationalSqlParser.ParameterizedHintContext) hiCtx; + List identifiers = paramHint.identifier(); + String hintName = identifiers.get(0).getText(); + String[] params = + identifiers.stream() + .skip(1) + .map(RelationalSqlParser.IdentifierContext::getText) + .toArray(String[]::new); + addParamHint(hintName.toUpperCase(), params, hintMap); } else if (hiCtx instanceof RelationalSqlParser.SimpleHintContext) { RelationalSqlParser.SimpleHintContext simpleHint = (RelationalSqlParser.SimpleHintContext) hiCtx; @@ -2434,20 +2434,35 @@ public Node visitSelectHint(RelationalSqlParser.SelectHintContext ctx) { private static final Map HINT_DEFINITIONS = ImmutableMap.of( "LEADER", - new HintDefinition(LeaderHint.hintName, LeaderHint::new, ImmutableSet.of("Follower")), + new HintDefinition( + LeaderHint.hintName, LeaderHint::new, ImmutableSet.of(FollowerHint.hintName)), "FOLLOWER", new HintDefinition( - FollowerHint.hintName, FollowerHint::new, ImmutableSet.of("Leader")) + FollowerHint.hintName, FollowerHint::new, ImmutableSet.of(LeaderHint.hintName)) // "MERGE_JOIN", new HintDefinition("MergeJoin", MergeJoinHint::new, // ImmutableSet.of("NestLoopJoin", "HashJoin")), // "NL_JOIN", new HintDefinition("NestLoopJoin", NestLoopJoinHint::new, // ImmutableSet.of("MergeJoin", "HashJoin")) ); + private void addParamHint(String hintName, String[] params, Map hintMap) { + HintDefinition definition = HINT_DEFINITIONS.get(hintName); + if (definition != null) { + Hint hint = definition.createHint(params); + String hintKey = hint.toString(); + if (!hintMap.containsKey(hintKey)) { + hintMap.put(hintKey, hint); + } + } + } + private void addSimpleHint(String hintName, Map hintMap) { HintDefinition definition = HINT_DEFINITIONS.get(hintName); if (definition != null) { - definition.addTo(hintMap); + Hint hint = definition.createHint(); + if (definition.canAddTo(hintMap)) { + hintMap.put(hint.toString(), hint); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java index 125e801176a4d..3217691a5e8e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java @@ -23,8 +23,22 @@ public class FollowerHint extends Hint { public static String hintName = "Follower"; + private String targetTable = null; - public FollowerHint() { + public FollowerHint(String... tables) { super(hintName); + if (tables.length > 0) { + this.targetTable = tables[0]; + } + } + + @Override + public boolean appliesToAll() { + return targetTable == null; + } + + @Override + public String toString() { + return targetTable == null ? hintName : hintName + "-" + targetTable; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java index 8bc6903320524..5575916cc7c40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java @@ -30,12 +30,8 @@ protected Hint(String hintName) { this.hintName = Objects.requireNonNull(hintName, "hintName can not be null"); } - protected void setHintName(String hintName) { - this.hintName = hintName; - } - - protected String getHintName() { - return hintName; + public boolean appliesToAll() { + return true; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java index ac0bd98f7584f..3f8e62a363c5d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java @@ -23,7 +23,7 @@ import java.util.Map; import java.util.Set; -import java.util.function.Supplier; +import java.util.function.Function; /** * Defines the properties and creation logic for a SQL hint. This class encapsulates the hint's key, @@ -32,19 +32,20 @@ public final class HintDefinition { private final String key; - private final Supplier supplier; + private final Function hintFactory; private final Set mutuallyExclusive; /** * Creates a new hint definition. * * @param key the key to use when storing the hint in the hint map - * @param supplier factory method to create the hint instance, receives the key as parameter + * @param hintFactory factory method to create the hint instance, receives the key as parameter * @param mutuallyExclusive set of hint keys that are mutually exclusive with this hint */ - public HintDefinition(String key, Supplier supplier, Set mutuallyExclusive) { + public HintDefinition( + String key, Function hintFactory, Set mutuallyExclusive) { this.key = key; - this.supplier = supplier; + this.hintFactory = hintFactory; this.mutuallyExclusive = mutuallyExclusive; } @@ -57,15 +58,6 @@ public String getKey() { return key; } - /** - * Gets the supplier that creates the hint instance. - * - * @return the hint supplier - */ - public Supplier getSupplier() { - return supplier; - } - /** * Gets the set of hint keys that are mutually exclusive with this hint. * @@ -81,7 +73,11 @@ public Set getMutuallyExclusive() { * @return the created hint */ public Hint createHint() { - return supplier.get(); + return hintFactory.apply(new String[0]); + } + + public Hint createHint(String... parameters) { + return hintFactory.apply(parameters != null ? parameters : new String[0]); } /** @@ -93,15 +89,4 @@ public Hint createHint() { public boolean canAddTo(Map hintMap) { return mutuallyExclusive.stream().noneMatch(hintMap::containsKey); } - - /** - * Adds this hint to the hint map if no mutually exclusive hints exist. - * - * @param hintMap the hint map to add to - */ - public void addTo(Map hintMap) { - if (canAddTo(hintMap)) { - hintMap.put(key, createHint()); - } - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java index b9c7f7fd82ec5..0a6e53da1494d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java @@ -23,8 +23,22 @@ public class LeaderHint extends Hint { public static String hintName = "Leader"; + private String targetTable = null; - public LeaderHint() { + public LeaderHint(String... tables) { super(hintName); + if (tables.length > 0) { + this.targetTable = tables[0]; + } + } + + @Override + public boolean appliesToAll() { + return targetTable == null; + } + + @Override + public String toString() { + return targetTable == null ? hintName : hintName + "-" + targetTable; } } diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 577ac1e739eff..4b04ef8d29f4b 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -1090,7 +1090,7 @@ selectHint ; hintItem - : identifier '(' ( identifier (',' identifier)* )? ')' #parameterizedHint + : identifier '(' identifier (',' identifier)* ')' #parameterizedHint | identifier #simpleHint ; From 404389e6faacd3b990685f3cb5f140901e6c299a Mon Sep 17 00:00:00 2001 From: shizy Date: Thu, 13 Nov 2025 11:27:49 +0800 Subject: [PATCH 03/20] follower & leader refactor # Conflicts: # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java --- .../queryengine/common/MPPQueryContext.java | 10 -- .../plan/relational/analyzer/Analysis.java | 16 ++++ .../analyzer/StatementAnalyzer.java | 93 +++++++++++++++++-- .../plan/relational/sql/ast/AstVisitor.java | 12 +++ .../sql/ast/ParameterizedHintItem.java | 80 ++++++++++++++++ .../sql/ast/QuerySpecification.java | 2 +- .../plan/relational/sql/ast/SelectHint.java | 38 ++++---- .../relational/sql/ast/SimpleHintItem.java | 73 +++++++++++++++ .../relational/sql/parser/AstBuilder.java | 82 +++++----------- .../relational/utils/hint/FollowerHint.java | 10 +- .../plan/relational/utils/hint/Hint.java | 13 +-- .../{HintDefinition.java => HintFactory.java} | 53 +++++------ .../relational/utils/hint/LeaderHint.java | 10 +- 13 files changed, 342 insertions(+), 150 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/{HintDefinition.java => HintFactory.java} (55%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index bdeae41060324..1fcd78f1c639e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -47,11 +47,9 @@ import java.time.ZoneId; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -186,14 +184,6 @@ public MPPQueryContext( this.initResultNodeContext(); } - public void setHintMap(Map hintMap) { - this.hintMap = hintMap; - } - - public Map getHintMap() { - return hintMap; - } - public void setReserveMemoryForSchemaTreeFunc(LongConsumer reserveMemoryForSchemaTreeFunc) { this.reserveMemoryForSchemaTreeFunc = reserveMemoryForSchemaTreeFunc; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 4a0fe9daa570c..68528f11130e8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -71,6 +71,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.With; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy; import com.google.common.collect.ArrayListMultimap; @@ -258,6 +259,9 @@ public class Analysis implements IAnalysis { private boolean isQuery = false; + // Hint map + private Map hintMap = new HashMap<>(); + // SqlParser is needed during query planning phase for executing uncorrelated scalar subqueries // in advance (predicate folding). The planner needs to parse and execute these subqueries // independently to utilize predicate pushdown optimization. @@ -268,6 +272,14 @@ public Analysis(@Nullable Statement root, Map, Expression> pa this.parameters = ImmutableMap.copyOf(requireNonNull(parameters, "parameters is null")); } + public void setHintMap(Map hintMap) { + this.hintMap = hintMap; + } + + public Map getHintMap() { + return hintMap; + } + public Map, Expression> getParameters() { return parameters; } @@ -850,6 +862,10 @@ public QualifiedName getRelationName(final Relation relation) { return relationNames.get(NodeRef.of(relation)); } + public List getRelationNames() { + return relationNames.values().stream().collect(toImmutableList()); + } + public void addAliased(final Relation relation) { aliasedRelations.add(NodeRef.of(relation)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 133f9b677360a..48124150e7a9e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -132,6 +132,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullIfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Offset; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ParameterizedHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeEnriched; @@ -161,6 +162,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTopics; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; @@ -194,7 +196,10 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedInsertStatement; import org.apache.iotdb.db.queryengine.plan.relational.type.CompatibleResolver; import org.apache.iotdb.db.queryengine.plan.relational.type.TypeManager; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.FollowerHint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.HintFactory; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeaderHint; import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -362,6 +367,14 @@ private enum UpdateKind { MERGE, } + // Hint definition and processing methods + private static final Map HINT_DEFINITIONS = + ImmutableMap.of( + "LEADER", + new HintFactory(LeaderHint.hintName, LeaderHint::new, true), + "FOLLOWER", + new HintFactory(FollowerHint.hintName, FollowerHint::new, true)); + /** * Visitor context represents local query scope (if exists). The invariant is that the local query * scopes hierarchy should always have outer query scope (if provided) as ancestor. @@ -1155,10 +1168,6 @@ protected Scope visitQuerySpecification(QuerySpecification node, Optional node.getWhere().ifPresent(where -> analyzeWhere(node, sourceScope, where)); List outputExpressions = analyzeSelect(node, sourceScope); - - Map hintMap = analyzeHint(node); - queryContext.setHintMap(hintMap); - Analysis.GroupingSetAnalysis groupByAnalysis = analyzeGroupBy(node, sourceScope, outputExpressions); analyzeHaving(node, sourceScope); @@ -1244,6 +1253,8 @@ protected Scope visitQuerySpecification(QuerySpecification node, Optional orderByScope.orElseThrow(() -> new NoSuchElementException("No value present"))); } + // select hint + analyzeHint(node, sourceScope); return outputScope; } @@ -1497,12 +1508,76 @@ private void analyzeWhere(Node node, Scope scope, Expression predicate) { analysis.setWhere(node, predicate); } - private Map analyzeHint(QuerySpecification node) { - Optional selectHint = node.getHintMap(); - if (selectHint.isPresent()) { - return selectHint.get().getHintMap(); + private void analyzeHint(QuerySpecification node, Scope scope) { + Optional selectHint = node.getSelectHint(); + selectHint.ifPresent(hint -> process(hint, scope)); + } + + @Override + public Scope visitSelectHint(SelectHint node, final Optional context) { + Map hintMap = new HashMap<>(); + for (Node hintItem : node.getHintItems()) { + if (hintItem instanceof ParameterizedHintItem) { + ParameterizedHintItem paramHint = (ParameterizedHintItem) hintItem; + String hintName = paramHint.getHintName(); + List params = paramHint.getParameters(); + addHint(hintName, params.toArray(new String[0]), hintMap); + } else if (hintItem instanceof SimpleHintItem) { + SimpleHintItem simpleHint = (SimpleHintItem) hintItem; + String hintName = simpleHint.getHintName(); + addHint(hintName, null, hintMap); + } + } + analysis.setHintMap(hintMap); + return createAndAssignScope(node, context); + } + + private boolean invalidHintParameters(String... params) { + if (params.length == 0) { + return false; + } + + List validTables = + analysis.getRelationNames().stream() + .map(QualifiedName::getSuffix) + .collect(toImmutableList()); + for (String tableName : params) { + if (!validTables.contains(tableName)) { + return true; + } + } + return false; + } + + private void addHint(String hintName, String[] params, Map hintMap) { + HintFactory definition = HINT_DEFINITIONS.get(hintName); + if (definition != null) { + // If this hint supports parameter expansion and has multiple parameters + if (params != null && definition.shouldExpandParameters()) { + for (String param : params) { + if (invalidHintParameters(param)) { + return; + } + + Hint hint = definition.createHint(param); + String hintKey = hint.getKey(); + if (!hintMap.containsKey(hintKey)) { + hintMap.put(hintKey, hint); + } + } + } else { + // Default behavior for single parameter or non-expanding hints + if (params != null && invalidHintParameters(params)) { + return; + } + + Hint hint = definition.createHint(params); + String hintKey = hint.getKey(); + if (!hintMap.containsKey(hintKey)) { + hintMap.put(hintKey, hint); + } + } } - return ImmutableMap.of(); } private List analyzeSelect(QuerySpecification node, Scope scope) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index 13925ae392029..4b670d050621e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -129,6 +129,18 @@ protected R visitSelect(Select node, C context) { return visitNode(node, context); } + protected R visitSelectHint(SelectHint node, C context) { + return visitNode(node, context); + } + + protected R visitSimpleHintItem(SimpleHintItem node, C context) { + return visitNode(node, context); + } + + protected R visitParameterizedHintItem(ParameterizedHintItem node, C context) { + return visitNode(node, context); + } + protected R visitRelation(Relation node, C context) { return visitNode(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java new file mode 100644 index 0000000000000..9857132ab1750 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java @@ -0,0 +1,80 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +/** Represents a parameterized hint, e.g., "LEADER(table1)" or "FOLLOWER(table2)". */ +public class ParameterizedHintItem extends Node { + private final String hintName; + private final List parameters; + + public ParameterizedHintItem(String hintName, List parameters) { + super(null); + this.hintName = hintName.toUpperCase(); + this.parameters = ImmutableList.copyOf(parameters); + } + + public String getHintName() { + return hintName; + } + + public List getParameters() { + return parameters; + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitParameterizedHintItem(this, context); + } + + @Override + public int hashCode() { + return Objects.hash(hintName, parameters); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + ParameterizedHintItem other = (ParameterizedHintItem) obj; + return Objects.equals(this.hintName, other.hintName) + && Objects.equals(this.parameters, other.parameters); + } + + @Override + public String toString() { + return hintName + "(" + String.join(", ", parameters) + ")"; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java index 3edf33320d37b..9a4afb6c2a1f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java @@ -142,7 +142,7 @@ public Optional getLimit() { return limit; } - public Optional getHintMap() { + public Optional getSelectHint() { return selectHint; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java index 6d9ee4b6ab1f3..721504393fa32 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java @@ -21,40 +21,36 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; - import com.google.common.collect.ImmutableList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; public class SelectHint extends Node { - Map hintMap; + private final List hintItems; - public SelectHint() { + public SelectHint(List hintItems) { super(null); - this.hintMap = new HashMap<>(); + this.hintItems = ImmutableList.copyOf(hintItems); } - public SelectHint(Map hintMap) { - super(null); - this.hintMap = hintMap; + public List getHintItems() { + return hintItems; } - public Map getHintMap() { - return hintMap; + @Override + public List getChildren() { + return hintItems; } @Override - public List getChildren() { - return ImmutableList.of(); + public R accept(AstVisitor visitor, C context) { + return visitor.visitSelectHint(this, context); } @Override public int hashCode() { - return Objects.hash(hintMap); + return Objects.hash(hintItems); } @Override @@ -66,25 +62,23 @@ public boolean equals(Object obj) { return false; } SelectHint other = (SelectHint) obj; - return Objects.equals(this.hintMap, other.hintMap); + return Objects.equals(this.hintItems, other.hintItems); } @Override public String toString() { - if (hintMap == null || hintMap.isEmpty()) { + if (hintItems == null || hintItems.isEmpty()) { return ""; } StringBuilder sb = new StringBuilder(); sb.append("/*+ "); - boolean first = true; - for (Map.Entry entry : hintMap.entrySet()) { - if (!first) { + for (int i = 0; i < hintItems.size(); i++) { + if (i > 0) { sb.append(" "); } - sb.append(entry.getValue().toString()); - first = false; + sb.append(hintItems.get(i).toString()); } sb.append(" */"); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java new file mode 100644 index 0000000000000..cecaa1047fff0 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java @@ -0,0 +1,73 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +/** Represents a simple hint without parameters, e.g., "LEADER". */ +public class SimpleHintItem extends Node { + private final String hintName; + + public SimpleHintItem(String hintName) { + super(null); + this.hintName = hintName.toUpperCase(); + } + + public String getHintName() { + return hintName; + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitSimpleHintItem(this, context); + } + + @Override + public int hashCode() { + return Objects.hash(hintName); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + SimpleHintItem other = (SimpleHintItem) obj; + return Objects.equals(this.hintName, other.hintName); + } + + @Override + public String toString() { + return hintName; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 36fb12aaa5743..5acedfd3fe72b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -141,6 +141,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OneOrMoreQuantifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ParameterizedHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternAlternation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternConcatenation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternPermutation; @@ -210,6 +211,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowVersion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; @@ -249,10 +251,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ZeroOrOneQuantifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.util.AstUtil; import org.apache.iotdb.db.queryengine.plan.relational.type.AuthorRType; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.FollowerHint; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.HintDefinition; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeaderHint; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; @@ -272,8 +270,6 @@ import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.tree.ParseTree; @@ -2242,7 +2238,7 @@ public Node visitQueryNoWith(RelationalSqlParser.QueryNoWithContext ctx) { orderBy, offset, limit, - query.getHintMap()), + query.getSelectHint()), Optional.empty(), Optional.empty(), Optional.empty(), @@ -2407,63 +2403,29 @@ public Node visitSelectAll(RelationalSqlParser.SelectAllContext ctx) { @Override public Node visitSelectHint(RelationalSqlParser.SelectHintContext ctx) { - Map hintMap = new HashMap<>(); - for (RelationalSqlParser.HintItemContext hiCtx : ctx.hintItem()) { - if (hiCtx instanceof RelationalSqlParser.ParameterizedHintContext) { - RelationalSqlParser.ParameterizedHintContext paramHint = - (RelationalSqlParser.ParameterizedHintContext) hiCtx; - List identifiers = paramHint.identifier(); - String hintName = identifiers.get(0).getText(); - String[] params = - identifiers.stream() - .skip(1) - .map(RelationalSqlParser.IdentifierContext::getText) - .toArray(String[]::new); - addParamHint(hintName.toUpperCase(), params, hintMap); - } else if (hiCtx instanceof RelationalSqlParser.SimpleHintContext) { - RelationalSqlParser.SimpleHintContext simpleHint = - (RelationalSqlParser.SimpleHintContext) hiCtx; - String hintName = simpleHint.identifier().getText(); - addSimpleHint(hintName.toUpperCase(), hintMap); - } + List hintItems = new ArrayList<>(); + for (RelationalSqlParser.HintItemContext hintItemCtx : ctx.hintItem()) { + hintItems.add(visit(hintItemCtx)); } + return new SelectHint(hintItems); + } - return new SelectHint(hintMap); - } - - private static final Map HINT_DEFINITIONS = - ImmutableMap.of( - "LEADER", - new HintDefinition( - LeaderHint.hintName, LeaderHint::new, ImmutableSet.of(FollowerHint.hintName)), - "FOLLOWER", - new HintDefinition( - FollowerHint.hintName, FollowerHint::new, ImmutableSet.of(LeaderHint.hintName)) - // "MERGE_JOIN", new HintDefinition("MergeJoin", MergeJoinHint::new, - // ImmutableSet.of("NestLoopJoin", "HashJoin")), - // "NL_JOIN", new HintDefinition("NestLoopJoin", NestLoopJoinHint::new, - // ImmutableSet.of("MergeJoin", "HashJoin")) - ); - - private void addParamHint(String hintName, String[] params, Map hintMap) { - HintDefinition definition = HINT_DEFINITIONS.get(hintName); - if (definition != null) { - Hint hint = definition.createHint(params); - String hintKey = hint.toString(); - if (!hintMap.containsKey(hintKey)) { - hintMap.put(hintKey, hint); - } - } + @Override + public Node visitParameterizedHint(RelationalSqlParser.ParameterizedHintContext ctx) { + List identifiers = ctx.identifier(); + String hintName = identifiers.get(0).getText(); + List params = + identifiers.stream() + .skip(1) + .map(x -> x.getText().toLowerCase()) + .collect(Collectors.toList()); + return new ParameterizedHintItem(hintName, params); } - private void addSimpleHint(String hintName, Map hintMap) { - HintDefinition definition = HINT_DEFINITIONS.get(hintName); - if (definition != null) { - Hint hint = definition.createHint(); - if (definition.canAddTo(hintMap)) { - hintMap.put(hint.toString(), hint); - } - } + @Override + public Node visitSimpleHint(RelationalSqlParser.SimpleHintContext ctx) { + String hintName = ctx.identifier().getText(); + return new SimpleHintItem(hintName); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java index 3217691a5e8e4..25700f9e78f75 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java @@ -22,19 +22,19 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; public class FollowerHint extends Hint { - public static String hintName = "Follower"; + public static String hintName = "follower"; + public static String category = "replica"; private String targetTable = null; public FollowerHint(String... tables) { - super(hintName); + super(hintName, category); if (tables.length > 0) { this.targetTable = tables[0]; } } - @Override - public boolean appliesToAll() { - return targetTable == null; + public String getKey() { + return targetTable == null ? category : category + "-" + targetTable; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java index 5575916cc7c40..ce07d51668ff8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java @@ -25,17 +25,14 @@ public abstract class Hint { protected String hintName; + protected String category; - protected Hint(String hintName) { + protected Hint(String hintName, String category) { this.hintName = Objects.requireNonNull(hintName, "hintName can not be null"); + this.category = Objects.requireNonNull(category, "category can not be null"); } - public boolean appliesToAll() { - return true; - } + public abstract String getKey(); - @Override - public String toString() { - return hintName; - } + public abstract String toString(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java similarity index 55% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java index 3f8e62a363c5d..07c920dd3b495 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java @@ -21,50 +21,48 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; -import java.util.Map; -import java.util.Set; import java.util.function.Function; /** * Defines the properties and creation logic for a SQL hint. This class encapsulates the hint's key, - * creation strategy, and which other hints it is mutually exclusive with. + * creation strategy. */ -public final class HintDefinition { +public final class HintFactory { private final String key; - private final Function hintFactory; - private final Set mutuallyExclusive; + private final Function factory; + private final boolean expandParameters; /** * Creates a new hint definition. * * @param key the key to use when storing the hint in the hint map * @param hintFactory factory method to create the hint instance, receives the key as parameter - * @param mutuallyExclusive set of hint keys that are mutually exclusive with this hint */ - public HintDefinition( - String key, Function hintFactory, Set mutuallyExclusive) { - this.key = key; - this.hintFactory = hintFactory; - this.mutuallyExclusive = mutuallyExclusive; + public HintFactory(String key, Function hintFactory) { + this(key, hintFactory, false); } /** - * Gets the hint key used in the hint map. + * Creates a new hint definition with parameter expansion option. * - * @return the hint key + * @param key the key to use when storing the hint in the hint map + * @param hintFactory factory method to create the hint instance + * @param expandParameters whether to expand array parameters into multiple hints */ - public String getKey() { - return key; + public HintFactory(String key, Function hintFactory, boolean expandParameters) { + this.key = key; + this.factory = hintFactory; + this.expandParameters = expandParameters; } /** - * Gets the set of hint keys that are mutually exclusive with this hint. + * Gets the hint name used to create hint instance. * - * @return the set of mutually exclusive hint keys + * @return the hint key */ - public Set getMutuallyExclusive() { - return mutuallyExclusive; + public String getKey() { + return key; } /** @@ -72,21 +70,16 @@ public Set getMutuallyExclusive() { * * @return the created hint */ - public Hint createHint() { - return hintFactory.apply(new String[0]); - } - public Hint createHint(String... parameters) { - return hintFactory.apply(parameters != null ? parameters : new String[0]); + return factory.apply(parameters != null ? parameters : new String[0]); } /** - * Checks if this hint is mutually exclusive with the given hint map. + * Checks whether this hint should expand parameters into multiple hints. * - * @param hintMap the current hint map to check against - * @return true if this hint can be added (no conflicts), false otherwise + * @return true if parameters should be expanded, false otherwise */ - public boolean canAddTo(Map hintMap) { - return mutuallyExclusive.stream().noneMatch(hintMap::containsKey); + public boolean shouldExpandParameters() { + return expandParameters; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java index 0a6e53da1494d..b450c0b9624dc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java @@ -22,19 +22,19 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; public class LeaderHint extends Hint { - public static String hintName = "Leader"; + public static String hintName = "leader"; + public static String category = "replica"; private String targetTable = null; public LeaderHint(String... tables) { - super(hintName); + super(hintName, category); if (tables.length > 0) { this.targetTable = tables[0]; } } - @Override - public boolean appliesToAll() { - return targetTable == null; + public String getKey() { + return targetTable == null ? category : category + "-" + targetTable; } @Override From 0769463c2884fd4cc300c16ac28a1f6e3ce40816 Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 18 Nov 2025 10:52:50 +0800 Subject: [PATCH 04/20] add ReplicaHint --- .../plan/AbstractFragmentParallelPlanner.java | 5 -- .../analyzer/StatementAnalyzer.java | 45 +++++---------- .../planner/distribute/AddExchangeNodes.java | 55 ++++++++++++++++++- .../TableDistributedPlanGenerator.java | 8 +++ .../distribute/TableDistributedPlanner.java | 2 +- .../relational/utils/hint/FollowerHint.java | 32 ++++++++--- .../relational/utils/hint/HintFactory.java | 12 ++-- .../relational/utils/hint/LeaderHint.java | 33 ++++++++--- .../relational/utils/hint/ReplicaHint.java | 48 ++++++++++++++++ 9 files changed, 180 insertions(+), 60 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java index caf8128009e1e..d7415320e048a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java @@ -118,11 +118,6 @@ protected TDataNodeLocation selectTargetDataNode(TRegionReplicaSet regionReplica } boolean selectRandomDataNode = ReadConsistencyLevel.WEAK == this.readConsistencyLevel; - // if (ReadConsistencyLevel.STRONG == this.readConsistencyLevel - // && queryContext.getHintMap().containsKey("Follower")) { - // selectRandomDataNode = true; - // } - // When planning fragment onto specific DataNode, the DataNode whose endPoint is in // black list won't be considered because it may have connection issue now. List availableDataNodes = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 48124150e7a9e..c57acfc9fe97b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1521,7 +1521,7 @@ public Scope visitSelectHint(SelectHint node, final Optional context) { ParameterizedHintItem paramHint = (ParameterizedHintItem) hintItem; String hintName = paramHint.getHintName(); List params = paramHint.getParameters(); - addHint(hintName, params.toArray(new String[0]), hintMap); + addHint(hintName, params, hintMap); } else if (hintItem instanceof SimpleHintItem) { SimpleHintItem simpleHint = (SimpleHintItem) hintItem; String hintName = simpleHint.getHintName(); @@ -1532,46 +1532,31 @@ public Scope visitSelectHint(SelectHint node, final Optional context) { return createAndAssignScope(node, context); } - private boolean invalidHintParameters(String... params) { - if (params.length == 0) { - return false; - } - - List validTables = - analysis.getRelationNames().stream() - .map(QualifiedName::getSuffix) - .collect(toImmutableList()); - for (String tableName : params) { - if (!validTables.contains(tableName)) { - return true; - } - } - return false; + private List intersect(List a, List b) { + return a.stream().filter(b::contains).collect(toImmutableList()); } - private void addHint(String hintName, String[] params, Map hintMap) { + private void addHint(String hintName, List paramTables, Map hintMap) { HintFactory definition = HINT_DEFINITIONS.get(hintName); if (definition != null) { - // If this hint supports parameter expansion and has multiple parameters - if (params != null && definition.shouldExpandParameters()) { - for (String param : params) { - if (invalidHintParameters(param)) { - return; - } + List existingTables = + analysis.getRelationNames().stream() + .map(QualifiedName::getSuffix) + .collect(toImmutableList()); + + List validTables = + paramTables != null ? intersect(paramTables, existingTables) : ImmutableList.of(); - Hint hint = definition.createHint(param); + if (definition.shouldExpandParameters()) { + for (String table : validTables) { + Hint hint = definition.createHint(ImmutableList.of(table)); String hintKey = hint.getKey(); if (!hintMap.containsKey(hintKey)) { hintMap.put(hintKey, hint); } } } else { - // Default behavior for single parameter or non-expanding hints - if (params != null && invalidHintParameters(params)) { - return; - } - - Hint hint = definition.createHint(params); + Hint hint = definition.createHint(validTables); String hintKey = hint.getKey(); if (!hintMap.containsKey(hintKey)) { hintMap.put(hintKey, hint); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java index a1c26e1b7017a..4d2286a895b23 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.distribute; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.plan.planner.distribution.NodeDistribution; @@ -35,6 +37,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.ReplicaHint; + +import java.util.List; +import java.util.Map; import static org.apache.iotdb.db.queryengine.plan.planner.distribution.NodeDistributionType.DIFFERENT_FROM_ALL_CHILDREN; import static org.apache.iotdb.db.queryengine.plan.planner.distribution.NodeDistributionType.NO_CHILD; @@ -95,9 +102,30 @@ public PlanNode visitPlan(PlanNode node, TableDistributedPlanGenerator.PlanConte @Override public PlanNode visitTableScan( TableScanNode node, TableDistributedPlanGenerator.PlanContext context) { + // Original region replica set + TRegionReplicaSet regionReplicaSet = node.getRegionReplicaSet(); + // Find applicable hint + ReplicaHint hint = findReplicaHint(node, context.hintMap); + + // Early return for simple cases + if (context.hintMap == null || regionReplicaSet == null || hint == null) { + context.nodeDistributionMap.put( + node.getPlanNodeId(), new NodeDistribution(SAME_WITH_ALL_CHILDREN, regionReplicaSet)); + return node; + } + + // Determine optimized locations based on hint + List optimizedLocations = + selectLocations(regionReplicaSet.getDataNodeLocations(), hint); + + // Create optimized region replica set + TRegionReplicaSet optimizedRegionReplicaSet = + new TRegionReplicaSet(regionReplicaSet.getRegionId(), optimizedLocations); + context.nodeDistributionMap.put( node.getPlanNodeId(), - new NodeDistribution(SAME_WITH_ALL_CHILDREN, node.getRegionReplicaSet())); + new NodeDistribution(SAME_WITH_ALL_CHILDREN, optimizedRegionReplicaSet)); + return node; } @@ -206,4 +234,29 @@ private PlanNode processTableDeviceSourceNode( new NodeDistribution(SAME_WITH_ALL_CHILDREN, node.getRegionReplicaSet())); return node; } + + /** + * Finds the applicable replica hint for the given table scan node. First checks for + * table-specific hint, then falls back to global hint. + */ + private ReplicaHint findReplicaHint(TableScanNode node, Map hintMap) { + if (hintMap == null || hintMap.isEmpty()) { + return null; + } + + String tableName = node.getQualifiedObjectName().getObjectName(); + String tableSpecificKey = "replica-" + tableName; + String globalKey = "replica-*"; + + return (ReplicaHint) hintMap.getOrDefault(tableSpecificKey, hintMap.get(globalKey)); + } + + /** + * Selects data node locations based on the provided hint using polymorphism. - ReplicaHint: Uses + * the hint's own selectLocations strategy - Other hints or null: Returns all original locations + */ + private List selectLocations( + List dataNodeLocations, ReplicaHint hint) { + return hint.selectLocations(dataNodeLocations); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 7072b5f519f73..33bfaedde0e11 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -101,6 +101,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Insert; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.table.DataNodeTreeViewSchemaUtils; @@ -1952,6 +1953,7 @@ public List visitUnion(UnionNode node, PlanContext context) { public static class PlanContext { final Map nodeDistributionMap; + final Map hintMap; boolean hasExchangeNode = false; boolean hasSortProperty = false; boolean pushDownGrouping = false; @@ -1961,6 +1963,12 @@ public static class PlanContext { public PlanContext() { this.nodeDistributionMap = new HashMap<>(); + this.hintMap = new HashMap<>(); + } + + public PlanContext(Map hintMap) { + this.nodeDistributionMap = new HashMap<>(); + this.hintMap = hintMap; } public NodeDistribution getNodeDistribution(PlanNodeId nodeId) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java index ff7686fe86891..465a113a8f9c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java @@ -100,7 +100,7 @@ public TableDistributedPlanner( public DistributedQueryPlan plan() { TableDistributedPlanGenerator.PlanContext planContext = - new TableDistributedPlanGenerator.PlanContext(); + new TableDistributedPlanGenerator.PlanContext(analysis.getHintMap()); PlanNode outputNodeWithExchange = generateDistributedPlanWithOptimize(planContext); List planText = null; if (mppQueryContext.isExplain() && mppQueryContext.isInnerTriggeredQuery()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java index 25700f9e78f75..1854a1d5ef63b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java @@ -21,24 +21,38 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; -public class FollowerHint extends Hint { +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; + +import java.util.List; + +public class FollowerHint extends ReplicaHint { public static String hintName = "follower"; - public static String category = "replica"; - private String targetTable = null; + private final String targetTable; - public FollowerHint(String... tables) { - super(hintName, category); - if (tables.length > 0) { - this.targetTable = tables[0]; + public FollowerHint(List tables) { + super(hintName); + if (tables == null || tables.size() > 1) { + throw new IllegalArgumentException("FollowerHint accepts empty or exactly one table"); } + targetTable = tables.isEmpty() ? "*" : tables.get(0); } + @Override public String getKey() { - return targetTable == null ? category : category + "-" + targetTable; + return category + "-" + targetTable; } @Override public String toString() { - return targetTable == null ? hintName : hintName + "-" + targetTable; + return hintName + "-" + targetTable; + } + + @Override + public List selectLocations(List dataNodeLocations) { + if (dataNodeLocations == null || dataNodeLocations.size() <= 1) { + return dataNodeLocations; + } + // Return only followers (all locations except the first/leader) + return dataNodeLocations.subList(1, dataNodeLocations.size()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java index 07c920dd3b495..1e1c8193277aa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; +import java.util.List; import java.util.function.Function; /** @@ -30,7 +31,7 @@ public final class HintFactory { private final String key; - private final Function factory; + private final Function, Hint> factory; private final boolean expandParameters; /** @@ -39,7 +40,7 @@ public final class HintFactory { * @param key the key to use when storing the hint in the hint map * @param hintFactory factory method to create the hint instance, receives the key as parameter */ - public HintFactory(String key, Function hintFactory) { + public HintFactory(String key, Function, Hint> hintFactory) { this(key, hintFactory, false); } @@ -50,7 +51,8 @@ public HintFactory(String key, Function hintFactory) { * @param hintFactory factory method to create the hint instance * @param expandParameters whether to expand array parameters into multiple hints */ - public HintFactory(String key, Function hintFactory, boolean expandParameters) { + public HintFactory( + String key, Function, Hint> hintFactory, boolean expandParameters) { this.key = key; this.factory = hintFactory; this.expandParameters = expandParameters; @@ -70,8 +72,8 @@ public String getKey() { * * @return the created hint */ - public Hint createHint(String... parameters) { - return factory.apply(parameters != null ? parameters : new String[0]); + public Hint createHint(List parameters) { + return factory.apply(parameters); } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java index b450c0b9624dc..6fc7a2346547f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java @@ -21,24 +21,39 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; -public class LeaderHint extends Hint { +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; + +import java.util.Collections; +import java.util.List; + +public class LeaderHint extends ReplicaHint { public static String hintName = "leader"; - public static String category = "replica"; - private String targetTable = null; + private final String targetTable; - public LeaderHint(String... tables) { - super(hintName, category); - if (tables.length > 0) { - this.targetTable = tables[0]; + public LeaderHint(List tables) { + super(hintName); + if (tables == null || tables.size() > 1) { + throw new IllegalArgumentException("LeaderHint accepts empty or exactly one table"); } + targetTable = tables.isEmpty() ? "*" : tables.get(0); } + @Override public String getKey() { - return targetTable == null ? category : category + "-" + targetTable; + return category + "-" + targetTable; } @Override public String toString() { - return targetTable == null ? hintName : hintName + "-" + targetTable; + return hintName + "-" + targetTable; + } + + @Override + public List selectLocations(List dataNodeLocations) { + if (dataNodeLocations == null || dataNodeLocations.size() <= 1) { + return dataNodeLocations; + } + // Return only the leader (first location) + return Collections.singletonList(dataNodeLocations.get(0)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java new file mode 100644 index 0000000000000..187dc27f85212 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java @@ -0,0 +1,48 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; + +import java.util.List; + +/** + * Abstract base class for replica-related hints. Provides common functionality for hints that deal + * with data node replica selection. + */ +public abstract class ReplicaHint extends Hint { + public static String category = "replica"; + + protected ReplicaHint(String hintName) { + super(hintName, category); + } + + /** + * Selects data node locations based on the replica strategy. Each replica hint implementation + * defines its own location selection logic. + * + * @param dataNodeLocations the available data node locations + * @return the selected locations based on replica hint strategy + */ + public abstract List selectLocations( + List dataNodeLocations); +} From 55937ddc41628e972e0c14e1920aa1d3d0afd8f4 Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 18 Nov 2025 15:44:34 +0800 Subject: [PATCH 05/20] add alias support --- ...ableModelStatementMemorySourceVisitor.java | 2 +- .../planner/plan/node/PlanGraphPrinter.java | 6 +++ .../plan/relational/analyzer/Analysis.java | 12 +++-- .../analyzer/StatementAnalyzer.java | 2 +- .../relational/planner/RelationPlanner.java | 3 +- .../planner/distribute/AddExchangeNodes.java | 5 +- .../TableDistributedPlanGenerator.java | 14 +++-- .../iterative/rule/PruneTableScanColumns.java | 3 +- .../node/AggregationTableScanNode.java | 54 +++++++++++++++++-- .../planner/node/DeviceTableScanNode.java | 49 ++++++++++++++++- .../planner/node/TableScanNode.java | 27 ++++++++++ .../UnaliasSymbolReferences.java | 3 +- .../planner/optimizations/Util.java | 3 +- .../relational/analyzer/TestPlanBuilder.java | 3 +- 14 files changed, 162 insertions(+), 24 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java index 932d941979223..2fcc8a608ada1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java @@ -92,7 +92,7 @@ public StatementMemorySource visitExplain( // Generate table model distributed plan final TableDistributedPlanGenerator.PlanContext planContext = - new TableDistributedPlanGenerator.PlanContext(); + new TableDistributedPlanGenerator.PlanContext(context.getAnalysis().getHintMap()); final PlanNode outputNodeWithExchange = new TableDistributedPlanner( context.getAnalysis(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java index 180241b4192cd..03789645d9dbc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java @@ -647,6 +647,9 @@ public List visitTableScan(TableScanNode node, GraphContext context) { List boxValue = new ArrayList<>(); boxValue.add(node.toString()); boxValue.add(String.format("QualifiedTableName: %s", node.getQualifiedObjectName().toString())); + if (node.getAlias() != null) { + boxValue.add(String.format("Alias: %s", node.getAlias().getValue())); + } boxValue.add(String.format("OutputSymbols: %s", node.getOutputSymbols())); if (deviceTableScanNode != null) { @@ -727,6 +730,9 @@ public List visitAggregationTableScan( List boxValue = new ArrayList<>(); boxValue.add(node.toString()); boxValue.add(String.format("QualifiedTableName: %s", node.getQualifiedObjectName().toString())); + if (node.getAlias() != null) { + boxValue.add(String.format("Alias: %s", node.getAlias().getValue())); + } boxValue.add(String.format("OutputSymbols: %s", node.getOutputSymbols())); int i = 0; for (org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Aggregation diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 68528f11130e8..2259ead7fa31c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -218,7 +218,7 @@ public class Analysis implements IAnalysis { private final Map, QualifiedName> relationNames = new LinkedHashMap<>(); - private final Set> aliasedRelations = new LinkedHashSet<>(); + private final Map, Identifier> aliasedRelations = new LinkedHashMap<>(); private final Map, TableFunctionInvocationAnalysis> tableFunctionAnalyses = new LinkedHashMap<>(); @@ -866,12 +866,16 @@ public List getRelationNames() { return relationNames.values().stream().collect(toImmutableList()); } - public void addAliased(final Relation relation) { - aliasedRelations.add(NodeRef.of(relation)); + public void addAliased(final Relation relation, Identifier alias) { + aliasedRelations.put(NodeRef.of(relation), alias); + } + + public Identifier getAliased(Relation relation) { + return aliasedRelations.get(NodeRef.of(relation)); } public boolean isAliased(Relation relation) { - return aliasedRelations.contains(NodeRef.of(relation)); + return aliasedRelations.containsKey(NodeRef.of(relation)); } public void addTableSchema( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index c57acfc9fe97b..32197c14fc72e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -3666,7 +3666,7 @@ protected Scope visitValues(Values node, Optional scope) { @Override protected Scope visitAliasedRelation(AliasedRelation relation, Optional scope) { analysis.setRelationName(relation, QualifiedName.of(ImmutableList.of(relation.getAlias()))); - analysis.addAliased(relation.getRelation()); + analysis.addAliased(relation.getRelation(), relation.getAlias()); Scope relationScope = process(relation.getRelation(), scope); RelationType relationType = relationScope.getRelationType(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index ded699588c1c2..a3564b86d800e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -394,7 +394,8 @@ private RelationPlan processPhysicalTable(Table table, Scope scope) { qualifiedObjectName, outputSymbols, tableColumnSchema, - tagAndAttributeIndexMap); + tagAndAttributeIndexMap, + analysis.getAliased(table)); return new RelationPlan(tableScanNode, scope, outputSymbols, outerContext); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java index 4d2286a895b23..9aaa219585beb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java @@ -244,7 +244,10 @@ private ReplicaHint findReplicaHint(TableScanNode node, Map hintMa return null; } - String tableName = node.getQualifiedObjectName().getObjectName(); + String tableName = + node.getAlias() != null + ? node.getAlias().getValue() + : node.getQualifiedObjectName().getObjectName(); String tableSpecificKey = "replica-" + tableName; String globalKey = "replica-*"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 33bfaedde0e11..a24e881abe55c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -743,7 +743,8 @@ private List constructDeviceTableScanByTags( node.getPushDownLimit(), node.getPushDownOffset(), node.isPushLimitToEachDevice(), - node.containsNonAlignedDevice()); + node.containsNonAlignedDevice(), + node.getAlias()); scanNode.setRegionReplicaSet(regionReplicaSets.get(0)); return scanNode; }); @@ -829,7 +830,8 @@ private List constructDeviceTableScanByRegionReplicaSet( node.getPushDownLimit(), node.getPushDownOffset(), node.isPushLimitToEachDevice(), - node.containsNonAlignedDevice()); + node.containsNonAlignedDevice(), + node.getAlias()); scanNode.setRegionReplicaSet(regionReplicaSet); return scanNode; }); @@ -1442,7 +1444,8 @@ private void buildRegionNodeMap( partialAggTableScanNode.getGroupingSets(), partialAggTableScanNode.getPreGroupedSymbols(), partialAggTableScanNode.getStep(), - partialAggTableScanNode.getGroupIdSymbol()) + partialAggTableScanNode.getGroupIdSymbol(), + partialAggTableScanNode.getAlias()) : new AggregationTreeDeviceViewScanNode( queryId.genPlanNodeId(), partialAggTableScanNode.getQualifiedObjectName(), @@ -1961,11 +1964,6 @@ public static class PlanContext { TRegionReplicaSet mostUsedRegion; boolean deviceCrossRegion; - public PlanContext() { - this.nodeDistributionMap = new HashMap<>(); - this.hintMap = new HashMap<>(); - } - public PlanContext(Map hintMap) { this.nodeDistributionMap = new HashMap<>(); this.hintMap = hintMap; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java index ffce0b6693e9d..08451ea7edb63 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java @@ -126,7 +126,8 @@ public static Optional pruneColumns(TableScanNode node, Set re deviceTableScanNode.getPushDownLimit(), deviceTableScanNode.getPushDownOffset(), deviceTableScanNode.isPushLimitToEachDevice(), - deviceTableScanNode.containsNonAlignedDevice())); + deviceTableScanNode.containsNonAlignedDevice(), + deviceTableScanNode.getAlias())); } } else if (node instanceof InformationSchemaTableScanNode) { // For the convenience of process in execution stage, column-prune for diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTableScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTableScanNode.java index 56d39f2f77dcb..1928db3c73120 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTableScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTableScanNode.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.Assignments; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; @@ -148,6 +149,50 @@ public AggregationTableScanNode( this.setOutputSymbols(constructOutputSymbols(groupingSets, aggregations)); } + public AggregationTableScanNode( + PlanNodeId id, + QualifiedObjectName qualifiedObjectName, + List outputSymbols, + Map assignments, + List deviceEntries, + Map tagAndAttributeIndexMap, + Ordering scanOrder, + Expression timePredicate, + Expression pushDownPredicate, + long pushDownLimit, + long pushDownOffset, + boolean pushLimitToEachDevice, + boolean containsNonAlignedDevice, + Assignments projection, + Map aggregations, + AggregationNode.GroupingSetDescriptor groupingSets, + List preGroupedSymbols, + AggregationNode.Step step, + Optional groupIdSymbol, + Identifier alias) { + this( + id, + qualifiedObjectName, + outputSymbols, + assignments, + deviceEntries, + tagAndAttributeIndexMap, + scanOrder, + timePredicate, + pushDownPredicate, + pushDownLimit, + pushDownOffset, + pushLimitToEachDevice, + containsNonAlignedDevice, + projection, + aggregations, + groupingSets, + preGroupedSymbols, + step, + groupIdSymbol); + this.alias = alias; + } + protected AggregationTableScanNode() {} private static List constructOutputSymbols( @@ -278,7 +323,8 @@ public AggregationTableScanNode clone() { groupingSets, preGroupedSymbols, step, - groupIdSymbol); + groupIdSymbol, + alias); } @Override @@ -336,7 +382,8 @@ public static AggregationTableScanNode combineAggregationAndTableScan( aggregationNode.getGroupingSets(), aggregationNode.getPreGroupedSymbols(), aggregationNode.getStep(), - aggregationNode.getGroupIdSymbol()); + aggregationNode.getGroupIdSymbol(), + tableScanNode.getAlias()); } public static AggregationTableScanNode combineAggregationAndTableScan( @@ -390,7 +437,8 @@ public static AggregationTableScanNode combineAggregationAndTableScan( aggregationNode.getGroupingSets(), aggregationNode.getPreGroupedSymbols(), step, - aggregationNode.getGroupIdSymbol()); + aggregationNode.getGroupIdSymbol(), + tableScanNode.getAlias()); } public boolean mayUseLastCache() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java index b91737739351a..cead343ddcee4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; import org.apache.tsfile.read.filter.basic.Filter; @@ -89,6 +90,18 @@ public DeviceTableScanNode( this.tagAndAttributeIndexMap = tagAndAttributeIndexMap; } + public DeviceTableScanNode( + PlanNodeId id, + QualifiedObjectName qualifiedObjectName, + List outputSymbols, + Map assignments, + Map tagAndAttributeIndexMap, + Identifier alias) { + super(id, qualifiedObjectName, outputSymbols, assignments); + this.tagAndAttributeIndexMap = tagAndAttributeIndexMap; + this.alias = alias; + } + public DeviceTableScanNode( PlanNodeId id, QualifiedObjectName qualifiedObjectName, @@ -120,6 +133,39 @@ public DeviceTableScanNode( this.containsNonAlignedDevice = containsNonAlignedDevice; } + public DeviceTableScanNode( + PlanNodeId id, + QualifiedObjectName qualifiedObjectName, + List outputSymbols, + Map assignments, + List deviceEntries, + Map tagAndAttributeIndexMap, + Ordering scanOrder, + Expression timePredicate, + Expression pushDownPredicate, + long pushDownLimit, + long pushDownOffset, + boolean pushLimitToEachDevice, + boolean containsNonAlignedDevice, + Identifier alias) { + super( + id, + qualifiedObjectName, + outputSymbols, + assignments, + pushDownPredicate, + pushDownLimit, + pushDownOffset); + this.deviceEntries = deviceEntries; + this.tagAndAttributeIndexMap = tagAndAttributeIndexMap; + this.scanOrder = scanOrder; + this.timePredicate = timePredicate; + this.pushDownPredicate = pushDownPredicate; + this.pushLimitToEachDevice = pushLimitToEachDevice; + this.containsNonAlignedDevice = containsNonAlignedDevice; + this.alias = alias; + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitDeviceTableScan(this, context); @@ -140,7 +186,8 @@ public DeviceTableScanNode clone() { pushDownLimit, pushDownOffset, pushLimitToEachDevice, - containsNonAlignedDevice); + containsNonAlignedDevice, + alias); } protected static void serializeMemberVariables( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java index 11807d9ceb499..f50f9f39b40f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import com.google.common.collect.ImmutableList; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -75,6 +76,9 @@ public abstract class TableScanNode extends SourceNode { // For query of schemaInfo, we only need the list of DataNodeLocation protected TRegionReplicaSet regionReplicaSet; + // alias name + protected Identifier alias; + public TableScanNode( PlanNodeId id, QualifiedObjectName qualifiedObjectName, @@ -177,6 +181,10 @@ public void open() throws Exception {} @Override public void close() throws Exception {} + public Identifier getAlias() { + return alias; + } + public QualifiedObjectName getQualifiedObjectName() { return this.qualifiedObjectName; } @@ -255,6 +263,13 @@ protected static void serializeMemberVariables( ReadWriteIOUtils.write(node.pushDownLimit, byteBuffer); ReadWriteIOUtils.write(node.pushDownOffset, byteBuffer); + + if (node.alias != null) { + ReadWriteIOUtils.write(true, byteBuffer); + Identifier.serialize(node.alias, byteBuffer); + } else { + ReadWriteIOUtils.write(false, byteBuffer); + } } protected static void serializeMemberVariables( @@ -290,6 +305,13 @@ protected static void serializeMemberVariables( ReadWriteIOUtils.write(node.pushDownLimit, stream); ReadWriteIOUtils.write(node.pushDownOffset, stream); + + if (node.alias != null) { + ReadWriteIOUtils.write(true, stream); + Identifier.serialize(node.alias, stream); + } else { + ReadWriteIOUtils.write(false, stream); + } } protected static void deserializeMemberVariables( @@ -327,6 +349,11 @@ protected static void deserializeMemberVariables( node.pushDownLimit = ReadWriteIOUtils.readLong(byteBuffer); node.pushDownOffset = ReadWriteIOUtils.readLong(byteBuffer); + + boolean hasAlias = ReadWriteIOUtils.readBool(byteBuffer); + if (hasAlias) { + node.alias = new Identifier(byteBuffer); + } } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index b7f2e9eaaa851..08cce14e375ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -237,7 +237,8 @@ public PlanAndMappings visitDeviceTableScan(DeviceTableScanNode node, UnaliasCon node.getPushDownLimit(), node.getPushDownOffset(), node.isPushLimitToEachDevice(), - node.containsNonAlignedDevice()), + node.containsNonAlignedDevice(), + node.getAlias()), mapping); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java index 706b24a567b3b..27a8ea66bf67c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java @@ -188,7 +188,8 @@ public static Pair split( node.getGroupingSets(), node.getPreGroupedSymbols(), PARTIAL, - node.getGroupIdSymbol()) + node.getGroupIdSymbol(), + node.getAlias()) : new AggregationTreeDeviceViewScanNode( queryId.genPlanNodeId(), node.getQualifiedObjectName(), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestPlanBuilder.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestPlanBuilder.java index 4b0136d892327..8422834c5cb3e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestPlanBuilder.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestPlanBuilder.java @@ -101,7 +101,8 @@ public TestPlanBuilder deviceTableScan( pushDownLimit, pushDownOffset, pushLimitToEachDevice, - containsNonAlignedDevice); + containsNonAlignedDevice, + null); return this; } } From d3016a8d716d0b4b0b44f89f98e293ad96a8c2a1 Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 18 Nov 2025 17:29:03 +0800 Subject: [PATCH 06/20] fix alias serialize --- .../plan/relational/planner/node/TableScanNode.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java index f50f9f39b40f7..c6bda2232621c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java @@ -266,7 +266,7 @@ protected static void serializeMemberVariables( if (node.alias != null) { ReadWriteIOUtils.write(true, byteBuffer); - Identifier.serialize(node.alias, byteBuffer); + node.alias.serialize(byteBuffer); } else { ReadWriteIOUtils.write(false, byteBuffer); } @@ -308,7 +308,7 @@ protected static void serializeMemberVariables( if (node.alias != null) { ReadWriteIOUtils.write(true, stream); - Identifier.serialize(node.alias, stream); + node.alias.serialize(stream); } else { ReadWriteIOUtils.write(false, stream); } From ac48e367c5bbad10d7cc0b96a74278a5c792229d Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 19 Nov 2025 14:04:31 +0800 Subject: [PATCH 07/20] Leader & Follower Hint without parameters --- .../plan/relational/analyzer/StatementAnalyzer.java | 2 +- .../relational/planner/distribute/AddExchangeNodes.java | 3 +-- .../plan/relational/utils/hint/FollowerHint.java | 6 +++--- .../queryengine/plan/relational/utils/hint/LeaderHint.java | 6 +++--- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 32197c14fc72e..0080dcc5f575c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1545,7 +1545,7 @@ private void addHint(String hintName, List paramTables, Map validTables = - paramTables != null ? intersect(paramTables, existingTables) : ImmutableList.of(); + paramTables != null ? intersect(paramTables, existingTables) : existingTables; if (definition.shouldExpandParameters()) { for (String table : validTables) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java index 9aaa219585beb..642bc5ce65526 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java @@ -249,9 +249,8 @@ private ReplicaHint findReplicaHint(TableScanNode node, Map hintMa ? node.getAlias().getValue() : node.getQualifiedObjectName().getObjectName(); String tableSpecificKey = "replica-" + tableName; - String globalKey = "replica-*"; - return (ReplicaHint) hintMap.getOrDefault(tableSpecificKey, hintMap.get(globalKey)); + return (ReplicaHint) hintMap.getOrDefault(tableSpecificKey, null); } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java index 1854a1d5ef63b..2c1d3b3a0bafc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java @@ -31,10 +31,10 @@ public class FollowerHint extends ReplicaHint { public FollowerHint(List tables) { super(hintName); - if (tables == null || tables.size() > 1) { - throw new IllegalArgumentException("FollowerHint accepts empty or exactly one table"); + if (tables == null || tables.size() != 1) { + throw new IllegalArgumentException("FollowerHint accepts exactly one table"); } - targetTable = tables.isEmpty() ? "*" : tables.get(0); + targetTable = tables.get(0); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java index 6fc7a2346547f..8c179846da986 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java @@ -32,10 +32,10 @@ public class LeaderHint extends ReplicaHint { public LeaderHint(List tables) { super(hintName); - if (tables == null || tables.size() > 1) { - throw new IllegalArgumentException("LeaderHint accepts empty or exactly one table"); + if (tables == null || tables.size() != 1) { + throw new IllegalArgumentException("LeaderHint accepts exactly one table"); } - targetTable = tables.isEmpty() ? "*" : tables.get(0); + targetTable = tables.get(0); } @Override From eb24b0ec74496eb89fc042295ffbf683bc5164a2 Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 19 Nov 2025 16:15:21 +0800 Subject: [PATCH 08/20] hintParameters --- .../queryengine/plan/relational/analyzer/Analysis.java | 4 ++-- .../plan/relational/analyzer/StatementAnalyzer.java | 6 ++++-- .../plan/relational/sql/parser/AstBuilder.java | 9 +++------ .../iotdb/db/relational/grammar/sql/RelationalSql.g4 | 10 ++++++++-- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 2259ead7fa31c..09ae588888bd8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -862,8 +862,8 @@ public QualifiedName getRelationName(final Relation relation) { return relationNames.get(NodeRef.of(relation)); } - public List getRelationNames() { - return relationNames.values().stream().collect(toImmutableList()); + public Map, QualifiedName> getRelationNames() { + return relationNames; } public void addAliased(final Relation relation, Identifier alias) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 0080dcc5f575c..055811213a570 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1539,9 +1539,11 @@ private List intersect(List a, List b) { private void addHint(String hintName, List paramTables, Map hintMap) { HintFactory definition = HINT_DEFINITIONS.get(hintName); if (definition != null) { + // filter the relation of aliased relation List existingTables = - analysis.getRelationNames().stream() - .map(QualifiedName::getSuffix) + analysis.getRelationNames().entrySet().stream() + .filter(entry -> !analysis.isAliased(entry.getKey().getNode())) + .map(entry -> entry.getValue().getSuffix()) .collect(toImmutableList()); List validTables = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 5acedfd3fe72b..f3b1a1ada2bac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -2412,13 +2412,10 @@ public Node visitSelectHint(RelationalSqlParser.SelectHintContext ctx) { @Override public Node visitParameterizedHint(RelationalSqlParser.ParameterizedHintContext ctx) { - List identifiers = ctx.identifier(); - String hintName = identifiers.get(0).getText(); + String hintName = ctx.identifier().getText(); + List identifiers = ctx.hintParameter(); List params = - identifiers.stream() - .skip(1) - .map(x -> x.getText().toLowerCase()) - .collect(Collectors.toList()); + identifiers.stream().map(x -> x.getText().toLowerCase()).collect(Collectors.toList()); return new ParameterizedHintItem(hintName, params); } diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 4b04ef8d29f4b..75b9c95702484 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -1086,14 +1086,20 @@ joinCriteria ; selectHint - : HINT_START hintItem (',' hintItem)* HINT_END + : HINT_START hintItem+ HINT_END ; hintItem - : identifier '(' identifier (',' identifier)* ')' #parameterizedHint + : identifier '(' hintParameter+ ')' #parameterizedHint | identifier #simpleHint ; +hintParameter + : identifier + | '{' + | '}' + ; + patternRecognition : aliasedRelation ( MATCH_RECOGNIZE '(' From 53f620ae6b5450b930fae7f582ffab6fc1682f21 Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 23 Dec 2025 11:28:10 +0800 Subject: [PATCH 09/20] fix: rebase master --- .../queryengine/plan/relational/analyzer/StatementAnalyzer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 055811213a570..22c547587cc58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -132,8 +132,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullIfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Offset; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ParameterizedHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ParameterizedHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeEnriched; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Property; From 9aa018a6ad1176d1187052ebc03987e2e8c2b489 Mon Sep 17 00:00:00 2001 From: shizy Date: Fri, 9 Jan 2026 15:38:51 +0800 Subject: [PATCH 10/20] leading hint --- .../queryengine/common/MPPQueryContext.java | 2 + .../analyzer/StatementAnalyzer.java | 29 +-- .../optimizations/CollectJoinConstraint.java | 73 ++++++ .../optimizations/LeadingJoinOptimizer.java | 83 +++++++ .../optimizations/LogicalOptimizeFactory.java | 4 +- .../sql/ast/ParameterizedHintItem.java | 13 ++ .../plan/relational/sql/ast/SelectHint.java | 12 + .../relational/sql/ast/SimpleHintItem.java | 12 + .../relational/utils/hint/JoinOrderHint.java | 30 +++ .../relational/utils/hint/LeadingHint.java | 219 ++++++++++++++++++ 10 files changed, 462 insertions(+), 15 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinOrderHint.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 1fcd78f1c639e..7e69c9c643228 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -47,9 +47,11 @@ import java.time.ZoneId; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 22c547587cc58..393c523702d16 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -200,6 +200,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.HintFactory; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeaderHint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeadingHint; import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -373,7 +374,9 @@ private enum UpdateKind { "LEADER", new HintFactory(LeaderHint.hintName, LeaderHint::new, true), "FOLLOWER", - new HintFactory(FollowerHint.hintName, FollowerHint::new, true)); + new HintFactory(FollowerHint.hintName, FollowerHint::new, true), + "LEADING", + new HintFactory(LeadingHint.hintName, LeadingHint::new, false)); /** * Visitor context represents local query scope (if exists). The invariant is that the local query @@ -1536,21 +1539,19 @@ private List intersect(List a, List b) { return a.stream().filter(b::contains).collect(toImmutableList()); } - private void addHint(String hintName, List paramTables, Map hintMap) { + private void addHint(String hintName, List parameters, Map hintMap) { HintFactory definition = HINT_DEFINITIONS.get(hintName); if (definition != null) { - // filter the relation of aliased relation - List existingTables = - analysis.getRelationNames().entrySet().stream() - .filter(entry -> !analysis.isAliased(entry.getKey().getNode())) - .map(entry -> entry.getValue().getSuffix()) - .collect(toImmutableList()); - - List validTables = - paramTables != null ? intersect(paramTables, existingTables) : existingTables; - if (definition.shouldExpandParameters()) { - for (String table : validTables) { + List existingTables = + analysis.getRelationNames().entrySet().stream() + .filter(entry -> !analysis.isAliased(entry.getKey().getNode())) + .map(entry -> entry.getValue().getSuffix()) + .collect(toImmutableList()); + for (String table : parameters) { + if (!existingTables.contains(table)) { + continue; + } Hint hint = definition.createHint(ImmutableList.of(table)); String hintKey = hint.getKey(); if (!hintMap.containsKey(hintKey)) { @@ -1558,7 +1559,7 @@ private void addHint(String hintName, List paramTables, Map { + private final Analysis analysis; + + public Rewriter(Analysis analysis) { + this.analysis = analysis; + } + + @Override + public PlanNode visitPlan(PlanNode node, Context context) { + Hint hint = analysis.getHintMap().getOrDefault(JoinOrderHint.category, null); + if (!(hint instanceof LeadingHint)) { + return node; + } + + for (PlanNode child : node.getChildren()) { + child.accept(this, context); + } + return node; + } + + @Override + public PlanNode visitJoin(JoinNode node, Context context) { + PlanNode leftNode = node.getLeftChild(); + PlanNode rightNode = node.getRightChild(); + return node; + } + + // @Override + // public PlanNode visitSemiJoin(SemiJoinNode node, Context context) { + // return visitTwoChildProcess(node, context); + // } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java new file mode 100644 index 0000000000000..14c3d1fd261f8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java @@ -0,0 +1,83 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.JoinOrderHint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeadingHint; + +import java.util.Set; + +import static com.google.common.collect.ImmutableSet.toImmutableSet; + +public class LeadingJoinOptimizer implements PlanOptimizer { + @Override + public PlanNode optimize(PlanNode plan, Context context) { + if (!context.getAnalysis().isQuery()) { + return plan; + } + + return plan.accept(new Rewriter(context.getAnalysis()), null); + } + + private static class Rewriter extends PlanVisitor { + private final Analysis analysis; + + public Rewriter(Analysis analysis) { + this.analysis = analysis; + } + + @Override + public PlanNode visitPlan(PlanNode node, Context context) { + Hint hint = analysis.getHintMap().getOrDefault(JoinOrderHint.category, null); + if (!(hint instanceof LeadingHint)) { + return node; + } + LeadingHint leadingHint = (LeadingHint) hint; + Set relationNames = + analysis.getRelationNames().values().stream() + .map(QualifiedName::getSuffix) + .collect(toImmutableSet()); + + if (!validateTableNamesMatch(relationNames, leadingHint)) { + return node; + } + + PlanNode leadingJoin = leadingHint.generateLeadingJoinPlan(); + if (leadingJoin != null) { + return leadingJoin; + } + return node; + } + + private boolean validateTableNamesMatch(Set relationNames, LeadingHint leadingHint) { + if (relationNames.size() != leadingHint.getTables().size()) { + return false; + } + return relationNames.containsAll(leadingHint.getTables()); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index 864b9a987abca..546eab97f318d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -388,7 +388,9 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new MergeLimitWithSort(), new MergeLimitOverProjectWithSort(), new PushTopKThroughUnion())), - new ParallelizeGrouping()); + new ParallelizeGrouping(), + new CollectJoinConstraint()); + // new LeadingJoinOptimizer()); this.planOptimizers = optimizerBuilder.build(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java index 9857132ab1750..cc3b98d9b45b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java @@ -22,12 +22,16 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.RamUsageEstimator; import java.util.List; import java.util.Objects; /** Represents a parameterized hint, e.g., "LEADER(table1)" or "FOLLOWER(table2)". */ public class ParameterizedHintItem extends Node { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(ParameterizedHintItem.class); + private final String hintName; private final List parameters; @@ -77,4 +81,13 @@ public boolean equals(Object obj) { public String toString() { return hintName + "(" + String.join(", ", parameters) + ")"; } + + @Override + public long ramBytesUsed() { + long size = INSTANCE_SIZE; + size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); + size += AstMemoryEstimationHelper.getEstimatedSizeOfStringList(parameters); + size += RamUsageEstimator.sizeOf(hintName); + return size; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java index 721504393fa32..ee583509a131e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java @@ -22,11 +22,15 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.RamUsageEstimator; import java.util.List; import java.util.Objects; public class SelectHint extends Node { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(SelectHint.class); + private final List hintItems; public SelectHint(List hintItems) { @@ -84,4 +88,12 @@ public String toString() { sb.append(" */"); return sb.toString(); } + + @Override + public long ramBytesUsed() { + long size = INSTANCE_SIZE; + size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); + size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeList(hintItems); + return size; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java index cecaa1047fff0..9eebb65fe9da2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java @@ -22,12 +22,16 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.RamUsageEstimator; import java.util.List; import java.util.Objects; /** Represents a simple hint without parameters, e.g., "LEADER". */ public class SimpleHintItem extends Node { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(SimpleHintItem.class); + private final String hintName; public SimpleHintItem(String hintName) { @@ -70,4 +74,12 @@ public boolean equals(Object obj) { public String toString() { return hintName; } + + @Override + public long ramBytesUsed() { + long size = INSTANCE_SIZE; + size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); + size += RamUsageEstimator.sizeOf(hintName); + return size; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinOrderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinOrderHint.java new file mode 100644 index 0000000000000..3bd3d2bf446d6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinOrderHint.java @@ -0,0 +1,30 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +public abstract class JoinOrderHint extends Hint { + public static String category = "join-order"; + + protected JoinOrderHint(String hintName) { + super(hintName, category); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java new file mode 100644 index 0000000000000..7cca5eddb0d10 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java @@ -0,0 +1,219 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +import org.apache.iotdb.commons.exception.IoTDBRuntimeException; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; + +import com.google.common.collect.Sets; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.Stack; + +import static org.apache.iotdb.rpc.TSStatusCode.INTERNAL_SERVER_ERROR; + +public class LeadingHint extends JoinOrderHint { + public static String hintName = "leading"; + private final List tables; + + private List addJoinParameters; + private List normalizedParameters; + + public LeadingHint(List parameters) { + super(hintName); + // /* leading(t3 {}) 会报错 + this.tables = new ArrayList<>(); + addJoinParameters = insertJoinIntoParameters(parameters); + normalizedParameters = parseIntoReversePolishNotation(addJoinParameters); + + if (tables.isEmpty()) { + throw new IllegalArgumentException("LeaderHint accepts one or more tables"); + } + if (hasDuplicateTable(tables)) { + throw new IllegalArgumentException("LeaderHint accepts no duplicate tables"); + } + } + + @Override + public String getKey() { + return category; + } + + public List getTables() { + return tables; + } + + public PlanNode generateLeadingJoinPlan() { + Stack stack = new Stack<>(); + for (String item : normalizedParameters) { + if (item.equals("join")) { + PlanNode rightChild = stack.pop(); + PlanNode leftChild = stack.pop(); + PlanNode joinPlan = makeJoinPlan(leftChild, rightChild); + if (joinPlan == null) { + return null; + } + stack.push(joinPlan); + } else { + // PlanNode logicalPlan = getLogicalPlanByName(item); + // ogicalPlan = makeFilterPlanIfExist(getFilters(), logicalPlan); + // stack.push(logicalPlan); + } + } + + PlanNode finalJoin = stack.pop(); + // we want all filters been removed + // if (Utils.enableAssert && !filters.isEmpty()) { + // throw new IllegalStateException( + // "Leading hint process failed: filter should be empty, but meet: " + filters + // ); + // } + if (finalJoin == null) { + throw new IoTDBRuntimeException( + "final join plan should not be null", INTERNAL_SERVER_ERROR.getStatusCode()); + } + return finalJoin; + } + + @Override + public String toString() { + if (tables == null || tables.isEmpty()) { + return hintName; + } + return hintName + "-" + String.join("-", tables); + } + + private boolean hasDuplicateTable(List tables) { + Set tableSet = Sets.newHashSet(); + for (String table : tables) { + if (!tableSet.add(table)) { + return true; + } + } + return false; + } + + public static List insertJoinIntoParameters(List list) { + List output = new ArrayList<>(); + + for (String item : list) { + if (item.equals("{")) { + output.add(item); + continue; + } else if (item.equals("}")) { + output.remove(output.size() - 1); + output.add(item); + } else { + output.add(item); + } + output.add("join"); + } + output.remove(output.size() - 1); + return output; + } + + public List parseIntoReversePolishNotation(List list) { + Stack s1 = new Stack<>(); + List s2 = new ArrayList<>(); + + for (String item : list) { + if (!(item.equals("{") || item.equals("}") || item.equals("join"))) { + tables.add(item); + s2.add(item); + } else if (item.equals("{")) { + s1.push(item); + } else if (item.equals("}")) { + while (!s1.peek().equals("{")) { + String pop = s1.pop(); + s2.add(pop); + } + s1.pop(); + } else { + while (!s1.isEmpty() && !s1.peek().equals("{")) { + s2.add(s1.pop()); + } + s1.push(item); + } + } + while (!s1.isEmpty()) { + s2.add(s1.pop()); + } + return s2; + } + + private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { + // List conditions = getJoinConditions( + // getFilters(), leftChild, rightChild); + // Pair, List> pair = JoinUtils.extractExpressionForHashTable( + // leftChild.getOutput(), rightChild.getOutput(), conditions); + // // leading hint would set status inside if not success + // JoinType joinType = computeJoinType(getBitmap(leftChild), + // getBitmap(rightChild), conditions); + // if (joinType == null) { + // this.setStatus(HintStatus.SYNTAX_ERROR); + // this.setErrorMessage("JoinType can not be null"); + // } else if (!isConditionJoinTypeMatched(conditions, joinType)) { + // this.setStatus(HintStatus.UNUSED); + // this.setErrorMessage("condition does not matched joinType"); + // } + // if (!this.isSuccess()) { + // return null; + // } + // get joinType + // LogicalJoin logicalJoin = new LogicalJoin<>(joinType, pair.first, + // pair.second, + // distributeHint, + // Optional.empty(), + // leftChild, + // rightChild, null); + // logicalJoin.getJoinReorderContext().setLeadingJoin(true); + // logicalJoin.setBitmap(LongBitmap.or(getBitmap(leftChild), getBitmap(rightChild))); + + PlanNode joinNode = planJoin(leftChild, rightChild); + return joinNode; + } + + private PlanNode planJoin(PlanNode leftPlan, PlanNode rightPlan) { + List leftOutputSymbols = leftPlan.getOutputSymbols(); + List rightOutputSymbols = rightPlan.getOutputSymbols(); + List criteria = new ArrayList<>(); + Optional asofCriteria = Optional.empty(); + + return new JoinNode( + new PlanNodeId("join"), + JoinNode.JoinType.INNER, + leftPlan, + rightPlan, + criteria, + asofCriteria, + leftOutputSymbols, + rightOutputSymbols, + Optional.empty(), + Optional.empty()); + } +} From f573cc0c109fca067d3d0d15df2bb9b64e3a4e5d Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 21 Jan 2026 10:16:15 +0800 Subject: [PATCH 11/20] EquiJoinClause part 1 --- .../relational/planner/RelationPlanner.java | 30 ++++++++- ...TransformFilteringSemiJoinToInnerJoin.java | 7 ++- .../relational/planner/node/JoinNode.java | 61 +++++++++++++++++-- .../PushPredicateIntoTableScan.java | 24 +++++++- .../UnaliasSymbolReferences.java | 5 +- .../relational/analyzer/AsofJoinTest.java | 14 ++--- .../plan/relational/analyzer/JoinTest.java | 14 ++++- .../analyzer/TableFunctionTest.java | 2 +- .../planner/CorrelatedSubqueryTest.java | 4 +- .../assertions/EquiJoinClauseProvider.java | 16 ++++- .../planner/assertions/JoinMatcher.java | 5 +- .../planner/assertions/PlanMatchPattern.java | 9 ++- 12 files changed, 162 insertions(+), 29 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index a3564b86d800e..1d0dab122f036 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -142,6 +142,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -513,7 +514,9 @@ If casts are redundant (due to column type and common type being equal), rightCoercions.put(rightOutput, right.getSymbol(rightField).toSymbolReference()); rightJoinColumns.put(identifier, rightOutput); - clauses.add(new JoinNode.EquiJoinClause(leftOutput, rightOutput)); + Set leftTables = new HashSet<>(left.getScope().getTables()); + Set rightTables = new HashSet<>(right.getScope().getTables()); + clauses.add(new JoinNode.EquiJoinClause(leftOutput, rightOutput, leftTables, rightTables)); } ProjectNode leftCoercion = @@ -729,7 +732,30 @@ public RelationPlan planJoin( Symbol leftSymbol = leftCoercions.get(leftComparisonExpressions.get(i)); Symbol rightSymbol = rightCoercions.get(rightComparisonExpressions.get(i)); - equiClauses.add(new JoinNode.EquiJoinClause(leftSymbol, rightSymbol)); + // Extract tables from expressions + Set leftDependencies = + SymbolsExtractor.extractNames( + leftComparisonExpressions.get(i), analysis.getColumnReferences()); + Set rightDependencies = + SymbolsExtractor.extractNames( + rightComparisonExpressions.get(i), analysis.getColumnReferences()); + + // Convert QualifiedName to Identifier (table name is the first part) + Set leftTables = new HashSet<>(); + for (QualifiedName name : leftDependencies) { + if (name.getPrefix().isPresent()) { + leftTables.add(new Identifier(name.getPrefix().get().getSuffix())); + } + } + Set rightTables = new HashSet<>(); + for (QualifiedName name : rightDependencies) { + if (name.getPrefix().isPresent()) { + rightTables.add(new Identifier(name.getPrefix().get().getSuffix())); + } + } + + equiClauses.add( + new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, leftTables, rightTables)); } else { postInnerJoinConditions.add( new ComparisonExpression( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java index 1f32fdbffe157..bdc35fd8ba255 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java @@ -34,6 +34,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Optional; @@ -130,7 +131,11 @@ public Result apply(FilterNode filterNode, Captures captures, Context context) { filteringSourceDistinct, ImmutableList.of( new JoinNode.EquiJoinClause( - semiJoin.getSourceJoinSymbol(), semiJoin.getFilteringSourceJoinSymbol())), + semiJoin.getSourceJoinSymbol(), + semiJoin.getFilteringSourceJoinSymbol(), + // should from SemiJoin + ImmutableSet.of(), + ImmutableSet.of())), Optional.empty(), semiJoin.getSource().getOutputSymbols(), ImmutableList.of(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index 5e6cc5818acc1..c3c79caeb2727 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; import com.google.common.collect.ImmutableList; @@ -37,6 +38,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -236,6 +238,16 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { for (EquiJoinClause equiJoinClause : criteria) { Symbol.serialize(equiJoinClause.getLeft(), byteBuffer); Symbol.serialize(equiJoinClause.getRight(), byteBuffer); + Set leftTables = equiJoinClause.getLeftTables(); + ReadWriteIOUtils.write(leftTables.size(), byteBuffer); + for (Identifier identifier : leftTables) { + identifier.serialize(byteBuffer); + } + Set rightTables = equiJoinClause.getRightTables(); + ReadWriteIOUtils.write(rightTables.size(), byteBuffer); + for (Identifier identifier : rightTables) { + identifier.serialize(byteBuffer); + } } if (asofCriteria.isPresent()) { @@ -268,6 +280,16 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { for (EquiJoinClause equiJoinClause : criteria) { Symbol.serialize(equiJoinClause.getLeft(), stream); Symbol.serialize(equiJoinClause.getRight(), stream); + Set leftTables = equiJoinClause.getLeftTables(); + ReadWriteIOUtils.write(leftTables.size(), stream); + for (Identifier identifier : leftTables) { + identifier.serialize(stream); + } + Set rightTables = equiJoinClause.getRightTables(); + ReadWriteIOUtils.write(rightTables.size(), stream); + for (Identifier identifier : rightTables) { + identifier.serialize(stream); + } } if (asofCriteria.isPresent()) { @@ -295,8 +317,19 @@ public static JoinNode deserialize(ByteBuffer byteBuffer) { int size = ReadWriteIOUtils.readInt(byteBuffer); List criteria = new ArrayList<>(size); while (size-- > 0) { - criteria.add( - new EquiJoinClause(Symbol.deserialize(byteBuffer), Symbol.deserialize(byteBuffer))); + Symbol left = Symbol.deserialize(byteBuffer); + Symbol right = Symbol.deserialize(byteBuffer); + int leftTablesSize = ReadWriteIOUtils.readInt(byteBuffer); + Set leftTables = new HashSet<>(leftTablesSize); + while (leftTablesSize-- > 0) { + leftTables.add(new Identifier(byteBuffer)); + } + int rightTablesSize = ReadWriteIOUtils.readInt(byteBuffer); + Set rightTables = new HashSet<>(rightTablesSize); + while (rightTablesSize-- > 0) { + rightTables.add(new Identifier(byteBuffer)); + } + criteria.add(new EquiJoinClause(left, right, leftTables, rightTables)); } Optional asofJoinClause = Optional.empty(); @@ -369,10 +402,15 @@ public String toString() { public static class EquiJoinClause { private final Symbol left; private final Symbol right; + private final Set leftTables; + private final Set rightTables; - public EquiJoinClause(Symbol left, Symbol right) { + public EquiJoinClause( + Symbol left, Symbol right, Set leftTables, Set rightTables) { this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); + this.leftTables = requireNonNull(leftTables, "leftTables is null"); + this.rightTables = requireNonNull(rightTables, "rightTables is null"); } public Symbol getLeft() { @@ -383,13 +421,21 @@ public Symbol getRight() { return right; } + public Set getLeftTables() { + return leftTables; + } + + public Set getRightTables() { + return rightTables; + } + public ComparisonExpression toExpression() { return new ComparisonExpression( ComparisonExpression.Operator.EQUAL, left.toSymbolReference(), right.toSymbolReference()); } public EquiJoinClause flip() { - return new EquiJoinClause(right, left); + return new EquiJoinClause(right, left, rightTables, leftTables); } public static List flipBatch(List input) { @@ -410,12 +456,15 @@ public boolean equals(Object obj) { EquiJoinClause other = (EquiJoinClause) obj; - return Objects.equals(this.left, other.left) && Objects.equals(this.right, other.right); + return Objects.equals(this.left, other.left) + && Objects.equals(this.right, other.right) + && Objects.equals(this.leftTables, other.leftTables) + && Objects.equals(this.rightTables, other.rightTables); } @Override public int hashCode() { - return Objects.hash(left, right); + return Objects.hash(left, right, leftTables, rightTables); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 55905f6768247..8de262a7d5e60 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -66,9 +66,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.utils.TimestampPrecisionUtils; @@ -856,7 +858,27 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { rightProjections.put(rightSymbol, rightExpression); } - equiJoinClauses.add(new JoinNode.EquiJoinClause(leftSymbol, rightSymbol)); + Set leftDependencies = + SymbolsExtractor.extractNames(equality.getLeft(), analysis.getColumnReferences()); + Set rightDependencies = + SymbolsExtractor.extractNames(equality.getRight(), analysis.getColumnReferences()); + + // Convert QualifiedName to Identifier (table name is the first part) + Set leftTables = new HashSet<>(); + for (QualifiedName name : leftDependencies) { + if (name.getPrefix().isPresent()) { + leftTables.add(new Identifier(name.getPrefix().get().getSuffix())); + } + } + Set rightTables = new HashSet<>(); + for (QualifiedName name : rightDependencies) { + if (name.getPrefix().isPresent()) { + rightTables.add(new Identifier(name.getPrefix().get().getSuffix())); + } + } + + equiJoinClauses.add( + new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, leftTables, rightTables)); } else { if (conjunct.equals(TRUE_LITERAL) && node.getAsofCriteria().isPresent()) { continue; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index 08cce14e375ec..5748f539879f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -813,7 +813,10 @@ public PlanAndMappings visitJoin(JoinNode node, UnaliasContext context) { for (JoinNode.EquiJoinClause clause : node.getCriteria()) { builder.add( new JoinNode.EquiJoinClause( - mapper.map(clause.getLeft()), mapper.map(clause.getRight()))); + mapper.map(clause.getLeft()), + mapper.map(clause.getRight()), + ImmutableSet.copyOf(clause.getLeftTables()), + ImmutableSet.copyOf(clause.getRightTables()))); } List newCriteria = builder.build(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java index 62ec85d27745b..dee921c73fe9b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java @@ -154,7 +154,7 @@ public void projectInAsofCriteriaTest() { builder .asofCriteria( ComparisonExpression.Operator.GREATER_THAN, "expr", "time_0") - .equiCriteria("tag1", "tag1_1") + .equiCriteria("tag1", "tag1_1", "table1", "table2") .left( sort( ImmutableList.of( @@ -210,9 +210,9 @@ public void sortEliminateTest() { .asofCriteria(ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") .equiCriteria( ImmutableList.of( - equiJoinClause("tag1", "tag1_1"), - equiJoinClause("tag2", "tag2_2"), - equiJoinClause("tag3", "tag3_3"))) + equiJoinClause("tag1", "tag1_1", "table1", "table2"), + equiJoinClause("tag2", "tag2_2", "table1", "table2"), + equiJoinClause("tag3", "tag3_3", "table1", "table2"))) .left(sort(table1)) .right(sort(table2))))); @@ -227,9 +227,9 @@ public void sortEliminateTest() { .asofCriteria(ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") .equiCriteria( ImmutableList.of( - equiJoinClause("tag1", "tag1_1"), - equiJoinClause("tag2", "tag2_2"), - equiJoinClause("tag3", "tag3_3"))) + equiJoinClause("tag1", "tag1_1", "table1", "table2"), + equiJoinClause("tag2", "tag2_2", "table1", "table2"), + equiJoinClause("tag3", "tag3_3", "table1", "table2"))) .left(exchange()) .right(exchange())))); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java index 1b178b50449da..1e161eea62061 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java @@ -53,6 +53,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; @@ -187,7 +188,12 @@ private void assertInnerJoinTest1(String sql) { joinNode = (JoinNode) getChildrenNode(logicalPlanNode, 3); List joinCriteria = Collections.singletonList( - new JoinNode.EquiJoinClause(Symbol.of("time"), Symbol.of("time_0"))); + new JoinNode.EquiJoinClause( + Symbol.of("time"), + Symbol.of("time_0"), + ImmutableSet.of(new Identifier("t1")), + ImmutableSet.of(new Identifier("t2")))); + assertJoinNodeEquals( joinNode, INNER, @@ -368,7 +374,11 @@ private void assertInnerJoinTest2(String sql, boolean joinUsing) { joinNode = (JoinNode) getChildrenNode(logicalPlanNode, 4); List joinCriteria = Collections.singletonList( - new JoinNode.EquiJoinClause(Symbol.of("time"), Symbol.of("time_0"))); + new JoinNode.EquiJoinClause( + Symbol.of("time"), + Symbol.of("time_0"), + ImmutableSet.of(new Identifier("t1")), + ImmutableSet.of(new Identifier("t2")))); assertJoinNodeEquals( joinNode, INNER, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java index 344c69cfc1a71..da2edb5051dab 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java @@ -281,7 +281,7 @@ public void testLeafFunction() { builder .left(sort(tableFunctionProcessor(tableFunctionMatcher1))) .right(sort(tableFunctionProcessor(tableFunctionMatcher2))) - .equiCriteria("output", "output_0")))); + .equiCriteria("output", "output_0", "a", "b")))); } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java index e7baf8078f112..c54df4742c43a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java @@ -84,7 +84,7 @@ public void testCorrelatedExistsSubquery() { JoinNode.JoinType.INNER, builder -> builder - .equiCriteria("s1", "s2_7") + .equiCriteria("s1", "s2_7", "t1", "t2") .left(sort(tableScan1)) .right( sort( @@ -133,7 +133,7 @@ public void testCorrelatedExistsSubquery() { JoinNode.JoinType.LEFT, builder -> builder - .equiCriteria("s1", "s2_3") + .equiCriteria("s1", "s2_3", "t1", "t2") .left(sort(tableScan1)) .right( sort( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java index c151b6874a6f8..b6e5280caf2ea 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java @@ -20,21 +20,33 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.assertions; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; + +import java.util.Set; import static java.util.Objects.requireNonNull; public class EquiJoinClauseProvider implements ExpectedValueProvider { private final SymbolAlias left; private final SymbolAlias right; + private final Set leftTables; + private final Set rightTables; - public EquiJoinClauseProvider(SymbolAlias left, SymbolAlias right) { + public EquiJoinClauseProvider( + SymbolAlias left, + SymbolAlias right, + Set leftTables, + Set rightTables) { this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); + this.leftTables = requireNonNull(leftTables, "leftTables is null"); + this.rightTables = requireNonNull(rightTables, "rightTables is null"); } @Override public JoinNode.EquiJoinClause getExpectedValue(SymbolAliases aliases) { - return new JoinNode.EquiJoinClause(left.toSymbol(aliases), right.toSymbol(aliases)); + return new JoinNode.EquiJoinClause( + left.toSymbol(aliases), right.toSymbol(aliases), leftTables, rightTables); } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java index 090e4a0bc3bc9..8ea44457e331d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java @@ -166,8 +166,9 @@ public Builder equiCriteria( } @CanIgnoreReturnValue - public Builder equiCriteria(String left, String right) { - this.equiCriteria = Optional.of(ImmutableList.of(equiJoinClause(left, right))); + public Builder equiCriteria(String left, String right, String leftTable, String rightTable) { + this.equiCriteria = + Optional.of(ImmutableList.of(equiJoinClause(left, right, leftTable, rightTable))); return this; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java index 03f79fd2ec29a..044de2f45f56e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java @@ -61,6 +61,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; @@ -682,8 +683,12 @@ public static PlanMatchPattern strictProject( } public static ExpectedValueProvider equiJoinClause( - String left, String right) { - return new EquiJoinClauseProvider(new SymbolAlias(left), new SymbolAlias(right)); + String left, String right, String leftTable, String rightTable) { + return new EquiJoinClauseProvider( + new SymbolAlias(left), + new SymbolAlias(right), + ImmutableSet.of(new Identifier(leftTable)), + ImmutableSet.of(new Identifier(rightTable))); } public static AsofJoinClauseProvider asofJoinClause( From 23d4188d74410f49aebd40139e6c189a652b43a1 Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 21 Jan 2026 14:36:14 +0800 Subject: [PATCH 12/20] left & right tables for JoinNode --- .../queryengine/common/MPPQueryContext.java | 10 +- .../plan/relational/analyzer/Scope.java | 17 +-- .../analyzer/StatementAnalyzer.java | 2 +- .../relational/planner/CteMaterializer.java | 3 +- .../relational/planner/RelationPlanner.java | 12 +- ...correlatedScalarSubqueryReconstructor.java | 3 +- .../iterative/rule/PruneJoinColumns.java | 4 +- ...TransformFilteringSemiJoinToInnerJoin.java | 7 +- .../relational/planner/node/JoinNode.java | 113 +++++++++++++++++- .../PushPredicateIntoTableScan.java | 16 ++- .../UnaliasSymbolReferences.java | 4 +- 11 files changed, 154 insertions(+), 37 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 7e69c9c643228..e89e84cda49b1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -41,7 +41,7 @@ import org.apache.iotdb.db.queryengine.statistics.QueryPlanStatistics; import org.apache.iotdb.db.utils.cte.CteDataStore; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Pair; @@ -146,7 +146,7 @@ public enum ExplainType { private boolean innerTriggeredQuery = false; // Tables in the subquery - private final Map, List> subQueryTables = new HashMap<>(); + private final Map, Set> subQueryTables = new HashMap<>(); public MPPQueryContext(QueryId queryId) { this.queryId = queryId; @@ -542,12 +542,12 @@ public void setCteQueries(Map, Query> cteQueries) { this.cteQueries = cteQueries; } - public void addSubQueryTables(Query query, List tables) { + public void addSubQueryTables(Query query, Set tables) { subQueryTables.put(NodeRef.of(query), tables); } - public List getTables(Query query) { - return subQueryTables.getOrDefault(NodeRef.of(query), ImmutableList.of()); + public Set getTables(Query query) { + return subQueryTables.getOrDefault(NodeRef.of(query), ImmutableSet.of()); } public void addCteExplainResult(Table table, Pair> cteExplainResult) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java index d06529c00506f..d178a5f30471e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java @@ -31,11 +31,12 @@ import com.google.common.collect.ImmutableMap; -import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Predicate; import static com.google.common.base.MoreObjects.toStringHelper; @@ -57,7 +58,7 @@ public class Scope { // Tables to access for the current relation. For CTE materialization and constant folding // subqueries, non-materialized CTEs in tables must be identified, and their definitions // attached to the subquery context. - private List tables; + private Set tables; public static Scope create() { return builder().build(); @@ -73,24 +74,24 @@ private Scope( RelationId relationId, RelationType relation, Map namedQueries, - List tables) { + Set tables) { this.parent = requireNonNull(parent, "parent is null"); this.relationId = requireNonNull(relationId, "relationId is null"); this.queryBoundary = queryBoundary; this.relation = requireNonNull(relation, "relation is null"); this.namedQueries = ImmutableMap.copyOf(requireNonNull(namedQueries, "namedQueries is null")); - this.tables = new ArrayList<>(requireNonNull(tables, "tables is null")); + this.tables = new HashSet<>(requireNonNull(tables, "tables is null")); } public void addTable(Table table) { tables.add(new Identifier(table.getName().getSuffix())); } - public void setTables(List tables) { + public void setTables(Set tables) { this.tables = tables; } - public List getTables() { + public Set getTables() { return tables; } @@ -349,7 +350,7 @@ public static final class Builder { private RelationId relationId = RelationId.anonymous(); private RelationType relationType = new RelationType(); private final Map namedQueries = new HashMap<>(); - private final List tables = new ArrayList<>(); + private final Set tables = new HashSet<>(); private Optional parent = Optional.empty(); private boolean queryBoundary; @@ -388,7 +389,7 @@ public Builder withNamedQuery(String name, WithQuery withQuery) { return this; } - public Builder withTables(List tables) { + public Builder withTables(Set tables) { this.tables.addAll(tables); return this; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 393c523702d16..4cebaffeca8a5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -3720,7 +3720,7 @@ protected Scope visitJoin(Join node, Optional scope) { joinConditionCheck(criteria); // Remember original tables before processing left - List originalTables = new ArrayList<>(); + Set originalTables = new HashSet<>(); scope.ifPresent(s -> originalTables.addAll(s.getTables())); Scope left = process(node.getLeft(), scope); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CteMaterializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CteMaterializer.java index 72d231bd60439..21eae05da71ff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CteMaterializer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CteMaterializer.java @@ -66,6 +66,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -108,7 +109,7 @@ public CteDataStore fetchCteQueryResult( try { Query q = query; if (with != null) { - List tables = context.getTables(query); + Set tables = context.getTables(query); List withQueries = with.getQueries().stream() .filter( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 1d0dab122f036..6bc78efb6896d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -537,7 +537,9 @@ If casts are redundant (due to column type and common type being equal), leftCoercion.getOutputSymbols(), rightCoercion.getOutputSymbols(), Optional.empty(), - Optional.empty()); + Optional.empty(), + left.getScope().getTables(), + right.getScope().getTables()); // Transform RIGHT JOIN to LEFT if (join.getJoinType() == JoinNode.JoinType.RIGHT) { join = join.flip(); @@ -777,7 +779,9 @@ public RelationPlan planJoin( leftPlanBuilder.getRoot().getOutputSymbols(), rightPlanBuilder.getRoot().getOutputSymbols(), Optional.empty(), - Optional.empty()); + Optional.empty(), + leftPlan.getScope().getTables(), + rightPlan.getScope().getTables()); if (type == RIGHT && asofCriteria == null) { root = ((JoinNode) root).flip(); } @@ -832,7 +836,9 @@ public RelationPlan planJoin( complexJoinExpressions.stream() .map(e -> coerceIfNecessary(analysis, e, translationMap.rewrite(e))) .collect(Collectors.toList()))), - Optional.empty()); + Optional.empty(), + leftPlan.getScope().getTables(), + rightPlan.getScope().getTables()); if (type == RIGHT && asofCriteria == null) { root = ((JoinNode) root).flip(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java index 268de07bdbc55..d4b0ef83f4865 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java @@ -55,6 +55,7 @@ import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -118,7 +119,7 @@ public Optional fetchUncorrelatedSubqueryResultForPredicate( Query query = subqueryExpression.getQuery(); Query q = query; if (with != null) { - List tables = context.getTables(query); + Set tables = context.getTables(query); List withQueries = with.getQueries().stream() .filter( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java index c09710346ed09..21f3df06ec7a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java @@ -48,6 +48,8 @@ protected Optional pushDownProjectOff( filteredCopy(joinNode.getLeftOutputSymbols(), referencedOutputs::contains), filteredCopy(joinNode.getRightOutputSymbols(), referencedOutputs::contains), joinNode.getFilter(), - joinNode.isSpillable())); + joinNode.isSpillable(), + joinNode.getLeftTables(), + joinNode.getRightTables())); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java index bdc35fd8ba255..1f32fdbffe157 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java @@ -34,7 +34,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Optional; @@ -131,11 +130,7 @@ public Result apply(FilterNode filterNode, Captures captures, Context context) { filteringSourceDistinct, ImmutableList.of( new JoinNode.EquiJoinClause( - semiJoin.getSourceJoinSymbol(), - semiJoin.getFilteringSourceJoinSymbol(), - // should from SemiJoin - ImmutableSet.of(), - ImmutableSet.of())), + semiJoin.getSourceJoinSymbol(), semiJoin.getFilteringSourceJoinSymbol())), Optional.empty(), semiJoin.getSource().getOutputSymbols(), ImmutableList.of(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index c3c79caeb2727..0f85e5990aa90 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -67,6 +67,9 @@ public class JoinNode extends TwoChildProcessNode { // private final Optional distributionType; // private final Map dynamicFilters; + private final Set leftTables; + private final Set rightTables; + public JoinNode( PlanNodeId id, JoinType joinType, @@ -77,7 +80,9 @@ public JoinNode( List leftOutputSymbols, List rightOutputSymbols, Optional filter, - Optional spillable) { + Optional spillable, + Set leftTables, + Set rightTables) { super(id); requireNonNull(joinType, "type is null"); requireNonNull(leftChild, "left is null"); @@ -96,6 +101,8 @@ public JoinNode( // requireNonNull(leftHashSymbol, "leftHashSymbol is null"); // requireNonNull(rightHashSymbol, "rightHashSymbol is null"); requireNonNull(spillable, "spillable is null"); + requireNonNull(leftTables, "leftTables is null"); + requireNonNull(rightTables, "rightTables is null"); this.joinType = joinType; this.leftChild = leftChild; @@ -109,6 +116,8 @@ public JoinNode( // this.maySkipOutputDuplicates = maySkipOutputDuplicates; // this.leftHashSymbol = leftHashSymbol; // this.rightHashSymbol = rightHashSymbol; + this.leftTables = ImmutableSet.copyOf(leftTables); + this.rightTables = ImmutableSet.copyOf(rightTables); Set leftSymbols = ImmutableSet.copyOf(leftChild.getOutputSymbols()); Set rightSymbols = ImmutableSet.copyOf(rightChild.getOutputSymbols()); @@ -136,6 +145,32 @@ public JoinNode( equiJoinClause)); } + public JoinNode( + PlanNodeId id, + JoinType joinType, + PlanNode leftChild, + PlanNode rightChild, + List criteria, + Optional asofCriteria, + List leftOutputSymbols, + List rightOutputSymbols, + Optional filter, + Optional spillable) { + this( + id, + joinType, + leftChild, + rightChild, + criteria, + asofCriteria, + leftOutputSymbols, + rightOutputSymbols, + filter, + spillable, + ImmutableSet.of(), + ImmutableSet.of()); + } + // only used for deserialize public JoinNode( PlanNodeId id, @@ -143,15 +178,21 @@ public JoinNode( List criteria, Optional asofCriteria, List leftOutputSymbols, - List rightOutputSymbols) { + List rightOutputSymbols, + Set leftTables, + Set rightTables) { super(id); requireNonNull(joinType, "type is null"); requireNonNull(criteria, "criteria is null"); + requireNonNull(leftTables, "leftTables is null"); + requireNonNull(rightTables, "rightTables is null"); this.leftOutputSymbols = leftOutputSymbols; this.rightOutputSymbols = rightOutputSymbols; this.filter = Optional.empty(); this.spillable = Optional.empty(); + this.leftTables = ImmutableSet.copyOf(leftTables); + this.rightTables = ImmutableSet.copyOf(rightTables); this.joinType = joinType; this.criteria = criteria; @@ -172,7 +213,9 @@ public JoinNode flip() { rightOutputSymbols, leftOutputSymbols, filter, - spillable); + spillable, + rightTables, + leftTables); } @Override @@ -193,7 +236,9 @@ public PlanNode replaceChildren(List newChildren) { leftOutputSymbols, rightOutputSymbols, filter, - spillable); + spillable, + leftTables, + rightTables); } @Override @@ -217,7 +262,9 @@ public PlanNode clone() { leftOutputSymbols, rightOutputSymbols, filter, - spillable); + spillable, + leftTables, + rightTables); joinNode.setLeftChild(null); joinNode.setRightChild(null); return joinNode; @@ -268,6 +315,16 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { for (Symbol rightOutputSymbol : rightOutputSymbols) { Symbol.serialize(rightOutputSymbol, byteBuffer); } + + // Serialize JoinNode-level leftTables and rightTables + ReadWriteIOUtils.write(this.leftTables.size(), byteBuffer); + for (Identifier identifier : this.leftTables) { + identifier.serialize(byteBuffer); + } + ReadWriteIOUtils.write(this.rightTables.size(), byteBuffer); + for (Identifier identifier : this.rightTables) { + identifier.serialize(byteBuffer); + } } @Override @@ -310,6 +367,16 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { for (Symbol rightOutputSymbol : rightOutputSymbols) { Symbol.serialize(rightOutputSymbol, stream); } + + // Serialize JoinNode-level leftTables and rightTables + ReadWriteIOUtils.write(this.leftTables.size(), stream); + for (Identifier identifier : this.leftTables) { + identifier.serialize(stream); + } + ReadWriteIOUtils.write(this.rightTables.size(), stream); + for (Identifier identifier : this.rightTables) { + identifier.serialize(stream); + } } public static JoinNode deserialize(ByteBuffer byteBuffer) { @@ -354,9 +421,28 @@ public static JoinNode deserialize(ByteBuffer byteBuffer) { rightOutputSymbols.add(Symbol.deserialize(byteBuffer)); } + // Deserialize JoinNode-level leftTables and rightTables + int joinLeftTablesSize = ReadWriteIOUtils.readInt(byteBuffer); + Set joinLeftTables = new HashSet<>(joinLeftTablesSize); + while (joinLeftTablesSize-- > 0) { + joinLeftTables.add(new Identifier(byteBuffer)); + } + int joinRightTablesSize = ReadWriteIOUtils.readInt(byteBuffer); + Set joinRightTables = new HashSet<>(joinRightTablesSize); + while (joinRightTablesSize-- > 0) { + joinRightTables.add(new Identifier(byteBuffer)); + } + PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); return new JoinNode( - planNodeId, joinType, criteria, asofJoinClause, leftOutputSymbols, rightOutputSymbols); + planNodeId, + joinType, + criteria, + asofJoinClause, + leftOutputSymbols, + rightOutputSymbols, + joinLeftTables, + joinRightTables); } public JoinType getJoinType() { @@ -387,6 +473,14 @@ public Optional isSpillable() { return spillable; } + public Set getLeftTables() { + return leftTables; + } + + public Set getRightTables() { + return rightTables; + } + public boolean isCrossJoin() { return !asofCriteria.isPresent() && criteria.isEmpty() @@ -405,6 +499,13 @@ public static class EquiJoinClause { private final Set leftTables; private final Set rightTables; + public EquiJoinClause(Symbol left, Symbol right) { + this.left = requireNonNull(left, "left is null"); + this.right = requireNonNull(right, "right is null"); + this.leftTables = ImmutableSet.of(); + this.rightTables = ImmutableSet.of(); + } + public EquiJoinClause( Symbol left, Symbol right, Set leftTables, Set rightTables) { this.left = requireNonNull(left, "left is null"); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 8de262a7d5e60..7c4d894494cfe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -961,7 +961,9 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { leftSource.getOutputSymbols(), rightSource.getOutputSymbols(), newJoinFilter, - node.isSpillable()); + node.isSpillable(), + node.getLeftTables(), + node.getRightTables()); } JoinNode outputJoinNode = (JoinNode) output; @@ -1364,7 +1366,9 @@ private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, Expression inheri node.getLeftOutputSymbols(), node.getRightOutputSymbols(), node.getFilter(), - node.isSpillable()); + node.isSpillable(), + node.getLeftTables(), + node.getRightTables()); } if (canConvertToLeftJoin) { return new JoinNode( @@ -1377,7 +1381,9 @@ private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, Expression inheri node.getLeftOutputSymbols(), node.getRightOutputSymbols(), node.getFilter(), - node.isSpillable()); + node.isSpillable(), + node.getLeftTables(), + node.getRightTables()); } else { // temp fix because right join is not supported for now. return node; @@ -1412,7 +1418,9 @@ private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, Expression inheri node.getLeftOutputSymbols(), node.getRightOutputSymbols(), node.getFilter(), - node.isSpillable()); + node.isSpillable(), + node.getLeftTables(), + node.getRightTables()); } private boolean canConvertOuterToInner( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index 5748f539879f7..5ac1a82022a1c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -858,7 +858,9 @@ public PlanAndMappings visitJoin(JoinNode node, UnaliasContext context) { newLeftOutputSymbols, newRightOutputSymbols, newFilter, - node.isSpillable()), + node.isSpillable(), + node.getLeftTables(), + node.getRightTables()), outputMapping); } From 05f36a89dcf4cd01c13c25b66ea8693d11de5b15 Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 21 Jan 2026 15:58:16 +0800 Subject: [PATCH 13/20] Update PushPredicateIntoTableScan --- .../PushPredicateIntoTableScan.java | 35 ++++++++----------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 7c4d894494cfe..9d7f9417f4993 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -70,7 +70,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.utils.TimestampPrecisionUtils; @@ -835,6 +834,15 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { node.getRightChild().getOutputSymbols().stream() .collect(toImmutableMap(key -> key, Symbol::toSymbolReference))); + // Build a map from (left, right) symbols to their (leftTables, rightTables) + // This preserves table info when rebuilding EquiJoinClause + Map, Pair, Set>> symbolToTablesMap = + node.getCriteria().stream() + .collect( + toImmutableMap( + clause -> new Pair<>(clause.getLeft(), clause.getRight()), + clause -> new Pair<>(clause.getLeftTables(), clause.getRightTables()))); + // Create new projections for the new join clauses List equiJoinClauses = new ArrayList<>(); ImmutableList.Builder joinFilterBuilder = ImmutableList.builder(); @@ -858,27 +866,14 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { rightProjections.put(rightSymbol, rightExpression); } - Set leftDependencies = - SymbolsExtractor.extractNames(equality.getLeft(), analysis.getColumnReferences()); - Set rightDependencies = - SymbolsExtractor.extractNames(equality.getRight(), analysis.getColumnReferences()); - - // Convert QualifiedName to Identifier (table name is the first part) - Set leftTables = new HashSet<>(); - for (QualifiedName name : leftDependencies) { - if (name.getPrefix().isPresent()) { - leftTables.add(new Identifier(name.getPrefix().get().getSuffix())); - } - } - Set rightTables = new HashSet<>(); - for (QualifiedName name : rightDependencies) { - if (name.getPrefix().isPresent()) { - rightTables.add(new Identifier(name.getPrefix().get().getSuffix())); - } - } + // Retrieve leftTables and rightTables from original criteria + Pair, Set> tables = + symbolToTablesMap.getOrDefault( + new Pair<>(leftSymbol, rightSymbol), + new Pair<>(ImmutableSet.of(), ImmutableSet.of())); equiJoinClauses.add( - new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, leftTables, rightTables)); + new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, tables.left, tables.right)); } else { if (conjunct.equals(TRUE_LITERAL) && node.getAsofCriteria().isPresent()) { continue; From d6bbb5116aed58b34f618fe3cb490b67bcc8e918 Mon Sep 17 00:00:00 2001 From: shizy Date: Thu, 22 Jan 2026 16:17:02 +0800 Subject: [PATCH 14/20] simple inner join reorder --- .../relational/planner/node/JoinNode.java | 8 + .../optimizations/CollectJoinConstraint.java | 62 ++++++- .../optimizations/LeadingJoinOptimizer.java | 40 ++-- .../optimizations/LogicalOptimizeFactory.java | 6 +- .../relational/utils/hint/LeadingHint.java | 171 +++++++++++++----- 5 files changed, 225 insertions(+), 62 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index 0f85e5990aa90..aeaf4d0dbb7c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -32,6 +32,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; @@ -498,12 +499,14 @@ public static class EquiJoinClause { private final Symbol right; private final Set leftTables; private final Set rightTables; + private final Set tables; public EquiJoinClause(Symbol left, Symbol right) { this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); this.leftTables = ImmutableSet.of(); this.rightTables = ImmutableSet.of(); + this.tables = ImmutableSet.of(); } public EquiJoinClause( @@ -512,6 +515,7 @@ public EquiJoinClause( this.right = requireNonNull(right, "right is null"); this.leftTables = requireNonNull(leftTables, "leftTables is null"); this.rightTables = requireNonNull(rightTables, "rightTables is null"); + this.tables = Sets.union(leftTables, rightTables); } public Symbol getLeft() { @@ -530,6 +534,10 @@ public Set getRightTables() { return rightTables; } + public Set getTables() { + return tables; + } + public ComparisonExpression toExpression() { return new ComparisonExpression( ComparisonExpression.Operator.EQUAL, left.toSymbolReference(), right.toSymbolReference()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java index 5922d7dc3bcae..5450c9b60f8fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java @@ -24,11 +24,21 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.JoinOrderHint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeadingHint; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.apache.tsfile.utils.Pair; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + public class CollectJoinConstraint implements PlanOptimizer { @Override public PlanNode optimize(PlanNode plan, Context context) { @@ -58,10 +68,41 @@ public PlanNode visitPlan(PlanNode node, Context context) { return node; } + @Override + public PlanNode visitDeviceTableScan(DeviceTableScanNode node, Context context) { + String tableName = node.getQualifiedObjectName().getObjectName(); + LeadingHint leading = (LeadingHint) analysis.getHintMap().get(JoinOrderHint.category); + leading.getRelationToScanMap().put(tableName, node); + return node; + } + @Override public PlanNode visitJoin(JoinNode node, Context context) { - PlanNode leftNode = node.getLeftChild(); - PlanNode rightNode = node.getRightChild(); + for (PlanNode child : node.getChildren()) { + child.accept(this, context); + } + + LeadingHint leading = (LeadingHint) analysis.getHintMap().get(JoinOrderHint.category); + Set leadingTables = + leading.getTables().stream().map(Identifier::new).collect(Collectors.toSet()); + + Set leftTables = node.getLeftTables(); + Set rightTables = node.getRightTables(); + Set totalJoinTables = ImmutableSet.of(); + + // join conjunctions + List criteria = node.getCriteria(); + for (JoinNode.EquiJoinClause equiJoin : criteria) { + Set equiJoinTables = Sets.intersection(leadingTables, equiJoin.getTables()); + totalJoinTables = Sets.union(totalJoinTables, equiJoinTables); + if (node.getJoinType() == JoinNode.JoinType.LEFT) { + // do something + } + leading.getFilters().add(new Pair<>(equiJoinTables, equiJoin.toExpression())); + // leading.putConditionJoinType(expression, join.getJoinType()); + } + collectJoinConstraintList(leading, leftTables, rightTables, node, totalJoinTables); + return node; } @@ -69,5 +110,22 @@ public PlanNode visitJoin(JoinNode node, Context context) { // public PlanNode visitSemiJoin(SemiJoinNode node, Context context) { // return visitTwoChildProcess(node, context); // } + + private void collectJoinConstraintList( + LeadingHint leading, + Set leftHand, + Set rightHand, + JoinNode join, + Set joinTables) { + Set totalTables = Sets.union(leftHand, rightHand); + if (join.getJoinType() == JoinNode.JoinType.INNER) { + leading.setInnerJoinTables(Sets.union(leading.getInnerJoinTables(), totalTables)); + return; + } + if (join.getJoinType() == JoinNode.JoinType.FULL) { + // full join + return; + } + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java index 14c3d1fd261f8..3bd3ebcdbd021 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java @@ -24,14 +24,16 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.JoinOrderHint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeadingHint; -import java.util.Set; +import com.google.common.collect.Sets; -import static com.google.common.collect.ImmutableSet.toImmutableSet; +import java.util.Set; +import java.util.stream.Collectors; public class LeadingJoinOptimizer implements PlanOptimizer { @Override @@ -56,13 +58,22 @@ public PlanNode visitPlan(PlanNode node, Context context) { if (!(hint instanceof LeadingHint)) { return node; } - LeadingHint leadingHint = (LeadingHint) hint; - Set relationNames = - analysis.getRelationNames().values().stream() - .map(QualifiedName::getSuffix) - .collect(toImmutableSet()); - if (!validateTableNamesMatch(relationNames, leadingHint)) { + PlanNode newNode = node.clone(); + for (PlanNode child : node.getChildren()) { + newNode.addChild(child.accept(this, context)); + } + return newNode; + } + + @Override + public PlanNode visitJoin(JoinNode node, Context context) { + LeadingHint leadingHint = (LeadingHint) analysis.getHintMap().get(JoinOrderHint.category); + Set currentTables = Sets.union(node.getLeftTables(), node.getRightTables()); + Set leadingTables = + leadingHint.getTables().stream().map(Identifier::new).collect(Collectors.toSet()); + + if (!currentTables.equals(leadingTables)) { return node; } @@ -73,11 +84,10 @@ public PlanNode visitPlan(PlanNode node, Context context) { return node; } - private boolean validateTableNamesMatch(Set relationNames, LeadingHint leadingHint) { - if (relationNames.size() != leadingHint.getTables().size()) { - return false; - } - return relationNames.containsAll(leadingHint.getTables()); - } + // @Override + // public PlanNode visitSemiJoin(SemiJoinNode node, Context context) { + // return visitTwoChildProcess(node, context); + // } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index 546eab97f318d..0f6a173d6891d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -284,6 +284,8 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new RemoveRedundantIdentityProjections())) .build()), simplifyOptimizer, + new CollectJoinConstraint(), + new LeadingJoinOptimizer(), new UnaliasSymbolReferences(plannerContext.getMetadata()), new IterativeOptimizer( plannerContext, @@ -388,9 +390,7 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new MergeLimitWithSort(), new MergeLimitOverProjectWithSort(), new PushTopKThroughUnion())), - new ParallelizeGrouping(), - new CollectJoinConstraint()); - // new LeadingJoinOptimizer()); + new ParallelizeGrouping()); this.planOptimizers = optimizerBuilder.build(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java index 7cca5eddb0d10..856efcec77f01 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java @@ -25,12 +25,22 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import org.apache.tsfile.utils.Pair; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.Stack; @@ -44,6 +54,11 @@ public class LeadingHint extends JoinOrderHint { private List addJoinParameters; private List normalizedParameters; + private final List, Expression>> filters = new ArrayList<>(); + private final Map relationToScanMap = new HashMap<>(); + + private Set innerJoinTables = ImmutableSet.of(); + public LeadingHint(List parameters) { super(hintName); // /* leading(t3 {}) 会报错 @@ -68,6 +83,22 @@ public List getTables() { return tables; } + public List, Expression>> getFilters() { + return filters; + } + + public Map getRelationToScanMap() { + return relationToScanMap; + } + + public Set getInnerJoinTables() { + return innerJoinTables; + } + + public void setInnerJoinTables(Set innerJoinTables) { + this.innerJoinTables = innerJoinTables; + } + public PlanNode generateLeadingJoinPlan() { Stack stack = new Stack<>(); for (String item : normalizedParameters) { @@ -80,9 +111,12 @@ public PlanNode generateLeadingJoinPlan() { } stack.push(joinPlan); } else { - // PlanNode logicalPlan = getLogicalPlanByName(item); - // ogicalPlan = makeFilterPlanIfExist(getFilters(), logicalPlan); - // stack.push(logicalPlan); + PlanNode logicalPlan = getPlanByName(item); + if (logicalPlan == null) { + return null; + } + logicalPlan = makeFilterPlanIfExist(getFilters(), logicalPlan); + stack.push(logicalPlan); } } @@ -166,54 +200,107 @@ public List parseIntoReversePolishNotation(List list) { return s2; } + public PlanNode getPlanByName(String name) { + if (!relationToScanMap.containsKey(name)) { + return null; + } + return relationToScanMap.get(name); + } + private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { - // List conditions = getJoinConditions( - // getFilters(), leftChild, rightChild); - // Pair, List> pair = JoinUtils.extractExpressionForHashTable( - // leftChild.getOutput(), rightChild.getOutput(), conditions); - // // leading hint would set status inside if not success - // JoinType joinType = computeJoinType(getBitmap(leftChild), - // getBitmap(rightChild), conditions); - // if (joinType == null) { - // this.setStatus(HintStatus.SYNTAX_ERROR); - // this.setErrorMessage("JoinType can not be null"); - // } else if (!isConditionJoinTypeMatched(conditions, joinType)) { - // this.setStatus(HintStatus.UNUSED); - // this.setErrorMessage("condition does not matched joinType"); - // } - // if (!this.isSuccess()) { - // return null; - // } - // get joinType - // LogicalJoin logicalJoin = new LogicalJoin<>(joinType, pair.first, - // pair.second, - // distributeHint, - // Optional.empty(), - // leftChild, - // rightChild, null); - // logicalJoin.getJoinReorderContext().setLeadingJoin(true); - // logicalJoin.setBitmap(LongBitmap.or(getBitmap(leftChild), getBitmap(rightChild))); - - PlanNode joinNode = planJoin(leftChild, rightChild); - return joinNode; - } - - private PlanNode planJoin(PlanNode leftPlan, PlanNode rightPlan) { - List leftOutputSymbols = leftPlan.getOutputSymbols(); - List rightOutputSymbols = rightPlan.getOutputSymbols(); - List criteria = new ArrayList<>(); + List conditions = getJoinConditions(getFilters(), leftChild, rightChild); + JoinNode.JoinType joinType = computeJoinType(); + if (joinType == null) { + return null; + } else if (!isConditionJoinTypeMatched()) { + return null; + } + + List leftOutputSymbols = leftChild.getOutputSymbols(); + List rightOutputSymbols = rightChild.getOutputSymbols(); Optional asofCriteria = Optional.empty(); + List criteria = new ArrayList<>(); + + for (Expression conjunct : conditions) { + ComparisonExpression equality = (ComparisonExpression) conjunct; + Symbol leftSymbol = Symbol.from(equality.getLeft()); + Symbol rightSymbol = Symbol.from(equality.getRight()); + + if (leftOutputSymbols.contains(leftSymbol) && rightOutputSymbols.contains(rightSymbol)) { + criteria.add(new JoinNode.EquiJoinClause(leftSymbol, rightSymbol)); + } else if (leftOutputSymbols.contains(rightSymbol) + && rightOutputSymbols.contains(leftSymbol)) { + criteria.add(new JoinNode.EquiJoinClause(rightSymbol, leftSymbol)); + } else { + throw new IllegalArgumentException("Invalid join condition"); + } + } return new JoinNode( new PlanNodeId("join"), - JoinNode.JoinType.INNER, - leftPlan, - rightPlan, + joinType, + leftChild, + rightChild, criteria, asofCriteria, leftOutputSymbols, rightOutputSymbols, Optional.empty(), - Optional.empty()); + Optional.empty(), + getTables(leftChild), + getTables(rightChild)); + } + + private List getJoinConditions( + List, Expression>> filters, PlanNode left, PlanNode right) { + List joinConditions = new ArrayList<>(); + for (int i = filters.size() - 1; i >= 0; i--) { + Pair, Expression> filterPair = filters.get(i); + Set tables = Sets.union(getTables(left), getTables(right)); + // left one is smaller set + if (tables.containsAll(filterPair.left)) { + joinConditions.add(filterPair.right); + filters.remove(i); + } + } + return joinConditions; + } + + private PlanNode makeFilterPlanIfExist( + List, Expression>> filters, PlanNode plan) { + if (filters.isEmpty()) { + return plan; + } + for (int i = filters.size() - 1; i >= 0; i--) { + Pair, Expression> filterPair = filters.get(i); + if (getTables(plan).containsAll(filterPair.left)) { + plan = new FilterNode(plan.getPlanNodeId(), plan, filterPair.right); + filters.remove(i); + } + } + return plan; + } + + private Set getTables(PlanNode root) { + if (root instanceof JoinNode) { + return Sets.union(((JoinNode) root).getLeftTables(), ((JoinNode) root).getRightTables()); + } else if (root instanceof DeviceTableScanNode) { + return ImmutableSet.of( + new Identifier(((DeviceTableScanNode) root).getQualifiedObjectName().getObjectName())); + } else if (root instanceof FilterNode) { + return getTables(((FilterNode) root).getChild()); + } else if (root instanceof ProjectNode) { + return getTables(((ProjectNode) root).getChild()); + } else { + return null; + } + } + + public JoinNode.JoinType computeJoinType() { + return JoinNode.JoinType.INNER; + } + + public boolean isConditionJoinTypeMatched() { + return true; } } From 86c881aa68279af5ee3d397bf4d3abec4054d057 Mon Sep 17 00:00:00 2001 From: shizy Date: Mon, 9 Feb 2026 15:04:14 +0800 Subject: [PATCH 15/20] left & right outer join --- .../plan/planner/plan/node/PlanNode.java | 12 ++ .../relational/planner/RelationPlanner.java | 12 +- .../iterative/rule/PruneJoinColumns.java | 4 +- .../relational/planner/node/JoinNode.java | 4 +- .../planner/node/TableScanNode.java | 9 + .../optimizations/CollectJoinConstraint.java | 86 +++++++- .../PushPredicateIntoTableScan.java | 16 +- .../UnaliasSymbolReferences.java | 4 +- .../relational/utils/hint/JoinConstraint.java | 80 ++++++++ .../relational/utils/hint/LeadingHint.java | 192 +++++++++++++++--- 10 files changed, 350 insertions(+), 69 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java index 4e3196c8d42c4..9746d06d2a10a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java @@ -23,6 +23,7 @@ import org.apache.iotdb.consensus.common.request.IConsensusRequest; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -32,8 +33,10 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; import static java.util.Objects.requireNonNull; @@ -78,6 +81,15 @@ public void markAsGeneratedByPipe() { public abstract void addChild(PlanNode child); + public Set getInputTables() { + Set tables = new HashSet<>(); + for (PlanNode child : getChildren()) { + tables.addAll(child.getInputTables()); + } + ; + return tables; + } + /** * If this plan node has to be serialized or deserialized, override this method. If this method is * overridden, the serialization and deserialization methods must be implemented. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 6bc78efb6896d..1d0dab122f036 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -537,9 +537,7 @@ If casts are redundant (due to column type and common type being equal), leftCoercion.getOutputSymbols(), rightCoercion.getOutputSymbols(), Optional.empty(), - Optional.empty(), - left.getScope().getTables(), - right.getScope().getTables()); + Optional.empty()); // Transform RIGHT JOIN to LEFT if (join.getJoinType() == JoinNode.JoinType.RIGHT) { join = join.flip(); @@ -779,9 +777,7 @@ public RelationPlan planJoin( leftPlanBuilder.getRoot().getOutputSymbols(), rightPlanBuilder.getRoot().getOutputSymbols(), Optional.empty(), - Optional.empty(), - leftPlan.getScope().getTables(), - rightPlan.getScope().getTables()); + Optional.empty()); if (type == RIGHT && asofCriteria == null) { root = ((JoinNode) root).flip(); } @@ -836,9 +832,7 @@ public RelationPlan planJoin( complexJoinExpressions.stream() .map(e -> coerceIfNecessary(analysis, e, translationMap.rewrite(e))) .collect(Collectors.toList()))), - Optional.empty(), - leftPlan.getScope().getTables(), - rightPlan.getScope().getTables()); + Optional.empty()); if (type == RIGHT && asofCriteria == null) { root = ((JoinNode) root).flip(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java index 21f3df06ec7a8..c09710346ed09 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java @@ -48,8 +48,6 @@ protected Optional pushDownProjectOff( filteredCopy(joinNode.getLeftOutputSymbols(), referencedOutputs::contains), filteredCopy(joinNode.getRightOutputSymbols(), referencedOutputs::contains), joinNode.getFilter(), - joinNode.isSpillable(), - joinNode.getLeftTables(), - joinNode.getRightTables())); + joinNode.isSpillable())); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index aeaf4d0dbb7c9..09994aaeb7704 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -168,8 +168,8 @@ public JoinNode( rightOutputSymbols, filter, spillable, - ImmutableSet.of(), - ImmutableSet.of()); + leftChild.getInputTables(), + rightChild.getInputTables()); } // only used for deserialize diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java index c6bda2232621c..dff5887961994 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java @@ -43,10 +43,12 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -114,6 +116,13 @@ public R accept(PlanVisitor visitor, C context) { return visitor.visitTableScan(this, context); } + @Override + public Set getInputTables() { + Set tables = new HashSet<>(); + tables.add(alias != null ? alias : new Identifier(qualifiedObjectName.getObjectName())); + return tables; + } + @Override public List getChildren() { return ImmutableList.of(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java index 5450c9b60f8fc..91b52a450e128 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.JoinConstraint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.JoinOrderHint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeadingHint; @@ -86,8 +87,8 @@ public PlanNode visitJoin(JoinNode node, Context context) { Set leadingTables = leading.getTables().stream().map(Identifier::new).collect(Collectors.toSet()); - Set leftTables = node.getLeftTables(); - Set rightTables = node.getRightTables(); + Set leftHand = node.getLeftTables(); + Set rightHand = node.getRightTables(); Set totalJoinTables = ImmutableSet.of(); // join conjunctions @@ -96,21 +97,25 @@ public PlanNode visitJoin(JoinNode node, Context context) { Set equiJoinTables = Sets.intersection(leadingTables, equiJoin.getTables()); totalJoinTables = Sets.union(totalJoinTables, equiJoinTables); if (node.getJoinType() == JoinNode.JoinType.LEFT) { - // do something + /* + SELECT * + FROM A + LEFT JOIN B ON A.id = B.id + LEFT JOIN C ON A.id = C.id; + + join conjunction A.id = C.id（A ⋈ B ⋈ C）, and join table set is {A, B, C}. + */ + equiJoinTables = Sets.union(equiJoinTables, leftHand); } leading.getFilters().add(new Pair<>(equiJoinTables, equiJoin.toExpression())); - // leading.putConditionJoinType(expression, join.getJoinType()); + leading.putConditionJoinType(equiJoin.toExpression(), node.getJoinType()); } - collectJoinConstraintList(leading, leftTables, rightTables, node, totalJoinTables); + // join constraint + collectJoinConstraintList(leading, leftHand, rightHand, node, totalJoinTables); return node; } - // @Override - // public PlanNode visitSemiJoin(SemiJoinNode node, Context context) { - // return visitTwoChildProcess(node, context); - // } - private void collectJoinConstraintList( LeadingHint leading, Set leftHand, @@ -123,9 +128,68 @@ private void collectJoinConstraintList( return; } if (join.getJoinType() == JoinNode.JoinType.FULL) { - // full join + JoinConstraint joinConstraint = + new JoinConstraint(leftHand, rightHand, leftHand, rightHand, JoinNode.JoinType.FULL); + leading.getJoinConstraintList().add(joinConstraint); return; } + Set minLeftHand = Sets.intersection(joinTables, leftHand); + Set innerJoinTables = + Sets.intersection(totalTables, leading.getInnerJoinTables()); + Set filterAndInnerBelow = Sets.union(joinTables, innerJoinTables); + Set minRightHand = Sets.intersection(filterAndInnerBelow, rightHand); + + for (JoinConstraint other : leading.getJoinConstraintList()) { + if (other.getJoinType() == JoinNode.JoinType.FULL) { + if (isOverlap(leftHand, other.getLeftHand()) + || isOverlap(leftHand, other.getRightHand())) { + minLeftHand = Sets.union(minLeftHand, other.getLeftHand()); + minLeftHand = Sets.union(minLeftHand, other.getRightHand()); + } + + if (isOverlap(rightHand, other.getLeftHand()) + || isOverlap(rightHand, other.getRightHand())) { + minRightHand = Sets.union(minRightHand, other.getLeftHand()); + minRightHand = Sets.union(minRightHand, other.getRightHand()); + } + /* Needn't do anything else with the full join */ + continue; + } + + // 当前join的左表包含之前某个join的右表 & join条件包含之前某个join的右表 + if (isOverlap(leftHand, other.getRightHand()) + && isOverlap(joinTables, other.getRightHand())) { + minLeftHand = Sets.union(minLeftHand, other.getLeftHand()); + minLeftHand = Sets.union(minLeftHand, other.getRightHand()); + } + + // 当前join的右表包含之前某个join的右表 + if (isOverlap(rightHand, other.getRightHand())) { + if (isOverlap(joinTables, other.getRightHand()) + || !isOverlap(joinTables, other.getMinLeftHand())) { + minRightHand = Sets.union(minRightHand, other.getLeftHand()); + minRightHand = Sets.union(minRightHand, other.getRightHand()); + } + } + } + if (minLeftHand.isEmpty()) { + minLeftHand = leftHand; + } + if (minRightHand.isEmpty()) { + minRightHand = rightHand; + } + + JoinConstraint joinConstraint = + new JoinConstraint( + minLeftHand, minRightHand, minLeftHand, minRightHand, join.getJoinType()); + leading.getJoinConstraintList().add(joinConstraint); + } + + private boolean isOverlap(Set set1, Set set2) { + if (set1 == null || set2 == null || set1.isEmpty() || set2.isEmpty()) { + return false; + } + return !Sets.intersection(set1, set2).isEmpty(); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 9d7f9417f4993..853d88027f8d7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -956,9 +956,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { leftSource.getOutputSymbols(), rightSource.getOutputSymbols(), newJoinFilter, - node.isSpillable(), - node.getLeftTables(), - node.getRightTables()); + node.isSpillable()); } JoinNode outputJoinNode = (JoinNode) output; @@ -1361,9 +1359,7 @@ private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, Expression inheri node.getLeftOutputSymbols(), node.getRightOutputSymbols(), node.getFilter(), - node.isSpillable(), - node.getLeftTables(), - node.getRightTables()); + node.isSpillable()); } if (canConvertToLeftJoin) { return new JoinNode( @@ -1376,9 +1372,7 @@ private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, Expression inheri node.getLeftOutputSymbols(), node.getRightOutputSymbols(), node.getFilter(), - node.isSpillable(), - node.getLeftTables(), - node.getRightTables()); + node.isSpillable()); } else { // temp fix because right join is not supported for now. return node; @@ -1413,9 +1407,7 @@ private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, Expression inheri node.getLeftOutputSymbols(), node.getRightOutputSymbols(), node.getFilter(), - node.isSpillable(), - node.getLeftTables(), - node.getRightTables()); + node.isSpillable()); } private boolean canConvertOuterToInner( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index 5ac1a82022a1c..5748f539879f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -858,9 +858,7 @@ public PlanAndMappings visitJoin(JoinNode node, UnaliasContext context) { newLeftOutputSymbols, newRightOutputSymbols, newFilter, - node.isSpillable(), - node.getLeftTables(), - node.getRightTables()), + node.isSpillable()), outputMapping); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java new file mode 100644 index 0000000000000..547929479dc20 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java @@ -0,0 +1,80 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; + +import java.util.Set; + +public class JoinConstraint { + private final Set minLeftHand; + private final Set minRightHand; + + private final Set leftHand; + private final Set rightHand; + + private final JoinNode.JoinType joinType; + + private boolean isReversed; + + public JoinConstraint( + Set minLeftHand, + Set minRightHand, + Set leftHand, + Set rightHand, + JoinNode.JoinType joinType) { + this.minLeftHand = minLeftHand; + this.minRightHand = minRightHand; + this.leftHand = leftHand; + this.rightHand = rightHand; + this.joinType = joinType; + } + + public JoinNode.JoinType getJoinType() { + return joinType; + } + + public Set getLeftHand() { + return leftHand; + } + + public Set getRightHand() { + return rightHand; + } + + public Set getMinLeftHand() { + return minLeftHand; + } + + public Set getMinRightHand() { + return minRightHand; + } + + public void setReversed(boolean reversed) { + isReversed = reversed; + } + + public boolean isReversed() { + return isReversed; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java index 856efcec77f01..5e65fb3943826 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java @@ -25,15 +25,14 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.tsfile.utils.Pair; @@ -59,6 +58,10 @@ public class LeadingHint extends JoinOrderHint { private Set innerJoinTables = ImmutableSet.of(); + private final List joinConstraintList = new ArrayList<>(); + + private final Map conditionJoinType = Maps.newLinkedHashMap(); + public LeadingHint(List parameters) { super(hintName); // /* leading(t3 {}) 会报错 @@ -83,10 +86,18 @@ public List getTables() { return tables; } + public List getJoinConstraintList() { + return joinConstraintList; + } + public List, Expression>> getFilters() { return filters; } + public void putConditionJoinType(Expression filter, JoinNode.JoinType joinType) { + conditionJoinType.put(filter, joinType); + } + public Map getRelationToScanMap() { return relationToScanMap; } @@ -122,11 +133,10 @@ public PlanNode generateLeadingJoinPlan() { PlanNode finalJoin = stack.pop(); // we want all filters been removed - // if (Utils.enableAssert && !filters.isEmpty()) { - // throw new IllegalStateException( - // "Leading hint process failed: filter should be empty, but meet: " + filters - // ); - // } + if (!filters.isEmpty()) { + throw new IllegalStateException( + "Leading hint process failed: filter should be empty, but meet: " + filters); + } if (finalJoin == null) { throw new IoTDBRuntimeException( "final join plan should not be null", INTERNAL_SERVER_ERROR.getStatusCode()); @@ -209,10 +219,11 @@ public PlanNode getPlanByName(String name) { private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { List conditions = getJoinConditions(getFilters(), leftChild, rightChild); - JoinNode.JoinType joinType = computeJoinType(); + JoinNode.JoinType joinType = + computeJoinType(leftChild.getInputTables(), rightChild.getInputTables(), conditions); if (joinType == null) { return null; - } else if (!isConditionJoinTypeMatched()) { + } else if (!isConditionJoinTypeMatched(conditions, joinType)) { return null; } @@ -246,9 +257,7 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { leftOutputSymbols, rightOutputSymbols, Optional.empty(), - Optional.empty(), - getTables(leftChild), - getTables(rightChild)); + Optional.empty()); } private List getJoinConditions( @@ -256,7 +265,7 @@ private List getJoinConditions( List joinConditions = new ArrayList<>(); for (int i = filters.size() - 1; i >= 0; i--) { Pair, Expression> filterPair = filters.get(i); - Set tables = Sets.union(getTables(left), getTables(right)); + Set tables = Sets.union(left.getInputTables(), right.getInputTables()); // left one is smaller set if (tables.containsAll(filterPair.left)) { joinConditions.add(filterPair.right); @@ -273,7 +282,7 @@ private PlanNode makeFilterPlanIfExist( } for (int i = filters.size() - 1; i >= 0; i--) { Pair, Expression> filterPair = filters.get(i); - if (getTables(plan).containsAll(filterPair.left)) { + if (plan.getInputTables().containsAll(filterPair.left)) { plan = new FilterNode(plan.getPlanNodeId(), plan, filterPair.right); filters.remove(i); } @@ -281,26 +290,151 @@ private PlanNode makeFilterPlanIfExist( return plan; } - private Set getTables(PlanNode root) { - if (root instanceof JoinNode) { - return Sets.union(((JoinNode) root).getLeftTables(), ((JoinNode) root).getRightTables()); - } else if (root instanceof DeviceTableScanNode) { - return ImmutableSet.of( - new Identifier(((DeviceTableScanNode) root).getQualifiedObjectName().getObjectName())); - } else if (root instanceof FilterNode) { - return getTables(((FilterNode) root).getChild()); - } else if (root instanceof ProjectNode) { - return getTables(((ProjectNode) root).getChild()); - } else { + // private Set getTables(PlanNode root) { + // if (root instanceof JoinNode) { + // return Sets.union(((JoinNode) root).getLeftTables(), ((JoinNode) root).getRightTables()); + // } else if (root instanceof DeviceTableScanNode) { + // return ImmutableSet.of( + // new Identifier(((DeviceTableScanNode) + // root).getQualifiedObjectName().getObjectName())); + // } else if (root instanceof FilterNode) { + // return getTables(((FilterNode) root).getChild()); + // } else if (root instanceof ProjectNode) { + // return getTables(((ProjectNode) root).getChild()); + // } else { + // return null; + // } + // } + + public JoinNode.JoinType computeJoinType( + Set left, Set right, List conditions) { + Pair joinConstraintBooleanPair = + getJoinConstraint(Sets.union(left, right), left, right); + if (!joinConstraintBooleanPair.right) { return null; + } else if (joinConstraintBooleanPair.left == null) { + return JoinNode.JoinType.INNER; + } else { + JoinConstraint joinConstraint = joinConstraintBooleanPair.left; + if (joinConstraint.isReversed()) { + JoinNode.JoinType joinType = joinConstraint.getJoinType(); + if (joinType == JoinNode.JoinType.LEFT) { + return JoinNode.JoinType.RIGHT; + } else if (joinType == JoinNode.JoinType.RIGHT) { + return JoinNode.JoinType.LEFT; + } else { + return joinType; + } + } } - } - - public JoinNode.JoinType computeJoinType() { return JoinNode.JoinType.INNER; } - public boolean isConditionJoinTypeMatched() { + public boolean isConditionJoinTypeMatched( + List conditions, JoinNode.JoinType joinType) { + for (Expression condition : conditions) { + JoinNode.JoinType originalJoinType = conditionJoinType.get(condition); + if (originalJoinType == joinType + || (originalJoinType == JoinNode.JoinType.LEFT && joinType == JoinNode.JoinType.RIGHT) + || (originalJoinType == JoinNode.JoinType.RIGHT && joinType == JoinNode.JoinType.LEFT)) { + continue; + } + return false; + } return true; } + + public Pair getJoinConstraint( + Set joinTables, Set leftHand, Set rightHand) { + boolean reversed = false; + boolean mustBeLeftJoin = false; + + JoinConstraint matchedJoinConstraint = null; + for (JoinConstraint joinConstraint : joinConstraintList) { + if (joinConstraint.getJoinType() == JoinNode.JoinType.FULL) { + if ((isEqual(joinConstraint.getLeftHand(), leftHand) + && isEqual(joinConstraint.getRightHand(), rightHand)) + || (isEqual(joinConstraint.getLeftHand(), rightHand) + && isEqual(joinConstraint.getRightHand(), leftHand))) { + if (matchedJoinConstraint != null) { + return new Pair<>(null, false); + } + matchedJoinConstraint = joinConstraint; + reversed = false; + break; + } else { + continue; + } + } + + if (!isOverlap(joinConstraint.getRightHand(), joinTables)) { + continue; + } + + if (joinTables.containsAll(joinConstraint.getMinRightHand())) { + continue; + } + + if ((joinConstraint.getMinLeftHand().containsAll(leftHand)) + && joinConstraint.getMinRightHand().containsAll(leftHand)) { + continue; + } + + if (joinConstraint.getMinLeftHand().containsAll(rightHand) + && joinConstraint.getMinRightHand().containsAll(rightHand)) { + continue; + } + + if (joinConstraint.getMinLeftHand().containsAll(leftHand) + && joinConstraint.getMinRightHand().containsAll(rightHand)) { + if (matchedJoinConstraint != null) { + return new Pair<>(null, false); + } + matchedJoinConstraint = joinConstraint; + reversed = false; + } else if (joinConstraint.getMinLeftHand().containsAll(rightHand) + && joinConstraint.getMinRightHand().containsAll(leftHand)) { + if (matchedJoinConstraint != null) { + return new Pair<>(null, false); + } + matchedJoinConstraint = joinConstraint; + reversed = true; + } else { + if (isOverlap(leftHand, joinConstraint.getMinRightHand()) + && isOverlap(rightHand, joinConstraint.getRightHand())) { + continue; + } + if (joinConstraint.getJoinType() == JoinNode.JoinType.LEFT + || isOverlap(joinTables, joinConstraint.getMinLeftHand())) { + return new Pair<>(null, false); + } + mustBeLeftJoin = true; + } + } + if (mustBeLeftJoin + && (matchedJoinConstraint == null + || matchedJoinConstraint.getJoinType() != JoinNode.JoinType.LEFT)) { + return new Pair<>(null, false); + } + // inner join + if (matchedJoinConstraint == null) { + return new Pair<>(null, true); + } + matchedJoinConstraint.setReversed(reversed); + return new Pair<>(matchedJoinConstraint, true); + } + + private boolean isEqual(Set set1, Set set2) { + if (set1 == null || set2 == null) { + return set1 == set2; + } + return set1.size() == set2.size() && set1.containsAll(set2); + } + + private boolean isOverlap(Set set1, Set set2) { + if (set1 == null || set2 == null || set1.isEmpty() || set2.isEmpty()) { + return false; + } + return !Sets.intersection(set1, set2).isEmpty(); + } } From 263053aed4f156f17cff497d4f557b344b460433 Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 10 Feb 2026 09:28:28 +0800 Subject: [PATCH 16/20] fix leading hint --- ...TransformFilteringSemiJoinToInnerJoin.java | 5 +- .../relational/planner/node/JoinNode.java | 8 -- .../optimizations/CollectJoinConstraint.java | 22 ++-- .../relational/utils/hint/JoinConstraint.java | 10 -- .../relational/utils/hint/LeadingHint.java | 105 ++++++++---------- 5 files changed, 60 insertions(+), 90 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java index 1f32fdbffe157..10f2eb51dd0ef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java @@ -130,7 +130,10 @@ public Result apply(FilterNode filterNode, Captures captures, Context context) { filteringSourceDistinct, ImmutableList.of( new JoinNode.EquiJoinClause( - semiJoin.getSourceJoinSymbol(), semiJoin.getFilteringSourceJoinSymbol())), + semiJoin.getSourceJoinSymbol(), + semiJoin.getFilteringSourceJoinSymbol(), + semiJoin.getSource().getInputTables(), + semiJoin.getFilteringSource().getInputTables())), Optional.empty(), semiJoin.getSource().getOutputSymbols(), ImmutableList.of(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index 09994aaeb7704..a0521bb7e5373 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -501,14 +501,6 @@ public static class EquiJoinClause { private final Set rightTables; private final Set tables; - public EquiJoinClause(Symbol left, Symbol right) { - this.left = requireNonNull(left, "left is null"); - this.right = requireNonNull(right, "right is null"); - this.leftTables = ImmutableSet.of(); - this.rightTables = ImmutableSet.of(); - this.tables = ImmutableSet.of(); - } - public EquiJoinClause( Symbol left, Symbol right, Set leftTables, Set rightTables) { this.left = requireNonNull(left, "left is null"); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java index 91b52a450e128..d1768e4eeca5a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java @@ -97,15 +97,14 @@ public PlanNode visitJoin(JoinNode node, Context context) { Set equiJoinTables = Sets.intersection(leadingTables, equiJoin.getTables()); totalJoinTables = Sets.union(totalJoinTables, equiJoinTables); if (node.getJoinType() == JoinNode.JoinType.LEFT) { - /* - SELECT * - FROM A - LEFT JOIN B ON A.id = B.id - LEFT JOIN C ON A.id = C.id; - - join conjunction A.id = C.id（A ⋈ B ⋈ C）, and join table set is {A, B, C}. - */ - equiJoinTables = Sets.union(equiJoinTables, leftHand); + // leading.getFilters() is used to determine which join the join condition should apply + // to. + // For LEFT join, we need to add rightHand tables as well. In the getJoinConditions + // function, + // we ensure that the tables joined by the Join include both the tables in the join + // condition + // and the right tables of the outer join, so that the join condition can be applied. + equiJoinTables = Sets.union(equiJoinTables, rightHand); } leading.getFilters().add(new Pair<>(equiJoinTables, equiJoin.toExpression())); leading.putConditionJoinType(equiJoin.toExpression(), node.getJoinType()); @@ -156,14 +155,15 @@ private void collectJoinConstraintList( continue; } - // 当前join的左表包含之前某个join的右表 & join条件包含之前某个join的右表 + // Current join's left table contains the right table of a previous join & join condition + // contains the right table of a previous join if (isOverlap(leftHand, other.getRightHand()) && isOverlap(joinTables, other.getRightHand())) { minLeftHand = Sets.union(minLeftHand, other.getLeftHand()); minLeftHand = Sets.union(minLeftHand, other.getRightHand()); } - // 当前join的右表包含之前某个join的右表 + // Current join's right table contains the right table of a previous join if (isOverlap(rightHand, other.getRightHand())) { if (isOverlap(joinTables, other.getRightHand()) || !isOverlap(joinTables, other.getMinLeftHand())) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java index 547929479dc20..243308cb6dd67 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java @@ -35,8 +35,6 @@ public class JoinConstraint { private final JoinNode.JoinType joinType; - private boolean isReversed; - public JoinConstraint( Set minLeftHand, Set minRightHand, @@ -69,12 +67,4 @@ public Set getMinLeftHand() { public Set getMinRightHand() { return minRightHand; } - - public void setReversed(boolean reversed) { - isReversed = reversed; - } - - public boolean isReversed() { - return isReversed; - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java index 5e65fb3943826..5b4566b2451ca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java @@ -220,7 +220,7 @@ public PlanNode getPlanByName(String name) { private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { List conditions = getJoinConditions(getFilters(), leftChild, rightChild); JoinNode.JoinType joinType = - computeJoinType(leftChild.getInputTables(), rightChild.getInputTables(), conditions); + computeJoinType(leftChild.getInputTables(), rightChild.getInputTables()); if (joinType == null) { return null; } else if (!isConditionJoinTypeMatched(conditions, joinType)) { @@ -238,10 +238,14 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { Symbol rightSymbol = Symbol.from(equality.getRight()); if (leftOutputSymbols.contains(leftSymbol) && rightOutputSymbols.contains(rightSymbol)) { - criteria.add(new JoinNode.EquiJoinClause(leftSymbol, rightSymbol)); + criteria.add( + new JoinNode.EquiJoinClause( + leftSymbol, rightSymbol, leftChild.getInputTables(), rightChild.getInputTables())); } else if (leftOutputSymbols.contains(rightSymbol) && rightOutputSymbols.contains(leftSymbol)) { - criteria.add(new JoinNode.EquiJoinClause(rightSymbol, leftSymbol)); + criteria.add( + new JoinNode.EquiJoinClause( + rightSymbol, leftSymbol, leftChild.getInputTables(), rightChild.getInputTables())); } else { throw new IllegalArgumentException("Invalid join condition"); } @@ -266,7 +270,7 @@ private List getJoinConditions( for (int i = filters.size() - 1; i >= 0; i--) { Pair, Expression> filterPair = filters.get(i); Set tables = Sets.union(left.getInputTables(), right.getInputTables()); - // left one is smaller set + // it should contain all tables in join conjunctions & right tables if it's left join if (tables.containsAll(filterPair.left)) { joinConditions.add(filterPair.right); filters.remove(i); @@ -290,53 +294,25 @@ private PlanNode makeFilterPlanIfExist( return plan; } - // private Set getTables(PlanNode root) { - // if (root instanceof JoinNode) { - // return Sets.union(((JoinNode) root).getLeftTables(), ((JoinNode) root).getRightTables()); - // } else if (root instanceof DeviceTableScanNode) { - // return ImmutableSet.of( - // new Identifier(((DeviceTableScanNode) - // root).getQualifiedObjectName().getObjectName())); - // } else if (root instanceof FilterNode) { - // return getTables(((FilterNode) root).getChild()); - // } else if (root instanceof ProjectNode) { - // return getTables(((ProjectNode) root).getChild()); - // } else { - // return null; - // } - // } - - public JoinNode.JoinType computeJoinType( - Set left, Set right, List conditions) { + public JoinNode.JoinType computeJoinType(Set left, Set right) { Pair joinConstraintBooleanPair = getJoinConstraint(Sets.union(left, right), left, right); if (!joinConstraintBooleanPair.right) { return null; - } else if (joinConstraintBooleanPair.left == null) { + } + if (joinConstraintBooleanPair.left == null) { return JoinNode.JoinType.INNER; - } else { - JoinConstraint joinConstraint = joinConstraintBooleanPair.left; - if (joinConstraint.isReversed()) { - JoinNode.JoinType joinType = joinConstraint.getJoinType(); - if (joinType == JoinNode.JoinType.LEFT) { - return JoinNode.JoinType.RIGHT; - } else if (joinType == JoinNode.JoinType.RIGHT) { - return JoinNode.JoinType.LEFT; - } else { - return joinType; - } - } } - return JoinNode.JoinType.INNER; + + JoinConstraint joinConstraint = joinConstraintBooleanPair.left; + return joinConstraint.getJoinType(); } public boolean isConditionJoinTypeMatched( List conditions, JoinNode.JoinType joinType) { for (Expression condition : conditions) { JoinNode.JoinType originalJoinType = conditionJoinType.get(condition); - if (originalJoinType == joinType - || (originalJoinType == JoinNode.JoinType.LEFT && joinType == JoinNode.JoinType.RIGHT) - || (originalJoinType == JoinNode.JoinType.RIGHT && joinType == JoinNode.JoinType.LEFT)) { + if (originalJoinType == joinType) { continue; } return false; @@ -344,9 +320,14 @@ public boolean isConditionJoinTypeMatched( return true; } + /** + * try to get join constraint. If it can not be found, it means join is inner join + * + * @return boolean value used for judging whether the join is legal, and should this join need to + * reverse + */ public Pair getJoinConstraint( Set joinTables, Set leftHand, Set rightHand) { - boolean reversed = false; boolean mustBeLeftJoin = false; JoinConstraint matchedJoinConstraint = null; @@ -360,54 +341,59 @@ && isEqual(joinConstraint.getRightHand(), leftHand))) { return new Pair<>(null, false); } matchedJoinConstraint = joinConstraint; - reversed = false; break; } else { continue; } } - if (!isOverlap(joinConstraint.getRightHand(), joinTables)) { + // join操作完全不涉及最小右表约束，跳过 + if (!isOverlap(joinConstraint.getMinRightHand(), joinTables)) { continue; } - if (joinTables.containsAll(joinConstraint.getMinRightHand())) { + // 如果当前join的所有表都在约束的"右边界"内，说明这个约束当前还没法应用，跳过 + if (joinConstraint.getMinRightHand().containsAll(joinTables)) { continue; } - if ((joinConstraint.getMinLeftHand().containsAll(leftHand)) - && joinConstraint.getMinRightHand().containsAll(leftHand)) { + // 当前join的左表包含最小左右约束，跳过 + if (leftHand.containsAll(joinConstraint.getMinLeftHand()) + && leftHand.containsAll(joinConstraint.getMinRightHand())) { continue; } - if (joinConstraint.getMinLeftHand().containsAll(rightHand) - && joinConstraint.getMinRightHand().containsAll(rightHand)) { + // 当前join的右表包含最小左右约束，跳过 + if (rightHand.containsAll(joinConstraint.getMinLeftHand()) + && rightHand.containsAll(joinConstraint.getMinRightHand())) { continue; } - if (joinConstraint.getMinLeftHand().containsAll(leftHand) - && joinConstraint.getMinRightHand().containsAll(rightHand)) { - if (matchedJoinConstraint != null) { - return new Pair<>(null, false); - } - matchedJoinConstraint = joinConstraint; - reversed = false; - } else if (joinConstraint.getMinLeftHand().containsAll(rightHand) - && joinConstraint.getMinRightHand().containsAll(leftHand)) { + if (leftHand.containsAll(joinConstraint.getMinLeftHand()) + && rightHand.containsAll(joinConstraint.getMinRightHand())) { + // 当前join的左表包含最小左约束，右表包含最小右约束 if (matchedJoinConstraint != null) { return new Pair<>(null, false); } matchedJoinConstraint = joinConstraint; - reversed = true; + } else if (rightHand.containsAll(joinConstraint.getMinLeftHand()) + && leftHand.containsAll(joinConstraint.getMinRightHand())) { + // 不支持left join转换为right join + return new Pair<>(null, false); } else { + // 当前join的左右表和最小右约束均有交集，跳过 + // minRightHand中的表被分散在了join的两边 if (isOverlap(leftHand, joinConstraint.getMinRightHand()) - && isOverlap(rightHand, joinConstraint.getRightHand())) { + && isOverlap(rightHand, joinConstraint.getMinRightHand())) { continue; } - if (joinConstraint.getJoinType() == JoinNode.JoinType.LEFT + // LEFT JOIN且joinTables 与 minLeftHand 有交集。如果当前JOIN执行完毕，再和minRightHand连接， + // 无法再满足 "minLeftHand作为左边界"的要求 + if (joinConstraint.getJoinType() != JoinNode.JoinType.LEFT || isOverlap(joinTables, joinConstraint.getMinLeftHand())) { return new Pair<>(null, false); } + // LEFT JOIN 且joinTables 与 minLeftHand 无交集 mustBeLeftJoin = true; } } @@ -420,7 +406,6 @@ && isOverlap(rightHand, joinConstraint.getRightHand())) { if (matchedJoinConstraint == null) { return new Pair<>(null, true); } - matchedJoinConstraint.setReversed(reversed); return new Pair<>(matchedJoinConstraint, true); } From e02eadd4a4cfb3fe7022f4a2ddb32197f9ab2626 Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 24 Feb 2026 11:41:59 +0800 Subject: [PATCH 17/20] fix leader/follower bug --- .../analyzer/StatementAnalyzer.java | 21 ++++++++++--------- .../planner/distribute/AddExchangeNodes.java | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 4cebaffeca8a5..5138bcf639f27 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1548,16 +1548,17 @@ private void addHint(String hintName, List parameters, Map .filter(entry -> !analysis.isAliased(entry.getKey().getNode())) .map(entry -> entry.getValue().getSuffix()) .collect(toImmutableList()); - for (String table : parameters) { - if (!existingTables.contains(table)) { - continue; - } - Hint hint = definition.createHint(ImmutableList.of(table)); - String hintKey = hint.getKey(); - if (!hintMap.containsKey(hintKey)) { - hintMap.put(hintKey, hint); - } - } + + List tablesToProcess = parameters == null ? existingTables : parameters; + + tablesToProcess.stream() + .filter(table -> parameters == null || existingTables.contains(table)) + .forEach( + table -> { + Hint hint = definition.createHint(ImmutableList.of(table)); + String hintKey = hint.getKey(); + hintMap.putIfAbsent(hintKey, hint); + }); } else { Hint hint = definition.createHint(parameters); String hintKey = hint.getKey(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java index 642bc5ce65526..07d456723826e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java @@ -108,7 +108,7 @@ public PlanNode visitTableScan( ReplicaHint hint = findReplicaHint(node, context.hintMap); // Early return for simple cases - if (context.hintMap == null || regionReplicaSet == null || hint == null) { + if (regionReplicaSet == null || hint == null) { context.nodeDistributionMap.put( node.getPlanNodeId(), new NodeDistribution(SAME_WITH_ALL_CHILDREN, regionReplicaSet)); return node; From ef84dd46d91cb01abbe36ebbd78749484fd0f767 Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 25 Feb 2026 16:17:51 +0800 Subject: [PATCH 18/20] handle filter & sort & mergesort node for leading hint --- .../analyzer/StatementAnalyzer.java | 21 ++++++---- .../optimizations/CollectJoinConstraint.java | 41 +++++++++++++++---- .../optimizations/LogicalOptimizeFactory.java | 6 +-- .../PushPredicateIntoTableScan.java | 19 +-------- 4 files changed, 53 insertions(+), 34 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 5138bcf639f27..aadccd713826f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1542,15 +1542,13 @@ private List intersect(List a, List b) { private void addHint(String hintName, List parameters, Map hintMap) { HintFactory definition = HINT_DEFINITIONS.get(hintName); if (definition != null) { - if (definition.shouldExpandParameters()) { - List existingTables = - analysis.getRelationNames().entrySet().stream() - .filter(entry -> !analysis.isAliased(entry.getKey().getNode())) - .map(entry -> entry.getValue().getSuffix()) - .collect(toImmutableList()); + List existingTables = + analysis.getRelationNames().values().stream() + .map(QualifiedName::getSuffix) + .collect(toImmutableList()); + if (definition.shouldExpandParameters()) { List tablesToProcess = parameters == null ? existingTables : parameters; - tablesToProcess.stream() .filter(table -> parameters == null || existingTables.contains(table)) .forEach( @@ -1560,6 +1558,15 @@ private void addHint(String hintName, List parameters, Map hintMap.putIfAbsent(hintKey, hint); }); } else { + // Skip if parameters contain tables that don't exist in the query + if (parameters != null) { + boolean hasInvalidTable = + parameters.stream().anyMatch(table -> !existingTables.contains(table)); + if (hasInvalidTable) { + return; + } + } + Hint hint = definition.createHint(parameters); String hintKey = hint.getKey(); if (!hintMap.containsKey(hintKey)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java index d1768e4eeca5a..9ced7f8aa22ea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java @@ -25,7 +25,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.JoinConstraint; @@ -71,12 +74,38 @@ public PlanNode visitPlan(PlanNode node, Context context) { @Override public PlanNode visitDeviceTableScan(DeviceTableScanNode node, Context context) { - String tableName = node.getQualifiedObjectName().getObjectName(); + Identifier alias = node.getAlias(); + String tableName = + alias != null ? alias.getValue() : node.getQualifiedObjectName().getObjectName(); LeadingHint leading = (LeadingHint) analysis.getHintMap().get(JoinOrderHint.category); leading.getRelationToScanMap().put(tableName, node); return node; } + @Override + public PlanNode visitMergeSort(MergeSortNode node, Context context) { + return visitSingleTableNode(node, context); + } + + public PlanNode visitFilter(FilterNode node, Context context) { + return visitSingleTableNode(node, context); + } + + public PlanNode visitSort(SortNode node, Context context) { + return visitSingleTableNode(node, context); + } + + private PlanNode visitSingleTableNode(PlanNode node, Context context) { + visitPlan(node, context); + Set tables = node.getInputTables(); + if (tables != null && tables.size() == 1) { + Identifier tableName = tables.iterator().next(); + LeadingHint leading = (LeadingHint) analysis.getHintMap().get(JoinOrderHint.category); + leading.getRelationToScanMap().put(tableName.getValue(), node); + } + return node; + } + @Override public PlanNode visitJoin(JoinNode node, Context context) { for (PlanNode child : node.getChildren()) { @@ -98,12 +127,10 @@ public PlanNode visitJoin(JoinNode node, Context context) { totalJoinTables = Sets.union(totalJoinTables, equiJoinTables); if (node.getJoinType() == JoinNode.JoinType.LEFT) { // leading.getFilters() is used to determine which join the join condition should apply - // to. - // For LEFT join, we need to add rightHand tables as well. In the getJoinConditions - // function, - // we ensure that the tables joined by the Join include both the tables in the join - // condition - // and the right tables of the outer join, so that the join condition can be applied. + // to. For LEFT join, we need to add rightHand tables as well. In the getJoinConditions + // function, we ensure that the tables joined by the Join include both the tables in the + // join condition and the right tables of the outer join, so that the join condition + // can be applied. equiJoinTables = Sets.union(equiJoinTables, rightHand); } leading.getFilters().add(new Pair<>(equiJoinTables, equiJoin.toExpression())); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index 0f6a173d6891d..bf82bda10b066 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -284,8 +284,6 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new RemoveRedundantIdentityProjections())) .build()), simplifyOptimizer, - new CollectJoinConstraint(), - new LeadingJoinOptimizer(), new UnaliasSymbolReferences(plannerContext.getMetadata()), new IterativeOptimizer( plannerContext, @@ -390,7 +388,9 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new MergeLimitWithSort(), new MergeLimitOverProjectWithSort(), new PushTopKThroughUnion())), - new ParallelizeGrouping()); + new ParallelizeGrouping(), + new CollectJoinConstraint(), + new LeadingJoinOptimizer()); this.planOptimizers = optimizerBuilder.build(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 853d88027f8d7..009456e4dd420 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -66,7 +66,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; @@ -834,15 +833,6 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { node.getRightChild().getOutputSymbols().stream() .collect(toImmutableMap(key -> key, Symbol::toSymbolReference))); - // Build a map from (left, right) symbols to their (leftTables, rightTables) - // This preserves table info when rebuilding EquiJoinClause - Map, Pair, Set>> symbolToTablesMap = - node.getCriteria().stream() - .collect( - toImmutableMap( - clause -> new Pair<>(clause.getLeft(), clause.getRight()), - clause -> new Pair<>(clause.getLeftTables(), clause.getRightTables()))); - // Create new projections for the new join clauses List equiJoinClauses = new ArrayList<>(); ImmutableList.Builder joinFilterBuilder = ImmutableList.builder(); @@ -866,14 +856,9 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { rightProjections.put(rightSymbol, rightExpression); } - // Retrieve leftTables and rightTables from original criteria - Pair, Set> tables = - symbolToTablesMap.getOrDefault( - new Pair<>(leftSymbol, rightSymbol), - new Pair<>(ImmutableSet.of(), ImmutableSet.of())); - equiJoinClauses.add( - new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, tables.left, tables.right)); + new JoinNode.EquiJoinClause( + leftSymbol, rightSymbol, node.getLeftTables(), node.getRightTables())); } else { if (conjunct.equals(TRUE_LITERAL) && node.getAsofCriteria().isPresent()) { continue; From e1b23334dbfa8766cf1ae2103bf13ce4453230d4 Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 25 Feb 2026 19:56:47 +0800 Subject: [PATCH 19/20] EquiJoinClause left & right table --- .../relational/planner/RelationPlanner.java | 83 +++++++++++++++---- ...TransformFilteringSemiJoinToInnerJoin.java | 18 +++- .../relational/planner/node/JoinNode.java | 68 +++++---------- .../planner/optimizations/JoinUtils.java | 38 +++++++++ .../PushPredicateIntoTableScan.java | 17 +++- .../UnaliasSymbolReferences.java | 4 +- .../relational/utils/hint/LeadingHint.java | 35 +++++++- 7 files changed, 192 insertions(+), 71 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 1d0dab122f036..7817bafecb035 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -142,7 +142,6 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -514,9 +513,28 @@ If casts are redundant (due to column type and common type being equal), rightCoercions.put(rightOutput, right.getSymbol(rightField).toSymbolReference()); rightJoinColumns.put(identifier, rightOutput); - Set leftTables = new HashSet<>(left.getScope().getTables()); - Set rightTables = new HashSet<>(right.getScope().getTables()); - clauses.add(new JoinNode.EquiJoinClause(leftOutput, rightOutput, leftTables, rightTables)); + // Extract tables from the actual fields used in the join condition + Field leftFieldObj = left.getScope().getRelationType().getFieldByIndex(leftField); + Identifier leftTable = + extractTableName(leftFieldObj) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for field %s in JOIN USING clause", + leftFieldObj.getName()))); + + Field rightFieldObj = right.getScope().getRelationType().getFieldByIndex(rightField); + Identifier rightTable = + extractTableName(rightFieldObj) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for field %s in JOIN USING clause", + rightFieldObj.getName()))); + + clauses.add(new JoinNode.EquiJoinClause(leftOutput, rightOutput, leftTable, rightTable)); } ProjectNode leftCoercion = @@ -740,22 +758,37 @@ public RelationPlan planJoin( SymbolsExtractor.extractNames( rightComparisonExpressions.get(i), analysis.getColumnReferences()); - // Convert QualifiedName to Identifier (table name is the first part) - Set leftTables = new HashSet<>(); - for (QualifiedName name : leftDependencies) { - if (name.getPrefix().isPresent()) { - leftTables.add(new Identifier(name.getPrefix().get().getSuffix())); - } - } - Set rightTables = new HashSet<>(); - for (QualifiedName name : rightDependencies) { - if (name.getPrefix().isPresent()) { - rightTables.add(new Identifier(name.getPrefix().get().getSuffix())); - } + if (leftDependencies.size() != 1 || rightDependencies.size() != 1) { + throw new IllegalStateException("Cannot find source table for symbol " + leftSymbol); } + QualifiedName leftQualifiedName = leftDependencies.iterator().next(); + QualifiedName rightQualifiedName = rightDependencies.iterator().next(); + + Identifier leftTable = + leftQualifiedName + .getPrefix() + .map(prefix -> new Identifier(prefix.getSuffix())) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in JOIN ON clause", + leftSymbol))); + + Identifier rightTable = + rightQualifiedName + .getPrefix() + .map(prefix -> new Identifier(prefix.getSuffix())) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in JOIN ON clause", + rightSymbol))); + equiClauses.add( - new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, leftTables, rightTables)); + new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, leftTable, rightTable)); } else { postInnerJoinConditions.add( new ComparisonExpression( @@ -1668,4 +1701,20 @@ public Map getVariableDefinitions() { return variableDefinitions; } } + + /** + * Extracts the table name from a field. Priority: 1) relation alias (if exists), 2) origin table + * name. + * + * @param field the field to extract the table name from + * @return the table identifier, or empty if not found + */ + private static Optional extractTableName(Field field) { + Optional fromAlias = + field.getRelationAlias().map(alias -> new Identifier(alias.getSuffix())); + if (fromAlias.isPresent()) { + return fromAlias; + } + return field.getOriginTable().map(originTable -> new Identifier(originTable.getObjectName())); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java index 10f2eb51dd0ef..910fe9e9ce8d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Capture; import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; @@ -132,8 +133,21 @@ public Result apply(FilterNode filterNode, Captures captures, Context context) { new JoinNode.EquiJoinClause( semiJoin.getSourceJoinSymbol(), semiJoin.getFilteringSourceJoinSymbol(), - semiJoin.getSource().getInputTables(), - semiJoin.getFilteringSource().getInputTables())), + JoinUtils.findSourceTable(semiJoin.getSource(), semiJoin.getSourceJoinSymbol()) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in semi-join source", + semiJoin.getSourceJoinSymbol()))), + JoinUtils.findSourceTable( + semiJoin.getFilteringSource(), semiJoin.getFilteringSourceJoinSymbol()) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in semi-join filtering source", + semiJoin.getFilteringSourceJoinSymbol()))))), Optional.empty(), semiJoin.getSource().getOutputSymbols(), ImmutableList.of(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index a0521bb7e5373..a7049702a2fce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -32,7 +32,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; @@ -286,16 +285,8 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { for (EquiJoinClause equiJoinClause : criteria) { Symbol.serialize(equiJoinClause.getLeft(), byteBuffer); Symbol.serialize(equiJoinClause.getRight(), byteBuffer); - Set leftTables = equiJoinClause.getLeftTables(); - ReadWriteIOUtils.write(leftTables.size(), byteBuffer); - for (Identifier identifier : leftTables) { - identifier.serialize(byteBuffer); - } - Set rightTables = equiJoinClause.getRightTables(); - ReadWriteIOUtils.write(rightTables.size(), byteBuffer); - for (Identifier identifier : rightTables) { - identifier.serialize(byteBuffer); - } + equiJoinClause.getLeftTable().serialize(byteBuffer); + equiJoinClause.getRightTable().serialize(byteBuffer); } if (asofCriteria.isPresent()) { @@ -338,16 +329,8 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { for (EquiJoinClause equiJoinClause : criteria) { Symbol.serialize(equiJoinClause.getLeft(), stream); Symbol.serialize(equiJoinClause.getRight(), stream); - Set leftTables = equiJoinClause.getLeftTables(); - ReadWriteIOUtils.write(leftTables.size(), stream); - for (Identifier identifier : leftTables) { - identifier.serialize(stream); - } - Set rightTables = equiJoinClause.getRightTables(); - ReadWriteIOUtils.write(rightTables.size(), stream); - for (Identifier identifier : rightTables) { - identifier.serialize(stream); - } + equiJoinClause.getLeftTable().serialize(stream); + equiJoinClause.getRightTable().serialize(stream); } if (asofCriteria.isPresent()) { @@ -387,17 +370,9 @@ public static JoinNode deserialize(ByteBuffer byteBuffer) { while (size-- > 0) { Symbol left = Symbol.deserialize(byteBuffer); Symbol right = Symbol.deserialize(byteBuffer); - int leftTablesSize = ReadWriteIOUtils.readInt(byteBuffer); - Set leftTables = new HashSet<>(leftTablesSize); - while (leftTablesSize-- > 0) { - leftTables.add(new Identifier(byteBuffer)); - } - int rightTablesSize = ReadWriteIOUtils.readInt(byteBuffer); - Set rightTables = new HashSet<>(rightTablesSize); - while (rightTablesSize-- > 0) { - rightTables.add(new Identifier(byteBuffer)); - } - criteria.add(new EquiJoinClause(left, right, leftTables, rightTables)); + Identifier leftTable = new Identifier(byteBuffer); + Identifier rightTable = new Identifier(byteBuffer); + criteria.add(new EquiJoinClause(left, right, leftTable, rightTable)); } Optional asofJoinClause = Optional.empty(); @@ -497,17 +472,16 @@ public String toString() { public static class EquiJoinClause { private final Symbol left; private final Symbol right; - private final Set leftTables; - private final Set rightTables; + private final Identifier leftTable; + private final Identifier rightTable; private final Set tables; - public EquiJoinClause( - Symbol left, Symbol right, Set leftTables, Set rightTables) { + public EquiJoinClause(Symbol left, Symbol right, Identifier leftTable, Identifier rightTable) { this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); - this.leftTables = requireNonNull(leftTables, "leftTables is null"); - this.rightTables = requireNonNull(rightTables, "rightTables is null"); - this.tables = Sets.union(leftTables, rightTables); + this.leftTable = requireNonNull(leftTable, "leftTable is null"); + this.rightTable = requireNonNull(rightTable, "rightTable is null"); + this.tables = ImmutableSet.of(leftTable, rightTable); } public Symbol getLeft() { @@ -518,12 +492,12 @@ public Symbol getRight() { return right; } - public Set getLeftTables() { - return leftTables; + public Identifier getLeftTable() { + return leftTable; } - public Set getRightTables() { - return rightTables; + public Identifier getRightTable() { + return rightTable; } public Set getTables() { @@ -536,7 +510,7 @@ public ComparisonExpression toExpression() { } public EquiJoinClause flip() { - return new EquiJoinClause(right, left, rightTables, leftTables); + return new EquiJoinClause(right, left, rightTable, leftTable); } public static List flipBatch(List input) { @@ -559,13 +533,13 @@ public boolean equals(Object obj) { return Objects.equals(this.left, other.left) && Objects.equals(this.right, other.right) - && Objects.equals(this.leftTables, other.leftTables) - && Objects.equals(this.rightTables, other.rightTables); + && Objects.equals(this.leftTable, other.leftTable) + && Objects.equals(this.rightTable, other.rightTable); } @Override public int hashCode() { - return Objects.hash(left, right, leftTables, rightTables); + return Objects.hash(left, right, leftTable, rightTable); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java index 9e751d2dd4d31..6542b46df93bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java @@ -19,18 +19,22 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.planner.EqualityInference; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.util.Collection; import java.util.Objects; +import java.util.Optional; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; @@ -404,4 +408,38 @@ public Expression getPostJoinPredicate() { return postJoinPredicate; } } + + /** + * Finds the source table for a given symbol by traversing the plan tree. Returns Optional.empty() + * if the symbol cannot be traced to a specific table. + * + * @param node the plan node to search + * @param symbol the symbol to find the source table for + * @return the table identifier if found, Optional.empty() otherwise + */ + public static Optional findSourceTable(PlanNode node, Symbol symbol) { + if (node instanceof DeviceTableScanNode) { + DeviceTableScanNode scanNode = (DeviceTableScanNode) node; + if (scanNode.getOutputSymbols().contains(symbol)) { + Identifier alias = scanNode.getAlias(); + if (alias != null) { + return Optional.of(alias); + } + return Optional.of(new Identifier(scanNode.getQualifiedObjectName().getObjectName())); + } + return Optional.empty(); + } + + // For other node types, recursively check children + for (PlanNode child : node.getChildren()) { + if (child.getOutputSymbols().contains(symbol)) { + Optional result = findSourceTable(child, symbol); + if (result.isPresent()) { + return result; + } + } + } + + return Optional.empty(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 009456e4dd420..30efc6ebf38d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -858,7 +858,22 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { equiJoinClauses.add( new JoinNode.EquiJoinClause( - leftSymbol, rightSymbol, node.getLeftTables(), node.getRightTables())); + leftSymbol, + rightSymbol, + JoinUtils.findSourceTable(node.getLeftChild(), leftSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in join left child", + leftSymbol))), + JoinUtils.findSourceTable(node.getRightChild(), rightSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in join right child", + rightSymbol))))); } else { if (conjunct.equals(TRUE_LITERAL) && node.getAsofCriteria().isPresent()) { continue; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index 5748f539879f7..4e8c16df0cc36 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -815,8 +815,8 @@ public PlanAndMappings visitJoin(JoinNode node, UnaliasContext context) { new JoinNode.EquiJoinClause( mapper.map(clause.getLeft()), mapper.map(clause.getRight()), - ImmutableSet.copyOf(clause.getLeftTables()), - ImmutableSet.copyOf(clause.getRightTables()))); + clause.getLeftTable(), + clause.getRightTable())); } List newCriteria = builder.build(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java index 5b4566b2451ca..31d0f35026379 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; @@ -240,12 +241,42 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { if (leftOutputSymbols.contains(leftSymbol) && rightOutputSymbols.contains(rightSymbol)) { criteria.add( new JoinNode.EquiJoinClause( - leftSymbol, rightSymbol, leftChild.getInputTables(), rightChild.getInputTables())); + leftSymbol, + rightSymbol, + JoinUtils.findSourceTable(leftChild, leftSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join left child", + leftSymbol))), + JoinUtils.findSourceTable(rightChild, rightSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join right child", + rightSymbol))))); } else if (leftOutputSymbols.contains(rightSymbol) && rightOutputSymbols.contains(leftSymbol)) { criteria.add( new JoinNode.EquiJoinClause( - rightSymbol, leftSymbol, leftChild.getInputTables(), rightChild.getInputTables())); + rightSymbol, + leftSymbol, + JoinUtils.findSourceTable(leftChild, rightSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join left child", + rightSymbol))), + JoinUtils.findSourceTable(rightChild, leftSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join right child", + leftSymbol))))); } else { throw new IllegalArgumentException("Invalid join condition"); } From 83ff041df9567b82d06711747c2f36e3888809a0 Mon Sep 17 00:00:00 2001 From: shizy Date: Thu, 26 Feb 2026 22:23:00 +0800 Subject: [PATCH 20/20] leading hint for asof join --- .../relational/planner/RelationPlanner.java | 78 +++++++------ .../relational/planner/node/JoinNode.java | 41 ++++++- .../optimizations/CollectJoinConstraint.java | 21 +++- .../relational/utils/hint/LeadingHint.java | 108 ++++++++++++++++-- .../relational/analyzer/AsofJoinTest.java | 40 ++++++- .../assertions/AsofJoinClauseProvider.java | 14 ++- .../assertions/EquiJoinClauseProvider.java | 17 +-- .../planner/assertions/JoinMatcher.java | 10 +- .../planner/assertions/PlanMatchPattern.java | 17 ++- 9 files changed, 263 insertions(+), 83 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 7817bafecb035..76cf9848d13c0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -735,6 +735,41 @@ public RelationPlan planJoin( rightPlanBuilder = rightCoercions.getSubPlan(); for (int i = 0; i < leftComparisonExpressions.size(); i++) { + // Extract tables from expressions + Set leftDependencies = + SymbolsExtractor.extractNames( + leftComparisonExpressions.get(i), analysis.getColumnReferences()); + Set rightDependencies = + SymbolsExtractor.extractNames( + rightComparisonExpressions.get(i), analysis.getColumnReferences()); + + if (leftDependencies.size() != 1 || rightDependencies.size() != 1) { + throw new IllegalStateException("Cannot find source table for symbol"); + } + + QualifiedName leftQualifiedName = leftDependencies.iterator().next(); + QualifiedName rightQualifiedName = rightDependencies.iterator().next(); + + Identifier leftTable = + leftQualifiedName + .getPrefix() + .map(prefix -> new Identifier(prefix.getSuffix())) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in JOIN ON clause"))); + + Identifier rightTable = + rightQualifiedName + .getPrefix() + .map(prefix -> new Identifier(prefix.getSuffix())) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in JOIN ON clause"))); + if (asofCriteria != null && i == 0) { Symbol leftSymbol = leftCoercions.get(leftComparisonExpressions.get(i)); Symbol rightSymbol = rightCoercions.get(rightComparisonExpressions.get(i)); @@ -742,7 +777,11 @@ public RelationPlan planJoin( asofJoinClause = Optional.of( new JoinNode.AsofJoinClause( - joinConditionComparisonOperators.get(i), leftSymbol, rightSymbol)); + joinConditionComparisonOperators.get(i), + leftSymbol, + rightSymbol, + leftTable, + rightTable)); continue; } @@ -750,43 +789,6 @@ public RelationPlan planJoin( Symbol leftSymbol = leftCoercions.get(leftComparisonExpressions.get(i)); Symbol rightSymbol = rightCoercions.get(rightComparisonExpressions.get(i)); - // Extract tables from expressions - Set leftDependencies = - SymbolsExtractor.extractNames( - leftComparisonExpressions.get(i), analysis.getColumnReferences()); - Set rightDependencies = - SymbolsExtractor.extractNames( - rightComparisonExpressions.get(i), analysis.getColumnReferences()); - - if (leftDependencies.size() != 1 || rightDependencies.size() != 1) { - throw new IllegalStateException("Cannot find source table for symbol " + leftSymbol); - } - - QualifiedName leftQualifiedName = leftDependencies.iterator().next(); - QualifiedName rightQualifiedName = rightDependencies.iterator().next(); - - Identifier leftTable = - leftQualifiedName - .getPrefix() - .map(prefix -> new Identifier(prefix.getSuffix())) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in JOIN ON clause", - leftSymbol))); - - Identifier rightTable = - rightQualifiedName - .getPrefix() - .map(prefix -> new Identifier(prefix.getSuffix())) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in JOIN ON clause", - rightSymbol))); - equiClauses.add( new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, leftTable, rightTable)); } else { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index a7049702a2fce..9334dde0611ee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -295,6 +295,8 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { ReadWriteIOUtils.write(asofJoinClause.getOperator().ordinal(), byteBuffer); Symbol.serialize(asofJoinClause.getLeft(), byteBuffer); Symbol.serialize(asofJoinClause.getRight(), byteBuffer); + asofJoinClause.getLeftTable().serialize(byteBuffer); + asofJoinClause.getRightTable().serialize(byteBuffer); } else { ReadWriteIOUtils.write(false, byteBuffer); } @@ -339,6 +341,8 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(asofJoinClause.getOperator().ordinal(), stream); Symbol.serialize(asofJoinClause.getLeft(), stream); Symbol.serialize(asofJoinClause.getRight(), stream); + asofJoinClause.getLeftTable().serialize(stream); + asofJoinClause.getRightTable().serialize(stream); } else { ReadWriteIOUtils.write(false, stream); } @@ -382,7 +386,9 @@ public static JoinNode deserialize(ByteBuffer byteBuffer) { new AsofJoinClause( ComparisonExpression.Operator.values()[ReadWriteIOUtils.readInt(byteBuffer)], Symbol.deserialize(byteBuffer), - Symbol.deserialize(byteBuffer))); + Symbol.deserialize(byteBuffer), + new Identifier(byteBuffer), + new Identifier(byteBuffer))); } size = ReadWriteIOUtils.readInt(byteBuffer); @@ -551,12 +557,23 @@ public String toString() { public static class AsofJoinClause { private final Symbol left; private final Symbol right; + private final Identifier leftTable; + private final Identifier rightTable; + private final Set tables; private final ComparisonExpression.Operator operator; - public AsofJoinClause(ComparisonExpression.Operator operator, Symbol left, Symbol right) { + public AsofJoinClause( + ComparisonExpression.Operator operator, + Symbol left, + Symbol right, + Identifier leftTable, + Identifier rightTable) { this.operator = operator; this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); + this.leftTable = requireNonNull(leftTable, "leftTable is null"); + this.rightTable = requireNonNull(rightTable, "rightTable is null"); + this.tables = ImmutableSet.of(leftTable, rightTable); } public Symbol getLeft() { @@ -567,6 +584,18 @@ public Symbol getRight() { return right; } + public Identifier getLeftTable() { + return leftTable; + } + + public Identifier getRightTable() { + return rightTable; + } + + public Set getTables() { + return tables; + } + public ComparisonExpression.Operator getOperator() { return operator; } @@ -577,7 +606,7 @@ public ComparisonExpression toExpression() { } public AsofJoinClause flip() { - return new AsofJoinClause(operator.flip(), right, left); + return new AsofJoinClause(operator.flip(), right, left, rightTable, leftTable); } public boolean isOperatorContainsGreater() { @@ -607,12 +636,14 @@ public boolean equals(Object obj) { return Objects.equals(this.operator, other.operator) && Objects.equals(this.left, other.left) - && Objects.equals(this.right, other.right); + && Objects.equals(this.right, other.right) + && Objects.equals(this.leftTable, other.leftTable) + && Objects.equals(this.rightTable, other.rightTable); } @Override public int hashCode() { - return Objects.hash(operator, left, right); + return Objects.hash(operator, left, right, leftTable, rightTable); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java index 9ced7f8aa22ea..cad983b631c5d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java @@ -39,7 +39,7 @@ import com.google.common.collect.Sets; import org.apache.tsfile.utils.Pair; -import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -121,8 +121,7 @@ public PlanNode visitJoin(JoinNode node, Context context) { Set totalJoinTables = ImmutableSet.of(); // join conjunctions - List criteria = node.getCriteria(); - for (JoinNode.EquiJoinClause equiJoin : criteria) { + for (JoinNode.EquiJoinClause equiJoin : node.getCriteria()) { Set equiJoinTables = Sets.intersection(leadingTables, equiJoin.getTables()); totalJoinTables = Sets.union(totalJoinTables, equiJoinTables); if (node.getJoinType() == JoinNode.JoinType.LEFT) { @@ -133,9 +132,23 @@ public PlanNode visitJoin(JoinNode node, Context context) { // can be applied. equiJoinTables = Sets.union(equiJoinTables, rightHand); } - leading.getFilters().add(new Pair<>(equiJoinTables, equiJoin.toExpression())); + leading.getEquiJoins().add(new Pair<>(equiJoinTables, equiJoin.toExpression())); leading.putConditionJoinType(equiJoin.toExpression(), node.getJoinType()); } + + Optional asofJoin = node.getAsofCriteria(); + if (asofJoin.isPresent()) { + JoinNode.AsofJoinClause asofJoinClause = asofJoin.get(); + Set asofJoinTables = + Sets.intersection(leadingTables, asofJoinClause.getTables()); + totalJoinTables = Sets.union(totalJoinTables, asofJoinTables); + if (node.getJoinType() == JoinNode.JoinType.LEFT) { + asofJoinTables = Sets.union(asofJoinTables, rightHand); + } + leading.setAsofJoin(new Pair<>(asofJoinTables, asofJoinClause.toExpression())); + leading.putConditionJoinType(asofJoinClause.toExpression(), node.getJoinType()); + } + // join constraint collectJoinConstraintList(leading, leftHand, rightHand, node, totalJoinTables); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java index 31d0f35026379..d7710d451dd38 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java @@ -54,7 +54,8 @@ public class LeadingHint extends JoinOrderHint { private List addJoinParameters; private List normalizedParameters; - private final List, Expression>> filters = new ArrayList<>(); + private final List, Expression>> equiJoins = new ArrayList<>(); + private Pair, Expression> asofJoin; private final Map relationToScanMap = new HashMap<>(); private Set innerJoinTables = ImmutableSet.of(); @@ -91,8 +92,16 @@ public List getJoinConstraintList() { return joinConstraintList; } - public List, Expression>> getFilters() { - return filters; + public List, Expression>> getEquiJoins() { + return equiJoins; + } + + public Pair, Expression> getAsofJoin() { + return asofJoin; + } + + public void setAsofJoin(Pair, Expression> asofJoin) { + this.asofJoin = asofJoin; } public void putConditionJoinType(Expression filter, JoinNode.JoinType joinType) { @@ -127,16 +136,16 @@ public PlanNode generateLeadingJoinPlan() { if (logicalPlan == null) { return null; } - logicalPlan = makeFilterPlanIfExist(getFilters(), logicalPlan); + logicalPlan = makeFilterPlanIfExist(getEquiJoins(), logicalPlan); stack.push(logicalPlan); } } PlanNode finalJoin = stack.pop(); // we want all filters been removed - if (!filters.isEmpty()) { + if (!equiJoins.isEmpty()) { throw new IllegalStateException( - "Leading hint process failed: filter should be empty, but meet: " + filters); + "Leading hint process failed: filter should be empty, but meet: " + equiJoins); } if (finalJoin == null) { throw new IoTDBRuntimeException( @@ -219,12 +228,14 @@ public PlanNode getPlanByName(String name) { } private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { - List conditions = getJoinConditions(getFilters(), leftChild, rightChild); JoinNode.JoinType joinType = computeJoinType(leftChild.getInputTables(), rightChild.getInputTables()); if (joinType == null) { return null; - } else if (!isConditionJoinTypeMatched(conditions, joinType)) { + } + + List equiJoins = getEquiJoinConditions(getEquiJoins(), leftChild, rightChild); + if (!isConditionJoinTypeMatched(equiJoins, joinType)) { return null; } @@ -233,8 +244,8 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { Optional asofCriteria = Optional.empty(); List criteria = new ArrayList<>(); - for (Expression conjunct : conditions) { - ComparisonExpression equality = (ComparisonExpression) conjunct; + for (Expression equiJoin : equiJoins) { + ComparisonExpression equality = (ComparisonExpression) equiJoin; Symbol leftSymbol = Symbol.from(equality.getLeft()); Symbol rightSymbol = Symbol.from(equality.getRight()); @@ -282,6 +293,62 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { } } + Expression asofJoin = getAsofJoinCondition(getAsofJoin(), leftChild, rightChild); + if (asofJoin != null) { + if (!isConditionJoinTypeMatched(asofJoin, joinType)) { + return null; + } + + ComparisonExpression asofJoinExpr = (ComparisonExpression) asofJoin; + Symbol leftSymbol = Symbol.from(asofJoinExpr.getLeft()); + Symbol rightSymbol = Symbol.from(asofJoinExpr.getRight()); + + if (leftOutputSymbols.contains(leftSymbol) && rightOutputSymbols.contains(rightSymbol)) { + asofCriteria = + Optional.of( + new JoinNode.AsofJoinClause( + asofJoinExpr.getOperator(), + leftSymbol, + rightSymbol, + JoinUtils.findSourceTable(leftChild, leftSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join left child", + leftSymbol))), + JoinUtils.findSourceTable(rightChild, rightSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join right child", + rightSymbol))))); + } else if (leftOutputSymbols.contains(rightSymbol) + && rightOutputSymbols.contains(leftSymbol)) { + asofCriteria = + Optional.of( + new JoinNode.AsofJoinClause( + asofJoinExpr.getOperator().flip(), + rightSymbol, + leftSymbol, + JoinUtils.findSourceTable(leftChild, rightSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join left child", + rightSymbol))), + JoinUtils.findSourceTable(rightChild, leftSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join right child", + leftSymbol))))); + } + } + return new JoinNode( new PlanNodeId("join"), joinType, @@ -295,7 +362,7 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { Optional.empty()); } - private List getJoinConditions( + private List getEquiJoinConditions( List, Expression>> filters, PlanNode left, PlanNode right) { List joinConditions = new ArrayList<>(); for (int i = filters.size() - 1; i >= 0; i--) { @@ -310,6 +377,20 @@ private List getJoinConditions( return joinConditions; } + private Expression getAsofJoinCondition( + Pair, Expression> filterPair, PlanNode left, PlanNode right) { + if (filterPair == null) { + return null; + } + + Set tables = Sets.union(left.getInputTables(), right.getInputTables()); + // it should contain all tables in join conjunctions & right tables if it's left join + if (tables.containsAll(filterPair.left)) { + return filterPair.right; + } + return null; + } + private PlanNode makeFilterPlanIfExist( List, Expression>> filters, PlanNode plan) { if (filters.isEmpty()) { @@ -351,6 +432,11 @@ public boolean isConditionJoinTypeMatched( return true; } + public boolean isConditionJoinTypeMatched(Expression condition, JoinNode.JoinType joinType) { + JoinNode.JoinType originalJoinType = conditionJoinType.get(condition); + return originalJoinType == joinType; + } + /** * try to get join constraint. If it can not be found, it means join is inner join * diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java index dee921c73fe9b..5648ce16a88ae 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java @@ -61,7 +61,12 @@ public void simpleTest() { JoinNode.JoinType.INNER, builder -> builder - .asofCriteria(ComparisonExpression.Operator.LESS_THAN, "time", "time_0") + .asofCriteria( + ComparisonExpression.Operator.LESS_THAN, + "time", + "time_0", + "table1", + "table2") .left( sort( ImmutableList.of(sort("time", ASCENDING, LAST)), @@ -85,7 +90,12 @@ public void simpleTest() { JoinNode.JoinType.INNER, builder -> builder - .asofCriteria(ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") + .asofCriteria( + ComparisonExpression.Operator.GREATER_THAN, + "time", + "time_0", + "table1", + "table2") .left( sort( ImmutableList.of(sort("time", DESCENDING, LAST)), @@ -118,7 +128,11 @@ public void toleranceTest() { builder -> builder .asofCriteria( - ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") + ComparisonExpression.Operator.GREATER_THAN, + "time", + "time_0", + "table1", + "table2") .left( sort( ImmutableList.of(sort("time", DESCENDING, LAST)), @@ -153,7 +167,11 @@ public void projectInAsofCriteriaTest() { builder -> builder .asofCriteria( - ComparisonExpression.Operator.GREATER_THAN, "expr", "time_0") + ComparisonExpression.Operator.GREATER_THAN, + "expr", + "time_0", + "table1", + "table2") .equiCriteria("tag1", "tag1_1", "table1", "table2") .left( sort( @@ -207,7 +225,12 @@ public void sortEliminateTest() { JoinNode.JoinType.INNER, builder -> builder - .asofCriteria(ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") + .asofCriteria( + ComparisonExpression.Operator.GREATER_THAN, + "time", + "time_0", + "table1", + "table2") .equiCriteria( ImmutableList.of( equiJoinClause("tag1", "tag1_1", "table1", "table2"), @@ -224,7 +247,12 @@ public void sortEliminateTest() { JoinNode.JoinType.INNER, builder -> builder - .asofCriteria(ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") + .asofCriteria( + ComparisonExpression.Operator.GREATER_THAN, + "time", + "time_0", + "table1", + "table2") .equiCriteria( ImmutableList.of( equiJoinClause("tag1", "tag1_1", "table1", "table2"), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AsofJoinClauseProvider.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AsofJoinClauseProvider.java index c02f41bc8ae06..2ac69c2bf818e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AsofJoinClauseProvider.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AsofJoinClauseProvider.java @@ -21,6 +21,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import static java.lang.String.format; @@ -29,13 +30,21 @@ public class AsofJoinClauseProvider implements ExpectedValueProvider { private final SymbolAlias left; private final SymbolAlias right; + private final Identifier leftTable; + private final Identifier rightTable; private final ComparisonExpression.Operator operator; public AsofJoinClauseProvider( - ComparisonExpression.Operator operator, SymbolAlias left, SymbolAlias right) { + ComparisonExpression.Operator operator, + SymbolAlias left, + SymbolAlias right, + Identifier leftTable, + Identifier rightTable) { this.operator = requireNonNull(operator, "operator is null"); this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); + this.leftTable = requireNonNull(leftTable, "leftTable is null"); + this.rightTable = requireNonNull(rightTable, "rightTable is null"); } public ComparisonExpression toExpression() { @@ -45,7 +54,8 @@ public ComparisonExpression toExpression() { @Override public JoinNode.AsofJoinClause getExpectedValue(SymbolAliases aliases) { - return new JoinNode.AsofJoinClause(operator, left.toSymbol(aliases), right.toSymbol(aliases)); + return new JoinNode.AsofJoinClause( + operator, left.toSymbol(aliases), right.toSymbol(aliases), leftTable, rightTable); } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java index b6e5280caf2ea..99ca1fc8af2df 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java @@ -22,31 +22,26 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; -import java.util.Set; - import static java.util.Objects.requireNonNull; public class EquiJoinClauseProvider implements ExpectedValueProvider { private final SymbolAlias left; private final SymbolAlias right; - private final Set leftTables; - private final Set rightTables; + private final Identifier leftTable; + private final Identifier rightTable; public EquiJoinClauseProvider( - SymbolAlias left, - SymbolAlias right, - Set leftTables, - Set rightTables) { + SymbolAlias left, SymbolAlias right, Identifier leftTable, Identifier rightTable) { this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); - this.leftTables = requireNonNull(leftTables, "leftTables is null"); - this.rightTables = requireNonNull(rightTables, "rightTables is null"); + this.leftTable = requireNonNull(leftTable, "leftTable is null"); + this.rightTable = requireNonNull(rightTable, "rightTable is null"); } @Override public JoinNode.EquiJoinClause getExpectedValue(SymbolAliases aliases) { return new JoinNode.EquiJoinClause( - left.toSymbol(aliases), right.toSymbol(aliases), leftTables, rightTables); + left.toSymbol(aliases), right.toSymbol(aliases), leftTable, rightTable); } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java index 8ea44457e331d..29b8314f5b50f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java @@ -174,8 +174,14 @@ public Builder equiCriteria(String left, String right, String leftTable, String } @CanIgnoreReturnValue - public Builder asofCriteria(ComparisonExpression.Operator operator, String left, String right) { - this.asofJoinCriteria = Optional.of(asofJoinClause(operator, left, right)); + public Builder asofCriteria( + ComparisonExpression.Operator operator, + String left, + String right, + String leftTable, + String rightTable) { + this.asofJoinCriteria = + Optional.of(asofJoinClause(operator, left, right, leftTable, rightTable)); return this; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java index 044de2f45f56e..b3bd90d5feef4 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java @@ -687,13 +687,22 @@ public static ExpectedValueProvider equiJoinClause( return new EquiJoinClauseProvider( new SymbolAlias(left), new SymbolAlias(right), - ImmutableSet.of(new Identifier(leftTable)), - ImmutableSet.of(new Identifier(rightTable))); + new Identifier(leftTable), + new Identifier(rightTable)); } public static AsofJoinClauseProvider asofJoinClause( - ComparisonExpression.Operator operator, String left, String right) { - return new AsofJoinClauseProvider(operator, new SymbolAlias(left), new SymbolAlias(right)); + ComparisonExpression.Operator operator, + String left, + String right, + String leftTable, + String rightTable) { + return new AsofJoinClauseProvider( + operator, + new SymbolAlias(left), + new SymbolAlias(right), + new Identifier(leftTable), + new Identifier(rightTable)); } public static SymbolAlias symbol(String alias) {