Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[opt](nereids) optimize rewrite of synchronize materialize view #45748

Merged
merged 1 commit into from
Dec 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -600,7 +600,9 @@ public void setQueryDistributedFinishTime() {
}

public void setQueryPlanFinishTime() {
this.queryPlanFinishTime = TimeUtils.getStartTimeMs();
if (queryPlanFinishTime == -1) {
this.queryPlanFinishTime = TimeUtils.getStartTimeMs();
}
}

public void setQueryScheduleFinishTime() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public boolean nullable() {
}

@Override
public String toSql() {
public String computeToSql() {
return slot.toSql();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public DataType getDataType() throws UnboundException {
}

@Override
public String toSql() {
public String computeToSql() {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append("(" + child() + ")");
alias.ifPresent(name -> stringBuilder.append(" AS " + name));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ public List<Expression> getArguments() {
}

@Override
public String toSql() throws UnboundException {
public String computeToSql() throws UnboundException {
String params = children.stream()
.map(Expression::toSql)
.collect(Collectors.joining(", "));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,18 @@ public String getInternalName() {
}

@Override
public String toSql() {
return nameParts.stream().map(Utils::quoteIfNeeded).reduce((left, right) -> left + "." + right).orElse("");
public String computeToSql() {
switch (nameParts.size()) {
case 1: return Utils.quoteIfNeeded(nameParts.get(0));
case 2: return Utils.quoteIfNeeded(nameParts.get(0)) + "." + Utils.quoteIfNeeded(nameParts.get(1));
case 3: return Utils.quoteIfNeeded(nameParts.get(0)) + "." + Utils.quoteIfNeeded(nameParts.get(1))
+ "." + Utils.quoteIfNeeded(nameParts.get(2));
default: {
return nameParts.stream().map(Utils::quoteIfNeeded)
.reduce((left, right) -> left + "." + right)
.orElse("");
}
}
}

public static UnboundSlot quoted(String name) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ public UnboundStar(List<String> qualifier, List<NamedExpression> exceptedSlots,
}

@Override
public String toSql() {
public String computeToSql() {
StringBuilder builder = new StringBuilder();
builder.append(Utils.qualifiedName(qualifier, "*"));
if (!exceptedSlots.isEmpty()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.DorisLexer;
import org.apache.doris.nereids.DorisParser;
import org.apache.doris.nereids.DorisParser.NonReservedContext;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.analyzer.UnboundSlot;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.parser.plsql.PLSqlLogicalPlanBuilder;
import org.apache.doris.nereids.trees.expressions.Expression;
Expand All @@ -35,6 +37,8 @@
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.antlr.v4.runtime.CharStreams;
Expand All @@ -45,14 +49,17 @@
import org.antlr.v4.runtime.TokenSource;
import org.antlr.v4.runtime.atn.PredictionMode;
import org.antlr.v4.runtime.misc.ParseCancellationException;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.lang.reflect.Method;
import java.util.BitSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import javax.annotation.Nullable;

Expand All @@ -66,6 +73,9 @@ public class NereidsParser {

private static final BitSet EXPLAIN_TOKENS = new BitSet();

private static final Set<String> NON_RESERVED_KEYWORDS;
private static final Map<String, Integer> LITERAL_TOKENS;

static {
EXPLAIN_TOKENS.set(DorisLexer.EXPLAIN);
EXPLAIN_TOKENS.set(DorisLexer.PARSED);
Expand All @@ -77,6 +87,25 @@ public class NereidsParser {
EXPLAIN_TOKENS.set(DorisLexer.PLAN);
EXPLAIN_TOKENS.set(DorisLexer.PROCESS);

ImmutableSet.Builder<String> nonReserveds = ImmutableSet.builder();
for (Method declaredMethod : NonReservedContext.class.getDeclaredMethods()) {
if (TerminalNode.class.equals(declaredMethod.getReturnType())
&& declaredMethod.getName().toUpperCase().equals(declaredMethod.getName())
&& declaredMethod.getParameterTypes().length == 0) {
String nonReserved = declaredMethod.getName();
nonReserveds.add(nonReserved);
}
}
NON_RESERVED_KEYWORDS = nonReserveds.build();

ImmutableMap.Builder<String, Integer> literalToTokenType = ImmutableMap.builder();
for (int tokenType = 0; tokenType <= DorisLexer.VOCABULARY.getMaxTokenType(); tokenType++) {
String literalName = DorisLexer.VOCABULARY.getLiteralName(tokenType);
if (literalName != null) {
literalToTokenType.put(literalName.substring(1, literalName.length() - 1), tokenType);
}
}
LITERAL_TOKENS = literalToTokenType.build();
}

/**
Expand Down Expand Up @@ -256,9 +285,33 @@ public List<Pair<LogicalPlan, StatementContext>> parseMultiple(String sql,
}

public Expression parseExpression(String expression) {
if (isSimpleIdentifier(expression)) {
return new UnboundSlot(expression);
}
return parse(expression, DorisParser::expression);
}

private static boolean isSimpleIdentifier(String expression) {
if (expression == null || expression.isEmpty()) {
return false;
}

boolean hasLetter = false;
for (int i = 0; i < expression.length(); i++) {
char c = expression.charAt(i);
if ((('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '_' || c == '$')) {
hasLetter = true;
} else if (!('0' <= c && c <= '9')) {
return false;
}
}
if (!hasLetter) {
return false;
}
String upperCase = expression.toUpperCase();
return (NON_RESERVED_KEYWORDS.contains(upperCase) || !LITERAL_TOKENS.containsKey(upperCase));
}

public DataType parseDataType(String dataType) {
return parse(dataType, DorisParser::dataType);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -219,8 +219,9 @@ public static String parseMvColumnToMvName(String mvName, Optional<String> aggTy
}

protected static boolean containsAllColumn(Expression expression, Set<String> mvColumnNames) {
if (mvColumnNames.contains(expression.toSql()) || mvColumnNames
.contains(org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBreaker(expression.toSql()))) {
String sql = expression.toSql();
if (mvColumnNames.contains(sql) || mvColumnNames
.contains(org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBreaker(sql))) {
return true;
}
if (expression.children().isEmpty()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -625,9 +625,13 @@ private SelectResult select(LogicalOlapScan scan, Set<Slot> requiredScanOutput,
aggFuncsDiff(aggregateFunctions, aggRewriteResult), groupingExprs).isOn())
.collect(Collectors.toSet());

Set<MaterializedIndex> candidatesWithRewritingIndexes = candidatesWithRewriting.stream()
.map(result -> result.index)
.collect(Collectors.toSet());

Set<MaterializedIndex> candidatesWithoutRewriting = indexesGroupByIsBaseOrNot
.getOrDefault(false, ImmutableList.of()).stream()
.filter(index -> !candidatesWithRewriting.contains(index))
.filter(index -> !candidatesWithRewritingIndexes.contains(index))
.filter(index -> preAggEnabledByHint(scan)
|| checkPreAggStatus(scan, index.getId(), predicates, aggregateFunctions, groupingExprs).isOn())
.collect(Collectors.toSet());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ public AggregateExpression withChildren(List<Expression> children) {
}

@Override
public String toSql() {
public String computeToSql() {
if (aggregateParam.aggMode.productAggregateBuffer) {
return "partial_" + function.toSql();
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ public DataType getDataType() throws UnboundException {
}

@Override
public String toSql() {
public String computeToSql() {
return child().toSql() + " AS `" + name.get() + "`";
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ public DataType getDataType() {
}

@Override
public String toSql() {
public String computeToSql() {
return child(0).toSql();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public List<DataType> expectedInputTypes() {
}

@Override
public String toSql() {
public String computeToSql() {
return "(" + left().toSql() + " " + symbol + " " + right().toSql() + ")";
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public BoundStar(List<Slot> children) {
);
}

public String toSql() {
public String computeToSql() {
return children.stream().map(Expression::toSql).collect(Collectors.joining(", "));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ public String toString() {
}

@Override
public String toSql() throws UnboundException {
public String computeToSql() throws UnboundException {
StringBuilder output = new StringBuilder("CASE");
for (Expression child : children()) {
if (child instanceof WhenClause) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public Cast withChildren(List<Expression> children) {
}

@Override
public String toSql() throws UnboundException {
public String computeToSql() throws UnboundException {
return "cast(" + child().toSql() + " as " + targetType.toSql() + ")";
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ public boolean equals(Object o) {
}

@Override
public String toSql() {
public String computeToSql() {
StringBuilder sb = new StringBuilder();
children().forEach(c -> sb.append(c.toSql()).append(","));
sb.deleteCharAt(sb.length() - 1);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ public DataType getDataType() throws UnboundException {
}

@Override
public String toSql() {
return "EXISTS (SUBQUERY) " + super.toSql();
public String computeToSql() {
return "EXISTS (SUBQUERY) " + super.computeToSql();
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.apache.doris.nereids.analyzer.Unbound;
import org.apache.doris.nereids.analyzer.UnboundVariable;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.trees.AbstractTreeNode;
import org.apache.doris.nereids.trees.expressions.ArrayItemReference.ArrayItemSlot;
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
Expand Down Expand Up @@ -68,6 +69,7 @@ public abstract class Expression extends AbstractTreeNode<Expression> implements
private final Supplier<Set<Slot>> inputSlots = Suppliers.memoize(
() -> collect(e -> e instanceof Slot && !(e instanceof ArrayItemSlot)));
private final int fastChildrenHashCode;
private final Supplier<String> toSqlCache = Suppliers.memoize(this::computeToSql);

protected Expression(Expression... children) {
super(children);
Expand Down Expand Up @@ -210,6 +212,10 @@ public int fastChildrenHashCode() {
return fastChildrenHashCode;
}

protected String computeToSql() {
throw new UnboundException("sql");
}

protected TypeCheckResult checkInputDataTypesInternal() {
return TypeCheckResult.SUCCESS;
}
Expand Down Expand Up @@ -301,6 +307,10 @@ public boolean isInferred() {
return inferred;
}

public final String toSql() {
return toSqlCache.get();
}

@Override
public Expression withChildren(List<Expression> children) {
throw new RuntimeException();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ public String toString() {
}

@Override
public String toSql() {
public String computeToSql() {
return compareExpr.toSql() + " IN " + options.stream()
.map(Expression::toSql).sorted()
.collect(Collectors.joining(", ", "(", ")"));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,8 @@ public boolean nullable() throws UnboundException {
}

@Override
public String toSql() {
return this.compareExpr.toSql() + " IN (" + super.toSql() + ")";
public String computeToSql() {
return this.compareExpr.toSql() + " IN (" + super.computeToSql() + ")";
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public IsNull withChildren(List<Expression> children) {
}

@Override
public String toSql() throws UnboundException {
public String computeToSql() throws UnboundException {
return child().toSql() + " IS NULL";
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ public DataType getDataType() {
}

@Override
public String toSql() {
return " (LISTQUERY) " + super.toSql();
public String computeToSql() {
return " (LISTQUERY) " + super.computeToSql();
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public boolean nullable() throws UnboundException {
}

@Override
public String toSql() {
public String computeToSql() {
return "(" + left().toSql() + " " + symbol + " " + right().toSql() + ")";
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ public String toString() {
}

@Override
public String toSql() {
public String computeToSql() {
return "( not " + child().toSql() + ")";
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public String toString() {
}

@Override
public String toSql() {
public String computeToSql() {
return orderKey.toSql();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public String toString() {
}

@Override
public String toSql() {
public String computeToSql() {
return "?";
}

Expand Down
Loading
Loading