diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/externalize/RelJson.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/externalize/RelJson.java index f2e414ec591..033e42f7d36 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/externalize/RelJson.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/externalize/RelJson.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.sql.engine.externalize; +import static java.util.Objects.requireNonNull; import static org.apache.calcite.sql.type.SqlTypeUtil.isApproximateNumeric; import static org.apache.ignite.internal.lang.IgniteStringFormatter.format; import static org.apache.ignite.internal.sql.engine.util.Commons.FRAMEWORK_CONFIG; @@ -24,9 +25,17 @@ import static org.apache.ignite.internal.util.IgniteUtils.igniteClassLoader; import static org.apache.ignite.lang.ErrorGroups.Common.INTERNAL_ERR; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableRangeSet; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; import java.lang.reflect.Constructor; import java.lang.reflect.Modifier; import java.math.BigDecimal; @@ -40,6 +49,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.calcite.avatica.AvaticaUtils; import org.apache.calcite.avatica.util.ByteString; @@ -61,6 +71,7 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.externalize.RelEnumTypes; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeFactoryImpl.JavaType; @@ -74,7 +85,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexOver; import org.apache.calcite.rex.RexSlot; -import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.rex.RexUnknownAs; import org.apache.calcite.rex.RexVariable; import org.apache.calcite.rex.RexWindow; import org.apache.calcite.rex.RexWindowBound; @@ -103,7 +114,13 @@ import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.validate.SqlNameMatchers; +import org.apache.calcite.util.DateString; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.NlsString; +import org.apache.calcite.util.RangeSets; +import org.apache.calcite.util.Sarg; +import org.apache.calcite.util.TimeString; +import org.apache.calcite.util.TimestampString; import org.apache.calcite.util.Util; import org.apache.ignite.internal.lang.IgniteInternalException; import org.apache.ignite.internal.sql.engine.prepare.bounds.ExactBounds; @@ -120,13 +137,22 @@ import org.apache.ignite.internal.sql.engine.type.IgniteTypeFactory; import org.apache.ignite.internal.sql.engine.util.Commons; import org.apache.ignite.internal.util.IgniteUtils; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; /** * Utilities for converting {@link RelNode} into JSON format. */ @SuppressWarnings({"rawtypes", "unchecked"}) class RelJson { - @SuppressWarnings("PublicInnerClass") + private static final ObjectMapper OBJECT_MAPPER = + new ObjectMapper() + .configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, true); + + private static final List VALUE_CLASSES = + ImmutableList.of(NlsString.class, BigDecimal.class, ByteString.class, + Boolean.class, TimestampString.class, DateString.class, TimeString.class); + @FunctionalInterface public interface RelFactory extends Function { /** {@inheritDoc} */ @@ -183,6 +209,7 @@ private static RelFactory relFactory(String typeName) { register(enumByName, JoinType.class); register(enumByName, Direction.class); register(enumByName, NullDirection.class); + register(enumByName, RexUnknownAs.class); register(enumByName, SqlTypeName.class); register(enumByName, SqlKind.class); register(enumByName, SqlSyntax.class); @@ -235,7 +262,7 @@ private static Class classForName(String typeName, boolean skipNotFound) { * Constructor. * TODO Documentation https://issues.apache.org/jira/browse/IGNITE-15859 */ - RelJson() { + public RelJson() { } Function factory(String type) { @@ -305,6 +332,12 @@ Object toJson(Object value) { return toJson((RelDataType) value); } else if (value instanceof RelDataTypeField) { return toJson((RelDataTypeField) value); + } else if (value instanceof Sarg) { + return toJson((Sarg) value); + } else if (value instanceof RangeSet) { + return toJson((RangeSet) value); + } else if (value instanceof Range) { + return toJson((Range) value); } else if (value instanceof ByteString) { return toJson((ByteString) value); } else if (value instanceof SearchBounds) { @@ -344,6 +377,32 @@ private Object toJson(AggregateCall node) { return map; } + private > Object toJson(Sarg node) { + final Map map = map(); + map.put("rangeSet", toJson(node.rangeSet)); + map.put("nullAs", RelEnumTypes.fromEnum(node.nullAs)); + return map; + } + + private static > List> toJson( + RangeSet rangeSet) { + List> list = new ArrayList<>(); + + try { + RangeSets.forEach(rangeSet, + RangeToJsonConverter.instance().andThen(list::add)); + } catch (Exception e) { + throw new RuntimeException("Failed to serialize RangeSet: ", e); + } + return list; + } + + /** Serializes a {@link Range} that can be deserialized using + * {@link org.apache.calcite.rel.externalize.RelJson#rangeFromJson(List)}. */ + private > List toJson(Range range) { + return RangeSets.map(range, RangeToJsonConverter.instance()); + } + private Object toJson(RelDataType node) { if (node instanceof JavaType) { Map map = map(); @@ -410,9 +469,6 @@ private Object toJson(CorrelationId node) { } private Object toJson(RexNode node) { - // removes calls to SEARCH and the included Sarg and converts them to comparisons - node = RexUtil.expandSearch(Commons.emptyCluster().getRexBuilder(), null, node); - Map map; switch (node.getKind()) { case FIELD_ACCESS: @@ -590,6 +646,10 @@ private Object toJson(SqlOperator operator) { map.put("name", operator.getName()); map.put("kind", toJson(operator.kind)); map.put("syntax", toJson(operator.getSyntax())); + + if (operator.getOperandTypeChecker() != null && operator.getAllowedSignatures() != null) { + map.put("signature", toJson(operator.getAllowedSignatures())); + } return map; } @@ -865,6 +925,12 @@ RexNode toRex(RelInput relInput, Object o) { Object literal = map.get("literal"); RelDataType type = toType(typeFactory, map.get("type")); + if (literal instanceof Map + && ((Map) literal).containsKey("rangeSet")) { + Sarg sarg = sargFromJson((Map) literal); + return rexBuilder.makeSearchArgumentLiteral(sarg, type); + } + if (literal == null) { return rexBuilder.makeNullLiteral(type); } @@ -876,6 +942,12 @@ RexNode toRex(RelInput relInput, Object o) { literal = new BigDecimal(((Number) literal).longValue()); } + // Stub, it need to be fixed https://issues.apache.org/jira/browse/IGNITE-23873 + if (type.getSqlTypeName() == SqlTypeName.DOUBLE && literal instanceof String + && Double.isNaN(Double.parseDouble(literal.toString()))) { + literal = Double.NaN; + } + if (literal instanceof BigInteger) { // If the literal is a BigInteger, RexBuilder assumes it represents a long value // within the valid range and converts it without checking the bounds. If the @@ -915,26 +987,141 @@ RexNode toRex(RelInput relInput, Object o) { } } + + /** Converts a JSON object to a {@code Sarg}. + * + *

For example, + * {@code {rangeSet: [["[", 0, 5, "]"], ["[", 10, "-", ")"]], + * nullAs: "UNKNOWN"}} represents the range x ≥ 0 and x ≤ 5 or + * x > 10. + */ + public static > Sarg sargFromJson( + Map map) { + final String nullAs = requireNonNull((String) map.get("nullAs"), "nullAs"); + final List> rangeSet = + requireNonNull((List>) map.get("rangeSet"), "rangeSet"); + + String enumName = RexUnknownAs.class.getSimpleName() + '#' + nullAs; + + return Sarg.of((RexUnknownAs) ENUM_BY_NAME.get(enumName), RelJson.rangeSetFromJson(rangeSet)); + } + + /** Converts a JSON list to a {@link RangeSet}. */ + private static > RangeSet rangeSetFromJson( + List> rangeSetsJson) { + final ImmutableRangeSet.Builder builder = ImmutableRangeSet.builder(); + try { + rangeSetsJson.forEach(list -> builder.add(rangeFromJson(list))); + } catch (Exception e) { + throw new RuntimeException("Error creating RangeSet from JSON: ", e); + } + return builder.build(); + } + + /** Creates a {@link Range} from a JSON object. + * + *

The JSON object is as serialized using {@link #toJson(Range)}, + * e.g. {@code ["[", ")", 10, "-"]}. + * + * @see RangeToJsonConverter */ + private static > Range rangeFromJson( + List list) { + switch (list.get(0)) { + case "all": + return Range.all(); + case "atLeast": + return Range.atLeast(rangeEndPointFromJson(list.get(1))); + case "atMost": + return Range.atMost(rangeEndPointFromJson(list.get(1))); + case "greaterThan": + return Range.greaterThan(rangeEndPointFromJson(list.get(1))); + case "lessThan": + return Range.lessThan(rangeEndPointFromJson(list.get(1))); + case "singleton": + return Range.singleton(rangeEndPointFromJson(list.get(1))); + case "closed": + return Range.closed(rangeEndPointFromJson(list.get(1)), + rangeEndPointFromJson(list.get(2))); + case "closedOpen": + return Range.closedOpen(rangeEndPointFromJson(list.get(1)), + rangeEndPointFromJson(list.get(2))); + case "openClosed": + return Range.openClosed(rangeEndPointFromJson(list.get(1)), + rangeEndPointFromJson(list.get(2))); + case "open": + return Range.open(rangeEndPointFromJson(list.get(1)), + rangeEndPointFromJson(list.get(2))); + default: + throw new AssertionError("unknown range type " + list.get(0)); + } + } + + static class ByteStringWrapper { + private ByteString value; + + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) + public ByteStringWrapper(@JsonProperty("value") String value) { + this.value = ByteString.ofBase64(value); + } + + @JsonProperty("value") + public ByteString getValue() { + return value; + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static > C rangeEndPointFromJson(Object o) { + Exception e = null; + for (Class clsType : VALUE_CLASSES) { + try { + if (clsType == ByteString.class) { + ByteStringWrapper wrapper = OBJECT_MAPPER.readValue((String) o, ByteStringWrapper.class); + + return (C) wrapper.getValue(); + } + + return (C) OBJECT_MAPPER.readValue((String) o, clsType); + } catch (JsonProcessingException ex) { + e = ex; + } + } + throw new RuntimeException( + "Error deserializing range endpoint (did not find compatible type): ", + e); + } + SqlOperator toOp(Map map) { // in case different operator has the same kind, check with both name and kind. String name = map.get("name").toString(); SqlKind sqlKind = toEnum(map.get("kind")); SqlSyntax sqlSyntax = toEnum(map.get("syntax")); + String sig = (String) map.get("signature"); + Predicate signature = s -> sig == null || sig.equals(s); List operators = new ArrayList<>(); FRAMEWORK_CONFIG.getOperatorTable().lookupOperatorOverloads( - new SqlIdentifier(name, new SqlParserPos(0, 0)), + new SqlIdentifier(name, SqlParserPos.ZERO), null, sqlSyntax, operators, SqlNameMatchers.liberal() ); + for (SqlOperator operator : operators) { + if (operator.kind == sqlKind && (operator.getOperandTypeChecker() == null || signature.test(operator.getAllowedSignatures()))) { + return operator; + } + } + + // Fallback still need for IgniteSqlOperatorTable.EQUALS and so on operators, can be removed + // after operandTypeChecker will be aligned for (SqlOperator operator : operators) { if (operator.kind == sqlKind) { return operator; } } + String cls = (String) map.get("class"); if (cls != null) { return AvaticaUtils.instantiatePlugin(SqlOperator.class, cls); @@ -1043,4 +1230,69 @@ private List toRexList(RelInput relInput, List operands) { } return list; } + + /** Implementation of {@link RangeSets.Handler} that converts a {@link Range} + * event to a list of strings. + * + * @param Range value type + */ + private static class RangeToJsonConverter + implements RangeSets.Handler<@NonNull V, List> { + private static final RangeToJsonConverter INSTANCE = + new RangeToJsonConverter<>(); + + private static > RangeToJsonConverter instance() { + return INSTANCE; + } + + @Override public List all() { + return ImmutableList.of("all"); + } + + @Override public List atLeast(@NonNull V lower) { + return ImmutableList.of("atLeast", toJson(lower)); + } + + @Override public List atMost(@NonNull V upper) { + return ImmutableList.of("atMost", toJson(upper)); + } + + @Override public List greaterThan(@NonNull V lower) { + return ImmutableList.of("greaterThan", toJson(lower)); + } + + @Override public List lessThan(@NonNull V upper) { + return ImmutableList.of("lessThan", toJson(upper)); + } + + @Override public List singleton(@NonNull V value) { + return ImmutableList.of("singleton", toJson(value)); + } + + @Override public List closed(@NonNull V lower, @NonNull V upper) { + return ImmutableList.of("closed", toJson(lower), toJson(upper)); + } + + @Override public List closedOpen(@NonNull V lower, + @NonNull V upper) { + return ImmutableList.of("closedOpen", toJson(lower), toJson(upper)); + } + + @Override public List openClosed(@NonNull V lower, + @NonNull V upper) { + return ImmutableList.of("openClosed", toJson(lower), toJson(upper)); + } + + @Override public List open(@NonNull V lower, @NonNull V upper) { + return ImmutableList.of("open", toJson(lower), toJson(upper)); + } + + private static String toJson(Object o) { + try { + return OBJECT_MAPPER.writeValueAsString(o); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize Range endpoint: ", e); + } + } + } } diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rel/ProjectableFilterableTableScan.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rel/ProjectableFilterableTableScan.java index 75c604d4fdc..3743dbd8820 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rel/ProjectableFilterableTableScan.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rel/ProjectableFilterableTableScan.java @@ -143,8 +143,7 @@ public RelNode accept(RexShuttle shuttle) { protected RelWriter explainTerms0(RelWriter pw) { if (condition != null) { - pw.item("filters", pw.nest() ? condition : - RexUtil.expandSearch(getCluster().getRexBuilder(), null, condition)); + pw.item("filters", condition); } return pw.itemIf("projects", projects, projects != null) diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/sql/fun/IgniteSqlOperatorTable.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/sql/fun/IgniteSqlOperatorTable.java index 7e14873ae62..de4c5f3b8d8 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/sql/fun/IgniteSqlOperatorTable.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/sql/fun/IgniteSqlOperatorTable.java @@ -509,6 +509,8 @@ public IgniteSqlOperatorTable() { register(EVERY); register(SOME); + register(SqlStdOperatorTable.SEARCH); + // IS ... operator. register(SqlStdOperatorTable.IS_NULL); register(SqlStdOperatorTable.IS_NOT_NULL); diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/DynamicParametersTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/DynamicParametersTest.java index 6f5b590d7df..b94444681b2 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/DynamicParametersTest.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/DynamicParametersTest.java @@ -131,16 +131,15 @@ public Stream testInExpression() { .parameterTypes(nullable(NativeTypes.INT32)) .fails("Values passed to IN operator must have compatible types"), + sql("SELECT CAST(? as INT) IN (1, 2)", "1") + .parameterTypes(nullable(NativeTypes.STRING)) + .project("SEARCH(CAST(?0):INTEGER, Sarg[1, 2])"), + sql("SELECT ? IN ('1', 2)", 2) .parameterTypes(nullable(NativeTypes.INT32)) .fails("Values in expression list must have compatible types") ); - // TODO https://issues.apache.org/jira/browse/IGNITE-23039 Add support for Sarg serialization/deserialization - // sql("SELECT ? IN (1, 2)", "1") - // .parameterTypes(nullable(NativeTypes.STRING)) - // .project("SEARCH(CAST(?0):INTEGER, Sarg[1, 2])"), - // TODO https://issues.apache.org/jira/browse/IGNITE-22084: Sql. Add support for row data type. // sql("SELECT (?,?) IN ((1,2))", 1, 2) // .parameterTypes(nullable(NativeTypes.INT32), nullable(NativeTypes.INT32))