Refine search
private static <F, T extends Comparable<T>> Domain createDomain(Type type, boolean hasNullValue, RangeStatistics<F> rangeStatistics, Function<F, T> function) { F min = rangeStatistics.getMin(); F max = rangeStatistics.getMax(); if (min != null && max != null) { return Domain.create(ValueSet.ofRanges(Range.range(type, function.apply(min), true, function.apply(max), true)), hasNullValue); } if (max != null) { return Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(type, function.apply(max))), hasNullValue); } if (min != null) { return Domain.create(ValueSet.ofRanges(Range.greaterThanOrEqual(type, function.apply(min))), hasNullValue); } return Domain.create(ValueSet.all(type), hasNullValue); }
@Test public void testBuildQueryNull() { TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of( COL1, Domain.create(ValueSet.ofRanges(greaterThan(BIGINT, 200L)), true))); Document query = MongoSession.buildQuery(tupleDomain); Document expected = new Document("$or", asList( new Document(COL1.getName(), new Document("$gt", 200L)), new Document(COL1.getName(), new Document("$exists", true).append("$eq", null)))); assertEquals(query, expected); } }
@Test public void testFromValueSetOfRangesUnbounded() { PrestoThriftValueSet thriftValueSet = fromValueSet(ValueSet.ofRanges(Range.greaterThanOrEqual(BIGINT, 0L))); assertNotNull(thriftValueSet.getRangeValueSet()); assertEquals(thriftValueSet.getRangeValueSet().getRanges(), ImmutableList.of( new PrestoThriftRange(new PrestoThriftMarker(longValue(0), EXACTLY), new PrestoThriftMarker(null, BELOW)))); }
@Test public void testBuildQueryOr() { TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of( COL1, Domain.create(ValueSet.ofRanges(lessThan(BIGINT, 100L), greaterThan(BIGINT, 200L)), false))); Document query = MongoSession.buildQuery(tupleDomain); Document expected = new Document("$or", asList( new Document(COL1.getName(), new Document("$lt", 100L)), new Document(COL1.getName(), new Document("$gt", 200L)))); assertEquals(query, expected); }
@Test public void testRoundTrip() { TupleDomain<Symbol> tupleDomain = withColumnDomains(ImmutableMap.<Symbol, Domain>builder() .put(C_BIGINT, Domain.singleValue(BIGINT, 1L)) .put(C_DOUBLE, Domain.onlyNull(DOUBLE)) .put(C_VARCHAR, Domain.notNull(VARCHAR)) .put(C_BOOLEAN, Domain.singleValue(BOOLEAN, true)) .put(C_BIGINT_1, Domain.singleValue(BIGINT, 2L)) .put(C_DOUBLE_1, Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(DOUBLE, 1.1), Range.equal(DOUBLE, 2.0), Range.range(DOUBLE, 3.0, false, 3.5, true)), true)) .put(C_VARCHAR_1, Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(VARCHAR, utf8Slice("2013-01-01")), Range.greaterThan(VARCHAR, utf8Slice("2013-10-01"))), false)) .put(C_TIMESTAMP, Domain.singleValue(TIMESTAMP, TIMESTAMP_VALUE)) .put(C_DATE, Domain.singleValue(DATE, DATE_VALUE)) .put(C_COLOR, Domain.singleValue(COLOR, COLOR_VALUE_1)) .put(C_HYPER_LOG_LOG, Domain.notNull(HYPER_LOG_LOG)) .build()); assertPredicateTranslates(toPredicate(tupleDomain), tupleDomain); }
@Test public void testFromValueSetOfRangesBounded() { PrestoThriftValueSet thriftValueSet = fromValueSet(ValueSet.ofRanges( range(BIGINT, -10L, true, -1L, false), range(BIGINT, -1L, false, 100L, true))); assertNotNull(thriftValueSet.getRangeValueSet()); assertEquals(thriftValueSet.getRangeValueSet().getRanges(), ImmutableList.of( new PrestoThriftRange(new PrestoThriftMarker(longValue(-10), EXACTLY), new PrestoThriftMarker(longValue(-1), BELOW)), new PrestoThriftRange(new PrestoThriftMarker(longValue(-1), ABOVE), new PrestoThriftMarker(longValue(100), EXACTLY)))); }
private static <F, T extends Comparable<T>> Domain createDomain(Type type, boolean hasNullValue, ParquetRangeStatistics<F> rangeStatistics, Function<F, T> function) { F min = rangeStatistics.getMin(); F max = rangeStatistics.getMax(); if (min != null && max != null) { return Domain.create(ValueSet.ofRanges(Range.range(type, function.apply(min), true, function.apply(max), true)), hasNullValue); } if (max != null) { return Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(type, function.apply(max))), hasNullValue); } if (min != null) { return Domain.create(ValueSet.ofRanges(Range.greaterThanOrEqual(type, function.apply(min))), hasNullValue); } return Domain.create(ValueSet.all(type), hasNullValue); } }
@Test public void testBuildQueryIn() { TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of( COL2, Domain.create(ValueSet.ofRanges(equal(createUnboundedVarcharType(), utf8Slice("hello")), equal(createUnboundedVarcharType(), utf8Slice("world"))), false))); Document query = MongoSession.buildQuery(tupleDomain); Document expected = new Document(COL2.getName(), new Document("$in", ImmutableList.of("hello", "world"))); assertEquals(query, expected); }
@Test public void testUnion() { assertUnion(Domain.all(BIGINT), Domain.all(BIGINT), Domain.all(BIGINT)); assertUnion(Domain.none(BIGINT), Domain.none(BIGINT), Domain.none(BIGINT)); assertUnion(Domain.all(BIGINT), Domain.none(BIGINT), Domain.all(BIGINT)); assertUnion(Domain.notNull(BIGINT), Domain.onlyNull(BIGINT), Domain.all(BIGINT)); assertUnion(Domain.singleValue(BIGINT, 0L), Domain.all(BIGINT), Domain.all(BIGINT)); assertUnion(Domain.singleValue(BIGINT, 0L), Domain.notNull(BIGINT), Domain.notNull(BIGINT)); assertUnion(Domain.singleValue(BIGINT, 0L), Domain.onlyNull(BIGINT), Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 0L)), true)); assertUnion(Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L)), true), Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 2L)), true), Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L), Range.equal(BIGINT, 2L)), true)); assertUnion(Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L)), true), Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L), Range.equal(BIGINT, 2L)), false), Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L), Range.equal(BIGINT, 2L)), true)); }
@Test public void testFromValueSetOfRangesUnbounded() { PrestoThriftValueSet thriftValueSet = fromValueSet(ValueSet.ofRanges(Range.greaterThanOrEqual(BIGINT, 0L))); assertNotNull(thriftValueSet.getRangeValueSet()); assertEquals(thriftValueSet.getRangeValueSet().getRanges(), ImmutableList.of( new PrestoThriftRange(new PrestoThriftMarker(longValue(0), EXACTLY), new PrestoThriftMarker(null, BELOW)))); }
private static TupleDomain<HiveColumnHandle> toCompactTupleDomain(TupleDomain<ColumnHandle> effectivePredicate, int threshold) { ImmutableMap.Builder<HiveColumnHandle, Domain> builder = ImmutableMap.builder(); effectivePredicate.getDomains().ifPresent(domains -> { for (Map.Entry<ColumnHandle, Domain> entry : domains.entrySet()) { HiveColumnHandle hiveColumnHandle = (HiveColumnHandle) entry.getKey(); ValueSet values = entry.getValue().getValues(); ValueSet compactValueSet = values.getValuesProcessor().<Optional<ValueSet>>transform( ranges -> ranges.getRangeCount() > threshold ? Optional.of(ValueSet.ofRanges(ranges.getSpan())) : Optional.empty(), discreteValues -> discreteValues.getValues().size() > threshold ? Optional.of(ValueSet.all(values.getType())) : Optional.empty(), allOrNone -> Optional.empty()) .orElse(values); builder.put(hiveColumnHandle, Domain.create(compactValueSet, entry.getValue().isNullAllowed())); } }); return TupleDomain.withColumnDomains(builder.build()); }
@Test public void testNotPushDoublePredicates() { IonSqlQueryBuilder queryBuilder = new IonSqlQueryBuilder(new TypeRegistry()); List<HiveColumnHandle> columns = ImmutableList.of( new HiveColumnHandle("quantity", HIVE_INT, parseTypeSignature(INTEGER), 0, REGULAR, Optional.empty()), new HiveColumnHandle("extendedprice", HIVE_DOUBLE, parseTypeSignature(StandardTypes.DOUBLE), 1, REGULAR, Optional.empty()), new HiveColumnHandle("discount", HIVE_DOUBLE, parseTypeSignature(StandardTypes.DOUBLE), 2, REGULAR, Optional.empty())); TupleDomain<HiveColumnHandle> tupleDomain = withColumnDomains( ImmutableMap.of( columns.get(0), Domain.create(ofRanges(Range.lessThan(BIGINT, 50L)), false), columns.get(1), Domain.create(ofRanges(Range.equal(DOUBLE, 0.05)), false), columns.get(2), Domain.create(ofRanges(Range.range(DOUBLE, 0.0, true, 0.02, true)), false))); assertEquals("SELECT s._1, s._2, s._3 FROM S3Object s WHERE ((case s._1 when '' then null else CAST(s._1 AS INT) end < 50))", queryBuilder.buildSql(columns, tupleDomain)); } }
@Test public void testInPredicateWithNull() { assertPredicateTranslates( in(C_BIGINT, Arrays.asList(1L, 2L, null)), withColumnDomains(ImmutableMap.of(C_BIGINT, Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L), Range.equal(BIGINT, 2L)), false)))); assertPredicateIsAlwaysFalse(not(in(C_BIGINT, Arrays.asList(1L, 2L, null)))); assertPredicateIsAlwaysFalse(in(C_BIGINT, Arrays.asList(new Long[] {null}))); assertPredicateIsAlwaysFalse(not(in(C_BIGINT, Arrays.asList(new Long[] {null})))); assertUnsupportedPredicate(isNull(in(C_BIGINT, Arrays.asList(1L, 2L, null)))); assertUnsupportedPredicate(isNotNull(in(C_BIGINT, Arrays.asList(1L, 2L, null)))); assertUnsupportedPredicate(isNull(in(C_BIGINT, Arrays.asList(new Long[] {null})))); assertUnsupportedPredicate(isNotNull(in(C_BIGINT, Arrays.asList(new Long[] {null})))); }
@Test public void testFromValueSetOfRangesBounded() { PrestoThriftValueSet thriftValueSet = fromValueSet(ValueSet.ofRanges( range(BIGINT, -10L, true, -1L, false), range(BIGINT, -1L, false, 100L, true))); assertNotNull(thriftValueSet.getRangeValueSet()); assertEquals(thriftValueSet.getRangeValueSet().getRanges(), ImmutableList.of( new PrestoThriftRange(new PrestoThriftMarker(longValue(-10), EXACTLY), new PrestoThriftMarker(longValue(-1), BELOW)), new PrestoThriftRange(new PrestoThriftMarker(longValue(-1), ABOVE), new PrestoThriftMarker(longValue(100), EXACTLY)))); }
private static Domain extractOrderableDomain(ComparisonExpression.Operator comparisonOperator, Type type, Object value, boolean complement) { checkArgument(value != null); switch (comparisonOperator) { case EQUAL: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.equal(type, value)), complement), false); case GREATER_THAN: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.greaterThan(type, value)), complement), false); case GREATER_THAN_OR_EQUAL: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.greaterThanOrEqual(type, value)), complement), false); case LESS_THAN: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.lessThan(type, value)), complement), false); case LESS_THAN_OR_EQUAL: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.lessThanOrEqual(type, value)), complement), false); case NOT_EQUAL: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.lessThan(type, value), Range.greaterThan(type, value)), complement), false); case IS_DISTINCT_FROM: // Need to potential complement the whole domain for IS_DISTINCT_FROM since it is null-aware return complementIfNecessary(Domain.create(ValueSet.ofRanges(Range.lessThan(type, value), Range.greaterThan(type, value)), true), complement); default: throw new AssertionError("Unhandled operator: " + comparisonOperator); } }
@Test public void testExtractFixedValues() { assertEquals( TupleDomain.extractFixedValues(TupleDomain.withColumnDomains( ImmutableMap.<ColumnHandle, Domain>builder() .put(A, Domain.all(DOUBLE)) .put(B, Domain.singleValue(VARCHAR, utf8Slice("value"))) .put(C, Domain.onlyNull(BIGINT)) .put(D, Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L)), true)) .build())).get(), ImmutableMap.of( B, NullableValue.of(VARCHAR, utf8Slice("value")), C, NullableValue.asNull(BIGINT))); }
@Test public void testFromInPredicate() { assertPredicateTranslates( in(C_BIGINT, ImmutableList.of(1L)), withColumnDomains(ImmutableMap.of(C_BIGINT, Domain.singleValue(BIGINT, 1L)))); assertPredicateTranslates( in(C_COLOR, ImmutableList.of(colorLiteral(COLOR_VALUE_1))), withColumnDomains(ImmutableMap.of(C_COLOR, Domain.singleValue(COLOR, COLOR_VALUE_1)))); assertPredicateTranslates( in(C_BIGINT, ImmutableList.of(1L, 2L)), withColumnDomains(ImmutableMap.of(C_BIGINT, Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L), Range.equal(BIGINT, 2L)), false)))); assertPredicateTranslates( in(C_COLOR, ImmutableList.of(colorLiteral(COLOR_VALUE_1), colorLiteral(COLOR_VALUE_2))), withColumnDomains(ImmutableMap.of(C_COLOR, Domain.create(ValueSet.of(COLOR, COLOR_VALUE_1, COLOR_VALUE_2), false)))); assertPredicateTranslates( not(in(C_BIGINT, ImmutableList.of(1L, 2L))), withColumnDomains(ImmutableMap.of(C_BIGINT, Domain.create(ValueSet.ofRanges(Range.lessThan(BIGINT, 1L), Range.range(BIGINT, 1L, false, 2L, false), Range.greaterThan(BIGINT, 2L)), false)))); assertPredicateTranslates( not(in(C_COLOR, ImmutableList.of(colorLiteral(COLOR_VALUE_1), colorLiteral(COLOR_VALUE_2)))), withColumnDomains(ImmutableMap.of(C_COLOR, Domain.create(ValueSet.of(COLOR, COLOR_VALUE_1, COLOR_VALUE_2).complement(), false)))); }
/** * Reduces the number of discrete components in the Domain if there are too many. */ public Domain simplify() { ValueSet simplifiedValueSet = values.getValuesProcessor().<Optional<ValueSet>>transform( ranges -> { if (ranges.getOrderedRanges().size() <= 32) { return Optional.empty(); } return Optional.of(ValueSet.ofRanges(ranges.getSpan())); }, discreteValues -> { if (discreteValues.getValues().size() <= 32) { return Optional.empty(); } return Optional.of(ValueSet.all(values.getType())); }, allOrNone -> Optional.empty()) .orElse(values); return Domain.create(simplifiedValueSet, nullAllowed); }
@Test public void testBuildQueryStringType() { TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of( COL1, Domain.create(ValueSet.ofRanges(range(createUnboundedVarcharType(), utf8Slice("hello"), false, utf8Slice("world"), true)), false), COL2, Domain.create(ValueSet.ofRanges(greaterThanOrEqual(createUnboundedVarcharType(), utf8Slice("a value"))), false))); Document query = MongoSession.buildQuery(tupleDomain); Document expected = new Document() .append(COL1.getName(), new Document().append("$gt", "hello").append("$lte", "world")) .append(COL2.getName(), new Document("$gte", "a value")); assertEquals(query, expected); }