public TypeProvider getTypes() { return TypeProvider.copyOf(symbols); } }
protected final OperatorFactory createHashProjectOperator(int operatorId, PlanNodeId planNodeId, List<Type> types) { ImmutableMap.Builder<Symbol, Type> symbolTypes = ImmutableMap.builder(); ImmutableMap.Builder<Symbol, Integer> symbolToInputMapping = ImmutableMap.builder(); ImmutableList.Builder<PageProjection> projections = ImmutableList.builder(); for (int channel = 0; channel < types.size(); channel++) { Symbol symbol = new Symbol("h" + channel); symbolTypes.put(symbol, types.get(channel)); symbolToInputMapping.put(symbol, channel); projections.add(new InputPageProjection(channel, types.get(channel))); } Optional<Expression> hashExpression = HashGenerationOptimizer.getHashExpression(ImmutableList.copyOf(symbolTypes.build().keySet())); verify(hashExpression.isPresent()); projections.add(new InterpretedPageProjection( hashExpression.get(), TypeProvider.copyOf(symbolTypes.build()), symbolToInputMapping.build(), localQueryRunner.getMetadata(), localQueryRunner.getSqlParser(), session)); return new FilterAndProjectOperator.FilterAndProjectOperatorFactory( operatorId, planNodeId, () -> new PageProcessor(Optional.empty(), projections.build()), ImmutableList.copyOf(Iterables.concat(types, ImmutableList.of(BIGINT))), getFilterAndProjectMinOutputPageSize(session), getFilterAndProjectMinOutputPageRowCount(session)); }
private PlanNodeStatsAssertion assertNormalized(PlanNodeStatsEstimate estimate) { TypeProvider types = TypeProvider.copyOf(estimate.getSymbolsWithKnownStatistics().stream() .collect(toImmutableMap(identity(), symbol -> BIGINT))); return assertNormalized(estimate, types); }
@Test public void testSymbolReference() { Symbol symbol = new Symbol("symbol"); ImmutableMap<Symbol, Integer> symbolToInputMappings = ImmutableMap.of(symbol, 0); assertProjection("symbol", true, symbolToInputMappings, TypeProvider.copyOf(ImmutableMap.of(symbol, BOOLEAN)), 0, createBlock(BOOLEAN, true)); assertProjection("symbol", null, symbolToInputMappings, TypeProvider.copyOf(ImmutableMap.of(symbol, BOOLEAN)), 0, createNullBlock(BOOLEAN)); assertProjection("symbol", 42L, symbolToInputMappings, TypeProvider.copyOf(ImmutableMap.of(symbol, BIGINT)), 0, createBlock(BIGINT, 42)); assertProjection("symbol", null, symbolToInputMappings, TypeProvider.copyOf(ImmutableMap.of(symbol, BIGINT)), 0, createNullBlock(BIGINT)); assertProjection("symbol", 11.1, symbolToInputMappings, TypeProvider.copyOf(ImmutableMap.of(symbol, DOUBLE)), 0, createBlock(DOUBLE, 11.1)); assertProjection("symbol", null, symbolToInputMappings, TypeProvider.copyOf(ImmutableMap.of(symbol, DOUBLE)), 0, createNullBlock(DOUBLE)); assertProjection("symbol", "foo", symbolToInputMappings, TypeProvider.copyOf(ImmutableMap.of(symbol, VARCHAR)), 0, createBlock(VARCHAR, "foo")); assertProjection("symbol", null, symbolToInputMappings, TypeProvider.copyOf(ImmutableMap.of(symbol, VARCHAR)), 0, createNullBlock(VARCHAR)); }
private PlanNodeCostEstimate calculateCumulativeCost(PlanNode node, CostCalculator costCalculator, StatsCalculator statsCalculator, Map<String, Type> types) { TypeProvider typeProvider = TypeProvider.copyOf(types.entrySet().stream() .collect(ImmutableMap.toImmutableMap(entry -> new Symbol(entry.getKey()), Map.Entry::getValue))); StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, typeProvider); CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.empty(), session, typeProvider); return costProvider.getCumulativeCost(node); }
private RowExpression rowExpression(String expression) { Expression inputReferenceExpression = new SymbolToInputRewriter(sourceLayout).rewrite(createExpression(expression, metadata, TypeProvider.copyOf(symbolTypes))); Map<Integer, Type> types = sourceLayout.entrySet().stream() .collect(toMap(Map.Entry::getValue, entry -> symbolTypes.get(entry.getKey()))); Map<NodeRef<Expression>, Type> expressionTypes = getExpressionTypesFromInput(TEST_SESSION, metadata, SQL_PARSER, types, inputReferenceExpression, emptyList(), WarningCollector.NOOP); return SqlToRowExpressionTranslator.translate(inputReferenceExpression, SCALAR, expressionTypes, metadata.getFunctionRegistry(), metadata.getTypeManager(), TEST_SESSION, true); }
private PlanNodeCostEstimate calculateCumulativeCost( CostCalculator costCalculator, PlanNode node, Function<PlanNode, PlanNodeCostEstimate> costs, Function<PlanNode, PlanNodeStatsEstimate> stats, Map<String, Type> types) { PlanNodeCostEstimate localCost = costCalculator.calculateCost( node, planNode -> requireNonNull(stats.apply(planNode), "no stats for node"), session, TypeProvider.copyOf(types.entrySet().stream() .collect(ImmutableMap.toImmutableMap(entry -> new Symbol(entry.getKey()), Map.Entry::getValue)))); PlanNodeCostEstimate sourcesCost = node.getSources().stream() .map(source -> requireNonNull(costs.apply(source), format("no cost for source: %s", source.getId()))) .reduce(PlanNodeCostEstimate.zero(), PlanNodeCostEstimate::add); return sourcesCost.add(localCost); }
TypeProvider typeProvider = TypeProvider.copyOf(allFragments.stream() .flatMap(f -> f.getSymbols().entrySet().stream()) .distinct()
metadata, sqlParser, TypeProvider.copyOf(lambdaArgumentSymbolTypes), lambdaExpression.getBody(), emptyList(),
private RowExpression rowExpression(String expression, Type type) { SymbolToInputRewriter symbolToInputRewriter = new SymbolToInputRewriter(sourceLayout); Expression inputReferenceExpression = symbolToInputRewriter.rewrite(createExpression(expression, METADATA, TypeProvider.copyOf(symbolTypes))); ImmutableMap.Builder<Integer, Type> builder = ImmutableMap.builder(); for (int i = 0; i < columnCount; i++) { builder.put(i, type); } Map<Integer, Type> types = builder.build(); Map<NodeRef<Expression>, Type> expressionTypes = getExpressionTypesFromInput(TEST_SESSION, METADATA, SQL_PARSER, types, inputReferenceExpression, emptyList(), WarningCollector.NOOP); return SqlToRowExpressionTranslator.translate(inputReferenceExpression, SCALAR, expressionTypes, METADATA.getFunctionRegistry(), METADATA.getTypeManager(), TEST_SESSION, true); }
private static void assertNotEquivalent(@Language("SQL") String left, @Language("SQL") String right) { ParsingOptions parsingOptions = new ParsingOptions(AS_DOUBLE /* anything */); Expression leftExpression = rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(left, parsingOptions)); Expression rightExpression = rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(right, parsingOptions)); Set<Symbol> symbols = extractUnique(ImmutableList.of(leftExpression, rightExpression)); TypeProvider types = TypeProvider.copyOf(symbols.stream() .collect(toMap(identity(), TestExpressionEquivalence::generateType))); assertFalse( EQUIVALENCE.areExpressionsEquivalent(TEST_SESSION, leftExpression, rightExpression, types), String.format("Expected (%s) and (%s) to not be equivalent", left, right)); assertFalse( EQUIVALENCE.areExpressionsEquivalent(TEST_SESSION, rightExpression, leftExpression, types), String.format("Expected (%s) and (%s) to not be equivalent", right, left)); }
private static void assertEquivalent(@Language("SQL") String left, @Language("SQL") String right) { ParsingOptions parsingOptions = new ParsingOptions(AS_DOUBLE /* anything */); Expression leftExpression = rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(left, parsingOptions)); Expression rightExpression = rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(right, parsingOptions)); Set<Symbol> symbols = extractUnique(ImmutableList.of(leftExpression, rightExpression)); TypeProvider types = TypeProvider.copyOf(symbols.stream() .collect(toMap(identity(), TestExpressionEquivalence::generateType))); assertTrue( EQUIVALENCE.areExpressionsEquivalent(TEST_SESSION, leftExpression, rightExpression, types), String.format("Expected (%s) and (%s) to be equivalent", left, right)); assertTrue( EQUIVALENCE.areExpressionsEquivalent(TEST_SESSION, rightExpression, leftExpression, types), String.format("Expected (%s) and (%s) to be equivalent", right, left)); }
private PlanNodeCostEstimate calculateCumulativeCostFragmentedPlan(PlanNode node, StatsCalculator statsCalculator, Map<String, Type> types) { TypeProvider typeProvider = TypeProvider.copyOf(types.entrySet().stream() .collect(ImmutableMap.toImmutableMap(entry -> new Symbol(entry.getKey()), Map.Entry::getValue))); StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, typeProvider); CostProvider costProvider = new CachingCostProvider(costCalculatorUsingExchanges, statsProvider, Optional.empty(), session, typeProvider); SubPlan subPlan = fragment(new Plan(node, typeProvider, StatsAndCosts.create(node, statsProvider, costProvider))); return subPlan.getFragment().getStatsAndCosts().getCosts().getOrDefault(node.getId(), PlanNodeCostEstimate.unknown()); }
taskContext, fragment.getRoot(), TypeProvider.copyOf(fragment.getSymbols()), fragment.getPartitioningScheme(), fragment.getStageExecutionDescriptor(),
private CostAssertionBuilder assertCostFragmentedPlan( PlanNode node, Map<String, PlanNodeCostEstimate> costs, Map<String, PlanNodeStatsEstimate> stats, Map<String, Type> types) { TypeProvider typeProvider = TypeProvider.copyOf(types.entrySet().stream() .collect(ImmutableMap.toImmutableMap(entry -> new Symbol(entry.getKey()), Map.Entry::getValue))); StatsProvider statsProvider = new CachingStatsProvider(statsCalculator(stats), session, typeProvider); CostProvider costProvider = new TestingCostProvider(costs, costCalculatorUsingExchanges, statsProvider, session, typeProvider); SubPlan subPlan = fragment(new Plan(node, typeProvider, StatsAndCosts.create(node, statsProvider, costProvider))); return new CostAssertionBuilder(subPlan.getFragment().getStatsAndCosts().getCosts().getOrDefault(node.getId(), PlanNodeCostEstimate.unknown())); }
@Test public void testDropNonOutputSymbols() { Symbol a = new Symbol("a"); Symbol b = new Symbol("b"); Symbol c = new Symbol("c"); PlanNodeStatsEstimate estimate = PlanNodeStatsEstimate.builder() .setOutputRowCount(40) .addSymbolStatistics(a, SymbolStatsEstimate.builder().setDistinctValuesCount(20).build()) .addSymbolStatistics(b, SymbolStatsEstimate.builder().setDistinctValuesCount(30).build()) .addSymbolStatistics(c, SymbolStatsEstimate.unknown()) .build(); PlanNodeStatsAssertion.assertThat(normalizer.normalize(estimate, ImmutableList.of(b, c), TypeProvider.copyOf(ImmutableMap.of(b, BIGINT, c, BIGINT)))) .symbolsWithKnownStats(b) .symbolStats(b, symbolAssert -> symbolAssert.distinctValuesCount(30)); }
private void testCapDistinctValuesByToDomainRangeLength(Type type, double ndv, Object low, Object high, double expectedNormalizedNdv) { Symbol symbol = new Symbol("x"); SymbolStatsEstimate symbolStats = SymbolStatsEstimate.builder() .setNullsFraction(0) .setDistinctValuesCount(ndv) .setLowValue(asStatsValue(low, type)) .setHighValue(asStatsValue(high, type)) .build(); PlanNodeStatsEstimate estimate = PlanNodeStatsEstimate.builder() .setOutputRowCount(10000000000L) .addSymbolStatistics(symbol, symbolStats).build(); assertNormalized(estimate, TypeProvider.copyOf(ImmutableMap.of(symbol, type))) .symbolStats(symbol, symbolAssert -> symbolAssert.distinctValuesCount(expectedNormalizedNdv)); }
.build(); standardTypes = TypeProvider.copyOf(ImmutableMap.<Symbol, Type>builder() .put(new Symbol("x"), DoubleType.DOUBLE) .put(new Symbol("y"), DoubleType.DOUBLE)
.build(); types = TypeProvider.copyOf(ImmutableMap.<Symbol, Type>builder() .put(new Symbol("u"), DoubleType.DOUBLE) .put(new Symbol("w"), DoubleType.DOUBLE)
protected final OperatorFactory createHashProjectOperator(int operatorId, PlanNodeId planNodeId, List<Type> types) { ImmutableMap.Builder<Symbol, Type> symbolTypes = ImmutableMap.builder(); ImmutableMap.Builder<Symbol, Integer> symbolToInputMapping = ImmutableMap.builder(); ImmutableList.Builder<PageProjection> projections = ImmutableList.builder(); for (int channel = 0; channel < types.size(); channel++) { Symbol symbol = new Symbol("h" + channel); symbolTypes.put(symbol, types.get(channel)); symbolToInputMapping.put(symbol, channel); projections.add(new InputPageProjection(channel, types.get(channel))); } Optional<Expression> hashExpression = HashGenerationOptimizer.getHashExpression(ImmutableList.copyOf(symbolTypes.build().keySet())); verify(hashExpression.isPresent()); projections.add(new InterpretedPageProjection( hashExpression.get(), TypeProvider.copyOf(symbolTypes.build()), symbolToInputMapping.build(), localQueryRunner.getMetadata(), localQueryRunner.getSqlParser(), session)); return new FilterAndProjectOperator.FilterAndProjectOperatorFactory( operatorId, planNodeId, () -> new PageProcessor(Optional.empty(), projections.build()), ImmutableList.copyOf(Iterables.concat(types, ImmutableList.of(BIGINT))), getFilterAndProjectMinOutputPageSize(session), getFilterAndProjectMinOutputPageRowCount(session)); }