public static <V> Constraint<V> alwaysFalse() { return new Constraint<>(TupleDomain.<V>none(), Optional.of(bindings -> false)); }
public static <V> Constraint<V> alwaysTrue() { return new Constraint<>(TupleDomain.<V>all(), Optional.empty()); }
/** * Get the physical layout for a inserting into an existing table. */ default Optional<ConnectorNewTableLayout> getInsertLayout(ConnectorSession session, ConnectorTableHandle tableHandle) { List<ConnectorTableLayout> layouts = getTableLayouts(session, tableHandle, new Constraint<>(TupleDomain.all(), map -> true), Optional.empty()) .stream() .map(ConnectorTableLayoutResult::getTableLayout) .filter(layout -> layout.getTablePartitioning().isPresent()) .collect(toList()); if (layouts.isEmpty()) { return Optional.empty(); } if (layouts.size() > 1) { throw new PrestoException(NOT_SUPPORTED, "Tables with multiple layouts can not be written"); } ConnectorTableLayout layout = layouts.get(0); ConnectorPartitioningHandle partitioningHandle = layout.getTablePartitioning().get().getPartitioningHandle(); Map<ColumnHandle, String> columnNamesByHandle = getColumnHandles(session, tableHandle).entrySet().stream() .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); List<String> partitionColumns = layout.getTablePartitioning().get().getPartitioningColumns().stream() .map(columnNamesByHandle::get) .collect(toList()); return Optional.of(new ConnectorNewTableLayout(partitioningHandle, partitionColumns)); }
private List<HivePartition> getOrComputePartitions(HiveTableLayoutHandle layoutHandle, ConnectorSession session, ConnectorTableHandle tableHandle) { if (layoutHandle.getPartitions().isPresent()) { return layoutHandle.getPartitions().get(); } else { TupleDomain<ColumnHandle> promisedPredicate = layoutHandle.getPromisedPredicate(); Predicate<Map<ColumnHandle, NullableValue>> predicate = convertToPredicate(promisedPredicate); List<ConnectorTableLayoutResult> tableLayoutResults = getTableLayouts(session, tableHandle, new Constraint<>(promisedPredicate, predicate), Optional.empty()); return ((HiveTableLayoutHandle) Iterables.getOnlyElement(tableLayoutResults).getTableLayout().getHandle()).getPartitions().get(); } }
@Override public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint) { TupleDomain<ColumnHandle> targetTupleDomain = constraint.transform(fieldIdToColumnHandle::get); Predicate<Map<ColumnHandle, NullableValue>> targetPredicate = convertToPredicate(targetTupleDomain); Constraint<ColumnHandle> targetConstraint = new Constraint<>(targetTupleDomain, targetPredicate); Iterable<List<Object>> records = () -> stream(partitionManager.getPartitions(metastore, sourceTableHandle, targetConstraint).getPartitions()) .map(hivePartition -> (List<Object>) IntStream.range(0, partitionColumns.size()) .mapToObj(fieldIdToColumnHandle::get) .map(columnHandle -> hivePartition.getKeys().get(columnHandle).getValue()) .collect(toList())) .iterator(); return new InMemoryRecordSet(partitionColumnTypes, records).cursor(); } };
@Override protected Optional<PlanNodeStatsEstimate> doCalculate(TableScanNode node, StatsProvider sourceStats, Lookup lookup, Session session, TypeProvider types) { // TODO Construct predicate like AddExchanges's LayoutConstraintEvaluator Constraint<ColumnHandle> constraint = new Constraint<>(node.getCurrentConstraint()); TableStatistics tableStatistics = metadata.getTableStatistics(session, node.getTable(), constraint); Map<Symbol, SymbolStatsEstimate> outputSymbolStats = new HashMap<>(); for (Map.Entry<Symbol, ColumnHandle> entry : node.getAssignments().entrySet()) { Symbol symbol = entry.getKey(); Optional<ColumnStatistics> columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getValue())); outputSymbolStats.put(symbol, columnStatistics.map(statistics -> toSymbolStatistics(tableStatistics, statistics)).orElse(SymbolStatsEstimate.unknown())); } return Optional.of(PlanNodeStatsEstimate.builder() .setOutputRowCount(tableStatistics.getRowCount().getValue()) .addSymbolStatistics(outputSymbolStats) .build()); }
private Constraint<ColumnHandle> getConstraint(Plan plan) { Optional<TableScanNode> scanNode = searchFrom(plan.getRoot()) .where(TableScanNode.class::isInstance) .findSingle(); if (!scanNode.isPresent()) { return Constraint.alwaysFalse(); } return new Constraint<>(scanNode.get().getCurrentConstraint()); }
Constraint<ColumnHandle> targetConstraint = new Constraint<>(targetTupleDomain, targetPredicate); Iterable<List<Object>> records = () -> stream(partitionManager.getPartitions(metastore, sourceTableHandle, targetConstraint).getPartitions())
private Constraint<ColumnHandle> constraint(TpchColumn<?> column, String... values) { List<TupleDomain<ColumnHandle>> valueDomains = stream(values) .map(value -> fixedValueTupleDomain(tpchMetadata, column, utf8Slice(value))) .collect(toList()); TupleDomain<ColumnHandle> domain = TupleDomain.columnWiseUnion(valueDomains); return new Constraint<>(domain, convertToPredicate(domain, column)); }
@Test public void testGetPartitionsWithBindings() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(newSession(), tableHandle, new Constraint<>(TupleDomain.withColumnDomains(ImmutableMap.of(intColumn, Domain.singleValue(BIGINT, 5L)))), Optional.empty()); assertExpectedTableLayout(getOnlyElement(tableLayoutResults).getTableLayout(), tableLayout); } }
session, handle, new Constraint<>(originalEnforcedConstraint), Optional.of(ImmutableSet.copyOf(scan.getAssignments().values()))); verify(layouts.size() == 1, "Expected exactly one layout for delete");
/** * Tests information schema predicate pushdown when both schema and table name are specified. */ @Test public void testInformationSchemaPredicatePushdown() { TransactionId transactionId = transactionManager.beginTransaction(false); ImmutableMap.Builder<ColumnHandle, Domain> domains = new ImmutableMap.Builder<>(); domains.put(new InformationSchemaColumnHandle("table_schema"), Domain.singleValue(VARCHAR, Slices.utf8Slice("test_schema"))); domains.put(new InformationSchemaColumnHandle("table_name"), Domain.singleValue(VARCHAR, Slices.utf8Slice("test_view"))); Constraint<ColumnHandle> constraint = new Constraint<>(TupleDomain.withColumnDomains(domains.build())); InformationSchemaMetadata informationSchemaMetadata = new InformationSchemaMetadata("test_catalog", metadata); List<ConnectorTableLayoutResult> layoutResults = informationSchemaMetadata.getTableLayouts( createNewSession(transactionId), new InformationSchemaTableHandle("test_catalog", "information_schema", "views"), constraint, Optional.empty()); assertEquals(layoutResults.size(), 1); ConnectorTableLayoutHandle handle = layoutResults.get(0).getTableLayout().getHandle(); assertTrue(handle instanceof InformationSchemaTableLayoutHandle); InformationSchemaTableLayoutHandle tableHandle = (InformationSchemaTableLayoutHandle) handle; assertEquals(tableHandle.getPrefixes(), ImmutableSet.of(new QualifiedTablePrefix("test_catalog", "test_schema", "test_view"))); }
private List<ConnectorSplit> getAllSplits(ConnectorTableHandle tableHandle, TupleDomain<ColumnHandle> tupleDomain) { try (Transaction transaction = newTransaction()) { ConnectorSession session = newSession(); ConnectorMetadata metadata = transaction.getMetadata(); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(tupleDomain), Optional.empty()); ConnectorTableLayoutHandle layoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); return getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, UNGROUPED_SCHEDULING)); } }
@Test(enabled = false) public void testPartitionSchemaNonCanonical() throws Exception { try (Transaction transaction = newTransaction()) { ConnectorSession session = newSession(); ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle table = getTableHandle(metadata, tablePartitionSchemaChangeNonCanonical); ColumnHandle column = metadata.getColumnHandles(session, table).get("t_boolean"); assertNotNull(column); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, table, new Constraint<>(TupleDomain.fromFixedValues(ImmutableMap.of(column, NullableValue.of(BOOLEAN, false)))), Optional.empty()); ConnectorTableLayoutHandle layoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); assertEquals(getAllPartitions(layoutHandle).size(), 1); assertEquals(getPartitionId(getAllPartitions(layoutHandle).get(0)), "t_boolean=0"); ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, UNGROUPED_SCHEDULING); ConnectorSplit split = getOnlyElement(getAllSplits(splitSource)); ImmutableList<ColumnHandle> columnHandles = ImmutableList.of(column); try (ConnectorPageSource ignored = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, columnHandles)) { fail("expected exception"); } catch (PrestoException e) { assertEquals(e.getErrorCode(), HIVE_INVALID_PARTITION_VALUE.toErrorCode()); } } }
static class HiveTransaction implements Transaction { private final HiveTransactionManager transactionManager; private final ConnectorTransactionHandle transactionHandle; private boolean closed; public HiveTransaction(HiveTransactionManager transactionManager, HiveMetadata hiveMetadata) { this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); this.transactionHandle = new HiveTransactionHandle(); transactionManager.put(transactionHandle, hiveMetadata); getMetastore().testOnlyThrowOnCleanupFailures(); } @Override public ConnectorMetadata getMetadata() { return transactionManager.get(transactionHandle); } @Override public SemiTransactionalHiveMetastore getMetastore(String schema) { return getMetastore(); } private SemiTransactionalHiveMetastore getMetastore() { return ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore();
@Test public void testGetPartitionSplitsTableOfflinePartition() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableOfflinePartition); assertNotNull(tableHandle); ColumnHandle dsColumn = metadata.getColumnHandles(session, tableHandle).get("ds"); assertNotNull(dsColumn); Domain domain = Domain.singleValue(createUnboundedVarcharType(), utf8Slice("2012-12-30")); TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of(dsColumn, domain)); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(tupleDomain), Optional.empty()); try { getSplitCount(splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle(), UNGROUPED_SCHEDULING)); fail("Expected PartitionOfflineException"); } catch (PartitionOfflineException e) { assertEquals(e.getTableName(), tableOfflinePartition); assertEquals(e.getPartition(), "ds=2012-12-30"); } } }
private MaterializedResult readTable( Transaction transaction, ConnectorTableHandle tableHandle, List<ColumnHandle> columnHandles, ConnectorSession session, TupleDomain<ColumnHandle> tupleDomain, OptionalInt expectedSplitCount, Optional<HiveStorageFormat> expectedStorageFormat) throws Exception { List<ConnectorTableLayoutResult> tableLayoutResults = transaction.getMetadata().getTableLayouts( session, tableHandle, new Constraint<>(tupleDomain), Optional.empty()); ConnectorTableLayoutHandle layoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); List<ConnectorSplit> splits = getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, UNGROUPED_SCHEDULING)); if (expectedSplitCount.isPresent()) { assertEquals(splits.size(), expectedSplitCount.getAsInt()); } ImmutableList.Builder<MaterializedRow> allRows = ImmutableList.builder(); for (ConnectorSplit split : splits) { try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, columnHandles)) { expectedStorageFormat.ifPresent(format -> assertPageSourceType(pageSource, format)); MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles)); allRows.addAll(result.getMaterializedRows()); } } return new MaterializedResult(allRows.build(), getTypes(columnHandles)); }
Constraint<ColumnHandle> constraint = new Constraint<>( TupleDomain.all(),
@Test public void testOrdersOrderStatusPredicatePushdown() { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName("sf1", ORDERS.getTableName())); TupleDomain<ColumnHandle> domain; ConnectorTableLayoutResult tableLayout; domain = fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("P")); tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, ORDER_STATUS))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), TupleDomain.all(), session); assertTupleDomainEquals(tableLayout.getTableLayout().getPredicate(), domain, session); domain = fixedValueTupleDomain(tpchMetadata, ORDER_KEY, 42L); tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, ORDER_STATUS))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), domain, session); assertTupleDomainEquals( tableLayout.getTableLayout().getPredicate(), // The most important thing about the expected value that it is NOT TupleDomain.none() (or equivalent). // Using concrete expected value instead of checking TupleDomain::isNone to make sure the test doesn't pass on some other wrong value. TupleDomain.columnWiseUnion( fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("F")), fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("O")), fixedValueTupleDomain(tpchMetadata, ORDER_STATUS, utf8Slice("P"))), session); }
tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, PartColumn.TYPE))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), TupleDomain.all(), session); assertTupleDomainEquals( tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, PartColumn.TYPE))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), TupleDomain.all(), session); assertTupleDomainEquals(tableLayout.getTableLayout().getPredicate(), TupleDomain.none(), session); tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, PartColumn.CONTAINER))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), TupleDomain.all(), session); assertTupleDomainEquals( tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, PartColumn.CONTAINER))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), TupleDomain.all(), session); assertTupleDomainEquals(tableLayout.getTableLayout().getPredicate(), TupleDomain.none(), session); tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, PartColumn.CONTAINER))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), TupleDomain.all(), session); assertTupleDomainEquals(tableLayout.getTableLayout().getPredicate(), domain, session); tableLayout = getTableOnlyLayout(tpchMetadata, session, tableHandle, new Constraint<>(domain, convertToPredicate(domain, PartColumn.TYPE, PartColumn.CONTAINER))); assertTupleDomainEquals(tableLayout.getUnenforcedConstraint(), TupleDomain.all(), session); assertTupleDomainEquals(tableLayout.getTableLayout().getPredicate(), TupleDomain.none(), session);