private Optional<HivePartition> parseValuesAndFilterPartition( SchemaTableName tableName, String partitionId, List<HiveColumnHandle> partitionColumns, List<Type> partitionColumnTypes, Constraint<ColumnHandle> constraint) { HivePartition partition = parsePartition(tableName, partitionId, partitionColumns, partitionColumnTypes, timeZone); Map<ColumnHandle, Domain> domains = constraint.getSummary().getDomains().get(); for (HiveColumnHandle column : partitionColumns) { NullableValue value = partition.getKeys().get(column); Domain allowedDomain = domains.get(column); if (allowedDomain != null && !allowedDomain.includesNullableValue(value.getValue())) { return Optional.empty(); } } if (constraint.predicate().isPresent() && !constraint.predicate().get().test(partition.getKeys())) { return Optional.empty(); } return Optional.of(partition); }
protected final OperatorFactory createTableScanOperator(int operatorId, PlanNodeId planNodeId, String tableName, String... columnNames) checkArgument(session.getCatalog().isPresent(), "catalog not set"); checkArgument(session.getSchema().isPresent(), "schema not set"); QualifiedObjectName qualifiedTableName = new QualifiedObjectName(session.getCatalog().get(), session.getSchema().get(), tableName); TableHandle tableHandle = metadata.getTableHandle(session, qualifiedTableName).orElse(null); checkArgument(tableHandle != null, "Table %s does not exist", qualifiedTableName); ImmutableList.Builder<ColumnHandle> columnHandlesBuilder = ImmutableList.builder(); for (String columnName : columnNames) { ColumnHandle columnHandle = allColumnHandles.get(columnName); List<TableLayoutResult> layouts = metadata.getLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); Split split = getLocalQuerySplit(session, layouts.get(0).getLayout().getHandle());
private List<HivePartition> getOrComputePartitions(HiveTableLayoutHandle layoutHandle, ConnectorSession session, ConnectorTableHandle tableHandle) { if (layoutHandle.getPartitions().isPresent()) { return layoutHandle.getPartitions().get(); } else { TupleDomain<ColumnHandle> promisedPredicate = layoutHandle.getPromisedPredicate(); Predicate<Map<ColumnHandle, NullableValue>> predicate = convertToPredicate(promisedPredicate); List<ConnectorTableLayoutResult> tableLayoutResults = getTableLayouts(session, tableHandle, new Constraint<>(promisedPredicate, predicate), Optional.empty()); return ((HiveTableLayoutHandle) Iterables.getOnlyElement(tableLayoutResults).getTableLayout().getHandle()).getPartitions().get(); } }
private Set<NullableValue> filterValues(Set<NullableValue> nullableValues, TpchColumn<?> column, Constraint<ColumnHandle> constraint) { return nullableValues.stream() .filter(convertToPredicate(constraint.getSummary(), toColumnHandle(column))) .filter(value -> !constraint.predicate().isPresent() || constraint.predicate().get().test(ImmutableMap.of(toColumnHandle(column), value))) .collect(toSet()); }
@Override public List<ConnectorTableLayoutResult> getTableLayouts(ConnectorSession session, ConnectorTableHandle table, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { if (constraint.getSummary().isNone()) { return ImmutableList.of(); } InformationSchemaTableHandle handle = checkTableHandle(table); Set<QualifiedTablePrefix> prefixes = calculatePrefixesWithSchemaName(session, constraint.getSummary(), constraint.predicate()); if (isTablesEnumeratingTable(handle.getSchemaTableName())) { Set<QualifiedTablePrefix> tablePrefixes = calculatePrefixesWithTableName(session, prefixes, constraint.getSummary(), constraint.predicate()); // in case of high number of prefixes it is better to populate all data and then filter if (tablePrefixes.size() <= MAX_PREFIXES_COUNT) { prefixes = tablePrefixes; } } if (prefixes.size() > MAX_PREFIXES_COUNT) { // in case of high number of prefixes it is better to populate all data and then filter prefixes = ImmutableSet.of(new QualifiedTablePrefix(catalogName)); } ConnectorTableLayout layout = new ConnectorTableLayout(new InformationSchemaTableLayoutHandle(handle, prefixes)); return ImmutableList.of(new ConnectorTableLayoutResult(layout, constraint.getSummary())); }
@Test public void testGetPartitionNamesUnpartitioned() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableUnpartitioned); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(newSession(), tableHandle, Constraint.alwaysTrue(), Optional.empty()); assertEquals(getAllPartitions(getOnlyElement(tableLayoutResults).getTableLayout().getHandle()).size(), 1); assertExpectedTableLayout(getOnlyElement(tableLayoutResults).getTableLayout(), unpartitionedTableLayout); } }
private static KdbTree loadKdbTree(String tableName, Session session, Metadata metadata, SplitManager splitManager, PageSourceManager pageSourceManager) QualifiedObjectName name = toQualifiedObjectName(tableName, session.getCatalog().get(), session.getSchema().get()); TableHandle tableHandle = metadata.getTableHandle(session, name) .orElseThrow(() -> new PrestoException(INVALID_SPATIAL_PARTITIONING, format("Table not found: %s", name))); Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle); List<ColumnHandle> visibleColumnHandles = columnHandles.values().stream() .filter(handle -> !metadata.getColumnMetadata(session, tableHandle, handle).isHidden()) .collect(toImmutableList()); checkSpatialPartitioningTable(visibleColumnHandles.size() == 1, "Expected single column for table %s, but found %s columns", name, columnHandles.size()); ColumnHandle kdbTreeColumn = Iterables.getOnlyElement(visibleColumnHandles); List<TableLayoutResult> layouts = metadata.getLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.of(ImmutableSet.of(kdbTreeColumn))); checkSpatialPartitioningTable(!layouts.isEmpty(), "Table is empty: %s", name); try (ConnectorPageSource pageSource = pageSourceManager.createPageSource(session, split, ImmutableList.of(kdbTreeColumn))) { do { getFutureValue(pageSource.isBlocked());
static class HiveTransaction implements Transaction { private final HiveTransactionManager transactionManager; private final ConnectorTransactionHandle transactionHandle; private boolean closed; public HiveTransaction(HiveTransactionManager transactionManager, HiveMetadata hiveMetadata) { this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); this.transactionHandle = new HiveTransactionHandle(); transactionManager.put(transactionHandle, hiveMetadata); getMetastore().testOnlyThrowOnCleanupFailures(); } @Override public ConnectorMetadata getMetadata() { return transactionManager.get(transactionHandle); } @Override public SemiTransactionalHiveMetastore getMetastore(String schema) { return getMetastore(); } private SemiTransactionalHiveMetastore getMetastore() { return ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore();
@Test public void testGetRecordsS3() throws Exception { HiveTransactionHandle transaction = new HiveTransactionHandle(); HiveMetadata metadata = metadataFactory.create(); ConnectorTableHandle table = getTableHandle(metadata, tableS3); List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(SESSION, table).values()); Map<String, Integer> columnIndex = indexColumns(columnHandles); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(SESSION, table, new Constraint<>(TupleDomain.all(), bindings -> true), Optional.empty()); HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); assertEquals(layoutHandle.getPartitions().get().size(), 1); ConnectorSplitSource splitSource = splitManager.getSplits(transaction, SESSION, layoutHandle); long sum = 0; for (ConnectorSplit split : getAllSplits(splitSource)) { try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction, SESSION, split, columnHandles)) { MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, getTypes(columnHandles)); for (MaterializedRow row : result) { sum += (Long) row.getField(columnIndex.get("t_bigint")); } } } assertEquals(sum, 78300); }
/** * Get the physical layout for a inserting into an existing table. */ default Optional<ConnectorNewTableLayout> getInsertLayout(ConnectorSession session, ConnectorTableHandle tableHandle) { List<ConnectorTableLayout> layouts = getTableLayouts(session, tableHandle, new Constraint<>(TupleDomain.all(), map -> true), Optional.empty()) .stream() .map(ConnectorTableLayoutResult::getTableLayout) .filter(layout -> layout.getTablePartitioning().isPresent()) .collect(toList()); if (layouts.isEmpty()) { return Optional.empty(); } if (layouts.size() > 1) { throw new PrestoException(NOT_SUPPORTED, "Tables with multiple layouts can not be written"); } ConnectorTableLayout layout = layouts.get(0); ConnectorPartitioningHandle partitioningHandle = layout.getTablePartitioning().get().getPartitioningHandle(); Map<ColumnHandle, String> columnNamesByHandle = getColumnHandles(session, tableHandle).entrySet().stream() .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); List<String> partitionColumns = layout.getTablePartitioning().get().getPartitioningColumns().stream() .map(columnNamesByHandle::get) .collect(toList()); return Optional.of(new ConnectorNewTableLayout(partitioningHandle, partitionColumns)); }
private Map<TpchColumn<?>, List<Object>> getColumnValuesRestrictions(TpchTable<?> tpchTable, Constraint<ColumnHandle> constraint) { TupleDomain<ColumnHandle> constraintSummary = constraint.getSummary(); if (constraintSummary.isAll()) { return emptyMap(); } else if (constraintSummary.isNone()) { Set<TpchColumn<?>> columns = ImmutableSet.copyOf(tpchTable.getColumns()); return asMap(columns, key -> emptyList()); } else { Map<ColumnHandle, Domain> domains = constraintSummary.getDomains().get(); Optional<Domain> orderStatusDomain = Optional.ofNullable(domains.get(toColumnHandle(ORDER_STATUS))); Optional<Map<TpchColumn<?>, List<Object>>> allowedColumnValues = orderStatusDomain.map(domain -> { List<Object> allowedValues = ORDER_STATUS_VALUES.stream() .filter(domain::includesNullableValue) .collect(toList()); return avoidTrivialOrderStatusRestriction(allowedValues); }); return allowedColumnValues.orElse(emptyMap()); } }
@Override public List<ConnectorTableLayoutResult> getTableLayouts(ConnectorSession session, ConnectorTableHandle table, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { MongoTableHandle tableHandle = (MongoTableHandle) table; Optional<Set<ColumnHandle>> partitioningColumns = Optional.empty(); //TODO: sharding key ImmutableList.Builder<LocalProperty<ColumnHandle>> localProperties = ImmutableList.builder(); MongoTable tableInfo = mongoSession.getTable(tableHandle.getSchemaTableName()); Map<String, ColumnHandle> columns = getColumnHandles(session, tableHandle); for (MongoIndex index : tableInfo.getIndexes()) { for (MongodbIndexKey key : index.getKeys()) { if (!key.getSortOrder().isPresent()) { continue; } if (columns.get(key.getName()) != null) { localProperties.add(new SortingProperty<>(columns.get(key.getName()), key.getSortOrder().get())); } } } ConnectorTableLayout layout = new ConnectorTableLayout( new MongoTableLayoutHandle(tableHandle, constraint.getSummary()), Optional.empty(), TupleDomain.all(), Optional.empty(), partitioningColumns, Optional.empty(), localProperties.build()); return ImmutableList.of(new ConnectorTableLayoutResult(layout, constraint.getSummary())); }
@Override protected Node visitShowStats(ShowStats node, Void context) { checkState(queryExplainer.isPresent(), "Query explainer must be provided for SHOW STATS SELECT"); if (node.getRelation() instanceof TableSubquery) { Query query = ((TableSubquery) node.getRelation()).getQuery(); QuerySpecification specification = (QuerySpecification) query.getQueryBody(); Plan plan = queryExplainer.get().getLogicalPlan(session, new Query(Optional.empty(), specification, Optional.empty(), Optional.empty()), parameters, warningCollector); validateShowStatsSubquery(node, query, specification, plan); Table table = (Table) specification.getFrom().get(); Constraint<ColumnHandle> constraint = getConstraint(plan); return rewriteShowStats(node, table, constraint); } else if (node.getRelation() instanceof Table) { Table table = (Table) node.getRelation(); return rewriteShowStats(node, table, Constraint.alwaysTrue()); } else { throw new IllegalArgumentException("Expected either TableSubquery or Table as relation"); } }
@Override protected Optional<PlanNodeStatsEstimate> doCalculate(TableScanNode node, StatsProvider sourceStats, Lookup lookup, Session session, TypeProvider types) { // TODO Construct predicate like AddExchanges's LayoutConstraintEvaluator Constraint<ColumnHandle> constraint = new Constraint<>(node.getCurrentConstraint()); TableStatistics tableStatistics = metadata.getTableStatistics(session, node.getTable(), constraint); Map<Symbol, SymbolStatsEstimate> outputSymbolStats = new HashMap<>(); for (Map.Entry<Symbol, ColumnHandle> entry : node.getAssignments().entrySet()) { Symbol symbol = entry.getKey(); Optional<ColumnStatistics> columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getValue())); outputSymbolStats.put(symbol, columnStatistics.map(statistics -> toSymbolStatistics(tableStatistics, statistics)).orElse(SymbolStatsEstimate.unknown())); } return Optional.of(PlanNodeStatsEstimate.builder() .setOutputRowCount(tableStatistics.getRowCount().getValue()) .addSymbolStatistics(outputSymbolStats) .build()); }
@Override public List<ConnectorTableLayoutResult> getTableLayouts( ConnectorSession session, ConnectorTableHandle table, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { TpcdsTableHandle tableHandle = (TpcdsTableHandle) table; ConnectorTableLayout layout = new ConnectorTableLayout( new TpcdsTableLayoutHandle(tableHandle), Optional.empty(), TupleDomain.all(), Optional.empty(), Optional.empty(), Optional.empty(), ImmutableList.of()); return ImmutableList.of(new ConnectorTableLayoutResult(layout, constraint.getSummary())); }
if (!result.isPresent()) { return context.defaultRewrite(node); TableScanNode tableScan = result.get(); if (!tableScan.getLayout().isPresent()) { List<TableLayoutResult> layouts = metadata.getLayouts(session, tableScan.getTable(), Constraint.alwaysTrue(), Optional.empty()); if (layouts.size() == 1) { layout = Iterables.getOnlyElement(layouts).getLayout(); ImmutableList.Builder<List<Expression>> rowsBuilder = ImmutableList.builder(); for (TupleDomain<ColumnHandle> domain : predicates.getPredicates()) { if (!domain.isNone()) { Map<ColumnHandle, NullableValue> entries = TupleDomain.extractFixedValues(domain).get(); ImmutableList.Builder<Expression> rowBuilder = ImmutableList.builder();
@Test public void testGetPartitionsWithBindings() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(newSession(), tableHandle, new Constraint<>(TupleDomain.withColumnDomains(ImmutableMap.of(intColumn, Domain.singleValue(BIGINT, 5L)))), Optional.empty()); assertExpectedTableLayout(getOnlyElement(tableLayoutResults).getTableLayout(), tableLayout); } }
.orElseThrow(() -> new AssertionError("Table does not exist: " + tableName)); assertEqualsIgnoreOrder(partitionNames, CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows().stream() .map(row -> "ds=" + row.getField(CREATE_TABLE_PARTITIONED_DATA.getTypes().size() - 1)) MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat)); assertEqualsIgnoreOrder(result.getMaterializedRows(), expectedResultBuilder.build().getMaterializedRows()); TupleDomain<ColumnHandle> tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(dsColumnHandle, NullableValue.of(createUnboundedVarcharType(), utf8Slice("2015-07-03")))); Constraint<ColumnHandle> constraint = new Constraint<>(tupleDomain, convertToPredicate(tupleDomain)); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, constraint, Optional.empty()); ConnectorTableLayoutHandle tableLayoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); metadata.metadataDelete(session, tableHandle, tableLayoutHandle); TupleDomain<ColumnHandle> tupleDomain2 = TupleDomain.withColumnDomains( ImmutableMap.of(dsColumnHandle, Domain.create(ValueSet.ofRanges(Range.range(createUnboundedVarcharType(), utf8Slice("2015-07-01"), true, utf8Slice("2015-07-02"), true)), false))); Constraint<ColumnHandle> constraint2 = new Constraint<>(tupleDomain2, convertToPredicate(tupleDomain2)); List<ConnectorTableLayoutResult> tableLayoutResults2 = metadata.getTableLayouts(session, tableHandle, constraint2, Optional.empty()); ConnectorTableLayoutHandle tableLayoutHandle2 = getOnlyElement(tableLayoutResults2).getTableLayout().getHandle(); metadata.metadataDelete(session, tableHandle, tableLayoutHandle2); ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName); List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values()); MaterializedResult actualAfterDelete2 = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat)); assertEqualsIgnoreOrder(actualAfterDelete2.getMaterializedRows(), ImmutableList.of());
@Override public List<ConnectorTableLayoutResult> getTableLayouts(ConnectorSession session, ConnectorTableHandle table, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { AtopTableHandle tableHandle = (AtopTableHandle) table; Optional<Map<ColumnHandle, Domain>> domains = constraint.getSummary().getDomains(); Domain endTimeDomain = Domain.all(TIMESTAMP_WITH_TIME_ZONE); Domain startTimeDomain = Domain.all(TIMESTAMP_WITH_TIME_ZONE); if (domains.isPresent()) { if (domains.get().containsKey(START_TIME_HANDLE)) { startTimeDomain = domains.get().get(START_TIME_HANDLE); } if (domains.get().containsKey(END_TIME_HANDLE)) { endTimeDomain = domains.get().get(END_TIME_HANDLE); } } AtopTableLayoutHandle layoutHandle = new AtopTableLayoutHandle(tableHandle, startTimeDomain, endTimeDomain); ConnectorTableLayout tableLayout = getTableLayout(session, layoutHandle); return ImmutableList.of(new ConnectorTableLayoutResult(tableLayout, constraint.getSummary())); }
return Optional.empty(); return Optional.empty(); .map(HiveColumnHandle::getTypeSignature) .map(typeManager::getType) .collect(toImmutableList()); column.getName(), typeManager.getType(column.getTypeSignature()), column.getComment().orElse(null), column.isHidden())) .collect(toImmutableList()); Constraint<ColumnHandle> targetConstraint = new Constraint<>(targetTupleDomain, targetPredicate); Iterable<List<Object>> records = () -> stream(partitionManager.getPartitions(metastore, sourceTableHandle, targetConstraint).getPartitions())