public static TableLayout fromConnectorLayout(ConnectorId connectorId, ConnectorTransactionHandle transactionHandle, ConnectorTableLayout layout) { return new TableLayout(new TableLayoutHandle(connectorId, transactionHandle, layout.getHandle()), layout); }
public boolean hasAllOutputs(TableScanNode node) { if (!layout.getColumns().isPresent()) { return true; } Set<ColumnHandle> columns = ImmutableSet.copyOf(layout.getColumns().get()); List<ColumnHandle> nodeColumnHandles = node.getOutputSymbols().stream() .map(node.getAssignments()::get) .collect(toImmutableList()); return columns.containsAll(nodeColumnHandles); }
@Override public ActualProperties visitTableScan(TableScanNode node, List<ActualProperties> inputProperties) { checkArgument(node.getLayout().isPresent(), "table layout has not yet been chosen"); TableLayout layout = metadata.getLayout(session, node.getLayout().get()); Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse(); ActualProperties.Builder properties = ActualProperties.builder(); // Globally constant assignments Map<ColumnHandle, NullableValue> globalConstants = new HashMap<>(); extractFixedValues(node.getCurrentConstraint()).orElse(ImmutableMap.of()) .entrySet().stream() .filter(entry -> !entry.getValue().isNull()) .forEach(entry -> globalConstants.put(entry.getKey(), entry.getValue())); Map<Symbol, NullableValue> symbolConstants = globalConstants.entrySet().stream() .filter(entry -> assignments.containsKey(entry.getKey())) .collect(toMap(entry -> assignments.get(entry.getKey()), Map.Entry::getValue)); properties.constants(symbolConstants); // Partitioning properties properties.global(deriveGlobalProperties(layout, assignments, globalConstants)); // Append the global constants onto the local properties to maximize their translation potential List<LocalProperty<ColumnHandle>> constantAppendedLocalProperties = ImmutableList.<LocalProperty<ColumnHandle>>builder() .addAll(globalConstants.keySet().stream().map(ConstantProperty::new).iterator()) .addAll(layout.getLocalProperties()) .build(); properties.local(LocalProperties.translate(constantAppendedLocalProperties, column -> Optional.ofNullable(assignments.get(column)))); return properties.build(); }
private Global deriveGlobalProperties(TableLayout layout, Map<ColumnHandle, Symbol> assignments, Map<ColumnHandle, NullableValue> constants) { Optional<List<Symbol>> streamPartitioning = layout.getStreamPartitioningColumns() .flatMap(columns -> translateToNonConstantSymbols(columns, assignments, constants)); if (planWithTableNodePartitioning(session) && layout.getTablePartitioning().isPresent()) { TablePartitioning tablePartitioning = layout.getTablePartitioning().get(); if (assignments.keySet().containsAll(tablePartitioning.getPartitioningColumns())) { List<Symbol> arguments = tablePartitioning.getPartitioningColumns().stream() .map(assignments::get) .collect(toImmutableList()); return partitionedOn(tablePartitioning.getPartitioningHandle(), arguments, streamPartitioning); } } if (streamPartitioning.isPresent()) { return streamPartitionedOn(streamPartitioning.get()); } return arbitraryPartition(); }
scan.getOutputSymbols(), scan.getAssignments(), Optional.of(layoutResult.getLayout().getHandle()), layoutResult.getLayout().getPredicate(), computeEnforced(originalEnforcedConstraint, layoutResult.getUnenforcedConstraint()));
Split split = getLocalQuerySplit(session, layouts.get(0).getLayout().getHandle());
@Override public List<TableLayoutResult> getLayouts(Session session, TableHandle table, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { if (constraint.getSummary().isNone()) { return ImmutableList.of(); } ConnectorId connectorId = table.getConnectorId(); ConnectorTableHandle connectorTable = table.getConnectorHandle(); CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); ConnectorTransactionHandle transaction = catalogMetadata.getTransactionHandleFor(connectorId); ConnectorSession connectorSession = session.toConnectorSession(connectorId); List<ConnectorTableLayoutResult> layouts = metadata.getTableLayouts(connectorSession, connectorTable, constraint, desiredColumns); return layouts.stream() .map(layout -> new TableLayoutResult(fromConnectorLayout(connectorId, transaction, layout.getTableLayout()), layout.getUnenforcedConstraint())) .collect(toImmutableList()); }
@Override public GroupedExecutionProperties visitTableScan(TableScanNode node, Void context) { Optional<TablePartitioning> tablePartitioning = metadata.getLayout(session, node.getLayout().get()).getTablePartitioning(); if (!tablePartitioning.isPresent()) { return GroupedExecutionProperties.notCapable(); } List<ConnectorPartitionHandle> partitionHandles = nodePartitioningManager.listPartitionHandles(session, tablePartitioning.get().getPartitioningHandle()); if (ImmutableList.of(NOT_PARTITIONED).equals(partitionHandles)) { return new GroupedExecutionProperties(false, false, ImmutableList.of()); } else { return new GroupedExecutionProperties(true, false, ImmutableList.of(node.getId())); } }
@Override public StreamProperties visitTableScan(TableScanNode node, List<StreamProperties> inputProperties) { checkArgument(node.getLayout().isPresent(), "table layout has not yet been chosen"); TableLayout layout = metadata.getLayout(session, node.getLayout().get()); Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse(); // Globally constant assignments Set<ColumnHandle> constants = new HashSet<>(); extractFixedValues(node.getCurrentConstraint()).orElse(ImmutableMap.of()) .entrySet().stream() .filter(entry -> !entry.getValue().isNull()) // TODO consider allowing nulls .forEach(entry -> constants.add(entry.getKey())); Optional<Set<Symbol>> streamPartitionSymbols = layout.getStreamPartitioningColumns() .flatMap(columns -> getNonConstantSymbols(columns, assignments, constants)); // if we are partitioned on empty set, we must say multiple of unknown partitioning, because // the connector does not guarantee a single split in this case (since it might not understand // that the value is a constant). if (streamPartitionSymbols.isPresent() && streamPartitionSymbols.get().isEmpty()) { return new StreamProperties(MULTIPLE, Optional.empty(), false); } return new StreamProperties(MULTIPLE, streamPartitionSymbols, false); }
if (layout == null || !layout.getDiscretePredicates().isPresent()) { return context.defaultRewrite(node); DiscretePredicates predicates = layout.getDiscretePredicates().get();
scan.getOutputSymbols(), scan.getAssignments(), Optional.of(layoutResult.getLayout().getHandle()), layoutResult.getLayout().getPredicate(), computeEnforced(originalEnforcedConstraint, layoutResult.getUnenforcedConstraint()));
Split split = getLocalQuerySplit(session, layouts.get(0).getLayout().getHandle());
private Global deriveGlobalProperties(TableLayout layout, Map<ColumnHandle, Symbol> assignments, Map<ColumnHandle, NullableValue> constants) { Optional<List<Symbol>> streamPartitioning = layout.getStreamPartitioningColumns() .flatMap(columns -> translateToNonConstantSymbols(columns, assignments, constants)); if (planWithTableNodePartitioning(session) && layout.getTablePartitioning().isPresent()) { TablePartitioning tablePartitioning = layout.getTablePartitioning().get(); if (assignments.keySet().containsAll(tablePartitioning.getPartitioningColumns())) { List<Symbol> arguments = tablePartitioning.getPartitioningColumns().stream() .map(assignments::get) .collect(toImmutableList()); return partitionedOn(tablePartitioning.getPartitioningHandle(), arguments, streamPartitioning); } } if (streamPartitioning.isPresent()) { return streamPartitionedOn(streamPartitioning.get()); } return arbitraryPartition(); }
@Override public List<TableLayoutResult> getLayouts(Session session, TableHandle table, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { if (constraint.getSummary().isNone()) { return ImmutableList.of(); } ConnectorId connectorId = table.getConnectorId(); ConnectorTableHandle connectorTable = table.getConnectorHandle(); CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); ConnectorTransactionHandle transaction = catalogMetadata.getTransactionHandleFor(connectorId); ConnectorSession connectorSession = session.toConnectorSession(connectorId); List<ConnectorTableLayoutResult> layouts = metadata.getTableLayouts(connectorSession, connectorTable, constraint, desiredColumns); return layouts.stream() .map(layout -> new TableLayoutResult(fromConnectorLayout(connectorId, transaction, layout.getTableLayout()), layout.getUnenforcedConstraint())) .collect(toImmutableList()); }
@Override public GroupedExecutionProperties visitTableScan(TableScanNode node, Void context) { Optional<TablePartitioning> tablePartitioning = metadata.getLayout(session, node.getLayout().get()).getTablePartitioning(); if (!tablePartitioning.isPresent()) { return GroupedExecutionProperties.notCapable(); } List<ConnectorPartitionHandle> partitionHandles = nodePartitioningManager.listPartitionHandles(session, tablePartitioning.get().getPartitioningHandle()); if (ImmutableList.of(NOT_PARTITIONED).equals(partitionHandles)) { return new GroupedExecutionProperties(false, false, ImmutableList.of()); } else { return new GroupedExecutionProperties(true, false, ImmutableList.of(node.getId())); } }
@Override public StreamProperties visitTableScan(TableScanNode node, List<StreamProperties> inputProperties) { checkArgument(node.getLayout().isPresent(), "table layout has not yet been chosen"); TableLayout layout = metadata.getLayout(session, node.getLayout().get()); Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse(); // Globally constant assignments Set<ColumnHandle> constants = new HashSet<>(); extractFixedValues(node.getCurrentConstraint()).orElse(ImmutableMap.of()) .entrySet().stream() .filter(entry -> !entry.getValue().isNull()) // TODO consider allowing nulls .forEach(entry -> constants.add(entry.getKey())); Optional<Set<Symbol>> streamPartitionSymbols = layout.getStreamPartitioningColumns() .flatMap(columns -> getNonConstantSymbols(columns, assignments, constants)); // if we are partitioned on empty set, we must say multiple of unknown partitioning, because // the connector does not guarantee a single split in this case (since it might not understand // that the value is a constant). if (streamPartitionSymbols.isPresent() && streamPartitionSymbols.get().isEmpty()) { return new StreamProperties(MULTIPLE, Optional.empty(), false); } return new StreamProperties(MULTIPLE, streamPartitionSymbols, false); }
if (layout == null || !layout.getDiscretePredicates().isPresent()) { return context.defaultRewrite(node); DiscretePredicates predicates = layout.getDiscretePredicates().get();
checkState(!layouts.isEmpty(), "No usable layouts for %s", node); if (layouts.stream().anyMatch(layout -> layout.getLayout().getPredicate().isNone())) { return ImmutableList.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of())); node.getOutputSymbols(), node.getAssignments(), Optional.of(layout.getLayout().getHandle()), layout.getLayout().getPredicate(), computeEnforced(newDomain, layout.getUnenforcedConstraint()));
try (SplitSource splitSource = splitManager.getSplits(session, layouts.get(0).getLayout().getHandle(), UNGROUPED_SCHEDULING)) { while (!Thread.currentThread().isInterrupted()) { SplitBatch splitBatch = getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), 1000));
@Override public TableLayout getLayout(Session session, TableLayoutHandle handle) { ConnectorId connectorId = handle.getConnectorId(); CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); ConnectorTransactionHandle transaction = catalogMetadata.getTransactionHandleFor(connectorId); return fromConnectorLayout(connectorId, transaction, metadata.getTableLayout(session.toConnectorSession(connectorId), handle.getConnectorHandle())); }