@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { return new FixedSplitSource(splits); } }
@SuppressWarnings("ObjectEquality") @Override public CompletableFuture<ConnectorSplitBatch> getNextBatch(ConnectorPartitionHandle partitionHandle, int maxSize) { if (!partitionHandle.equals(NOT_PARTITIONED)) { throw new IllegalArgumentException("partitionHandle must be NOT_PARTITIONED"); } int remainingSplits = splits.size() - offset; int size = Math.min(remainingSplits, maxSize); List<ConnectorSplit> results = splits.subList(offset, offset + size); offset += size; return completedFuture(new ConnectorSplitBatch(results, isFinished())); }
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { KuduTableLayoutHandle layoutHandle = (KuduTableLayoutHandle) layout; List<KuduSplit> splits = clientSession.buildKuduSplits(layoutHandle); return new FixedSplitSource(splits); } }
@SuppressWarnings("ObjectEquality") @Override public CompletableFuture<ConnectorSplitBatch> getNextBatch(ConnectorPartitionHandle partitionHandle, int maxSize) { if (!partitionHandle.equals(NOT_PARTITIONED)) { throw new IllegalArgumentException("partitionHandle must be NOT_PARTITIONED"); } int remainingSplits = splits.size() - offset; int size = Math.min(remainingSplits, maxSize); List<ConnectorSplit> results = splits.subList(offset, offset + size); offset += size; return completedFuture(new ConnectorSplitBatch(results, isFinished())); }
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { TpcdsTableHandle tableHandle = ((TpcdsTableLayoutHandle) layout).getTable(); Set<Node> nodes = nodeManager.getRequiredWorkerNodes(); checkState(!nodes.isEmpty(), "No TPCDS nodes available"); int totalParts = nodes.size() * splitsPerNode; int partNumber = 0; // Split the data using split and skew by the number of nodes available. ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder(); for (Node node : nodes) { for (int i = 0; i < splitsPerNode; i++) { splits.add(new TpcdsSplit(tableHandle, partNumber, totalParts, ImmutableList.of(node.getHostAndPort()), noSexism)); partNumber++; } } return new FixedSplitSource(splits.build()); } }
private static ConnectorSplitSource createFixedSplitSource(int splitCount, Supplier<ConnectorSplit> splitFactory) { ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder(); for (int i = 0; i < splitCount; i++) { splits.add(splitFactory.get()); } return new FixedSplitSource(splits.build()); }
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { TpchTableLayoutHandle tableLayoutHandle = (TpchTableLayoutHandle) layout; TpchTableHandle tableHandle = tableLayoutHandle.getTable(); Set<Node> nodes = nodeManager.getRequiredWorkerNodes(); int totalParts = nodes.size() * splitsPerNode; int partNumber = 0; // Split the data using split and skew by the number of nodes available. ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder(); for (Node node : nodes) { for (int i = 0; i < splitsPerNode; i++) { splits.add(new TpchSplit(tableHandle, partNumber, totalParts, ImmutableList.of(node.getHostAndPort()), tableLayoutHandle.getPredicate())); partNumber++; } } return new FixedSplitSource(splits.build()); } }
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { MongoTableLayoutHandle tableLayout = (MongoTableLayoutHandle) layout; MongoTableHandle tableHandle = tableLayout.getTable(); MongoSplit split = new MongoSplit( tableHandle.getSchemaTableName(), tableLayout.getTupleDomain(), addresses); return new FixedSplitSource(ImmutableList.of(split)); } }
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { InformationSchemaTableLayoutHandle handle = (InformationSchemaTableLayoutHandle) layout; List<HostAddress> localAddress = ImmutableList.of(nodeManager.getCurrentNode().getHostAndPort()); ConnectorSplit split = new InformationSchemaSplit(handle.getTable(), handle.getPrefixes(), localAddress); return new FixedSplitSource(ImmutableList.of(split)); } }
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { LocalFileTableLayoutHandle layoutHandle = (LocalFileTableLayoutHandle) layout; LocalFileTableHandle tableHandle = layoutHandle.getTable(); TupleDomain<LocalFileColumnHandle> effectivePredicate = layoutHandle.getConstraint() .transform(LocalFileColumnHandle.class::cast); List<ConnectorSplit> splits = nodeManager.getAllNodes().stream() .map(node -> new LocalFileSplit(node.getHostAndPort(), tableHandle.getSchemaTableName(), effectivePredicate)) .collect(Collectors.toList()); return new FixedSplitSource(splits); } }
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { JmxTableLayoutHandle jmxLayout = (JmxTableLayoutHandle) layout; JmxTableHandle tableHandle = jmxLayout.getTable(); TupleDomain<ColumnHandle> predicate = jmxLayout.getConstraint(); //TODO is there a better way to get the node column? Optional<JmxColumnHandle> nodeColumnHandle = tableHandle.getColumnHandles().stream() .filter(jmxColumnHandle -> jmxColumnHandle.getColumnName().equals(NODE_COLUMN_NAME)) .findFirst(); checkState(nodeColumnHandle.isPresent(), "Failed to find %s column", NODE_COLUMN_NAME); List<ConnectorSplit> splits = nodeManager.getAllNodes().stream() .filter(node -> { NullableValue value = NullableValue.of(createUnboundedVarcharType(), utf8Slice(node.getNodeIdentifier())); return predicate.overlaps(fromFixedValues(ImmutableMap.of(nodeColumnHandle.get(), value))); }) .map(node -> new JmxSplit(tableHandle, ImmutableList.of(node.getHostAndPort()))) .collect(toList()); return new FixedSplitSource(splits); } }
@Override public ConnectorSplitSource getSplits(JdbcTableLayoutHandle layoutHandle) { JdbcTableHandle tableHandle = layoutHandle.getTable(); JdbcSplit jdbcSplit = new JdbcSplit( connectorId, tableHandle.getCatalogName(), tableHandle.getSchemaName(), tableHandle.getTableName(), layoutHandle.getTupleDomain(), Optional.empty()); return new FixedSplitSource(ImmutableList.of(jdbcSplit)); }
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layoutHandle, SplitSchedulingStrategy splitSchedulingStrategy) { MemoryTableLayoutHandle layout = (MemoryTableLayoutHandle) layoutHandle; List<MemoryDataFragment> dataFragments = layout.getDataFragments(); ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder(); for (MemoryDataFragment dataFragment : dataFragments) { for (int i = 0; i < splitsPerNode; i++) { splits.add( new MemorySplit( layout.getTable(), i, splitsPerNode, dataFragment.getHostAddress(), dataFragment.getRows())); } } return new FixedSplitSource(splits.build()); } }
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { CassandraTableLayoutHandle layoutHandle = (CassandraTableLayoutHandle) layout; CassandraTableHandle cassandraTableHandle = layoutHandle.getTable(); List<CassandraPartition> partitions = layoutHandle.getPartitions(); if (partitions.isEmpty()) { return new FixedSplitSource(ImmutableList.of()); } // if this is an unpartitioned table, split into equal ranges if (partitions.size() == 1) { CassandraPartition cassandraPartition = partitions.get(0); if (cassandraPartition.isUnpartitioned() || cassandraPartition.isIndexedColumnPredicatePushdown()) { CassandraTable table = cassandraSession.getTable(cassandraTableHandle.getSchemaTableName()); List<ConnectorSplit> splits = getSplitsByTokenRange(table, cassandraPartition.getPartitionId(), getSplitsPerNode(session)); return new FixedSplitSource(splits); } } return new FixedSplitSource(getSplitsForPartitions(cassandraTableHandle, partitions, layoutHandle.getClusteringPredicates())); }
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layoutHandle, SplitSchedulingStrategy splitSchedulingStrategy) { BlackHoleTableLayoutHandle layout = (BlackHoleTableLayoutHandle) layoutHandle; ImmutableList.Builder<BlackHoleSplit> builder = ImmutableList.builder(); for (int i = 0; i < layout.getSplitCount(); i++) { builder.add( new BlackHoleSplit( layout.getPagesPerSplit(), layout.getRowsPerPage(), layout.getFieldsLength(), layout.getPageProcessingDelay())); } return new FixedSplitSource(builder.build()); } }
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle handle, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { ExampleTableLayoutHandle layoutHandle = (ExampleTableLayoutHandle) layout; ExampleTableHandle tableHandle = layoutHandle.getTable(); ExampleTable table = exampleClient.getTable(tableHandle.getSchemaName(), tableHandle.getTableName()); // this can happen if table is removed during a query checkState(table != null, "Table %s.%s no longer exists", tableHandle.getSchemaName(), tableHandle.getTableName()); List<ConnectorSplit> splits = new ArrayList<>(); for (URI uri : table.getSources()) { splits.add(new ExampleSplit(connectorId, tableHandle.getSchemaName(), tableHandle.getTableName(), uri)); } Collections.shuffle(splits); return new FixedSplitSource(splits); } }
HostAddress address = nodeManager.getCurrentNode().getHostAndPort(); ConnectorSplit split = new SystemSplit(tableHandle.getConnectorId(), tableHandle, address, constraint); return new FixedSplitSource(ImmutableList.of(split)); splits.add(new SystemSplit(tableHandle.getConnectorId(), tableHandle, node.getHostAndPort(), constraint)); return new FixedSplitSource(splits.build());
return new FixedSplitSource(builder.build());
return new FixedSplitSource(cSplits.build());
return new FixedSplitSource(ImmutableList.of());