@Override public ToIntFunction<ConnectorSplit> getSplitBucketFunction(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorPartitioningHandle partitioningHandle) { return value -> ((TpchSplit) value).getPartNumber(); }
@Override public RecordSet getRecordSet(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit split, List<? extends ColumnHandle> columns) { TpchSplit tpchSplit = (TpchSplit) split; String tableName = tpchSplit.getTableHandle().getTableName(); TpchTable<?> tpchTable = TpchTable.getTable(tableName); return getRecordSet(tpchTable, columns, tpchSplit.getTableHandle().getScaleFactor(), tpchSplit.getPartNumber(), tpchSplit.getTotalParts(), tpchSplit.getPredicate()); }
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { TpchTableLayoutHandle tableLayoutHandle = (TpchTableLayoutHandle) layout; TpchTableHandle tableHandle = tableLayoutHandle.getTable(); Set<Node> nodes = nodeManager.getRequiredWorkerNodes(); int totalParts = nodes.size() * splitsPerNode; int partNumber = 0; // Split the data using split and skew by the number of nodes available. ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder(); for (Node node : nodes) { for (int i = 0; i < splitsPerNode; i++) { splits.add(new TpchSplit(tableHandle, partNumber, totalParts, ImmutableList.of(node.getHostAndPort()), tableLayoutHandle.getPredicate())); partNumber++; } } return new FixedSplitSource(splits.build()); } }
@Override public RecordSet getRecordSet(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit split, List<? extends ColumnHandle> columns) { TpchSplit tpchSplit = (TpchSplit) split; String tableName = tpchSplit.getTableHandle().getTableName(); TpchTable<?> tpchTable = TpchTable.getTable(tableName); return getRecordSet(tpchTable, columns, tpchSplit.getTableHandle().getScaleFactor(), tpchSplit.getPartNumber(), tpchSplit.getTotalParts(), tpchSplit.getPredicate()); }
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { TpchTableLayoutHandle tableLayoutHandle = (TpchTableLayoutHandle) layout; TpchTableHandle tableHandle = tableLayoutHandle.getTable(); Set<Node> nodes = nodeManager.getRequiredWorkerNodes(); int totalParts = nodes.size() * splitsPerNode; int partNumber = 0; // Split the data using split and skew by the number of nodes available. ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder(); for (Node node : nodes) { for (int i = 0; i < splitsPerNode; i++) { splits.add(new TpchSplit(tableHandle, partNumber, totalParts, ImmutableList.of(node.getHostAndPort()), tableLayoutHandle.getPredicate())); partNumber++; } } return new FixedSplitSource(splits.build()); } }
@Override public ToIntFunction<ConnectorSplit> getSplitBucketFunction(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorPartitioningHandle partitioningHandle) { return value -> ((TpchSplit) value).getPartNumber(); }