@Override public ListenableFuture<SplitBatch> getNextBatch(ConnectorPartitionHandle partitionHandle, Lifespan lifespan, int maxSize) { ListenableFuture<ConnectorSplitBatch> nextBatch = toListenableFuture(source.getNextBatch(partitionHandle, maxSize)); return Futures.transform(nextBatch, splitBatch -> { ImmutableList.Builder<Split> result = ImmutableList.builder(); for (ConnectorSplit connectorSplit : splitBatch.getSplits()) { result.add(new Split(connectorId, transactionHandle, connectorSplit, lifespan)); } return new SplitBatch(result.build(), splitBatch.isNoMoreSplits()); }, directExecutor()); }
@Test public void testScheduleLocal() { Split split = new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitLocal()); Set<Split> splits = ImmutableSet.of(split); Map.Entry<Node, Split> assignment = Iterables.getOnlyElement(nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments().entries()); assertEquals(assignment.getKey().getHostAndPort(), split.getAddresses().get(0)); assertEquals(assignment.getValue(), split); }
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkState(this.split == null, "Table scan split already set"); if (finished) { return Optional::empty; } this.split = split; Object splitInfo = split.getInfo(); if (splitInfo != null) { operatorContext.setInfoSupplier(() -> new SplitOperatorInfo(splitInfo)); } blocked.set(null); if (split.getConnectorSplit() instanceof EmptySplit) { source = new EmptySplitPageSource(); } return () -> { if (source instanceof UpdatablePageSource) { return Optional.of((UpdatablePageSource) source); } return Optional.empty(); }; }
@Override public ConnectorPageSource createPageSource(Session session, Split split, List<ColumnHandle> columns) { requireNonNull(split, "split is null"); requireNonNull(columns, "columns is null"); ConnectorSession connectorSession = session.toConnectorSession(split.getConnectorId()); return getPageSourceProvider(split).createPageSource(split.getTransactionHandle(), connectorSession, split.getConnectorSplit(), columns); }
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkArgument(split.getConnectorId().equals(REMOTE_CONNECTOR_ID), "split is not a remote split"); URI location = ((RemoteSplit) split.getConnectorSplit()).getLocation(); exchangeClient.addLocation(location); return Optional::empty; }
boolean splitWaitingForAnyNode = false; for (Split split : splits) { if (!split.isRemotelyAccessible()) { List<Node> candidateNodes = selectExactNodes(nodeMap, split.getAddresses(), includeCoordinator); if (candidateNodes.isEmpty()) { log.debug("No nodes available to schedule %s. Available nodes %s", split, nodeMap.getNodesByHost().keys()); int chosenDepth = 0; Set<NetworkLocation> locations = new HashSet<>(); for (HostAddress host : split.getAddresses()) { locations.add(networkLocationCache.get(host));
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkState(this.split == null, "Table scan split already set"); if (finished) { return Optional::empty; } this.split = (TestingSplit) split.getConnectorSplit(); blocked.set(null); return Optional::empty; }
private ToIntFunction<Split> getSplitToBucket(Session session, PartitioningHandle partitioningHandle) { ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); ToIntFunction<ConnectorSplit> splitBucketFunction = partitioningProvider.getSplitBucketFunction( partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), partitioningHandle.getConnectorHandle()); checkArgument(splitBucketFunction != null, "No partitioning %s", partitioningHandle); return split -> { int bucket; if (split.getConnectorSplit() instanceof EmptySplit) { bucket = split.getLifespan().isTaskWide() ? 0 : split.getLifespan().getId(); } else { bucket = splitBucketFunction.applyAsInt(split.getConnectorSplit()); } if (!split.getLifespan().isTaskWide()) { checkArgument(split.getLifespan().getId() == bucket); } return bucket; }; }
private ConnectorPageSourceProvider getPageSourceProvider(Split split) { ConnectorPageSourceProvider provider = pageSourceProviders.get(split.getConnectorId()); checkArgument(provider != null, "No page stream provider for '%s", split.getConnectorId()); return provider; } }
@Override public String getInfo() { return (partitionedSplit == null) ? "" : partitionedSplit.getSplit().getInfo().toString(); }
@GuardedBy("this") private void mergeIntoPendingSplits(PlanNodeId planNodeId, Set<ScheduledSplit> scheduledSplits, Set<Lifespan> noMoreSplitsForLifespan, boolean noMoreSplits) { checkHoldsLock(); DriverSplitRunnerFactory partitionedDriverFactory = driverRunnerFactoriesWithSplitLifeCycle.get(planNodeId); PendingSplitsForPlanNode pendingSplitsForPlanNode = pendingSplitsByPlanNode.get(planNodeId); partitionedDriverFactory.splitsAdded(scheduledSplits.size()); for (ScheduledSplit scheduledSplit : scheduledSplits) { Lifespan lifespan = scheduledSplit.getSplit().getLifespan(); checkLifespan(partitionedDriverFactory.getPipelineExecutionStrategy(), lifespan); pendingSplitsForPlanNode.getLifespan(lifespan).addSplit(scheduledSplit); schedulingLifespanManager.addLifespanIfAbsent(lifespan); } for (Lifespan lifespanWithNoMoreSplits : noMoreSplitsForLifespan) { checkLifespan(partitionedDriverFactory.getPipelineExecutionStrategy(), lifespanWithNoMoreSplits); pendingSplitsForPlanNode.getLifespan(lifespanWithNoMoreSplits).noMoreSplits(); schedulingLifespanManager.addLifespanIfAbsent(lifespanWithNoMoreSplits); } if (noMoreSplits) { pendingSplitsForPlanNode.setNoMoreSplits(); } }
boolean splitWaitingForAnyNode = false; for (Split split : splits) { if (!split.isRemotelyAccessible()) { List<Node> candidateNodes = selectExactNodes(nodeMap, split.getAddresses(), includeCoordinator); if (candidateNodes.isEmpty()) { log.debug("No nodes available to schedule %s. Available nodes %s", split, nodeMap.getNodesByHost().keys()); int chosenDepth = 0; Set<NetworkLocation> locations = new HashSet<>(); for (HostAddress host : split.getAddresses()) { locations.add(networkLocationCache.get(host));
@Override public ConnectorPageSource createPageSource(Session session, Split split, List<ColumnHandle> columns) { requireNonNull(split, "split is null"); requireNonNull(columns, "columns is null"); ConnectorSession connectorSession = session.toConnectorSession(split.getConnectorId()); return getPageSourceProvider(split).createPageSource(split.getTransactionHandle(), connectorSession, split.getConnectorSplit(), columns); }
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkState(this.split == null, "Table scan split already set"); if (finished) { return Optional::empty; } this.split = (TestingSplit) split.getConnectorSplit(); blocked.set(null); return Optional::empty; }
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkArgument(split.getConnectorId().equals(REMOTE_CONNECTOR_ID), "split is not a remote split"); URI location = ((RemoteSplit) split.getConnectorSplit()).getLocation(); exchangeClient.addLocation(location); return Optional::empty; }
private ToIntFunction<Split> getSplitToBucket(Session session, PartitioningHandle partitioningHandle) { ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); ToIntFunction<ConnectorSplit> splitBucketFunction = partitioningProvider.getSplitBucketFunction( partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), partitioningHandle.getConnectorHandle()); checkArgument(splitBucketFunction != null, "No partitioning %s", partitioningHandle); return split -> { int bucket; if (split.getConnectorSplit() instanceof EmptySplit) { bucket = split.getLifespan().isTaskWide() ? 0 : split.getLifespan().getId(); } else { bucket = splitBucketFunction.applyAsInt(split.getConnectorSplit()); } if (!split.getLifespan().isTaskWide()) { checkArgument(split.getLifespan().getId() == bucket); } return bucket; }; }
private ConnectorPageSourceProvider getPageSourceProvider(Split split) { ConnectorPageSourceProvider provider = pageSourceProviders.get(split.getConnectorId()); checkArgument(provider != null, "No page stream provider for '%s", split.getConnectorId()); return provider; } }
@Override public String getInfo() { return (partitionedSplit == null) ? "" : partitionedSplit.getSplit().getInfo().toString(); }
@GuardedBy("this") private void mergeIntoPendingSplits(PlanNodeId planNodeId, Set<ScheduledSplit> scheduledSplits, Set<Lifespan> noMoreSplitsForLifespan, boolean noMoreSplits) { checkHoldsLock(); DriverSplitRunnerFactory partitionedDriverFactory = driverRunnerFactoriesWithSplitLifeCycle.get(planNodeId); PendingSplitsForPlanNode pendingSplitsForPlanNode = pendingSplitsByPlanNode.get(planNodeId); partitionedDriverFactory.splitsAdded(scheduledSplits.size()); for (ScheduledSplit scheduledSplit : scheduledSplits) { Lifespan lifespan = scheduledSplit.getSplit().getLifespan(); checkLifespan(partitionedDriverFactory.getPipelineExecutionStrategy(), lifespan); pendingSplitsForPlanNode.getLifespan(lifespan).addSplit(scheduledSplit); schedulingLifespanManager.addLifespanIfAbsent(lifespan); } for (Lifespan lifespanWithNoMoreSplits : noMoreSplitsForLifespan) { checkLifespan(partitionedDriverFactory.getPipelineExecutionStrategy(), lifespanWithNoMoreSplits); pendingSplitsForPlanNode.getLifespan(lifespanWithNoMoreSplits).noMoreSplits(); schedulingLifespanManager.addLifespanIfAbsent(lifespanWithNoMoreSplits); } if (noMoreSplits) { pendingSplitsForPlanNode.setNoMoreSplits(); } }
@Override public ListenableFuture<SplitBatch> getNextBatch(ConnectorPartitionHandle partitionHandle, Lifespan lifespan, int maxSize) { ListenableFuture<ConnectorSplitBatch> nextBatch = toListenableFuture(source.getNextBatch(partitionHandle, maxSize)); return Futures.transform(nextBatch, splitBatch -> { ImmutableList.Builder<Split> result = ImmutableList.builder(); for (ConnectorSplit connectorSplit : splitBatch.getSplits()) { result.add(new Split(connectorId, transactionHandle, connectorSplit, lifespan)); } return new SplitBatch(result.build(), splitBatch.isNoMoreSplits()); }, directExecutor()); }