continue; driver.updateSource(sourceUpdate);
public Driver createDriver(DriverContext driverContext, @Nullable ScheduledSplit partitionedSplit) { Driver driver = driverFactory.createDriver(driverContext); // record driver so other threads add unpartitioned sources can see the driver // NOTE: this MUST be done before reading unpartitionedSources, so we see a consistent view of the unpartitioned sources drivers.add(new WeakReference<>(driver)); if (partitionedSplit != null) { // TableScanOperator requires partitioned split to be added before the first call to process driver.updateSource(new TaskSource(partitionedSplit.getPlanNodeId(), ImmutableSet.of(partitionedSplit), true)); } // add unpartitioned sources Optional<PlanNodeId> sourceId = driver.getSourceId(); if (sourceId.isPresent()) { TaskSource taskSource = unpartitionedSources.get(sourceId.get()); if (taskSource != null) { driver.updateSource(taskSource); } } status.decrementPendingCreation(pipelineContext.getPipelineId(), driverContext.getLifespan()); closeDriverFactoryIfFullyCreated(); return driver; }
public IndexedData streamIndexDataForSingleKey(UpdateRequest updateRequest) { Page indexKeyTuple = updateRequest.getPage().getRegion(0, 1); PageBuffer pageBuffer = new PageBuffer(100); DriverFactory driverFactory = indexBuildDriverFactoryProvider.createStreaming(pageBuffer, indexKeyTuple); Driver driver = driverFactory.createDriver(pipelineContext.addDriverContext()); PageRecordSet pageRecordSet = new PageRecordSet(keyTypes, indexKeyTuple); PlanNodeId planNodeId = driverFactory.getSourceId().get(); ScheduledSplit split = new ScheduledSplit(0, planNodeId, new Split(INDEX_CONNECTOR_ID, new ConnectorTransactionHandle() {}, new IndexSplit(pageRecordSet))); driver.updateSource(new TaskSource(planNodeId, ImmutableSet.of(split), true)); return new StreamingIndexedData(outputTypes, keyTypes, indexKeyTuple, pageBuffer, driver); }
PlanNodeId sourcePlanNodeId = driverFactory.getSourceId().get(); ScheduledSplit split = new ScheduledSplit(0, sourcePlanNodeId, new Split(INDEX_CONNECTOR_ID, new ConnectorTransactionHandle() {}, new IndexSplit(recordSetForLookupSource))); driver.updateSource(new TaskSource(sourcePlanNodeId, ImmutableSet.of(split), true)); while (!driver.isFinished()) { ListenableFuture<?> process = driver.process();
DriverContext driverContext = taskContext.addPipelineContext(driverFactory.getPipelineId(), driverFactory.isInputDriver(), driverFactory.isOutputDriver(), partitioned).addDriverContext(); Driver driver = driverFactory.createDriver(driverContext); driver.updateSource(new TaskSource(split.getPlanNodeId(), ImmutableSet.of(split), true)); drivers.add(driver);
assertFalse(driver.isFinished()); driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));
assertFalse(driver.isFinished()); driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));
assertFalse(driver.isFinished()); Split testSplit = new Split(new ConnectorId("test"), TestingTransactionHandle.create(), new TestSplit()); driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, testSplit)), true));
private Driver createDriver(DriverContext driverContext, @Nullable ScheduledSplit partitionedSplit) { Driver driver = driverFactory.createDriver(driverContext); // record driver so other threads add unpartitioned sources can see the driver // NOTE: this MUST be done before reading unpartitionedSources, so we see a consistent view of the unpartitioned sources drivers.add(new WeakReference<>(driver)); if (partitionedSplit != null) { // TableScanOperator requires partitioned split to be added before the first call to process driver.updateSource(new TaskSource(partitionedSourceId, ImmutableSet.of(partitionedSplit), true)); } // add unpartitioned sources for (TaskSource source : unpartitionedSources.values()) { driver.updateSource(source); } pendingCreation.decrementAndGet(); closeDriverFactoryIfFullyCreated(); return driver; }
public void addSources(List<TaskSource> sources) { requireNonNull(sources, "sources is null"); checkState(!Thread.holdsLock(this), "Can not add sources while holding a lock on the %s", getClass().getSimpleName()); try (SetThreadName ignored = new SetThreadName("Task-%s", taskId)) { // update our record of sources and schedule drivers for new partitioned splits Map<PlanNodeId, TaskSource> updatedUnpartitionedSources = updateSources(sources); // tell existing drivers about the new splits; it is safe to update drivers // multiple times and out of order because sources contain full record of // the unpartitioned splits for (TaskSource source : updatedUnpartitionedSources.values()) { // tell all the existing drivers this source is finished for (WeakReference<Driver> driverReference : drivers) { Driver driver = driverReference.get(); // the driver can be GCed due to a failure or a limit if (driver != null) { driver.updateSource(source); } else { // remove the weak reference from the list to avoid a memory leak // NOTE: this is a concurrent safe operation on a CopyOnWriteArrayList drivers.remove(driverReference); } } } // we may have transitioned to no more splits, so check for completion checkTaskCompletion(); } }
driver.updateSource(new TaskSource(sourcePlanNodeId, ImmutableSet.of(new ScheduledSplit(0, new Split("index", new ConnectorTransactionHandle() {}, new IndexSplit(recordSetForLookupSource)))), true)); while (!driver.isFinished()) { ListenableFuture<?> process = driver.process();
public IndexedData streamIndexDataForSingleKey(UpdateRequest updateRequest) { Page indexKeyTuple = new Page(sliceBlocks(updateRequest.getBlocks(), 0, 1)); PageBuffer pageBuffer = new PageBuffer(100); DriverFactory driverFactory = indexBuildDriverFactoryProvider.createStreaming(pageBuffer, indexKeyTuple); Driver driver = driverFactory.createDriver(pipelineContext.addDriverContext()); PageRecordSet pageRecordSet = new PageRecordSet(keyTypes, indexKeyTuple); PlanNodeId planNodeId = Iterables.getOnlyElement(driverFactory.getSourceIds()); driver.updateSource(new TaskSource(planNodeId, ImmutableSet.of(new ScheduledSplit(0, new Split("index", new ConnectorTransactionHandle() {}, new IndexSplit(pageRecordSet)))), true)); return new StreamingIndexedData(outputTypes, keyTypes, indexKeyTuple, pageBuffer, driver); }
driver.updateSource(source);
assertFalse(driver.isFinished()); driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, newMockSplit())), true));
assertFalse(driver.isFinished()); driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, newMockSplit())), true));