@VisibleForTesting public static Driver createDriver(DriverContext driverContext, Operator firstOperator, Operator... otherOperators) { requireNonNull(driverContext, "driverContext is null"); requireNonNull(firstOperator, "firstOperator is null"); requireNonNull(otherOperators, "otherOperators is null"); ImmutableList<Operator> operators = ImmutableList.<Operator>builder() .add(firstOperator) .add(otherOperators) .build(); return createDriver(driverContext, operators); }
public synchronized Driver createDriver(DriverContext driverContext) { checkState(!closed, "DriverFactory is already closed"); requireNonNull(driverContext, "driverContext is null"); checkState(!closedLifespans.contains(driverContext.getLifespan()), "DriverFatory is already closed for driver group %s", driverContext.getLifespan()); encounteredLifespans.add(driverContext.getLifespan()); ImmutableList.Builder<Operator> operators = ImmutableList.builder(); for (OperatorFactory operatorFactory : operatorFactories) { Operator operator = operatorFactory.createOperator(driverContext); operators.add(operator); } return Driver.createDriver(driverContext, operators.build()); }
private RevocableMemoryOperator setupConsumeRevocableMemory(DataSize reservedPerPage, long numberOfPages) { AtomicReference<RevocableMemoryOperator> createOperator = new AtomicReference<>(); setUp(() -> { DriverContext driverContext = taskContext.addPipelineContext(0, false, false, false).addDriverContext(); OperatorContext revokableOperatorContext = driverContext.addOperatorContext( Integer.MAX_VALUE, new PlanNodeId("revokable_operator"), TableScanOperator.class.getSimpleName()); OutputFactory outputFactory = new PageConsumerOutputFactory(types -> (page -> {})); Operator outputOperator = outputFactory.createOutputOperator(2, new PlanNodeId("output"), ImmutableList.of(), Function.identity(), new TestingPagesSerdeFactory()).createOperator(driverContext); RevocableMemoryOperator revocableMemoryOperator = new RevocableMemoryOperator(revokableOperatorContext, reservedPerPage, numberOfPages); createOperator.set(revocableMemoryOperator); Driver driver = Driver.createDriver(driverContext, revocableMemoryOperator, outputOperator); return ImmutableList.of(driver); }); return createOperator.get(); }
private void instantiateBuildDrivers(BuildSideSetup buildSideSetup, TaskContext taskContext) { PipelineContext buildPipeline = taskContext.addPipelineContext(1, true, true, false); List<Driver> buildDrivers = new ArrayList<>(); List<HashBuilderOperator> buildOperators = new ArrayList<>(); for (int i = 0; i < buildSideSetup.getPartitionCount(); i++) { DriverContext buildDriverContext = buildPipeline.addDriverContext(); HashBuilderOperator buildOperator = buildSideSetup.getBuildOperatorFactory().createOperator(buildDriverContext); Driver driver = Driver.createDriver( buildDriverContext, buildSideSetup.getBuildSideSourceOperatorFactory().createOperator(buildDriverContext), buildOperator); buildDrivers.add(driver); buildOperators.add(buildOperator); } buildSideSetup.setDriversAndOperators(buildDrivers, buildOperators); }
@Test public void testBrokenOperatorCloseWhileProcessing() throws Exception { BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"), false); final Driver driver = Driver.createDriver(driverContext, brokenOperator, createSinkOperator(ImmutableList.of())); assertSame(driver.getDriverContext(), driverContext); // block thread in operator processing Future<Boolean> driverProcessFor = executor.submit(new Callable<Boolean>() { @Override public Boolean call() { return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone(); } }); brokenOperator.waitForLocked(); driver.close(); assertTrue(driver.isFinished()); try { driverProcessFor.get(1, TimeUnit.SECONDS); fail("Expected InterruptedException"); } catch (ExecutionException e) { assertDriverInterrupted(e.getCause()); } }
@Test(invocationCount = 1_000, timeOut = 10_000) public void testConcurrentClose() { List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT); OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"); ValuesOperator source = new ValuesOperator(operatorContext, rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); Operator sink = createSinkOperator(types); Driver driver = Driver.createDriver(driverContext, source, sink); // let these threads race scheduledExecutor.submit(() -> driver.processFor(new Duration(1, TimeUnit.NANOSECONDS))); // don't want to call isFinishedInternal in processFor scheduledExecutor.submit(() -> driver.close()); while (!driverContext.isDone()) { Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS); } }
@Test public void testBrokenOperatorProcessWhileClosing() throws Exception { BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"), true); final Driver driver = Driver.createDriver(driverContext, brokenOperator, createSinkOperator(ImmutableList.of())); assertSame(driver.getDriverContext(), driverContext); // block thread in operator close Future<Boolean> driverClose = executor.submit(new Callable<Boolean>() { @Override public Boolean call() { driver.close(); return true; } }); brokenOperator.waitForLocked(); assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone()); assertTrue(driver.isFinished()); brokenOperator.unlock(); assertTrue(driverClose.get()); }
@Test public void testAbruptFinish() { List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT); ValuesOperator source = new ValuesOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"), rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); PageConsumerOperator sink = createSinkOperator(types); Driver driver = Driver.createDriver(driverContext, source, sink); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); driver.close(); assertTrue(driver.isFinished()); // finish is only called in normal operations assertFalse(source.isFinished()); assertFalse(sink.isFinished()); // close is always called (values operator doesn't have a closed state) assertTrue(sink.isClosed()); }
@Test public void testNormalFinish() { List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT); ValuesOperator source = new ValuesOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"), rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); Operator sink = createSinkOperator(types); Driver driver = Driver.createDriver(driverContext, source, sink); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS)); assertTrue(blocked.isDone()); assertTrue(driver.isFinished()); assertTrue(sink.isFinished()); assertTrue(source.isFinished()); }
@Test public void testMemoryRevocationRace() { List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT); TableScanOperator source = new AlwaysBlockedMemoryRevokingTableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "scan"), new PlanNodeId("source"), new PageSourceProvider() { @Override public ConnectorPageSource createPageSource(Session session, Split split, List<ColumnHandle> columns) { return new FixedPageSource(rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); } }, ImmutableList.of()); Driver driver = Driver.createDriver(driverContext, source, createSinkOperator(types)); // the table scan operator will request memory revocation with requestMemoryRevoking() // while the driver is still not done with the processFor() method and before it moves to // updateDriverBlockedFuture() method. assertTrue(driver.processFor(new Duration(100, TimeUnit.MILLISECONDS)).isDone()); }
Driver driver = Driver.createDriver(driverContext, source, sink);
final Driver driver = Driver.createDriver(driverContext, source, brokenOperator);
ValuesOperatorFactory valuesOperatorFactory = new ValuesOperatorFactory(0, new PlanNodeId("values"), buildPages.build()); LocalExchangeSinkOperatorFactory sinkOperatorFactory = new LocalExchangeSinkOperatorFactory(localExchangeFactory, 1, new PlanNodeId("sink"), localExchangeSinkFactoryId, Function.identity()); Driver sourceDriver = Driver.createDriver(collectDriverContext, valuesOperatorFactory.createOperator(collectDriverContext), sinkOperatorFactory.createOperator(collectDriverContext));
@Test(dataProvider = "hashEnabledValues", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node user memory limit of.*") public void testMemoryLimit(boolean hashEnabled) { DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION, new DataSize(100, BYTE)) .addPipelineContext(0, true, true, false) .addDriverContext(); OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), ValuesOperator.class.getSimpleName()); List<Type> buildTypes = ImmutableList.of(BIGINT); RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), buildTypes); Operator buildOperator = new ValuesOperator(operatorContext, rowPagesBuilder .addSequencePage(10000, 20) .build()); SetBuilderOperatorFactory setBuilderOperatorFactory = new SetBuilderOperatorFactory( 1, new PlanNodeId("test"), buildTypes.get(0), 0, rowPagesBuilder.getHashChannel(), 10, new JoinCompiler(createTestMetadataManager(), new FeaturesConfig())); Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext); Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator); while (!driver.isFinished()) { driver.process(); } } }
ImmutableList.of()); PageConsumerOperator sink = createSinkOperator(types); Driver driver = Driver.createDriver(driverContext, source, sink); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished());
Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext); Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator); while (!driver.isFinished()) { driver.process();
Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext); Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator); while (!driver.isFinished()) { driver.process();
Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext); Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator); while (!driver.isFinished()) { driver.process();
Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext); Driver driver = Driver.createDriver(driverContext, buildOperator, setBuilderOperator); while (!driver.isFinished()) { driver.process();
Driver driver = Driver.createDriver(driverContext, valuesOperator, nestedLoopBuildOperator);