@Override public ListenableFuture<?> processFor(Duration duration) { Driver driver; synchronized (this) { // if close() was called before we get here, there's not point in even creating the driver if (closed) { return Futures.immediateFuture(null); } if (this.driver == null) { this.driver = driverSplitRunnerFactory.createDriver(driverContext, partitionedSplit); } driver = this.driver; } return driver.processFor(duration); }
@Override public Boolean call() { return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone(); } });
@Override public Boolean call() { return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone(); } });
@Test public void testBrokenOperatorProcessWhileClosing() throws Exception { BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"), true); final Driver driver = Driver.createDriver(driverContext, brokenOperator, createSinkOperator(ImmutableList.of())); assertSame(driver.getDriverContext(), driverContext); // block thread in operator close Future<Boolean> driverClose = executor.submit(new Callable<Boolean>() { @Override public Boolean call() { driver.close(); return true; } }); brokenOperator.waitForLocked(); assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone()); assertTrue(driver.isFinished()); brokenOperator.unlock(); assertTrue(driverClose.get()); }
@Test(invocationCount = 1_000, timeOut = 10_000) public void testConcurrentClose() { List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT); OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"); ValuesOperator source = new ValuesOperator(operatorContext, rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); Operator sink = createSinkOperator(types); Driver driver = Driver.createDriver(driverContext, source, sink); // let these threads race scheduledExecutor.submit(() -> driver.processFor(new Duration(1, TimeUnit.NANOSECONDS))); // don't want to call isFinishedInternal in processFor scheduledExecutor.submit(() -> driver.close()); while (!driverContext.isDone()) { Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS); } }
@Test public void testNormalFinish() { List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT); ValuesOperator source = new ValuesOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"), rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); Operator sink = createSinkOperator(types); Driver driver = Driver.createDriver(driverContext, source, sink); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS)); assertTrue(blocked.isDone()); assertTrue(driver.isFinished()); assertTrue(sink.isFinished()); assertTrue(source.isFinished()); }
@Test public void testMemoryRevocationRace() { List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT); TableScanOperator source = new AlwaysBlockedMemoryRevokingTableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "scan"), new PlanNodeId("source"), new PageSourceProvider() { @Override public ConnectorPageSource createPageSource(Session session, Split split, List<ColumnHandle> columns) { return new FixedPageSource(rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); } }, ImmutableList.of()); Driver driver = Driver.createDriver(driverContext, source, createSinkOperator(types)); // the table scan operator will request memory revocation with requestMemoryRevoking() // while the driver is still not done with the processFor() method and before it moves to // updateDriverBlockedFuture() method. assertTrue(driver.processFor(new Duration(100, TimeUnit.MILLISECONDS)).isDone()); }
assertFalse(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone()); assertFalse(driver.isFinished()); assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone()); assertTrue(driver.isFinished());
assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone()); assertFalse(driver.isFinished()); assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone()); assertFalse(driver.isFinished());
driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, testSplit)), true)); ListenableFuture<?> blocked = driver.processFor(new Duration(1, NANOSECONDS)); blocked = driver.processFor(new Duration(1, NANOSECONDS)); assertFalse(blocked.isDone()); assertFalse(source.getOperatorContext().isWaitingForMemory().isDone()); blocked = driver.processFor(new Duration(1, NANOSECONDS)); assertTrue(blocked.isDone());
@Override public ListenableFuture<?> processFor(Duration duration) { Driver driver; synchronized (this) { // if close() was called before we get here, there's not point in even creating the driver if (closed) { return Futures.immediateFuture(null); } if (this.driver == null) { this.driver = driverSplitRunnerFactory.createDriver(driverContext, partitionedSplit); } driver = this.driver; } return driver.processFor(duration); }
@Override public Boolean call() throws Exception { return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone(); } });
@Override public Boolean call() throws Exception { return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone(); } });
@Test public void testBrokenOperatorProcessWhileClosing() throws Exception { BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"), true); final Driver driver = new Driver(driverContext, brokenOperator, createSinkOperator(brokenOperator)); assertSame(driver.getDriverContext(), driverContext); // block thread in operator close Future<Boolean> driverClose = executor.submit(new Callable<Boolean>() { @Override public Boolean call() throws Exception { driver.close(); return true; } }); brokenOperator.waitForLocked(); assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone()); assertTrue(driver.isFinished()); brokenOperator.unlock(); assertTrue(driverClose.get()); }
@Test public void testNormalFinish() { List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT); ValuesOperator source = new ValuesOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"), types, rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); MaterializingOperator sink = createSinkOperator(source); Driver driver = new Driver(driverContext, source, sink); assertSame(driver.getDriverContext(), driverContext); assertFalse(driver.isFinished()); ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS)); assertTrue(blocked.isDone()); assertTrue(driver.isFinished()); assertTrue(sink.isFinished()); assertTrue(source.isFinished()); }
assertFalse(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone()); assertFalse(driver.isFinished()); assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone()); assertTrue(driver.isFinished());
assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone()); assertFalse(driver.isFinished()); assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone()); assertFalse(driver.isFinished());