@Test public void testIncreasingBatchSize() { int rows = 1024; // We deliberately do not set the ticker, so that the expression is always cheap and the batch size gets doubled until other limits are hit TestingTicker testingTicker = new TestingTicker(); ExpressionProfiler profiler = new ExpressionProfiler(testingTicker, SPLIT_RUN_QUANTA); PageProcessor pageProcessor = new PageProcessor( Optional.empty(), ImmutableList.of(new InputPageProjection(0, BIGINT)), OptionalInt.of(1), profiler); Slice[] slices = new Slice[rows]; Arrays.fill(slices, Slices.allocate(rows)); Page inputPage = new Page(createSlicesBlock(slices)); Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, inputPage); long previousPositionCount = 1; long totalPositionCount = 0; while (totalPositionCount < rows) { Optional<Page> page = output.next(); assertTrue(page.isPresent()); long positionCount = page.get().getPositionCount(); totalPositionCount += positionCount; // skip the first read && skip the last read, which can be a partial page if (positionCount > 1 && totalPositionCount != rows) { assertEquals(positionCount, previousPositionCount * 2); } previousPositionCount = positionCount; } }
private void assertAllTimeSpentInQueueing(QueryState expectedState, Consumer<QueryStateMachine> stateTransition) { TestingTicker ticker = new TestingTicker(); QueryStateMachine stateMachine = createQueryStateMachineWithTicker(ticker); ticker.increment(7, MILLISECONDS); stateTransition.accept(stateMachine); assertEquals(stateMachine.getQueryState(), expectedState); QueryStats queryStats = stateMachine.getQueryInfo(Optional.empty()).getQueryStats(); assertEquals(queryStats.getQueuedTime(), new Duration(7, MILLISECONDS)); assertEquals(queryStats.getResourceWaitingTime(), new Duration(0, MILLISECONDS)); assertEquals(queryStats.getTotalPlanningTime(), new Duration(0, MILLISECONDS)); assertEquals(queryStats.getExecutionTime(), new Duration(0, MILLISECONDS)); assertEquals(queryStats.getFinishingTime(), new Duration(0, MILLISECONDS)); }
@Test public void testDecreasingBatchSize() { int rows = 1024; // We set the expensive expression threshold to 0, so the expression is always considered expensive and the batch size gets halved until it becomes 1 TestingTicker testingTicker = new TestingTicker(); ExpressionProfiler profiler = new ExpressionProfiler(testingTicker, new Duration(0, MILLISECONDS)); PageProcessor pageProcessor = new PageProcessor( Optional.empty(), ImmutableList.of(new InputPageProjection(0, BIGINT)), OptionalInt.of(512), profiler); Slice[] slices = new Slice[rows]; Arrays.fill(slices, Slices.allocate(rows)); Page inputPage = new Page(createSlicesBlock(slices)); Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, inputPage); long previousPositionCount = 1; long totalPositionCount = 0; while (totalPositionCount < rows) { Optional<Page> page = output.next(); assertTrue(page.isPresent()); long positionCount = page.get().getPositionCount(); totalPositionCount += positionCount; // the batch size doesn't get smaller than 1 if (positionCount > 1 && previousPositionCount != 1) { assertEquals(positionCount, previousPositionCount / 2); } previousPositionCount = positionCount; } }
@Test public void testFailureInterval() { TestingTicker ticker = new TestingTicker(); ticker.increment(1, NANOSECONDS); Backoff backoff = new Backoff(1, new Duration(15, SECONDS), ticker, ImmutableList.of(new Duration(10, MILLISECONDS))); ticker.increment(10, MICROSECONDS); // verify initial state assertEquals(backoff.getFailureCount(), 0); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 0); // first failure, should never fail assertFalse(backoff.failure()); assertEquals(backoff.getFailureCount(), 1); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 0); ticker.increment(14, SECONDS); // second failure within the limit, should not fail assertFalse(backoff.failure()); assertEquals(backoff.getFailureCount(), 2); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 14); ticker.increment(1, SECONDS); // final failure after the limit causes failure assertTrue(backoff.failure()); assertEquals(backoff.getFailureCount(), 3); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 15); }
throws Exception TestingTicker ticker = new TestingTicker(); AtomicReference<Duration> tickerIncrement = new AtomicReference<>(new Duration(0, TimeUnit.SECONDS));
@Test public void testMinTries() { TestingTicker ticker = new TestingTicker(); ticker.increment(1, NANOSECONDS); Backoff backoff = new Backoff(3, new Duration(1, NANOSECONDS), ticker, ImmutableList.of(new Duration(10, MILLISECONDS))); ticker.increment(10, MICROSECONDS); // verify initial state assertEquals(backoff.getFailureCount(), 0); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 0); // first failure, should never fail assertFalse(backoff.failure()); assertEquals(backoff.getFailureCount(), 1); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 0); ticker.increment(14, SECONDS); // second failure under min failures, should not fail assertFalse(backoff.failure()); assertEquals(backoff.getFailureCount(), 2); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 14); ticker.increment(1, SECONDS); // last try failed assertTrue(backoff.failure()); assertEquals(backoff.getFailureCount(), 3); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 15); }
throws Exception TestingTicker ticker = new TestingTicker(); TaskExecutor taskExecutor = new TaskExecutor(4, 8, 3, 4, ticker); taskExecutor.start();
@Test public void testStartRequest() { TestingTicker ticker = new TestingTicker(); ticker.increment(1, NANOSECONDS); Backoff backoff = new Backoff(1, new Duration(15, SECONDS), ticker, ImmutableList.of(new Duration(10, MILLISECONDS))); ticker.increment(10, MICROSECONDS); assertFalse(backoff.failure()); assertEquals(backoff.getFailureCount(), 1); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 0); assertEquals(backoff.getFailureRequestTimeTotal().roundTo(SECONDS), 0); ticker.increment(7, SECONDS); backoff.startRequest(); ticker.increment(7, SECONDS); assertFalse(backoff.failure()); assertEquals(backoff.getFailureCount(), 2); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 14); // failed request took 7 seconds. assertEquals(backoff.getFailureRequestTimeTotal().roundTo(SECONDS), 7); ticker.increment(1, SECONDS); backoff.startRequest(); ticker.increment(1, SECONDS); assertTrue(backoff.failure()); assertEquals(backoff.getFailureCount(), 3); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 16); // failed requests took 7+1 seconds. assertEquals(backoff.getFailureRequestTimeTotal().roundTo(SECONDS), 8); }
TestingTicker ticker = new TestingTicker(); ticker.increment(1, NANOSECONDS);
@Test public void testPlanningTimeDuration() TestingTicker mockTicker = new TestingTicker(); QueryStateMachine stateMachine = createQueryStateMachineWithTicker(mockTicker); assertState(stateMachine, QUEUED);
@Test public void testTaskHandle() { TestingTicker ticker = new TestingTicker(); TaskExecutor taskExecutor = new TaskExecutor(4, 8, 3, 4, ticker); taskExecutor.start(); try { TaskId taskId = new TaskId("test", 0, 0); TaskHandle taskHandle = taskExecutor.addTask(taskId, () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()); Phaser beginPhase = new Phaser(); beginPhase.register(); Phaser verificationComplete = new Phaser(); verificationComplete.register(); TestingJob driver1 = new TestingJob(ticker, new Phaser(), beginPhase, verificationComplete, 10, 0); TestingJob driver2 = new TestingJob(ticker, new Phaser(), beginPhase, verificationComplete, 10, 0); // force enqueue a split taskExecutor.enqueueSplits(taskHandle, true, ImmutableList.of(driver1)); assertEquals(taskHandle.getRunningLeafSplits(), 0); // normal enqueue a split taskExecutor.enqueueSplits(taskHandle, false, ImmutableList.of(driver2)); assertEquals(taskHandle.getRunningLeafSplits(), 1); // let the split continue to run beginPhase.arriveAndDeregister(); verificationComplete.arriveAndDeregister(); } finally { taskExecutor.stop(); } }
@Test(invocationCount = 100) public void testQuantaFairness() { TestingTicker ticker = new TestingTicker(); TaskExecutor taskExecutor = new TaskExecutor(1, 2, 3, 4, ticker); taskExecutor.start(); ticker.increment(20, MILLISECONDS); try { TaskHandle shortQuantaTaskHandle = taskExecutor.addTask(new TaskId("shortQuanta", 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()); TaskHandle longQuantaTaskHandle = taskExecutor.addTask(new TaskId("longQuanta", 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()); Phaser globalPhaser = new Phaser(); TestingJob shortQuantaDriver = new TestingJob(ticker, new Phaser(), new Phaser(), globalPhaser, 10, 10); TestingJob longQuantaDriver = new TestingJob(ticker, new Phaser(), new Phaser(), globalPhaser, 10, 20); taskExecutor.enqueueSplits(shortQuantaTaskHandle, true, ImmutableList.of(shortQuantaDriver)); taskExecutor.enqueueSplits(longQuantaTaskHandle, true, ImmutableList.of(longQuantaDriver)); for (int i = 0; i < 11; i++) { globalPhaser.arriveAndAwaitAdvance(); } assertTrue(shortQuantaDriver.getCompletedPhases() >= 7 && shortQuantaDriver.getCompletedPhases() <= 8); assertTrue(longQuantaDriver.getCompletedPhases() >= 3 && longQuantaDriver.getCompletedPhases() <= 4); globalPhaser.arriveAndDeregister(); } finally { taskExecutor.stop(); } }
TestingTicker ticker = new TestingTicker();
constant(10L, BIGINT)); TestingTicker testingTicker = new TestingTicker(); PageFunctionCompiler functionCompiler = new PageFunctionCompiler(createTestMetadataManager(), 0); Supplier<PageProjection> projectionSupplier = functionCompiler.compileProjection(add10Expression, Optional.empty());
@Test(invocationCount = 100) public void testLevelMovement() { TestingTicker ticker = new TestingTicker(); TaskExecutor taskExecutor = new TaskExecutor(2, 2, 3, 4, ticker); taskExecutor.start(); ticker.increment(20, MILLISECONDS); try { TaskHandle testTaskHandle = taskExecutor.addTask(new TaskId("test", 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()); Phaser globalPhaser = new Phaser(); globalPhaser.bulkRegister(3); int quantaTimeMills = 500; int phasesPerSecond = 1000 / quantaTimeMills; int totalPhases = LEVEL_THRESHOLD_SECONDS[LEVEL_THRESHOLD_SECONDS.length - 1] * phasesPerSecond; TestingJob driver1 = new TestingJob(ticker, globalPhaser, new Phaser(), new Phaser(), totalPhases, quantaTimeMills); TestingJob driver2 = new TestingJob(ticker, globalPhaser, new Phaser(), new Phaser(), totalPhases, quantaTimeMills); taskExecutor.enqueueSplits(testTaskHandle, true, ImmutableList.of(driver1, driver2)); int completedPhases = 0; for (int i = 0; i < (LEVEL_THRESHOLD_SECONDS.length - 1); i++) { for (; (completedPhases / phasesPerSecond) < LEVEL_THRESHOLD_SECONDS[i + 1]; completedPhases++) { globalPhaser.arriveAndAwaitAdvance(); } assertEquals(testTaskHandle.getPriority().getLevel(), i + 1); } globalPhaser.arriveAndDeregister(); } finally { taskExecutor.stop(); } }
TestingTicker ticker = new TestingTicker(); TaskExecutor taskExecutor = new TaskExecutor(4, 16, 1, maxDriversPerTask, splitQueue, ticker); taskExecutor.start();
throws Exception TestingTicker ticker = new TestingTicker(); TaskExecutor taskExecutor = new TaskExecutor(1, 3, 3, 4, new MultilevelSplitQueue(2), ticker); taskExecutor.start();
@Test public void testCompression() throws Exception { QuantileDigest digest = new QuantileDigest(1, 0, new TestingTicker()); for (int loop = 0; loop < 2; ++loop) { addRange(digest, 0, 15); digest.compress(); digest.validate(); } }
@Test public void testCountDecays() { TestingTicker ticker = new TestingTicker(); DecayCounter counter = new DecayCounter(ExponentialDecay.oneMinute(), ticker); counter.add(1); ticker.increment(1, TimeUnit.MINUTES); assertTrue(Math.abs(counter.getCount() - 1 / Math.E) < 1e-9); }
@Test public void testAddAfterRescale() { TestingTicker ticker = new TestingTicker(); DecayCounter counter = new DecayCounter(ExponentialDecay.oneMinute(), ticker); counter.add(1); ticker.increment(1, TimeUnit.MINUTES); counter.add(2); double expected = 2 + 1 / Math.E; assertTrue(Math.abs(counter.getCount() - expected) < 1e-9); } }