Refine search
@Test public void testRampup() { SplitConcurrencyController controller = new SplitConcurrencyController(1, new Duration(1, SECONDS)); for (int i = 0; i < 10; i++) { controller.update(SECONDS.toNanos(2), 0, i + 1); assertEquals(controller.getTargetConcurrency(), i + 2); } }
private void assertAllTimeSpentInQueueing(QueryState expectedState, Consumer<QueryStateMachine> stateTransition) { TestingTicker ticker = new TestingTicker(); QueryStateMachine stateMachine = createQueryStateMachineWithTicker(ticker); ticker.increment(7, MILLISECONDS); stateTransition.accept(stateMachine); assertEquals(stateMachine.getQueryState(), expectedState); QueryStats queryStats = stateMachine.getQueryInfo(Optional.empty()).getQueryStats(); assertEquals(queryStats.getQueuedTime(), new Duration(7, MILLISECONDS)); assertEquals(queryStats.getResourceWaitingTime(), new Duration(0, MILLISECONDS)); assertEquals(queryStats.getTotalPlanningTime(), new Duration(0, MILLISECONDS)); assertEquals(queryStats.getExecutionTime(), new Duration(0, MILLISECONDS)); assertEquals(queryStats.getFinishingTime(), new Duration(0, MILLISECONDS)); }
@Test public void testRampdown() { SplitConcurrencyController controller = new SplitConcurrencyController(10, new Duration(1, SECONDS)); for (int i = 0; i < 9; i++) { controller.update(SECONDS.toNanos(2), 1, 10 - i); controller.splitFinished(SECONDS.toNanos(30), 1, 10 - i); assertEquals(controller.getTargetConcurrency(), 10 - i - 1); } }
@Test public void testExplicitPropertyMappings() { Map<String, String> properties = new ImmutableMap.Builder<String, String>() .put(SECURITY_CONFIG_FILE, "/test.json") .put(SECURITY_REFRESH_PERIOD, "1s") .build(); FileBasedAccessControlConfig expected = new FileBasedAccessControlConfig() .setConfigFile("/test.json") .setRefreshPeriod(new Duration(1, TimeUnit.SECONDS)); assertFullMapping(properties, expected); }
public static void assertExpectedDriverStats(DriverStats actual) assertEquals(actual.getLifespan(), Lifespan.driverGroup(21)); assertEquals(actual.getCreateTime(), new DateTime(1, UTC)); assertEquals(actual.getStartTime(), new DateTime(2, UTC)); assertEquals(actual.getEndTime(), new DateTime(3, UTC)); assertEquals(actual.getQueuedTime(), new Duration(4, NANOSECONDS)); assertEquals(actual.getElapsedTime(), new Duration(5, NANOSECONDS)); assertEquals(actual.getUserMemoryReservation(), new DataSize(6, BYTE)); assertEquals(actual.getSystemMemoryReservation(), new DataSize(8, BYTE)); assertEquals(actual.getTotalScheduledTime(), new Duration(9, NANOSECONDS)); assertEquals(actual.getTotalCpuTime(), new Duration(10, NANOSECONDS)); assertEquals(actual.getTotalBlockedTime(), new Duration(12, NANOSECONDS)); assertEquals(actual.getRawInputDataSize(), new DataSize(13, BYTE)); assertEquals(actual.getRawInputPositions(), 14); assertEquals(actual.getRawInputReadTime(), new Duration(15, NANOSECONDS)); assertEquals(actual.getProcessedInputDataSize(), new DataSize(16, BYTE));
@Test public void testUpdateLevelWithCap() { MultilevelSplitQueue splitQueue = new MultilevelSplitQueue(2); TaskHandle handle0 = new TaskHandle(new TaskId("test0", 0, 0), splitQueue, () -> 1, 1, new Duration(1, SECONDS), OptionalInt.empty()); long quantaNanos = MINUTES.toNanos(10); handle0.addScheduledNanos(quantaNanos); long cappedNanos = Math.min(quantaNanos, LEVEL_CONTRIBUTION_CAP); for (int i = 0; i < (LEVEL_THRESHOLD_SECONDS.length - 1); i++) { long thisLevelTime = Math.min(SECONDS.toNanos(LEVEL_THRESHOLD_SECONDS[i + 1] - LEVEL_THRESHOLD_SECONDS[i]), cappedNanos); assertEquals(splitQueue.getLevelScheduledTime(i), thisLevelTime); cappedNanos -= thisLevelTime; } }
@Test public void testExplicitPropertyMappings() { Map<String, String> properties = new ImmutableMap.Builder<String, String>() .put("transaction.idle-check-interval", "1s") .put("transaction.idle-timeout", "10s") .put("transaction.max-finishing-concurrency", "100") .build(); TransactionManagerConfig expected = new TransactionManagerConfig() .setIdleCheckInterval(new Duration(1, TimeUnit.SECONDS)) .setIdleTimeout(new Duration(10, TimeUnit.SECONDS)) .setMaxFinishingConcurrency(100); assertFullMapping(properties, expected); } }
public static void assertExpectedTaskStats(TaskStats actual) assertEquals(actual.getCreateTime(), new DateTime(1, UTC)); assertEquals(actual.getFirstStartTime(), new DateTime(2, UTC)); assertEquals(actual.getLastStartTime(), new DateTime(100, UTC)); assertEquals(actual.getLastEndTime(), new DateTime(101, UTC)); assertEquals(actual.getEndTime(), new DateTime(3, UTC)); assertEquals(actual.getElapsedTime(), new Duration(4, NANOSECONDS)); assertEquals(actual.getQueuedTime(), new Duration(5, NANOSECONDS)); assertEquals(actual.getTotalDrivers(), 6); assertEquals(actual.getSystemMemoryReservation(), new DataSize(14, BYTE)); assertEquals(actual.getTotalScheduledTime(), new Duration(15, NANOSECONDS)); assertEquals(actual.getTotalCpuTime(), new Duration(16, NANOSECONDS)); assertEquals(actual.getTotalBlockedTime(), new Duration(18, NANOSECONDS)); assertEquals(actual.getRawInputDataSize(), new DataSize(19, BYTE));
@Test public void testRapidAdjustForQuickSplits() { SplitConcurrencyController controller = new SplitConcurrencyController(10, new Duration(1, SECONDS)); for (int i = 0; i < 9; i++) { controller.update(MILLISECONDS.toNanos(200), 1, 10 - i); controller.splitFinished(MILLISECONDS.toNanos(100), 1, 10 - i); assertEquals(controller.getTargetConcurrency(), 10 - i - 1); } controller.update(SECONDS.toNanos(30), 0, 1); for (int i = 0; i < 10; i++) { controller.update(SECONDS.toNanos(200), 0, i + 1); controller.splitFinished(MILLISECONDS.toNanos(100), 0, i + 1); assertEquals(controller.getTargetConcurrency(), i + 2); } } }
@Test public void testDefaults() { ConfigAssertions.assertRecordedDefaults(ConfigAssertions.recordDefaults(FailureDetectorConfig.class) .setExpirationGraceInterval(new Duration(10, TimeUnit.MINUTES)) .setFailureRatioThreshold(0.1) .setHeartbeatInterval(new Duration(500, TimeUnit.MILLISECONDS)) .setWarmupInterval(new Duration(5, TimeUnit.SECONDS)) .setEnabled(true)); }
public static void assertExpectedOperatorStats(OperatorStats actual) assertEquals(actual.getStageId(), 0); assertEquals(actual.getOperatorId(), 41); assertEquals(actual.getOperatorType(), "test"); assertEquals(actual.getTotalDrivers(), 1); assertEquals(actual.getAddInputCalls(), 2); assertEquals(actual.getAddInputWall(), new Duration(3, NANOSECONDS)); assertEquals(actual.getAddInputCpu(), new Duration(4, NANOSECONDS)); assertEquals(actual.getRawInputDataSize(), new DataSize(5, BYTE)); assertEquals(actual.getInputDataSize(), new DataSize(6, BYTE)); assertEquals(actual.getGetOutputCalls(), 9); assertEquals(actual.getGetOutputWall(), new Duration(10, NANOSECONDS)); assertEquals(actual.getGetOutputCpu(), new Duration(11, NANOSECONDS)); assertEquals(actual.getOutputDataSize(), new DataSize(12, BYTE)); assertEquals(actual.getOutputPositions(), 13); assertEquals(actual.getPhysicalWrittenDataSize(), new DataSize(14, BYTE)); assertEquals(actual.getBlockedWall(), new Duration(15, NANOSECONDS)); assertEquals(actual.getFinishCalls(), 16); assertEquals(actual.getFinishWall(), new Duration(17, NANOSECONDS)); assertEquals(actual.getFinishCpu(), new Duration(18, NANOSECONDS)); assertEquals(actual.getUserMemoryReservation(), new DataSize(19, BYTE));
@Test public void testLevelContributionCap() { MultilevelSplitQueue splitQueue = new MultilevelSplitQueue(2); TaskHandle handle0 = new TaskHandle(new TaskId("test0", 0, 0), splitQueue, () -> 1, 1, new Duration(1, SECONDS), OptionalInt.empty()); TaskHandle handle1 = new TaskHandle(new TaskId("test1", 0, 0), splitQueue, () -> 1, 1, new Duration(1, SECONDS), OptionalInt.empty()); for (int i = 0; i < (LEVEL_THRESHOLD_SECONDS.length - 1); i++) { long levelAdvanceTime = SECONDS.toNanos(LEVEL_THRESHOLD_SECONDS[i + 1] - LEVEL_THRESHOLD_SECONDS[i]); handle0.addScheduledNanos(levelAdvanceTime); assertEquals(handle0.getPriority().getLevel(), i + 1); handle1.addScheduledNanos(levelAdvanceTime); assertEquals(handle1.getPriority().getLevel(), i + 1); assertEquals(splitQueue.getLevelScheduledTime(i), 2 * Math.min(levelAdvanceTime, LEVEL_CONTRIBUTION_CAP)); assertEquals(splitQueue.getLevelScheduledTime(i + 1), 0); } }
@Test public void testExplicitPropertyMappings() { Map<String, String> properties = new ImmutableMap.Builder<String, String>() .put("failure-detector.expiration-grace-interval", "5m") .put("failure-detector.warmup-interval", "60s") .put("failure-detector.heartbeat-interval", "10s") .put("failure-detector.threshold", "0.5") .put("failure-detector.enabled", "false") .build(); FailureDetectorConfig expected = new FailureDetectorConfig() .setExpirationGraceInterval(new Duration(5, TimeUnit.MINUTES)) .setWarmupInterval(new Duration(60, TimeUnit.SECONDS)) .setHeartbeatInterval(new Duration(10, TimeUnit.SECONDS)) .setFailureRatioThreshold(0.5) .setEnabled(false); ConfigAssertions.assertFullMapping(properties, expected); } }
public static void assertExpectedQueryStats(QueryStats actual) assertEquals(actual.getCreateTime(), new DateTime(1, UTC)); assertEquals(actual.getExecutionStartTime(), new DateTime(2, UTC)); assertEquals(actual.getLastHeartbeat(), new DateTime(3, UTC)); assertEquals(actual.getEndTime(), new DateTime(4, UTC)); assertEquals(actual.getElapsedTime(), new Duration(6, NANOSECONDS)); assertEquals(actual.getQueuedTime(), new Duration(5, NANOSECONDS)); assertEquals(actual.getExecutionTime(), new Duration(41, NANOSECONDS)); assertEquals(actual.getAnalysisTime(), new Duration(7, NANOSECONDS)); assertEquals(actual.getDistributedPlanningTime(), new Duration(8, NANOSECONDS)); assertEquals(actual.getTotalPlanningTime(), new Duration(100, NANOSECONDS)); assertEquals(actual.getFinishingTime(), new Duration(200, NANOSECONDS)); assertEquals(actual.getTotalTasks(), 9); assertEquals(actual.getSpilledDataSize(), new DataSize(690, BYTE)); assertEquals(actual.getTotalScheduledTime(), new Duration(20, NANOSECONDS)); assertEquals(actual.getTotalCpuTime(), new Duration(21, NANOSECONDS)); assertEquals(actual.getTotalBlockedTime(), new Duration(23, NANOSECONDS)); assertEquals(actual.getRawInputDataSize(), new DataSize(24, BYTE));
@Test public void testCreateTableAsSelectDifferentCatalog() throws Exception { handle.execute("CREATE TABLE \"my_test_table2\" (column1 BIGINT, column2 DOUBLE)"); SqlParser parser = new SqlParser(); Query query = new Query(CATALOG, SCHEMA, ImmutableList.of(), "CREATE TABLE public.my_test_table2 AS SELECT 1 column1, 2E0 column2", ImmutableList.of(), null, null, ImmutableMap.of()); QueryRewriter rewriter = new QueryRewriter(parser, URL, QualifiedName.of("other_catalog", "other_schema", "tmp_"), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), 1, new Duration(10, SECONDS)); Query rewrittenQuery = rewriter.shadowQuery(query); assertEquals(rewrittenQuery.getPreQueries().size(), 1); CreateTableAsSelect createTableAs = (CreateTableAsSelect) parser.createStatement(rewrittenQuery.getPreQueries().get(0)); assertEquals(createTableAs.getName().getParts().size(), 3); assertEquals(createTableAs.getName().getPrefix().get(), QualifiedName.of("other_catalog", "other_schema")); assertTrue(createTableAs.getName().getSuffix().startsWith("tmp_")); assertFalse(createTableAs.getName().getSuffix().contains("my_test_table")); }
@Test public void testExplicitPropertyMappings() { Map<String, String> properties = new ImmutableMap.Builder<String, String>() .put("mysql.auto-reconnect", "false") .put("mysql.max-reconnects", "4") .put("mysql.connection-timeout", "4s").build(); MySqlConfig expected = new MySqlConfig() .setAutoReconnect(false) .setMaxReconnects(4) .setConnectionTimeout(new Duration(4, TimeUnit.SECONDS)); assertFullMapping(properties, expected); } }
public static void assertExpectedPipelineStats(PipelineStats actual) assertEquals(actual.getFirstStartTime(), new DateTime(100, UTC)); assertEquals(actual.getLastStartTime(), new DateTime(101, UTC)); assertEquals(actual.getLastEndTime(), new DateTime(102, UTC)); assertEquals(actual.isInputPipeline(), true); assertEquals(actual.isOutputPipeline(), false); assertEquals(actual.getElapsedTime().getCount(), 9.0); assertEquals(actual.getTotalScheduledTime(), new Duration(10, NANOSECONDS)); assertEquals(actual.getTotalCpuTime(), new Duration(11, NANOSECONDS)); assertEquals(actual.getTotalBlockedTime(), new Duration(13, NANOSECONDS)); assertEquals(actual.getRawInputDataSize(), new DataSize(14, BYTE));
@Test public void testDecreasingBatchSize() { int rows = 1024; // We set the expensive expression threshold to 0, so the expression is always considered expensive and the batch size gets halved until it becomes 1 TestingTicker testingTicker = new TestingTicker(); ExpressionProfiler profiler = new ExpressionProfiler(testingTicker, new Duration(0, MILLISECONDS)); PageProcessor pageProcessor = new PageProcessor( Optional.empty(), ImmutableList.of(new InputPageProjection(0, BIGINT)), OptionalInt.of(512), profiler); Slice[] slices = new Slice[rows]; Arrays.fill(slices, Slices.allocate(rows)); Page inputPage = new Page(createSlicesBlock(slices)); Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, inputPage); long previousPositionCount = 1; long totalPositionCount = 0; while (totalPositionCount < rows) { Optional<Page> page = output.next(); assertTrue(page.isPresent()); long positionCount = page.get().getPositionCount(); totalPositionCount += positionCount; // the batch size doesn't get smaller than 1 if (positionCount > 1 && previousPositionCount != 1) { assertEquals(positionCount, previousPositionCount / 2); } previousPositionCount = positionCount; } }