protected void generateRandomInputPage() { RowPagesBuilder buildPagesBuilder = rowPagesBuilder(types); for (int i = 0; i < PAGE_SIZE; i++) { Object[] values = types.stream() .map(this::generateRandomValue) .collect(toList()).toArray(); buildPagesBuilder.row(values); } inputPage = getOnlyElement(buildPagesBuilder.build()); }
public static List<Page> createPagesWithDistinctHashKeys(Type type, int pageCount, int positionCountPerPage) { RowPagesBuilder rowPagesBuilder = rowPagesBuilder(true, ImmutableList.of(0), type); for (int i = 0; i < pageCount; i++) { rowPagesBuilder.addSequencePage(positionCountPerPage, positionCountPerPage * i); } return rowPagesBuilder.build(); }
protected void generateInputPage(int... initialValues) { RowPagesBuilder buildPagesBuilder = rowPagesBuilder(types); buildPagesBuilder.addSequencePage(PAGE_SIZE, initialValues); inputPage = getOnlyElement(buildPagesBuilder.build()); }
@Test public void testEmptyProbe() { TaskContext taskContext = createTaskContext(); RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)) .row(POLYGON_A, "A") .row(null, "null") .pageBreak() .row(POLYGON_B, "B"); RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)); MaterializedResult expected = resultBuilder(taskContext.getSession(), ImmutableList.of(VARCHAR, VARCHAR)).build(); assertSpatialJoin(taskContext, INNER, buildPages, probePages, expected); }
@Test(expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node user memory limit of.*", dataProvider = "testMemoryLimitProvider") public void testMemoryLimit(boolean parallelBuild, boolean buildHashEnabled) { TaskContext taskContext = TestingTaskContext.createTaskContext(executor, scheduledExecutor, TEST_SESSION, new DataSize(100, BYTE)); RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT)) .addSequencePage(10, 20, 30, 40); BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.empty(), false, SINGLE_STREAM_SPILLER_FACTORY); instantiateBuildDrivers(buildSideSetup, taskContext); buildLookupSource(buildSideSetup); }
@Test public void testPageSorterSorted() { List<Type> types = ImmutableList.of(BIGINT, DOUBLE, VARCHAR); List<Integer> sortChannels = Ints.asList(0); List<SortOrder> sortOrders = ImmutableList.of(ASC_NULLS_FIRST); List<Page> inputPages = RowPagesBuilder.rowPagesBuilder(types) .row(-12L, 2.2, "a") .row(-2L, 2.2, "b") .pageBreak() .row(1L, 2.2, "d") .row(2L, 1.1, "c") .build(); List<Page> expectedPages = RowPagesBuilder.rowPagesBuilder(types) .row(-12L, 2.2, "a") .row(-2L, 2.2, "b") .row(1L, 2.2, "d") .row(2L, 1.1, "c") .build(); assertSorted(inputPages, expectedPages, types, sortChannels, sortOrders, 100); }
protected void initializeBuildPages() { RowPagesBuilder buildPagesBuilder = rowPagesBuilder(buildHashEnabled, hashChannels, ImmutableList.of(VARCHAR, BIGINT, BIGINT)); int maxValue = BUILD_ROWS_NUMBER / buildRowsRepetition + 40; int rows = 0; while (rows < BUILD_ROWS_NUMBER) { int newRows = Math.min(BUILD_ROWS_NUMBER - rows, ROWS_PER_PAGE); buildPagesBuilder.addSequencePage(newRows, (rows + 20) % maxValue, (rows + 30) % maxValue, (rows + 40) % maxValue); buildPagesBuilder.pageBreak(); rows += newRows; } types = buildPagesBuilder.getTypes(); buildPages = buildPagesBuilder.build(); hashChannel = buildPagesBuilder.getHashChannel() .map(OptionalInt::of).orElse(OptionalInt.empty()); } }
@Override public ConnectorPageSource createPageSource(Session session, Split split, List<ColumnHandle> columns) { return new FixedPageSource(rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); } },
@Test(expectedExceptions = IllegalStateException.class) public void addInputFailsOnBlockedOperator() { Operator operator = createTableWriterOperator(new BlockingPageSink()); operator.addInput(rowPagesBuilder(BIGINT).row(42).build().get(0)); assertFalse(operator.isBlocked().isDone()); assertFalse(operator.needsInput()); operator.addInput(rowPagesBuilder(BIGINT).row(42).build().get(0)); }
@Override public ConnectorPageSource createPageSource(Session session, Split split, List<ColumnHandle> columns) { return new FixedPageSource(rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); } },
@Test public void testEmptyInput() { RowPagesBuilder rowPagesBuilder = RowPagesBuilder.rowPagesBuilder(BOOLEAN, VARCHAR, BIGINT); List<Page> input = rowPagesBuilder.build(); MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, BIGINT).build(); assertOperatorEquals(operatorFactory, driverContext, input, expected); }
@Override public ConnectorPageSource createPageSource(Session session, Split split, List<ColumnHandle> columns) { return new FixedPageSource(rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); } },
@Test public void testSinglePage() { RowPagesBuilder rowPagesBuilder = RowPagesBuilder.rowPagesBuilder(BOOLEAN, VARCHAR, BIGINT); List<Page> input = rowPagesBuilder .row(false, "a", 5) .build(); MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, BIGINT) .row("a", 1L, 5L) .build(); assertOperatorEquals(operatorFactory, driverContext, input, expected); }
private void testSimpleSpiller(Spiller spiller) throws ExecutionException, InterruptedException { RowPagesBuilder builder = RowPagesBuilder.rowPagesBuilder(TYPES); builder.addSequencePage(10, 0, 5, 10, 15); builder.pageBreak(); builder.addSequencePage(10, 0, -5, -10, -15); List<Page> firstSpill = builder.build(); builder = RowPagesBuilder.rowPagesBuilder(TYPES); builder.addSequencePage(10, 10, 15, 20, 25); builder.pageBreak(); builder.addSequencePage(10, -10, -15, -20, -25); List<Page> secondSpill = builder.build(); testSpiller(TYPES, spiller, firstSpill, secondSpill); }
@Test public void testPageSorter() { List<Type> types = ImmutableList.of(BIGINT, DOUBLE, VARCHAR); List<Integer> sortChannels = Ints.asList(0); List<SortOrder> sortOrders = ImmutableList.of(ASC_NULLS_FIRST); List<Page> inputPages = RowPagesBuilder.rowPagesBuilder(types) .row(2L, 1.1, "d") .row(1L, 2.2, "c") .pageBreak() .row(-2L, 2.2, "b") .row(-12L, 2.2, "a") .build(); List<Page> expectedPages = RowPagesBuilder.rowPagesBuilder(types) .row(-12L, 2.2, "a") .row(-2L, 2.2, "b") .pageBreak() .row(1L, 2.2, "c") .row(2L, 1.1, "d") .build(); assertSorted(inputPages, expectedPages, types, sortChannels, sortOrders, 100); }
@Test public void testPageSorterForceExpansion() { List<Type> types = ImmutableList.of(BIGINT, DOUBLE, VARCHAR); List<Integer> sortChannels = Ints.asList(0); List<SortOrder> sortOrders = ImmutableList.of(ASC_NULLS_FIRST); List<Page> inputPages = RowPagesBuilder.rowPagesBuilder(types) .row(2L, 1.1, "c") .row(1L, 2.2, "d") .pageBreak() .row(-2L, 2.2, "b") .row(-12L, 2.2, "a") .build(); List<Page> expectedPages = RowPagesBuilder.rowPagesBuilder(types) .row(-12L, 2.2, "a") .row(-2L, 2.2, "b") .pageBreak() .row(1L, 2.2, "d") .row(2L, 1.1, "c") .build(); assertSorted(inputPages, expectedPages, types, sortChannels, sortOrders, 2); }
@Test public void testEmptyBuild() { TaskContext taskContext = createTaskContext(); RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)); RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)) .row(POINT_X, "x") .row(null, "null") .row(POINT_Y, "y") .pageBreak() .row(POINT_Z, "z") .pageBreak() .row(POINT_W, "w"); MaterializedResult expected = resultBuilder(taskContext.getSession(), ImmutableList.of(VARCHAR, VARCHAR)).build(); assertSpatialJoin(taskContext, INNER, buildPages, probePages, expected); }
@Test public void testPageSorterMultipleChannels() { List<Type> types = ImmutableList.of(BIGINT, DOUBLE, VARCHAR); List<Integer> sortChannels = Ints.asList(0, 1, 2); List<SortOrder> sortOrders = Collections.nCopies(sortChannels.size(), ASC_NULLS_FIRST); List<Page> inputPages = RowPagesBuilder.rowPagesBuilder(types) .row(2L, 1.1, "d") .row(1L, 2.2, "c") .pageBreak() .row(1L, 2.2, "b") .row(1L, 2.2, "a") .pageBreak() .row(1L, 2.2, null) .row(1L, null, "z") .row(1L, null, null) .build(); List<Page> expectedPages = RowPagesBuilder.rowPagesBuilder(types) .row(1L, null, null) .row(1L, null, "z") .row(1L, 2.2, null) .row(1L, 2.2, "a") .row(1L, 2.2, "b") .row(1L, 2.2, "c") .row(2L, 1.1, "d") .build(); assertSorted(inputPages, expectedPages, types, sortChannels, sortOrders, 100); }
@Test public void testEmptyStreams() throws Exception { List<Type> types = ImmutableList.of(INTEGER, BIGINT, DOUBLE); MaterializedResult actual = mergeSortedPages( types, ImmutableList.of(0, 1), ImmutableList.of(ASC_NULLS_FIRST, ASC_NULLS_FIRST), ImmutableList.of( rowPagesBuilder(types) .pageBreak() .pageBreak() .build(), rowPagesBuilder(types) .pageBreak() .build(), rowPagesBuilder(types) .pageBreak() .build(), rowPagesBuilder(types) .build())); MaterializedResult expected = resultBuilder(TEST_SESSION, types) .build(); assertEquals(actual, expected); }
@Test public void testSingleGroupingValue() { RowPagesBuilder rowPagesBuilder = RowPagesBuilder.rowPagesBuilder(BOOLEAN, VARCHAR, BIGINT); List<Page> input = rowPagesBuilder .row(true, "a", 1) .row(false, "a", 2) .row(true, "a", 3) .row(false, "a", 4) .row(true, "a", 5) .pageBreak() .row(false, "a", 6) .row(true, "a", 7) .row(false, "a", 8) .pageBreak() .pageBreak() .row(true, "a", 9) .row(false, "a", 10) .build(); MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, BIGINT) .row("a", 10L, 55L) .build(); assertOperatorEquals(operatorFactory, driverContext, input, expected); } }