private static Block[] getSingleValueBlocks(Page page, int position) { Block[] blocks = page.getBlocks(); Block[] row = new Block[blocks.length]; for (int i = 0; i < blocks.length; i++) { row[i] = blocks[i].getSingleValueBlock(position); } return row; }
@Override public int hashPosition(int position, Page page) { Block[] blocks = page.getBlocks(); int result = HashGenerationOptimizer.INITIAL_HASH_VALUE; for (int i = 0; i < hashChannels.length; i++) { Type type = hashChannelTypes.get(i); result = (int) CombineHashFunction.getHash(result, TypeUtils.hashPosition(type, blocks[hashChannels[i]], position)); } return result; }
public static Page getHashPage(Page page, List<? extends Type> types, List<Integer> hashChannels) { Block[] blocks = Arrays.copyOf(page.getBlocks(), page.getChannelCount() + 1); ImmutableList.Builder<Type> hashTypes = ImmutableList.builder(); Block[] hashBlocks = new Block[hashChannels.size()]; int hashBlockIndex = 0; for (int channel : hashChannels) { hashTypes.add(types.get(channel)); hashBlocks[hashBlockIndex++] = blocks[channel]; } blocks[page.getChannelCount()] = getHashBlock(hashTypes.build(), hashBlocks); return new Page(blocks); }
private long mergeWithGlobalCandidates(Page page) { long sizeDelta = 0; Block[] blocks = page.getBlocks(); for (int position = 0; position < page.getPositionCount(); position++) { if (globalCandidates.size() < n || compare(position, blocks, globalCandidates.peek()) < 0) { sizeDelta += addRow(position, blocks); } } return sizeDelta; }
@Override public long getJoinPosition(int position, Page page) { return getJoinPosition(position, page, pagesHashStrategy.hashRow(position, page.getBlocks())); }
@Override public long getJoinPosition(int position, Page page, int rawHash) { int pos = getHashPosition(rawHash, mask); while (key[pos] != -1) { if (positionEqualsCurrentRow(key[pos], position, page.getBlocks())) { return key[pos]; } // increment position and mask to handler wrap around pos = (pos + 1) & mask; } return -1; }
@Override public boolean advanceNextPosition() { while (positionIterator == null || !positionIterator.hasNext()) { if (!pageAndPositionsIterator.hasNext()) { return false; } PageAndPositions pageAndPositions = pageAndPositionsIterator.next(); page = pageAndPositions.getUpdateRequest().getPage(); blocks = page.getBlocks(); checkState(types.size() == blocks.length); positionIterator = pageAndPositions.getPositions().iterator(); } position = positionIterator.nextInt(); return true; }
private static int findGroupEnd(Page page, PagesHashStrategy pagesHashStrategy, int startPosition) { checkArgument(page.getPositionCount() > 0, "Must have at least one position"); checkPositionIndex(startPosition, page.getPositionCount(), "startPosition out of bounds"); // Short circuit if the whole page has the same value if (pagesHashStrategy.rowEqualsRow(startPosition, page.getBlocks(), page.getPositionCount() - 1, page.getBlocks())) { return page.getPositionCount(); } // TODO: do position binary search int endPosition = startPosition + 1; while (endPosition < page.getPositionCount() && pagesHashStrategy.rowEqualsRow(endPosition - 1, page.getBlocks(), endPosition, page.getBlocks())) { endPosition++; } return endPosition; }
@Override public void addInput(Page page) { requireNonNull(page, "page is null"); checkState(!finishing, "Operator is finishing"); checkState(outputPage == null, "Operator still has pending output"); operatorContext.setMemoryReservation(markDistinctHash.getEstimatedSize()); Block markerBlock = markDistinctHash.markDistinctRows(page); // add the new boolean column to the page Block[] sourceBlocks = page.getBlocks(); Block[] outputBlocks = new Block[sourceBlocks.length + 1]; // +1 for the single boolean output channel System.arraycopy(sourceBlocks, 0, outputBlocks, 0, sourceBlocks.length); outputBlocks[sourceBlocks.length] = markerBlock; outputPage = new Page(outputBlocks); }
@Override public int process(ConnectorSession session, Page page, int start, int end, PageBuilder pageBuilder) { int position = start; Block[] inputBlocks = page.getBlocks(); for (; position < end && !pageBuilder.isFull(); position++) { if (filterFunction.filter(position, page.getBlocks())) { pageBuilder.declarePosition(); for (int i = 0; i < projections.size(); i++) { // todo: if the projection function increases the size of the data significantly, this could cause the servers to OOM projections.get(i).project(position, inputBlocks, pageBuilder.getBlockBuilder(i)); } } } return position; }
public PagesWriter append(Page page) { requireNonNull(page, "page is null"); Block[] blocks = page.getBlocks(); output.writeInt(page.getPositionCount()); output.writeInt(blocks.length); for (int i = 0; i < blocks.length; i++) { writeBlock(serde, output, blocks[i]); } return this; } }
@Override public long getJoinPosition(int position, Page page) { Block[] blocks = page.getBlocks(); long joinPosition = indexedData.getJoinPosition(position, page); if (joinPosition == UNLOADED_INDEX_KEY) { indexedData.close(); // Close out the old indexedData indexedData = indexLoader.getIndexedDataForKeys(position, blocks); joinPosition = indexedData.getJoinPosition(position, page); checkState(joinPosition != UNLOADED_INDEX_KEY); } // INVARIANT: position is -1 or a valid position greater than or equal to zero return joinPosition; }
@Override public Page processColumnar(ConnectorSession session, Page page, List<? extends Type> types) { int[] selectedPositions = filterPage(page); if (selectedPositions.length == 0) { return null; } if (projections.isEmpty()) { return new Page(selectedPositions.length); } PageBuilder pageBuilder = new PageBuilder(types); Block[] inputBlocks = page.getBlocks(); for (int projectionIndex = 0; projectionIndex < projections.size(); projectionIndex++) { ProjectionFunction projection = projections.get(projectionIndex); projectColumnar(selectedPositions, pageBuilder.getBlockBuilder(projectionIndex), inputBlocks, projection); } pageBuilder.declarePositions(selectedPositions.length); return pageBuilder.build(); }
private static List<Page> stripRowNumberColumn(List<Page> input) { return input.stream() .map(page -> { Block[] blocks = Arrays.copyOf(page.getBlocks(), page.getChannelCount() - 1); return new Page(blocks); }) .collect(toImmutableList()); } }
@Override public void writePage(Page page) { Block[] columns = page.getBlocks(); for (int position = 0; position < page.getPositionCount(); position++) { recordWriter.addRow(columns, position); } }
@Override public boolean contains(int position, Page page, int[] hashChannels) { int rawHash = hashStrategy.hashRow(position, page.getBlocks()); int hashPosition = getHashPosition(rawHash, mask); // look for a slot containing this key while (groupAddressByHash[hashPosition] != -1) { long address = groupAddressByHash[hashPosition]; if (hashStrategy.positionEqualsRow(decodeSliceIndex(address), decodePosition(address), position, page, hashChannels)) { // found an existing slot for this key return true; } // increment position and mask to handle wrap around hashPosition = (hashPosition + 1) & mask; } return false; }
private static Page[] reverseColumns(Page[] pages) { Page[] newPages = new Page[pages.length]; for (int i = 0; i < pages.length; i++) { Page page = pages[i]; if (page.getPositionCount() == 0) { newPages[i] = page; } else { Block[] newBlocks = Arrays.copyOf(page.getBlocks(), page.getChannelCount()); Collections.reverse(Arrays.asList(newBlocks)); newPages[i] = new Page(page.getPositionCount(), newBlocks); } } return newPages; }
private static Page[] offsetColumns(Page[] pages, int offset) { Page[] newPages = new Page[pages.length]; for (int i = 0; i < pages.length; i++) { Page page = pages[i]; Block[] newBlocks = new Block[page.getChannelCount() + offset]; for (int channel = 0; channel < offset; channel++) { newBlocks[channel] = createNullRLEBlock(page.getPositionCount()); } for (int channel = 0; channel < page.getBlocks().length; channel++) { newBlocks[channel + offset] = page.getBlocks()[channel]; } newPages[i] = new Page(page.getPositionCount(), newBlocks); } return newPages; }
public static List<Page> appendSampleWeight(List<Page> input, int sampleWeight) { return input.stream() .map(page -> { BlockBuilder builder = BIGINT.createBlockBuilder(new BlockBuilderStatus(), page.getPositionCount()); for (int i = 0; i < page.getPositionCount(); i++) { BIGINT.writeLong(builder, sampleWeight); } Block[] blocks = new Block[page.getChannelCount() + 1]; System.arraycopy(page.getBlocks(), 0, blocks, 0, page.getChannelCount()); blocks[blocks.length - 1] = builder.build(); return new Page(blocks); }) .collect(toImmutableList()); }
private static Page[] maskPages(boolean maskValue, Page... pages) { Page[] maskedPages = new Page[pages.length]; for (int i = 0; i < pages.length; i++) { Page page = pages[i]; BlockBuilder blockBuilder = BOOLEAN.createBlockBuilder(new BlockBuilderStatus(), page.getPositionCount()); for (int j = 0; j < page.getPositionCount(); j++) { BOOLEAN.writeBoolean(blockBuilder, maskValue); } Block[] sourceBlocks = page.getBlocks(); Block[] outputBlocks = new Block[sourceBlocks.length + 1]; // +1 for the single boolean output channel System.arraycopy(sourceBlocks, 0, outputBlocks, 0, sourceBlocks.length); outputBlocks[sourceBlocks.length] = blockBuilder.build(); maskedPages[i] = new Page(outputBlocks); } return maskedPages; }