public AbstractTreeIndexBulkLoader(float fillFactor) throws HyracksDataException { leafFrame = leafFrameFactory.createFrame(); interiorFrame = interiorFrameFactory.createFrame(); metaFrame = freePageManager.createMetadataFrame(); queue = bufferCache.createFIFOQueue(); if (!isEmptyTree(leafFrame)) { throw HyracksDataException.create(ErrorCode.CANNOT_BULK_LOAD_NON_EMPTY_TREE); } this.cmp = MultiComparator.create(cmpFactories); leafFrame.setMultiComparator(cmp); interiorFrame.setMultiComparator(cmp); tupleWriter = leafFrame.getTupleWriter(); NodeFrontier leafFrontier = new NodeFrontier(leafFrame.createTupleReference()); leafFrontier.pageId = freePageManager.takePage(metaFrame); leafFrontier.page = bufferCache.confiscatePage(BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId)); interiorFrame.setPage(leafFrontier.page); interiorFrame.initBuffer((byte) 0); interiorMaxBytes = (int) (interiorFrame.getBuffer().capacity() * fillFactor); leafFrame.setPage(leafFrontier.page); leafFrame.initBuffer((byte) 0); leafMaxBytes = (int) (leafFrame.getBuffer().capacity() * fillFactor); slotSize = leafFrame.getSlotSize(); nodeFrontiers.add(leafFrontier); pagesToWrite = new ArrayList<>(); compressedPageWriter = bufferCache.getCompressedPageWriter(fileId); }
private boolean positionToNextLeaf(boolean skipCurrent) throws HyracksDataException { while (frame.getLevel() != 0 || skipCurrent || frame.getTupleCount() == 0) { if (++currentPageId > maxPageId) { break; } releasePage(); ICachedPage nextPage = acquireNextPage(); page = nextPage; frame.setPage(page); tupleIndex = 0; skipCurrent = false; } if (currentPageId <= maxPageId) { return true; } else { return false; } }
@Override public void resetByTupleIndex(ITreeIndexFrame frame, int tupleIndex) { resetByTupleOffset(frame.getBuffer().array(), frame.getTupleOffset(tupleIndex)); }
public void add(ITreeIndexFrame frame) { numPages++; numTuples += frame.getTupleCount(); sumFillFactors += (double) (frame.getBuffer().capacity() - frame.getTotalFreeSpace()) / (double) frame.getBuffer().capacity(); }
@Override public int getSlotEndOff() { return frame.getBuffer().capacity() - (frame.getTupleCount() * slotSize); }
interiorFrame.setPage(frontier.page); if (tupleBytes > interiorFrame.getMaxTupleSize(BTree.this.bufferCache.getPageSize())) { throw HyracksDataException.create(ErrorCode.RECORD_IS_TOO_LARGE, tupleBytes, interiorFrame.getMaxTupleSize(BTree.this.bufferCache.getPageSize())); int spaceUsed = interiorFrame.getBuffer().capacity() - interiorFrame.getTotalFreeSpace(); if (spaceUsed + spaceNeeded > interiorMaxBytes) { ISplitKey copyKey = splitKey.duplicate(leafFrame.getTupleWriter().createTupleReference()); tuple = copyKey.getTuple(); frontier.lastTuple.resetByTupleIndex(interiorFrame, interiorFrame.getTupleCount() - 1); int splitKeySize = tupleWriter.bytesRequired(frontier.lastTuple, 0, cmp.getKeyFieldCount()); splitKey.initData(splitKeySize); interiorFrame.setPage(frontier.page); interiorFrame.initBuffer((byte) level);
@Override public void add(ITupleReference tuple) throws HyracksDataException { try { int tupleSize = Math.max(leafFrame.getBytesRequiredToWriteTuple(tuple), interiorFrame.getBytesRequiredToWriteTuple(tuple)); NodeFrontier leafFrontier = nodeFrontiers.get(0); int spaceNeeded = tupleWriter.bytesRequired(tuple) + slotSize; int spaceUsed = leafFrame.getBuffer().capacity() - leafFrame.getTotalFreeSpace(); leafFrame.compress(); spaceUsed = leafFrame.getBuffer().capacity() - leafFrame.getTotalFreeSpace(); if (leafFrame.getTupleCount() == 0) { bufferCache.returnPage(leafFrontier.page, false); } else { leafFrontier.lastTuple.resetByTupleIndex(leafFrame, leafFrame.getTupleCount() - 1); if (verifyInput) { verifyInputTuple(tuple, leafFrontier.lastTuple); int headerSize = Math.max(leafFrame.getPageHeaderSize(), interiorFrame.getPageHeaderSize()); final int multiplier = (int) Math.ceil((double) tupleSize / (bufferCache.getPageSize() - headerSize)); leafFrame.setPage(leafFrontier.page); leafFrame.initBuffer((byte) 0); ((IBTreeLeafFrame) leafFrame).setLargeFlag(true); } else { final long dpid = BufferedFileHandle.getDiskPageId(getFileId(), leafFrontier.pageId); leafFrontier.page = bufferCache.confiscatePage(dpid); leafFrame.setPage(leafFrontier.page);
if (lowerFrame.getTupleCount() == 0) { return; interiorFrame.setPage(frontier.page); pagesToWrite.add(frontier.page); lowerFrame = prevInteriorFrame; lowerFrame.setPage(frontier.page); interiorFrame.setPage(frontier.page); interiorFrame.initBuffer((byte) level); interiorFrame.insert(mbrTuple, AbstractSlotManager.GREATEST_KEY_INDICATOR); interiorFrame.getBuffer().putInt( interiorFrame.getTupleOffset(interiorFrame.getTupleCount() - 1) + mbrTuple.getTupleSize(), prevNodeFrontierPages.get(level - 1)); } else if (interiorFrame.hasSpaceInsert(mbrTuple) == FrameOpSpaceStatus.SUFFICIENT_CONTIGUOUS_SPACE && !toRoot) { interiorFrame.insert(mbrTuple, -1); interiorFrame.getBuffer().putInt( interiorFrame.getTupleOffset(interiorFrame.getTupleCount() - 1) + mbrTuple.getTupleSize(), prevNodeFrontierPages.get(level - 1)); lowerFrame.setPage(frontier.page); propagateBulk(level + 1, true, pagesToWrite);
@Override public void add(ITupleReference tuple) throws HyracksDataException { try { int leafFrameTupleSize = leafFrame.getBytesRequiredToWriteTuple(tuple); int interiorFrameTupleSize = interiorFrame.getBytesRequiredToWriteTuple(tuple); int tupleSize = Math.max(leafFrameTupleSize, interiorFrameTupleSize); if (tupleSize > maxTupleSize) { int spaceUsed = leafFrame.getBuffer().capacity() - leafFrame.getTotalFreeSpace(); leafFrame.compress(); spaceUsed = leafFrame.getBuffer().capacity() - leafFrame.getTotalFreeSpace(); leafFrontier.page = bufferCache .confiscatePage(BufferedFileHandle.getDiskPageId(getFileId(), leafFrontier.pageId)); leafFrame.setPage(leafFrontier.page); leafFrame.initBuffer((byte) 0); leafFrame.setPage(leafFrontier.page); leafFrame.insert(tuple, AbstractSlotManager.GREATEST_KEY_INDICATOR); } catch (HyracksDataException e) { handleException();
interiorFrame.setPage(n.page); mbrTuple.resetByTupleOffset(mbr.array(), 0); interiorFrame.insert(mbrTuple, -1); interiorFrame.getBuffer().putInt( interiorFrame.getTupleOffset(interiorFrame.getTupleCount() - 1) + mbrTuple.getTupleSize(), prevPageId); lowerFrame.setPage(n.page); ((RTreeNSMFrame) lowerFrame).adjustMBR(); interiorFrameTupleWriter.writeTupleFields(((RTreeNSMFrame) lowerFrame).getMBRTuples(), 0, mbr, 0);
RTreeTypeAwareTupleWriter rTreeTupleWriterLeftFrame = ((RTreeTypeAwareTupleWriter) tupleWriter); RTreeTypeAwareTupleWriter rTreeTupleWriterRightFrame = ((RTreeTypeAwareTupleWriter) rightFrame.getTupleWriter()); rightFrame.insert(frameTuple, -1); ((UnorderedSlotManager) slotManager).modifySlot(slotManager.getSlotOff(seed1), -1); totalBytes += leftRTreeFrame.getTupleSize(frameTuple); remainingTuplestoBeInsertedInRightFrame = leftRTreeFrame.getTupleCount() / 2 - rightFrame.getTupleCount(); if (remainingTuplestoBeInsertedInRightFrame == 0) { break; keyValueProviders) || leftRTreeFrame.getTupleCount() - k <= remainingTuplestoBeInsertedInRightFrame) { rightFrame.insert(frameTuple, -1); rec[0].enlarge(frameTuple, keyValueProviders); ((UnorderedSlotManager) slotManager).modifySlot(slotManager.getSlotOff(k), -1); rightFrame.compact(); leftRTreeFrame.compact(); if (rightFrame.hasSpaceInsert(tuple) == FrameOpSpaceStatus.SUFFICIENT_CONTIGUOUS_SPACE) { rightFrame.insert(tuple, -1); } else { leftRTreeFrame.insert(tuple, -1); leftRTreeFrame.insert(tuple, -1); } else { rightFrame.insert(tuple, -1);
protected void addLevel() throws HyracksDataException { NodeFrontier frontier = new NodeFrontier(tupleWriter.createTupleReference()); frontier.page = bufferCache.confiscatePage(IBufferCache.INVALID_DPID); frontier.pageId = -1; frontier.lastTuple.setFieldCount(cmp.getKeyFieldCount()); interiorFrame.setPage(frontier.page); interiorFrame.initBuffer((byte) nodeFrontiers.size()); nodeFrontiers.add(frontier); }
@Override public boolean doHasNext() throws HyracksDataException { if (numTuples <= currentTupleIndex) { return false; } // We don't latch pages since this code is only used by flush () before // bulk-loading the r-tree to disk and flush is not concurrent. // ICachedPage node1 = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, tPointers[currentTupleIndex * 2]), false); try { leafFrame1.setPage(node1); frameTuple1.resetByTupleOffset(leafFrame1.getBuffer().array(), tPointers[currentTupleIndex * 2 + 1]); } finally { bufferCache.unpin(node1); } return true; }
@Override public void setFrame(ITreeIndexFrame frame) { this.frame = (BTreeFieldPrefixNSMLeafFrame) frame; this.buf = frame.getBuffer(); }
public byte getTreeHeight(ITreeIndexFrame frame) throws HyracksDataException { ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false); rootNode.acquireReadLatch(); try { frame.setPage(rootNode); return frame.getLevel(); } finally { rootNode.releaseReadLatch(); bufferCache.unpin(rootNode); } }
@Override public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey, IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache) throws HyracksDataException { ByteBuffer right = rightFrame.getBuffer(); int tupleCount = getTupleCount(); int src = rightFrame.getSlotManager().getSlotEndOff(); int dest = rightFrame.getSlotManager().getSlotEndOff() + tuplesToLeft * rightFrame.getSlotManager().getSlotSize(); int length = rightFrame.getSlotManager().getSlotSize() * tuplesToRight; System.arraycopy(right.array(), src, right.array(), dest, length); right.putInt(Constants.TUPLE_COUNT_OFFSET, tuplesToRight); rightFrame.compact(); compact(); targetFrame.insert(savedSplitKey.getTuple(), targetTupleIndex);
int deltaPages = page.getFrameSizeMultiplier() - rightFrame.getPage().getFrameSizeMultiplier(); if (deltaPages > 0) { ((BTreeNSMLeafFrame) rightFrame).growCapacity(extraPageBlockHelper, bufferCache, deltaPages); ByteBuffer right = rightFrame.getBuffer(); int src = rightFrame.getSlotManager().getSlotEndOff(); int dest = rightFrame.getSlotManager().getSlotEndOff() + tuplesToLeft * rightFrame.getSlotManager().getSlotSize(); int length = rightFrame.getSlotManager().getSlotSize() * tuplesToRight; System.arraycopy(right.array(), src, right.array(), dest, length); right.putInt(Constants.TUPLE_COUNT_OFFSET, tuplesToRight); rightFrame.compact(); compact();
@Override public boolean doHasNext() throws HyracksDataException { if (currentPageId > maxPageId) { return false; } if (tupleIndex >= frame.getTupleCount()) { boolean nextLeafExists = positionToNextLeaf(true); if (nextLeafExists) { frameTuple.resetByTupleIndex(frame, tupleIndex); return true; } else { return false; } } frameTuple.resetByTupleIndex(frame, tupleIndex); return true; }
public static String printFrameTuples(ITreeIndexFrame frame, ISerializerDeserializer[] fieldSerdes) throws HyracksDataException { StringBuilder strBuilder = new StringBuilder(); ITreeIndexTupleReference tuple = frame.createTupleReference(); for (int i = 0; i < frame.getTupleCount(); i++) { tuple.resetByTupleIndex(frame, i); String tupleString = TupleUtils.printTuple(tuple, fieldSerdes); strBuilder.append(tupleString); if (i != frame.getTupleCount() - 1) { strBuilder.append(" | "); } } return strBuilder.toString(); } }
try { metaFrame.setPage(page); leafFrame.setPage(page); interiorFrame.setPage(page); if (leafFrame.isLeaf()) { if (pageId == rootPage) { treeIndexStats.addRoot(leafFrame); treeIndexStats.add(leafFrame); } else if (interiorFrame.isInterior()) { if (pageId == rootPage) { treeIndexStats.addRoot(interiorFrame);