@Override public void setPage(ICachedPage page) { this.page = page; this.buf = page.getBuffer(); }
@Override public void setPage(ICachedPage page) { this.page = page; this.buf = page.getBuffer(); }
@Override public ByteBuffer getBuffer() { return page.getBuffer(); }
@Override public ByteBuffer getBuffer() { return page.getBuffer(); }
public void setEOF() { cPage.getBuffer().putLong(maxEntryOffset + ENTRY_LENGTH, EOF); }
private void allocateAndInitMetaDataPage() throws HyracksDataException { if (metaDataPage == null) { metaDataPage = bufferCache.confiscatePage(BufferedFileHandle.getDiskPageId(fileId, METADATA_PAGE_ID)); } metaDataPage.getBuffer().putInt(NUM_PAGES_OFFSET, numPages); metaDataPage.getBuffer().putInt(NUM_HASHES_USED_OFFSET, numHashes); metaDataPage.getBuffer().putLong(NUM_ELEMENTS_OFFSET, actualNumElements); metaDataPage.getBuffer().putLong(NUM_BITS_OFFSET, numBits); metaDataPage.getBuffer().putInt(VERSION_OFFSET, BLOCKED_BLOOM_FILTER_VERSION); }
public void writePageInfo(int compressedPageId, long offset, long size) { final int entryOffset = compressedPageId * ENTRY_LENGTH % bufferCache.getPageSize(); //Put page offset cPage.getBuffer().putLong(entryOffset, offset); //Put page size cPage.getBuffer().putLong(entryOffset + SIZE_ENTRY_OFFSET, size); //Keep the max entry offset to set EOF (if needed) maxEntryOffset = Math.max(maxEntryOffset, entryOffset); numOfEntries++; }
@Override public void setPage(ICachedPage page) { this.page = page; this.buf = page.getBuffer(); slotManager.setFrame(this); }
private void readBloomFilterMetaData() throws HyracksDataException { if (bufferCache.getNumPagesOfFile(fileId) == 0) { numPages = 0; numHashes = 0; numElements = 0; numBits = 0; version = DEFAULT_BLOOM_FILTER_VERSION; return; } ICachedPage metaPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, METADATA_PAGE_ID), false); metaPage.acquireReadLatch(); try { numPages = metaPage.getBuffer().getInt(NUM_PAGES_OFFSET); numHashes = metaPage.getBuffer().getInt(NUM_HASHES_USED_OFFSET); numElements = metaPage.getBuffer().getLong(NUM_ELEMENTS_OFFSET); numBits = metaPage.getBuffer().getLong(NUM_BITS_OFFSET); version = metaPage.getBuffer().getInt(VERSION_OFFSET); } finally { metaPage.releaseReadLatch(); bufferCache.unpin(metaPage); } }
@Override public void add(ITupleReference tuple) throws HyracksDataException { if (numPages == 0) { throw HyracksDataException.create(ErrorCode.CANNOT_ADD_TUPLES_TO_DUMMY_BLOOM_FILTER); } actualNumElements++; MurmurHash128Bit.hash3_x64_128(tuple, keyFields, SEED, hashes); long hash = Math.abs(hashes[0] % numBits); long groupId = hash / NUM_BITS_PER_BLOCK; int pageId = (int) (groupId / numBlocksPerPage); long groupStartIndex = (groupId % numBlocksPerPage) * NUM_BITS_PER_BLOCK; ICachedPage page = pages[pageId]; ByteBuffer buffer = page.getBuffer(); for (int i = 1; i < numHashes; ++i) { hash = Math.abs((hashes[0] + i * hashes[1]) % NUM_BITS_PER_BLOCK); int byteIndex = (int) ((hash + groupStartIndex) >> 3); // divide 8 byte b = buffer.get(byteIndex); int bitIndex = (int) (hash & 0x07); // mod 8 b = (byte) (b | (1 << bitIndex)); buffer.put(byteIndex, b); } }
@Override public void doNext() throws HyracksDataException { if (currentOffsetForScan + 2 * elementSize > bufferCache.getPageSize()) { // Read the next page. currentOffsetForScan = 0; loadPages(); } else { currentOffsetForScan += elementSize; } currentElementIxForScan++; tuple.reset(page.getBuffer().array(), currentOffsetForScan); }
protected void appendInvertedList(ITupleReference keyTuple, int startField) throws HyracksDataException { if (!invListBuilder.appendElement(keyTuple, startField, numInvListKeys)) { pinNextPage(); invListBuilder.setTargetBuffer(currentPage.getBuffer().array(), 0); if (!invListBuilder.appendElement(keyTuple, startField, numInvListKeys)) { throw new IllegalStateException( "Failed to append element to inverted list after switching to a new page."); } } }
@Override public void resizePage(ICachedPage cPage, int multiplier, IExtraPageBlockHelper extraPageBlockHelper) { ByteBuffer oldBuffer = cPage.getBuffer(); int origMultiplier = cPage.getFrameSizeMultiplier(); if (origMultiplier == multiplier) {
protected void startNewList(ITupleReference tokenTuple) throws HyracksDataException { if (!invListBuilder.startNewList(tokenTuple, numTokenFields)) { pinNextPage(); invListBuilder.setTargetBuffer(currentPage.getBuffer().array(), 0); if (!invListBuilder.startNewList(tokenTuple, numTokenFields)) { throw new IllegalStateException("Failed to create first inverted list."); } } currentInvListStartPageId = currentPageId; currentInvListStartOffset = invListBuilder.getPos(); }
unpinWhenExit = true; ByteBuffer buffer = page.getBuffer();
private void growCapacity(IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache, int deltaPages) throws HyracksDataException { int framePagesOld = page.getFrameSizeMultiplier(); int newMultiplier = framePagesOld + deltaPages; // we need to get the old slot offsets before we grow int oldSlotEnd = slotManager.getSlotEndOff(); int oldSlotStart = slotManager.getSlotStartOff() + slotManager.getSlotSize(); bufferCache.resizePage(getPage(), newMultiplier, extraPageBlockHelper); buf = getPage().getBuffer(); // fixup the slots System.arraycopy(buf.array(), oldSlotEnd, buf.array(), slotManager.getSlotEndOff(), oldSlotStart - oldSlotEnd); // fixup total free space counter buf.putInt(TOTAL_FREE_SPACE_OFFSET, buf.getInt(TOTAL_FREE_SPACE_OFFSET) + (bufferCache.getPageSize() * deltaPages)); }
private boolean legacyContains(long[] hashes) throws HyracksDataException { for (int i = 0; i < numHashes; ++i) { long hash = Math.abs((hashes[0] + i * hashes[1]) % numBits); // we increment the page id by one, since the metadata page id of the filter is 0. ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, (int) (hash / numBitsPerPage) + 1), false); page.acquireReadLatch(); try { ByteBuffer buffer = page.getBuffer(); int byteIndex = (int) (hash % numBitsPerPage) >> 3; // divide by 8 byte b = buffer.get(byteIndex); int bitIndex = (int) (hash % numBitsPerPage) & 0x07; // mod 8 if (!((b & (1L << bitIndex)) != 0)) { return false; } } finally { page.releaseReadLatch(); bufferCache.unpin(page); } } return true; }
private void init() throws HyracksDataException { final int numOfPages = bufferCache.getNumPagesOfFile(fileId); //Maximum number of entries in a page final int numOfEntriesPerPage = bufferCache.getPageSize() / ENTRY_LENGTH; //get the last page which may contain less entries than maxNumOfEntries final long dpid = getDiskPageId(numOfPages - 1); final ICachedPage page = bufferCache.pin(dpid, false); try { final ByteBuffer buf = page.getBuffer(); //Start at 1 since it is impossible to have EOF at the first entry of a page int i = 1; //Seek EOF and count number of entries while (i < numOfEntriesPerPage && buf.getLong(i * ENTRY_LENGTH) != EOF) { i++; } totalNumOfPages = (numOfPages - 1) * numOfEntriesPerPage + i; } finally { bufferCache.unpin(page); } }
private void setCompressedPageInfo(int compressedPageId, ICachedPageInternal compressedPage) throws HyracksDataException { ensureState(READABLE); if (totalNumOfPages == 0) { /* * It seems it is legal to pin empty file. * Return the page information as it is not compressed. */ compressedPage.setCompressedPageOffset(0); compressedPage.setCompressedPageSize(bufferCache.getPageSize()); return; } final ICachedPage page = pinAndGetPage(compressedPageId); try { // No need for read latches as pages are immutable. final ByteBuffer buf = page.getBuffer(); final int entryOffset = compressedPageId * ENTRY_LENGTH % bufferCache.getPageSize(); compressedPage.setCompressedPageOffset(buf.getLong(entryOffset)); compressedPage.setCompressedPageSize((int) buf.getLong(entryOffset + SIZE_ENTRY_OFFSET)); } finally { bufferCache.unpin(page); } }
public AbstractOnDiskInvertedIndexBulkLoader(float btreeFillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, int startPageId) throws HyracksDataException { this.verifyInput = verifyInput; this.invListCmp = MultiComparator.create(invListCmpFactories); if (verifyInput) { allCmp = MultiComparator.create(btree.getComparatorFactories(), invListCmpFactories); } else { allCmp = null; } this.btreeTupleBuilder = new ArrayTupleBuilder(btree.getFieldCount()); this.btreeTupleReference = new ArrayTupleReference(); this.lastTupleBuilder = new ArrayTupleBuilder(numTokenFields + numInvListKeys); this.lastTuple = new ArrayTupleReference(); this.btreeBulkloader = btree.createBulkLoader(btreeFillFactor, verifyInput, numElementsHint, checkIfEmptyIndex); currentPageId = startPageId; currentPage = bufferCache.confiscatePage(BufferedFileHandle.getDiskPageId(fileId, currentPageId)); invListBuilder.setTargetBuffer(currentPage.getBuffer().array(), 0); queue = bufferCache.createFIFOQueue(); }