@Override public ITupleReference getFilterMinTuple() { ILSMComponentFilter filter = operationalComponents.get( currentCursor < numMemoryComponents ? currentCursor : outputElement.getCursorIndex() + currentCursor) .getLSMComponentFilter(); return filter == null ? null : filter.getMinTuple(); }
private void initPriorityQueues() throws HyracksDataException { int pqInitSize = (rangeCursors.length > 0) ? rangeCursors.length : 1; tokenQueue = new PriorityQueue<>(pqInitSize, tokenQueueCmp); keyQueue = new PriorityQueue<>(pqInitSize, keyQueueCmp); tokenQueueElements = new PriorityQueueElement[pqInitSize]; keyQueueElements = new PriorityQueueElement[pqInitSize]; for (int i = 0; i < pqInitSize; i++) { tokenQueueElements[i] = new PriorityQueueElement(i); keyQueueElements[i] = new PriorityQueueElement(i); } for (int i = 0; i < rangeCursors.length; i++) { if (rangeCursors[i].hasNext()) { rangeCursors[i].next(); tokenQueueElements[i].reset(rangeCursors[i].getTuple()); tokenQueue.offer(tokenQueueElements[i]); } else { rangeCursors[i].close(); } } searchNextToken(); }
if (canCallProceed) { if (includeMutableComponent) { resultOfSearchCallbackProceed = searchCallback.proceed(queueHead.getTuple()); if (!resultOfSearchCallbackProceed) { tupleBuilder = new ArrayTupleBuilder(cmp.getKeyFieldCount()); TupleUtils.copyTuple(tupleBuilder, queueHead.getTuple(), cmp.getKeyFieldCount()); copyTuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray()); if (queueHead == null || cmp.compare(copyTuple, queueHead.getTuple()) != 0) { if (compare(cmp, outputElement.getTuple(), queueHead.getTuple()) == 0) {
if (outputElement != null && outputElement.getCursorIndex() == i) { pushIntoQueueFromCursorAndReplaceThisElement(outputElement); needPushElementIntoQueue = false; switchComponentTupleBuilders[i] = new ArrayTupleBuilder(cmp.getKeyFieldCount()); TupleUtils.copyTuple(switchComponentTupleBuilders[i], element.getTuple(), cmp.getKeyFieldCount());
private void redoMemoryComponentSearchIfNeeded() throws HyracksDataException { if (!includeMutableComponent) { return; } // if the last n records, none were from memory and there are writers inside the component, // we need to re-do the search so the cursor doesn't block modifications due to latches over page if (tupleFromMemoryComponentCount == 0 && ((AbstractLSMMemoryComponent) operationalComponents.get(0)).getWriterCount() > 0) { // When we reach here, we know that the mutable component element is not the outputElement // since if it was the output element, the tupleFromMemoryComponentCount would be at least 1 PriorityQueueElement mutableElement = remove(outputPriorityQueue, 0); if (mutableElement != null) { // if the element is null, then there is nothing to do since no latches are held if (tupleBuilder == null) { tupleBuilder = new ArrayTupleBuilder(cmp.getKeyFieldCount()); } TupleUtils.copyTuple(tupleBuilder, mutableElement.getTuple(), cmp.getKeyFieldCount()); copyTuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray()); // Unlatches/unpins the leaf page of the index. rangeCursors[0].close(); // Re-traverses the index. reusablePred.setLowKey(copyTuple, true); btreeAccessors[0].search(rangeCursors[0], reusablePred); pushIntoQueueFromCursorAndReplaceThisElement(mutableElement); } } tupleFromMemoryComponentCount = 0; }
public void initPriorityQueue() throws HyracksDataException { int pqInitSize = (rangeCursors.length > 0) ? rangeCursors.length : 1; if (outputPriorityQueue == null) { outputPriorityQueue = new PriorityQueue<>(pqInitSize, pqCmp); pqes = new PriorityQueueElement[pqInitSize]; for (int i = 0; i < pqInitSize; i++) { pqes[i] = new PriorityQueueElement(i); } for (int i = 0; i < rangeCursors.length; i++) { pushIntoQueueFromCursorAndReplaceThisElement(pqes[i]); } } else { outputPriorityQueue.clear(); // did size change? if (pqInitSize == pqes.length) { // size is the same -> re-use for (int i = 0; i < rangeCursors.length; i++) { pqes[i].reset(null); pushIntoQueueFromCursorAndReplaceThisElement(pqes[i]); } } else { // size changed (due to flushes, merges, etc) -> re-create pqes = new PriorityQueueElement[pqInitSize]; for (int i = 0; i < rangeCursors.length; i++) { pqes[i] = new PriorityQueueElement(i); pushIntoQueueFromCursorAndReplaceThisElement(pqes[i]); } } } }
/** * Check deleted-keys BTrees whether they contain the key in the checkElement's tuple. */ protected boolean isDeleted(PriorityQueueElement keyElement) throws HyracksDataException { ITupleReference keyTuple = keyElement.getTuple(); int end = keyElement.getCursorIndex(); for (int i = 0; i < end; i++) { if (bloomFilters[i] != null && !bloomFilters[i].contains(keyTuple, hashes)) { continue; } deletedKeysBTreeCursors[i].close(); deletedKeysBTreeAccessors.get(i).search(deletedKeysBTreeCursors[i], deletedKeyBTreeSearchPred); try { if (deletedKeysBTreeCursors[i].hasNext()) { return true; } } finally { deletedKeysBTreeCursors[i].close(); } } return false; }
@Override public boolean doHasNext() throws HyracksDataException { if (foundNext) { return true; } while (super.doHasNext()) { super.doNext(); LSMBTreeTupleReference diskTuple = (LSMBTreeTupleReference) super.doGetTuple(); if (diskTuple.isAntimatter()) { if (setAntiMatterTuple(diskTuple, outputElement.getCursorIndex())) { foundNext = true; return true; } } else { //matter tuple setMatterTuple(diskTuple, outputElement.getCursorIndex()); foundNext = true; return true; } } return false; }
/** * Check deleted-keys BTrees whether they contain the key in the checkElement's tuple. */ @Override protected boolean isDeleted(PriorityQueueElement checkElement) throws HyracksDataException { keysOnlyTuple.reset(checkElement.getTuple()); int end = checkElement.getCursorIndex(); for (int i = 0; i < end; i++) { if (bloomFilters[i] != null && !bloomFilters[i].contains(keysOnlyTuple, hashes)) { continue; } deletedKeysBTreeAccessors.get(i).search(deletedKeysBTreeCursors[i], keySearchPred); try { if (deletedKeysBTreeCursors[i].hasNext()) { return true; } } finally { deletedKeysBTreeCursors[i].close(); } } return false; }
@Override public int compare(PriorityQueueElement elementA, PriorityQueueElement elementB) { int result; try { result = cmp.selectiveFieldCompare(elementA.getTuple(), elementB.getTuple(), comparatorFields); if (result != 0) { return result; } } catch (HyracksDataException e) { throw new IllegalArgumentException(e); } if (elementA.getCursorIndex() > elementB.getCursorIndex()) { return 1; } else { return -1; } } }
private void pushIntoKeyQueueAndReplace(PriorityQueueElement keyElement) throws HyracksDataException { int cursorIndex = keyElement.getCursorIndex(); if (rangeCursors[cursorIndex].hasNext()) { rangeCursors[cursorIndex].next(); TokenKeyPairTuple tuple = (TokenKeyPairTuple) rangeCursors[cursorIndex].getTuple(); if (tuple.isNewToken()) { // if this element is a new token, then the current inverted list has exuasted PriorityQueueElement tokenElement = tokenQueueElements[cursorIndex]; tokenElement.reset(tuple); tokenQueue.offer(tokenElement); } else { keyElement.reset(tuple.getKeyTuple()); keyQueue.offer(keyElement); } } else { rangeCursors[cursorIndex].close(); } }
@Override public int compare(PriorityQueueElement elementA, PriorityQueueElement elementB) { int result; try { result = cmp.compare(elementA.getTuple(), elementB.getTuple()); if (result != 0) { return result; } } catch (HyracksDataException e) { throw new IllegalArgumentException(e); } if (elementA.getCursorIndex() > elementB.getCursorIndex()) { return 1; } else { return -1; } }
@Override public int compare(PriorityQueueElement elementA, PriorityQueueElement elementB) { int result; try { result = cmp.compare(elementA.getTuple(), elementB.getTuple()); if (result != 0) { return result; } } catch (HyracksDataException e) { throw new IllegalArgumentException(e); } // the components in the component list are in descending order of creation time // we want older components to be returned first return elementA.getCursorIndex() > elementB.getCursorIndex() ? -1 : 1; } }
private ITupleReference getTokenTuple(PriorityQueueElement tokenElement) { return ((TokenKeyPairTuple) tokenElement.getTuple()).getTokenTuple(); }
protected void pushIntoQueueFromCursorAndReplaceThisElement(PriorityQueueElement e) throws HyracksDataException { int cursorIndex = e.getCursorIndex(); if (rangeCursors[cursorIndex].hasNext()) { rangeCursors[cursorIndex].next(); e.reset(rangeCursors[cursorIndex].getTuple()); outputPriorityQueue.offer(e); return; } rangeCursors[cursorIndex].close(); if (cursorIndex == 0) { includeMutableComponent = false; } }
protected boolean isDeleted(PriorityQueueElement checkElement) throws HyracksDataException { return ((ILSMTreeTupleReference) checkElement.getTuple()).isAntimatter(); }
@Override public void doNext() throws HyracksDataException { outputKeyElement = keyQueue.poll(); outputTuple.setKeyTuple(outputKeyElement.getTuple()); needPushElementIntoKeyQueue = true; }
@Override public ITupleReference doGetTuple() { return outputElement.getTuple(); }