public DiskRangeList getIncompleteCbs( DiskRangeList ranges, long baseOffset, BooleanRef gotAllData) { DiskRangeList prev = ranges.prev; if (prev == null) { prev = new MutateHelper(ranges); } DiskRangeList current = ranges; gotAllData.value = true; // Assume by default that we would find everything. while (current != null) { // We assume ranges in "ranges" are non-overlapping; thus, we will save next in advance. DiskRangeList check = current; current = current.next; if (check.hasData()) continue; Integer badLength = cache.get(Long.valueOf(check.getOffset() + baseOffset)); if (badLength == null || badLength < check.getLength()) { gotAllData.value = false; continue; } // We could just remove here and handle the missing tail during read, but that can be // dangerous; let's explicitly add an incomplete CB. check.replaceSelfWith(new IncompleteCb(check.getOffset(), check.getEnd())); } return prev.next; }
if (currentNotCached.getOffset() >= currentCached.getOffset()) { if (currentNotCached.getEnd() <= currentCached.getEnd()) { currentNotCached.replaceSelfWith(currentCached); return null; } else { currentNotCached.insertPartBefore(currentCached); return currentNotCached; gotAllData.value = false; if (currentNotCached.getEnd() <= currentCached.getEnd()) { currentNotCached.insertPartAfter(currentCached); return null; // No more matches expected. } else { DiskRangeList tail = new DiskRangeList(currentCached.getEnd(), currentNotCached.getEnd()); currentNotCached.insertPartAfter(currentCached); currentCached.insertAfter(tail); return tail;
/** Splits current element in the list, using DiskRange::slice. * @param cOffset the position to split the list * @return the split list */ public final DiskRangeList split(long cOffset) { DiskRangeList right = insertAfterInternal((DiskRangeList)this.sliceAndShift(cOffset, end, 0)); DiskRangeList left = replaceSelfWith((DiskRangeList)this.sliceAndShift(offset, cOffset, 0)); checkOrder(left, right, left); // Prev/next are already checked in the calls. return left; }
/** * Inserts an intersecting range after current in the list and adjusts offset accordingly. * @param other the new element to insert * @return the new element. */ public DiskRangeList insertPartAfter(DiskRangeList other) { // The only allowed non-overlapping option is extra bytes at the end. if (other.offset > this.end || other.offset <= this.offset || other.end <= this.offset) { assertInvalidOrder(this.prev == null ? this : this.prev, this, other); } this.end = other.offset; return insertAfter(other); }
private final static void checkOrder(DiskRangeList prev, DiskRangeList next, DiskRangeList ref) { if (prev.getEnd() <= next.getOffset()) return; assertInvalidOrder(ref.prev == null ? ref : ref.prev, prev, next); }
if (range.hasData()) { range = range.next; continue; int len = (int) (range.getEnd() - range.getOffset()); long off = range.getOffset(); if (zcr != null) { file.seek(base + off); BufferChunk bc = new BufferChunk(partial, off); if (!hasReplaced) { range.replaceSelfWith(bc); hasReplaced = true; } else { range.insertAfter(bc); bb = ByteBuffer.wrap(buffer); range = range.replaceSelfWith(new BufferChunk(bb, range.getOffset()));
while (range != null) { if (!inRange) { if (range.getEnd() <= offset) { range = range.next; continue; // Skip until we are in range. if (range.getOffset() < offset) { buffers.add(range.sliceAndShift(offset, Math.min(streamEnd, range.getEnd()), -offset)); if (range.getEnd() >= streamEnd) break; // Partial first buffer is also partial last buffer. range = range.next; continue; } else if (range.getOffset() >= streamEnd) { break; if (range.getEnd() > streamEnd) { buffers.add(range.sliceAndShift(range.getOffset(), streamEnd, -offset)); break; buffers.add(range.sliceAndShift(range.getOffset(), range.getEnd(), -offset)); if (range.getEnd() == streamEnd) break; range = range.next;
@Override public int read(byte[] array, final int arrayOffset, final int len) throws IOException { long readStartPos = position; DiskRangeList drl = new DiskRangeList(readStartPos, readStartPos + len); DataCache.BooleanRef gotAllData = new DataCache.BooleanRef(); drl = cache.getFileData(fileKey, drl, 0, new DataCache.DiskRangeListFactory() { long sizeRead = 0; while (drl != null) { assert drl.hasData(); long from = drl.getOffset(), to = drl.getEnd(); int offsetFromReadStart = (int)(from - readStartPos), candidateSize = (int)(to - from); ByteBuffer data = drl.getData().duplicate(); data.get(array, arrayOffset + offsetFromReadStart, candidateSize); cache.releaseBuffer(((CacheChunk)drl).getBuffer()); DiskRangeList candidate = current; current = current.next; long from = candidate.getOffset(), to = candidate.getEnd(); if (candidate.hasData()) { ByteBuffer data = candidate.getData().duplicate(); data.get(array, arrayOffset + offsetFromReadStart, candidateSize); cache.releaseBuffer(((CacheChunk)candidate).getBuffer());
if (streamOffset > current.getOffset()) { current = current.split(streamOffset).next; assert partOffset <= current.getOffset(); if (partOffset == current.getOffset() && current instanceof CacheChunk) { assert current.getOffset() == partOffset && current.getEnd() == partEnd; lastUncompressed = (CacheChunk)current; current = current.next; continue; if (current.getOffset() >= partEnd) { continue; // We have no data at all for this part of the stream (could be unneeded), skip. boolean noMoreDataForPart = (next == null || next.getOffset() >= partEnd); if (noMoreDataForPart && hasEntirePartTo < partEnd && candidateCached != null) { if (current.getEnd() > partEnd) { current = current.split(partEnd); + current.getOffset() + ", " + current.getEnd() + ")"); trace.logUncompressedData(current.getOffset(), current.getEnd()); BufferChunk curBc = (BufferChunk)current; hasEntirePartTo = (hasEntirePartTo == current.getOffset()) ? current.getEnd() : -1; if (hasEntirePartTo == -1) {
private static IncompleteCb addIncompleteCompressionBuffer(long cbStartOffset, DiskRangeList target, int extraChunkCountToLog, boolean isTracingEnabled, IoTrace trace) { IncompleteCb icb = new IncompleteCb(cbStartOffset, target.getEnd()); if (isTracingEnabled) { LOG.trace("Replacing " + target + " (and " + extraChunkCountToLog + " previous chunks) with " + icb + " in the buffers"); } trace.logInvalidOrcCb(cbStartOffset, target.getEnd()); target.replaceSelfWith(icb); return icb; }
public void addOrMerge(long offset, long end, boolean doMerge, boolean doLogNew) { if (doMerge && tail != null && tail.merge(offset, end)) return; if (doLogNew) { LOG.debug("Creating new range; last range (which can include some previous adds) was " + tail); } DiskRangeList node = new DiskRangeList(offset, end); if (tail == null) { head = tail = node; } else { tail = tail.insertAfter(node); } }
for(OrcProto.Stream stream: footer.getStreamsList()) { while (range != null && range.getEnd() <= offset) { range = range.next; if (stream.hasKind() && range.getOffset() <= offset) { switch (stream.getKind()) { case ROW_INDEX: if (included == null || included[column]) { ByteBuffer bb = range.getData().duplicate(); bb.position((int) (offset - range.getOffset())); bb.limit((int) (bb.position() + stream.getLength())); indexes[column] = OrcProto.RowIndex.parseFrom( case BLOOM_FILTER_UTF8: if (sargColumns != null && sargColumns[column]) { ByteBuffer bb = range.getData().duplicate(); bb.position((int) (offset - range.getOffset())); bb.limit((int) (bb.position() + stream.getLength())); bloomFilterIndices[column] = OrcProto.BloomFilterIndex.parseFrom
private DiskRangeList dr(int from, int to) { return new DiskRangeList(from, to); }
new DiskRangeList(offset, offset + si.getFooterLength()), 0, false); if (hasCache) { LlapBufferOrBuffers cacheBuf = metadataCache.putStripeTail( stripeKey, footerRange.getData().duplicate(), cacheTag, isStopped); ByteBuffer bb = footerRange.getData().duplicate();
cache.putIncompleteCbs(fileKey1, new DiskRange[] { new DiskRangeList(0, 3) }, 0, null); cp.verifyEquals(1); DiskRangeList result = cache.getIncompleteCbs( fileKey1, new DiskRangeList(0, 3), 0, gotAllData); assertTrue(gotAllData.value); verifyResult(result, INCOMPLETE, 0, 3); cache.putIncompleteCbs(fileKey1, new DiskRange[] { new DiskRangeList(5, 6) }, 0, null); cp.verifyEquals(3); DiskRangeList ranges = new DiskRangeList(0, 3); ranges.insertAfter(new DiskRangeList(4, 6)); result = cache.getIncompleteCbs(fileKey1, ranges, 0, gotAllData); assertFalse(gotAllData.value); verifyResult(result, INCOMPLETE, 0, 3, DRL, 4, 6); ranges = new DiskRangeList(0, 3); ranges.insertAfter(new DiskRangeList(3, 5)).insertAfter(new DiskRangeList(5, 6)); result = cache.getIncompleteCbs(fileKey1, ranges, 0, gotAllData); assertFalse(gotAllData.value); verifyResult(result, INCOMPLETE, 0, 3, DRL, 3, 5, INCOMPLETE, 5, 6); result = cache.getIncompleteCbs(fileKey1, new DiskRangeList(5, 6), 0, gotAllData); assertTrue(gotAllData.value); verifyResult(result, INCOMPLETE, 5, 6); result = cache.getIncompleteCbs(fileKey1, new DiskRangeList(4, 5), 0, gotAllData); assertFalse(gotAllData.value); verifyResult(result, DRL, 4, 5);
List<ByteBuffer> toReleaseCopies, List<ProcCacheChunk> toDecompress, List<IncompleteCb> badEstimates) throws IOException { if (cOffset > current.getOffset()) { current = current.split(cOffset).next; next = current.next; if (next != null && (endCOffset >= 0 && currentOffset < endCOffset) && next.getOffset() >= endCOffset) { throw new IOException("Expected data at " + currentOffset + " (reading until " + endCOffset + "), but the next buffer starts at " + next.getOffset()); lastUncompressed = (newCached == null) ? lastUncompressed : newCached; next = (newCached != null) ? newCached.next : null; currentOffset = (next != null) ? next.getOffset() : -1;
FileCache<ConcurrentSkipListMap<Long, LlapDataBuffer>> subCache = cache.get(fileKey); if (subCache == null || !subCache.incRef()) { long totalMissed = ranges.getTotalLength(); metrics.incrCacheRequestedBytes(totalMissed); if (qfCounters != null) { metrics.incrCacheRequestedBytes(current.getLength()); while (current != null) { if (current.hasData()) { bytesHit += current.getLength(); } else { if (gotAllData.value) { isInvalid = true; bytesMissed += current.getLength(); DiskRangeList current = prev.next; while (current != null) { if (!current.hasData()) { isInvalid = true; break;
while (true) { if ((prev == null) || (prev.getEnd() <= streamStartOffset)) break;