private static Map<StreamId, DiskRange> getDiskRanges(List<Stream> streams) { ImmutableMap.Builder<StreamId, DiskRange> streamDiskRanges = ImmutableMap.builder(); long stripeOffset = 0; for (Stream stream : streams) { int streamLength = toIntExact(stream.getLength()); // ignore zero byte streams if (streamLength > 0) { streamDiskRanges.put(new StreamId(stream), new DiskRange(stripeOffset, streamLength)); } stripeOffset += streamLength; } return streamDiskRanges.build(); }
/** * Returns the minimal DiskRange that encloses both this DiskRange * and otherDiskRange. If there was a gap between the ranges the * new range will cover that gap. */ public DiskRange span(DiskRange otherDiskRange) { requireNonNull(otherDiskRange, "otherDiskRange is null"); long start = Math.min(this.offset, otherDiskRange.getOffset()); long end = Math.max(getEnd(), otherDiskRange.getEnd()); return new DiskRange(start, toIntExact(end - start)); }
public Map<StreamId, OrcInputStream> readDiskRanges(long stripeOffset, Map<StreamId, DiskRange> diskRanges, AggregatedMemoryContext systemMemoryUsage) throws IOException { // // Note: this code does not use the Java 8 stream APIs to avoid any extra object allocation // // transform ranges to have an absolute offset in file ImmutableMap.Builder<StreamId, DiskRange> diskRangesBuilder = ImmutableMap.builder(); for (Entry<StreamId, DiskRange> entry : diskRanges.entrySet()) { DiskRange diskRange = entry.getValue(); diskRangesBuilder.put(entry.getKey(), new DiskRange(stripeOffset + diskRange.getOffset(), diskRange.getLength())); } diskRanges = diskRangesBuilder.build(); // read ranges Map<StreamId, OrcDataSourceInput> streamsData = orcDataSource.readFully(diskRanges); // transform streams to OrcInputStream ImmutableMap.Builder<StreamId, OrcInputStream> streamsBuilder = ImmutableMap.builder(); for (Entry<StreamId, OrcDataSourceInput> entry : streamsData.entrySet()) { OrcDataSourceInput sourceInput = entry.getValue(); streamsBuilder.put(entry.getKey(), new OrcInputStream(orcDataSource.getId(), sourceInput.getInput(), decompressor, systemMemoryUsage, sourceInput.getRetainedSizeInBytes())); } return streamsBuilder.build(); }
@VisibleForTesting void readCacheAt(long offset) throws IOException { DiskRange newCacheRange = regionFinder.getRangeFor(offset); cachePosition = newCacheRange.getOffset(); cacheLength = newCacheRange.getLength(); if (cache.length < cacheLength) { cache = new byte[cacheLength]; } dataSource.readFully(newCacheRange.getOffset(), cache, 0, cacheLength); }
@Override public long getSize() { return diskRange.getLength(); }
public Map<StreamId, OrcInputStream> readDiskRanges(long stripeOffset, Map<StreamId, DiskRange> diskRanges, AbstractAggregatedMemoryContext systemMemoryUsage) throws IOException { // // Note: this code does not use the Java 8 stream APIs to avoid any extra object allocation // // transform ranges to have an absolute offset in file ImmutableMap.Builder<StreamId, DiskRange> diskRangesBuilder = ImmutableMap.builder(); for (Entry<StreamId, DiskRange> entry : diskRanges.entrySet()) { DiskRange diskRange = entry.getValue(); diskRangesBuilder.put(entry.getKey(), new DiskRange(stripeOffset + diskRange.getOffset(), diskRange.getLength())); } diskRanges = diskRangesBuilder.build(); // read ranges Map<StreamId, FixedLengthSliceInput> streamsData = orcDataSource.readFully(diskRanges); // transform streams to OrcInputStream String sourceName = orcDataSource.toString(); ImmutableMap.Builder<StreamId, OrcInputStream> streamsBuilder = ImmutableMap.builder(); for (Entry<StreamId, FixedLengthSliceInput> entry : streamsData.entrySet()) { streamsBuilder.put(entry.getKey(), new OrcInputStream(sourceName, entry.getValue(), compressionKind, bufferSize, systemMemoryUsage)); } return streamsBuilder.build(); }
@Override public DiskRange getRangeFor(long desiredOffset) { // Assumption: range are always read in order // Assumption: bytes that are not part of any range are never read for (; index < diskRanges.size(); index++) { DiskRange range = diskRanges.get(index); if (range.getEnd() > desiredOffset) { checkArgument(range.getOffset() <= desiredOffset); return range; } } throw new IllegalArgumentException("Invalid desiredOffset " + desiredOffset); }
private <K> Map<K, OrcDataSourceInput> readLargeDiskRanges(Map<K, DiskRange> diskRanges) { if (diskRanges.isEmpty()) { return ImmutableMap.of(); } ImmutableMap.Builder<K, OrcDataSourceInput> slices = ImmutableMap.builder(); for (Entry<K, DiskRange> entry : diskRanges.entrySet()) { DiskRange diskRange = entry.getValue(); int bufferSize = toIntExact(streamBufferSize.toBytes()); FixedLengthSliceInput sliceInput = new LazySliceInput(diskRange.getLength(), new LazyChunkedSliceLoader(diskRange, bufferSize)); slices.put(entry.getKey(), new OrcDataSourceInput(sliceInput, bufferSize)); } return slices.build(); }
@Override public void load(long position, SliceBufferReference bufferReference, int length) { try { readFully(diskRange.getOffset() + position, bufferReference.getBuffer(), 0, length); } catch (IOException e) { throw new UncheckedIOException(e); } }
public static LinearProbeRangeFinder createTinyStripesRangeFinder(List<StripeInformation> stripes, DataSize maxMergeDistance, DataSize tinyStripeThreshold) { if (stripes.size() == 0) { return new LinearProbeRangeFinder(ImmutableList.of()); } List<DiskRange> scratchDiskRanges = stripes.stream() .map(stripe -> new DiskRange(stripe.getOffset(), toIntExact(stripe.getTotalLength()))) .collect(Collectors.toList()); List<DiskRange> diskRanges = mergeAdjacentDiskRanges(scratchDiskRanges, maxMergeDistance, tinyStripeThreshold); return new LinearProbeRangeFinder(diskRanges); } }
/** * Get a slice for the disk range from the provided buffers. The buffers ranges do not have * to exactly match {@code diskRange}, but {@code diskRange} must be completely contained within * one of the buffer ranges. */ public static Slice getDiskRangeSlice(DiskRange diskRange, Map<DiskRange, byte[]> buffers) { for (Entry<DiskRange, byte[]> bufferEntry : buffers.entrySet()) { DiskRange bufferRange = bufferEntry.getKey(); byte[] buffer = bufferEntry.getValue(); if (bufferRange.contains(diskRange)) { int offset = toIntExact(diskRange.getOffset() - bufferRange.getOffset()); return Slices.wrappedBuffer(buffer, offset, diskRange.getLength()); } } throw new IllegalStateException("No matching buffer for disk range"); } }
/** * Returns the minimal DiskRange that encloses both this DiskRange * and otherDiskRange. If there was a gap between the ranges the * new range will cover that gap. */ public DiskRange span(DiskRange otherDiskRange) { requireNonNull(otherDiskRange, "otherDiskRange is null"); long start = Math.min(this.offset, otherDiskRange.getOffset()); long end = Math.max(getEnd(), otherDiskRange.getEnd()); return new DiskRange(start, toIntExact(end - start)); }
public Map<StreamId, OrcInputStream> readDiskRanges(long stripeOffset, Map<StreamId, DiskRange> diskRanges, AggregatedMemoryContext systemMemoryUsage) throws IOException { // // Note: this code does not use the Java 8 stream APIs to avoid any extra object allocation // // transform ranges to have an absolute offset in file ImmutableMap.Builder<StreamId, DiskRange> diskRangesBuilder = ImmutableMap.builder(); for (Entry<StreamId, DiskRange> entry : diskRanges.entrySet()) { DiskRange diskRange = entry.getValue(); diskRangesBuilder.put(entry.getKey(), new DiskRange(stripeOffset + diskRange.getOffset(), diskRange.getLength())); } diskRanges = diskRangesBuilder.build(); // read ranges Map<StreamId, OrcDataSourceInput> streamsData = orcDataSource.readFully(diskRanges); // transform streams to OrcInputStream ImmutableMap.Builder<StreamId, OrcInputStream> streamsBuilder = ImmutableMap.builder(); for (Entry<StreamId, OrcDataSourceInput> entry : streamsData.entrySet()) { OrcDataSourceInput sourceInput = entry.getValue(); streamsBuilder.put(entry.getKey(), new OrcInputStream(orcDataSource.getId(), sourceInput.getInput(), decompressor, systemMemoryUsage, sourceInput.getRetainedSizeInBytes())); } return streamsBuilder.build(); }