/** * Used to iterate through the columns of a row. * @param sstable SSTable we are reading ffrom. * @param file Reading using this file. * @param key Key of this row. * @param dataSize length of row data * @param checkData if true, do its best to deserialize and check the coherence of row data */ public SSTableIdentityIterator(SSTableReader sstable, RandomAccessReader file, DecoratedKey key, long dataSize, boolean checkData) { this(sstable.metadata, file, file.getPath(), key, dataSize, checkData, sstable, ColumnSerializer.Flag.LOCAL); }
public String getPath() { // if input is from file, then return that path, otherwise it's from streaming if (in instanceof RandomAccessReader) { RandomAccessReader file = (RandomAccessReader) in; return file.getPath(); } else { throw new UnsupportedOperationException(); } }
@Override public void seek(long newPosition) { if (newPosition < 0) throw new IllegalArgumentException("new position should not be negative"); if (newPosition > length()) // it is save to call length() in read-only mode throw new IllegalArgumentException(String.format("unable to seek to position %d in %s (%d bytes) in read-only mode", newPosition, getPath(), length())); current = newPosition; if (newPosition > (bufferOffset + validBufferBytes) || newPosition < bufferOffset) reBuffer(); }
@Override public void seek(long newPosition) { if (newPosition < 0) throw new IllegalArgumentException("new position should not be negative"); if (buffer == null) throw new IllegalStateException("Attempted to seek in a closed RAR"); long bufferOffset = bufferHolder.offset(); if (newPosition >= bufferOffset && newPosition < bufferOffset + buffer.limit()) { buffer.position((int) (newPosition - bufferOffset)); return; } if (newPosition > length()) throw new IllegalArgumentException(String.format("Unable to seek to position %d in %s (%d bytes) in read-only mode", newPosition, getPath(), length())); reBufferAt(newPosition); }
@Override public void seek(long newPosition) { if (newPosition < 0) throw new IllegalArgumentException("new position should not be negative"); if (buffer == null) throw new IllegalStateException("Attempted to seek in a closed RAR"); long bufferOffset = bufferHolder.offset(); if (newPosition >= bufferOffset && newPosition < bufferOffset + buffer.limit()) { buffer.position((int) (newPosition - bufferOffset)); return; } if (newPosition > length()) throw new IllegalArgumentException(String.format("Unable to seek to position %d in %s (%d bytes) in read-only mode", newPosition, getPath(), length())); reBufferAt(newPosition); }
@Override public void seek(long newPosition) { if (newPosition < 0) throw new IllegalArgumentException("new position should not be negative"); if (buffer == null) throw new IllegalStateException("Attempted to seek in a closed RAR"); long bufferOffset = bufferHolder.offset(); if (newPosition >= bufferOffset && newPosition < bufferOffset + buffer.limit()) { buffer.position((int) (newPosition - bufferOffset)); return; } if (newPosition > length()) throw new IllegalArgumentException(String.format("Unable to seek to position %d in %s (%d bytes) in read-only mode", newPosition, getPath(), length())); reBufferAt(newPosition); }
@Override public void seek(long newPosition) { if (newPosition < 0) throw new IllegalArgumentException("new position should not be negative"); if (buffer == null) throw new IllegalStateException("Attempted to seek in a closed RAR"); long bufferOffset = bufferHolder.offset(); if (newPosition >= bufferOffset && newPosition < bufferOffset + buffer.limit()) { buffer.position((int) (newPosition - bufferOffset)); return; } if (newPosition > length()) throw new IllegalArgumentException(String.format("Unable to seek to position %d in %s (%d bytes) in read-only mode", newPosition, getPath(), length())); reBufferAt(newPosition); }
@Override public void onRemoval(RemovalNotification<CacheKey, CacheBucket> notification) { CacheBucket bucket = notification.getValue(); if (bucket == null) return; // set discarded before deallocating the readers, to ensure we don't leak any bucket.discarded = true; Queue<RandomAccessReader> q = bucket.queue; boolean first = true; for (RandomAccessReader reader = q.poll() ; reader != null ; reader = q.poll()) { if (logger.isDebugEnabled() && first) { logger.debug("Evicting cold readers for {}", reader.getPath()); first = false; } memoryUsage.addAndGet(-1 * reader.getTotalBufferSize()); reader.deallocate(); } } };
@Override // -1 will be returned if there is nothing to read; higher-level methods like readInt // or readFully (from RandomAccessFile) will throw EOFException but this should not public int read(byte[] buff, int offset, int length) { if (buffer == null) throw new AssertionError("Attempted to read from closed RAR"); if (length == 0) return 0; if (isEOF()) return -1; if (current >= bufferOffset + buffer.length || validBufferBytes == -1) reBuffer(); assert current >= bufferOffset && current < bufferOffset + validBufferBytes : String.format("File (%s), current offset %d, buffer offset %d, buffer limit %d", getPath(), current, bufferOffset, validBufferBytes); int toCopy = Math.min(length, validBufferBytes - bufferCursor()); System.arraycopy(buffer, bufferCursor(), buff, offset, toCopy); current += toCopy; return toCopy; }
public static SSTableIdentityIterator create(SSTableReader sstable, RandomAccessReader file, DecoratedKey key) { try { DeletionTime partitionLevelDeletion = DeletionTime.serializer.deserialize(file); SerializationHelper helper = new SerializationHelper(sstable.metadata, sstable.descriptor.version.correspondingMessagingVersion(), SerializationHelper.Flag.LOCAL); SSTableSimpleIterator iterator = SSTableSimpleIterator.create(sstable.metadata, file, sstable.header, helper, partitionLevelDeletion); return new SSTableIdentityIterator(sstable, key, partitionLevelDeletion, file.getPath(), iterator); } catch (IOException e) { sstable.markSuspect(); throw new CorruptSSTableException(e, file.getPath()); } }
public static SSTableIdentityIterator create(SSTableReader sstable, RandomAccessReader file, DecoratedKey key) { try { DeletionTime partitionLevelDeletion = DeletionTime.serializer.deserialize(file); SerializationHelper helper = new SerializationHelper(sstable.metadata, sstable.descriptor.version.correspondingMessagingVersion(), SerializationHelper.Flag.LOCAL); SSTableSimpleIterator iterator = SSTableSimpleIterator.create(sstable.metadata, file, sstable.header, helper, partitionLevelDeletion); return new SSTableIdentityIterator(sstable, key, partitionLevelDeletion, file.getPath(), iterator); } catch (IOException e) { sstable.markSuspect(); throw new CorruptSSTableException(e, file.getPath()); } }
public static SSTableIdentityIterator create(SSTableReader sstable, RandomAccessReader file, DecoratedKey key) { try { DeletionTime partitionLevelDeletion = DeletionTime.serializer.deserialize(file); SerializationHelper helper = new SerializationHelper(sstable.metadata, sstable.descriptor.version.correspondingMessagingVersion(), SerializationHelper.Flag.LOCAL); SSTableSimpleIterator iterator = SSTableSimpleIterator.create(sstable.metadata, file, sstable.header, helper, partitionLevelDeletion); return new SSTableIdentityIterator(sstable, key, partitionLevelDeletion, file.getPath(), iterator); } catch (IOException e) { sstable.markSuspect(); throw new CorruptSSTableException(e, file.getPath()); } }
public static SSTableIdentityIterator create(SSTableReader sstable, RandomAccessReader file, DecoratedKey key) { try { DeletionTime partitionLevelDeletion = DeletionTime.serializer.deserialize(file); SerializationHelper helper = new SerializationHelper(sstable.metadata, sstable.descriptor.version.correspondingMessagingVersion(), SerializationHelper.Flag.LOCAL); SSTableSimpleIterator iterator = SSTableSimpleIterator.create(sstable.metadata, file, sstable.header, helper, partitionLevelDeletion); return new SSTableIdentityIterator(sstable, key, partitionLevelDeletion, file.getPath(), iterator); } catch (IOException e) { sstable.markSuspect(); throw new CorruptSSTableException(e, file.getPath()); } }
@SuppressWarnings("resource") public SyncSegment nextSegment(int startPosition, int nextSectionStartPosition) throws IOException { int totalPlainTextLength = reader.readInt(); currentSegmentEndPosition = nextSectionStartPosition - 1; nextLogicalStart += SYNC_MARKER_SIZE; FileDataInput input = new EncryptedFileSegmentInputStream(reader.getPath(), nextLogicalStart, 0, totalPlainTextLength, chunkProvider); nextLogicalStart += totalPlainTextLength; return new SyncSegment(input, startPosition, nextSectionStartPosition, (int)nextLogicalStart, tolerateSegmentErrors(nextSectionStartPosition, reader.length())); } }
@SuppressWarnings("resource") public SyncSegment nextSegment(int startPosition, int nextSectionStartPosition) throws IOException { int totalPlainTextLength = reader.readInt(); currentSegmentEndPosition = nextSectionStartPosition - 1; nextLogicalStart += SYNC_MARKER_SIZE; FileDataInput input = new EncryptedFileSegmentInputStream(reader.getPath(), nextLogicalStart, 0, totalPlainTextLength, chunkProvider); nextLogicalStart += totalPlainTextLength; return new SyncSegment(input, startPosition, nextSectionStartPosition, (int)nextLogicalStart, tolerateSegmentErrors(nextSectionStartPosition, reader.length())); } }
@SuppressWarnings("resource") public SyncSegment nextSegment(int startPosition, int nextSectionStartPosition) throws IOException { int totalPlainTextLength = reader.readInt(); currentSegmentEndPosition = nextSectionStartPosition - 1; nextLogicalStart += SYNC_MARKER_SIZE; FileDataInput input = new EncryptedFileSegmentInputStream(reader.getPath(), nextLogicalStart, 0, totalPlainTextLength, chunkProvider); nextLogicalStart += totalPlainTextLength; return new SyncSegment(input, startPosition, nextSectionStartPosition, (int)nextLogicalStart, tolerateSegmentErrors(nextSectionStartPosition, reader.length())); } }
@SuppressWarnings("resource") private boolean tryAppend(DecoratedKey prevKey, DecoratedKey key, SSTableRewriter writer) { // OrderCheckerIterator will check, at iteration time, that the rows are in the proper order. If it detects // that one row is out of order, it will stop returning them. The remaining rows will be sorted and added // to the outOfOrder set that will be later written to a new SSTable. OrderCheckerIterator sstableIterator = new OrderCheckerIterator(getIterator(key), cfs.metadata.comparator); try (UnfilteredRowIterator iterator = withValidation(sstableIterator, dataFile.getPath())) { if (prevKey != null && prevKey.compareTo(key) > 0) { saveOutOfOrderRow(prevKey, key, iterator); return false; } if (writer.tryAppend(iterator) == null) emptyRows++; else goodRows++; } if (sstableIterator.hasRowsOutOfOrder()) { outputHandler.warn(String.format("Out of order rows found in partition: %s", key)); outOfOrder.add(sstableIterator.getRowsOutOfOrder()); } return true; }
@SuppressWarnings("resource") public SyncSegment nextSegment(final int startPosition, final int nextSectionStartPosition) throws IOException { reader.seek(startPosition); int uncompressedLength = reader.readInt(); int compressedLength = nextSectionStartPosition - (int)reader.getPosition(); if (compressedLength > compressedBuffer.length) compressedBuffer = new byte[(int) (1.2 * compressedLength)]; reader.readFully(compressedBuffer, 0, compressedLength); if (uncompressedLength > uncompressedBuffer.length) uncompressedBuffer = new byte[(int) (1.2 * uncompressedLength)]; int count = compressor.uncompress(compressedBuffer, 0, compressedLength, uncompressedBuffer, 0); nextLogicalStart += SYNC_MARKER_SIZE; FileDataInput input = new FileSegmentInputStream(ByteBuffer.wrap(uncompressedBuffer, 0, count), reader.getPath(), nextLogicalStart); nextLogicalStart += uncompressedLength; return new SyncSegment(input, startPosition, nextSectionStartPosition, (int)nextLogicalStart, tolerateSegmentErrors(nextSectionStartPosition, reader.length())); } }
@SuppressWarnings("resource") public SyncSegment nextSegment(final int startPosition, final int nextSectionStartPosition) throws IOException { reader.seek(startPosition); int uncompressedLength = reader.readInt(); int compressedLength = nextSectionStartPosition - (int)reader.getPosition(); if (compressedLength > compressedBuffer.length) compressedBuffer = new byte[(int) (1.2 * compressedLength)]; reader.readFully(compressedBuffer, 0, compressedLength); if (uncompressedLength > uncompressedBuffer.length) uncompressedBuffer = new byte[(int) (1.2 * uncompressedLength)]; int count = compressor.uncompress(compressedBuffer, 0, compressedLength, uncompressedBuffer, 0); nextLogicalStart += SYNC_MARKER_SIZE; FileDataInput input = new FileSegmentInputStream(ByteBuffer.wrap(uncompressedBuffer, 0, count), reader.getPath(), nextLogicalStart); nextLogicalStart += uncompressedLength; return new SyncSegment(input, startPosition, nextSectionStartPosition, (int)nextLogicalStart, tolerateSegmentErrors(nextSectionStartPosition, reader.length())); } }
@SuppressWarnings("resource") public SyncSegment nextSegment(final int startPosition, final int nextSectionStartPosition) throws IOException { reader.seek(startPosition); int uncompressedLength = reader.readInt(); int compressedLength = nextSectionStartPosition - (int)reader.getPosition(); if (compressedLength > compressedBuffer.length) compressedBuffer = new byte[(int) (1.2 * compressedLength)]; reader.readFully(compressedBuffer, 0, compressedLength); if (uncompressedLength > uncompressedBuffer.length) uncompressedBuffer = new byte[(int) (1.2 * uncompressedLength)]; int count = compressor.uncompress(compressedBuffer, 0, compressedLength, uncompressedBuffer, 0); nextLogicalStart += SYNC_MARKER_SIZE; FileDataInput input = new FileSegmentInputStream(ByteBuffer.wrap(uncompressedBuffer, 0, count), reader.getPath(), nextLogicalStart); nextLogicalStart += uncompressedLength; return new SyncSegment(input, startPosition, nextSectionStartPosition, (int)nextLogicalStart, tolerateSegmentErrors(nextSectionStartPosition, reader.length())); } }