public Future<Pair<KeyCacheKey, RowIndexEntry>> deserialize(DataInputStream input, ColumnFamilyStore cfs) throws IOException { //Keyspace and CF name are deserialized by AutoSaving cache and used to fetch the CFS provided as a //parameter so they aren't deserialized here, even though they are serialized by this serializer int keyLength = input.readInt(); if (keyLength > FBUtilities.MAX_UNSIGNED_SHORT) { throw new IOException(String.format("Corrupted key cache. Key length of %d is longer than maximum of %d", keyLength, FBUtilities.MAX_UNSIGNED_SHORT)); } ByteBuffer key = ByteBufferUtil.read(input, keyLength); int generation = input.readInt(); input.readBoolean(); // backwards compatibility for "promoted indexes" boolean SSTableReader reader = null; if (cfs == null || !cfs.isKeyCacheEnabled() || (reader = findDesc(generation, cfs.getSSTables())) == null) { RowIndexEntry.Serializer.skip(input); return null; } RowIndexEntry entry = reader.metadata.comparator.rowIndexEntrySerializer().deserialize(input, reader.descriptor.version); return Futures.immediateFuture(Pair.create(new KeyCacheKey(cfs.metadata.ksAndCFName, reader.descriptor, key), entry)); }
protected DecoratedKey computeNext() { try { if (in.isEOF()) return endOfData(); DecoratedKey key = StorageService.getPartitioner().decorateKey(ByteBufferUtil.readWithShortLength(in)); RowIndexEntry.Serializer.skip(in); // skip remainder of the entry return key; } catch (IOException e) { throw new RuntimeException(e); } }
protected DecoratedKey computeNext() { try { if (in.isEOF()) return endOfData(); keyPosition = in.getFilePointer(); DecoratedKey key = partitioner.decorateKey(ByteBufferUtil.readWithShortLength(in.get())); RowIndexEntry.Serializer.skip(in.get(), desc.version); // skip remainder of the entry return key; } catch (IOException e) { throw new RuntimeException(e); } }
/** * Finds and returns the first key beyond a given token in this SSTable or null if no such key exists. */ public DecoratedKey firstKeyBeyond(PartitionPosition token) { if (token.compareTo(first) < 0) return first; long sampledPosition = getIndexScanPosition(token); if (ifile == null) return null; String path = null; try (FileDataInput in = ifile.createReader(sampledPosition)) { path = in.getPath(); while (!in.isEOF()) { ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in); DecoratedKey indexDecoratedKey = decorateKey(indexKey); if (indexDecoratedKey.compareTo(token) > 0) return indexDecoratedKey; RowIndexEntry.Serializer.skip(in, descriptor.version); } } catch (IOException e) { markSuspect(); throw new CorruptSSTableException(e, path); } return null; }
private void seekToCurrentRangeStart() { long indexPosition = sstable.getIndexScanPosition(currentRange.left); ifile.seek(indexPosition); try { while (!ifile.isEOF()) { indexPosition = ifile.getFilePointer(); DecoratedKey indexDecoratedKey = sstable.decorateKey(ByteBufferUtil.readWithShortLength(ifile)); if (indexDecoratedKey.compareTo(currentRange.left) > 0 || currentRange.contains(indexDecoratedKey)) { // Found, just read the dataPosition and seek into index and data files long dataPosition = RowIndexEntry.Serializer.readPosition(ifile, sstable.descriptor.version); ifile.seek(indexPosition); dfile.seek(dataPosition); break; } else { RowIndexEntry.Serializer.skip(ifile, sstable.descriptor.version); } } } catch (IOException e) { sstable.markSuspect(); throw new CorruptSSTableException(e, sstable.getFilename()); } }
/** * Finds and returns the first key beyond a given token in this SSTable or null if no such key exists. */ public DecoratedKey firstKeyBeyond(PartitionPosition token) { if (token.compareTo(first) < 0) return first; long sampledPosition = getIndexScanPosition(token); if (ifile == null) return null; String path = null; try (FileDataInput in = ifile.createReader(sampledPosition)) { path = in.getPath(); while (!in.isEOF()) { ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in); DecoratedKey indexDecoratedKey = decorateKey(indexKey); if (indexDecoratedKey.compareTo(token) > 0) return indexDecoratedKey; RowIndexEntry.Serializer.skip(in, descriptor.version); } } catch (IOException e) { markSuspect(); throw new CorruptSSTableException(e, path); } return null; }
private void seekToCurrentRangeStart() { long indexPosition = sstable.getIndexScanPosition(currentRange.left); ifile.seek(indexPosition); try { while (!ifile.isEOF()) { indexPosition = ifile.getFilePointer(); DecoratedKey indexDecoratedKey = sstable.decorateKey(ByteBufferUtil.readWithShortLength(ifile)); if (indexDecoratedKey.compareTo(currentRange.left) > 0 || currentRange.contains(indexDecoratedKey)) { // Found, just read the dataPosition and seek into index and data files long dataPosition = RowIndexEntry.Serializer.readPosition(ifile, sstable.descriptor.version); ifile.seek(indexPosition); dfile.seek(dataPosition); break; } else { RowIndexEntry.Serializer.skip(ifile, sstable.descriptor.version); } } } catch (IOException e) { sstable.markSuspect(); throw new CorruptSSTableException(e, sstable.getFilename()); } }
private void seekToCurrentRangeStart() { long indexPosition = sstable.getIndexScanPosition(currentRange.left); ifile.seek(indexPosition); try { while (!ifile.isEOF()) { indexPosition = ifile.getFilePointer(); DecoratedKey indexDecoratedKey = sstable.decorateKey(ByteBufferUtil.readWithShortLength(ifile)); if (indexDecoratedKey.compareTo(currentRange.left) > 0 || currentRange.contains(indexDecoratedKey)) { // Found, just read the dataPosition and seek into index and data files long dataPosition = RowIndexEntry.Serializer.readPosition(ifile, sstable.descriptor.version); ifile.seek(indexPosition); dfile.seek(dataPosition); break; } else { RowIndexEntry.Serializer.skip(ifile, sstable.descriptor.version); } } } catch (IOException e) { sstable.markSuspect(); throw new CorruptSSTableException(e, sstable.getFilename()); } }
private void seekToCurrentRangeStart() { long indexPosition = sstable.getIndexScanPosition(currentRange.left); ifile.seek(indexPosition); try { while (!ifile.isEOF()) { indexPosition = ifile.getFilePointer(); DecoratedKey indexDecoratedKey = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(ifile)); if (indexDecoratedKey.compareTo(currentRange.left) > 0 || currentRange.contains(indexDecoratedKey)) { // Found, just read the dataPosition and seek into index and data files long dataPosition = ifile.readLong(); ifile.seek(indexPosition); dfile.seek(dataPosition); break; } else { RowIndexEntry.Serializer.skip(ifile); } } } catch (IOException e) { sstable.markSuspect(); throw new CorruptSSTableException(e, sstable.getFilename()); } }
private IndexSummary buildSummaryAtLevel(int newSamplingLevel) throws IOException { // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary. RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))); try { long indexSize = primaryIndex.length(); try (IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata.getMinIndexInterval(), newSamplingLevel)) { long indexPosition; while ((indexPosition = primaryIndex.getFilePointer()) != indexSize) { summaryBuilder.maybeAddEntry(partitioner.decorateKey(ByteBufferUtil.readWithShortLength(primaryIndex)), indexPosition); RowIndexEntry.Serializer.skip(primaryIndex); } return summaryBuilder.build(partitioner); } } finally { FileUtils.closeQuietly(primaryIndex); } }
private IndexSummary buildSummaryAtLevel(int newSamplingLevel) throws IOException { // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary. RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))); try { long indexSize = primaryIndex.length(); try (IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata.params.minIndexInterval, newSamplingLevel)) { long indexPosition; while ((indexPosition = primaryIndex.getFilePointer()) != indexSize) { summaryBuilder.maybeAddEntry(decorateKey(ByteBufferUtil.readWithShortLength(primaryIndex)), indexPosition); RowIndexEntry.Serializer.skip(primaryIndex, descriptor.version); } return summaryBuilder.build(getPartitioner()); } } finally { FileUtils.closeQuietly(primaryIndex); } }
private IndexSummary buildSummaryAtLevel(int newSamplingLevel) throws IOException { // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary. RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))); try { long indexSize = primaryIndex.length(); try (IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata.params.minIndexInterval, newSamplingLevel)) { long indexPosition; while ((indexPosition = primaryIndex.getFilePointer()) != indexSize) { summaryBuilder.maybeAddEntry(decorateKey(ByteBufferUtil.readWithShortLength(primaryIndex)), indexPosition); RowIndexEntry.Serializer.skip(primaryIndex, descriptor.version); } return summaryBuilder.build(getPartitioner()); } } finally { FileUtils.closeQuietly(primaryIndex); } }
private IndexSummary buildSummaryAtLevel(int newSamplingLevel) throws IOException { // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary. RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))); try { long indexSize = primaryIndex.length(); try (IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata.params.minIndexInterval, newSamplingLevel)) { long indexPosition; while ((indexPosition = primaryIndex.getFilePointer()) != indexSize) { summaryBuilder.maybeAddEntry(decorateKey(ByteBufferUtil.readWithShortLength(primaryIndex)), indexPosition); RowIndexEntry.Serializer.skip(primaryIndex, descriptor.version); } return summaryBuilder.build(getPartitioner()); } } finally { FileUtils.closeQuietly(primaryIndex); } }
/** @return An estimate of the number of keys contained in the given index file. */ long estimateRowsFromIndex(RandomAccessReader ifile) throws IOException { // collect sizes for the first 10000 keys, or first 10 megabytes of data final int SAMPLES_CAP = 10000, BYTES_CAP = (int)Math.min(10000000, ifile.length()); int keys = 0; while (ifile.getFilePointer() < BYTES_CAP && keys < SAMPLES_CAP) { ByteBufferUtil.skipShortLength(ifile); RowIndexEntry.Serializer.skip(ifile); keys++; } assert keys > 0 && ifile.getFilePointer() > 0 && ifile.length() > 0 : "Unexpected empty index file: " + ifile; long estimatedRows = ifile.length() / (ifile.getFilePointer() / keys); ifile.seek(0); return estimatedRows; }
/** @return An estimate of the number of keys contained in the given index file. */ protected long estimateRowsFromIndex(RandomAccessReader ifile) throws IOException { // collect sizes for the first 10000 keys, or first 10 megabytes of data final int SAMPLES_CAP = 10000, BYTES_CAP = (int)Math.min(10000000, ifile.length()); int keys = 0; while (ifile.getFilePointer() < BYTES_CAP && keys < SAMPLES_CAP) { ByteBufferUtil.skipShortLength(ifile); RowIndexEntry.Serializer.skip(ifile, descriptor.version); keys++; } assert keys > 0 && ifile.getFilePointer() > 0 && ifile.length() > 0 : "Unexpected empty index file: " + ifile; long estimatedRows = ifile.length() / (ifile.getFilePointer() / keys); ifile.seek(0); return estimatedRows; }
protected DecoratedKey computeNext() { try { if (in.isEOF()) return endOfData(); keyPosition = in.getFilePointer(); DecoratedKey key = partitioner.decorateKey(ByteBufferUtil.readWithShortLength(in.get())); RowIndexEntry.Serializer.skip(in.get(), desc.version); // skip remainder of the entry return key; } catch (IOException e) { throw new RuntimeException(e); } }
protected DecoratedKey computeNext() { try { if (in.isEOF()) return endOfData(); keyPosition = in.getFilePointer(); DecoratedKey key = partitioner.decorateKey(ByteBufferUtil.readWithShortLength(in.get())); RowIndexEntry.Serializer.skip(in.get(), desc.version); // skip remainder of the entry return key; } catch (IOException e) { throw new RuntimeException(e); } }
/** @return An estimate of the number of keys contained in the given index file. */ protected long estimateRowsFromIndex(RandomAccessReader ifile) throws IOException { // collect sizes for the first 10000 keys, or first 10 megabytes of data final int SAMPLES_CAP = 10000, BYTES_CAP = (int)Math.min(10000000, ifile.length()); int keys = 0; while (ifile.getFilePointer() < BYTES_CAP && keys < SAMPLES_CAP) { ByteBufferUtil.skipShortLength(ifile); RowIndexEntry.Serializer.skip(ifile, descriptor.version); keys++; } assert keys > 0 && ifile.getFilePointer() > 0 && ifile.length() > 0 : "Unexpected empty index file: " + ifile; long estimatedRows = ifile.length() / (ifile.getFilePointer() / keys); ifile.seek(0); return estimatedRows; }
/** @return An estimate of the number of keys contained in the given index file. */ protected long estimateRowsFromIndex(RandomAccessReader ifile) throws IOException { // collect sizes for the first 10000 keys, or first 10 megabytes of data final int SAMPLES_CAP = 10000, BYTES_CAP = (int)Math.min(10000000, ifile.length()); int keys = 0; while (ifile.getFilePointer() < BYTES_CAP && keys < SAMPLES_CAP) { ByteBufferUtil.skipShortLength(ifile); RowIndexEntry.Serializer.skip(ifile, descriptor.version); keys++; } assert keys > 0 && ifile.getFilePointer() > 0 && ifile.length() > 0 : "Unexpected empty index file: " + ifile; long estimatedRows = ifile.length() / (ifile.getFilePointer() / keys); ifile.seek(0); return estimatedRows; }
/** @return An estimate of the number of keys contained in the given index file. */ protected long estimateRowsFromIndex(RandomAccessReader ifile) throws IOException { // collect sizes for the first 10000 keys, or first 10 megabytes of data final int SAMPLES_CAP = 10000, BYTES_CAP = (int)Math.min(10000000, ifile.length()); int keys = 0; while (ifile.getFilePointer() < BYTES_CAP && keys < SAMPLES_CAP) { ByteBufferUtil.skipShortLength(ifile); RowIndexEntry.Serializer.skip(ifile, descriptor.version); keys++; } assert keys > 0 && ifile.getFilePointer() > 0 && ifile.length() > 0 : "Unexpected empty index file: " + ifile; long estimatedRows = ifile.length() / (ifile.getFilePointer() / keys); ifile.seek(0); return estimatedRows; }