@Override public boolean advanceNextPosition() { if (nanoStart == 0) { nanoStart = System.nanoTime(); } try { if (iterator.hasNext()) { serializer.reset(); Entry<Key, Value> row = iterator.next(); for (Entry<Key, Value> entry : WholeRowIterator.decodeRow(row.getKey(), row.getValue()).entrySet()) { bytesRead += entry.getKey().getSize() + entry.getValue().getSize(); serializer.deserialize(entry); } return true; } else { return false; } } catch (IOException e) { throw new PrestoException(IO_ERROR, "Caught IO error from serializer on read", e); } }
int weight() { int keyWeight = ClassSize.align(prevKey.getSize()) + ClassSize.OBJECT + SizeConstants.SIZEOF_LONG + 4 * (ClassSize.ARRAY + ClassSize.REFERENCE); return 2 * SizeConstants.SIZEOF_INT + ClassSize.REFERENCE + ClassSize.OBJECT + keyWeight; } }
private boolean isGiantKey(Key k) { double mean = keyLenStats.getMean(); double stddev = keyLenStats.getStandardDeviation(); return k.getSize() > mean + Math.max(9 * mean, 4 * stddev); }
public void add(Key k) { int size = k.getSize(); stats.addValue(size); logHistogram.add(size); }
public void append(Key key, Value value) { if (sampler.accept(key)) { entries.add(new SampleEntry(key, value)); dataSize += key.getSize() + value.getSize(); } }
int estimateMemoryUsed() { return getKey().getSize() + getValue().get().length + (9 * 32); // overhead is 32 per object } }
int numBytes() { return getKey().getSize() + getValue().get().length; }
private void fill() { end = 0; index = 0; if (source.hasNext()) source.doNextPreCheck(); int amountRead = 0; // as we keep filling, increase the read ahead buffer if (nextEntries.length < MAX_READ_AHEAD_ENTRIES) nextEntries = new PreAllocatedArray<>( Math.min(nextEntries.length * 2, MAX_READ_AHEAD_ENTRIES)); while (source.hasNext() && end < nextEntries.length) { Entry<Key,Value> ne = source.next(); nextEntries.set(end++, ne); amountRead += ne.getKey().getSize() + ne.getValue().getSize(); if (amountRead > READ_AHEAD_BYTES) break; } }
Key key = iter.getTopKey(); Value value = iter.getTopValue(); if (key.getSize() + value.getSize() < opts.maxSize) { small.append(key, value); } else {
private void prepKeys() throws IOException { long kvBufSize = 0; if (topKey != null) return; Text currentRow; do { if (!sourceIter.hasTop()) return; currentRow = new Text(sourceIter.getTopKey().getRow()); keys.clear(); values.clear(); while (sourceIter.hasTop() && sourceIter.getTopKey().getRow().equals(currentRow)) { Key sourceTopKey = sourceIter.getTopKey(); Value sourceTopValue = sourceIter.getTopValue(); keys.add(new Key(sourceTopKey)); values.add(new Value(sourceTopValue)); kvBufSize += sourceTopKey.getSize() + sourceTopValue.getSize() + 128; if (kvBufSize > maxBufferSize) { throw new BufferOverflowException( "Exceeded buffer size of " + maxBufferSize + " for row: " + sourceTopKey.getRow()); } sourceIter.next(); } } while (!filter(currentRow, keys, values)); topKey = new Key(currentRow); topValue = rowEncoder(keys, values); }
@Override public void append(Key key, Value val) { // ensure the key provided by the user has the correct prefix if (!key.equals(prefixKey, getKeyPrefix())) throw new IllegalArgumentException("Key prefixes are not equal " + key + " " + prefixKey); // Transformation could have produced a key that falls outside // of the seek range, or one that the user cannot see. Check // these before adding it to the output list. if (includeTransformedKey(key)) { // try to defend against a scan or compaction using all memory in a tablet server if (appened > maxBufferSize) throw new BufferOverflowException( "Exceeded buffer size of " + maxBufferSize + ", prefixKey: " + prefixKey); if (getSource().hasTop() && key == getSource().getTopKey()) key = new Key(key); keys.add(new Pair<>(key, new Value(val))); appened += (key.getSize() + val.getSize() + 128); } } });
kvHistogram.add(key.getSize() + value.getSize());
if ((closeKey.getSize() <= averageKeySize || blockWriter.getRawSize() > maxBlockSize) && !isGiantKey(closeKey)) { closeBlock(closeKey, false); } else if (((long) key.getSize() + (long) value.getSize() + blockWriter.getRawSize() + 128L) >= Integer.MAX_VALUE) { closeBlock(closeKey, false); entries++; keyLenStats.addValue(key.getSize());
private boolean isGiantKey(Key k) { double mean = keyLenStats.getMean(); double stddev = keyLenStats.getStandardDeviation(); return k.getSize() > mean + Math.max(9 * mean, 4 * stddev); }
public void add(Key k) { int size = k.getSize(); stats.addValue(size); logHistogram.add(size); }
public void append(Key key, Value value) throws IOException { if (sampler.accept(key)) { entries.add(new SampleEntry(key, value)); dataSize += key.getSize() + value.getSize(); } }
int estimateMemoryUsed() { return getKey().getSize() + getValue().get().length + (9 * 32); // overhead is 32 per object } }
int estimateMemoryUsed() { return getKey().getSize() + getValue().get().length + (9 * 32); // overhead is 32 per object } }
int numBytes() { return getKey().getSize() + getValue().get().length; }
int numBytes() { return getKey().getSize() + getValue().get().length; }