@Override public boolean advanceNextPosition() { if (nanoStart == 0) { nanoStart = System.nanoTime(); } try { if (iterator.hasNext()) { serializer.reset(); Entry<Key, Value> row = iterator.next(); for (Entry<Key, Value> entry : WholeRowIterator.decodeRow(row.getKey(), row.getValue()).entrySet()) { bytesRead += entry.getKey().getSize() + entry.getValue().getSize(); serializer.deserialize(entry); } return true; } else { return false; } } catch (IOException e) { throw new PrestoException(IO_ERROR, "Caught IO error from serializer on read", e); } }
public void put(Key key, Value value) { // Always a MemKey, so account for the kvCount int bytesInMemory.addAndGet(key.getLength() + 4); bytesInMemory.addAndGet(value.getSize()); if (map.put(key, value) == null) size.incrementAndGet(); }
public static Value encode(Value value, int kv) { byte[] combinedBytes = new byte[value.getSize() + 4]; System.arraycopy(value.get(), 0, combinedBytes, 4, value.getSize()); combinedBytes[0] = (byte) (kv >>> 24); combinedBytes[1] = (byte) (kv >>> 16); combinedBytes[2] = (byte) (kv >>> 8); combinedBytes[3] = (byte) (kv); return new Value(combinedBytes); }
/** * Creates a new Value based on another. * * @param ibw * may not be null. */ public Value(final Value ibw) { this(ibw.get(), 0, ibw.getSize()); }
public void append(Key key, Value value) { if (sampler.accept(key)) { entries.add(new SampleEntry(key, value)); dataSize += key.getSize() + value.getSize(); } }
public static MemValue decode(Value v) { byte[] originalBytes = new byte[v.getSize() - 4]; byte[] combined = v.get(); System.arraycopy(combined, 4, originalBytes, 0, originalBytes.length); int kv = (combined[0] << 24) + ((combined[1] & 0xFF) << 16) + ((combined[2] & 0xFF) << 8) + (combined[3] & 0xFF); return new MemValue(new Value(originalBytes), kv); } }
/** Does not show timestamps if timestampFormat is null */ public static String formatEntry(Entry<Key,Value> entry, DateFormat timestampFormat) { StringBuilder sb = new StringBuilder(); Key key = entry.getKey(); Text buffer = new Text(); // append row appendText(sb, key.getRow(buffer)).append(" "); // append column family appendText(sb, key.getColumnFamily(buffer)).append(":"); // append column qualifier appendText(sb, key.getColumnQualifier(buffer)).append(" "); // append visibility expression sb.append(new ColumnVisibility(key.getColumnVisibility(buffer))); // append timestamp if (timestampFormat != null) { tmpDate.get().setTime(entry.getKey().getTimestamp()); sb.append(" ").append(timestampFormat.format(tmpDate.get())); } Value value = entry.getValue(); // append value if (value != null && value.getSize() > 0) { sb.append("\t"); appendValue(sb, value); } return sb.toString(); }
public static String formatEntry(Entry<Key,Value> entry, FormatterConfig config) { // originally from BinaryFormatter StringBuilder sb = new StringBuilder(); Key key = entry.getKey(); Text buffer = new Text(); final int shownLength = config.getShownLength(); appendText(sb, key.getRow(buffer), shownLength).append(" "); appendText(sb, key.getColumnFamily(buffer), shownLength).append(":"); appendText(sb, key.getColumnQualifier(buffer), shownLength).append(" "); sb.append(new ColumnVisibility(key.getColumnVisibility(buffer))); // append timestamp if (config.willPrintTimestamps() && config.getDateFormatSupplier() != null) { tmpDate.get().setTime(entry.getKey().getTimestamp()); sb.append(" ").append(config.getDateFormatSupplier().get().format(tmpDate.get())); } // append value Value value = entry.getValue(); if (value != null && value.getSize() > 0) { sb.append("\t"); appendValue(sb, value, shownLength); } return sb.toString(); }
public static String formatEntry(Entry<Key,Value> entry, boolean printTimestamps, int shownLength) { StringBuilder sb = new StringBuilder(); Key key = entry.getKey(); // append row appendText(sb, key.getRow(), shownLength).append(" "); // append column family appendText(sb, key.getColumnFamily(), shownLength).append(":"); // append column qualifier appendText(sb, key.getColumnQualifier(), shownLength).append(" "); // append visibility expression sb.append(new ColumnVisibility(key.getColumnVisibility())); // append timestamp if (printTimestamps) sb.append(" ").append(entry.getKey().getTimestamp()); // append value Value value = entry.getValue(); if (value != null && value.getSize() > 0) { sb.append("\t"); appendValue(sb, value, shownLength); } return sb.toString(); }
private void fill() { end = 0; index = 0; if (source.hasNext()) source.doNextPreCheck(); int amountRead = 0; // as we keep filling, increase the read ahead buffer if (nextEntries.length < MAX_READ_AHEAD_ENTRIES) nextEntries = new PreAllocatedArray<>( Math.min(nextEntries.length * 2, MAX_READ_AHEAD_ENTRIES)); while (source.hasNext() && end < nextEntries.length) { Entry<Key,Value> ne = source.next(); nextEntries.set(end++, ne); amountRead += ne.getKey().getSize() + ne.getValue().getSize(); if (amountRead > READ_AHEAD_BYTES) break; } }
Key key = iter.getTopKey(); Value value = iter.getTopValue(); if (key.getSize() + value.getSize() < opts.maxSize) { small.append(key, value); } else {
private void prepKeys() throws IOException { long kvBufSize = 0; if (topKey != null) return; Text currentRow; do { if (!sourceIter.hasTop()) return; currentRow = new Text(sourceIter.getTopKey().getRow()); keys.clear(); values.clear(); while (sourceIter.hasTop() && sourceIter.getTopKey().getRow().equals(currentRow)) { Key sourceTopKey = sourceIter.getTopKey(); Value sourceTopValue = sourceIter.getTopValue(); keys.add(new Key(sourceTopKey)); values.add(new Value(sourceTopValue)); kvBufSize += sourceTopKey.getSize() + sourceTopValue.getSize() + 128; if (kvBufSize > maxBufferSize) { throw new BufferOverflowException( "Exceeded buffer size of " + maxBufferSize + " for row: " + sourceTopKey.getRow()); } sourceIter.next(); } } while (!filter(currentRow, keys, values)); topKey = new Key(currentRow); topValue = rowEncoder(keys, values); }
@Override public void append(Key key, Value val) { // ensure the key provided by the user has the correct prefix if (!key.equals(prefixKey, getKeyPrefix())) throw new IllegalArgumentException("Key prefixes are not equal " + key + " " + prefixKey); // Transformation could have produced a key that falls outside // of the seek range, or one that the user cannot see. Check // these before adding it to the output list. if (includeTransformedKey(key)) { // try to defend against a scan or compaction using all memory in a tablet server if (appened > maxBufferSize) throw new BufferOverflowException( "Exceeded buffer size of " + maxBufferSize + ", prefixKey: " + prefixKey); if (getSource().hasTop() && key == getSource().getTopKey()) key = new Key(key); keys.add(new Pair<>(key, new Value(val))); appened += (key.getSize() + val.getSize() + 128); } } });
kvHistogram.add(key.getSize() + value.getSize());
@Override public void accept(Key k, Value v) { keyStats.accept(k.getLength()); rowStats.accept(k.getRowData().length()); familyStats.accept(k.getColumnFamilyData().length()); qualifierStats.accept(k.getColumnQualifierData().length()); visibilityStats.accept(k.getColumnVisibilityData().length()); valueStats.accept(v.getSize()); total++; }
/** * Creates a new Value based on another. * * @param ibw * may not be null. */ public Value(final Value ibw) { this(ibw.get(), 0, ibw.getSize()); }
} else if (((long) key.getSize() + (long) value.getSize() + blockWriter.getRawSize() + 128L) >= Integer.MAX_VALUE) { closeBlock(closeKey, false);
public void append(Key key, Value value) throws IOException { entries++; size += key.getLength() + (value == null ? 0 : value.getSize()); delegate.append(key, value); }
public void append(Key key, Value value) throws IOException { if (sampler.accept(key)) { entries.add(new SampleEntry(key, value)); dataSize += key.getSize() + value.getSize(); } }
valbs = new MutableByteSequence(val.get(), 0, val.getSize());