/** * Write out a KeyValue in the manner in which we used to when KeyValue was a Writable. * @param kv * @param out * @return Length written on stream * @throws IOException * @see #create(DataInput) for the inverse function */ public static long write(final KeyValue kv, final DataOutput out) throws IOException { // This is how the old Writables write used to serialize KVs. Need to figure way to make it // work for all implementations. int length = kv.getLength(); out.writeInt(length); out.write(kv.getBuffer(), kv.getOffset(), length); return (long) length + Bytes.SIZEOF_INT; }
/** * Write out a KeyValue in the manner in which we used to when KeyValue was a * Writable. * * @param kv * @param out * @return Length written on stream * @throws IOException * @see #create(DataInput) for the inverse function */ public static long write(final KeyValue kv, final DataOutput out) throws IOException { // This is how the old Writables write used to serialize KVs. Need to figure // way to make it // work for all implementations. int length = kv.getLength(); out.writeInt(length); out.write(kv.getBuffer(), kv.getOffset(), length); return (long) length + Bytes.SIZEOF_INT; }
public static void appendToByteBuffer(final ByteBuffer bb, final KeyValue kv, final boolean includeMvccVersion) { // keep pushing the limit out. assume enough capacity bb.limit(bb.position() + kv.getLength()); bb.put(kv.getBuffer(), kv.getOffset(), kv.getLength()); if (includeMvccVersion) { int numMvccVersionBytes = WritableUtils.getVIntSize(kv.getSequenceId()); ByteBufferUtils.extendLimit(bb, numMvccVersionBytes); ByteBufferUtils.writeVLong(bb, kv.getSequenceId()); } }
for (Cell cell : kvs) { KeyValue currentCell = KeyValueUtil.copyToNewKeyValue(cell); out.write(currentCell.getBuffer(), currentCell.getOffset(), currentCell.getLength()); if (includesMvcc) {
@Test public void testKeyValue() throws Exception { final String name = "testKeyValue2"; byte[] row = name.getBytes(); byte[] fam = "fam".getBytes(); byte[] qf = "qf".getBytes(); long ts = System.currentTimeMillis(); byte[] val = "val".getBytes(); KeyValue kv = new KeyValue(row, fam, qf, ts, val); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(baos); long l = KeyValueUtil.write(kv, dos); dos.close(); byte [] mb = baos.toByteArray(); ByteArrayInputStream bais = new ByteArrayInputStream(mb); DataInputStream dis = new DataInputStream(bais); KeyValue deserializedKv = KeyValueUtil.create(dis); assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer())); assertEquals(kv.getOffset(), deserializedKv.getOffset()); assertEquals(kv.getLength(), deserializedKv.getLength()); }
@Test public void testToCell() throws Exception { KeyValue kv1 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); KeyValue kv2 = new KeyValue(Bytes.toBytes("bbb"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); KeyValue kv3 = new KeyValue(Bytes.toBytes("ccc"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); byte[] arr = new byte[kv1.getLength() + kv2.getLength() + kv3.getLength()]; System.arraycopy(kv1.getBuffer(), kv1.getOffset(), arr, 0, kv1.getLength()); System.arraycopy(kv2.getBuffer(), kv2.getOffset(), arr, kv1.getLength(), kv2.getLength()); System.arraycopy(kv3.getBuffer(), kv3.getOffset(), arr, kv1.getLength() + kv2.getLength(), kv3.getLength()); ByteBuffer dbb = ByteBuffer.allocateDirect(arr.length); dbb.put(arr); ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, kv1.getLength(), kv2.getLength()); CellProtos.Cell cell = ProtobufUtil.toCell(offheapKV); Cell newOffheapKV = ProtobufUtil.toCell(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY), cell); assertTrue(CellComparatorImpl.COMPARATOR.compare(offheapKV, newOffheapKV) == 0); }
ByteBuffer offheapKVBB = ByteBuffer.allocateDirect(keyValue.getLength()); ByteBufferUtils.copyFromArrayToBuffer(offheapKVBB, keyValue.getBuffer(), keyValue.getOffset(), keyValue.getLength()); ByteBufferKeyValue offheapKV = new ExtendedOffheapKeyValue(offheapKVBB, 0, keyValue.getLength(), 0); ByteBuffer offheapKVBB = ByteBuffer.allocateDirect(keyValue.getLength()); ByteBufferUtils.copyFromArrayToBuffer(offheapKVBB, keyValue.getBuffer(), keyValue.getOffset(), keyValue.getLength()); ByteBufferKeyValue offheapKV = new ExtendedOffheapKeyValue(offheapKVBB, 0, keyValue.getLength(), 0);
@Test public void testToCell() throws Exception { KeyValue kv1 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); KeyValue kv2 = new KeyValue(Bytes.toBytes("bbb"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); KeyValue kv3 = new KeyValue(Bytes.toBytes("ccc"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); byte[] arr = new byte[kv1.getLength() + kv2.getLength() + kv3.getLength()]; System.arraycopy(kv1.getBuffer(), kv1.getOffset(), arr, 0, kv1.getLength()); System.arraycopy(kv2.getBuffer(), kv2.getOffset(), arr, kv1.getLength(), kv2.getLength()); System.arraycopy(kv3.getBuffer(), kv3.getOffset(), arr, kv1.getLength() + kv2.getLength(), kv3.getLength()); ByteBuffer dbb = ByteBuffer.allocateDirect(arr.length); dbb.put(arr); ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, kv1.getLength(), kv2.getLength()); CellProtos.Cell cell = ProtobufUtil.toCell(offheapKV); Cell newOffheapKV = ProtobufUtil.toCell(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY), cell); assertTrue(CellComparatorImpl.COMPARATOR.compare(offheapKV, newOffheapKV) == 0); } }
/** * Convert list of KeyValues to byte buffer. * @param keyValues list of KeyValues to be converted. * @return buffer with content from key values */ public static ByteBuffer convertKvToByteBuffer(List<KeyValue> keyValues, boolean includesMemstoreTS) { int totalSize = 0; for (KeyValue kv : keyValues) { totalSize += kv.getLength(); if (includesMemstoreTS) { totalSize += WritableUtils.getVIntSize(kv.getSequenceId()); } } ByteBuffer result = ByteBuffer.allocate(totalSize); for (KeyValue kv : keyValues) { result.put(kv.getBuffer(), kv.getOffset(), kv.getLength()); if (includesMemstoreTS) { ByteBufferUtils.writeVLong(result, kv.getSequenceId()); } } return result; }
KeyValue codecKv = KeyValueUtil.ensureKeyValue(c); if (codecKv == null || 0 != Bytes.compareTo( codecKv.getBuffer(), codecKv.getOffset(), codecKv.getLength(), currentKv.getBuffer(), currentKv.getOffset(), currentKv.getLength())) { if (codecKv == null) { Math.min(codecKv.getLength(), currentKv.getLength()); while (prefix < limitLength && codecKv.getBuffer()[prefix + codecKv.getOffset()] == currentKv.getBuffer()[prefix + currentKv.getOffset()]) { prefix++; "\n prefix " + prefix + "\n codecKv '" + Bytes.toStringBinary(codecKv.getBuffer(), codecKv.getOffset(), prefix) + "' diff '" + Bytes.toStringBinary(codecKv.getBuffer(), codecKv.getOffset() + prefix, codecKv.getLength() - prefix) + "'" + "\n currentKv '" + Bytes.toStringBinary( currentKv.getBuffer(), currentKv.getOffset(), prefix) + "' diff '" + Bytes.toStringBinary(currentKv.getBuffer(), currentKv.getOffset() + prefix, currentKv.getLength() - prefix) + "'" );
/** * Get the offset into the Result byte array to the key. * @param r */ static int getKeyOffset(Result r) { KeyValue firstKV = PhoenixKeyValueUtil.maybeCopyCell(r.rawCells()[0]); return firstKV.getOffset(); }
/** * Get the offset into the Result byte array to the key. * @param r * @return */ static int getKeyOffset(Result r) { // Special case for when Result was instantiated via KeyValue array (no bytes in that case) versus returned from a scanner return (r.getBytes() == null ? r.raw()[0].getOffset() : (r.getBytes().getOffset() + Bytes.SIZEOF_INT /* KV length in Result */)) + KeyValue.ROW_OFFSET /* key offset in KV */ + Bytes.SIZEOF_SHORT /* key length */; }
public static int write(Tuple result, DataOutput out) throws IOException { int size = 0; for(int i = 0; i < result.size(); i++) { KeyValue kv = result.getValue(i); size += kv.getLength(); size += Bytes.SIZEOF_INT; // kv.getLength } WritableUtils.writeVInt(out, size); for(int i = 0; i < result.size(); i++) { KeyValue kv = result.getValue(i); out.writeInt(kv.getLength()); out.write(kv.getBuffer(), kv.getOffset(), kv.getLength()); } return size; } }
public static int write(Tuple result, DataOutput out) throws IOException { int size = 0; for(int i = 0; i < result.size(); i++) { KeyValue kv = PhoenixKeyValueUtil.maybeCopyCell(result.getValue(i)); size += kv.getLength(); size += Bytes.SIZEOF_INT; // kv.getLength } WritableUtils.writeVInt(out, size); for(int i = 0; i < result.size(); i++) { KeyValue kv = PhoenixKeyValueUtil.maybeCopyCell(result.getValue(i)); out.writeInt(kv.getLength()); out.write(kv.getBuffer(), kv.getOffset(), kv.getLength()); } return size; } }
@SuppressWarnings("deprecation") @Override protected void writeToBuffer(MappedByteBuffer buffer, Tuple e) { KeyValue kv = PhoenixKeyValueUtil.maybeCopyCell(e.getValue(0)); buffer.putInt(kv.getLength() + Bytes.SIZEOF_INT); buffer.putInt(kv.getLength()); buffer.put(kv.getBuffer(), kv.getOffset(), kv.getLength()); }
@Test public void testToCell() throws Exception { KeyValue kv1 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); KeyValue kv2 = new KeyValue(Bytes.toBytes("bbb"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); KeyValue kv3 = new KeyValue(Bytes.toBytes("ccc"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); byte[] arr = new byte[kv1.getLength() + kv2.getLength() + kv3.getLength()]; System.arraycopy(kv1.getBuffer(), kv1.getOffset(), arr, 0, kv1.getLength()); System.arraycopy(kv2.getBuffer(), kv2.getOffset(), arr, kv1.getLength(), kv2.getLength()); System.arraycopy(kv3.getBuffer(), kv3.getOffset(), arr, kv1.getLength() + kv2.getLength(), kv3.getLength()); ByteBuffer dbb = ByteBuffer.allocateDirect(arr.length); dbb.put(arr); ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, kv1.getLength(), kv2.getLength()); CellProtos.Cell cell = ProtobufUtil.toCell(offheapKV); Cell newOffheapKV = ProtobufUtil.toCell(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY), cell); assertTrue(CellComparatorImpl.COMPARATOR.compare(offheapKV, newOffheapKV) == 0); }
RowKeyValueAccessor accessor = new RowKeyValueAccessor(pkColumns, 3); int offset = accessor.getOffset(keyValue.getRowArray(), keyValue.getRowOffset()); int length = accessor.getLength(keyValue.getRowArray(), offset, keyValue.getOffset()+keyValue.getLength()); ImmutableBytesWritable ptr = new ImmutableBytesWritable(keyValue.getRowArray(), offset, length);
@Override protected void writeToBuffer(MappedByteBuffer buffer, ResultEntry e) { int totalLen = 0; List<KeyValue> keyValues = toKeyValues(e); for (KeyValue kv : keyValues) { totalLen += (kv.getLength() + Bytes.SIZEOF_INT); } buffer.putInt(totalLen); for (KeyValue kv : keyValues) { buffer.putInt(kv.getLength()); buffer.put(kv.getBuffer(), kv.getOffset(), kv .getLength()); } ImmutableBytesWritable[] sortKeys = e.sortKeys; buffer.putInt(sortKeys.length); for (ImmutableBytesWritable sortKey : sortKeys) { if (sortKey != null) { buffer.putInt(sortKey.getLength()); buffer.put(sortKey.get(), sortKey.getOffset(), sortKey.getLength()); } else { buffer.putInt(0); } } }