/** * Do not use unless you have to. Used internally for compacting and testing. Use * {@link #getRowArray()}, {@link #getFamilyArray()}, {@link #getQualifierArray()}, and * {@link #getValueArray()} if accessing a KeyValue client-side. * @return Copy of the key portion only. */ public byte [] getKey() { int keylength = getKeyLength(); byte [] key = new byte[keylength]; System.arraycopy(getBuffer(), getKeyOffset(), key, 0, keylength); return key; }
/** * Write out a KeyValue in the manner in which we used to when KeyValue was a Writable. * @param kv * @param out * @return Length written on stream * @throws IOException * @see #create(DataInput) for the inverse function */ public static long write(final KeyValue kv, final DataOutput out) throws IOException { // This is how the old Writables write used to serialize KVs. Need to figure way to make it // work for all implementations. int length = kv.getLength(); out.writeInt(length); out.write(kv.getBuffer(), kv.getOffset(), length); return (long) length + Bytes.SIZEOF_INT; }
/** * @return True if this KeyValue has a LATEST_TIMESTAMP timestamp. */ public boolean isLatestTimestamp() { return Bytes.equals(getBuffer(), getTimestampOffset(), Bytes.SIZEOF_LONG, HConstants.LATEST_TIMESTAMP_BYTES, 0, Bytes.SIZEOF_LONG); }
/** * Write out a KeyValue in the manner in which we used to when KeyValue was a * Writable. * * @param kv * @param out * @return Length written on stream * @throws IOException * @see #create(DataInput) for the inverse function */ public static long write(final KeyValue kv, final DataOutput out) throws IOException { // This is how the old Writables write used to serialize KVs. Need to figure // way to make it // work for all implementations. int length = kv.getLength(); out.writeInt(length); out.write(kv.getBuffer(), kv.getOffset(), length); return (long) length + Bytes.SIZEOF_INT; }
private static Cell getOffheapCell(byte [] row, byte [] family, byte [] qualifier) { KeyValue kvCell = new KeyValue(row, family, qualifier, 0L, Type.Put, row); ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length); ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length); return new ByteBufferKeyValue(buf, 0, buf.capacity(), 0L); }
private Cell createOffheapKVWithTags(int noOfTags) { List<Tag> tags = new ArrayList<>(); for (int i = 0; i < noOfTags; i++) { tags.add(new ArrayBackedTag((byte) i, "tagValue" + i)); } KeyValue kv = new KeyValue(ROW, CF, Q, 1234L, V, tags); ByteBuffer dbb = ByteBuffer.allocateDirect(kv.getBuffer().length); ByteBufferUtils.copyFromArrayToBuffer(dbb, kv.getBuffer(), 0, kv.getBuffer().length); ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, 0, kv.getBuffer().length, 0); return offheapKV; } }
private static Cell createByteBufferKeyValueFromKeyValue(KeyValue kv) { ByteBuffer bb = ByteBuffer.wrap(kv.getBuffer()); return new ByteBufferKeyValue(bb, 0, bb.remaining()); }
public String getKeyString() { return Bytes.toStringBinary(getBuffer(), getKeyOffset(), getKeyLength()); }
private ByteBufferKeyValue createOffheapKV(int noOfTags) { byte[] row = Bytes.toBytes("myRow"); byte[] cf = Bytes.toBytes("myCF"); byte[] q = Bytes.toBytes("myQualifier"); byte[] value = Bytes.toBytes("myValue"); List<Tag> tags = new ArrayList<>(noOfTags); for (int i = 1; i <= noOfTags; i++) { tags.add(new ArrayBackedTag((byte) i, Bytes.toBytes("tagValue" + i))); } KeyValue kv = new KeyValue(row, cf, q, HConstants.LATEST_TIMESTAMP, value, tags); ByteBuffer dbb = ByteBuffer.allocateDirect(kv.getBuffer().length); dbb.put(kv.getBuffer()); return new ByteBufferKeyValue(dbb, 0, kv.getBuffer().length); } }
public static void appendToByteBuffer(final ByteBuffer bb, final KeyValue kv, final boolean includeMvccVersion) { // keep pushing the limit out. assume enough capacity bb.limit(bb.position() + kv.getLength()); bb.put(kv.getBuffer(), kv.getOffset(), kv.getLength()); if (includeMvccVersion) { int numMvccVersionBytes = WritableUtils.getVIntSize(kv.getSequenceId()); ByteBufferUtils.extendLimit(bb, numMvccVersionBytes); ByteBufferUtils.writeVLong(bb, kv.getSequenceId()); } }
@Test public void testKeyValue() throws Exception { final String name = "testKeyValue2"; byte[] row = name.getBytes(); byte[] fam = "fam".getBytes(); byte[] qf = "qf".getBytes(); long ts = System.currentTimeMillis(); byte[] val = "val".getBytes(); KeyValue kv = new KeyValue(row, fam, qf, ts, val); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(baos); long l = KeyValueUtil.write(kv, dos); dos.close(); byte [] mb = baos.toByteArray(); ByteArrayInputStream bais = new ByteArrayInputStream(mb); DataInputStream dis = new DataInputStream(bais); KeyValue deserializedKv = KeyValueUtil.create(dis); assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer())); assertEquals(kv.getOffset(), deserializedKv.getOffset()); assertEquals(kv.getLength(), deserializedKv.getLength()); }
@Test public void testCreateKeyValueFromKey() { KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("myValue")); int initialPadding = 10; int endingPadding = 20; int keyLen = kv.getKeyLength(); byte[] tmpArr = new byte[initialPadding + endingPadding + keyLen]; System.arraycopy(kv.getBuffer(), kv.getKeyOffset(), tmpArr, initialPadding, keyLen); KeyValue kvFromKey = KeyValueUtil.createKeyValueFromKey(tmpArr, initialPadding, keyLen); assertEquals(keyLen, kvFromKey.getKeyLength()); assertEquals(KeyValue.ROW_OFFSET + keyLen, kvFromKey.getBuffer().length); System.err.println("kv=" + kv); System.err.println("kvFromKey=" + kvFromKey); assertEquals(kvFromKey.toString(), kv.toString().replaceAll("=[0-9]+", "=0")); }
@Test public void testToCell() throws Exception { KeyValue kv1 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); KeyValue kv2 = new KeyValue(Bytes.toBytes("bbb"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); KeyValue kv3 = new KeyValue(Bytes.toBytes("ccc"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); byte[] arr = new byte[kv1.getLength() + kv2.getLength() + kv3.getLength()]; System.arraycopy(kv1.getBuffer(), kv1.getOffset(), arr, 0, kv1.getLength()); System.arraycopy(kv2.getBuffer(), kv2.getOffset(), arr, kv1.getLength(), kv2.getLength()); System.arraycopy(kv3.getBuffer(), kv3.getOffset(), arr, kv1.getLength() + kv2.getLength(), kv3.getLength()); ByteBuffer dbb = ByteBuffer.allocateDirect(arr.length); dbb.put(arr); ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, kv1.getLength(), kv2.getLength()); CellProtos.Cell cell = ProtobufUtil.toCell(offheapKV); Cell newOffheapKV = ProtobufUtil.toCell(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY), cell); assertTrue(CellComparatorImpl.COMPARATOR.compare(offheapKV, newOffheapKV) == 0); }
@Test public void testToCell() throws Exception { KeyValue kv1 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); KeyValue kv2 = new KeyValue(Bytes.toBytes("bbb"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); KeyValue kv3 = new KeyValue(Bytes.toBytes("ccc"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); byte[] arr = new byte[kv1.getLength() + kv2.getLength() + kv3.getLength()]; System.arraycopy(kv1.getBuffer(), kv1.getOffset(), arr, 0, kv1.getLength()); System.arraycopy(kv2.getBuffer(), kv2.getOffset(), arr, kv1.getLength(), kv2.getLength()); System.arraycopy(kv3.getBuffer(), kv3.getOffset(), arr, kv1.getLength() + kv2.getLength(), kv3.getLength()); ByteBuffer dbb = ByteBuffer.allocateDirect(arr.length); dbb.put(arr); ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, kv1.getLength(), kv2.getLength()); CellProtos.Cell cell = ProtobufUtil.toCell(offheapKV); Cell newOffheapKV = ProtobufUtil.toCell(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY), cell); assertTrue(CellComparatorImpl.COMPARATOR.compare(offheapKV, newOffheapKV) == 0); } }
@Test public void testCloneCellFieldsFromByteBufferedCell() { byte[] r = Bytes.toBytes("row1"); byte[] f = Bytes.toBytes("cf1"); byte[] q = Bytes.toBytes("qual1"); byte[] v = Bytes.toBytes("val1"); byte[] tags = Bytes.toBytes("tag1"); KeyValue kv = new KeyValue(r, f, q, 0, q.length, 1234L, KeyValue.Type.Put, v, 0, v.length, tags); ByteBuffer buffer = ByteBuffer.wrap(kv.getBuffer()); Cell bbCell = new ByteBufferKeyValue(buffer, 0, buffer.remaining()); byte[] rDest = CellUtil.cloneRow(bbCell); assertTrue(Bytes.equals(r, rDest)); byte[] fDest = CellUtil.cloneFamily(bbCell); assertTrue(Bytes.equals(f, fDest)); byte[] qDest = CellUtil.cloneQualifier(bbCell); assertTrue(Bytes.equals(q, qDest)); byte[] vDest = CellUtil.cloneValue(bbCell); assertTrue(Bytes.equals(v, vDest)); byte[] tDest = new byte[tags.length]; PrivateCellUtil.copyTagsTo(bbCell, tDest, 0); assertTrue(Bytes.equals(tags, tDest)); }
private void nullFilterTests(Filter filter) throws Exception { ((SingleColumnValueFilter) filter).setFilterIfMissing(true); KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_1); assertTrue("null1", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); byte[] buffer = cell.getBuffer(); Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("null1", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); assertFalse("null1FilterRow", filter.filterRow()); filter.reset(); cell = new KeyValue(ROW, COLUMN_FAMILY, Bytes.toBytes("qual2"), FULLSTRING_2); assertTrue("null2", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); buffer = cell.getBuffer(); c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("null2", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); assertTrue("null2FilterRow", filter.filterRow()); }
private void substrFilterTests(Filter filter) throws Exception { KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_1); assertTrue("substrTrue", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); byte[] buffer = cell.getBuffer(); Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("substrTrue", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_2); assertTrue("substrFalse", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); buffer = cell.getBuffer(); c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("substrFalse", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); assertFalse("substrFilterAllRemaining", filter.filterAllRemaining()); assertFalse("substrFilterNotNull", filter.filterRow()); }
private void regexPatternFilterTests(Filter filter) throws Exception { KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_1); assertTrue("regexTrue", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); byte[] buffer = cell.getBuffer(); Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("regexTrue", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); assertFalse("regexFilterAllRemaining", filter.filterAllRemaining()); assertFalse("regexFilterNotNull", filter.filterRow()); }
private void regexFilterTests(Filter filter) throws Exception { KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_1); assertTrue("regexTrue", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); byte[] buffer = cell.getBuffer(); Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("regexTrue", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_2); assertTrue("regexFalse", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); buffer = cell.getBuffer(); c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("regexFalse", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); assertFalse("regexFilterAllRemaining", filter.filterAllRemaining()); assertFalse("regexFilterNotNull", filter.filterRow()); }
static void writeCell(PositionedByteRange pbr, KeyValue kv) throws Exception { pbr.putInt(kv.getKeyLength()); pbr.putInt(kv.getValueLength()); pbr.put(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength()); pbr.put(kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()); int tagsLen = kv.getTagsLength(); pbr.put((byte) (tagsLen >> 8 & 0xff)); pbr.put((byte) (tagsLen & 0xff)); pbr.put(kv.getTagsArray(), kv.getTagsOffset(), tagsLen); pbr.putVLong(kv.getSequenceId()); }