private static Cell convertKv(Cell kv, Map<byte[], byte[]> cfRenameMap) { if(cfRenameMap != null) { // If there's a rename mapping for this CF, create a new KeyValue byte[] newCfName = cfRenameMap.get(CellUtil.cloneFamily(kv)); if (newCfName != null) { kv = new KeyValue(kv.getRowArray(), // row buffer kv.getRowOffset(), // row offset kv.getRowLength(), // row length newCfName, // CF buffer 0, // CF offset newCfName.length, // CF length kv.getQualifierArray(), // qualifier buffer kv.getQualifierOffset(), // qualifier offset kv.getQualifierLength(), // qualifier length kv.getTimestamp(), // timestamp KeyValue.Type.codeToType(kv.getTypeByte()), // KV Type kv.getValueArray(), // value buffer kv.getValueOffset(), // value offset kv.getValueLength()); // value length } } return kv; }
Scan s = new Scan(Bytes.toBytes(startkey)); s.setCaching(recordcount); if (this.usePageFilter) { s.setFilter(new PageFilter(recordcount)); } else { for (String field : fields) { s.addColumn(columnFamilyBytes, Bytes.toBytes(field)); for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { String key = Bytes.toString(rr.getRow()); if (debug) { System.out.println("Got scan result for key: " + key); for (KeyValue kv : rr.raw()) { rowResult.put( Bytes.toString(kv.getQualifier()), new ByteArrayByteIterator(kv.getValue()));
/** * {@inheritDoc} */ @Override public void put(Put put) throws IOException { byte[] row = put.getRow(); NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowData = forceFind(data, row, new TreeMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>>(Bytes.BYTES_COMPARATOR)); for (byte[] family : put.getFamilyMap().keySet()) { if (columnFamilies.contains(new String(family, StandardCharsets.UTF_8)) == false) { throw new RuntimeException("Not Exists columnFamily : " + new String(family, StandardCharsets.UTF_8)); } NavigableMap<byte[], NavigableMap<Long, byte[]>> familyData = forceFind(rowData, family, new TreeMap<byte[], NavigableMap<Long, byte[]>>(Bytes.BYTES_COMPARATOR)); for (KeyValue kv : put.getFamilyMap().get(family)) { kv.updateLatestStamp(Bytes.toBytes(System.currentTimeMillis())); byte[] qualifier = kv.getQualifier(); NavigableMap<Long, byte[]> qualifierData = forceFind(familyData, qualifier, new TreeMap<Long, byte[]>()); qualifierData.put(kv.getTimestamp(), kv.getValue()); } } }
/** * * @return Timestamp */ @Override public long getTimestamp() { return getTimestamp(getKeyLength()); }
/** * @return Qualifier offset */ @Override public int getQualifierOffset() { return getQualifierOffset(getFamilyOffset()); }
/** * @return Qualifier length */ @Override public int getQualifierLength() { return getQualifierLength(getRowLength(),getFamilyLength()); }
public void testHBaseSerDeWithBackwardEvolvedSchema() throws SerDeException, IOException { byte[] cfa = "cola".getBytes(); byte[] qualAvro = "avro".getBytes(); byte[] rowKey = Bytes.toBytes("test-row1"); // Data List<Cell> kvs = new ArrayList<Cell>(); byte[] avroData = getTestAvroBytesFromSchema(RECORD_SCHEMA_EVOLVED); kvs.add(new KeyValue(rowKey, cfa, qualAvro, avroData)); Result r = Result.create(kvs); Put p = new Put(rowKey); // Post serialization, separators are automatically inserted between different fields in the // struct. Currently there is not way to disable that. So the work around here is to pad the // data with the separator bytes before creating a "Put" object p.add(new KeyValue(rowKey, cfa, qualAvro, avroData)); Object[] expectedFieldsData = {new String("test-row1"), new String("[[42, true, 42432234234]]")}; // Create, initialize, and test the SerDe HBaseSerDe serDe = new HBaseSerDe(); Configuration conf = new Configuration(); Properties tbl = createPropertiesForHiveAvroBackwardEvolvedSchema(); serDe.initialize(conf, tbl); deserializeAndSerializeHiveAvro(serDe, r, p, expectedFieldsData, EXPECTED_DESERIALIZED_AVRO_STRING); }
final TableName tableName = TableName.valueOf(name.getMethodName()); long l = System.currentTimeMillis(); Admin admin = TEST_UTIL.getAdmin(); createTable(admin, tableName); Scan scan = createScan(); Configuration conf = TEST_UTIL.getConfiguration(); conf.setBoolean("hbase.mapreduce.bulkload.assign.sequenceNumbers", true); final LoadIncrementalHFiles bulkload = new LoadIncrementalHFiles(conf); try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { bulkload.doBulkLoad(hfilePath, admin, table, locator); Result result = scanner.next(); result = scanAfterBulkLoad(scanner, result, "version2"); Put put0 = new Put(Bytes.toBytes("row1")); put0.add(new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("col"), Bytes.toBytes("q"), l, Bytes .toBytes("version3"))); table.put(put0); result = scanner.next(); while (result != null) { List<Cell> cells = result.getColumnCells(Bytes.toBytes("col"), Bytes.toBytes("q")); for (Cell _c : cells) { if (Bytes.toString(_c.getRowArray(), _c.getRowOffset(), _c.getRowLength()) .equals("row1")) { System.out
/** * Test GroupingTableMapper class */ @Test public void testGroupingTableMapper() throws Exception { GroupingTableMapper mapper = new GroupingTableMapper(); Configuration configuration = new Configuration(); configuration.set(GroupingTableMapper.GROUP_COLUMNS, "family1:clm family2:clm"); mapper.setConf(configuration); Result result = mock(Result.class); @SuppressWarnings("unchecked") Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Result>.Context context = mock(Mapper.Context.class); context.write(any(), any()); List<Cell> keyValue = new ArrayList<>(); byte[] row = {}; keyValue.add(new KeyValue(row, Bytes.toBytes("family2"), Bytes.toBytes("clm"), Bytes .toBytes("value1"))); keyValue.add(new KeyValue(row, Bytes.toBytes("family1"), Bytes.toBytes("clm"), Bytes .toBytes("value2"))); when(result.listCells()).thenReturn(keyValue); mapper.map(null, result, context); // template data byte[][] data = { Bytes.toBytes("value1"), Bytes.toBytes("value2") }; ImmutableBytesWritable ibw = mapper.createGroupKey(data); verify(context).write(ibw, result); }
public void testHBaseSerDeCustomStructValue() throws IOException, SerDeException { byte[] cfa = "cola".getBytes(); byte[] qualStruct = "struct".getBytes(); TestStruct testStruct = new TestStruct("A", "B", "C", false, (byte) 0); byte[] key = testStruct.getBytes(); // Data List<Cell> kvs = new ArrayList<Cell>(); byte[] testData = testStruct.getBytes(); kvs.add(new KeyValue(key, cfa, qualStruct, testData)); Result r = Result.create(kvs); byte[] putKey = testStruct.getBytesWithDelimiters(); Put p = new Put(putKey); // Post serialization, separators are automatically inserted between different fields in the // struct. Currently there is not way to disable that. So the work around here is to pad the // data with the separator bytes before creating a "Put" object p.add(new KeyValue(putKey, cfa, qualStruct, Bytes.padTail(testData, 2))); // Create, initialize, and test the SerDe HBaseSerDe serDe = new HBaseSerDe(); Configuration conf = new Configuration(); Properties tbl = createPropertiesForValueStruct(); serDe.initialize(conf, tbl); deserializeAndSerializeHBaseValueStruct(serDe, r, p); }
@Test public void testMultipleVersionsSimple() throws Exception { DefaultMemStore m = new DefaultMemStore(new Configuration(), CellComparatorImpl.COMPARATOR); byte [] row = Bytes.toBytes("testRow"); byte [] family = Bytes.toBytes("testFamily"); byte [] qf = Bytes.toBytes("testQualifier"); long [] stamps = {1,2,3}; byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"), Bytes.toBytes("value2")}; KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]); KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]); KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]); m.add(key0, null); m.add(key1, null); m.add(key2, null); assertTrue("Expected memstore to hold 3 values, actually has " + m.getActive().getCellsCount(), m.getActive().getCellsCount() == 3); }
@Test public void testResultSizeEstimation() throws Exception { byte [] ROW1 = Bytes.toBytes("testRow1"); byte [] ROW2 = Bytes.toBytes("testRow2"); byte [] FAMILY = Bytes.toBytes("testFamily"); byte [] QUALIFIER = Bytes.toBytes("testQualifier"); byte [] VALUE = Bytes.toBytes("testValue"); final TableName tableName = TableName.valueOf(name.getMethodName()); byte[][] FAMILIES = new byte[][] { FAMILY }; Table table = TEST_UTIL.createTable(tableName, FAMILIES); Put p = new Put(ROW1); p.add(new KeyValue(ROW1, FAMILY, QUALIFIER, Long.MAX_VALUE, VALUE)); table.put(p); p = new Put(ROW2); p.add(new KeyValue(ROW2, FAMILY, QUALIFIER, Long.MAX_VALUE, VALUE)); table.put(p); Scan s = new Scan(); s.setMaxResultSize(SCANNER_DATA_LIMIT); ResultScanner rs = table.getScanner(s); int count = 0; while(rs.next() != null) { count++; } assertEquals("Result size estimation did not work properly", 2, count); rs.close(); table.close(); }
@Test public void testReverseScanner_FromMemStore_SingleCF_LargerKey() throws IOException { byte[] rowC = Bytes.toBytes("rowC"); byte[] rowA = Bytes.toBytes("rowA"); byte[] rowB = Bytes.toBytes("rowB"); long ts = 1; this.region = initHRegion(tableName, method, families); KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); Put put = null; put = new Put(rowC); put.add(kv1); put.add(kv11); region.put(put);
@Override public void process(long now, HRegion region, List<Mutation> mutations, WALEdit walEdit) throws IOException { // Scan current counter List<Cell> kvs = new ArrayList<>(); Scan scan = new Scan(row, row); scan.addColumn(FAM, COUNTER); doScan(region, scan, kvs); counter = kvs.isEmpty() ? 0 : Bytes.toInt(CellUtil.cloneValue(kvs.iterator().next())); // Assert counter value assertEquals(expectedCounter, counter); // Increment counter and send it to both memstore and wal edit counter += 1; expectedCounter += 1; Put p = new Put(row); KeyValue kv = new KeyValue(row, FAM, COUNTER, now, Bytes.toBytes(counter)); p.add(kv); mutations.add(p); walEdit.add(kv); // We can also inject some meta data to the walEdit KeyValue metaKv = new KeyValue( row, WALEdit.METAFAMILY, Bytes.toBytes("I just increment counter"), Bytes.toBytes(counter)); walEdit.add(metaKv); }
@Test public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException { byte[] row1 = Bytes.toBytes("row1"); byte[] qf1 = Bytes.toBytes("qualifier1"); byte[] qf2 = Bytes.toBytes("qualifier2"); byte[] fam1 = Bytes.toBytes("fam1"); byte[][] families = { fam1 }; KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); put = new Put(row1); put.add(kv13); put.add(kv12); put.add(kv11); put.add(kv23); expected.add(kv12); Scan scan = new Scan(row1); scan.addColumn(fam1, qf1); scan.setMaxVersions(MAX_VERSIONS); List<Cell> actual = new ArrayList<>(); InternalScanner scanner = region.getScanner(scan);
@Test public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException { byte[] row1 = Bytes.toBytes("row1"); byte[] qf1 = Bytes.toBytes("qualifier1"); byte[] qf2 = Bytes.toBytes("qualifier2"); byte[] fam1 = Bytes.toBytes("fam1"); byte[][] families = { fam1 }; KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); put = new Put(row1); put.add(kv13); put.add(kv12); put.add(kv11); put.add(kv23); expected.add(kv22); Scan scan = new Scan(row1); scan.addFamily(fam1); scan.setMaxVersions(MAX_VERSIONS); List<Cell> actual = new ArrayList<>(); InternalScanner scanner = region.getScanner(scan);
private Table init(Admin admin, long l, Scan scan, TableName tableName) throws Exception { Table table = TEST_UTIL.getConnection().getTable(tableName); Put put0 = new Put(Bytes.toBytes("row1")); put0.add(new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("col"), Bytes.toBytes("q"), l, Bytes.toBytes("version0"))); table.put(put0); admin.flush(tableName); Put put1 = new Put(Bytes.toBytes("row2")); put1.add(new KeyValue(Bytes.toBytes("row2"), Bytes.toBytes("col"), Bytes.toBytes("q"), l, Bytes .toBytes("version0"))); table.put(put1); admin.flush(tableName); put0 = new Put(Bytes.toBytes("row1")); put0.add(new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("col"), Bytes.toBytes("q"), l, Bytes .toBytes("version1"))); table.put(put0); admin.flush(tableName); admin.compact(tableName); ResultScanner scanner = table.getScanner(scan); Result result = scanner.next(); List<Cell> cells = result.getColumnCells(Bytes.toBytes("col"), Bytes.toBytes("q")); Assert.assertEquals(1, cells.size()); Cell _c = cells.get(0); Assert.assertEquals("version1", Bytes.toString(_c.getValueArray(), _c.getValueOffset(), _c.getValueLength())); scanner.close(); return table; }
@Test public void testMaxResultSizeIsSetToDefault() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); Table ht = TEST_UTIL.createTable(tableName, FAMILY); TEST_UTIL.getConfiguration().getLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE); byte[] cellValue = Bytes.createMaxByteArray(cellSize); put = new Put(ROWS[row]); for (int qual = 0; qual < QUALIFIERS.length; qual++) { KeyValue kv = new KeyValue(ROWS[row], FAMILY, QUALIFIERS[qual], cellValue); put.add(kv); Scan scan = new Scan();
@SuppressWarnings("deprecation") @Test @Ignore("not maintaining") public void testMapper2() throws IOException { mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name); mapDriver.addInput(new Text("52010tech"), new Text("35.432")); List<Pair<RowKeyWritable, KeyValue>> result = mapDriver.run(); assertEquals(2, result.size()); byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 }; ImmutableBytesWritable key = new ImmutableBytesWritable(bytes); Pair<RowKeyWritable, KeyValue> p1 = result.get(0); Pair<RowKeyWritable, KeyValue> p2 = result.get(1); assertEquals(key, p1.getFirst()); assertEquals("cf1", new String(p1.getSecond().getFamily(), StandardCharsets.UTF_8)); assertEquals("usd_amt", new String(p1.getSecond().getQualifier(), StandardCharsets.UTF_8)); assertEquals("35.43", new String(p1.getSecond().getValue(), StandardCharsets.UTF_8)); assertEquals(key, p2.getFirst()); assertEquals("cf1", new String(p2.getSecond().getFamily(), StandardCharsets.UTF_8)); assertEquals("item_count", new String(p2.getSecond().getQualifier(), StandardCharsets.UTF_8)); assertEquals("2", new String(p2.getSecond().getValue(), StandardCharsets.UTF_8)); } }
@Test public void testPutIteration() throws IOException { Put p = new Put(ROW); for (int i = 0; i < COUNT; i++) { byte [] bytes = Bytes.toBytes(i); p.addColumn(bytes, bytes, TIMESTAMP, bytes); } int index = 0; for (CellScanner cellScanner = p.cellScanner(); cellScanner.advance();) { Cell cell = cellScanner.current(); byte [] bytes = Bytes.toBytes(index++); cell.equals(new KeyValue(ROW, bytes, bytes, TIMESTAMP, bytes)); } assertEquals(COUNT, index); }