@SuppressWarnings("deprecation") @Test @Ignore("not maintaining") public void testMapper2() throws IOException { mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name); mapDriver.addInput(new Text("52010tech"), new Text("35.432")); List<Pair<RowKeyWritable, KeyValue>> result = mapDriver.run(); assertEquals(2, result.size()); byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 }; ImmutableBytesWritable key = new ImmutableBytesWritable(bytes); Pair<RowKeyWritable, KeyValue> p1 = result.get(0); Pair<RowKeyWritable, KeyValue> p2 = result.get(1); assertEquals(key, p1.getFirst()); assertEquals("cf1", new String(p1.getSecond().getFamily(), StandardCharsets.UTF_8)); assertEquals("usd_amt", new String(p1.getSecond().getQualifier(), StandardCharsets.UTF_8)); assertEquals("35.43", new String(p1.getSecond().getValue(), StandardCharsets.UTF_8)); assertEquals(key, p2.getFirst()); assertEquals("cf1", new String(p2.getSecond().getFamily(), StandardCharsets.UTF_8)); assertEquals("item_count", new String(p2.getSecond().getQualifier(), StandardCharsets.UTF_8)); assertEquals("2", new String(p2.getSecond().getValue(), StandardCharsets.UTF_8)); } }
/** * * @return next element of HbaseData. */ public HBaseData next() { HBaseData data = null; Result result = resultsIter.next(); counter++; List<KeyValue> values = result.list(); for (KeyValue value : values) { data = new HBaseData(new String(value.getFamily()), value.getRow()); break; } data.setColumns(values); return data; }
public boolean isRowDeleted(Collection<KeyValue> pendingUpdates) { int nDeleteCF = 0; for (KeyValue kv : pendingUpdates) { if (kv.getType() == KeyValue.Type.DeleteFamily.getCode()) { nDeleteCF++; boolean isEmptyCF = Bytes.compareTo(kv.getFamily(), dataEmptyKeyValueCF) == 0; // This is what a delete looks like on the client side for immutable indexing... if (isEmptyCF) { return true; } } } // This is what a delete looks like on the server side for mutable indexing... return nDeleteCF == this.nDataCFs; }
for (KeyValue value : values) data = new HBaseData(columnFamily != null ? columnFamily : new String(value.getFamily()), value.getRow()); break;
data = new HBaseData(columnFamily != null ? columnFamily : new String(value.getFamily()), value.getRow()); break;
/** * check to see if the kvs in the update match any of the passed columns. Generally, this is useful to for an index * codec to determine if a given update should even be indexed. This assumes that for any index, there are going to * small number of columns, versus the number of kvs in any one batch. */ public static boolean updateMatchesColumns(Collection<KeyValue> update, List<ColumnReference> columns) { // check to see if the kvs in the new update even match any of the columns requested // assuming that for any index, there are going to small number of columns, versus the number of // kvs in any one batch. boolean matches = false; outer: for (KeyValue kv : update) { for (ColumnReference ref : columns) { if (ref.matchesFamily(kv.getFamily()) && ref.matchesQualifier(kv.getQualifier())) { matches = true; // if a single column matches a single kv, we need to build a whole scanner break outer; } } } return matches; }
/** * Check to see if the kvs in the update match any of the passed columns. Generally, this is useful to for an index * codec to determine if a given update should even be indexed. This assumes that for any index, there are going to * small number of kvs, versus the number of columns in any one batch. * <p> * This employs the same logic as {@link #updateMatchesColumns(Collection, List)}, but is flips the iteration logic * to search columns before kvs. */ public static boolean columnMatchesUpdate(List<ColumnReference> columns, Collection<KeyValue> update) { boolean matches = false; outer: for (ColumnReference ref : columns) { for (KeyValue kv : update) { if (ref.matchesFamily(kv.getFamily()) && ref.matchesQualifier(kv.getQualifier())) { matches = true; // if a single column matches a single kv, we need to build a whole scanner break outer; } } } return matches; }
private boolean hasIndexedColumnChanged(ValueGetter oldState, Collection<KeyValue> pendingUpdates) throws IOException { if (pendingUpdates.isEmpty()) { return false; } Map<ColumnReference,KeyValue> newState = Maps.newHashMapWithExpectedSize(pendingUpdates.size()); for (KeyValue kv : pendingUpdates) { newState.put(new ColumnReference(kv.getFamily(), kv.getQualifier()), kv); } for (ColumnReference ref : indexedColumns) { KeyValue newValue = newState.get(ref); if (newValue != null) { // Indexed column was potentially changed ImmutableBytesPtr oldValue = oldState.getLatestValue(ref); // If there was no old value or the old value is different than the new value, the index row needs to be deleted if (oldValue == null || Bytes.compareTo(oldValue.get(), oldValue.getOffset(), oldValue.getLength(), newValue.getBuffer(), newValue.getValueOffset(), newValue.getValueLength()) != 0){ return true; } } } return false; }
public Increment newIncrement(long timestamp) { Increment inc = new Increment(SchemaUtil.getSequenceKey(key.getTenantId(), key.getSchemaName(), key.getSequenceName())); // It doesn't matter what we set the amount too - we always use the values we get // from the Get we do to prevent any race conditions. All columns that get added // are returned with their current value try { inc.setTimeRange(MetaDataProtocol.MIN_TABLE_TIMESTAMP, timestamp); } catch (IOException e) { throw new RuntimeException(e); // Impossible } for (KeyValue kv : SEQUENCE_KV_COLUMNS) { // We don't care about the amount, as we'll add what gets looked up on the server-side inc.addColumn(kv.getFamily(), kv.getQualifier(), AMOUNT); } return inc; }
for (KeyValue value : result.list()) data = new HBaseData(columnFamily != null ? columnFamily : new String(value.getFamily()), value.getRow()); break;
@SuppressWarnings("deprecation") public Delete buildDeleteMutation(ValueGetter oldState, ImmutableBytesWritable dataRowKeyPtr, Collection<KeyValue> pendingUpdates, long ts) throws IOException { byte[] indexRowKey = this.buildRowKey(oldState, dataRowKeyPtr); // Delete the entire row if any of the indexed columns changed if (oldState == null || isRowDeleted(pendingUpdates) || hasIndexedColumnChanged(oldState, pendingUpdates)) { // Deleting the entire row Delete delete = new Delete(indexRowKey, ts, null); delete.setWriteToWAL(!indexWALDisabled); return delete; } Delete delete = null; // Delete columns for missing key values for (KeyValue kv : pendingUpdates) { if (kv.getType() != KeyValue.Type.Put.getCode()) { ColumnReference ref = new ColumnReference(kv.getFamily(), kv.getQualifier()); if (coveredColumns.contains(ref)) { if (delete == null) { delete = new Delete(indexRowKey); delete.setWriteToWAL(!indexWALDisabled); } delete.deleteColumns(ref.getFamily(), IndexUtil.getIndexColumnName(ref.getFamily(), ref.getQualifier()), ts); } } } return delete; }
private void assertNResult(Result result, byte[] row, byte[] family, byte[] qualifier, long[] stamps, byte[][] values, int start, int end) throws IOException { assertTrue( "Expected row [" + Bytes.toString(row) + "] " + "Got row [" + Bytes.toString(result.getRow()) + "]", equals(row, result.getRow())); int expectedResults = end - start + 1; assertEquals(expectedResults, result.size()); KeyValue[] keys = result.raw(); for (int i = 0; i < keys.length; i++) { byte[] value = values[end - i]; long ts = stamps[end - i]; KeyValue key = keys[i]; assertTrue("(" + i + ") Expected family [" + Bytes.toString(family) + "] " + "Got family [" + Bytes.toString(key.getFamily()) + "]", equals(family, key.getFamily())); assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier) + "] " + "Got qualifier [" + Bytes.toString(key.getQualifier()) + "]", equals(qualifier, key.getQualifier())); assertTrue("Expected ts [" + ts + "] " + "Got ts [" + key.getTimestamp() + "]", ts == key.getTimestamp()); assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] " + "Got value [" + Bytes.toString(key.getValue()) + "]", equals(value, key.getValue())); } }
public KeyValue getProjectedKeyValue(KeyValue kv) { if (type == ProjectionType.TABLE) { byte[] cf = ByteUtil.concat(tablePrefix, SEPERATOR, kv.getFamily()); return KeyValueUtil.newKeyValue(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(), cf, kv.getQualifier(), kv.getTimestamp(), kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()); } if (type == ProjectionType.CF) { byte[] cf = cfProjectionMap.get(new ImmutableBytesPtr(kv.getFamily())); if (cf == null) return kv; return KeyValueUtil.newKeyValue(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(), cf, kv.getQualifier(), kv.getTimestamp(), kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()); } Map<ImmutableBytesPtr, Pair<byte[], byte[]>> map = cqProjectionMap.get(new ImmutableBytesPtr(kv.getFamily())); if (map == null) return kv; Pair<byte[], byte[]> col = map.get(new ImmutableBytesPtr(kv.getQualifier())); if (col == null) return kv; return KeyValueUtil.newKeyValue(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(), col.getFirst(), col.getSecond(), kv.getTimestamp(), kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()); } }
@SuppressWarnings("deprecation") @Test @Ignore("not maintaining") public void testMapper2() throws IOException { mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name); mapDriver.addInput(new Text("52010tech"), new Text("35.432")); List<Pair<ImmutableBytesWritable, KeyValue>> result = mapDriver.run(); assertEquals(2, result.size()); byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 }; ImmutableBytesWritable key = new ImmutableBytesWritable(bytes); Pair<ImmutableBytesWritable, KeyValue> p1 = result.get(0); Pair<ImmutableBytesWritable, KeyValue> p2 = result.get(1); assertEquals(key, p1.getFirst()); assertEquals("cf1", new String(p1.getSecond().getFamily())); assertEquals("usd_amt", new String(p1.getSecond().getQualifier())); assertEquals("35.43", new String(p1.getSecond().getValue())); assertEquals(key, p2.getFirst()); assertEquals("cf1", new String(p2.getSecond().getFamily())); assertEquals("item_count", new String(p2.getSecond().getQualifier())); assertEquals("2", new String(p2.getSecond().getValue())); } }
Put dataMutation = new Put(rowKeyPtr.copyBytes()); for (KeyValue kv : dataKeyValues) { valueMap.put(new ColumnReference(kv.getFamily(),kv.getQualifier()), kv.getValue()); dataMutation.add(kv);
Integer lid = region.getLock(null, row, true); try { byte[] family = keyValue.getFamily(); byte[] qualifier = keyValue.getQualifier();
KeyValue newCurrentValueKV = KeyValueUtil.newKeyValue(row, currentValueKV.getFamily(), currentValueKV.getQualifier(), currentValueKV.getTimestamp(), valueBuffer); put.add(newCurrentValueKV); @SuppressWarnings("unchecked")
private void validate(KeyValue kv, byte[] row, byte[] family, byte[] qualifier, long ts, Type type, byte[] value) throws IOException { DataOutputBuffer out = new DataOutputBuffer(); kv.write(out); out.close(); byte[] data = out.getData(); // read it back in KeyValue read = new KeyValue(); DataInputBuffer in = new DataInputBuffer(); in.reset(data, data.length); read.readFields(in); in.close(); // validate that its the same assertTrue("Row didn't match!", Bytes.equals(row, read.getRow())); assertTrue("Family didn't match!", Bytes.equals(family, read.getFamily())); assertTrue("Qualifier didn't match!", Bytes.equals(qualifier, read.getQualifier())); assertTrue("Value didn't match!", Bytes.equals(value, read.getValue())); assertEquals("Timestamp didn't match", ts, read.getTimestamp()); assertEquals("Type didn't match", type.getCode(), read.getType()); }