data.get(row).get(kv.getFamily()).clear(); } else { data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
/** * {@inheritDoc} */ @Override public void put(Put put) throws IOException { byte[] row = put.getRow(); NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowData = forceFind(data, row, new TreeMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>>(Bytes.BYTES_COMPARATOR)); for (byte[] family : put.getFamilyMap().keySet()) { if (columnFamilies.contains(new String(family, StandardCharsets.UTF_8)) == false) { throw new RuntimeException("Not Exists columnFamily : " + new String(family, StandardCharsets.UTF_8)); } NavigableMap<byte[], NavigableMap<Long, byte[]>> familyData = forceFind(rowData, family, new TreeMap<byte[], NavigableMap<Long, byte[]>>(Bytes.BYTES_COMPARATOR)); for (KeyValue kv : put.getFamilyMap().get(family)) { kv.updateLatestStamp(Bytes.toBytes(System.currentTimeMillis())); byte[] qualifier = kv.getQualifier(); NavigableMap<Long, byte[]> qualifierData = forceFind(familyData, qualifier, new TreeMap<Long, byte[]>()); qualifierData.put(kv.getTimestamp(), kv.getValue()); } } }
Bytes.toString(kv.getQualifier()), new ByteArrayByteIterator(kv.getValue())); if (debug) { System.out.println("Result for field: " + Bytes.toString(kv.getQualifier()) + " is: " + Bytes.toString(kv.getValue()));
Bytes.toString(kv.getQualifier()), new ByteArrayByteIterator(kv.getValue()));
@SuppressWarnings("deprecation") @Test @Ignore("not maintaining") public void testMapper2() throws IOException { mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name); mapDriver.addInput(new Text("52010tech"), new Text("35.432")); List<Pair<RowKeyWritable, KeyValue>> result = mapDriver.run(); assertEquals(2, result.size()); byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 }; ImmutableBytesWritable key = new ImmutableBytesWritable(bytes); Pair<RowKeyWritable, KeyValue> p1 = result.get(0); Pair<RowKeyWritable, KeyValue> p2 = result.get(1); assertEquals(key, p1.getFirst()); assertEquals("cf1", new String(p1.getSecond().getFamily(), StandardCharsets.UTF_8)); assertEquals("usd_amt", new String(p1.getSecond().getQualifier(), StandardCharsets.UTF_8)); assertEquals("35.43", new String(p1.getSecond().getValue(), StandardCharsets.UTF_8)); assertEquals(key, p2.getFirst()); assertEquals("cf1", new String(p2.getSecond().getFamily(), StandardCharsets.UTF_8)); assertEquals("item_count", new String(p2.getSecond().getQualifier(), StandardCharsets.UTF_8)); assertEquals("2", new String(p2.getSecond().getValue(), StandardCharsets.UTF_8)); } }
private void removeIfPresent(Mutation m, byte[] family, byte[] qualifier) { Map<byte[],List<KeyValue>> familyMap = m.getFamilyMap(); List<KeyValue> kvs = familyMap.get(family); if (kvs != null) { Iterator<KeyValue> iterator = kvs.iterator(); while (iterator.hasNext()) { KeyValue kv = iterator.next(); if (Bytes.compareTo(kv.getQualifier(), qualifier) == 0) { iterator.remove(); } } } }
@SuppressWarnings({ "rawtypes", "deprecation" }) protected Map getCIFObject(String key) { LOGGER.debug("=======Pinging HBase For: {}", key); Get get = new Get(key.getBytes()); Result rs; Map output = new HashMap(); try { rs = table.get(get); for (KeyValue kv : rs.raw()) output.put(new String(kv.getQualifier()), "Y"); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } return output; }
public void setColumns(List<KeyValue> columns) { for(KeyValue column : columns) { putColumn(column.getQualifier(), column.getValue()); } }
/** * check to see if the kvs in the update match any of the passed columns. Generally, this is useful to for an index * codec to determine if a given update should even be indexed. This assumes that for any index, there are going to * small number of columns, versus the number of kvs in any one batch. */ public static boolean updateMatchesColumns(Collection<KeyValue> update, List<ColumnReference> columns) { // check to see if the kvs in the new update even match any of the columns requested // assuming that for any index, there are going to small number of columns, versus the number of // kvs in any one batch. boolean matches = false; outer: for (KeyValue kv : update) { for (ColumnReference ref : columns) { if (ref.matchesFamily(kv.getFamily()) && ref.matchesQualifier(kv.getQualifier())) { matches = true; // if a single column matches a single kv, we need to build a whole scanner break outer; } } } return matches; }
/** * Check to see if the kvs in the update match any of the passed columns. Generally, this is useful to for an index * codec to determine if a given update should even be indexed. This assumes that for any index, there are going to * small number of kvs, versus the number of columns in any one batch. * <p> * This employs the same logic as {@link #updateMatchesColumns(Collection, List)}, but is flips the iteration logic * to search columns before kvs. */ public static boolean columnMatchesUpdate(List<ColumnReference> columns, Collection<KeyValue> update) { boolean matches = false; outer: for (ColumnReference ref : columns) { for (KeyValue kv : update) { if (ref.matchesFamily(kv.getFamily()) && ref.matchesQualifier(kv.getQualifier())) { matches = true; // if a single column matches a single kv, we need to build a whole scanner break outer; } } } return matches; }
private boolean hasIndexedColumnChanged(ValueGetter oldState, Collection<KeyValue> pendingUpdates) throws IOException { if (pendingUpdates.isEmpty()) { return false; } Map<ColumnReference,KeyValue> newState = Maps.newHashMapWithExpectedSize(pendingUpdates.size()); for (KeyValue kv : pendingUpdates) { newState.put(new ColumnReference(kv.getFamily(), kv.getQualifier()), kv); } for (ColumnReference ref : indexedColumns) { KeyValue newValue = newState.get(ref); if (newValue != null) { // Indexed column was potentially changed ImmutableBytesPtr oldValue = oldState.getLatestValue(ref); // If there was no old value or the old value is different than the new value, the index row needs to be deleted if (oldValue == null || Bytes.compareTo(oldValue.get(), oldValue.getOffset(), oldValue.getLength(), newValue.getBuffer(), newValue.getValueOffset(), newValue.getValueLength()) != 0){ return true; } } } return false; }
if (ref.matchesFamily(next.getFamily()) && ref.matchesQualifier(next.getQualifier())) { byte[] v = next.getValue(); totalValueLength += v.length; byte[] lastQual = next.getQualifier(); byte[] nextQual = null; while ((next = kvs.next()) != null) { break; nextQual = next.getQualifier();
public Increment newIncrement(long timestamp) { Increment inc = new Increment(SchemaUtil.getSequenceKey(key.getTenantId(), key.getSchemaName(), key.getSequenceName())); // It doesn't matter what we set the amount too - we always use the values we get // from the Get we do to prevent any race conditions. All columns that get added // are returned with their current value try { inc.setTimeRange(MetaDataProtocol.MIN_TABLE_TIMESTAMP, timestamp); } catch (IOException e) { throw new RuntimeException(e); // Impossible } for (KeyValue kv : SEQUENCE_KV_COLUMNS) { // We don't care about the amount, as we'll add what gets looked up on the server-side inc.addColumn(kv.getFamily(), kv.getQualifier(), AMOUNT); } return inc; }
@Override public void put(Put put) throws IOException { addToPutLog(put); byte[] row = put.getRow(); NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowData = forceFind(data, row, new TreeMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>>(Bytes.BYTES_COMPARATOR)); for (byte[] family : put.getFamilyMap().keySet()){ NavigableMap<byte[], NavigableMap<Long, byte[]>> familyData = forceFind(rowData, family, new TreeMap<byte[], NavigableMap<Long, byte[]>>(Bytes.BYTES_COMPARATOR)); for (KeyValue kv : put.getFamilyMap().get(family)){ kv.updateLatestStamp(Bytes.toBytes(System.currentTimeMillis())); byte[] qualifier = kv.getQualifier(); NavigableMap<Long, byte[]> qualifierData = forceFind(familyData, qualifier, new TreeMap<Long, byte[]>()); qualifierData.put(kv.getTimestamp(), kv.getValue()); } } }
@SuppressWarnings("deprecation") public Delete buildDeleteMutation(ValueGetter oldState, ImmutableBytesWritable dataRowKeyPtr, Collection<KeyValue> pendingUpdates, long ts) throws IOException { byte[] indexRowKey = this.buildRowKey(oldState, dataRowKeyPtr); // Delete the entire row if any of the indexed columns changed if (oldState == null || isRowDeleted(pendingUpdates) || hasIndexedColumnChanged(oldState, pendingUpdates)) { // Deleting the entire row Delete delete = new Delete(indexRowKey, ts, null); delete.setWriteToWAL(!indexWALDisabled); return delete; } Delete delete = null; // Delete columns for missing key values for (KeyValue kv : pendingUpdates) { if (kv.getType() != KeyValue.Type.Put.getCode()) { ColumnReference ref = new ColumnReference(kv.getFamily(), kv.getQualifier()); if (coveredColumns.contains(ref)) { if (delete == null) { delete = new Delete(indexRowKey); delete.setWriteToWAL(!indexWALDisabled); } delete.deleteColumns(ref.getFamily(), IndexUtil.getIndexColumnName(ref.getFamily(), ref.getQualifier()), ts); } } } return delete; }
private void assertNResult(Result result, byte[] row, byte[] family, byte[] qualifier, long[] stamps, byte[][] values, int start, int end) throws IOException { assertTrue( "Expected row [" + Bytes.toString(row) + "] " + "Got row [" + Bytes.toString(result.getRow()) + "]", equals(row, result.getRow())); int expectedResults = end - start + 1; assertEquals(expectedResults, result.size()); KeyValue[] keys = result.raw(); for (int i = 0; i < keys.length; i++) { byte[] value = values[end - i]; long ts = stamps[end - i]; KeyValue key = keys[i]; assertTrue("(" + i + ") Expected family [" + Bytes.toString(family) + "] " + "Got family [" + Bytes.toString(key.getFamily()) + "]", equals(family, key.getFamily())); assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier) + "] " + "Got qualifier [" + Bytes.toString(key.getQualifier()) + "]", equals(qualifier, key.getQualifier())); assertTrue("Expected ts [" + ts + "] " + "Got ts [" + key.getTimestamp() + "]", ts == key.getTimestamp()); assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] " + "Got value [" + Bytes.toString(key.getValue()) + "]", equals(value, key.getValue())); } }
public KeyValue getProjectedKeyValue(KeyValue kv) { if (type == ProjectionType.TABLE) { byte[] cf = ByteUtil.concat(tablePrefix, SEPERATOR, kv.getFamily()); return KeyValueUtil.newKeyValue(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(), cf, kv.getQualifier(), kv.getTimestamp(), kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()); } if (type == ProjectionType.CF) { byte[] cf = cfProjectionMap.get(new ImmutableBytesPtr(kv.getFamily())); if (cf == null) return kv; return KeyValueUtil.newKeyValue(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(), cf, kv.getQualifier(), kv.getTimestamp(), kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()); } Map<ImmutableBytesPtr, Pair<byte[], byte[]>> map = cqProjectionMap.get(new ImmutableBytesPtr(kv.getFamily())); if (map == null) return kv; Pair<byte[], byte[]> col = map.get(new ImmutableBytesPtr(kv.getQualifier())); if (col == null) return kv; return KeyValueUtil.newKeyValue(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(), col.getFirst(), col.getSecond(), kv.getTimestamp(), kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()); } }
@SuppressWarnings("deprecation") @Test @Ignore("not maintaining") public void testMapper2() throws IOException { mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name); mapDriver.addInput(new Text("52010tech"), new Text("35.432")); List<Pair<ImmutableBytesWritable, KeyValue>> result = mapDriver.run(); assertEquals(2, result.size()); byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 }; ImmutableBytesWritable key = new ImmutableBytesWritable(bytes); Pair<ImmutableBytesWritable, KeyValue> p1 = result.get(0); Pair<ImmutableBytesWritable, KeyValue> p2 = result.get(1); assertEquals(key, p1.getFirst()); assertEquals("cf1", new String(p1.getSecond().getFamily())); assertEquals("usd_amt", new String(p1.getSecond().getQualifier())); assertEquals("35.43", new String(p1.getSecond().getValue())); assertEquals(key, p2.getFirst()); assertEquals("cf1", new String(p2.getSecond().getFamily())); assertEquals("item_count", new String(p2.getSecond().getQualifier())); assertEquals("2", new String(p2.getSecond().getValue())); } }
Put dataMutation = new Put(rowKeyPtr.copyBytes()); for (KeyValue kv : dataKeyValues) { valueMap.put(new ColumnReference(kv.getFamily(),kv.getQualifier()), kv.getValue()); dataMutation.add(kv);
private void validate(KeyValue kv, byte[] row, byte[] family, byte[] qualifier, long ts, Type type, byte[] value) throws IOException { DataOutputBuffer out = new DataOutputBuffer(); kv.write(out); out.close(); byte[] data = out.getData(); // read it back in KeyValue read = new KeyValue(); DataInputBuffer in = new DataInputBuffer(); in.reset(data, data.length); read.readFields(in); in.close(); // validate that its the same assertTrue("Row didn't match!", Bytes.equals(row, read.getRow())); assertTrue("Family didn't match!", Bytes.equals(family, read.getFamily())); assertTrue("Qualifier didn't match!", Bytes.equals(qualifier, read.getQualifier())); assertTrue("Value didn't match!", Bytes.equals(value, read.getValue())); assertEquals("Timestamp didn't match", ts, read.getTimestamp()); assertEquals("Type didn't match", type.getCode(), read.getType()); }