private byte[] createDistributedRowKey(byte[] row) { return rowKeyDistributor == null ? row : rowKeyDistributor.getDistributedKey(row); }
@Override public byte[] getActualRowKey(ConsumerConfig consumerConfig, byte[] originalRowKey) { return rowKeyDistributor.getDistributedKey(originalRowKey); }
private byte[] createDistributedRowKey(byte[] row) { return rowKeyDistributor == null ? row : rowKeyDistributor.getDistributedKey(row); }
@Override protected void updateState(Iterable<byte[]> rows, int size, byte[] value) throws IOException { List<Put> puts = Lists.newArrayListWithCapacity(size); for (byte[] row : rows) { Put put = new Put(keyDistributor.getDistributedKey(row)); put.add(QueueEntryRow.COLUMN_FAMILY, stateColumnName, value); puts.add(put); } hTable.put(puts); hTable.flushCommits(); }
@Override protected boolean claimFifoEntry(byte[] row, byte[] value, byte[] oldValue) throws IOException { Put put = new Put(keyDistributor.getDistributedKey(row)); put.add(QueueEntryRow.COLUMN_FAMILY, stateColumnName, value); return hTable.checkAndPut(put.getRow(), QueueEntryRow.COLUMN_FAMILY, stateColumnName, oldValue, put); }
@Override protected void undoState(Iterable<byte[]> rows, int size) throws IOException { List<Delete> deletes = Lists.newArrayListWithCapacity(size); for (byte[] row : rows) { Delete delete = new Delete(keyDistributor.getDistributedKey(row)); delete.deleteColumns(QueueEntryRow.COLUMN_FAMILY, stateColumnName); deletes.add(delete); } hTable.delete(deletes); hTable.flushCommits(); }
@Override public byte[] getActualRowKey(ConsumerConfig consumerConfig, byte[] originalRowKey) { return rowKeyDistributor.getDistributedKey(getShardedKey(consumerConfig, consumerConfig.getInstanceId(), originalRowKey)); }
@Override public void getRowKeys(Iterable<ConsumerGroupConfig> consumerGroupConfigs, QueueEntry queueEntry, byte[] rowKeyPrefix, long writePointer, int counter, Collection<byte[]> rowKeys) { byte[] rowKey = new byte[rowKeyPrefix.length + Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT]; Bytes.putBytes(rowKey, 0, rowKeyPrefix, 0, rowKeyPrefix.length); Bytes.putLong(rowKey, rowKeyPrefix.length, writePointer); Bytes.putInt(rowKey, rowKey.length - Bytes.SIZEOF_INT, counter); rowKeys.add(rowKeyDistributor.getDistributedKey(rowKey)); }
private int writeTestData(long origKeyPrefix, int numRows, int rowKeySeed, int seekIntervalMinValue, int seekIntervalMaxValue) throws IOException { int valuesCountInSeekInterval = 0; for (int i = 0; i < numRows; i++) { int val = rowKeySeed + i - i * (i % 2) * 2; // i.e. 500, 499, 502, 497, 504, ... valuesCountInSeekInterval += (val >= seekIntervalMinValue && val <= seekIntervalMaxValue) ? 1 : 0; byte[] key = Bytes.toBytes(origKeyPrefix + val); byte[] distributedKey = keyDistributor.getDistributedKey(key); byte[] value = Bytes.toBytes(val); hTable.put(new Put(distributedKey).add(CF, QUAL, value)); } return valuesCountInSeekInterval; }
@Override public void persist(Iterator<RawPayloadTableEntry> entries) throws IOException { List<Put> batchPuts = new ArrayList<>(); while (entries.hasNext()) { RawPayloadTableEntry tableEntry = entries.next(); Put put = tableUtil.buildPut(rowKeyDistributor.getDistributedKey(tableEntry.getKey())) .add(columnFamily, COL, tableEntry.getValue()) .build(); batchPuts.add(put); } try { if (!batchPuts.isEmpty()) { hTable.put(batchPuts); if (!hTable.isAutoFlush()) { hTable.flushCommits(); } } } catch (IOException e) { throw exceptionHandler.handle(e); } }
@Override public void persist(Iterator<RawPayloadTableEntry> entries) throws IOException { List<Put> batchPuts = new ArrayList<>(); while (entries.hasNext()) { RawPayloadTableEntry tableEntry = entries.next(); Put put = tableUtil.buildPut(rowKeyDistributor.getDistributedKey(tableEntry.getKey())) .add(columnFamily, COL, tableEntry.getValue()) .build(); batchPuts.add(put); } try { if (!batchPuts.isEmpty()) { hTable.put(batchPuts); if (!hTable.isAutoFlush()) { hTable.flushCommits(); } } } catch (IOException e) { throw exceptionHandler.handle(e); } }
/** Testing simple get. */ @Test public void testGet() throws IOException, InterruptedException { // Testing simple get byte[] key = new byte[] {123, 124, 122}; byte[] distributedKey = keyDistributor.getDistributedKey(key); byte[] value = Bytes.toBytes("some"); hTable.put(new Put(distributedKey).add(CF, QUAL, value)); Result result = hTable.get(new Get(distributedKey)); Assert.assertArrayEquals(key, keyDistributor.getOriginalKey(result.getRow())); Assert.assertArrayEquals(value, result.getValue(CF, QUAL)); }
@Override protected void persist(Iterator<RawMessageTableEntry> entries) throws IOException { List<Put> batchPuts = new ArrayList<>(); while (entries.hasNext()) { RawMessageTableEntry entry = entries.next(); PutBuilder putBuilder = tableUtil.buildPut(rowKeyDistributor.getDistributedKey(entry.getKey())); if (entry.getTxPtr() != null) { putBuilder.add(columnFamily, TX_COL, entry.getTxPtr()); } if (entry.getPayload() != null) { putBuilder.add(columnFamily, PAYLOAD_COL, entry.getPayload()); } batchPuts.add(putBuilder.build()); } try { if (!batchPuts.isEmpty()) { hTable.put(batchPuts); if (!hTable.isAutoFlush()) { hTable.flushCommits(); } } } catch (IOException e) { throw exceptionHandler.handle(e); } }
@Override protected void persist(Iterator<RawMessageTableEntry> entries) throws IOException { List<Put> batchPuts = new ArrayList<>(); while (entries.hasNext()) { RawMessageTableEntry entry = entries.next(); PutBuilder putBuilder = tableUtil.buildPut(rowKeyDistributor.getDistributedKey(entry.getKey())); if (entry.getTxPtr() != null) { putBuilder.add(columnFamily, TX_COL, entry.getTxPtr()); } if (entry.getPayload() != null) { putBuilder.add(columnFamily, PAYLOAD_COL, entry.getPayload()); } batchPuts.add(putBuilder.build()); } try { if (!batchPuts.isEmpty()) { hTable.put(batchPuts); if (!hTable.isAutoFlush()) { hTable.flushCommits(); } } } catch (IOException e) { throw exceptionHandler.handle(e); } }
@Override public void getRowKeys(Iterable<ConsumerGroupConfig> consumerGroupConfigs, QueueEntry queueEntry, byte[] rowKeyPrefix, long writePointer, int counter, Collection<byte[]> rowKeys) { byte[] rowKey = new byte[rowKeyPrefix.length + Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT]; Bytes.putBytes(rowKey, 0, rowKeyPrefix, 0, rowKeyPrefix.length); Bytes.putLong(rowKey, rowKeyPrefix.length, writePointer); Bytes.putInt(rowKey, rowKey.length - Bytes.SIZEOF_INT, counter); // Generates all row keys, one per consumer group. for (ConsumerGroupConfig config : consumerGroupConfigs) { DequeueStrategy dequeueStrategy = config.getDequeueStrategy(); // Default for FIFO int instanceId = -1; if (dequeueStrategy != DequeueStrategy.FIFO) { if (dequeueStrategy == DequeueStrategy.ROUND_ROBIN) { instanceId = QueueEntryRow.getRoundRobinConsumerInstance(writePointer, counter, config.getGroupSize()); } else if (dequeueStrategy == DequeueStrategy.HASH) { instanceId = QueueEntryRow.getHashConsumerInstance(queueEntry.getHashKeys(), config.getHashKey(), config.getGroupSize()); } else { throw new IllegalArgumentException("Unsupported consumer strategy: " + dequeueStrategy); } } rowKeys.add(rowKeyDistributor.getDistributedKey(getShardedKey(config, instanceId, rowKey))); } }