@Test public void testGetSplitKeys() { int buckets = 16; AbstractRowKeyDistributor distributor = new RowKeyDistributorByHashPrefix( new RowKeyDistributorByHashPrefix.OneByteSimpleHash(buckets)); // Number of splits will be no less than user asked. If splits > buckets, the number of splits will bumped to // next multiple of bucket that is no less than user splits requested. // it should return one key less than required splits count, because HBase will take care of the first automatically Assert.assertEquals(getSplitSize(buckets, 12) - 1, HBaseTableUtil.getSplitKeys(12, buckets, distributor).length); Assert.assertEquals(getSplitSize(buckets, 16) - 1, HBaseTableUtil.getSplitKeys(16, buckets, distributor).length); // at least #buckets - 1, but no less than user asked Assert.assertEquals(buckets - 1, HBaseTableUtil.getSplitKeys(6, buckets, distributor).length); Assert.assertEquals(buckets - 1, HBaseTableUtil.getSplitKeys(2, buckets, distributor).length); // "1" can be used for queue tables that we know are not "hot", so we do not pre-split in this case Assert.assertEquals(0, HBaseTableUtil.getSplitKeys(1, buckets, distributor).length); // allows up to 255 * 8 - 1 splits Assert.assertEquals(255 * buckets - 1, HBaseTableUtil.getSplitKeys(255 * buckets, buckets, distributor).length); try { HBaseTableUtil.getSplitKeys(256 * buckets, buckets, distributor); Assert.fail("getSplitKeys(256) should have thrown IllegalArgumentException"); } catch (IllegalArgumentException e) { // expected } try { HBaseTableUtil.getSplitKeys(0, buckets, distributor); Assert.fail("getSplitKeys(0) should have thrown IllegalArgumentException"); } catch (IllegalArgumentException e) { // expected } }
@Override public void create() throws IOException { // Create the queue table TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf); for (String key : properties.stringPropertyNames()) { tdBuilder.addProperty(key, properties.getProperty(key)); } ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(QueueEntryRow.COLUMN_FAMILY), hConf); tdBuilder.addColumnFamily(cfdBuilder.build()); // Add coprocessors CoprocessorJar coprocessorJar = createCoprocessorJar(); for (Class<? extends Coprocessor> coprocessor : coprocessorJar.getCoprocessors()) { tdBuilder.addCoprocessor( coprocessorManager.getCoprocessorDescriptor(coprocessor, coprocessorJar.getPriority(coprocessor))); } // Create queue table with splits. The distributor bucket size is the same as splits. int splits = cConf.getInt(QueueConstants.ConfigKeys.QUEUE_TABLE_PRESPLITS); AbstractRowKeyDistributor distributor = new RowKeyDistributorByHashPrefix( new RowKeyDistributorByHashPrefix.OneByteSimpleHash(splits)); byte[][] splitKeys = HBaseTableUtil.getSplitKeys(splits, splits, distributor); tdBuilder.addProperty(QueueConstants.DISTRIBUTOR_BUCKETS, Integer.toString(splits)); createQueueTable(tdBuilder, splitKeys); }
@Override protected StreamConsumer create(TableId tableId, StreamConfig streamConfig, ConsumerConfig consumerConfig, StreamConsumerStateStore stateStore, StreamConsumerState beginConsumerState, FileReader<StreamEventOffset, Iterable<StreamFileOffset>> reader, @Nullable ReadFilter extraFilter) throws IOException { int splits = cConf.getInt(Constants.Stream.CONSUMER_TABLE_PRESPLITS); AbstractRowKeyDistributor distributor = new RowKeyDistributorByHashPrefix( new RowKeyDistributorByHashPrefix.OneByteSimpleHash(splits)); byte[][] splitKeys = HBaseTableUtil.getSplitKeys(splits, splits, distributor); TableId hBaseTableId = tableUtil.createHTableId(new NamespaceId(tableId.getNamespace()), tableId.getTableName()); TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(hBaseTableId, cConf); ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(QueueEntryRow.COLUMN_FAMILY), hConf); tdBuilder.addColumnFamily(cfdBuilder.build()); tdBuilder.addProperty(QueueConstants.DISTRIBUTOR_BUCKETS, Integer.toString(splits)); try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) { ddlExecutor.createTableIfNotExists(tdBuilder.build(), splitKeys); } HTable hTable = tableUtil.createHTable(hConf, hBaseTableId); hTable.setWriteBufferSize(Constants.Stream.HBASE_WRITE_BUFFER_SIZE); hTable.setAutoFlushTo(false); return new HBaseStreamFileConsumer(cConf, streamConfig, consumerConfig, tableUtil, hTable, reader, stateStore, beginConsumerState, extraFilter, createKeyDistributor(hTable.getTableDescriptor())); }
byte[][] splitKeys = HBaseTableUtil.getSplitKeys( splits, splits, new RowKeyDistributorByHashPrefix(new OneByteSimpleHash(splits))); ddlExecutor.createTableIfNotExists(tdBuilder.build(), splitKeys);
byte[][] splitKeys = HBaseTableUtil.getSplitKeys( splits, splits, new RowKeyDistributorByHashPrefix(new OneByteSimpleHash(splits))); ddlExecutor.createTableIfNotExists(tdBuilder.build(), splitKeys);