private void createQueueTable(TableDescriptorBuilder tdBuilder, byte[][] splitKeys) throws IOException { int prefixBytes = (type == QueueConstants.QueueType.SHARDED_QUEUE) ? ShardedHBaseQueueStrategy.PREFIX_BYTES : SaltedHBaseQueueStrategy.SALT_BYTES; String prefix = Integer.toString(prefixBytes); tdBuilder.addProperty(HBaseQueueAdmin.PROPERTY_PREFIX_BYTES, prefix); LOG.info("Create queue table with prefix bytes {}", prefix); try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) { ddlExecutor.createTableIfNotExists(tdBuilder.build(), splitKeys); } } }
/** * Get {@link TableDescriptorBuilder} with default properties set. * @param tableId id of the table for which the descriptor is to be returned * @param cConf the instance of the {@link CConfiguration} * @return the builder with default properties set */ public static TableDescriptorBuilder getTableDescriptorBuilder(TableId tableId, CConfiguration cConf) { String tablePrefix = cConf.get(Constants.Dataset.TABLE_PREFIX); TableName tableName = HTableNameConverter.toTableName(tablePrefix, tableId); TableDescriptorBuilder tdBuilder = new TableDescriptorBuilder(tableName.getNamespaceAsString(), tableName.getQualifierAsString()); tdBuilder .addProperty(Constants.Dataset.TABLE_PREFIX, tablePrefix) .addProperty(HBaseTableUtil.CDAP_VERSION, ProjectInfo.getVersion().toString()) .addProperty(HBaseTableUtil.CDAP_HBASE_VERSION, HBaseVersion.get().getMajorVersion()); return tdBuilder; }
/** * Get {@link TableDescriptorBuilder} with default properties set. * @param tableId id of the table for which the descriptor is to be returned * @param cConf the instance of the {@link CConfiguration} * @return the builder with default properties set */ public static TableDescriptorBuilder getTableDescriptorBuilder(TableId tableId, CConfiguration cConf) { String tablePrefix = cConf.get(Constants.Dataset.TABLE_PREFIX); TableName tableName = HTableNameConverter.toTableName(tablePrefix, tableId); TableDescriptorBuilder tdBuilder = new TableDescriptorBuilder(tableName.getNamespaceAsString(), tableName.getQualifierAsString()); tdBuilder .addProperty(Constants.Dataset.TABLE_PREFIX, tablePrefix) .addProperty(HBaseTableUtil.CDAP_VERSION, ProjectInfo.getVersion().toString()) .addProperty(HBaseTableUtil.CDAP_HBASE_VERSION, HBaseVersion.get().getMajorVersion()); return tdBuilder; }
@Override public void create() throws IOException { // Create the queue table TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf); for (String key : properties.stringPropertyNames()) { tdBuilder.addProperty(key, properties.getProperty(key)); } ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(QueueEntryRow.COLUMN_FAMILY), hConf); tdBuilder.addColumnFamily(cfdBuilder.build()); // Add coprocessors CoprocessorJar coprocessorJar = createCoprocessorJar(); for (Class<? extends Coprocessor> coprocessor : coprocessorJar.getCoprocessors()) { tdBuilder.addCoprocessor( coprocessorManager.getCoprocessorDescriptor(coprocessor, coprocessorJar.getPriority(coprocessor))); } // Create queue table with splits. The distributor bucket size is the same as splits. int splits = cConf.getInt(QueueConstants.ConfigKeys.QUEUE_TABLE_PRESPLITS); AbstractRowKeyDistributor distributor = new RowKeyDistributorByHashPrefix( new RowKeyDistributorByHashPrefix.OneByteSimpleHash(splits)); byte[][] splitKeys = HBaseTableUtil.getSplitKeys(splits, splits, distributor); tdBuilder.addProperty(QueueConstants.DISTRIBUTOR_BUCKETS, Integer.toString(splits)); createQueueTable(tdBuilder, splitKeys); }
.addProperty(Constants.MessagingSystem.HBASE_MESSAGING_TABLE_PREFIX_NUM_BYTES, Integer.toString(1)) .addProperty(Constants.MessagingSystem.KEY_DISTRIBUTOR_BUCKETS_ATTR, Integer.toString(splits)) .addProperty(Constants.MessagingSystem.HBASE_METADATA_TABLE_NAMESPACE, metadataTableId.getNamespace()) .addProperty(HTableDescriptor.SPLIT_POLICY, cConf.get(Constants.MessagingSystem.TABLE_HBASE_SPLIT_POLICY)) .addCoprocessor(coprocessorManager.getCoprocessorDescriptor(coprocessor, Coprocessor.PRIORITY_USER));
.addProperty(Constants.MessagingSystem.HBASE_MESSAGING_TABLE_PREFIX_NUM_BYTES, Integer.toString(1)) .addProperty(Constants.MessagingSystem.KEY_DISTRIBUTOR_BUCKETS_ATTR, Integer.toString(splits)) .addProperty(Constants.MessagingSystem.HBASE_METADATA_TABLE_NAMESPACE, metadataTableId.getNamespace()) .addProperty(HTableDescriptor.SPLIT_POLICY, cConf.get(Constants.MessagingSystem.TABLE_HBASE_SPLIT_POLICY)) .addCoprocessor(coprocessorManager.getCoprocessorDescriptor(coprocessor, Coprocessor.PRIORITY_USER));
tdBuilder.addProperty(Table.PROPERTY_READLESS_INCREMENT, "true"); tdBuilder.addProperty(Constants.Dataset.TABLE_TX_DISABLED, "true"); if (supportsReadlessIncrements) { tdBuilder.addProperty(HTableDescriptor.SPLIT_POLICY, splitsPolicy);
tdBuilder.addProperty(Table.PROPERTY_READLESS_INCREMENT, "true"); tdBuilder.addProperty(Constants.Dataset.TABLE_TX_DISABLED, "true"); if (supportsReadlessIncrements) { tdBuilder.addProperty(HTableDescriptor.SPLIT_POLICY, splitsPolicy);
@Override protected StreamConsumer create(TableId tableId, StreamConfig streamConfig, ConsumerConfig consumerConfig, StreamConsumerStateStore stateStore, StreamConsumerState beginConsumerState, FileReader<StreamEventOffset, Iterable<StreamFileOffset>> reader, @Nullable ReadFilter extraFilter) throws IOException { int splits = cConf.getInt(Constants.Stream.CONSUMER_TABLE_PRESPLITS); AbstractRowKeyDistributor distributor = new RowKeyDistributorByHashPrefix( new RowKeyDistributorByHashPrefix.OneByteSimpleHash(splits)); byte[][] splitKeys = HBaseTableUtil.getSplitKeys(splits, splits, distributor); TableId hBaseTableId = tableUtil.createHTableId(new NamespaceId(tableId.getNamespace()), tableId.getTableName()); TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(hBaseTableId, cConf); ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(QueueEntryRow.COLUMN_FAMILY), hConf); tdBuilder.addColumnFamily(cfdBuilder.build()); tdBuilder.addProperty(QueueConstants.DISTRIBUTOR_BUCKETS, Integer.toString(splits)); try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) { ddlExecutor.createTableIfNotExists(tdBuilder.build(), splitKeys); } HTable hTable = tableUtil.createHTable(hConf, hBaseTableId); hTable.setWriteBufferSize(Constants.Stream.HBASE_WRITE_BUFFER_SIZE); hTable.setAutoFlushTo(false); return new HBaseStreamFileConsumer(cConf, streamConfig, consumerConfig, tableUtil, hTable, reader, stateStore, beginConsumerState, extraFilter, createKeyDistributor(hTable.getTableDescriptor())); }