@Override public void create() throws IOException { ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(DATA_COLUMN_FAMILY), hConf); TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf) .addColumnFamily(cfdBuilder.build()); try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) { ddlExecutor.createTableIfNotExists(tdBuilder.build(), null); } }
@Override public void create() throws IOException { // Create the queue table TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf); for (String key : properties.stringPropertyNames()) { tdBuilder.addProperty(key, properties.getProperty(key)); } ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(QueueEntryRow.COLUMN_FAMILY), hConf); tdBuilder.addColumnFamily(cfdBuilder.build()); // Add coprocessors CoprocessorJar coprocessorJar = createCoprocessorJar(); for (Class<? extends Coprocessor> coprocessor : coprocessorJar.getCoprocessors()) { tdBuilder.addCoprocessor( coprocessorManager.getCoprocessorDescriptor(coprocessor, coprocessorJar.getPriority(coprocessor))); } // Create queue table with splits. The distributor bucket size is the same as splits. int splits = cConf.getInt(QueueConstants.ConfigKeys.QUEUE_TABLE_PRESPLITS); AbstractRowKeyDistributor distributor = new RowKeyDistributorByHashPrefix( new RowKeyDistributorByHashPrefix.OneByteSimpleHash(splits)); byte[][] splitKeys = HBaseTableUtil.getSplitKeys(splits, splits, distributor); tdBuilder.addProperty(QueueConstants.DISTRIBUTOR_BUCKETS, Integer.toString(splits)); createQueueTable(tdBuilder, splitKeys); }
/** * Get {@link TableDescriptorBuilder} with default properties set. * @param tableId id of the table for which the descriptor is to be returned * @param cConf the instance of the {@link CConfiguration} * @return the builder with default properties set */ public static TableDescriptorBuilder getTableDescriptorBuilder(TableId tableId, CConfiguration cConf) { String tablePrefix = cConf.get(Constants.Dataset.TABLE_PREFIX); TableName tableName = HTableNameConverter.toTableName(tablePrefix, tableId); TableDescriptorBuilder tdBuilder = new TableDescriptorBuilder(tableName.getNamespaceAsString(), tableName.getQualifierAsString()); tdBuilder .addProperty(Constants.Dataset.TABLE_PREFIX, tablePrefix) .addProperty(HBaseTableUtil.CDAP_VERSION, ProjectInfo.getVersion().toString()) .addProperty(HBaseTableUtil.CDAP_HBASE_VERSION, HBaseVersion.get().getMajorVersion()); return tdBuilder; }
private void createQueueTable(TableDescriptorBuilder tdBuilder, byte[][] splitKeys) throws IOException { int prefixBytes = (type == QueueConstants.QueueType.SHARDED_QUEUE) ? ShardedHBaseQueueStrategy.PREFIX_BYTES : SaltedHBaseQueueStrategy.SALT_BYTES; String prefix = Integer.toString(prefixBytes); tdBuilder.addProperty(HBaseQueueAdmin.PROPERTY_PREFIX_BYTES, prefix); LOG.info("Create queue table with prefix bytes {}", prefix); try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) { ddlExecutor.createTableIfNotExists(tdBuilder.build(), splitKeys); } } }
.addColumnFamily(cfdBuilder.build()) .addProperty(Constants.MessagingSystem.HBASE_MESSAGING_TABLE_PREFIX_NUM_BYTES, Integer.toString(1)) .addProperty(Constants.MessagingSystem.KEY_DISTRIBUTOR_BUCKETS_ATTR, Integer.toString(splits)) .addProperty(Constants.MessagingSystem.HBASE_METADATA_TABLE_NAMESPACE, metadataTableId.getNamespace()) .addProperty(HTableDescriptor.SPLIT_POLICY, cConf.get(Constants.MessagingSystem.TABLE_HBASE_SPLIT_POLICY)) .addCoprocessor(coprocessorManager.getCoprocessorDescriptor(coprocessor, Coprocessor.PRIORITY_USER)); ddlExecutor.createTableIfNotExists(tdBuilder.build(), splitKeys);
@Override protected StreamConsumer create(TableId tableId, StreamConfig streamConfig, ConsumerConfig consumerConfig, StreamConsumerStateStore stateStore, StreamConsumerState beginConsumerState, FileReader<StreamEventOffset, Iterable<StreamFileOffset>> reader, @Nullable ReadFilter extraFilter) throws IOException { int splits = cConf.getInt(Constants.Stream.CONSUMER_TABLE_PRESPLITS); AbstractRowKeyDistributor distributor = new RowKeyDistributorByHashPrefix( new RowKeyDistributorByHashPrefix.OneByteSimpleHash(splits)); byte[][] splitKeys = HBaseTableUtil.getSplitKeys(splits, splits, distributor); TableId hBaseTableId = tableUtil.createHTableId(new NamespaceId(tableId.getNamespace()), tableId.getTableName()); TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(hBaseTableId, cConf); ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(QueueEntryRow.COLUMN_FAMILY), hConf); tdBuilder.addColumnFamily(cfdBuilder.build()); tdBuilder.addProperty(QueueConstants.DISTRIBUTOR_BUCKETS, Integer.toString(splits)); try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) { ddlExecutor.createTableIfNotExists(tdBuilder.build(), splitKeys); } HTable hTable = tableUtil.createHTable(hConf, hBaseTableId); hTable.setWriteBufferSize(Constants.Stream.HBASE_WRITE_BUFFER_SIZE); hTable.setAutoFlushTo(false); return new HBaseStreamFileConsumer(cConf, streamConfig, consumerConfig, tableUtil, hTable, reader, stateStore, beginConsumerState, extraFilter, createKeyDistributor(hTable.getTableDescriptor())); }
.addColumnFamily(cfdBuilder.build()) .addProperty(Constants.MessagingSystem.HBASE_MESSAGING_TABLE_PREFIX_NUM_BYTES, Integer.toString(1)) .addProperty(Constants.MessagingSystem.KEY_DISTRIBUTOR_BUCKETS_ATTR, Integer.toString(splits)) .addProperty(Constants.MessagingSystem.HBASE_METADATA_TABLE_NAMESPACE, metadataTableId.getNamespace()) .addProperty(HTableDescriptor.SPLIT_POLICY, cConf.get(Constants.MessagingSystem.TABLE_HBASE_SPLIT_POLICY)) .addCoprocessor(coprocessorManager.getCoprocessorDescriptor(coprocessor, Coprocessor.PRIORITY_USER)); ddlExecutor.createTableIfNotExists(tdBuilder.build(), splitKeys);
tdBuilder.addProperty(Table.PROPERTY_READLESS_INCREMENT, "true"); tdBuilder.addProperty(Constants.Dataset.TABLE_TX_DISABLED, "true"); if (supportsReadlessIncrements) { tdBuilder.addColumnFamily(cfdBuilder.build()); tdBuilder.addCoprocessor( coprocessorManager.getCoprocessorDescriptor(coprocessor, coprocessorJar.getPriority(coprocessor))); tdBuilder.addProperty(HTableDescriptor.SPLIT_POLICY, splitsPolicy); ddlExecutor.createTableIfNotExists(tdBuilder.build(), splits); try { Map<String, String> permissions = TableProperties.getTablePermissions(spec.getProperties());
@Override public void create() throws IOException { ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(DATA_COLUMN_FAMILY), hConf); TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf) .addColumnFamily(cfdBuilder.build()); try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) { ddlExecutor.createTableIfNotExists(tdBuilder.build(), null); } }
/** * Get {@link TableDescriptorBuilder} with default properties set. * @param tableId id of the table for which the descriptor is to be returned * @param cConf the instance of the {@link CConfiguration} * @return the builder with default properties set */ public static TableDescriptorBuilder getTableDescriptorBuilder(TableId tableId, CConfiguration cConf) { String tablePrefix = cConf.get(Constants.Dataset.TABLE_PREFIX); TableName tableName = HTableNameConverter.toTableName(tablePrefix, tableId); TableDescriptorBuilder tdBuilder = new TableDescriptorBuilder(tableName.getNamespaceAsString(), tableName.getQualifierAsString()); tdBuilder .addProperty(Constants.Dataset.TABLE_PREFIX, tablePrefix) .addProperty(HBaseTableUtil.CDAP_VERSION, ProjectInfo.getVersion().toString()) .addProperty(HBaseTableUtil.CDAP_HBASE_VERSION, HBaseVersion.get().getMajorVersion()); return tdBuilder; }
tdBuilder.addProperty(Table.PROPERTY_READLESS_INCREMENT, "true"); tdBuilder.addProperty(Constants.Dataset.TABLE_TX_DISABLED, "true"); if (supportsReadlessIncrements) { tdBuilder.addColumnFamily(cfdBuilder.build()); tdBuilder.addCoprocessor( coprocessorManager.getCoprocessorDescriptor(coprocessor, coprocessorJar.getPriority(coprocessor))); tdBuilder.addProperty(HTableDescriptor.SPLIT_POLICY, splitsPolicy); ddlExecutor.createTableIfNotExists(tdBuilder.build(), splits); try { Map<String, String> permissions = TableProperties.getTablePermissions(spec.getProperties());
/** * Creates the configuration HBase table if it does not exist. */ @VisibleForTesting void createTableIfNecessary() throws IOException { try (HBaseDDLExecutor ddlExecutor = new HBaseDDLExecutorFactory(cConf, hConf).get()) { HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get(); TableId tableId = tableUtil.createHTableId(NamespaceId.SYSTEM, TABLE_NAME); ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(FAMILY), hConf); TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf).addColumnFamily(cfdBuilder.build()); ddlExecutor.createTableIfNotExists(tdBuilder.build(), null); } } }
/** * Creates the configuration HBase table if it does not exist. */ @VisibleForTesting void createTableIfNecessary() throws IOException { try (HBaseDDLExecutor ddlExecutor = new HBaseDDLExecutorFactory(cConf, hConf).get()) { HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get(); TableId tableId = tableUtil.createHTableId(NamespaceId.SYSTEM, TABLE_NAME); ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(FAMILY), hConf); TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf).addColumnFamily(cfdBuilder.build()); ddlExecutor.createTableIfNotExists(tdBuilder.build(), null); } } }
@Override public MetadataTable createMetadataTable() throws IOException { TableId tableId = tableUtil.createHTableId(NamespaceId.SYSTEM, metadataTableName); HTable hTable = null; // If the table descriptor is in the cache, we assume the table exists. if (!tableDescriptors.containsKey(tableId)) { synchronized (this) { if (!tableDescriptors.containsKey(tableId)) { try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) { ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(COLUMN_FAMILY), hConf); TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf).addColumnFamily(cfdBuilder.build()); ddlExecutor.createTableIfNotExists(tdBuilder.build(), null); hTable = tableUtil.createHTable(hConf, tableId); tableDescriptors.put(tableId, hTable.getTableDescriptor()); } } } } if (hTable == null) { hTable = tableUtil.createHTable(hConf, tableId); } return new HBaseMetadataTable(tableUtil, hTable, COLUMN_FAMILY, cConf.getInt(Constants.MessagingSystem.HBASE_SCAN_CACHE_ROWS), createExceptionHandler(tableId)); }
@Override public MetadataTable createMetadataTable(String tableName) throws IOException { TableId tableId = tableUtil.createHTableId(NamespaceId.SYSTEM, tableName); HTable hTable = null; // If the table descriptor is in the cache, we assume the table exists. if (!tableDescriptors.containsKey(tableId)) { synchronized (this) { if (!tableDescriptors.containsKey(tableId)) { try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) { ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(COLUMN_FAMILY), hConf); TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf).addColumnFamily(cfdBuilder.build()); ddlExecutor.createTableIfNotExists(tdBuilder.build(), null); hTable = tableUtil.createHTable(hConf, tableId); tableDescriptors.put(tableId, hTable.getTableDescriptor()); } } } } if (hTable == null) { hTable = tableUtil.createHTable(hConf, tableId); } return new HBaseMetadataTable(tableUtil, hTable, COLUMN_FAMILY, cConf.getInt(Constants.MessagingSystem.HBASE_SCAN_CACHE_ROWS), createExceptionHandler(tableId)); }
private void create(TableId tableId) throws IOException { HBaseTableUtil tableUtil = getTableUtil(); TableId htableId = tableUtil.createHTableId(new NamespaceId(tableId.getNamespace()), tableId.getTableName()); ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder("d", hAdmin.getConfiguration()); TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(htableId, cConf); tdBuilder.addColumnFamily(cfdBuilder.build()); ddlExecutor.createTableIfNotExists(tdBuilder.build(), null); }
@Override public synchronized StreamConsumerStateStore create(StreamConfig streamConfig) throws IOException { NamespaceId namespace = streamConfig.getStreamId().getParent(); TableId streamStateStoreTableId = StreamUtils.getStateStoreTableId(namespace); TableId hbaseTableId = tableUtil.createHTableId(new NamespaceId(streamStateStoreTableId.getNamespace()), streamStateStoreTableId.getTableName()); boolean tableExist; try (HBaseAdmin admin = new HBaseAdmin(hConf)) { tableExist = tableUtil.tableExists(admin, hbaseTableId); } if (!tableExist) { try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) { TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(hbaseTableId, cConf); ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(QueueEntryRow.COLUMN_FAMILY), hConf); tdBuilder.addColumnFamily(cfdBuilder.build()); ddlExecutor.createTableIfNotExists(tdBuilder.build(), null); } } HTable hTable = tableUtil.createHTable(hConf, hbaseTableId); hTable.setWriteBufferSize(Constants.Stream.HBASE_WRITE_BUFFER_SIZE); hTable.setAutoFlushTo(false); return new HBaseStreamConsumerStateStore(streamConfig, hTable); }