KVTableImpl(DatasetContext datasetContext, String tableName, Configuration hConf, HBaseTableUtil tableUtil) throws IOException { this.tableUtil = tableUtil; TableId tableId = tableUtil.createHTableId(new NamespaceId(datasetContext.getNamespaceId()), tableName); this.table = this.tableUtil.createHTable(hConf, tableId); }
public HTable createHTable(TableId hTableId) throws IOException { HTable consumerTable = hBaseTableUtil.createHTable(hConf, hTableId); // TODO: make configurable consumerTable.setWriteBufferSize(DEFAULT_WRITE_BUFFER_SIZE); consumerTable.setAutoFlushTo(false); return consumerTable; }
KVTableImpl(DatasetContext datasetContext, String tableName, Configuration hConf, HBaseTableUtil tableUtil) throws IOException { this.tableUtil = tableUtil; TableId tableId = tableUtil.createHTableId(new NamespaceId(datasetContext.getNamespaceId()), tableName); this.table = this.tableUtil.createHTable(hConf, tableId); }
private static Map<String, Long> getMapFromTable(String rowType) throws IOException { HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get(); HTable hTable = tableUtil.createHTable(hConf, getReplicationStateTableId(tableUtil)); // Scan the table to scan for all regions. ScanBuilder scan = getScanBuilder(tableUtil, rowType); Result result; HashMap<String, Long> timeMap = new HashMap<>(); try (ResultScanner resultScanner = hTable.getScanner(scan.build())) { while ((result = resultScanner.next()) != null) { ReplicationStatusKey key = new ReplicationStatusKey(result.getRow()); String region = key.getRegionName(); Long timestamp = getTimeFromResult(result, rowType); if (timeMap.get(region) == null || timestamp > timeMap.get(region)) { timeMap.put(region, timestamp); } } } catch (Exception e) { LOG.error("Error while reading table.", e); throw Throwables.propagate(e); } finally { hTable.close(); } return timeMap; }
public HBaseMetricsTable(DatasetContext datasetContext, DatasetSpecification spec, Configuration hConf, HBaseTableUtil tableUtil, CConfiguration cConf) throws IOException { this.tableUtil = tableUtil; this.tableId = tableUtil.createHTableId(new NamespaceId(datasetContext.getNamespaceId()), spec.getName()); initializeVars(cConf, spec); HTable hTable = tableUtil.createHTable(hConf, tableId); // todo: make configurable hTable.setWriteBufferSize(HBaseTableUtil.DEFAULT_WRITE_BUFFER_SIZE); hTable.setAutoFlushTo(false); this.hTable = hTable; this.columnFamily = TableProperties.getColumnFamilyBytes(spec.getProperties()); }
private static void dumpReplicationStateTable() throws Exception { System.out.println("\nThis is all the HBase regions on the Cluster:"); HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get(); HTable hTable = tableUtil.createHTable(hConf, getReplicationStateTableId(tableUtil)); ScanBuilder scan = tableUtil.buildScan(); scan.addColumn(Bytes.toBytes(ReplicationConstants.ReplicationStatusTool.TIME_FAMILY), Bytes.toBytes(ReplicationConstants.ReplicationStatusTool.WRITE_TIME_ROW_TYPE)); scan.addColumn(Bytes.toBytes(ReplicationConstants.ReplicationStatusTool.TIME_FAMILY), Bytes.toBytes(ReplicationConstants.ReplicationStatusTool.REPLICATE_TIME_ROW_TYPE)); Result result; try (ResultScanner resultScanner = hTable.getScanner(scan.build())) { while ((result = resultScanner.next()) != null) { ReplicationStatusKey key = new ReplicationStatusKey(result.getRow()); String rowType = key.getRowType(); String region = key.getRegionName(); UUID rsID = key.getRsID(); Long writeTime = getTimeFromResult(result, ReplicationConstants.ReplicationStatusTool.WRITE_TIME_ROW_TYPE); Long replicateTime = getTimeFromResult(result, ReplicationConstants.ReplicationStatusTool.REPLICATE_TIME_ROW_TYPE); System.out.println("Key=>rowType:" + rowType + ":region:" + region + ":RSID:" + rsID + " writeTime:" + writeTime + ":replicateTime:" + replicateTime); } } finally { hTable.close(); } }
public HBaseMetricsTable(DatasetContext datasetContext, DatasetSpecification spec, Configuration hConf, HBaseTableUtil tableUtil, CConfiguration cConf) throws IOException { this.tableUtil = tableUtil; this.tableId = tableUtil.createHTableId(new NamespaceId(datasetContext.getNamespaceId()), spec.getName()); initializeV3Vars(cConf, spec); HTable hTable = tableUtil.createHTable(hConf, tableId); // todo: make configurable hTable.setWriteBufferSize(HBaseTableUtil.DEFAULT_WRITE_BUFFER_SIZE); hTable.setAutoFlushTo(false); this.hTable = hTable; this.columnFamily = TableProperties.getColumnFamilyBytes(spec.getProperties()); }
@Override public HTable createTable(TableId tableId) throws Exception { HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get(); HTableDescriptorBuilder tableDesc = tableUtil.buildHTableDescriptor(tableId); HColumnDescriptor columnDesc = new HColumnDescriptor(FAMILY); columnDesc.setMaxVersions(Integer.MAX_VALUE); columnDesc.setValue(IncrementHandlerState.PROPERTY_TRANSACTIONAL, "false"); tableDesc.addFamily(columnDesc); tableDesc.addCoprocessor(IncrementHandler.class.getName()); HTableDescriptor htd = tableDesc.build(); TEST_HBASE.getHBaseAdmin().createTable(htd); TEST_HBASE.waitUntilTableAvailable(htd.getName(), 5000); return tableUtil.createHTable(conf, tableId); }
@Override public MetadataTable createMetadataTable() throws IOException { TableId tableId = tableUtil.createHTableId(NamespaceId.SYSTEM, metadataTableName); HTable hTable = null; // If the table descriptor is in the cache, we assume the table exists. if (!tableDescriptors.containsKey(tableId)) { synchronized (this) { if (!tableDescriptors.containsKey(tableId)) { try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) { ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(COLUMN_FAMILY), hConf); TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf).addColumnFamily(cfdBuilder.build()); ddlExecutor.createTableIfNotExists(tdBuilder.build(), null); hTable = tableUtil.createHTable(hConf, tableId); tableDescriptors.put(tableId, hTable.getTableDescriptor()); } } } } if (hTable == null) { hTable = tableUtil.createHTable(hConf, tableId); } return new HBaseMetadataTable(tableUtil, hTable, COLUMN_FAMILY, cConf.getInt(Constants.MessagingSystem.HBASE_SCAN_CACHE_ROWS), createExceptionHandler(tableId)); }
@Override public HTable createTable(TableId tableId) throws Exception { HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get(); HTableDescriptorBuilder tableDesc = tableUtil.buildHTableDescriptor(tableId); HColumnDescriptor columnDesc = new HColumnDescriptor(FAMILY); columnDesc.setMaxVersions(Integer.MAX_VALUE); columnDesc.setValue(IncrementHandlerState.PROPERTY_TRANSACTIONAL, "false"); tableDesc.addFamily(columnDesc); tableDesc.addCoprocessor(IncrementHandler.class.getName()); HTableDescriptor htd = tableDesc.build(); TEST_HBASE.getHBaseAdmin().createTable(htd); TEST_HBASE.waitUntilTableAvailable(htd.getName(), 5000); return tableUtil.createHTable(conf, tableId); }
private void writeSome(String namespace, String tableName) throws IOException { HBaseTableUtil tableUtil = getTableUtil(); TableId hTableId = tableUtil.createHTableId(new NamespaceId(namespace), tableName); try (HTable table = tableUtil.createHTable(TEST_HBASE.getConfiguration(), hTableId)) { // writing at least couple megs to reflect in "megabyte"-based metrics for (int i = 0; i < 8; i++) { Put put = new Put(Bytes.toBytes("row" + i)); put.add(Bytes.toBytes("d"), Bytes.toBytes("col" + i), new byte[1024 * 1024]); table.put(put); } } }
@Override public HTable createTable(TableId tableId) throws Exception { HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get(); HTableDescriptorBuilder tableDesc = tableUtil.buildHTableDescriptor(tableId); HColumnDescriptor columnDesc = new HColumnDescriptor(FAMILY); columnDesc.setMaxVersions(Integer.MAX_VALUE); columnDesc.setValue(IncrementHandlerState.PROPERTY_TRANSACTIONAL, "false"); tableDesc.addFamily(columnDesc); tableDesc.addCoprocessor(IncrementHandler.class.getName()); HTableDescriptor htd = tableDesc.build(); TEST_HBASE.getHBaseAdmin().createTable(htd); TEST_HBASE.waitUntilTableAvailable(htd.getName(), 5000); return tableUtil.createHTable(conf, tableId); }
@VisibleForTesting HBaseTable(DatasetContext datasetContext, DatasetSpecification spec, Map<String, String> args, CConfiguration cConf, Configuration hConf, HBaseTableUtil tableUtil, TransactionCodec txCodec) throws IOException { super(PrefixedNamespaces.namespace(cConf, datasetContext.getNamespaceId(), spec.getName()), TableProperties.getReadlessIncrementSupport(spec.getProperties()), spec.getProperties()); TableId hBaseTableId = tableUtil.createHTableId(new NamespaceId(datasetContext.getNamespaceId()), spec.getName()); HTable hTable = tableUtil.createHTable(hConf, hBaseTableId); // todo: make configurable hTable.setWriteBufferSize(HBaseTableUtil.DEFAULT_WRITE_BUFFER_SIZE); hTable.setAutoFlushTo(false); this.tableUtil = tableUtil; this.hTable = hTable; this.hTableName = Bytes.toStringBinary(hTable.getTableName()); this.columnFamily = TableProperties.getColumnFamilyBytes(spec.getProperties()); this.txCodec = txCodec; // Overriding the hbase tx change prefix so it resembles the hbase table name more closely, since the HBase // table name is not the same as the dataset name anymore this.nameAsTxChangePrefix = Bytes.add(new byte[]{(byte) this.hTableName.length()}, Bytes.toBytes(this.hTableName)); this.safeReadlessIncrements = args.containsKey(SAFE_INCREMENTS) && Boolean.valueOf(args.get(SAFE_INCREMENTS)); this.txMaxLifetimeMillis = Bytes.toBytes(TimeUnit.SECONDS.toMillis( cConf.getInt(TxConstants.Manager.CFG_TX_MAX_LIFETIME, TxConstants.Manager.DEFAULT_TX_MAX_LIFETIME))); this.arguments = args; this.properties = spec.getProperties(); }
@Override public HTable createTable(TableId tableId) throws Exception { HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get(); HTableDescriptorBuilder tableDesc = tableUtil.buildHTableDescriptor(tableId); HColumnDescriptor columnDesc = new HColumnDescriptor(FAMILY); columnDesc.setMaxVersions(Integer.MAX_VALUE); columnDesc.setValue(IncrementHandlerState.PROPERTY_TRANSACTIONAL, "false"); tableDesc.addFamily(columnDesc); tableDesc.addCoprocessor(IncrementHandler.class.getName()); HTableDescriptor htd = tableDesc.build(); TEST_HBASE.getHBaseAdmin().createTable(htd); TEST_HBASE.waitUntilTableAvailable(htd.getName(), 5000); return tableUtil.createHTable(conf, tableId); }
@VisibleForTesting HBaseTable(DatasetContext datasetContext, DatasetSpecification spec, Map<String, String> args, CConfiguration cConf, Configuration hConf, HBaseTableUtil tableUtil, TransactionCodec txCodec) throws IOException { super(PrefixedNamespaces.namespace(cConf, datasetContext.getNamespaceId(), spec.getName()), TableProperties.getReadlessIncrementSupport(spec.getProperties()), spec.getProperties()); TableId hBaseTableId = tableUtil.createHTableId(new NamespaceId(datasetContext.getNamespaceId()), spec.getName()); HTable hTable = tableUtil.createHTable(hConf, hBaseTableId); // todo: make configurable hTable.setWriteBufferSize(HBaseTableUtil.DEFAULT_WRITE_BUFFER_SIZE); hTable.setAutoFlushTo(false); this.tableUtil = tableUtil; this.hTable = hTable; this.hTableName = Bytes.toStringBinary(hTable.getTableName()); this.columnFamily = TableProperties.getColumnFamilyBytes(spec.getProperties()); this.txCodec = txCodec; // Overriding the hbase tx change prefix so it resembles the hbase table name more closely, since the HBase // table name is not the same as the dataset name anymore this.nameAsTxChangePrefix = Bytes.add(new byte[]{(byte) this.hTableName.length()}, Bytes.toBytes(this.hTableName)); this.safeReadlessIncrements = args.containsKey(SAFE_INCREMENTS) && Boolean.valueOf(args.get(SAFE_INCREMENTS)); this.txMaxLifetimeMillis = Bytes.toBytes(TimeUnit.SECONDS.toMillis( cConf.getInt(TxConstants.Manager.CFG_TX_MAX_LIFETIME, TxConstants.Manager.DEFAULT_TX_MAX_LIFETIME))); this.arguments = args; this.properties = spec.getProperties(); }
@Override public HTable createTable(TableId tableId) throws Exception { HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get(); HTableDescriptorBuilder tableDesc = tableUtil.buildHTableDescriptor(tableId); HColumnDescriptor columnDesc = new HColumnDescriptor(FAMILY); columnDesc.setMaxVersions(Integer.MAX_VALUE); columnDesc.setValue(IncrementHandlerState.PROPERTY_TRANSACTIONAL, "false"); tableDesc.addFamily(columnDesc); tableDesc.addCoprocessor(IncrementHandler.class.getName()); HTableDescriptor htd = tableDesc.build(); TEST_HBASE.getHBaseAdmin().createTable(htd); TEST_HBASE.waitUntilTableAvailable(htd.getName(), 5000); return tableUtil.createHTable(conf, tableId); }
@Override protected StreamConsumer create(TableId tableId, StreamConfig streamConfig, ConsumerConfig consumerConfig, StreamConsumerStateStore stateStore, StreamConsumerState beginConsumerState, FileReader<StreamEventOffset, Iterable<StreamFileOffset>> reader, @Nullable ReadFilter extraFilter) throws IOException { int splits = cConf.getInt(Constants.Stream.CONSUMER_TABLE_PRESPLITS); AbstractRowKeyDistributor distributor = new RowKeyDistributorByHashPrefix( new RowKeyDistributorByHashPrefix.OneByteSimpleHash(splits)); byte[][] splitKeys = HBaseTableUtil.getSplitKeys(splits, splits, distributor); TableId hBaseTableId = tableUtil.createHTableId(new NamespaceId(tableId.getNamespace()), tableId.getTableName()); TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(hBaseTableId, cConf); ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(QueueEntryRow.COLUMN_FAMILY), hConf); tdBuilder.addColumnFamily(cfdBuilder.build()); tdBuilder.addProperty(QueueConstants.DISTRIBUTOR_BUCKETS, Integer.toString(splits)); try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) { ddlExecutor.createTableIfNotExists(tdBuilder.build(), splitKeys); } HTable hTable = tableUtil.createHTable(hConf, hBaseTableId); hTable.setWriteBufferSize(Constants.Stream.HBASE_WRITE_BUFFER_SIZE); hTable.setAutoFlushTo(false); return new HBaseStreamFileConsumer(cConf, streamConfig, consumerConfig, tableUtil, hTable, reader, stateStore, beginConsumerState, extraFilter, createKeyDistributor(hTable.getTableDescriptor())); }
@Override public HTable createTable(TableId tableId) throws Exception { HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get(); HTableDescriptorBuilder tableDesc = tableUtil.buildHTableDescriptor(tableId); HColumnDescriptor columnDesc = new HColumnDescriptor(FAMILY); columnDesc.setMaxVersions(Integer.MAX_VALUE); columnDesc.setValue(IncrementHandlerState.PROPERTY_TRANSACTIONAL, "false"); tableDesc.addFamily(columnDesc); tableDesc.addCoprocessor(IncrementHandler.class.getName()); HTableDescriptor htd = tableDesc.build(); TEST_HBASE.getHBaseAdmin().createTable(htd); TEST_HBASE.waitUntilTableAvailable(htd.getName(), 5000); return tableUtil.createHTable(conf, tableId); }
@Override public synchronized StreamConsumerStateStore create(StreamConfig streamConfig) throws IOException { NamespaceId namespace = streamConfig.getStreamId().getParent(); TableId streamStateStoreTableId = StreamUtils.getStateStoreTableId(namespace); TableId hbaseTableId = tableUtil.createHTableId(new NamespaceId(streamStateStoreTableId.getNamespace()), streamStateStoreTableId.getTableName()); boolean tableExist; try (HBaseAdmin admin = new HBaseAdmin(hConf)) { tableExist = tableUtil.tableExists(admin, hbaseTableId); } if (!tableExist) { try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) { TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(hbaseTableId, cConf); ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(QueueEntryRow.COLUMN_FAMILY), hConf); tdBuilder.addColumnFamily(cfdBuilder.build()); ddlExecutor.createTableIfNotExists(tdBuilder.build(), null); } } HTable hTable = tableUtil.createHTable(hConf, hbaseTableId); hTable.setWriteBufferSize(Constants.Stream.HBASE_WRITE_BUFFER_SIZE); hTable.setAutoFlushTo(false); return new HBaseStreamConsumerStateStore(streamConfig, hTable); }
@Override public HTable createTable(TableId tableId) throws Exception { HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get(); HTableDescriptorBuilder tableDesc = tableUtil.buildHTableDescriptor(tableId); HColumnDescriptor columnDesc = new HColumnDescriptor(FAMILY); columnDesc.setMaxVersions(Integer.MAX_VALUE); columnDesc.setValue(IncrementHandlerState.PROPERTY_TRANSACTIONAL, "false"); tableDesc.addFamily(columnDesc); tableDesc.addCoprocessor(IncrementHandler.class.getName()); HTableDescriptor htd = tableDesc.build(); TEST_HBASE.getHBaseAdmin().createTable(htd); TEST_HBASE.waitUntilTableAvailable(htd.getName(), 5000); return tableUtil.createHTable(conf, tableId); }