private Connection getHBaseConnection(KylinConfig kylinConfig) throws IOException { return HBaseConnection.get(kylinConfig.getStorageUrl()); }
public static Configuration getCurrentHBaseConfiguration() { if (configThreadLocal.get() == null) { StorageURL storageUrl = KylinConfig.getInstanceFromEnv().getStorageUrl(); configThreadLocal.set(newHBaseConfiguration(storageUrl)); } return configThreadLocal.get(); }
public CubeMigrationCheckCLI(KylinConfig kylinConfig, Boolean isFix) throws IOException { this.dstCfg = kylinConfig; this.ifFix = isFix; Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl()); hbaseAdmin = conn.getAdmin(); issueExistHTables = Lists.newArrayList(); inconsistentHTables = Lists.newArrayList(); }
protected void cleanUnusedHBaseTables() throws IOException { if ("hbase".equals(config.getStorageUrl().getScheme()) && !"".equals(config.getMetadataUrl().getScheme())) { final int deleteTimeoutMin = 10; // Unit minute try { // use reflection to isolate NoClassDef errors when HBase is not available Class hbaseCleanUpUtil = Class.forName("org.apache.kylin.rest.job.StorageCleanJobHbaseUtil"); Method cleanUnusedHBaseTables = hbaseCleanUpUtil.getDeclaredMethod("cleanUnusedHBaseTables", boolean.class, int.class); hbaseGarbageTables = (List<String>) cleanUnusedHBaseTables.invoke(hbaseCleanUpUtil, delete, deleteTimeoutMin); } catch (Throwable e) { logger.error("Error during HBase clean up", e); } } }
Writer() throws IOException { Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); table = conn.getBufferedMutator(htableName); }
public static void deleteHTable(TableName tableName) throws IOException { Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin(); try { if (admin.tableExists(tableName)) { logger.info("disabling hbase table " + tableName); admin.disableTable(tableName); logger.info("deleting hbase table " + tableName); admin.deleteTable(tableName); } } finally { IOUtils.closeQuietly(admin); } }
@SuppressWarnings("unused") // used by reflection public static HBaseResponse getHBaseInfo(String tableName, KylinConfig config) throws IOException { if (!config.getStorageUrl().getScheme().equals("hbase")) return null; Connection conn = HBaseConnection.get(config.getStorageUrl()); HBaseResponse hr = null; long tableSize = 0; int regionCount = 0; HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn); Map<byte[], Long> sizeMap = cal.getRegionSizeMap(); for (long s : sizeMap.values()) { tableSize += s; } regionCount = sizeMap.size(); // Set response. hr = new HBaseResponse(); hr.setTableSize(tableSize); hr.setRegionCount(regionCount); return hr; } }
private void convertToResourceStore(KylinConfig kylinConfig, String tableName, ResourceStore store, ResultConverter converter) throws IOException { Table table = null; ResultScanner rs = null; Scan scan = new Scan(); try { table = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(tableName)); rs = table.getScanner(scan); converter.convertResult(rs, store); store.checkAndPutResource(MIGRATE_OK_PREFIX + tableName, new StringEntity(tableName + " migrated"), StringEntity.serializer); } finally { IOUtils.closeQuietly(rs); IOUtils.closeQuietly(table); } }
private void clean() throws IOException { Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); Admin hbaseAdmin = conn.getAdmin(); for (HTableDescriptor descriptor : hbaseAdmin.listTables()) { String name = descriptor.getNameAsString().toLowerCase(Locale.ROOT); if (name.startsWith("kylin") || name.startsWith("_kylin")) { String x = descriptor.getValue(IRealizationConstants.HTableTag); System.out.println("table name " + descriptor.getNameAsString() + " host: " + x); System.out.println(descriptor); System.out.println(); descriptor.setValue(IRealizationConstants.HTableOwner, "whoami@kylin.apache.org"); hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor); } } hbaseAdmin.close(); }
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */ public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException { Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin(); try { if (admin.tableExists(tableName)) { logger.info("disabling hbase table " + tableName); admin.disableTable(tableName); logger.info("deleting hbase table " + tableName); admin.deleteTable(tableName); } HTableDescriptor tableDesc = new HTableDescriptor(tableName); tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); tableDesc.addFamily(createColumnFamily(kylinConfig, cfName, false)); logger.info("creating hbase table " + tableName); admin.createTable(tableDesc, null); Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons"); logger.info("create hbase table " + tableName + " done."); } finally { IOUtils.closeQuietly(admin); } }
private void init() { this.config = KylinConfig.getInstanceFromEnv(); this.storageUrl = config.getStorageUrl(); this.starSchemaUrl = config.getHiveUrl(); this.starSchemaUser = config.getHiveUser(); this.starSchemaPassword = config.getHivePassword(); }
public HBaseLookupTable(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshot) { String tableName = extTableSnapshot.getStorageLocationIdentifier(); this.lookupTableName = TableName.valueOf(tableName); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); Connection connection = HBaseConnection.get(kylinConfig.getStorageUrl()); try { table = connection.getTable(lookupTableName); } catch (IOException e) { throw new RuntimeException("error when connect HBase", e); } String[] keyColumns = extTableSnapshot.getKeyColumns(); encoder = new HBaseLookupRowEncoder(tableDesc, keyColumns, extTableSnapshot.getShardNum()); }
private static void show() throws IOException { Map<String, List<String>> envs = Maps.newHashMap(); // get all kylin hbase tables KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl()); Admin hbaseAdmin = conn.getAdmin(); String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix(); HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*"); for (HTableDescriptor desc : tableDescriptors) { String host = desc.getValue(IRealizationConstants.HTableTag); if (StringUtils.isEmpty(host)) { add("unknown", desc.getNameAsString(), envs); } else { add(host, desc.getNameAsString(), envs); } } for (Map.Entry<String, List<String>> entry : envs.entrySet()) { System.out.println(entry.getKey() + " has htable count: " + entry.getValue().size()); } hbaseAdmin.close(); }
Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl()); Admin hbaseAdmin = conn.getAdmin(); String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
Reader() throws IOException { Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); table = conn.getTable(htableName); Scan scan = new Scan(); scan.addFamily(CF_B); scan.setCaching(1024); scan.setCacheBlocks(true); scanner = table.getScanner(scan); }
private void cleanUnusedHBaseTables(Configuration conf) throws IOException { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
Table aclHtable = null; try { aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
Table aclHtable = null; try { aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
Connection conn = HBaseConnection.get(srcConfig.getStorageUrl()); hbaseAdmin = conn.getAdmin();
private void alter() throws IOException { Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); Admin hbaseAdmin = null; try { hbaseAdmin = conn.getAdmin(); HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName)); hbaseAdmin.disableTable(table.getTableName()); table.setValue(metadataKey, metadataValue); hbaseAdmin.modifyTable(table.getTableName(), table); hbaseAdmin.enableTable(table.getTableName()); } finally { if (hbaseAdmin != null) { hbaseAdmin.close(); } } }