private static long getNextPriority(HTableDescriptor desc) { String value = desc.getValue(COUNTER_KEY); long priority; // get the current priority if (value == null) { priority = MIN_PRIORITY; } else { priority = Long.parseLong(value) + 1; } return priority; }
assertEquals(originalTableDescriptor.getConfiguration().size(), cloneHtd.getConfiguration().size()); assertEquals(TEST_CUSTOM_VALUE, cloneHtd.getValue(TEST_CUSTOM_VALUE)); assertEquals(TEST_CONF_CUSTOM_VALUE, cloneHtd.getConfigurationValue(TEST_CONF_CUSTOM_VALUE)); assertEquals(originalTableDescriptor.getValues(), cloneHtd.getValues());
public void check(List<String> segFullNameList) { issueExistHTables = Lists.newArrayList(); inconsistentHTables = Lists.newArrayList(); for (String segFullName : segFullNameList) { String[] sepNameList = segFullName.split(","); try { HTableDescriptor hTableDescriptor = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0])); String host = hTableDescriptor.getValue(IRealizationConstants.HTableTag); if (!dstCfg.getMetadataUrlPrefix().equalsIgnoreCase(host)) { inconsistentHTables.add(segFullName); } } catch (IOException e) { issueExistHTables.add(segFullName); continue; } } }
private static List<String> filterByGitCommit(Admin hbaseAdmin, List<String> tableNames) throws IOException { List<String> result = Lists.newLinkedList(); List<String> filteredList = Lists.newLinkedList(); String commitInfo = KylinVersion.getGitCommitInfo(); if (StringUtils.isEmpty(commitInfo)) { return tableNames; } logger.info("Commit Information: " + commitInfo); for (String tableName : tableNames) { HTableDescriptor tableDesc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName)); String gitTag = tableDesc.getValue(IRealizationConstants.HTableGitTag); if (commitInfo.equals(gitTag)) { filteredList.add(tableName); } else { result.add(tableName); } } logger.info("Filtered tables don't need to deploy coprocessors: " + filteredList); return result; }
public void check(List<String> segFullNameList) { issueExistHTables = Lists.newArrayList(); inconsistentHTables = Lists.newArrayList(); for (String segFullName : segFullNameList) { String[] sepNameList = StringUtil.splitByComma(segFullName); try { HTableDescriptor hTableDescriptor = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0])); String host = hTableDescriptor.getValue(IRealizationConstants.HTableTag); if (!dstCfg.getMetadataUrlPrefix().equalsIgnoreCase(host)) { inconsistentHTables.add(segFullName); } } catch (IOException e) { issueExistHTables.add(segFullName); continue; } } }
/** * Get the kv {@link Entry} in the descriptor for the specified class * * @param desc {@link HTableDescriptor} to read * @param clazz To search for * @return The {@link Pair} of {@literal <key, value>} in the table, if that class is * present. {@code NULL} otherwise. */ private static Pair<String, String> getKeyValueForClass( HTableDescriptor desc, Class<? extends Constraint> clazz) { // get the serialized version of the constraint String key = serializeConstraintClass(clazz); String value = desc.getValue(key); return value == null ? null : new Pair<>(key, value); }
private void updateHtable(String tableName) throws IOException { HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName)); if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) { desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix()); hbaseAdmin.disableTable(TableName.valueOf(tableName)); hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc); hbaseAdmin.enableTable(TableName.valueOf(tableName)); updatedResources.add(tableName); } }
@Override protected String createMetaStoreUUID() throws IOException { try (final Admin hbaseAdmin = HBaseConnection.get(metadataUrl).getAdmin()) { final String metaStoreName = metadataUrl.getIdentifier(); final HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(metaStoreName)); String uuid = desc.getValue(HBaseConnection.HTABLE_UUID_TAG); if (uuid != null) return uuid; return UUID.randomUUID().toString(); } catch (Exception e) { return null; } }
public void fixInconsistent() throws IOException { if (ifFix == true) { for (String segFullName : inconsistentHTables) { String[] sepNameList = segFullName.split(","); HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0])); logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix()); hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0])); desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix()); hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc); hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0])); } } else { logger.info("------ Inconsistent HTables Needed To Be Fixed ------"); for (String hTable : inconsistentHTables) { String[] sepNameList = hTable.split(","); logger.info(sepNameList[0] + " belonging to cube " + sepNameList[1]); } logger.info("----------------------------------------------------"); } }
private void clean() throws IOException { Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); Admin hbaseAdmin = conn.getAdmin(); for (HTableDescriptor descriptor : hbaseAdmin.listTables()) { String name = descriptor.getNameAsString().toLowerCase(Locale.ROOT); if (name.startsWith("kylin") || name.startsWith("_kylin")) { String x = descriptor.getValue(IRealizationConstants.HTableTag); System.out.println("table name " + descriptor.getNameAsString() + " host: " + x); System.out.println(descriptor); System.out.println(); descriptor.setValue(IRealizationConstants.HTableOwner, "whoami@kylin.apache.org"); hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor); } } hbaseAdmin.close(); }
if (admin.tableExists(TableName.valueOf(table))) { HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table))); String host = tableDescriptor.getValue(IRealizationConstants.HTableTag); if (metadataUrlPrefix.equalsIgnoreCase(host)) { if (admin.isTableEnabled(TableName.valueOf(table))) {
public void fixInconsistent() throws IOException { if (ifFix == true) { for (String segFullName : inconsistentHTables) { String[] sepNameList = StringUtil.splitByComma(segFullName); HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0])); logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix()); hbaseAdmin.disableTable(sepNameList[0]); desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix()); hbaseAdmin.modifyTable(sepNameList[0], desc); hbaseAdmin.enableTable(sepNameList[0]); } } else { logger.info("------ Inconsistent HTables Needed To Be Fixed ------"); for (String hTable : inconsistentHTables) { String[] sepNameList = StringUtil.splitByComma(hTable); logger.info(sepNameList[0] + " belonging to cube " + sepNameList[1]); } logger.info("----------------------------------------------------"); } }
private static void show() throws IOException { Map<String, List<String>> envs = Maps.newHashMap(); // get all kylin hbase tables KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl()); Admin hbaseAdmin = conn.getAdmin(); String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix(); HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*"); for (HTableDescriptor desc : tableDescriptors) { String host = desc.getValue(IRealizationConstants.HTableTag); if (StringUtils.isEmpty(host)) { add("unknown", desc.getNameAsString(), envs); } else { add(host, desc.getNameAsString(), envs); } } for (Map.Entry<String, List<String>> entry : envs.entrySet()) { System.out.println(entry.getKey() + " has htable count: " + entry.getValue().size()); } hbaseAdmin.close(); }
List<String> allTablesNeedToBeDropped = Lists.newArrayList(); for (HTableDescriptor desc : tableDescriptors) { String host = desc.getValue(IRealizationConstants.HTableTag); if (tag.equalsIgnoreCase(host)) { allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
List<String> allTablesNeedToBeDropped = new ArrayList<String>(); for (HTableDescriptor desc : tableDescriptors) { String host = desc.getValue(IRealizationConstants.HTableTag); if (kylinConfig.getMetadataUrlPrefix().equalsIgnoreCase(host)) {
List<String> allTablesNeedToBeDropped = new ArrayList<String>(); for (HTableDescriptor desc : tableDescriptors) { String host = desc.getValue(IRealizationConstants.HTableTag); if (!metastoreWhitelistSet.contains(host)) { logger.info("HTable {} is recognized as orphan because its tag is {}", desc.getTableName(), host);
@Test public void test() throws IOException { HBaseAdmin hBaseAdmin = mock(HBaseAdmin.class); HTableDescriptor[] hds = new HTableDescriptor[2]; HTableDescriptor d1 = mock(HTableDescriptor.class); HTableDescriptor d2 = mock(HTableDescriptor.class); hds[0] = d1; hds[1] = d2; when(d1.getValue("KYLIN_HOST")).thenReturn("../examples/test_metadata/"); when(d2.getValue("KYLIN_HOST")).thenReturn("../examples/test_metadata/"); when(d1.getTableName()).thenReturn(TableName.valueOf("KYLIN_J9TE08D9IA")); String toBeDel = "to-be-del"; when(d2.getTableName()).thenReturn(TableName.valueOf(toBeDel)); when(hBaseAdmin.listTables("KYLIN_.*")).thenReturn(hds); when(hBaseAdmin.tableExists(toBeDel)).thenReturn(true); when(hBaseAdmin.isTableEnabled(toBeDel)).thenReturn(false); StorageCleanJobHbaseUtil.cleanUnusedHBaseTables(hBaseAdmin, true, 100000); ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class); verify(hBaseAdmin).deleteTable(captor.capture()); assertEquals(Lists.newArrayList(toBeDel), captor.getAllValues()); } }
/** * Test that we add and remove strings from settings properly. * @throws Exception */ @Test public void testAddGetRemoveString() throws Exception { HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName())); String key = "Some"; String value = "value"; desc.setValue(key, value); assertEquals(value, desc.getValue(key)); desc.remove(key); assertEquals(null, desc.getValue(key)); String keyShouldNotNull = "Some2"; String valueIsNull = null; desc.setValue(keyShouldNotNull, valueIsNull); assertEquals(valueIsNull, desc.getValue(keyShouldNotNull)); desc.remove(keyShouldNotNull); assertEquals(null, desc.getValue(keyShouldNotNull)); }
String host = desc.getValue(IRealizationConstants.HTableTag); if (!host.equalsIgnoreCase(kylinConfig.getMetadataUrlPrefix())) { logger.warn("This server doesn't own this table: " + tableName);
assertEquals("wrong table descriptor returned", desc.getTableName(), tableAname); value = desc.getValue(attrName); assertFalse("missing HTD attribute value", value == null); assertFalse("HTD attribute value is incorrect",