private static void updateLatestPriority(HTableDescriptor desc, long priority) { // update the max priority desc.setValue(COUNTER_KEY, Long.toString(priority)); }
/** * Write the given key and associated configuration to the * {@link HTableDescriptor} */ private static void writeConstraint(HTableDescriptor desc, String key, Configuration conf) throws IOException { // store the key and conf in the descriptor desc.setValue(key, serializeConfiguration(conf)); }
/** * @return a table descriptor */ @JsonIgnore public HTableDescriptor getTableDescriptor() { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(getName())); for (Map.Entry<QName, Object> e: getAny().entrySet()) { htd.setValue(e.getKey().getLocalPart(), e.getValue().toString()); } for (ColumnSchemaModel column: getColumns()) { HColumnDescriptor hcd = new HColumnDescriptor(column.getName()); for (Map.Entry<QName, Object> e: column.getAny().entrySet()) { hcd.setValue(e.getKey().getLocalPart(), e.getValue().toString()); } htd.addFamily(hcd); } return htd; }
private void updateHtable(String tableName) throws IOException { HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName)); if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) { desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix()); hbaseAdmin.disableTable(TableName.valueOf(tableName)); hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc); hbaseAdmin.enableTable(TableName.valueOf(tableName)); updatedResources.add(tableName); } }
public void fixInconsistent() throws IOException { if (ifFix == true) { for (String segFullName : inconsistentHTables) { String[] sepNameList = segFullName.split(","); HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0])); logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix()); hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0])); desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix()); hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc); hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0])); } } else { logger.info("------ Inconsistent HTables Needed To Be Fixed ------"); for (String hTable : inconsistentHTables) { String[] sepNameList = hTable.split(","); logger.info(sepNameList[0] + " belonging to cube " + sepNameList[1]); } logger.info("----------------------------------------------------"); } }
private void clean() throws IOException { Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); Admin hbaseAdmin = conn.getAdmin(); for (HTableDescriptor descriptor : hbaseAdmin.listTables()) { String name = descriptor.getNameAsString().toLowerCase(Locale.ROOT); if (name.startsWith("kylin") || name.startsWith("_kylin")) { String x = descriptor.getValue(IRealizationConstants.HTableTag); System.out.println("table name " + descriptor.getNameAsString() + " host: " + x); System.out.println(descriptor); System.out.println(); descriptor.setValue(IRealizationConstants.HTableOwner, "whoami@kylin.apache.org"); hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor); } } hbaseAdmin.close(); }
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */ public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException { Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin(); try { if (admin.tableExists(tableName)) { logger.info("disabling hbase table " + tableName); admin.disableTable(tableName); logger.info("deleting hbase table " + tableName); admin.deleteTable(tableName); } HTableDescriptor tableDesc = new HTableDescriptor(tableName); tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); tableDesc.addFamily(createColumnFamily(kylinConfig, cfName, false)); logger.info("creating hbase table " + tableName); admin.createTable(tableDesc, null); Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons"); logger.info("create hbase table " + tableName + " done."); } finally { IOUtils.closeQuietly(admin); } }
HTableDescriptor hTableDesc = new HTableDescriptor(tableName); hTableDesc.setCompactionEnabled(false); hTableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); hTableDesc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix()); hTableDesc.setValue(IRealizationConstants.HTableCreationTime, String.valueOf(System.currentTimeMillis())); String commitInfo = KylinVersion.getGitCommitInfo(); if (!StringUtils.isEmpty(commitInfo)) { hTableDesc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
HTableDescriptor htd = new HTableDescriptor(name); for (Map.Entry<QName,Object> e: model.getAny().entrySet()) { htd.setValue(e.getKey().getLocalPart(), e.getValue().toString());
public void fixInconsistent() throws IOException { if (ifFix == true) { for (String segFullName : inconsistentHTables) { String[] sepNameList = StringUtil.splitByComma(segFullName); HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0])); logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix()); hbaseAdmin.disableTable(sepNameList[0]); desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix()); hbaseAdmin.modifyTable(sepNameList[0], desc); hbaseAdmin.enableTable(sepNameList[0]); } } else { logger.info("------ Inconsistent HTables Needed To Be Fixed ------"); for (String hTable : inconsistentHTables) { String[] sepNameList = StringUtil.splitByComma(hTable); logger.info(sepNameList[0] + " belonging to cube " + sepNameList[1]); } logger.info("----------------------------------------------------"); } }
/********************************* Master related hooks **********************************/ @Override public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException { // Need to create the new system table for labels here if (!MetaTableAccessor.tableExists(ctx.getEnvironment().getConnection(), LABELS_TABLE_NAME)) { HTableDescriptor labelsTable = new HTableDescriptor(LABELS_TABLE_NAME); HColumnDescriptor labelsColumn = new HColumnDescriptor(LABELS_TABLE_FAMILY); labelsColumn.setBloomFilterType(BloomType.NONE); labelsColumn.setBlockCacheEnabled(false); // We will cache all the labels. No need of normal // table block cache. labelsTable.addFamily(labelsColumn); // Let the "labels" table having only one region always. We are not expecting too many labels in // the system. labelsTable.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); try (Admin admin = ctx.getEnvironment().getConnection().getAdmin()) { admin.createTable(labelsTable); } } }
/** * Test setting up a customized split policy */ @Test public void testCustomPolicy() throws IOException { HTableDescriptor myHtd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); myHtd.setValue(HTableDescriptor.SPLIT_POLICY, KeyPrefixRegionSplitPolicy.class.getName()); myHtd.setValue(KeyPrefixRegionSplitPolicy.PREFIX_LENGTH_KEY, String.valueOf(2)); HRegion myMockRegion = Mockito.mock(HRegion.class); Mockito.doReturn(myHtd).when(myMockRegion).getTableDescriptor(); Mockito.doReturn(stores).when(myMockRegion).getStores(); HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); Mockito.doReturn(Optional.of(Bytes.toBytes("abcd"))).when(mockStore).getSplitPoint(); stores.add(mockStore); KeyPrefixRegionSplitPolicy policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy .create(myMockRegion, conf); assertEquals("ab", Bytes.toString(policy.getSplitPoint())); Mockito.doReturn(true).when(myMockRegion).shouldForceSplit(); Mockito.doReturn(Bytes.toBytes("efgh")).when(myMockRegion) .getExplicitSplitPoint(); policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy .create(myMockRegion, conf); assertEquals("ef", Bytes.toString(policy.getSplitPoint())); }
@Test public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException { HTableDescriptor myHtd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); myHtd.setValue(HTableDescriptor.SPLIT_POLICY, DelimitedKeyPrefixRegionSplitPolicy.class.getName()); myHtd.setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ","); HRegion myMockRegion = Mockito.mock(HRegion.class); Mockito.doReturn(myHtd).when(myMockRegion).getTableDescriptor(); Mockito.doReturn(stores).when(myMockRegion).getStores(); HStore mockStore = Mockito.mock(HStore.class); Mockito.doReturn(2000L).when(mockStore).getSize(); Mockito.doReturn(true).when(mockStore).canSplit(); Mockito.doReturn(Optional.of(Bytes.toBytes("ab,cd"))).when(mockStore).getSplitPoint(); stores.add(mockStore); DelimitedKeyPrefixRegionSplitPolicy policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy .create(myMockRegion, conf); assertEquals("ab", Bytes.toString(policy.getSplitPoint())); Mockito.doReturn(true).when(myMockRegion).shouldForceSplit(); Mockito.doReturn(Bytes.toBytes("efg,h")).when(myMockRegion) .getExplicitSplitPoint(); policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy .create(myMockRegion, conf); assertEquals("efg", Bytes.toString(policy.getSplitPoint())); Mockito.doReturn(Bytes.toBytes("ijk")).when(myMockRegion) .getExplicitSplitPoint(); assertEquals("ijk", Bytes.toString(policy.getSplitPoint())); }
desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
private void createTableWithNonDefaultProperties() throws Exception { final long startTime = System.currentTimeMillis(); final String sourceTableNameAsString = STRING_TABLE_NAME + startTime; originalTableName = TableName.valueOf(sourceTableNameAsString); // enable replication on a column family HColumnDescriptor maxVersionsColumn = new HColumnDescriptor(MAX_VERSIONS_FAM); HColumnDescriptor bloomFilterColumn = new HColumnDescriptor(BLOOMFILTER_FAM); HColumnDescriptor dataBlockColumn = new HColumnDescriptor(COMPRESSED_FAM); HColumnDescriptor blockSizeColumn = new HColumnDescriptor(BLOCKSIZE_FAM); maxVersionsColumn.setMaxVersions(MAX_VERSIONS); bloomFilterColumn.setBloomFilterType(BLOOM_TYPE); dataBlockColumn.setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE); blockSizeColumn.setBlocksize(BLOCK_SIZE); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(sourceTableNameAsString)); htd.addFamily(maxVersionsColumn); htd.addFamily(bloomFilterColumn); htd.addFamily(dataBlockColumn); htd.addFamily(blockSizeColumn); htd.setValue(TEST_CUSTOM_VALUE, TEST_CUSTOM_VALUE); htd.setConfiguration(TEST_CONF_CUSTOM_VALUE, TEST_CONF_CUSTOM_VALUE); assertTrue(htd.getConfiguration().size() > 0); admin.createTable(htd); Table original = UTIL.getConnection().getTable(originalTableName); originalTableName = TableName.valueOf(sourceTableNameAsString); originalTableDescriptor = admin.getTableDescriptor(originalTableName); originalTableDescription = originalTableDescriptor.toStringCustomizedValues(); original.close(); }
private void alter() throws IOException { Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); Admin hbaseAdmin = null; try { hbaseAdmin = conn.getAdmin(); HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName)); hbaseAdmin.disableTable(table.getTableName()); table.setValue(metadataKey, metadataValue); hbaseAdmin.modifyTable(table.getTableName(), table); hbaseAdmin.enableTable(table.getTableName()); } finally { if (hbaseAdmin != null) { hbaseAdmin.close(); } } }
htd.setValue(HTableDescriptor.SPLIT_POLICY, ConstantSizeRegionSplitPolicy.class.getName()); htd.addFamily(new HColumnDescriptor(FAMILY1)); htd.addFamily(new HColumnDescriptor(FAMILY2));
/** * Test that we add and remove strings from settings properly. * @throws Exception */ @Test public void testAddGetRemoveString() throws Exception { HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName())); String key = "Some"; String value = "value"; desc.setValue(key, value); assertEquals(value, desc.getValue(key)); desc.remove(key); assertEquals(null, desc.getValue(key)); String keyShouldNotNull = "Some2"; String valueIsNull = null; desc.setValue(keyShouldNotNull, valueIsNull); assertEquals(valueIsNull, desc.getValue(keyShouldNotNull)); desc.remove(keyShouldNotNull); assertEquals(null, desc.getValue(keyShouldNotNull)); }
@Test // HBASE-6308: Test CP classloader is the CoprocessorClassLoader public void testPrivateClassLoader() throws Exception { File jarFile = buildCoprocessorJar(cpName4); // create a table that references the jar HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName4)); htd.addFamily(new HColumnDescriptor("test")); htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" + Coprocessor.PRIORITY_USER); Admin admin = TEST_UTIL.getAdmin(); admin.createTable(htd); waitForTable(htd.getTableName()); // verify that the coprocessor was loaded correctly boolean found = false; MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster(); for (HRegion region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) { if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName4)) { Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4); if (cp != null) { found = true; assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader", cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class); } } } assertTrue("Class " + cpName4 + " was missing on a region", found); }
@Test // HBASE-3516: Test CP Class loading from local file system public void testClassLoadingFromLocalFS() throws Exception { File jarFile = buildCoprocessorJar(cpName3); // create a table that references the jar HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName3)); htd.addFamily(new HColumnDescriptor("test")); htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" + Coprocessor.PRIORITY_USER); Admin admin = TEST_UTIL.getAdmin(); admin.createTable(htd); waitForTable(htd.getTableName()); // verify that the coprocessor was loaded boolean found = false; MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster(); for (HRegion region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) { if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName3)) { found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null); } } assertTrue("Class " + cpName3 + " was missing on a region", found); }