private static void createHTableIfNeeded(Connection conn, String tableName) throws IOException { Admin hbase = conn.getAdmin(); try { boolean tableExist = false; try { hbase.getTableDescriptor(TableName.valueOf(tableName)); tableExist = true; } catch (TableNotFoundException e) { //do nothing? } if (tableExist) { logger.info("HTable '{}' already exists", tableName); return; } logger.info("Creating HTable '{}'", tableName); HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); HColumnDescriptor fd = new HColumnDescriptor(CF); fd.setBlocksize(CELL_SIZE); desc.addFamily(fd); hbase.createTable(desc); logger.info("HTable '{}' created", tableName); } finally { hbase.close(); } }
private void assertReadOnly(HColumnDescriptor hcd) { try { hcd.setBlocksize(10); fail("ImmutableHColumnDescriptor can't be modified!!!"); } catch (UnsupportedOperationException e) { } }
@Test(expected=IllegalArgumentException.class) public void testAddDuplicateFamilies() { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); byte[] familyName = Bytes.toBytes("cf"); HColumnDescriptor hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(1000); htd.addFamily(hcd); assertEquals(1000, htd.getFamily(familyName).getBlocksize()); hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(2000); htd.addFamily(hcd); }
private void setupMockColumnFamiliesForBlockSize(Table table, Map<String, Integer> familyToDataBlockEncoding) throws IOException { HTableDescriptor mockTableDescriptor = new HTableDescriptor(TABLE_NAMES[0]); for (Entry<String, Integer> entry : familyToDataBlockEncoding.entrySet()) { mockTableDescriptor.addFamily(new HColumnDescriptor(entry.getKey()) .setMaxVersions(1) .setBlocksize(entry.getValue()) .setBlockCacheEnabled(false) .setTimeToLive(0)); } Mockito.doReturn(mockTableDescriptor).when(table).getTableDescriptor(); }
@Test public void testBlocksScanned() throws Exception { byte [] tableName = Bytes.toBytes("TestBlocksScanned"); HTableDescriptor table = new HTableDescriptor(TableName.valueOf(tableName)); table.addFamily( new HColumnDescriptor(FAMILY) .setMaxVersions(10) .setBlockCacheEnabled(true) .setBlocksize(BLOCK_SIZE) .setCompressionType(Compression.Algorithm.NONE) ); _testBlocksScanned(table); }
@Test public void testBlocksScannedWithEncoding() throws Exception { byte [] tableName = Bytes.toBytes("TestBlocksScannedWithEncoding"); HTableDescriptor table = new HTableDescriptor(TableName.valueOf(tableName)); table.addFamily( new HColumnDescriptor(FAMILY) .setMaxVersions(10) .setBlockCacheEnabled(true) .setDataBlockEncoding(DataBlockEncoding.FAST_DIFF) .setBlocksize(BLOCK_SIZE) .setCompressionType(Compression.Algorithm.NONE) ); _testBlocksScanned(table); }
cf.setBlocksize(kylinConfig.getHbaseDefaultBlockSize()); } else { cf.setBlocksize(kylinConfig.getHbaseSmallFamilyBlockSize());
/** * Create an HTableDescriptor from provided TestOptions. */ protected static HTableDescriptor getTableDescriptor(TestOptions opts) { HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(opts.tableName)); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); HColumnDescriptor familyDesc = new HColumnDescriptor(familyName); familyDesc.setDataBlockEncoding(opts.blockEncoding); familyDesc.setCompressionType(opts.compression); familyDesc.setBloomFilterType(opts.bloomType); familyDesc.setBlocksize(opts.blockSize); if (opts.inMemoryCF) { familyDesc.setInMemory(true); } familyDesc.setInMemoryCompaction(opts.inMemoryCompaction); tableDesc.addFamily(familyDesc); } if (opts.replicas != DEFAULT_OPTS.replicas) { tableDesc.setRegionReplication(opts.replicas); } if (opts.splitPolicy != null && !opts.splitPolicy.equals(DEFAULT_OPTS.splitPolicy)) { tableDesc.setRegionSplitPolicyClassName(opts.splitPolicy); } return tableDesc; }
@Before public void prepareRegion() throws IOException { region = TEST_UTIL.createTestRegion("TestTimestampFilterSeekHint" + regionCount++, new HColumnDescriptor(FAMILY) .setBlocksize(1024) .setMaxVersions(MAX_VERSIONS) ); for (long i = 0; i <MAX_VERSIONS - 2; i++) { Put p = new Put(RK_BYTES, i); p.addColumn(FAMILY_BYTES, QUAL_BYTES, Bytes.toBytes(RandomStringUtils.randomAlphabetic(255))); region.put(p); } region.flush(true); } }
private void createTableWithNonDefaultProperties() throws Exception { final long startTime = System.currentTimeMillis(); final String sourceTableNameAsString = STRING_TABLE_NAME + startTime; originalTableName = TableName.valueOf(sourceTableNameAsString); // enable replication on a column family HColumnDescriptor maxVersionsColumn = new HColumnDescriptor(MAX_VERSIONS_FAM); HColumnDescriptor bloomFilterColumn = new HColumnDescriptor(BLOOMFILTER_FAM); HColumnDescriptor dataBlockColumn = new HColumnDescriptor(COMPRESSED_FAM); HColumnDescriptor blockSizeColumn = new HColumnDescriptor(BLOCKSIZE_FAM); maxVersionsColumn.setMaxVersions(MAX_VERSIONS); bloomFilterColumn.setBloomFilterType(BLOOM_TYPE); dataBlockColumn.setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE); blockSizeColumn.setBlocksize(BLOCK_SIZE); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(sourceTableNameAsString)); htd.addFamily(maxVersionsColumn); htd.addFamily(bloomFilterColumn); htd.addFamily(dataBlockColumn); htd.addFamily(blockSizeColumn); htd.setValue(TEST_CUSTOM_VALUE, TEST_CUSTOM_VALUE); htd.setConfiguration(TEST_CONF_CUSTOM_VALUE, TEST_CONF_CUSTOM_VALUE); assertTrue(htd.getConfiguration().size() > 0); admin.createTable(htd); Table original = UTIL.getConnection().getTable(originalTableName); originalTableName = TableName.valueOf(sourceTableNameAsString); originalTableDescriptor = admin.getTableDescriptor(originalTableName); originalTableDescription = originalTableDescriptor.toStringCustomizedValues(); original.close(); }
@Test public void testModifyFamily() { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); byte[] familyName = Bytes.toBytes("cf"); HColumnDescriptor hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(1000); hcd.setDFSReplication((short) 3); htd.addFamily(hcd); assertEquals(1000, htd.getFamily(familyName).getBlocksize()); assertEquals(3, htd.getFamily(familyName).getDFSReplication()); hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(2000); hcd.setDFSReplication((short) 1); htd.modifyFamily(hcd); assertEquals(2000, htd.getFamily(familyName).getBlocksize()); assertEquals(1, htd.getFamily(familyName).getDFSReplication()); }
@Test public void testModifyNonExistingColumnFamily() throws IOException { Admin admin = TEST_UTIL.getAdmin(); HColumnDescriptor cfDescriptor = new HColumnDescriptor(FAMILY_1); int blockSize = cfDescriptor.getBlocksize(); // Create a table with one families HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME); baseHtd.addFamily(new HColumnDescriptor(FAMILY_0)); admin.createTable(baseHtd); admin.disableTable(TABLE_NAME); try { // Verify the table descriptor verifyTableDescriptor(TABLE_NAME, FAMILY_0); int newBlockSize = 2 * blockSize; cfDescriptor.setBlocksize(newBlockSize); // Modify a column family that is not in the table. try { admin.modifyColumnFamily(TABLE_NAME, cfDescriptor); Assert.fail("Modify a non-exist column family should fail"); } catch (InvalidFamilyOperationException e) { // Expected. } } finally { admin.deleteTable(TABLE_NAME); } }
public Table createTable(TableName tableName, byte[][] families, int numVersions, int blockSize, String cpName) throws IOException { HTableDescriptor desc = new HTableDescriptor(tableName); for (byte[] family : families) { HColumnDescriptor hcd = new HColumnDescriptor(family) .setMaxVersions(numVersions) .setBlocksize(blockSize); if (isNewVersionBehaviorEnabled()) { hcd.setNewVersionBehavior(true); } desc.addFamily(hcd); } if(cpName != null) { desc.addCoprocessor(cpName); } getAdmin().createTable(desc); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are // assigned waitUntilAllRegionsAssigned(tableName); return getConnection().getTable(tableName); }
/** * Create a table. * @param tableName * @param families * @param numVersions * @param blockSize * @return A Table instance for the created table. * @throws IOException */ public Table createTable(TableName tableName, byte[][] families, int numVersions, int blockSize) throws IOException { HTableDescriptor desc = new HTableDescriptor(tableName); for (byte[] family : families) { HColumnDescriptor hcd = new HColumnDescriptor(family) .setMaxVersions(numVersions) .setBlocksize(blockSize); if (isNewVersionBehaviorEnabled()) { hcd.setNewVersionBehavior(true); } desc.addFamily(hcd); } getAdmin().createTable(desc); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are // assigned waitUntilAllRegionsAssigned(tableName); return getConnection().getTable(tableName); }
@Test public void testModifyColumnFamily() throws IOException { Admin admin = TEST_UTIL.getAdmin(); HColumnDescriptor cfDescriptor = new HColumnDescriptor(FAMILY_0); int blockSize = cfDescriptor.getBlocksize(); // Create a table with one families HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME); baseHtd.addFamily(cfDescriptor); admin.createTable(baseHtd); admin.disableTable(TABLE_NAME); try { // Verify the table descriptor verifyTableDescriptor(TABLE_NAME, FAMILY_0); int newBlockSize = 2 * blockSize; cfDescriptor.setBlocksize(newBlockSize); // Modify colymn family admin.modifyColumnFamily(TABLE_NAME, cfDescriptor); HTableDescriptor htd = admin.getTableDescriptor(TABLE_NAME); HColumnDescriptor hcfd = htd.getFamily(FAMILY_0); assertTrue(hcfd.getBlocksize() == newBlockSize); } finally { admin.deleteTable(TABLE_NAME); } }
checkTableIsLegal(htd); hcd.setBlocksize(0); checkTableIsIllegal(htd); hcd.setBlocksize(1024 * 1024 * 128); // 128M checkTableIsIllegal(htd); hcd.setBlocksize(1024); checkTableIsLegal(htd);
private void assertReadOnly(HColumnDescriptor hcd) { try { hcd.setBlocksize(10); fail("ImmutableHColumnDescriptor can't be modified!!!"); } catch (UnsupportedOperationException e) { } }
@Test(expected=IllegalArgumentException.class) public void testAddDuplicateFamilies() { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); byte[] familyName = Bytes.toBytes("cf"); HColumnDescriptor hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(1000); htd.addFamily(hcd); assertEquals(1000, htd.getFamily(familyName).getBlocksize()); hcd = new HColumnDescriptor(familyName); hcd.setBlocksize(2000); htd.addFamily(hcd); }
.setCacheDataInL1(true)); final int v = 123; hcd.setBlocksize(v); hcd.setTimeToLive(v); hcd.setBlockCacheEnabled(!HColumnDescriptor.DEFAULT_BLOCKCACHE);
@Test public void testCreateHtable() throws IOException { HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf("testhbase")); tableDesc.setValue("KYLIN_HOST", "dev01"); HColumnDescriptor cf = new HColumnDescriptor("f"); cf.setMaxVersions(1); cf.setInMemory(true); cf.setBlocksize(4 * 1024 * 1024); // set to 4MB tableDesc.addFamily(cf); Configuration conf = HBaseConfiguration.create(); HBaseAdmin admin = new HBaseAdmin(conf); admin.createTable(tableDesc); admin.close(); }