private void setupMockColumnFamiliesForBloomType(Table table, Map<String, BloomType> familyToDataBlockEncoding) throws IOException { HTableDescriptor mockTableDescriptor = new HTableDescriptor(TABLE_NAMES[0]); for (Entry<String, BloomType> entry : familyToDataBlockEncoding.entrySet()) { mockTableDescriptor.addFamily(new HColumnDescriptor(entry.getKey()) .setMaxVersions(1) .setBloomFilterType(entry.getValue()) .setBlockCacheEnabled(false) .setTimeToLive(0)); } Mockito.doReturn(mockTableDescriptor).when(table).getTableDescriptor(); }
/********************************* Master related hooks **********************************/ @Override public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException { // Need to create the new system table for labels here if (!MetaTableAccessor.tableExists(ctx.getEnvironment().getConnection(), LABELS_TABLE_NAME)) { HTableDescriptor labelsTable = new HTableDescriptor(LABELS_TABLE_NAME); HColumnDescriptor labelsColumn = new HColumnDescriptor(LABELS_TABLE_FAMILY); labelsColumn.setBloomFilterType(BloomType.NONE); labelsColumn.setBlockCacheEnabled(false); // We will cache all the labels. No need of normal // table block cache. labelsTable.addFamily(labelsColumn); // Let the "labels" table having only one region always. We are not expecting too many labels in // the system. labelsTable.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); try (Admin admin = ctx.getEnvironment().getConnection().getAdmin()) { admin.createTable(labelsTable); } } }
/** * This utility method creates a new Hbase HColumnDescriptor object based on a * Thrift ColumnDescriptor "struct". * * @param in Thrift ColumnDescriptor object * @return HColumnDescriptor * @throws IllegalArgument if the column name is empty */ static public HColumnDescriptor colDescFromThrift(ColumnDescriptor in) throws IllegalArgument { Compression.Algorithm comp = Compression.getCompressionAlgorithmByName(in.compression.toLowerCase(Locale.ROOT)); BloomType bt = BloomType.valueOf(in.bloomFilterType); if (in.name == null || !in.name.hasRemaining()) { throw new IllegalArgument("column name is empty"); } byte [] parsedName = CellUtil.parseColumn(Bytes.getBytes(in.name))[0]; HColumnDescriptor col = new HColumnDescriptor(parsedName) .setMaxVersions(in.maxVersions) .setCompressionType(comp) .setInMemory(in.inMemory) .setBlockCacheEnabled(in.blockCacheEnabled) .setTimeToLive(in.timeToLive > 0 ? in.timeToLive : Integer.MAX_VALUE) .setBloomFilterType(bt); return col; }
/** * Create a set of column descriptors with the combination of compression, * encoding, bloom codecs available. * @param prefix family names prefix * @return the list of column descriptors */ public static List<HColumnDescriptor> generateColumnDescriptors(final String prefix) { List<HColumnDescriptor> htds = new ArrayList<>(); long familyId = 0; for (Compression.Algorithm compressionType: getSupportedCompressionAlgorithms()) { for (DataBlockEncoding encodingType: DataBlockEncoding.values()) { for (BloomType bloomType: BloomType.values()) { String name = String.format("%s-cf-!@#&-%d!@#", prefix, familyId); HColumnDescriptor htd = new HColumnDescriptor(name); htd.setCompressionType(compressionType); htd.setDataBlockEncoding(encodingType); htd.setBloomFilterType(bloomType); htds.add(htd); familyId++; } } } return htds; }
cf.setBloomFilterType(BloomType.NONE); cf.setScope(kylinConfig.getHBaseReplicationScope()); return cf;
/** * Create an HTableDescriptor from provided TestOptions. */ protected static HTableDescriptor getTableDescriptor(TestOptions opts) { HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(opts.tableName)); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); HColumnDescriptor familyDesc = new HColumnDescriptor(familyName); familyDesc.setDataBlockEncoding(opts.blockEncoding); familyDesc.setCompressionType(opts.compression); familyDesc.setBloomFilterType(opts.bloomType); familyDesc.setBlocksize(opts.blockSize); if (opts.inMemoryCF) { familyDesc.setInMemory(true); } familyDesc.setInMemoryCompaction(opts.inMemoryCompaction); tableDesc.addFamily(familyDesc); } if (opts.replicas != DEFAULT_OPTS.replicas) { tableDesc.setRegionReplication(opts.replicas); } if (opts.splitPolicy != null && !opts.splitPolicy.equals(DEFAULT_OPTS.splitPolicy)) { tableDesc.setRegionSplitPolicyClassName(opts.splitPolicy); } return tableDesc; }
private void createTableWithNonDefaultProperties() throws Exception { final long startTime = System.currentTimeMillis(); final String sourceTableNameAsString = STRING_TABLE_NAME + startTime; originalTableName = TableName.valueOf(sourceTableNameAsString); // enable replication on a column family HColumnDescriptor maxVersionsColumn = new HColumnDescriptor(MAX_VERSIONS_FAM); HColumnDescriptor bloomFilterColumn = new HColumnDescriptor(BLOOMFILTER_FAM); HColumnDescriptor dataBlockColumn = new HColumnDescriptor(COMPRESSED_FAM); HColumnDescriptor blockSizeColumn = new HColumnDescriptor(BLOCKSIZE_FAM); maxVersionsColumn.setMaxVersions(MAX_VERSIONS); bloomFilterColumn.setBloomFilterType(BLOOM_TYPE); dataBlockColumn.setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE); blockSizeColumn.setBlocksize(BLOCK_SIZE); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(sourceTableNameAsString)); htd.addFamily(maxVersionsColumn); htd.addFamily(bloomFilterColumn); htd.addFamily(dataBlockColumn); htd.addFamily(blockSizeColumn); htd.setValue(TEST_CUSTOM_VALUE, TEST_CUSTOM_VALUE); htd.setConfiguration(TEST_CONF_CUSTOM_VALUE, TEST_CONF_CUSTOM_VALUE); assertTrue(htd.getConfiguration().size() > 0); admin.createTable(htd); Table original = UTIL.getConnection().getTable(originalTableName); originalTableName = TableName.valueOf(sourceTableNameAsString); originalTableDescriptor = admin.getTableDescriptor(originalTableName); originalTableDescription = originalTableDescriptor.toStringCustomizedValues(); original.close(); }
.setBloomFilterType(BloomType.ROWCOL);
conf.setInt("hbase.hstore.compactionThreshold", 10000); HColumnDescriptor hcd = new HColumnDescriptor(FAMILY_BYTES).setBlockCacheEnabled(true) .setBloomFilterType(bloomType); HTableDescriptor htd = new HTableDescriptor(TABLE); htd.addFamily(hcd);
.setBloomFilterType(BloomType.ROWCOL); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE)); htd.addFamily(hcd);
@Test public void testThreeStoreFiles() throws IOException { region = TEST_UTIL.createTestRegion(TABLE_NAME, new HColumnDescriptor(FAMILY) .setCompressionType(Compression.Algorithm.GZ) .setBloomFilterType(bloomType) .setMaxVersions(TestMultiColumnScanner.MAX_VERSIONS)); createStoreFile(new int[] {1, 2, 6}); createStoreFile(new int[] {1, 2, 3, 7}); createStoreFile(new int[] {1, 9}); scanColSet(new int[]{1, 4, 6, 7}, new int[]{1, 6, 7}); HBaseTestingUtility.closeRegionAndWAL(region); }
hcd.setBloomFilterType(BloomType.ROWCOL); admin.modifyColumnFamily(LABELS_TABLE_NAME, hcd); fail("Lables table should not get altered by user.");
new HColumnDescriptor(FAMILY) .setCompressionType(comprAlgo) .setBloomFilterType(bloomType) .setMaxVersions(3) );
.setBloomFilterType(BloomType.ROWCOL);
public static void main(String[] args) throws IOException, InterruptedException { // vv HColumnDescriptorExample HColumnDescriptor desc = new HColumnDescriptor("colfam1") .setValue("test-key", "test-value") .setBloomFilterType(BloomType.ROWCOL); System.out.println("Column Descriptor: " + desc); System.out.print("Values: "); for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> entry : desc.getValues().entrySet()) { System.out.print(Bytes.toString(entry.getKey().get()) + " -> " + Bytes.toString(entry.getValue().get()) + ", "); } System.out.println(); System.out.println("Defaults: " + HColumnDescriptor.getDefaultValues()); System.out.println("Custom: " + desc.toStringCustomizedValues()); System.out.println("Units:"); System.out.println(HColumnDescriptor.TTL + " -> " + desc.getUnit(HColumnDescriptor.TTL)); System.out.println(HColumnDescriptor.BLOCKSIZE + " -> " + desc.getUnit(HColumnDescriptor.BLOCKSIZE)); // ^^ HColumnDescriptorExample } }
.setInMemory(true) .setScope(HConstants.REPLICATION_SCOPE_LOCAL) .setBloomFilterType(BloomType.NONE) .setCacheDataInL1(true)); final int v = 123; hcd.setScope(v); hcd.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF); hcd.setBloomFilterType(BloomType.ROW); hcd.setCompressionType(Algorithm.SNAPPY); hcd.setMobEnabled(true);
private static HColumnDescriptor getHColumnDesciptor(ColumnFamilyDescriptor descriptor) { HColumnDescriptor hFamily = new HColumnDescriptor(descriptor.getName()); hFamily.setMaxVersions(descriptor.getMaxVersions()); hFamily.setCompressionType(Compression.Algorithm.valueOf(descriptor.getCompressionType().name())); hFamily.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf( descriptor.getBloomType().name())); for (Map.Entry<String, String> property : descriptor.getProperties().entrySet()) { hFamily.setValue(property.getKey(), property.getValue()); } return hFamily; }
private static HColumnDescriptor getHColumnDesciptor(ColumnFamilyDescriptor descriptor) { HColumnDescriptor hFamily = new HColumnDescriptor(descriptor.getName()); hFamily.setMaxVersions(descriptor.getMaxVersions()); hFamily.setCompressionType(Compression.Algorithm.valueOf(descriptor.getCompressionType().name())); hFamily.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf( descriptor.getBloomType().name())); for (Map.Entry<String, String> property : descriptor.getProperties().entrySet()) { hFamily.setValue(property.getKey(), property.getValue()); } return hFamily; }
private static HColumnDescriptor getHColumnDesciptor(ColumnFamilyDescriptor descriptor) { HColumnDescriptor hFamily = new HColumnDescriptor(descriptor.getName()); hFamily.setMaxVersions(descriptor.getMaxVersions()); hFamily.setCompressionType(Compression.Algorithm.valueOf(descriptor.getCompressionType().name())); hFamily.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf( descriptor.getBloomType().name())); for (Map.Entry<String, String> property : descriptor.getProperties().entrySet()) { hFamily.setValue(property.getKey(), property.getValue()); } return hFamily; }
.setInMemory(true) .setScope(HConstants.REPLICATION_SCOPE_LOCAL) .setBloomFilterType(BloomType.NONE) .setCacheDataInL1(true)); final int v = 123; hcd.setScope(v); hcd.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF); hcd.setBloomFilterType(BloomType.ROW); hcd.setCompressionType(Algorithm.SNAPPY); hcd.setMobEnabled(true);