private void createWriteTable(int numberOfServers) throws IOException { int numberOfRegions = (int)(numberOfServers * regionsLowerLimit); LOG.info("Number of live regionservers {}, pre-splitting the canary table into {} regions " + "(current lower limit of regions per server is {} and you can change it with config {}).", numberOfServers, numberOfRegions, regionsLowerLimit, HConstants.HBASE_CANARY_WRITE_PERSERVER_REGIONS_LOWERLIMIT_KEY); HTableDescriptor desc = new HTableDescriptor(writeTableName); HColumnDescriptor family = new HColumnDescriptor(CANARY_TABLE_FAMILY_NAME); family.setMaxVersions(1); family.setTimeToLive(writeDataTTL); desc.addFamily(family); byte[][] splits = new RegionSplitter.HexStringSplit().split(numberOfRegions); admin.createTable(desc, splits); } }
/** * Create the mob store file */ private Path createMobStoreFile(Configuration conf, String family) throws IOException { HColumnDescriptor hcd = new HColumnDescriptor(family); hcd.setMaxVersions(4); hcd.setMobEnabled(true); return createMobStoreFile(hcd); }
@Before public void setUp() throws Exception { this.htd = UTIL.createTableDescriptor(name.getMethodName()); if (name.getMethodName().equals("testCompactionSeqId")) { UTIL.getConfiguration().set("hbase.hstore.compaction.kv.max", "10"); UTIL.getConfiguration().set( DefaultStoreEngine.DEFAULT_COMPACTOR_CLASS_KEY, DummyCompactor.class.getName()); HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); hcd.setMaxVersions(65536); this.htd.addFamily(hcd); } this.r = UTIL.createLocalHRegion(htd, null, null); }
HTableDescriptor copy = new HTableDescriptor(htd); assertTrue(htd.equals(copy)); HTableDescriptor modifiedHtd = new HTableDescriptor(this.admin.getTableDescriptor(tableName)); assertFalse(htd.equals(modifiedHtd)); assertTrue(copy.equals(modifiedHtd)); int maxversions = hcd.getMaxVersions(); final int newMaxVersions = maxversions + 1; hcd.setMaxVersions(newMaxVersions); final byte [] hcdName = hcd.getName(); expectedException = false; HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName); xtracol.setValue(xtracolName, xtracolName); expectedException = false;
private static void createTable(Admin admin, TableName tableName) throws IOException { HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor("col"); hcd.setMaxVersions(3); desc.addFamily(hcd); admin.createTable(desc); }
@Test public void testHColumnDescriptorCachedMaxVersions() throws Exception { HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); hcd.setMaxVersions(5); // Verify the max version assertEquals(5, hcd.getMaxVersions()); // modify the max version hcd.setValue(Bytes.toBytes(HConstants.VERSIONS), Bytes.toBytes("8")); // Verify the max version assertEquals(8, hcd.getMaxVersions()); }
private void createTestTable(TableName tname, byte[] cf) throws Exception { HTableDescriptor htd = new HTableDescriptor(tname); HColumnDescriptor hcd = new HColumnDescriptor(cf); hcd.setMaxVersions(100); htd.addFamily(hcd); htd.setOwner(USER_OWNER); createTable(TEST_UTIL, htd, new byte[][] { Bytes.toBytes("s") }); }
/** * This utility method creates a new Hbase HColumnDescriptor object based on a * Thrift ColumnDescriptor "struct". * * @param in Thrift ColumnDescriptor object * @return HColumnDescriptor * @throws IllegalArgument if the column name is empty */ static public HColumnDescriptor colDescFromThrift(ColumnDescriptor in) throws IllegalArgument { Compression.Algorithm comp = Compression.getCompressionAlgorithmByName(in.compression.toLowerCase(Locale.ROOT)); BloomType bt = BloomType.valueOf(in.bloomFilterType); if (in.name == null || !in.name.hasRemaining()) { throw new IllegalArgument("column name is empty"); } byte [] parsedName = CellUtil.parseColumn(Bytes.getBytes(in.name))[0]; HColumnDescriptor col = new HColumnDescriptor(parsedName) .setMaxVersions(in.maxVersions) .setCompressionType(comp) .setInMemory(in.inMemory) .setBlockCacheEnabled(in.blockCacheEnabled) .setTimeToLive(in.timeToLive > 0 ? in.timeToLive : Integer.MAX_VALUE) .setBloomFilterType(bt); return col; }
public HTableDescriptor createTableDescriptor(final TableName tableName, byte[][] families, int maxVersions) { HTableDescriptor desc = new HTableDescriptor(tableName); for (byte[] family : families) { HColumnDescriptor hcd = new HColumnDescriptor(family) .setMaxVersions(maxVersions); if (isNewVersionBehaviorEnabled()) { hcd.setNewVersionBehavior(true); } desc.addFamily(hcd); } return desc; }
public static HColumnDescriptor createColumnFamily(KylinConfig kylinConfig, String cfName, boolean isMemoryHungry) { HColumnDescriptor cf = new HColumnDescriptor(cfName); cf.setMaxVersions(1);
private void setupMockColumnFamiliesForDataBlockEncoding(Table table, Map<String, DataBlockEncoding> familyToDataBlockEncoding) throws IOException { HTableDescriptor mockTableDescriptor = new HTableDescriptor(TABLE_NAMES[0]); for (Entry<String, DataBlockEncoding> entry : familyToDataBlockEncoding.entrySet()) { mockTableDescriptor.addFamily(new HColumnDescriptor(entry.getKey()) .setMaxVersions(1) .setDataBlockEncoding(entry.getValue()) .setBlockCacheEnabled(false) .setTimeToLive(0)); } Mockito.doReturn(mockTableDescriptor).when(table).getTableDescriptor(); }
@Before public void prepareRegion() throws IOException { region = TEST_UTIL.createTestRegion("TestTimestampFilterSeekHint" + regionCount++, new HColumnDescriptor(FAMILY) .setBlocksize(1024) .setMaxVersions(MAX_VERSIONS) ); for (long i = 0; i <MAX_VERSIONS - 2; i++) { Put p = new Put(RK_BYTES, i); p.addColumn(FAMILY_BYTES, QUAL_BYTES, Bytes.toBytes(RandomStringUtils.randomAlphabetic(255))); region.put(p); } region.flush(true); } }
private void setupMockColumnFamiliesForCompression(Table table, Map<String, Compression.Algorithm> familyToCompression) throws IOException { HTableDescriptor mockTableDescriptor = new HTableDescriptor(TABLE_NAMES[0]); for (Entry<String, Compression.Algorithm> entry : familyToCompression.entrySet()) { mockTableDescriptor.addFamily(new HColumnDescriptor(entry.getKey()) .setMaxVersions(1) .setCompressionType(entry.getValue()) .setBlockCacheEnabled(false) .setTimeToLive(0)); } Mockito.doReturn(mockTableDescriptor).when(table).getTableDescriptor(); }
private void init(Configuration conf, long mobThreshold) throws Exception { this.conf = conf; this.mobCellThreshold = mobThreshold; HBaseTestingUtility UTIL = new HBaseTestingUtility(conf); compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3); htd = UTIL.createTableDescriptor(name.getMethodName()); hcd = new HColumnDescriptor(COLUMN_FAMILY); hcd.setMobEnabled(true); hcd.setMobThreshold(mobThreshold); hcd.setMaxVersions(1); htd.modifyFamily(hcd); RegionInfo regionInfo = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); region = HBaseTestingUtility .createRegionAndWAL(regionInfo, UTIL.getDataTestDir(), conf, htd, new MobFileCache(conf)); fs = FileSystem.get(conf); }
private void setupMockColumnFamiliesForBloomType(Table table, Map<String, BloomType> familyToDataBlockEncoding) throws IOException { HTableDescriptor mockTableDescriptor = new HTableDescriptor(TABLE_NAMES[0]); for (Entry<String, BloomType> entry : familyToDataBlockEncoding.entrySet()) { mockTableDescriptor.addFamily(new HColumnDescriptor(entry.getKey()) .setMaxVersions(1) .setBloomFilterType(entry.getValue()) .setBlockCacheEnabled(false) .setTimeToLive(0)); } Mockito.doReturn(mockTableDescriptor).when(table).getTableDescriptor(); }
@Test public void testThreeStoreFiles() throws IOException { region = TEST_UTIL.createTestRegion(TABLE_NAME, new HColumnDescriptor(FAMILY) .setCompressionType(Compression.Algorithm.GZ) .setBloomFilterType(bloomType) .setMaxVersions(TestMultiColumnScanner.MAX_VERSIONS)); createStoreFile(new int[] {1, 2, 6}); createStoreFile(new int[] {1, 2, 3, 7}); createStoreFile(new int[] {1, 9}); scanColSet(new int[]{1, 4, 6, 7}, new int[]{1, 6, 7}); HBaseTestingUtility.closeRegionAndWAL(region); }
private void setupMockColumnFamiliesForBlockSize(Table table, Map<String, Integer> familyToDataBlockEncoding) throws IOException { HTableDescriptor mockTableDescriptor = new HTableDescriptor(TABLE_NAMES[0]); for (Entry<String, Integer> entry : familyToDataBlockEncoding.entrySet()) { mockTableDescriptor.addFamily(new HColumnDescriptor(entry.getKey()) .setMaxVersions(1) .setBlocksize(entry.getValue()) .setBlockCacheEnabled(false) .setTimeToLive(0)); } Mockito.doReturn(mockTableDescriptor).when(table).getTableDescriptor(); }
new HColumnDescriptor(FAMILY) .setCompressionType(comprAlgo) .setBloomFilterType(bloomType) .setMaxVersions(3) );
@Test public void testFlushNonMobFile() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor(family); hcd.setMaxVersions(4); desc.addFamily(hcd); testFlushFile(desc); }
@Test public void testPb() throws DeserializationException { HColumnDescriptor hcd = new HColumnDescriptor( new HColumnDescriptor(HConstants.CATALOG_FAMILY) .setInMemory(true) .setScope(HConstants.REPLICATION_SCOPE_LOCAL) hcd.setBlockCacheEnabled(!HColumnDescriptor.DEFAULT_BLOCKCACHE); hcd.setValue("a", "b"); hcd.setMaxVersions(v); assertEquals(v, hcd.getMaxVersions()); hcd.setMinVersions(v);