/** * Create a "mock" HStore that uses the tmpDir specified by the user and * the store dir to compact as source. */ private static HStore getStore(final Configuration conf, final FileSystem fs, final Path tableDir, final TableDescriptor htd, final RegionInfo hri, final String familyName, final Path tempDir) throws IOException { HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, hri) { @Override public Path getTempDir() { return tempDir; } }; HRegion region = new HRegion(regionFs, null, conf, htd, null); return new HStore(region, htd.getColumnFamily(Bytes.toBytes(familyName)), conf); } }
protected HStore instantiateHStore(final ColumnFamilyDescriptor family) throws IOException { if (family.isMobEnabled()) { if (HFile.getFormatVersion(this.conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) { throw new IOException("A minimum HFile version of " + HFile.MIN_FORMAT_VERSION_WITH_TAGS + " is required for MOB feature. Consider setting " + HFile.FORMAT_VERSION_KEY + " accordingly."); } return new HMobStore(this, family, this.conf); } return new HStore(this, family, this.conf); }
private HStore init(String methodName, Configuration conf, TableDescriptorBuilder builder, ColumnFamilyDescriptor hcd, MyStoreHook hook, boolean switchToPread) throws IOException { initHRegion(methodName, conf, builder, hcd, hook, switchToPread); if (hook == null) { store = new HStore(region, hcd, conf); } else { store = new MyStore(region, hcd, conf, hook, switchToPread); } return store; }
protected void compactingSetUp() throws Exception { super.internalSetUp(); Configuration conf = new Configuration(); conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, true); conf.setFloat(MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY, 0.2f); conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000); HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf); HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("foobar")); htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(TableName.valueOf("foobar"), null, null, false); WAL wal = hbaseUtility.createWal(conf, hbaseUtility.getDataTestDir(), info); this.region = HRegion.createHRegion(info, hbaseUtility.getDataTestDir(), conf, htd, wal, true); //this.region = hbaseUtility.createTestRegion("foobar", hcd); this.regionServicesForStores = region.getRegionServicesForStores(); this.store = new HStore(region, hcd, conf); long globalMemStoreLimit = (long) (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage() .getMax() * MemorySizeUtil.getGlobalMemStoreHeapPercent(conf, false)); chunkCreator = ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, globalMemStoreLimit, 0.4f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null); assertTrue(chunkCreator != null); }
/** * Setting up a Store * @throws IOException with error */ protected void initialize() throws IOException { Path basedir = new Path(DIR); String logName = "logs"; Path logdir = new Path(DIR, logName); HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("family")); FileSystem fs = FileSystem.get(conf); fs.delete(logdir, true); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("table"))); htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); hlog = new FSHLog(fs, basedir, logName, conf); hlog.init(); ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); region = HRegion.createHRegion(info, basedir, conf, htd, hlog); region.close(); Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName()); region = new HRegion(tableDir, hlog, fs, conf, info, htd, null); store = new HStore(region, hcd, conf); TEST_FILE = region.getRegionFileSystem().createTempName(); fs.createNewFile(TEST_FILE); }
@Before public void setUp() throws IOException { // parameterized tests add [#] suffix get rid of [ and ]. table = Bytes.toBytes(name.getMethodName().replaceAll("[\\[\\]]", "_")); conf = TEST_UTIL.getConfiguration(); conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION); conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false); conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, false); conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, false); fs = HFileSystem.get(conf); // Create the schema ColumnFamilyDescriptor hcd = cowType .modifyFamilySchema( ColumnFamilyDescriptorBuilder.newBuilder(family).setBloomFilterType(BloomType.ROWCOL)) .build(); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setColumnFamily(hcd).build(); // Create a store based on the schema String id = TestCacheOnWriteInSchema.class.getName(); Path logdir = new Path(FSUtils.getRootDir(conf), AbstractFSWALProvider.getWALDirectoryName(id)); fs.delete(logdir, true); RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); walFactory = new WALFactory(conf, id); region = TEST_UTIL.createLocalHRegion(info, htd, walFactory.getWAL(info)); region.setBlockCache(BlockCacheFactory.createBlockCache(conf)); store = new HStore(region, hcd, conf); }
this.store.close(); this.store = new HStore(this.store.getHRegion(), this.store.getColumnFamilyDescriptor(), c); assertEquals(2, this.store.getStorefilesCount());
protected HStore instantiateHStore(final HColumnDescriptor family) throws IOException { return new HStore(this, family, this.conf); }
/** * Create a "mock" HStore that uses the tmpDir specified by the user and * the store dir to compact as source. */ private static HStore getStore(final Configuration conf, final FileSystem fs, final Path tableDir, final HTableDescriptor htd, final HRegionInfo hri, final String familyName, final Path tempDir) throws IOException { HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, hri) { @Override public Path getTempDir() { return tempDir; } }; HRegion region = new HRegion(regionFs, null, conf, htd, null); return new HStore(region, htd.getFamily(Bytes.toBytes(familyName)), conf); } }
/** * Create a "mock" HStore that uses the tmpDir specified by the user and * the store dir to compact as source. */ private static HStore getStore(final Configuration conf, final FileSystem fs, final Path tableDir, final TableDescriptor htd, final RegionInfo hri, final String familyName, final Path tempDir) throws IOException { HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, hri) { @Override public Path getTempDir() { return tempDir; } }; HRegion region = new HRegion(regionFs, null, conf, htd, null); return new HStore(region, htd.getColumnFamily(Bytes.toBytes(familyName)), conf); } }
/** * Create a "mock" HStore that uses the tmpDir specified by the user and * the store dir to compact as source. */ private static HStore getStore(final Configuration conf, final FileSystem fs, final Path tableDir, final TableDescriptor htd, final RegionInfo hri, final String familyName, final Path tempDir) throws IOException { HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, hri) { @Override public Path getTempDir() { return tempDir; } }; HRegion region = new HRegion(regionFs, null, conf, htd, null); return new HStore(region, htd.getColumnFamily(Bytes.toBytes(familyName)), conf); } }
private HStore init(String methodName, Configuration conf, TableDescriptorBuilder builder, ColumnFamilyDescriptor hcd, MyStoreHook hook, boolean switchToPread) throws IOException { initHRegion(methodName, conf, builder, hcd, hook, switchToPread); if (hook == null) { store = new HStore(region, hcd, conf); } else { store = new MyStore(region, hcd, conf, hook, switchToPread); } return store; }
protected void compactingSetUp() throws Exception { super.internalSetUp(); Configuration conf = new Configuration(); conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, true); conf.setFloat(MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY, 0.2f); conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000); HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf); HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("foobar")); htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(TableName.valueOf("foobar"), null, null, false); WAL wal = hbaseUtility.createWal(conf, hbaseUtility.getDataTestDir(), info); this.region = HRegion.createHRegion(info, hbaseUtility.getDataTestDir(), conf, htd, wal, true); //this.region = hbaseUtility.createTestRegion("foobar", hcd); this.regionServicesForStores = region.getRegionServicesForStores(); this.store = new HStore(region, hcd, conf); long globalMemStoreLimit = (long) (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage() .getMax() * MemorySizeUtil.getGlobalMemStoreHeapPercent(conf, false)); chunkCreator = ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, globalMemStoreLimit, 0.4f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null); assertTrue(chunkCreator != null); }
/** * Setting up a Store * @throws IOException with error */ protected void initialize() throws IOException { Path basedir = new Path(DIR); String logName = "logs"; Path logdir = new Path(DIR, logName); HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("family")); FileSystem fs = FileSystem.get(conf); fs.delete(logdir, true); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("table"))); htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); hlog = new FSHLog(fs, basedir, logName, conf); ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); region = HRegion.createHRegion(info, basedir, conf, htd, hlog); region.close(); Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName()); region = new HRegion(tableDir, hlog, fs, conf, info, htd, null); store = new HStore(region, hcd, conf); TEST_FILE = region.getRegionFileSystem().createTempName(); fs.createNewFile(TEST_FILE); }
@Before public void setUp() throws IOException { // parameterized tests add [#] suffix get rid of [ and ]. table = Bytes.toBytes(name.getMethodName().replaceAll("[\\[\\]]", "_")); conf = TEST_UTIL.getConfiguration(); conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION); conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false); conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, false); conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, false); CacheConfig.instantiateBlockCache(conf); fs = HFileSystem.get(conf); // Create the schema ColumnFamilyDescriptor hcd = cowType .modifyFamilySchema( ColumnFamilyDescriptorBuilder.newBuilder(family).setBloomFilterType(BloomType.ROWCOL)) .build(); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setColumnFamily(hcd).build(); // Create a store based on the schema String id = TestCacheOnWriteInSchema.class.getName(); Path logdir = new Path(FSUtils.getRootDir(conf), AbstractFSWALProvider.getWALDirectoryName(id)); fs.delete(logdir, true); RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); walFactory = new WALFactory(conf, id); region = TEST_UTIL.createLocalHRegion(info, htd, walFactory.getWAL(info)); store = new HStore(region, hcd, conf); }
this.store.close(); this.store = new HStore(this.store.getHRegion(), this.store.getColumnFamilyDescriptor(), c); assertEquals(2, this.store.getStorefilesCount());