public static HRegion createHRegion(final RegionInfo info, final Path rootDir, final Configuration conf, final TableDescriptor hTableDescriptor, final WAL wal) throws IOException { return createHRegion(info, rootDir, conf, hTableDescriptor, wal, true); }
/** * Creates, flushes, and closes a new region. */ public static HRegion createHDFSRegionDir(Configuration conf, RegionInfo hri, TableDescriptor htd) throws IOException { // Create HRegion Path root = FSUtils.getRootDir(conf); HRegion region = HRegion.createHRegion(hri, root, conf, htd, null); // Close the new region to flush to disk. Close log file too. region.close(); return region; }
/** * Create new set of regions on the specified file-system. * @param conf {@link Configuration} * @param rootDir Root directory for HBase instance * @param tableDescriptor description of the table * @param newRegion {@link RegionInfo} that describes the region to create * @param task {@link RegionFillTask} custom code to populate region after creation * @throws IOException */ public static RegionInfo createRegion(final Configuration conf, final Path rootDir, final TableDescriptor tableDescriptor, final RegionInfo newRegion, final RegionFillTask task) throws IOException { // 1. Create HRegion // The WAL subsystem will use the default rootDir rather than the passed in rootDir // unless I pass along via the conf. Configuration confForWAL = new Configuration(conf); confForWAL.set(HConstants.HBASE_DIR, rootDir.toString()); HRegion region = HRegion.createHRegion(newRegion, rootDir, conf, tableDescriptor, null, false); try { // 2. Custom user code to interact with the created region if (task != null) { task.fillRegion(region); } } finally { // 3. Close the new region to flush to disk. Close log file too. region.close(); } return region.getRegionInfo(); }
private HRegion createHRegion(TableDescriptor td, RegionInfo info, String dir, WAL wal, Durability durability) throws IOException { Path path = new Path(DIR, dir); if (FS.exists(path)) { if (!FS.delete(path, true)) { throw new IOException("Failed delete of " + path); } } ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); return HRegion.createHRegion(info, path, CONF, td, wal); } }
private static void bootstrap(final Path rd, final Configuration c) throws IOException { LOG.info("BOOTSTRAP: creating hbase:meta region"); try { // Bootstrapping, make sure blockcache is off. Else, one will be // created here in bootstrap and it'll need to be cleaned up. Better to // not make it in first place. Turn off block caching for bootstrap. // Enable after. TableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME); HRegion meta = HRegion.createHRegion(RegionInfoBuilder.FIRST_META_REGIONINFO, rd, c, setInfoFamilyCachingForMeta(metaDescriptor, false), null); meta.close(); } catch (IOException e) { e = e instanceof RemoteException ? ((RemoteException)e).unwrapRemoteException() : e; LOG.error("bootstrap", e); throw e; } }
private HRegion testRegionWithFamiliesAndSpecifiedTableName(TableName tableName, byte[]... families) throws IOException { HRegionInfo hRegionInfo = new HRegionInfo(tableName); HTableDescriptor hTableDescriptor = new HTableDescriptor(tableName); for (byte[] family : families) { hTableDescriptor.addFamily(new HColumnDescriptor(family)); } ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); // TODO We need a way to do this without creating files return HRegion.createHRegion(hRegionInfo, new Path(testFolder.newFolder().toURI()), conf, hTableDescriptor, log); }
/** * Create a region with it's own WAL. Be sure to call * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} to clean up all resources. */ public static HRegion createRegionAndWAL(final RegionInfo info, final Path rootDir, final Configuration conf, final TableDescriptor htd, boolean initialize) throws IOException { ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); WAL wal = createWal(conf, rootDir, info); return HRegion.createHRegion(info, rootDir, conf, htd, wal, initialize); }
/** * Create an HRegion that writes to the local tmp dirs with specified wal * @param info regioninfo * @param desc table descriptor * @param wal wal for this region. * @return created hregion * @throws IOException */ public HRegion createLocalHRegion(RegionInfo info, TableDescriptor desc, WAL wal) throws IOException { return HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), desc, wal); }
/** * This borrows code from MasterFileSystem.bootstrap(). Explicitly creates it's own WAL, so be * sure to close it as well as the region when you're finished. * @param walFactoryID A unique identifier for WAL factory. Filesystem implementations will use * this ID to make a directory inside WAL directory path. * @return an open hbase:meta HRegion */ private HRegion createNewMeta(String walFactoryID) throws IOException { Path rootdir = FSUtils.getRootDir(getConf()); Configuration c = getConf(); RegionInfo metaHRI = RegionInfoBuilder.FIRST_META_REGIONINFO; TableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME); MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false); // The WAL subsystem will use the default rootDir rather than the passed in rootDir // unless I pass along via the conf. Configuration confForWAL = new Configuration(c); confForWAL.set(HConstants.HBASE_DIR, rootdir.toString()); WAL wal = new WALFactory(confForWAL, walFactoryID).getWAL(metaHRI); HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor, wal); MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true); return meta; }
HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf, byte[]... families) throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); for (byte[] family : families) { htd.addFamily(new HColumnDescriptor(family)); } ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); Path path = new Path(DIR + callingMethod); WAL wal = HBaseTestingUtility.createWal(conf, path, info); HRegion r = HRegion.createHRegion(info, path, conf, htd, wal); // this following piece is a hack. currently a coprocessorHost // is secretly loaded at OpenRegionHandler. we don't really // start a region server here, so just manually create cphost // and set it to region. RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf); r.setCoprocessorHost(host); return r; }
private HRegion createHRegion(WALFactory wals, Durability durability) throws IOException { TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^A-Za-z0-9-_]", "_")); TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build(); RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build(); Path path = new Path(DIR, tableName.getNameAsString()); if (FS.exists(path)) { if (!FS.delete(path, true)) { throw new IOException("Failed delete of " + path); } } ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); return HRegion.createHRegion(info, path, CONF, htd, wals.getWAL(info)); }
protected void compactingSetUp() throws Exception { super.internalSetUp(); Configuration conf = new Configuration(); conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, true); conf.setFloat(MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY, 0.2f); conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000); HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf); HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("foobar")); htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(TableName.valueOf("foobar"), null, null, false); WAL wal = hbaseUtility.createWal(conf, hbaseUtility.getDataTestDir(), info); this.region = HRegion.createHRegion(info, hbaseUtility.getDataTestDir(), conf, htd, wal, true); //this.region = hbaseUtility.createTestRegion("foobar", hcd); this.regionServicesForStores = region.getRegionServicesForStores(); this.store = new HStore(region, hcd, conf); long globalMemStoreLimit = (long) (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage() .getMax() * MemorySizeUtil.getGlobalMemStoreHeapPercent(conf, false)); chunkCreator = ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, globalMemStoreLimit, 0.4f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null); assertTrue(chunkCreator != null); }
/** * Setting up a Store * @throws IOException with error */ protected void initialize() throws IOException { Path basedir = new Path(DIR); String logName = "logs"; Path logdir = new Path(DIR, logName); HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("family")); FileSystem fs = FileSystem.get(conf); fs.delete(logdir, true); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("table"))); htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); hlog = new FSHLog(fs, basedir, logName, conf); hlog.init(); ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); region = HRegion.createHRegion(info, basedir, conf, htd, hlog); region.close(); Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName()); region = new HRegion(tableDir, hlog, fs, conf, info, htd, null); store = new HStore(region, hcd, conf); TEST_FILE = region.getRegionFileSystem().createTempName(); fs.createNewFile(TEST_FILE); }
private HRegion initHRegion(TableDescriptor htd, byte[] startKey, byte[] stopKey, int replicaId) throws IOException { Configuration conf = TEST_UTIL.getConfiguration(); conf.set("hbase.wal.provider", walProvider); conf.setBoolean("hbase.hregion.mvcc.preassign", false); Path tableDir = FSUtils.getTableDir(testDir, htd.getTableName()); RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(startKey) .setEndKey(stopKey).setReplicaId(replicaId).setRegionId(0).build(); fileSystem = tableDir.getFileSystem(conf); final Configuration walConf = new Configuration(conf); FSUtils.setRootDir(walConf, tableDir); this.walConf = walConf; wals = new WALFactory(walConf, "log_" + replicaId); ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); HRegion region = HRegion.createHRegion(info, TEST_UTIL.getDefaultRootDirPath(), conf, htd, wals.getWAL(info)); return region; }
string+"-"+string, 1); when(rss.getExecutorService()).thenReturn(es); primaryRegion = HRegion.createHRegion(primaryHri, rootDir, CONF, htd, walPrimary); primaryRegion.close(); List<HRegion> regions = new ArrayList<>();
return HRegion.createHRegion(regionInfo, dir, getConf(), htd, wal);
@Test public void testShouldFlushMeta() throws Exception { // write an edit in the META and ensure the shouldFlush (that the periodic memstore // flusher invokes) returns true after SYSTEM_CACHE_FLUSH_INTERVAL (even though // the MEMSTORE_PERIODIC_FLUSH_INTERVAL is set to a higher value) Configuration conf = new Configuration(); conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, HRegion.SYSTEM_CACHE_FLUSH_INTERVAL * 10); HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf); Path testDir = hbaseUtility.getDataTestDir(); EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest(); EnvironmentEdgeManager.injectEdge(edge); edge.setCurrentTimeMillis(1234); WALFactory wFactory = new WALFactory(conf, "1234"); HRegion meta = HRegion.createHRegion(RegionInfoBuilder.FIRST_META_REGIONINFO, testDir, conf, FSTableDescriptors.createMetaTableDescriptor(conf), wFactory.getWAL(RegionInfoBuilder.FIRST_META_REGIONINFO)); // parameterized tests add [#] suffix get rid of [ and ]. TableDescriptor desc = TableDescriptorBuilder .newBuilder(TableName.valueOf(name.getMethodName().replaceAll("[\\[\\]]", "_"))) .setColumnFamily(ColumnFamilyDescriptorBuilder.of("foo")).build(); RegionInfo hri = RegionInfoBuilder.newBuilder(desc.getTableName()) .setStartKey(Bytes.toBytes("row_0200")).setEndKey(Bytes.toBytes("row_0300")).build(); HRegion r = HRegion.createHRegion(hri, testDir, conf, desc, wFactory.getWAL(hri)); addRegionToMETA(meta, r); edge.setCurrentTimeMillis(1234 + 100); StringBuilder sb = new StringBuilder(); assertTrue(meta.shouldFlush(sb) == false); edge.setCurrentTimeMillis(edge.currentTime() + HRegion.SYSTEM_CACHE_FLUSH_INTERVAL + 1); assertTrue(meta.shouldFlush(sb) == true); }
region = HRegion.createHRegion(hri, rootDir, TEST_UTIL.getConfiguration(), htd, rss.getWAL(hri)); region = HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null);
HRegion region0 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd, wal); // FIXME: Uses private type region0.close(); region0.getWAL().close();
HRegion region0 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd); region0.close(); region0.getLog().closeAndDelete();