/** @return {@link FileSystem} being used by this region */ public FileSystem getFilesystem() { return fs.getFileSystem(); }
@Override public FileSystem getFileSystem() { return this.fs.getFileSystem(); }
private boolean familyHasReferenceFile(HRegionFileSystem fileSystem, String family) throws IOException { List<Path> referenceFiles = getReferenceFilePaths(fileSystem.getFileSystem(), fileSystem.getStoreDir(family)); for (Path referenceFile : referenceFiles) { FileStatus status = fileSystem.getFileSystem().getFileLinkStatus(referenceFile); if (status.getModificationTime() < timestamp) { LOG.info("Including store: " + family + " for compaction for region: " + fileSystem .getRegionInfo().getEncodedName() + " (reference store files)"); return true; } } return false; }
private Path splitStoreFile(final HRegionFileSystem regionFs, final HRegionInfo hri, final String family, final HStoreFile sf, final byte[] splitKey, boolean isTopRef) throws IOException { FileSystem fs = regionFs.getFileSystem(); Path path = regionFs.splitStoreFile(hri, family, sf, splitKey, isTopRef, null); if (null == path) { return null; } Path regionDir = regionFs.commitDaughterRegion(hri); return new Path(new Path(regionDir, family), path.getName()); }
/** * Returns the store files' LocatedFileStatus which available for the family. * This methods performs the filtering based on the valid store files. * @param familyName Column Family Name * @return a list of store files' LocatedFileStatus for the specified family. */ public static List<LocatedFileStatus> getStoreFilesLocatedStatus( final HRegionFileSystem regionfs, final String familyName, final boolean validate) throws IOException { Path familyDir = regionfs.getStoreDir(familyName); List<LocatedFileStatus> locatedFileStatuses = FSUtils.listLocatedStatus( regionfs.getFileSystem(), familyDir); if (locatedFileStatuses == null) { if (LOG.isTraceEnabled()) { LOG.trace("No StoreFiles for: " + familyDir); } return null; } List<LocatedFileStatus> validStoreFiles = Lists.newArrayList(); for (LocatedFileStatus status : locatedFileStatuses) { if (validate && !StoreFileInfo.isValid(status)) { LOG.warn("Invalid StoreFile: " + status.getPath()); } else { validStoreFiles.add(status); } } return validStoreFiles; }
private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles, List<StoreFileInfo> storeFiles, long referenceFileTimestamp) throws IOException { FileSystem fileSystem = mock(FileSystem.class); if (hasReferenceFiles) { FileStatus fileStatus = mock(FileStatus.class); doReturn(referenceFileTimestamp).when(fileStatus).getModificationTime(); doReturn(fileStatus).when(fileSystem).getFileLinkStatus(isA(Path.class)); } HRegionFileSystem mockSystem = mock(HRegionFileSystem.class); doReturn(info).when(mockSystem).getRegionInfo(); doReturn(regionStoreDir).when(mockSystem).getStoreDir(FAMILY); doReturn(hasReferenceFiles).when(mockSystem).hasReferences(anyString()); doReturn(storeFiles).when(mockSystem).getStoreFiles(anyString()); doReturn(fileSystem).when(mockSystem).getFileSystem(); return mockSystem; }
long getModTime(final HRegion r) throws IOException { FileStatus[] statuses = r.getRegionFileSystem().getFileSystem().listStatus( new Path(r.getRegionFileSystem().getRegionDir(), HRegionFileSystem.REGION_INFO_FILE)); assertTrue(statuses != null && statuses.length == 1); return statuses[0].getModificationTime(); }
long getModTime(final HRegion r) throws IOException { FileStatus[] statuses = r.getRegionFileSystem().getFileSystem().listStatus( new Path(r.getRegionFileSystem().getRegionDir(), HRegionFileSystem.REGION_INFO_FILE)); assertTrue(statuses != null && statuses.length == 1); return statuses[0].getModificationTime(); }
/** * Useful when reopening a closed region (normally for unit tests) * @param other original object * @param reporter An interface we can report progress against. * @return new HRegion */ public static HRegion openHRegion(final HRegion other, final CancelableProgressable reporter) throws IOException { HRegionFileSystem regionFs = other.getRegionFileSystem(); HRegion r = newHRegion(regionFs.getTableDir(), other.getWAL(), regionFs.getFileSystem(), other.baseConf, other.getRegionInfo(), other.getTableDescriptor(), null); return r.openHRegion(reporter); }
@Test public void testSkipRecoveredEditsReplayAllIgnored() throws Exception { byte[] family = Bytes.toBytes("family"); this.region = initHRegion(tableName, method, CONF, family); Path regiondir = region.getRegionFileSystem().getRegionDir(); FileSystem fs = region.getRegionFileSystem().getFileSystem(); Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir); for (int i = 1000; i < 1050; i += 10) { Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); FSDataOutputStream dos = fs.create(recoveredEdits); dos.writeInt(i); dos.close(); } long minSeqId = 2000; Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1)); FSDataOutputStream dos = fs.create(recoveredEdits); dos.close(); Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); for (HStore store : region.getStores()) { maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId); } long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, null); assertEquals(minSeqId, seqId); }
HRegion region = servingRegions.get(0); FileSystem fs = region.getRegionFileSystem().getFileSystem();
FileSystem fs = region.getRegionFileSystem().getFileSystem(); HBaseTestingUtility.closeRegionAndWAL(region);
try { Path regiondir = region.getRegionFileSystem().getRegionDir(); FileSystem fs = region.getRegionFileSystem().getFileSystem(); byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
try { Path regiondir = region.getRegionFileSystem().getRegionDir(); FileSystem fs = region.getRegionFileSystem().getFileSystem(); byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
FileSystem fs = r.getRegionFileSystem().getFileSystem(); Path path = r.getRegionFileSystem().getStoreDir(famStr); waitUntilFilesShowup(fs, path, 1);
FSDataOutputStream out = regionFS.getFileSystem().create(errFile); out.writeInt(1); out.close();
((HFileSystem) regionFs.getFileSystem()).getStoragePolicyName(storeTempDir)); for (FileStatus status : storeFiles) { assertEquals("ONE_SSD", ((HFileSystem) regionFs.getFileSystem()).getStoragePolicyName(status.getPath()));
try { Path regiondir = region.getRegionFileSystem().getRegionDir(); FileSystem fs = region.getRegionFileSystem().getFileSystem(); byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); byte[][] columns = region.getTableDescriptor().getColumnFamilyNames().toArray(new byte[0][]);
@Test public void testReadAndWriteHRegionInfoFile() throws IOException, InterruptedException { HBaseTestingUtility htu = new HBaseTestingUtility(); HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO; Path basedir = htu.getDataTestDir(); // Create a region. That'll write the .regioninfo file. FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(htu.getConfiguration()); HRegion r = HBaseTestingUtility.createRegionAndWAL(hri, basedir, htu.getConfiguration(), fsTableDescriptors.get(TableName.META_TABLE_NAME)); // Get modtime on the file. long modtime = getModTime(r); HBaseTestingUtility.closeRegionAndWAL(r); Thread.sleep(1001); r = HRegion.openHRegion(basedir, hri, fsTableDescriptors.get(TableName.META_TABLE_NAME), null, htu.getConfiguration()); // Ensure the file is not written for a second time. long modtime2 = getModTime(r); assertEquals(modtime, modtime2); // Now load the file. org.apache.hadoop.hbase.client.RegionInfo deserializedHri = HRegionFileSystem.loadRegionInfoFileContent( r.getRegionFileSystem().getFileSystem(), r.getRegionFileSystem().getRegionDir()); assertTrue(org.apache.hadoop.hbase.client.RegionInfo.COMPARATOR.compare(hri, deserializedHri) == 0); HBaseTestingUtility.closeRegionAndWAL(r); }
@Test public void testReadAndWriteRegionInfoFile() throws IOException, InterruptedException { HBaseTestingUtility htu = new HBaseTestingUtility(); RegionInfo ri = RegionInfoBuilder.FIRST_META_REGIONINFO; Path basedir = htu.getDataTestDir(); // Create a region. That'll write the .regioninfo file. FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(htu.getConfiguration()); HRegion r = HBaseTestingUtility.createRegionAndWAL(convert(ri), basedir, htu.getConfiguration(), fsTableDescriptors.get(TableName.META_TABLE_NAME)); // Get modtime on the file. long modtime = getModTime(r); HBaseTestingUtility.closeRegionAndWAL(r); Thread.sleep(1001); r = HRegion.openHRegion(basedir, convert(ri), fsTableDescriptors.get(TableName.META_TABLE_NAME), null, htu.getConfiguration()); // Ensure the file is not written for a second time. long modtime2 = getModTime(r); assertEquals(modtime, modtime2); // Now load the file. RegionInfo deserializedRi = HRegionFileSystem.loadRegionInfoFileContent( r.getRegionFileSystem().getFileSystem(), r.getRegionFileSystem().getRegionDir()); HBaseTestingUtility.closeRegionAndWAL(r); }