@Override public Path familyOpen(final HRegionFileSystem snapshotRegionFs, final byte[] familyName) { Path familyDir = snapshotRegionFs.getStoreDir(Bytes.toString(familyName)); return familyDir; }
/** * Create the store directory for the specified family name * @param familyName Column Family Name * @return {@link Path} to the directory of the specified family * @throws IOException if the directory creation fails. */ Path createStoreDir(final String familyName) throws IOException { Path storeDir = getStoreDir(familyName); if(!fs.exists(storeDir) && !createDir(storeDir)) throw new IOException("Failed creating "+storeDir); return storeDir; }
/** * Return Qualified Path of the specified family/file * * @param familyName Column Family Name * @param fileName File Name * @return The qualified Path for the specified family/file */ Path getStoreFilePath(final String familyName, final String fileName) { Path familyDir = getStoreDir(familyName); return new Path(familyDir, fileName).makeQualified(fs.getUri(), fs.getWorkingDirectory()); }
/** * Return the store file information of the specified family/file. * * @param familyName Column Family Name * @param fileName File Name * @return The {@link StoreFileInfo} for the specified family/file */ StoreFileInfo getStoreFileInfo(final String familyName, final String fileName) throws IOException { Path familyDir = getStoreDir(familyName); return ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo, regionInfoForFs, familyName, new Path(familyDir, fileName)); }
/** * Set the directory of CF to the specified storage policy. <br> * <i>"LAZY_PERSIST"</i>, <i>"ALL_SSD"</i>, <i>"ONE_SSD"</i>, <i>"HOT"</i>, <i>"WARM"</i>, * <i>"COLD"</i> <br> * <br> * See {@link org.apache.hadoop.hdfs.protocol.HdfsConstants} for more details. * @param familyName The name of column family. * @param policyName The name of the storage policy: 'HOT', 'COLD', etc. * See see hadoop 2.6+ org.apache.hadoop.hdfs.protocol.HdfsConstants for possible list e.g * 'COLD', 'WARM', 'HOT', 'ONE_SSD', 'ALL_SSD', 'LAZY_PERSIST'. */ public void setStoragePolicy(String familyName, String policyName) { FSUtils.setStoragePolicy(this.fs, getStoreDir(familyName), policyName); }
/** * Returns true if the specified family has reference files * @param familyName Column Family Name * @return true if family contains reference files * @throws IOException */ public boolean hasReferences(final String familyName) throws IOException { Path storeDir = getStoreDir(familyName); FileStatus[] files = FSUtils.listStatus(fs, storeDir); if (files != null) { for(FileStatus stat: files) { if(stat.isDirectory()) { continue; } if(StoreFileInfo.isReference(stat.getPath())) { if (LOG.isTraceEnabled()) LOG.trace("Reference " + stat.getPath()); return true; } } } return false; }
/** * Generate the filename in the main family store directory for moving the file from a build/temp * location. * @param familyName Family that will gain the file * @param buildPath {@link Path} to the file to commit. * @param seqNum Sequence Number to append to the file name (less then 0 if no sequence number) * @param generateNewName False if you want to keep the buildPath name * @return The new {@link Path} of the to be committed file * @throws IOException */ private Path preCommitStoreFile(final String familyName, final Path buildPath, final long seqNum, final boolean generateNewName) throws IOException { Path storeDir = getStoreDir(familyName); if(!fs.exists(storeDir) && !createDir(storeDir)) throw new IOException("Failed creating " + storeDir); String name = buildPath.getName(); if (generateNewName) { name = generateUniqueName((seqNum < 0) ? null : "_SeqId_" + seqNum + "_"); } Path dstPath = new Path(storeDir, name); if (!fs.exists(buildPath)) { throw new FileNotFoundException(buildPath.toString()); } if (LOG.isDebugEnabled()) { LOG.debug("Committing " + buildPath + " as " + dstPath); } return dstPath; }
/** * Get the storage policy of the directory of CF. * @param familyName The name of column family. * @return Storage policy name, or {@code null} if not using {@link HFileSystem} or exception * thrown when trying to get policy */ @Nullable public String getStoragePolicyName(String familyName) { if (this.fs instanceof HFileSystem) { Path storeDir = getStoreDir(familyName); return ((HFileSystem) this.fs).getStoragePolicyName(storeDir); } return null; }
/** * Returns the store files available for the family. * This methods performs the filtering based on the valid store files. * @param familyName Column Family Name * @return a set of {@link StoreFileInfo} for the specified family. */ public Collection<StoreFileInfo> getStoreFiles(final String familyName, final boolean validate) throws IOException { Path familyDir = getStoreDir(familyName); FileStatus[] files = FSUtils.listStatus(this.fs, familyDir); if (files == null) { if (LOG.isTraceEnabled()) { LOG.trace("No StoreFiles for: " + familyDir); } return null; } ArrayList<StoreFileInfo> storeFiles = new ArrayList<>(files.length); for (FileStatus status: files) { if (validate && !StoreFileInfo.isValid(status)) { LOG.warn("Invalid StoreFile: " + status.getPath()); continue; } StoreFileInfo info = ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo, regionInfoForFs, familyName, status.getPath()); storeFiles.add(info); } return storeFiles; }
/** * Remove the region family from disk, archiving the store files. * @param familyName Column Family Name * @throws IOException if an error occours during the archiving */ public void deleteFamily(final String familyName) throws IOException { // archive family store files HFileArchiver.archiveFamily(fs, conf, regionInfoForFs, tableDir, Bytes.toBytes(familyName)); // delete the family folder Path familyDir = getStoreDir(familyName); if(fs.exists(familyDir) && !deleteDir(familyDir)) throw new IOException("Could not delete family " + familyName + " from FileSystem for region " + regionInfoForFs.getRegionNameAsString() + "(" + regionInfoForFs.getEncodedName() + ")"); }
/** * Returns the store files' LocatedFileStatus which available for the family. * This methods performs the filtering based on the valid store files. * @param familyName Column Family Name * @return a list of store files' LocatedFileStatus for the specified family. */ public static List<LocatedFileStatus> getStoreFilesLocatedStatus( final HRegionFileSystem regionfs, final String familyName, final boolean validate) throws IOException { Path familyDir = regionfs.getStoreDir(familyName); List<LocatedFileStatus> locatedFileStatuses = FSUtils.listLocatedStatus( regionfs.getFileSystem(), familyDir); if (locatedFileStatuses == null) { if (LOG.isTraceEnabled()) { LOG.trace("No StoreFiles for: " + familyDir); } return null; } List<LocatedFileStatus> validStoreFiles = Lists.newArrayList(); for (LocatedFileStatus status : locatedFileStatuses) { if (validate && !StoreFileInfo.isValid(status)) { LOG.warn("Invalid StoreFile: " + status.getPath()); } else { validStoreFiles.add(status); } } return validStoreFiles; }
private boolean familyHasReferenceFile(HRegionFileSystem fileSystem, String family) throws IOException { List<Path> referenceFiles = getReferenceFilePaths(fileSystem.getFileSystem(), fileSystem.getStoreDir(family)); for (Path referenceFile : referenceFiles) { FileStatus status = fileSystem.getFileSystem().getFileLinkStatus(referenceFile); if (status.getModificationTime() < timestamp) { LOG.info("Including store: " + family + " for compaction for region: " + fileSystem .getRegionInfo().getEncodedName() + " (reference store files)"); return true; } } return false; }
/** * Writes the compaction WAL record. * @param filesCompacted Files compacted (input). * @param newFiles Files from compaction. */ private void writeCompactionWalRecord(Collection<HStoreFile> filesCompacted, Collection<HStoreFile> newFiles) throws IOException { if (region.getWAL() == null) { return; } List<Path> inputPaths = filesCompacted.stream().map(HStoreFile::getPath).collect(Collectors.toList()); List<Path> outputPaths = newFiles.stream().map(HStoreFile::getPath).collect(Collectors.toList()); RegionInfo info = this.region.getRegionInfo(); CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info, family.getName(), inputPaths, outputPaths, fs.getStoreDir(getColumnFamilyDescriptor().getNameAsString())); // Fix reaching into Region to get the maxWaitForSeqId. // Does this method belong in Region altogether given it is making so many references up there? // Could be Region#writeCompactionMarker(compactionDescriptor); WALUtil.writeCompactionMarker(this.region.getWAL(), this.region.getReplicationScope(), this.region.getRegionInfo(), compactionDescriptor, this.region.getMVCC()); }
private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles, List<StoreFileInfo> storeFiles, long referenceFileTimestamp) throws IOException { FileSystem fileSystem = mock(FileSystem.class); if (hasReferenceFiles) { FileStatus fileStatus = mock(FileStatus.class); doReturn(referenceFileTimestamp).when(fileStatus).getModificationTime(); doReturn(fileStatus).when(fileSystem).getFileLinkStatus(isA(Path.class)); } HRegionFileSystem mockSystem = mock(HRegionFileSystem.class); doReturn(info).when(mockSystem).getRegionInfo(); doReturn(regionStoreDir).when(mockSystem).getStoreDir(FAMILY); doReturn(hasReferenceFiles).when(mockSystem).hasReferences(anyString()); doReturn(storeFiles).when(mockSystem).getStoreFiles(anyString()); doReturn(fileSystem).when(mockSystem).getFileSystem(); return mockSystem; }
Path path = r.getRegionFileSystem().getStoreDir(famStr); waitUntilFilesShowup(fs, path, 1);
Path storePath = regionFs.getStoreDir(Bytes.toString(FAMILIES[0])); FileStatus[] storeFiles = FSUtils.listStatus(fs, storePath); assertNotNull(storeFiles);
.getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family, storeFiles, Lists.newArrayList(newFile), region.getRegionFileSystem().getStoreDir(Bytes.toString(family)));
testConf, fs, FSUtils.getTableDir(testDir, hri.getTable()), hriClone); Path dstPath = cloneRegionFs.getStoreDir(TEST_FAMILY); HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName()); Path linkFilePath = new Path(dstPath,
/** * Return Qualified Path of the specified family/file * * @param familyName Column Family Name * @param fileName File Name * @return The qualified Path for the specified family/file */ Path getStoreFilePath(final String familyName, final String fileName) { Path familyDir = getStoreDir(familyName); return new Path(familyDir, fileName).makeQualified(this.fs); }
/** * Returns true if the specified family has reference files * @param familyName Column Family Name * @return true if family contains reference files * @throws IOException */ public boolean hasReferences(final String familyName) throws IOException { FileStatus[] files = FSUtils.listStatus(fs, getStoreDir(familyName), new FSUtils.ReferenceFileFilter(fs)); return files != null && files.length > 0; }