/** * Creates a new instance of {@link alluxio.util.webui.UIFileInfo}. * * @param status underlying {@link URIStatus} */ public UIFileInfo(URIStatus status) { // detect the extended acls boolean hasExtended = status.getAcl().hasExtended() || !status.getDefaultAcl().isEmpty(); mId = status.getFileId(); mName = status.getName(); mAbsolutePath = status.getPath(); mBlockSizeBytes = status.getBlockSizeBytes(); mSize = status.getLength(); mCreationTimeMs = status.getCreationTimeMs(); mLastModificationTimeMs = status.getLastModificationTimeMs(); mInAlluxio = (100 == status.getInAlluxioPercentage()); mInAlluxioPercentage = status.getInAlluxioPercentage(); mIsDirectory = status.isFolder(); mPinned = status.isPinned(); mOwner = status.getOwner(); mGroup = status.getGroup(); mMode = FormatUtils.formatMode((short) status.getMode(), status.isFolder(), hasExtended); mPersistenceState = status.getPersistenceState(); mFileLocations = new ArrayList<>(); }
/** * Creates a new instance of {@link alluxio.util.webui.UIFileInfo}. * * @param info underlying {@link FileInfo} */ public UIFileInfo(FileInfo info) { this(new URIStatus(info)); }
private void printLsString(URIStatus status, boolean hSize) { // detect the extended acls boolean hasExtended = status.getAcl().hasExtended() || !status.getDefaultAcl().isEmpty(); System.out.print(formatLsString(hSize, SecurityUtils.isSecurityEnabled(mFsContext.getConf()), status.isFolder(), FormatUtils.formatMode((short) status.getMode(), status.isFolder(), hasExtended), status.getOwner(), status.getGroup(), status.getLength(), status.getLastModificationTimeMs(), status.getInAlluxioPercentage(), status.getPersistenceState(), status.getPath(), mFsContext.getConf().get(PropertyKey.USER_DATE_FORMAT_PATTERN))); }
public void fields() { FileInfo fileInfo = FileInfoTest.createRandom(); URIStatus uriStatus = new URIStatus(fileInfo); Assert.assertEquals(uriStatus.getBlockIds(), fileInfo.getBlockIds()); Assert.assertEquals(uriStatus.getBlockSizeBytes(), fileInfo.getBlockSizeBytes()); Assert.assertEquals(uriStatus.getCreationTimeMs(), fileInfo.getCreationTimeMs()); Assert.assertEquals(uriStatus.getFileId(), fileInfo.getFileId()); Assert.assertEquals(uriStatus.getGroup(), fileInfo.getGroup()); Assert.assertEquals(uriStatus.getInMemoryPercentage(), fileInfo.getInMemoryPercentage()); Assert.assertEquals(uriStatus.getLastModificationTimeMs(), fileInfo.getLastModificationTimeMs()); Assert.assertEquals(uriStatus.getLength(), fileInfo.getLength()); Assert.assertEquals(uriStatus.getName(), fileInfo.getName()); Assert.assertEquals(uriStatus.getPath(), fileInfo.getPath()); Assert.assertEquals(uriStatus.getMode(), fileInfo.getMode()); Assert.assertEquals(uriStatus.getPersistenceState(), fileInfo.getPersistenceState()); Assert.assertEquals(uriStatus.getTtl(), fileInfo.getTtl()); Assert.assertEquals(uriStatus.getTtlAction(), fileInfo.getTtlAction()); Assert.assertEquals(uriStatus.getUfsPath(), fileInfo.getUfsPath()); Assert.assertEquals(uriStatus.getOwner(), fileInfo.getOwner()); Assert.assertEquals(uriStatus.isCacheable(), fileInfo.isCacheable()); Assert.assertEquals(uriStatus.isCompleted(), fileInfo.isCompleted()); Assert.assertEquals(uriStatus.isFolder(), fileInfo.isFolder()); Assert.assertEquals(uriStatus.isPersisted(), fileInfo.isPersisted()); Assert.assertEquals(uriStatus.isPinned(), fileInfo.isPinned());
@Override public FileStatus[] listStatus(Path path) throws IOException { LOG.debug("listStatus({})", path); if (mStatistics != null) { mStatistics.incrementReadOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); List<URIStatus> statuses; try { statuses = mFileSystem.listStatus(uri); } catch (FileDoesNotExistException e) { throw new FileNotFoundException(HadoopUtils.getPathWithoutScheme(path)); } catch (AlluxioException e) { throw new IOException(e); } FileStatus[] ret = new FileStatus[statuses.size()]; for (int k = 0; k < statuses.size(); k++) { URIStatus status = statuses.get(k); ret[k] = new FileStatus(status.getLength(), status.isFolder(), getReplica(status), status.getBlockSizeBytes(), status.getLastModificationTimeMs(), status.getCreationTimeMs(), new FsPermission((short) status.getMode()), status.getOwner(), status.getGroup(), new Path(mAlluxioHeader + status.getPath())); } return ret; }
switch (formatSpecifier) { case 'b': resp = status.isFolder() ? "NA" : String.valueOf(status.getFileBlockInfos().size()); break; case 'g': resp = status.getGroup(); break; case 'u': resp = status.getOwner(); break; case 'y': SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); resp = sdf.format(new Date(status.getLastModificationTimeMs())); break; case 'z': resp = status.isFolder() ? "NA" : String.valueOf(status.getLength()); break; case 'N': resp = status.getName(); break; case 'Y': resp = String.valueOf(status.getLastModificationTimeMs()); break; default:
try { URIStatus status = mFileSystem.getStatus(turi); if (!status.isCompleted()) { long size = status.getLength(); stat.st_size.set(size); final long ctime_sec = status.getLastModificationTimeMs() / 1000; final long ctime_nsec = (status.getLastModificationTimeMs() % 1000) * 1000; stat.st_uid.set(AlluxioFuseUtils.getUid(status.getOwner())); stat.st_gid.set(AlluxioFuseUtils.getGidFromGroupName(status.getGroup())); } else { stat.st_uid.set(UID); int mode = status.getMode(); if (status.isFolder()) { mode |= FileStat.S_IFDIR; } else {
/** * @param source an Alluxio URI * @param fileSystem the Alluxio file system * @return whether the URI is a file or a directory which contains files (including recursively) * @throws Exception if an unexpected exception occurs */ private static boolean hasFiles(AlluxioURI source, FileSystem fileSystem) throws Exception { Stack<AlluxioURI> dirsToCheck = new Stack<>(); dirsToCheck.add(source); while (!dirsToCheck.isEmpty()) { try { for (URIStatus status : fileSystem.listStatus(dirsToCheck.pop())) { if (!status.isFolder()) { return true; } dirsToCheck.push(new AlluxioURI(status.getPath())); } } catch (FileDoesNotExistException e) { // This probably means another worker has deleted the directory already, so we can probably // return false here. To be safe though, we will fall through and complete the search. } } return false; }
URIStatus tokenStatus = new URIStatus(new FileInfo().setPath( mName + AlluxioURI.SEPARATOR + mContinuationToken)); startIndex = Collections.binarySearch(objectsList, tokenStatus, new URIStatusComparator()); String objectKey = status.getPath().substring(mName.length() + 1); if (mKeyCount >= mMaxKeys) { mIsTruncated = true; status.getPath().substring(mName.length() + 1), S3RestUtils.toS3Date(status.getLastModificationTimeMs()), S3Constants.S3_EMPTY_ETAG, String.valueOf(status.getLength()), S3Constants.S3_STANDARD_STORAGE_CLASS)); mKeyCount++;
long sizeInMem = 0; for (URIStatus status : statuses) { if (!status.isFolder()) { long size = status.getLength(); totalSize += size; sizeInMem += size * status.getInMemoryPercentage(); sizeInAlluxio += size * status.getInMemoryPercentage(); } else { for (URIStatus status : statuses) { if (!status.isFolder()) { long totalSize = status.getLength(); String sizeMessage = readable ? FormatUtils.getSizeFromBytes(totalSize) : String.valueOf(totalSize); String inAlluxioMessage = getFormattedValues(readable, status.getInAlluxioPercentage() * totalSize / 100, totalSize); String inMemMessage = addMemory ? getFormattedValues(readable, status.getInMemoryPercentage() * totalSize / 100, totalSize) : ""; printInfo(sizeMessage, inAlluxioMessage, inMemMessage, status.getPath());
@Override protected void runPlainPath(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(path); System.out.println("# file: " + status.getPath()); System.out.println("# owner: " + status.getOwner()); System.out.println("# group: " + status.getGroup()); for (String entry : status.getAcl().toStringEntries()) { System.out.println(entry); } List<String> defaultAclEntries = status.getDefaultAcl().toStringEntries(); for (String entry: defaultAclEntries) { System.out.println(entry); } }
@Override public FileStatus[] listStatus(Path path) throws IOException { AlluxioURI tPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); Path hdfsPath = HadoopUtils.getHDFSPath(tPath, mUnderFSAddress); LOG.info("listStatus({}): HDFS Path: {}", path, hdfsPath); if (mStatistics != null) { mStatistics.incrementReadOps(1); } List<URIStatus> statuses; try { statuses = mFileSystem.listStatus(tPath); } catch (AlluxioException e) { throw new IOException(e); } FileStatus[] ret = new FileStatus[statuses.size()]; for (int k = 0; k < statuses.size(); k++) { URIStatus status = statuses.get(k); // TODO(hy): Replicate 3 with the number of disk replications. ret[k] = new FileStatus(status.getLength(), status.isFolder(), 3, status.getBlockSizeBytes(), status.getCreationTimeMs(), status.getCreationTimeMs(), null, null, null, new Path(mAlluxioHeader + status.getPath())); } return ret; }
/** * Tests whether the path is a directory. * * @param status the {@link URIStatus} status as the input of the command * @return true if the path is a directory or false if it is not a directory */ private boolean isDir(URIStatus status) { return status.isFolder(); }
private long[] countHelper(AlluxioURI path) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(path); if (!status.isFolder()) { return new long[]{ 1L, 0L, status.getLength() }; } long[] rtn = new long[]{ 0L, 1L, 0L }; List<URIStatus> statuses; try { statuses = mFileSystem.listStatus(path); } catch (AlluxioException e) { throw new IOException(e.getMessage()); } for (URIStatus uriStatus : statuses) { long[] toAdd = countHelper(new AlluxioURI(uriStatus.getPath())); rtn[0] += toAdd[0]; rtn[1] += toAdd[1]; rtn[2] += toAdd[2]; } return rtn; }
/** * {@inheritDoc} * * If the file does not exist in Alluxio, query it from HDFS. */ @Override public FileStatus getFileStatus(Path path) throws IOException { AlluxioURI tPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); Path hdfsPath = HadoopUtils.getHDFSPath(tPath, mUnderFSAddress); LOG.info("getFileStatus({}): HDFS Path: {} Alluxio Path: {}{}", path, hdfsPath, mAlluxioHeader, tPath); if (mStatistics != null) { mStatistics.incrementReadOps(1); } URIStatus fileStatus; try { fileStatus = mFileSystem.getStatus(tPath); } catch (FileDoesNotExistException e) { throw new FileNotFoundException(e.getMessage()); } catch (AlluxioException e) { throw new IOException(e); } FileStatus ret = new FileStatus(fileStatus.getLength(), fileStatus.isFolder(), BLOCK_REPLICATION_CONSTANT, fileStatus.getBlockSizeBytes(), fileStatus.getCreationTimeMs(), fileStatus.getCreationTimeMs(), new FsPermission((short) fileStatus.getPermission()), fileStatus.getUserName(), fileStatus.getGroupName(), new Path(mAlluxioHeader + tPath)); return ret; }
/** * @param status the status of the part file * @return the {@link Part} parsed from the status */ public static Part fromURIStatus(URIStatus status) { Part result = new Part(); result.setPartNumber(Integer.parseInt(status.getName())); result.setLastModified(S3RestUtils.toS3Date(status.getLastModificationTimeMs())); result.setSize(status.getLength()); return result; }
private void queueNonPersistedRecursive(URIStatus status, Queue<AlluxioURI> toPersist) throws AlluxioException, IOException { AlluxioURI uri = new AlluxioURI(status.getPath()); if (status.isFolder()) { List<URIStatus> statuses = mFileSystem.listStatus(uri); for (URIStatus s : statuses) { queueNonPersistedRecursive(s, toPersist); } } else if (!status.isPersisted()) { toPersist.add(uri); } }
/** * @param blockId id of the block * @return a {@link Protocol.OpenUfsBlockOptions} based on the block id and options */ public Protocol.OpenUfsBlockOptions getOpenUfsBlockOptions(long blockId) { Preconditions.checkArgument(mStatus.getBlockIds().contains(blockId), "blockId"); boolean readFromUfs = mStatus.isPersisted(); // In case it is possible to fallback to read UFS blocks, also fill in the options. boolean storedAsUfsBlock = mStatus.getPersistenceState().equals("TO_BE_PERSISTED"); readFromUfs = readFromUfs || storedAsUfsBlock; if (!readFromUfs) { return Protocol.OpenUfsBlockOptions.getDefaultInstance(); } long blockStart = BlockId.getSequenceNumber(blockId) * mStatus.getBlockSizeBytes(); BlockInfo info = getBlockInfo(blockId); Protocol.OpenUfsBlockOptions openUfsBlockOptions = Protocol.OpenUfsBlockOptions.newBuilder() .setUfsPath(mStatus.getUfsPath()).setOffsetInFile(blockStart).setBlockSize(info.getLength()) .setMaxUfsReadConcurrency(mProtoOptions.getMaxUfsReadConcurrency()) .setNoCache(!ReadType.fromProto(mProtoOptions.getReadType()).isCache()) .setMountId(mStatus.getMountId()).build(); if (storedAsUfsBlock) { // On client-side, we do not have enough mount information to fill in the UFS file path. // Instead, we unset the ufsPath field and fill in a flag ufsBlock to indicate the UFS file // path can be derived from mount id and the block ID. Also because the entire file is only // one block, we set the offset in file to be zero. openUfsBlockOptions = openUfsBlockOptions.toBuilder().clearUfsPath().setBlockInUfsTier(true) .setOffsetInFile(0).build(); } return openUfsBlockOptions; }