/** * Tests whether the directory is not empty. * * @param status the {@link URIStatus} status as the input of the command * @return true if the directory is not empty or false if it is empty */ private boolean isNonEmptyDir(URIStatus status) { return status.isFolder() && status.getLength() > 0; }
/** * Tests whether the file is zero length. * * @param status the {@link URIStatus} status as the input of the command * @return true if the file is zero length or false if it is not zero length */ private boolean isZeroLengthFile(URIStatus status) { return !status.isFolder() && status.getLength() == 0; }
@Override protected void runPlainPath(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(path); int numOfBytes = Constants.KB; if (cl.hasOption('c')) { numOfBytes = (int) FormatUtils.parseSpaceSize(cl.getOptionValue('c')); Preconditions.checkArgument(numOfBytes > 0, "specified bytes must be > 0"); } if (status.isFolder()) { throw new IOException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(path)); } OpenFilePOptions options = OpenFilePOptions.getDefaultInstance(); try (FileInStream is = mFileSystem.openFile(path, options)) { byte[] buf = new byte[numOfBytes]; long bytesToRead; if (status.getLength() > numOfBytes) { bytesToRead = numOfBytes; } else { bytesToRead = status.getLength(); } is.skip(status.getLength() - bytesToRead); int read = is.read(buf); if (read != -1) { System.out.write(buf, 0, read); } } }
@Override protected void runPlainPath(AlluxioURI plainPath, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(plainPath); if (status.isFolder()) { throw new IOException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(plainPath)); } OpenFilePOptions options = OpenFilePOptions.getDefaultInstance(); try (FileInStream is = mFileSystem.openFile(plainPath, options)) { long bytesToRead; if (status.getLength() > mNumOfBytes) { bytesToRead = mNumOfBytes; } else { bytesToRead = status.getLength(); } byte[] buf = new byte[(int) bytesToRead]; int read = is.read(buf); if (read != -1) { System.out.write(buf, 0, read); } } }
break; case 'z': resp = status.isFolder() ? "NA" : String.valueOf(status.getLength()); break; case 'N':
/** * @param status the status of the part file * @return the {@link Part} parsed from the status */ public static Part fromURIStatus(URIStatus status) { Part result = new Part(); result.setPartNumber(Integer.parseInt(status.getName())); result.setLastModified(S3RestUtils.toS3Date(status.getLastModificationTimeMs())); result.setSize(status.getLength()); return result; }
@Override public FileStatus[] listStatus(Path path) throws IOException { LOG.debug("listStatus({})", path); if (mStatistics != null) { mStatistics.incrementReadOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); List<URIStatus> statuses; try { statuses = mFileSystem.listStatus(uri); } catch (FileDoesNotExistException e) { throw new FileNotFoundException(HadoopUtils.getPathWithoutScheme(path)); } catch (AlluxioException e) { throw new IOException(e); } FileStatus[] ret = new FileStatus[statuses.size()]; for (int k = 0; k < statuses.size(); k++) { URIStatus status = statuses.get(k); ret[k] = new FileStatus(status.getLength(), status.isFolder(), getReplica(status), status.getBlockSizeBytes(), status.getLastModificationTimeMs(), status.getCreationTimeMs(), new FsPermission((short) status.getMode()), status.getOwner(), status.getGroup(), new Path(mAlluxioHeader + status.getPath())); } return ret; }
S3RestUtils.toS3Date(status.getLastModificationTimeMs()), S3Constants.S3_EMPTY_ETAG, String.valueOf(status.getLength()), S3Constants.S3_STANDARD_STORAGE_CLASS)); mKeyCount++;
protected FileInStream(URIStatus status, InStreamOptions options, FileSystemContext context) { AlluxioConfiguration conf = context.getConf(); mPassiveCachingEnabled = conf.getBoolean(PropertyKey.USER_FILE_PASSIVE_CACHE_ENABLED); mBlockWorkerClientReadRetry = conf.getInt(PropertyKey.USER_BLOCK_WORKER_CLIENT_READ_RETRY); mStatus = status; mOptions = options; mBlockStore = AlluxioBlockStore.create(context); mContext = context; mLength = mStatus.getLength(); mBlockSize = mStatus.getBlockSizeBytes(); mPosition = 0; mBlockInStream = null; mCachedPositionedReadStream = null; mLastBlockIdCached = 0; }
/** * {@inheritDoc} * * If the file does not exist in Alluxio, query it from HDFS. */ @Override public FileStatus getFileStatus(Path path) throws IOException { LOG.debug("getFileStatus({})", path); if (mStatistics != null) { mStatistics.incrementReadOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); URIStatus fileStatus; try { fileStatus = mFileSystem.getStatus(uri); } catch (FileDoesNotExistException e) { throw new FileNotFoundException(e.getMessage()); } catch (AlluxioException e) { throw new IOException(e); } return new FileStatus(fileStatus.getLength(), fileStatus.isFolder(), getReplica(fileStatus), fileStatus.getBlockSizeBytes(), fileStatus.getLastModificationTimeMs(), fileStatus.getCreationTimeMs(), new FsPermission((short) fileStatus.getMode()), fileStatus.getOwner(), fileStatus.getGroup(), new Path(mAlluxioHeader + uri)); }
private long[] countHelper(AlluxioURI path) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(path); if (!status.isFolder()) { return new long[]{ 1L, 0L, status.getLength() }; } long[] rtn = new long[]{ 0L, 1L, 0L }; List<URIStatus> statuses; try { statuses = mFileSystem.listStatus(path); } catch (AlluxioException e) { throw new IOException(e.getMessage()); } for (URIStatus uriStatus : statuses) { long[] toAdd = countHelper(new AlluxioURI(uriStatus.getPath())); rtn[0] += toAdd[0]; rtn[1] += toAdd[1]; rtn[2] += toAdd[2]; } return rtn; }
for (URIStatus status : statuses) { if (!status.isFolder()) { long size = status.getLength(); totalSize += size; sizeInMem += size * status.getInMemoryPercentage(); for (URIStatus status : statuses) { if (!status.isFolder()) { long totalSize = status.getLength(); String sizeMessage = readable ? FormatUtils.getSizeFromBytes(totalSize) : String.valueOf(totalSize);
private void printLsString(URIStatus status, boolean hSize) { // detect the extended acls boolean hasExtended = status.getAcl().hasExtended() || !status.getDefaultAcl().isEmpty(); System.out.print(formatLsString(hSize, SecurityUtils.isSecurityEnabled(mFsContext.getConf()), status.isFolder(), FormatUtils.formatMode((short) status.getMode(), status.isFolder(), hasExtended), status.getOwner(), status.getGroup(), status.getLength(), status.getLastModificationTimeMs(), status.getInAlluxioPercentage(), status.getPersistenceState(), status.getPath(), mFsContext.getConf().get(PropertyKey.USER_DATE_FORMAT_PATTERN))); }
long size = status.getLength(); stat.st_size.set(size);
OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build(); try (FileInStream is = fs.openFile(absolutePath, options)) { int len = (int) Math.min(5 * Constants.KB, status.getLength() - offset); byte[] data = new byte[len]; long skipped = is.skip(offset);
/** * Creates a new instance of {@link alluxio.util.webui.UIFileInfo}. * * @param status underlying {@link URIStatus} */ public UIFileInfo(URIStatus status) { // detect the extended acls boolean hasExtended = status.getAcl().hasExtended() || !status.getDefaultAcl().isEmpty(); mId = status.getFileId(); mName = status.getName(); mAbsolutePath = status.getPath(); mBlockSizeBytes = status.getBlockSizeBytes(); mSize = status.getLength(); mCreationTimeMs = status.getCreationTimeMs(); mLastModificationTimeMs = status.getLastModificationTimeMs(); mInAlluxio = (100 == status.getInAlluxioPercentage()); mInAlluxioPercentage = status.getInAlluxioPercentage(); mIsDirectory = status.isFolder(); mPinned = status.isPinned(); mOwner = status.getOwner(); mGroup = status.getGroup(); mMode = FormatUtils.formatMode((short) status.getMode(), status.isFolder(), hasExtended); mPersistenceState = status.getPersistenceState(); mFileLocations = new ArrayList<>(); }
Assert.assertEquals(uriStatus.getLastModificationTimeMs(), fileInfo.getLastModificationTimeMs()); Assert.assertEquals(uriStatus.getLength(), fileInfo.getLength()); Assert.assertEquals(uriStatus.getName(), fileInfo.getName()); Assert.assertEquals(uriStatus.getPath(), fileInfo.getPath());
/** * Creates a new file input stream. * * @param status the file status * @param options the client options */ public FileInStream(URIStatus status, InStreamOptions options) { mStatus = status; mBlockSize = status.getBlockSizeBytes(); mFileLength = status.getLength(); mContext = FileSystemContext.INSTANCE; mAlluxioStorageType = options.getAlluxioStorageType(); mShouldCacheCurrentBlock = mAlluxioStorageType.isStore(); mClosed = false; mLocationPolicy = options.getLocationPolicy(); if (mShouldCacheCurrentBlock) { Preconditions.checkNotNull(options.getLocationPolicy(), PreconditionMessage.FILE_WRITE_LOCATION_POLICY_UNSPECIFIED); } }
/** * @param status the status of the part file * @return the {@link Part} parsed from the status */ public static Part fromURIStatus(URIStatus status) { Part result = new Part(); result.setPartNumber(Integer.parseInt(status.getName())); result.setLastModified(S3RestUtils.toS3Date(status.getLastModificationTimeMs())); result.setSize(status.getLength()); return result; }
/** * Creates a new instance of {@link UIFileInfo}. * * @param status underlying {@link URIStatus} */ public UIFileInfo(URIStatus status) { mId = status.getFileId(); mName = status.getName(); mAbsolutePath = status.getPath(); mBlockSizeBytes = status.getBlockSizeBytes(); mSize = status.getLength(); mCreationTimeMs = status.getCreationTimeMs(); mLastModificationTimeMs = status.getLastModificationTimeMs(); mInAlluxio = (100 == status.getInAlluxioPercentage()); mInAlluxioPercentage = status.getInAlluxioPercentage(); mIsDirectory = status.isFolder(); mPinned = status.isPinned(); mOwner = status.getOwner(); mGroup = status.getGroup(); mMode = FormatUtils.formatMode((short) status.getMode(), status.isFolder()); mPersistenceState = status.getPersistenceState(); mFileLocations = new ArrayList<>(); }