mStatistics.incrementReadOps(1);
@Override public FileStatus[] listStatus(Path path) throws IOException { LOG.debug("listStatus({})", path); if (mStatistics != null) { mStatistics.incrementReadOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); List<URIStatus> statuses; try { statuses = mFileSystem.listStatus(uri); } catch (FileDoesNotExistException e) { throw new FileNotFoundException(HadoopUtils.getPathWithoutScheme(path)); } catch (AlluxioException e) { throw new IOException(e); } FileStatus[] ret = new FileStatus[statuses.size()]; for (int k = 0; k < statuses.size(); k++) { URIStatus status = statuses.get(k); ret[k] = new FileStatus(status.getLength(), status.isFolder(), getReplica(status), status.getBlockSizeBytes(), status.getLastModificationTimeMs(), status.getCreationTimeMs(), new FsPermission((short) status.getMode()), status.getOwner(), status.getGroup(), new Path(mAlluxioHeader + status.getPath())); } return ret; }
/** * {@inheritDoc} * * If the file does not exist in Alluxio, query it from HDFS. */ @Override public FileStatus getFileStatus(Path path) throws IOException { LOG.debug("getFileStatus({})", path); if (mStatistics != null) { mStatistics.incrementReadOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); URIStatus fileStatus; try { fileStatus = mFileSystem.getStatus(uri); } catch (FileDoesNotExistException e) { throw new FileNotFoundException(e.getMessage()); } catch (AlluxioException e) { throw new IOException(e); } return new FileStatus(fileStatus.getLength(), fileStatus.isFolder(), getReplica(fileStatus), fileStatus.getBlockSizeBytes(), fileStatus.getLastModificationTimeMs(), fileStatus.getCreationTimeMs(), new FsPermission((short) fileStatus.getMode()), fileStatus.getOwner(), fileStatus.getGroup(), new Path(mAlluxioHeader + uri)); }
/** * Returns number of bytes within blocks with future generation stamp. These * are bytes that will be potentially deleted if we forceExit from safe mode. * * @return number of bytes. */ public long getBytesWithFutureGenerationStamps() throws IOException { statistics.incrementReadOps(1); storageStatistics.incrementOpCounter(OpType.GET_BYTES_WITH_FUTURE_GS); return dfs.getBytesInFutureBlocks(); }
@Override public FileStatus getFileStatus(Path f) throws IOException { statistics.incrementReadOps(1); return makeQualified(getHdfsFileStatus(f), f); }
@Override public ContentSummary getContentSummary(final Path p) throws IOException { statistics.incrementReadOps(1); final HttpOpParam.Op op = GetOpParam.Op.GETCONTENTSUMMARY; return new FsPathResponseRunner<ContentSummary>(op, p) { @Override ContentSummary decodeResponse(Map<?,?> json) { return JsonUtil.toContentSummary(json); } }.run(); }
@Override public FSDataInputStream open(final Path f, final int bufferSize ) throws IOException { statistics.incrementReadOps(1); return new FSDataInputStream(new WebHdfsInputStream(f, bufferSize)); }
/** * Deprecated. Prefer {@link FileSystem#getAllStoragePolicies()} * @throws IOException */ @Deprecated public BlockStoragePolicy[] getStoragePolicies() throws IOException { statistics.incrementReadOps(1); storageStatistics.incrementOpCounter(OpType.GET_STORAGE_POLICIES); return dfs.getStoragePolicies(); }
private BlockLocation[] getFileBlockLocationsImpl(final FileStatus stat, final long start, final long len, final boolean updateStats) throws IOException { if (updateStats) { statistics.incrementReadOps(1); System.out.println("STATS: getFileBlockLocationsImpl - " + stat.getPath()); } checkAccess(); List<BlockLocation> result = new ArrayList<BlockLocation>(); MockFile file = findFile(stat.getPath()); if (file != null) { for(MockBlock block: file.blocks) { if (OrcInputFormat.SplitGenerator.getOverlap(block.offset, block.length, start, len) > 0) { String[] topology = new String[block.hosts.length]; for(int i=0; i < topology.length; ++i) { topology[i] = "/rack/ " + block.hosts[i]; } result.add(new BlockLocation(block.hosts, block.hosts, topology, block.offset, block.length)); } } return result.toArray(new BlockLocation[result.size()]); } return new BlockLocation[0]; }
private BlockLocation[] getFileBlockLocationsImpl(final FileStatus stat, final long start, final long len, final boolean updateStats) throws IOException { if (updateStats) { statistics.incrementReadOps(1); } checkAccess(); List<BlockLocation> result = new ArrayList<BlockLocation>(); MockFile file = findFile(stat.getPath()); if (file != null) { for(MockBlock block: file.blocks) { if (getOverlap(block.offset, block.length, start, len) > 0) { String[] topology = new String[block.hosts.length]; for(int i=0; i < topology.length; ++i) { topology[i] = "/rack/ " + block.hosts[i]; } result.add(new BlockLocation(block.hosts, block.hosts, topology, block.offset, block.length)); } } return result.toArray(new BlockLocation[result.size()]); } return new BlockLocation[0]; }
private List<LocatedFileStatus> listLocatedFileStatuses(Path path) throws IOException { statistics.incrementReadOps(1); System.out.println("STATS: listLocatedFileStatuses - " + path); checkAccess(); path = path.makeQualified(this); List<LocatedFileStatus> result = new ArrayList<>(); String pathname = path.toString(); String pathnameAsDir = pathname + "/"; Set<String> dirs = new TreeSet<String>(); MockFile file = findFile(path); if (file != null) { result.add(createLocatedStatus(file)); return result; } findMatchingLocatedFiles(files, pathnameAsDir, dirs, result); findMatchingLocatedFiles(globalFiles, pathnameAsDir, dirs, result); // for each directory add it once for(String dir: dirs) { result.add(createLocatedDirectory(new MockPath(this, pathnameAsDir + dir))); } return result; }
private List<LocatedFileStatus> listLocatedFileStatuses(Path path) throws IOException { statistics.incrementReadOps(1); checkAccess(); path = path.makeQualified(this); List<LocatedFileStatus> result = new ArrayList<>(); String pathname = path.toString(); String pathnameAsDir = pathname + "/"; Set<String> dirs = new TreeSet<String>(); MockFile file = findFile(path); if (file != null) { result.add(createLocatedStatus(file)); return result; } findMatchingLocatedFiles(files, pathnameAsDir, dirs, result); findMatchingLocatedFiles(globalFiles, pathnameAsDir, dirs, result); // for each directory add it once for(String dir: dirs) { result.add(createLocatedDirectory(new MockPath(this, pathnameAsDir + dir))); } return result; }
@Override public FileStatus[] listStatus(Path path) throws IOException { statistics.incrementReadOps(1); System.out.println("STATS: listStatus - " + path); checkAccess(); path = path.makeQualified(this); List<FileStatus> result = new ArrayList<FileStatus>(); String pathname = path.toString(); String pathnameAsDir = pathname + "/"; Set<String> dirs = new TreeSet<String>(); MockFile file = findFile(path); if (file != null) { return new FileStatus[]{createStatus(file)}; } findMatchingFiles(files, pathnameAsDir, dirs, result); findMatchingFiles(globalFiles, pathnameAsDir, dirs, result); // for each directory add it once for(String dir: dirs) { result.add(createDirectory(new MockPath(this, pathnameAsDir + dir))); } return result.toArray(new FileStatus[result.size()]); }
@Override public FileStatus[] listStatus(Path path) throws IOException { statistics.incrementReadOps(1); checkAccess(); path = path.makeQualified(this); List<FileStatus> result = new ArrayList<FileStatus>(); String pathname = path.toString(); String pathnameAsDir = pathname + "/"; Set<String> dirs = new TreeSet<String>(); MockFile file = findFile(path); if (file != null) { return new FileStatus[]{createStatus(file)}; } findMatchingFiles(files, pathnameAsDir, dirs, result); findMatchingFiles(globalFiles, pathnameAsDir, dirs, result); // for each directory add it once for(String dir: dirs) { result.add(createDirectory(new MockPath(this, pathnameAsDir + dir))); } return result.toArray(new FileStatus[result.size()]); }
@Override public FileStatus getFileStatus(Path path) throws IOException { statistics.incrementReadOps(1); System.out.println("STATS: getFileStatus - " + path); checkAccess(); path = path.makeQualified(this); String pathnameAsDir = path.toString() + "/"; MockFile file = findFile(path); if (file != null) return createStatus(file); for (MockFile dir : files) { if (dir.path.toString().startsWith(pathnameAsDir)) { return createDirectory(path); } } for (MockFile dir : globalFiles) { if (dir.path.toString().startsWith(pathnameAsDir)) { return createDirectory(path); } } throw new FileNotFoundException("File " + path + " does not exist"); }
@Override public FileStatus getFileStatus(Path path) throws IOException { statistics.incrementReadOps(1); checkAccess(); path = path.makeQualified(this); String pathnameAsDir = path.toString() + "/"; MockFile file = findFile(path); if (file != null) return createStatus(file); for (MockFile dir : files) { if (dir.path.toString().startsWith(pathnameAsDir)) { return createDirectory(path); } } for (MockFile dir : globalFiles) { if (dir.path.toString().startsWith(pathnameAsDir)) { return createDirectory(path); } } throw new FileNotFoundException("File " + path + " does not exist"); }
/** * Attempts to open the specified file for reading. * * @param path the file name to open * @param bufferSize stream buffer size in bytes, currently unused * @return an {@link FSDataInputStream} at the indicated path of a file */ // TODO(calvin): Consider respecting the buffer size option @Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { LOG.debug("open({}, {})", path, bufferSize); if (mStatistics != null) { mStatistics.incrementReadOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); return new FSDataInputStream(new HdfsFileInputStream(mFileSystem, uri, mStatistics)); }
@Override public FSDataInputStream open(Path path, int i) throws IOException { statistics.incrementReadOps(1); System.out.println("STATS: open - " + path); checkAccess(); MockFile file = findFile(path); if (file != null) return new FSDataInputStream(new MockInputStream(file)); throw new IOException("File not found: " + path); }
@Override public FSDataInputStream open(Path path, int i) throws IOException { statistics.incrementReadOps(1); checkAccess(); MockFile file = findFile(path); if (file != null) return new FSDataInputStream(new MockInputStream(file)); throw new IOException("File not found: " + path); }
private static FileSystem.Statistics combineFileSystemStatistics(final FileSystem.Statistics s1, final FileSystem.Statistics s2) { FileSystem.Statistics result = new FileSystem.Statistics(s1); result.incrementReadOps(s2.getReadOps()); result.incrementLargeReadOps(s2.getLargeReadOps()); result.incrementWriteOps(s2.getWriteOps()); result.incrementBytesRead(s2.getBytesRead()); result.incrementBytesWritten(s2.getBytesWritten()); return result; }