@Override public long getFileLength() throws IOException { return _fs.getFileStatus(_path).getLen(); }
public boolean isReplayWALFinished(String wal) throws IOException { Path walPath = new Path(walRootDir, wal); return fs.getFileStatus(walPath).getLen() == 0; }
/** Construct given a path and a configuration. */ public HadoopInput(Path path, Configuration conf) throws IOException { this.stream = path.getFileSystem(conf).open(path); this.len = path.getFileSystem(conf).getFileStatus(path).getLen(); }
@Override public void close(boolean abort) throws IOException { recordWriter.close(abort); if (!abort) { length = target.getFileSystem(conf).getFileStatus(target).getLen(); } } };
/** Construct given a path and a {@code FileSystem}. */ public FsInput(Path path, FileSystem fileSystem) throws IOException { this.len = fileSystem.getFileStatus(path).getLen(); this.stream = fileSystem.open(path); }
/** * Returns new or changed files in the given directory. The offsets are used to find * changed files. */ public List<FileStatus> scanForChangedFiles(String subDir, Map<String, Long> currentOffsets) throws IOException { Path dirPath = new Path(basePath, subDir); FileSystem fileSystem = basePath.getFileSystem(conf); List<FileStatus> newFiles = new ArrayList<>(); if (!fileSystem.exists(dirPath)) { return newFiles; } for (FileStatus status : fileSystem.listStatus(dirPath)) { String fileName = status.getPath().getName(); Long offset = currentOffsets.get(fileName); // If the offset was never added or offset < fileSize. if (offset == null || offset < status.getLen()) { newFiles.add(status); } } return newFiles; }
private static TableDescriptor readTableDescriptor(FileSystem fs, FileStatus status) throws IOException { int len = Ints.checkedCast(status.getLen()); byte [] content = new byte[len]; FSDataInputStream fsDataInputStream = fs.open(status.getPath()); try { fsDataInputStream.readFully(content); } finally { fsDataInputStream.close(); } TableDescriptor htd = null; try { htd = TableDescriptorBuilder.parseFrom(content); } catch (DeserializationException e) { throw new IOException("content=" + Bytes.toShort(content), e); } return htd; }
/** * Checks whether existing file in filesystem is the expected file (compares length and modificaiton time). */ private boolean existsAndIsExpectedFile(FileStatus status) throws IOException { if (!getFS().exists(status.getPath())) { return false; } FileStatus currentFileStatus = getFS().getFileStatus(status.getPath()); if (currentFileStatus.getLen() != status.getLen() || currentFileStatus.getModificationTime() > status.getModificationTime()) { return false; } return true; }
/** * implementation shared with deprecated HLogInputFormat */ List<InputSplit> getSplits(final JobContext context, final String startKey, final String endKey) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); boolean ignoreMissing = conf.getBoolean(WALPlayer.IGNORE_MISSING_FILES, false); Path[] inputPaths = getInputPaths(conf); long startTime = conf.getLong(startKey, Long.MIN_VALUE); long endTime = conf.getLong(endKey, Long.MAX_VALUE); List<FileStatus> allFiles = new ArrayList<FileStatus>(); for(Path inputPath: inputPaths){ FileSystem fs = inputPath.getFileSystem(conf); try { List<FileStatus> files = getFiles(fs, inputPath, startTime, endTime); allFiles.addAll(files); } catch (FileNotFoundException e) { if (ignoreMissing) { LOG.warn("File "+ inputPath +" is missing. Skipping it."); continue; } throw e; } } List<InputSplit> splits = new ArrayList<InputSplit>(allFiles.size()); for (FileStatus file : allFiles) { splits.add(new WALSplit(file.getPath().toString(), file.getLen(), startTime, endTime)); } return splits; }
private boolean getNextPath() throws Exception { while (iterPath.hasNext()) { currPath = iterPath.next(); currDesc = iterPartDesc.next(); if (isNonNativeTable) { return true; } FileSystem fs = currPath.getFileSystem(job); if (fs.exists(currPath)) { for (FileStatus fStat : listStatusUnderPath(fs, currPath)) { if (fStat.getLen() > 0) { return true; } } } } return false; }
private long dumpFromOffset(PathData item, long offset) throws IOException { long fileSize = item.refreshStatus().getLen(); if (offset > fileSize) return fileSize; // treat a negative offset as relative to end of the file, floor of 0 if (offset < 0) { offset = Math.max(fileSize + offset, 0); } FSDataInputStream in = item.fs.open(item.path); try { in.seek(offset); // use conf so the system configured io block size is used IOUtils.copyBytes(in, System.out, getConf(), false); offset = in.getPos(); } finally { in.close(); } return offset; } }
Path p = new Path(partInfo.getLocation()); if (p.getFileSystem(conf).isFile(p)) { sizeInBytes += p.getFileSystem(conf).getFileStatus(p).getLen(); } else { FileStatus[] fileStatuses = p.getFileSystem(conf).listStatus(p); if (fileStatuses != null) { for (FileStatus child : fileStatuses) { sizeInBytes += child.getLen();
@Override public long getFileSize(String fileId) throws FileBasedHelperException { Preconditions.checkNotNull(fileId, "fileId is required"); Path p = new Path(fileId); try { FileStatus status = fileSystem.getFileStatus(p); return status.getLen(); } catch (IOException e) { throw new FileBasedHelperException("Failed to get metadata on " + fileId, e); } }
@Override public void close(boolean abort) throws IOException { writer.close(); if (!abort) { length = target.getFileSystem(conf).getFileStatus(target).getLen(); } } };
public ProxyFsInput(Path path, FileSystem fs) throws IOException { this.len = fs.getFileStatus(path).getLen(); this.stream = fs.open(path); }