/** Construct given a {@link FileContext} and a {@link Path}. */ public AvroFSInput(final FileContext fc, final Path p) throws IOException { FileStatus status = fc.getFileStatus(p); this.len = status.getLen(); this.stream = fc.open(p); }
CreateFlag.CREATE, CreateFlag.OVERWRITE) : EnumSet.of(CreateFlag.CREATE); InputStream in = open(qSrc); try (OutputStream out = create(qDst, createFlag)) { IOUtils.copyBytes(in, out, conf, true);
@Override public FSDataInputStream open(Path f) throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException, IOException { return fc.open(f); }
/** Construct given a {@link FileContext} and a {@link Path}. */ public AvroFSInput(final FileContext fc, final Path p) throws IOException { FileStatus status = fc.getFileStatus(p); this.len = status.getLen(); this.stream = fc.open(p); }
public static byte[] readFile(FileContext fc, Path path, int len) throws IOException { DataInputStream dis = fc.open(path); byte[] buffer = new byte[len]; IOUtils.readFully(dis, buffer, 0, len); dis.close(); return buffer; }
@Override public Object load(int operatorId, long windowId) throws IOException { Path lPath = new Path(path + Path.SEPARATOR + String.valueOf(operatorId) + Path.SEPARATOR + Long.toHexString(windowId)); logger.debug("Loading: {}", lPath); FSDataInputStream stream = fileContext.open(lPath); try { return retrieve(stream); } finally { stream.close(); } }
public synchronized Configuration loadConfFile() throws IOException { FileContext fc = FileContext.getFileContext(confFile.toUri(), conf); Configuration jobConf = new Configuration(false); jobConf.addResource(fc.open(confFile), confFile.toString()); return jobConf; } }
@Override public FSDataInputStream open(Path f) throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException, IOException { return fc.open(f); }
public synchronized Configuration loadConfFile() throws IOException { FileContext fc = FileContext.getFileContext(confFile.toUri(), conf); Configuration jobConf = new Configuration(false); jobConf.addResource(fc.open(confFile), confFile.toString(), true); return jobConf; } }
/** Construct given a {@link FileContext} and a {@link Path}. */ public AvroFSInput(final FileContext fc, final Path p) throws IOException { FileStatus status = fc.getFileStatus(p); this.len = status.getLen(); this.stream = fc.open(p); }
@Override public Configuration loadConfFile() throws IOException { Path confPath = getConfFile(); FileContext fc = FileContext.getFileContext(confPath.toUri(), conf); Configuration jobConf = new Configuration(false); jobConf.addResource(fc.open(confPath), confPath.toString()); return jobConf; }
@Test public void testInputStreamClosedTwice() throws IOException { //HADOOP-4760 according to Closeable#close() closing already-closed //streams should have no effect. Path src = getTestRootPath(fc, "test/hadoop/file"); createFile(src); FSDataInputStream in = fc.open(src); in.close(); in.close(); }
public LogReader(Configuration conf, Path remoteAppLogFile) throws IOException { FileContext fileContext = FileContext.getFileContext(remoteAppLogFile.toUri(), conf); this.fsDataIStream = fileContext.open(remoteAppLogFile); reader = new TFile.Reader(this.fsDataIStream, fileContext.getFileStatus( remoteAppLogFile).getLen(), conf); this.scanner = reader.createScanner(); }
public LogReader(Configuration conf, Path remoteAppLogFile) throws IOException { FileContext fileContext = FileContext.getFileContext(remoteAppLogFile.toUri(), conf); this.fsDataIStream = fileContext.open(remoteAppLogFile); reader = new TFile.Reader(this.fsDataIStream, fileContext.getFileStatus( remoteAppLogFile).getLen(), conf); this.scanner = reader.createScanner(); }
@Override public Configuration loadConfFile() throws IOException { FileContext fc = FileContext.getFileContext(configFile.toUri(), conf); Configuration jobConf = new Configuration(false); jobConf.addResource(fc.open(configFile), configFile.toString()); return jobConf; }
public static byte[] readFile(FileContext fc, Path path, int len) throws IOException { DataInputStream dis = fc.open(path); byte[] buffer = new byte[len]; IOUtils.readFully(dis, buffer, 0, len); dis.close(); return buffer; }
public byte[] readFile(Path path, int len) throws IOException { DataInputStream dis = fc.open(path); byte[] buffer = new byte[len]; IOUtils.readFully(dis, buffer, 0, len); dis.close(); return buffer; }
/** Read operation randomly picks a file in the test space and reads * the entire file */ private void read() throws IOException { String fileName = files.get(r.nextInt(files.size())); long startTime = Time.now(); InputStream in = fc.open(new Path(fileName)); executionTime[OPEN] += (Time.now()-startTime); totalNumOfOps[OPEN]++; while (in.read(buffer) != -1) {} in.close(); }
@Test public void testInputStreamClosedTwice() throws IOException { //HADOOP-4760 according to Closeable#close() closing already-closed //streams should have no effect. Path src = getTestRootPath(fc, "test/hadoop/file"); createFile(src); FSDataInputStream in = fc.open(src); in.close(); in.close(); }
@Override public Object run() throws IOException { FileContext myfc = FileContext.getFileContext(conf); myfc.open(link).read(); return null; } });