@Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { URLConnection conn = path.toUri().toURL().openConnection(); InputStream in = conn.getInputStream(); return new FSDataInputStream(new HttpDataInputStream(in)); }
/** * Open the FileLink for read. * <p> * It uses a wrapper of FSDataInputStream that is agnostic to the location * of the file, even if the file switches between locations. * * @param fs {@link FileSystem} on which to open the FileLink * @return InputStream for reading the file link. * @throws IOException on unexpected error. */ public FSDataInputStream open(final FileSystem fs) throws IOException { return new FSDataInputStream(new FileLinkInputStream(fs, this)); }
/** * Opens an FSDataInputStream at the indicated Path. * @param f the file name to open * @param bufferSize the size of the buffer to be used. */ @Override public FSDataInputStream open(Path f, int bufferSize) throws IOException, UnresolvedLinkException { return new FSDataInputStream( new ChecksumFSInputChecker(this, f, bufferSize)); }
@Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { return new FSDataInputStream(getCtx(path).cloneWithBufferSize(bufferSize)); }
/** * Convert an instance of {@link InputStream} to a {@link FSDataInputStream} that is {@link Seekable} and * {@link PositionedReadable}. * * @see SeekableFSInputStream * */ public static FSDataInputStream convertStream(InputStream in) throws IOException { return new FSDataInputStream(new SeekableFSInputStream(in)); }
/** * Open the FileLink for read. * <p> * It uses a wrapper of FSDataInputStream that is agnostic to the location * of the file, even if the file switches between locations. * * @param fs {@link FileSystem} on which to open the FileLink * @param bufferSize the size of the buffer to be used. * @return InputStream for reading the file link. * @throws IOException on unexpected error. */ public FSDataInputStream open(final FileSystem fs, int bufferSize) throws IOException { return new FSDataInputStream(new FileLinkInputStream(fs, this, bufferSize)); }
private void nextInput() throws IOException{ currentInput++; Path path = inFiles.get(currentInput); FSDataInputStream input = new FSDataInputStream(Util.openFromFS(path)); reader = new DataFileStream<>(input, new GenericDatumReader<>()); if (schema == null) { // if this is the first file, the schema gets saved schema = reader.getSchema(); } else if (!schema.equals(reader.getSchema())) { // subsequent files have to have equal schemas throw new IOException("schemas dont match"); } }
@Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { return closer.register(new FSDataInputStream( new SeekableFSInputStream( new BufferedInputStream( client.files().get(toFileId(path)).executeMediaAsInputStream(), bufferSize)))); }
@Override public FSDataInputStream open(Path path) throws IOException { return closer.register(new FSDataInputStream( new SeekableFSInputStream( new BufferedInputStream( client.files().get(toFileId(path)).executeMediaAsInputStream())))); }
/** * Attempts to open the specified file for reading. * * @param path the file name to open * @param bufferSize stream buffer size in bytes, currently unused * @return an {@link FSDataInputStream} at the indicated path of a file */ // TODO(calvin): Consider respecting the buffer size option @Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { LOG.debug("open({}, {})", path, bufferSize); if (mStatistics != null) { mStatistics.incrementReadOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); return new FSDataInputStream(new HdfsFileInputStream(mFileSystem, uri, mStatistics)); }
@Override public FSDataInputStream open(PathHandle fd, int bufferSize) throws IOException { if (!(fd instanceof LocalFileSystemPathHandle)) { fd = new LocalFileSystemPathHandle(fd.bytes()); } LocalFileSystemPathHandle id = (LocalFileSystemPathHandle) fd; id.verify(getFileStatus(new Path(id.getPath()))); return new FSDataInputStream(new BufferedFSInputStream( new LocalFSFileInputStream(new Path(id.getPath())), bufferSize)); }
/** {@inheritDoc} */ @Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { return new FSDataInputStream(new InStream(checkExists(convert(f)))); }
@Override public FSDataInputStream open(Path path, int bufferSize) { return new FSDataInputStream( new BufferedFSInputStream( new PrestoS3InputStream(s3, getBucketName(uri), path, maxAttempts, maxBackoffTime, maxRetryTime), bufferSize)); }
/** {@inheritDoc} */ @Override public FSDataInputStream open(Path f, int bufSize) throws IOException { A.notNull(f, "f"); enterBusy(); try { IgfsPath path = convert(f); HadoopIgfsStreamDelegate stream = seqReadsBeforePrefetchOverride ? rmtClient.open(path, seqReadsBeforePrefetch) : rmtClient.open(path); long logId = -1; if (clientLog.isLogEnabled()) { logId = IgfsLogger.nextId(); clientLog.logOpen(logId, path, bufSize, stream.length()); } if (LOG.isDebugEnabled()) LOG.debug("Opening input stream [thread=" + Thread.currentThread().getName() + ", path=" + path + ", bufSize=" + bufSize + ']'); HadoopIgfsInputStream igfsIn = new HadoopIgfsInputStream(stream, stream.length(), bufSize, LOG, clientLog, logId); if (LOG.isDebugEnabled()) LOG.debug("Opened input stream [path=" + path + ", delegate=" + stream + ']'); return new FSDataInputStream(igfsIn); } finally { leaveBusy(); } }
/** {@inheritDoc} */ @Override public FSDataInputStream open(Path f, int bufSize) throws IOException { A.notNull(f, "f"); enterBusy(); try { IgfsPath path = convert(f); HadoopIgfsStreamDelegate stream = seqReadsBeforePrefetchOverride ? rmtClient.open(path, seqReadsBeforePrefetch) : rmtClient.open(path); long logId = -1; if (clientLog.isLogEnabled()) { logId = IgfsLogger.nextId(); clientLog.logOpen(logId, path, bufSize, stream.length()); } if (LOG.isDebugEnabled()) LOG.debug("Opening input stream [thread=" + Thread.currentThread().getName() + ", path=" + path + ", bufSize=" + bufSize + ']'); HadoopIgfsInputStream igfsIn = new HadoopIgfsInputStream(stream, stream.length(), bufSize, LOG, clientLog, logId); if (LOG.isDebugEnabled()) LOG.debug("Opened input stream [path=" + path + ", delegate=" + stream + ']'); return new FSDataInputStream(igfsIn); } finally { leaveBusy(); } }
@Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { getFileStatus(f); return new FSDataInputStream(new BufferedFSInputStream( new LocalFSFileInputStream(f), bufferSize)); }
@Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { SftpGetMonitor monitor = new SftpGetMonitor(); try { ChannelSftp channelSftp = this.fsHelper.getSftpChannel(); InputStream is = channelSftp.get(HadoopUtils.toUriPath(path), monitor); return new FSDataInputStream(new BufferedFSInputStream(new SftpFsHelper.SftpFsFileInputStream(is, channelSftp), bufferSize)); } catch (SftpException e) { throw new IOException(e); } }
@Override public FSDataInputStream open(Path path, int i) throws IOException { statistics.incrementReadOps(1); System.out.println("STATS: open - " + path); checkAccess(); MockFile file = findFile(path); if (file != null) return new FSDataInputStream(new MockInputStream(file)); throw new IOException("File not found: " + path); }
@Override public FSDataInputStream open(Path path, int i) throws IOException { statistics.incrementReadOps(1); checkAccess(); MockFile file = findFile(path); if (file != null) return new FSDataInputStream(new MockInputStream(file)); throw new IOException("File not found: " + path); }
@Test public void testSeekSkip() throws IOException { verifyInputStream = spy(new FSDataInputStream(new SeekableByteArrayInputStream(new byte[2 * HadoopDataInputStream.MIN_SKIP_BYTES]))); testInputStream = new HadoopDataInputStream(verifyInputStream); seekAndAssert(10); seekAndAssert(10 + HadoopDataInputStream.MIN_SKIP_BYTES + 1); seekAndAssert(testInputStream.getPos() - 1); seekAndAssert(testInputStream.getPos() + 1); seekAndAssert(testInputStream.getPos() - HadoopDataInputStream.MIN_SKIP_BYTES); seekAndAssert(testInputStream.getPos()); seekAndAssert(0); seekAndAssert(testInputStream.getPos() + HadoopDataInputStream.MIN_SKIP_BYTES); seekAndAssert(testInputStream.getPos() + HadoopDataInputStream.MIN_SKIP_BYTES - 1); try { seekAndAssert(-1); Assert.fail(); } catch (Exception ignore) { } try { seekAndAssert(-HadoopDataInputStream.MIN_SKIP_BYTES - 1); Assert.fail(); } catch (Exception ignore) { } }