/** * Determine if a stream can do a byte buffer read via read(ByteBuffer buf) */ private static boolean streamHasByteBufferRead(InputStream stream) { if (!(stream instanceof ByteBufferReadable)) { return false; } if (!(stream instanceof FSDataInputStream)) { return true; } return ((FSDataInputStream)stream).getWrappedStream() instanceof ByteBufferReadable; }
FSDataInputStream stream = this.getStream(this.shouldUseHBaseChecksum()); if (stream != null) { InputStream wrappedStream = stream.getWrappedStream();
/** * Determine if a stream can do a byte buffer read via read(ByteBuffer buf) */ private static boolean streamHasByteBufferRead(InputStream stream) { if (!(stream instanceof ByteBufferReadable)) { return false; } if (!(stream instanceof FSDataInputStream)) { return true; } return ((FSDataInputStream)stream).getWrappedStream() instanceof ByteBufferReadable; }
/** * Determine if a stream can do a byte buffer read via read(ByteBuffer buf) */ private static boolean streamHasByteBufferRead(InputStream stream) { if (!(stream instanceof ByteBufferReadable)) { return false; } if (!(stream instanceof FSDataInputStream)) { return true; } return ((FSDataInputStream)stream).getWrappedStream() instanceof ByteBufferReadable; }
/** * Determine if a stream can do a byte buffer read via read(ByteBuffer buf) */ private static boolean streamHasByteBufferRead(InputStream stream) { if (!(stream instanceof ByteBufferReadable)) { return false; } if (!(stream instanceof FSDataInputStream)) { return true; } return ((FSDataInputStream)stream).getWrappedStream() instanceof ByteBufferReadable; }
/** * Determine if a stream can do a byte buffer read via read(ByteBuffer buf) */ private static boolean streamHasByteBufferRead(InputStream stream) { if (!(stream instanceof ByteBufferReadable)) { return false; } if (!(stream instanceof FSDataInputStream)) { return true; } return ((FSDataInputStream)stream).getWrappedStream() instanceof ByteBufferReadable; }
/** * Determine if a stream can do a byte buffer read via read(ByteBuffer buf) */ private static boolean streamHasByteBufferRead(InputStream stream) { if (!(stream instanceof ByteBufferReadable)) { return false; } if (!(stream instanceof FSDataInputStream)) { return true; } return ((FSDataInputStream)stream).getWrappedStream() instanceof ByteBufferReadable; }
@Override @LimitedPrivate({"HDFS"}) public InputStream getWrappedStream() { return underlyingIs.getWrappedStream(); }
@Override @LimitedPrivate({"HDFS"}) public InputStream getWrappedStream() { return underlyingIs.getWrappedStream(); }
public FSDataInputChannel(FSDataInputStream inputStream) throws IOException { if (inputStream.getWrappedStream() instanceof ByteBufferReadable) { this.isDirectRead = true; } else { /* LocalFileSystem, S3 does not support ByteBufferReadable */ this.channel = Channels.newChannel(inputStream); } this.inputStream = inputStream; this.size = inputStream.getPos() + inputStream.available(); }
public FSDataInputChannel(FSDataInputStream inputStream) throws IOException { if (inputStream.getWrappedStream() instanceof ByteBufferReadable) { this.isDirectRead = true; } else { /* LocalFileSystem, S3 does not support ByteBufferReadable */ this.channel = Channels.newChannel(inputStream); } this.inputStream = inputStream; this.size = inputStream.getPos() + inputStream.available(); }
public static void readFully(FSDataInputStream reader, long offset, ByteBuffer buffer) throws IOException { if (offset >= 0) { reader.seek(offset); } InputStream is = reader.getWrappedStream(); if (!(is instanceof ByteBufferReadable)) { logger.trace("Using read bytes method"); byte[] bytes = new byte[buffer.remaining()]; reader.readFully(bytes); buffer.put(bytes); } else { while (buffer.hasRemaining()) { int pos = buffer.position(); int rt = reader.read(buffer); if (rt < 0) { throw new IOException("End of stream"); } buffer.position(pos + rt); } } Preconditions.checkState(!buffer.hasRemaining()); }
private void tryGetLocalFile() { if (tryGetLocalFileTimes >= TRY_GET_LOCAL_FILE_LIMIT) { return; } if (isSingleBlock && HDFS_READ_HACK_ENABLE) { try { InputStream is = input.getWrappedStream(); if (is instanceof DFSInputStream) { BlockReader blockReader = MemoryUtil.getDFSInputStream_blockReader(is); if (blockReader != null && blockReader.isShortCircuit()) { localFile = MemoryUtil.getBlockReaderLocal_dataIn(blockReader); } } } catch (Throwable e) { logger.debug("HDFS READ HACK failed.", e); } } tryGetLocalFileTimes++; }
HoodieLogFileReader(FileSystem fs, HoodieLogFile logFile, Schema readerSchema, int bufferSize, boolean readBlockLazily, boolean reverseReader) throws IOException { this.inputStream = new FSDataInputStream( new BufferedFSInputStream((FSInputStream) fs.open(logFile.getPath(), bufferSize).getWrappedStream(), bufferSize)); this.logFile = logFile; this.readerSchema = readerSchema; this.readBlockLazily = readBlockLazily; this.reverseReader = reverseReader; if (this.reverseReader) { this.reverseLogFilePosition = this.lastReverseLogFilePosition = fs .getFileStatus(logFile.getPath()).getLen(); } addShutDownHook(); }
@Test public void testGetWrappedInputStream() throws IOException { Path src = getTestRootPath(fSys, "test/hadoop/file"); createFile(src); FSDataInputStream in = fSys.open(src); InputStream is = in.getWrappedStream(); in.close(); Assert.assertNotNull(is); }
@Test public void testGetWrappedInputStream() throws IOException { Path src = getTestRootPath(fSys, "test/hadoop/file"); createFile(src); FSDataInputStream in = fSys.open(src); InputStream is = in.getWrappedStream(); in.close(); Assert.assertNotNull(is); }
public static void check(FileSystem fs, Path p, long length) throws IOException { int i = -1; try { final FileStatus status = fs.getFileStatus(p); FSDataInputStream in = fs.open(p); if (in.getWrappedStream() instanceof DFSInputStream) { long len = ((DFSInputStream)in.getWrappedStream()).getFileLength(); assertEquals(length, len); } else { assertEquals(length, status.getLen()); } for(i++; i < length; i++) { assertEquals((byte)i, (byte)in.read()); } i = -(int)length; assertEquals(-1, in.read()); //EOF in.close(); } catch(IOException ioe) { throw new IOException("p=" + p + ", length=" + length + ", i=" + i, ioe); } }
@Test public void testStreamType() throws Exception { Path testPath = new Path(TEST_ROOT_DIR, "testStreamType"); localFs.create(testPath).close(); FSDataInputStream in = null; localFs.setVerifyChecksum(true); in = localFs.open(testPath); assertTrue("stream is input checker", in.getWrappedStream() instanceof FSInputChecker); localFs.setVerifyChecksum(false); in = localFs.open(testPath); assertFalse("stream is not input checker", in.getWrappedStream() instanceof FSInputChecker); }
@Test public void testStreamType() throws Exception { Path testPath = new Path(TEST_ROOT_DIR, "testStreamType"); localFs.create(testPath).close(); FSDataInputStream in = null; localFs.setVerifyChecksum(true); in = localFs.open(testPath); assertTrue("stream is input checker", in.getWrappedStream() instanceof FSInputChecker); localFs.setVerifyChecksum(false); in = localFs.open(testPath); assertFalse("stream is not input checker", in.getWrappedStream() instanceof FSInputChecker); }
@Test public void testReaderWithLocalFS() throws Exception { Path tablePath = new Path(testDir, "testReaderWithLocalFS"); Path filePath = new Path(tablePath, "data.dat"); FileSystem fileSystem = filePath.getFileSystem(conf); assertTrue(fileSystem instanceof LocalFileSystem); FSDataOutputStream out = fs.create(filePath, true); out.write(LINE.getBytes(Charset.defaultCharset())); out.write('\n'); out.close(); assertTrue(fs.exists(filePath)); FSDataInputStream inputStream = fs.open(filePath); assertFalse(inputStream.getWrappedStream() instanceof ByteBufferReadable); ByteBufLineReader lineReader = new ByteBufLineReader(new FSDataInputChannel(inputStream)); assertEquals(LINE, lineReader.readLine()); lineReader.seek(0); assertEquals(LINE, lineReader.readLine()); assertNull(lineReader.readLine()); lineReader.close(); fs.close(); }