private boolean readFile(FileSystem fs) throws IOException, AlluxioException { boolean pass = true; for (int i = 0; i < mNumFiles; i++) { AlluxioURI filePath = new AlluxioURI(mFileFolder + "/part-" + i); LOG.debug("Reading data from {}", filePath); FileInStream is = fs.openFile(filePath); URIStatus status = fs.getStatus(filePath); ByteBuffer buf = ByteBuffer.allocate((int) status.getBlockSizeBytes()); is.read(buf.array()); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mNumFiles; k++) { pass = pass && (buf.getInt() == k); } is.close(); } return pass; }
/** * Tests seeking to the beginning of the last block after reaching EOF. */ @Test public void seekToLastBlockAfterReachingEOF() throws IOException { mTestStream.read(new byte[(int) FILE_LENGTH]); mTestStream.seek(FILE_LENGTH - BLOCK_LENGTH); byte[] block = new byte[(int) BLOCK_LENGTH]; mTestStream.read(block); assertArrayEquals(BufferUtils.getIncreasingByteArray( (int) (FILE_LENGTH - BLOCK_LENGTH), (int) BLOCK_LENGTH), block); }
/** * Tests that {@link FileInStream#remaining()} is correctly updated during reads, skips, and * seeks. */ @Test public void testRemaining() throws IOException { assertEquals(FILE_LENGTH, mTestStream.remaining()); mTestStream.read(); assertEquals(FILE_LENGTH - 1, mTestStream.remaining()); mTestStream.read(new byte[150]); assertEquals(FILE_LENGTH - 151, mTestStream.remaining()); mTestStream.skip(140); assertEquals(FILE_LENGTH - 291, mTestStream.remaining()); mTestStream.seek(310); assertEquals(FILE_LENGTH - 310, mTestStream.remaining()); mTestStream.seek(130); assertEquals(FILE_LENGTH - 130, mTestStream.remaining()); }
private boolean readFile(FileSystem fileSystem) throws IOException, AlluxioException { boolean pass = true; LOG.debug("Reading data..."); final long startTimeMs = CommonUtils.getCurrentMs(); FileInStream is = fileSystem.openFile(mFilePath, mReadOptions); ByteBuffer buf = ByteBuffer.allocate((int) is.remaining()); is.read(buf.array()); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < NUMBERS; k++) { pass = pass && (buf.getInt() == k); } is.close(); LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "readFile file " + mFilePath)); return pass; } }
/** * Tests skipping backwards when the seek buffer size is smaller than block size. */ @Test public void seekBackwardSmallSeekBuffer() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int readAmount = (int) (BLOCK_LENGTH / 2); byte[] buffer = new byte[readAmount]; mTestStream.read(buffer); mTestStream.seek(readAmount - 1); validatePartialCaching(0, readAmount); }
@Override public int read(byte[] b) throws IOException { return read(b, 0, b.length); }
/** * Tests seeking with incomplete block caching enabled. It seeks forward for more than a block. */ @Test public void longSeekForwardCachingPartiallyReadBlocks() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int seekAmount = (int) (BLOCK_LENGTH / 4 + BLOCK_LENGTH); int readAmount = (int) (BLOCK_LENGTH / 2); byte[] buffer = new byte[readAmount]; mTestStream.read(buffer); // Seek backward. mTestStream.seek(readAmount + seekAmount); // Block 0 is cached though it is not fully read. validatePartialCaching(0, readAmount); // Block 1 is being cached though its prefix it not read. validatePartialCaching(1, 0); mTestStream.close(); validatePartialCaching(1, 0); }
@Override protected void runPlainPath(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(path); int numOfBytes = Constants.KB; if (cl.hasOption('c')) { numOfBytes = (int) FormatUtils.parseSpaceSize(cl.getOptionValue('c')); Preconditions.checkArgument(numOfBytes > 0, "specified bytes must be > 0"); } if (status.isFolder()) { throw new IOException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(path)); } OpenFilePOptions options = OpenFilePOptions.getDefaultInstance(); try (FileInStream is = mFileSystem.openFile(path, options)) { byte[] buf = new byte[numOfBytes]; long bytesToRead; if (status.getLength() > numOfBytes) { bytesToRead = numOfBytes; } else { bytesToRead = status.getLength(); } is.skip(status.getLength() - bytesToRead); int read = is.read(buf); if (read != -1) { System.out.write(buf, 0, read); } } }
@Test public void seekAndClose() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int seekAmount = (int) (BLOCK_LENGTH / 2); mTestStream.seek(seekAmount); mTestStream.close(); // Block 0 is cached though it is not fully read. validatePartialCaching(0, 0); }
@Override public void close() throws IOException { if (mClosed) { return; } mInputStream.close(); mClosed = true; }
@Override public long skip(long n) throws IOException { if (n <= 0) { return 0; } long toSkip = Math.min(n, mLength - mPosition); seek(mPosition + toSkip); return toSkip; }
@Override public long skip(long n) throws IOException { if (mClosed) { throw new IOException("Cannot skip bytes in a closed stream."); } return mInputStream.skip(n); } }
@Override public FileInStream openFile(AlluxioURI path, OpenFilePOptions options) throws FileDoesNotExistException, IOException, AlluxioException { checkUri(path); URIStatus status = getStatus(path); if (status.isFolder()) { throw new FileDoesNotExistException( ExceptionMessage.CANNOT_READ_DIRECTORY.getMessage(status.getName())); } InStreamOptions inStreamOptions = new InStreamOptions(status, options, mFsContext.getConf()); return new FileInStream(status, inStreamOptions, mFsContext); }
@Override public int available() throws IOException { if (mClosed) { throw new IOException("Cannot query available bytes from a closed stream."); } return (int) mInputStream.remaining(); }
@Override public long getPos() throws IOException { return mInputStream.getPos(); }
/** * Tests seeking with incomplete block caching enabled. It seeks backward within 1 block. */ @Test public void shortSeekBackwardCachingPartiallyReadBlocks() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int seekAmount = (int) (BLOCK_LENGTH / 4); int readAmount = (int) (BLOCK_LENGTH * 2 - BLOCK_LENGTH / 2); byte[] buffer = new byte[readAmount]; mTestStream.read(buffer); // Seek backward. mTestStream.seek(readAmount - seekAmount); // Block 1 is cached though it is not fully read. validatePartialCaching(1, (int) BLOCK_LENGTH / 2); // Seek many times. It will cache block 1 only once. for (int i = 0; i <= seekAmount; i++) { mTestStream.seek(readAmount - seekAmount - i); } validatePartialCaching(1, (int) BLOCK_LENGTH / 2); }
@Override public int read(byte[] buffer, int offset, int length) throws IOException { if (mClosed) { throw new IOException(ExceptionMessage.READ_CLOSED_STREAM.getMessage()); } int bytesRead = mInputStream.read(buffer, offset, length); if (bytesRead != -1 && mStatistics != null) { mStatistics.incrementBytesRead(bytesRead); } return bytesRead; }
private boolean readFile(FileSystem fileSystem) throws IOException, AlluxioException { boolean pass = true; LOG.debug("Reading data..."); final long startTimeMs = CommonUtils.getCurrentMs(); FileInStream is = fileSystem.openFile(mFilePath, mReadOptions); ByteBuffer buf = ByteBuffer.allocate((int) is.remaining()); is.read(buf.array()); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < NUMBERS; k++) { pass = pass && (buf.getInt() == k); } is.close(); LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "readFile file " + mFilePath)); return pass; } }
/** * Tests skip, particularly that skipping the start of a block will cause us not to cache it, and * cancels the existing cache stream. */ @Test public void testSkip() throws IOException { int skipAmount = (int) (BLOCK_LENGTH / 2); int readAmount = (int) (BLOCK_LENGTH * 2); byte[] buffer = new byte[readAmount]; // Skip halfway into block 1 mTestStream.skip(skipAmount); // Read two blocks from 0.5 to 2.5 mTestStream.read(buffer); assertArrayEquals(BufferUtils.getIncreasingByteArray(skipAmount, readAmount), buffer); assertEquals(0, mTestStream.skip(0)); // Skip the next half block, bringing us to block 3 assertEquals(BLOCK_LENGTH / 2, mTestStream.skip(BLOCK_LENGTH / 2)); assertEquals(BufferUtils.byteToInt((byte) (BLOCK_LENGTH * 3)), mTestStream.read()); }
/** * Tests seeking with incomplete block caching enabled. It seeks forward for more than a block * and then seek to the file beginning. */ @Test public void seekBackwardToFileBeginning() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int seekAmount = (int) (BLOCK_LENGTH / 4 + BLOCK_LENGTH); // Seek forward. mTestStream.seek(seekAmount); // Block 1 is partially cached though it is not fully read. validatePartialCaching(1, 0); // Seek backward. mTestStream.seek(0); // Block 1 is fully cached though it is not fully read. validatePartialCaching(1, 0); mTestStream.close(); // block 0 is cached validatePartialCaching(0, 0); }