@Override public int read() throws IOException { return fsDataInputStream.read(); }
@Override public int read(byte[] b, int off, int len) throws IOException { return this.stream.read(b, off, len); }
@Override public int read(byte[] b, int off, int len) throws IOException { return stream.read(b, off, len); }
final public ByteBuffer read(ByteBufferPool bufferPool, int maxLength) throws IOException, UnsupportedOperationException { return read(bufferPool, maxLength, EMPTY_READ_OPTIONS_SET); }
@Override public int read(@Nonnull byte[] buffer, int offset, int length) throws IOException { return fsDataInputStream.read(buffer, offset, length); }
private void copyBytes(FSDataInputStream is, FSDataOutputStream out, long bytesToCopy) throws IOException { byte[] buf = new byte[bufferSize]; int n; while ((n = is.read(buf)) != -1 && bytesToCopy > 0) { out.write(buf, 0, (int) Math.min(n, bytesToCopy)); bytesToCopy -= n; } }
@Override public int readByteBuffer(FSDataInputStream file, ByteBuffer dest) throws IOException { int pos = dest.position(); int result = file.read(dest); if (result > 0) { // Ensure this explicitly since versions before 2.7 read doesn't do it. dest.position(pos + result); } return result; } @Override
public static String readAsString(final FileSystem fs, final Path fromMetadataPath) throws IOException { try (FSDataInputStream stream = fs.open(fromMetadataPath)) { byte[] buffer = new byte[1024]; ByteArrayOutputStream sb = new ByteArrayOutputStream(); int read = stream.read(buffer); while (read != -1) { sb.write(buffer, 0, read); read = stream.read(buffer); } return new String(sb.toByteArray(), "UTF-8"); } }
@Override public int read(long position, byte[] buffer, int offset, int length) throws IOException { int n; try { n = in.read(position, buffer, offset, length); } catch (FileNotFoundException e) { n = tryOpen().read(position, buffer, offset, length); } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() n = tryOpen().read(position, buffer, offset, length); } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() n = tryOpen().read(position, buffer, offset, length); } return n; }
@Override public int read() throws IOException { int res; try { res = in.read(); } catch (FileNotFoundException e) { res = tryOpen().read(); } catch (NullPointerException e) { // HDFS 1.x - DFSInputStream.getBlockAt() res = tryOpen().read(); } catch (AssertionError e) { // assert in HDFS 1.x - DFSInputStream.getBlockAt() res = tryOpen().read(); } if (res > 0) pos += 1; return res; }
private static void skipBuffer(FSDataInputStream in, byte v) throws IOException { byte[] data = new byte[8192]; try { int n; while ((n = in.read(data)) == data.length) { for (int i = 0; i < data.length; ++i) { if (data[i] != v) throw new Exception("File changed"); } } } catch (Exception e) { } } }
/** {@inheritDoc} */ @Override public synchronized int read() throws IOException { readStart(); int res; try { res = is.read(); } finally { readEnd(); } if (res != -1) total++; return res; }
/** {@inheritDoc} */ @Override public synchronized int read(byte[] b) throws IOException { readStart(); int res; try { res = is.read(b); } finally { readEnd(); } if (res != -1) total += res; return res; }
public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException { FileStatus fst = fs.getFileStatus(src); long len = fst.getLen(); len = len / 2 ; // create a truncated hfile FSDataOutputStream fdos = fs.create(dst); byte[] buf = new byte[(int)len]; FSDataInputStream fdis = fs.open(src); fdis.read(buf); fdos.write(buf); fdis.close(); fdos.close(); }
@Test public void testPositionalReadNoExtra() throws IOException { long position = 0; int bufOffset = 0; int necessaryLen = 10; int extraLen = 0; int totalLen = necessaryLen + extraLen; byte[] buf = new byte[totalLen]; FSDataInputStream in = mock(FSDataInputStream.class); when(in.read(position, buf, bufOffset, totalLen)).thenReturn(totalLen); boolean ret = HFileBlock.positionalReadWithExtra(in, position, buf, bufOffset, necessaryLen, extraLen); assertFalse("Expect false return when no extra bytes requested", ret); verify(in).read(position, buf, bufOffset, totalLen); verifyNoMoreInteractions(in); }
@Test public void testReadRequestRangeNotSatisfiable() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); s3.setGetObjectHttpErrorCode(SC_REQUESTED_RANGE_NOT_SATISFIABLE); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) { assertEquals(inputStream.read(), -1); } } }
private void fileIsGoodAvro(Path path) throws IOException { DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(); try (FSDataInputStream in = fs.open(path, 0); FileOutputStream out = new FileOutputStream("target/FOO.avro")) { byte[] buffer = new byte[100]; int bytesRead; while ((bytesRead = in.read(buffer)) > 0) { out.write(buffer, 0, bytesRead); } } java.io.File file = new File("target/FOO.avro"); try (DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(file, datumReader)) { GenericRecord user = null; while (dataFileReader.hasNext()) { user = dataFileReader.next(user); } } file.delete(); } }
@SuppressWarnings("ResultOfMethodCallIgnored") @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + SC_NOT_FOUND + ".*") public void testReadNotFound() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); s3.setGetObjectHttpErrorCode(SC_NOT_FOUND); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) { inputStream.read(); } } }
@SuppressWarnings("ResultOfMethodCallIgnored") @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + SC_FORBIDDEN + ".*") public void testReadForbidden() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); s3.setGetObjectHttpErrorCode(SC_FORBIDDEN); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) { inputStream.read(); } } }
@SuppressWarnings({"ResultOfMethodCallIgnored", "OverlyStrongTypeCast", "ConstantConditions"}) @Test public void testReadRetryCounters() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { int maxRetries = 2; MockAmazonS3 s3 = new MockAmazonS3(); s3.setGetObjectHttpErrorCode(SC_INTERNAL_SERVER_ERROR); Configuration configuration = new Configuration(); configuration.set(S3_MAX_BACKOFF_TIME, "1ms"); configuration.set(S3_MAX_RETRY_TIME, "5s"); configuration.setInt(S3_MAX_CLIENT_RETRIES, maxRetries); fs.initialize(new URI("s3n://test-bucket/"), configuration); fs.setS3Client(s3); try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) { inputStream.read(); } catch (Throwable expected) { assertInstanceOf(expected, AmazonS3Exception.class); assertEquals(((AmazonS3Exception) expected).getStatusCode(), SC_INTERNAL_SERVER_ERROR); assertEquals(PrestoS3FileSystem.getFileSystemStats().getReadRetries().getTotalCount(), maxRetries); assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetObjectRetries().getTotalCount(), (maxRetries + 1L) * maxRetries); } } }