@Override public InputStream open(String path, OpenOptions options) throws IOException { path = stripPath(path); FileInputStream inputStream = new FileInputStream(path); try { ByteStreams.skipFully(inputStream, options.getOffset()); } catch (IOException e) { inputStream.close(); throw e; } return inputStream; }
private static void skipHelper(long n, int expect, InputStream in) throws IOException { ByteStreams.skipFully(in, n); assertEquals(expect, in.read()); in.close(); }
@Nonnull public static DexBackedOdexFile fromInputStream(@Nonnull Opcodes opcodes, @Nonnull InputStream is) throws IOException { DexUtil.verifyOdexHeader(is); is.reset(); byte[] odexBuf = new byte[OdexHeaderItem.ITEM_SIZE]; ByteStreams.readFully(is, odexBuf); int dexOffset = OdexHeaderItem.getDexOffset(odexBuf); if (dexOffset > OdexHeaderItem.ITEM_SIZE) { ByteStreams.skipFully(is, dexOffset - OdexHeaderItem.ITEM_SIZE); } byte[] dexBuf = ByteStreams.toByteArray(is); return new DexBackedOdexFile(opcodes, odexBuf, dexBuf); }
@Override public Object call() throws Exception { FileSystem fs = FileSystem.get(p.toUri(), CONF); FSDataInputStream s = null; try { s = fs.open(p); // NOTE: // The following line degrades performance of HDFS load from S3 API: s.readFully(skip,b,0,b.length); // Google API's simple seek has better performance // Load of 300MB file via Google API ~ 14sec, via s.readFully ~ 5min (under the same condition) ByteStreams.skipFully(s, skip_); ByteStreams.readFully(s, b); assert v.isPersisted(); } finally { Utils.close(s); } return null; } }, true, v._max);
@Override public byte[] load(Value v) { Key k = v._key; // key for value if (k._kb[0] != Key.DVEC) throw H2O.unimpl(); // Load only from values stored in vector long skip = FileVec.chunkOffset(k); // Compute skip for this value long start_io_ms = System.currentTimeMillis(); final byte[] b = MemoryManager.malloc1(v._max); String[] keyComp = decodeKey(k); String clientUri = keyComp[0]; String fpath = keyComp[1]; TachyonFS tfs = null; InputStream is = null; try { tfs = (TachyonFS) (Persist.I[Value.TACHYON].createClient(clientUri)); long start_ns = System.nanoTime(); // Blocking i/o call timing - without counting repeats is = tfs.getFile(fpath).getInStream(ReadType.NO_CACHE); ByteStreams.skipFully(is, skip); ByteStreams.readFully(is, b); TimeLine.record_IOclose(start_ns, start_io_ms, 1/* read */, v._max, Value.TACHYON); return b; } catch (IOException e) { throw new RuntimeException(Log.err("File load failed: ", e)); } finally { if (is!=null) Utils.close(is); } }
@SuppressWarnings("CheckReturnValue") // these calls to skip always return 0 public void testSkip() throws Exception { MultiInputStream multi = new MultiInputStream( Collections.singleton( new ByteSource() { @Override public InputStream openStream() { return new ByteArrayInputStream(newPreFilledByteArray(0, 50)) { @Override public long skip(long n) { return 0; } }; } }) .iterator()); assertEquals(0, multi.skip(-1)); assertEquals(0, multi.skip(-1)); assertEquals(0, multi.skip(0)); ByteStreams.skipFully(multi, 20); assertEquals(20, multi.read()); }
skipFully(input, compressedDataSize);
@Override public InputStream getInput() throws IOException { final InputStream inputStream = new FileInputStream(file); ByteStreams.skipFully(inputStream, offset); return inputStream; } }
protected Payload doSlice(InputStream content, long offset, long length) { try { ByteStreams.skipFully(content, offset); } catch (IOException ioe) { throw Throwables.propagate(ioe); } return new InputStreamPayload(ByteStreams.limit(content, length)); }
protected Payload doSlice(InputStream content, long offset, long length) { try { ByteStreams.skipFully(content, offset); } catch (IOException ioe) { throw Throwables.propagate(ioe); } return new InputStreamPayload(ByteStreams.limit(content, length)); }
protected Payload doSlice(InputStream content, long offset, long length) { try { ByteStreams.skipFully(content, offset); } catch (IOException ioe) { throw Throwables.propagate(ioe); } return new InputStreamPayload(ByteStreams.limit(content, length)); }
protected Payload doSlice(InputStream content, long offset, long length) { try { ByteStreams.skipFully(content, offset); } catch (IOException ioe) { throw Throwables.propagate(ioe); } return Payloads.newInputStreamPayload(ByteStreams.limit(content, length)); }
protected Payload doSlice(InputStream content, long offset, long length) { try { ByteStreams.skipFully(content, offset); } catch (IOException ioe) { throw Throwables.propagate(ioe); } return Payloads.newInputStreamPayload(ByteStreams.limit(content, length)); }
protected Payload doSlice(InputStream content, long offset, long length) { try { ByteStreams.skipFully(content, offset); } catch (IOException ioe) { throw Throwables.propagate(ioe); } return Payloads.newInputStreamPayload(ByteStreams.limit(content, length)); }
@Override public InputStream getInput() throws IOException { InputStream in = supplier.getInput(); if (offset > 0) { try { skipFully(in, offset); } catch (IOException e) { Closeables.closeQuietly(in); throw e; } } return new LimitInputStream(in, length); } };
@Override public InputStream getInput() throws IOException { InputStream in = supplier.getInput(); if (offset > 0) { try { skipFully(in, offset); } catch (IOException e) { Closeables.closeQuietly(in); throw e; } } return new LimitInputStream(in, length); } };
@Nonnull public static DexBackedOdexFile fromInputStream(@Nonnull Opcodes opcodes, @Nonnull InputStream is) throws IOException { DexUtil.verifyOdexHeader(is); is.reset(); byte[] odexBuf = new byte[OdexHeaderItem.ITEM_SIZE]; ByteStreams.readFully(is, odexBuf); int dexOffset = OdexHeaderItem.getDexOffset(odexBuf); if (dexOffset > OdexHeaderItem.ITEM_SIZE) { ByteStreams.skipFully(is, dexOffset - OdexHeaderItem.ITEM_SIZE); } byte[] dexBuf = ByteStreams.toByteArray(is); return new DexBackedOdexFile(opcodes, odexBuf, dexBuf); }
private static void skipHelper(long n, int expect, InputStream in) throws IOException { ByteStreams.skipFully(in, n); assertEquals(expect, in.read()); in.close(); }
protected void testReadOffEnd(SliceInputTester tester, Slice slice) { SliceInput input = createSliceInput(slice); try { ByteStreams.skipFully(input, slice.length() - tester.valueSize() + 1); } catch (IOException e) { throw Throwables.propagate(e); } tester.verifyReadOffEnd(input); }
protected void testReadOffEnd(SliceInputTester tester, Slice slice) { SliceInput input = createSliceInput(slice); try { ByteStreams.skipFully(input, slice.length() - tester.valueSize() + 1); } catch (IOException e) { throw Throwables.propagate(e); } tester.verifyReadOffEnd(input); }