@Override protected InputStream open(long offset) throws IOException { TachyonFS tfs = (TachyonFS) (Persist.I[Value.TACHYON].createClient(clientURI)); InputStream is = tfs.getFile(fpath).getInStream(ReadType.NO_CACHE); is.skip(offset); return is; } }
@Override public byte[] load(Value v) { Key k = v._key; // key for value if (k._kb[0] != Key.DVEC) throw H2O.unimpl(); // Load only from values stored in vector long skip = FileVec.chunkOffset(k); // Compute skip for this value long start_io_ms = System.currentTimeMillis(); final byte[] b = MemoryManager.malloc1(v._max); String[] keyComp = decodeKey(k); String clientUri = keyComp[0]; String fpath = keyComp[1]; TachyonFS tfs = null; InputStream is = null; try { tfs = (TachyonFS) (Persist.I[Value.TACHYON].createClient(clientUri)); long start_ns = System.nanoTime(); // Blocking i/o call timing - without counting repeats is = tfs.getFile(fpath).getInStream(ReadType.NO_CACHE); ByteStreams.skipFully(is, skip); ByteStreams.readFully(is, b); TimeLine.record_IOclose(start_ns, start_io_ms, 1/* read */, v._max, Value.TACHYON); return b; } catch (IOException e) { throw new RuntimeException(Log.err("File load failed: ", e)); } finally { if (is!=null) Utils.close(is); } }
private boolean read(TachyonFS tachyonClient) throws IOException { boolean pass = true; LOG.debug("Reading data..."); RawTable rawTable = tachyonClient.getRawTable(mId); ByteBuffer metadata = rawTable.getMetadata(); LOG.debug("Metadata: "); metadata.order(ByteOrder.nativeOrder()); for (int k = -mMetadataLength; k < 0; k ++) { pass = pass && (metadata.getInt() == k); } for (int column = 0; column < COLS; column ++) { RawColumn rawColumn = rawTable.getRawColumn(column); TachyonFile tFile = rawColumn.getPartition(0); FileInStream is = tFile.getInStream(mReadType); ByteBuffer buf = ByteBuffer.allocate(mDataLength * 4); is.read(buf.array()); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mDataLength; k ++) { pass = pass && (buf.getInt() == k); } is.close(); } return pass; }
mTachyonFileInputStream = mTachyonFile.getInStream(ReadType.CACHE);