@Override protected InputStream open(long offset) throws IOException { TachyonFS tfs = (TachyonFS) (Persist.I[Value.TACHYON].createClient(clientURI)); InputStream is = tfs.getFile(fpath).getInStream(ReadType.NO_CACHE); is.skip(offset); return is; } }
@Override public FSDataOutputStream append(Path cPath, int bufferSize, Progressable progress) throws IOException { LOG.info("append(" + cPath + ", " + bufferSize + ", " + progress + ")"); if (mStatistics != null) { mStatistics.incrementWriteOps(1); } TachyonURI path = new TachyonURI(Utils.getPathWithoutScheme(cPath)); long fileId = mTFS.getFileId(path); TachyonFile file = mTFS.getFile(fileId); if (file.length() > 0) { LOG.warn("This maybe an error."); } return new FSDataOutputStream(file.getOutStream(), mStatistics); }
TachyonFile file = mTFS.getFile(fileId); file.setUFSConf(getConf());
@Override public byte[] load(Value v) { Key k = v._key; // key for value if (k._kb[0] != Key.DVEC) throw H2O.unimpl(); // Load only from values stored in vector long skip = FileVec.chunkOffset(k); // Compute skip for this value long start_io_ms = System.currentTimeMillis(); final byte[] b = MemoryManager.malloc1(v._max); String[] keyComp = decodeKey(k); String clientUri = keyComp[0]; String fpath = keyComp[1]; TachyonFS tfs = null; InputStream is = null; try { tfs = (TachyonFS) (Persist.I[Value.TACHYON].createClient(clientUri)); long start_ns = System.nanoTime(); // Blocking i/o call timing - without counting repeats is = tfs.getFile(fpath).getInStream(ReadType.NO_CACHE); ByteStreams.skipFully(is, skip); ByteStreams.readFully(is, b); TimeLine.record_IOclose(start_ns, start_io_ms, 1/* read */, v._max, Value.TACHYON); return b; } catch (IOException e) { throw new RuntimeException(Log.err("File load failed: ", e)); } finally { if (is!=null) Utils.close(is); } }
/** * Gets <code>TachyonFile</code> based on the file id. * * NOTE: This *will* use cached file metadata, and so will not see changes to dynamic properties, * such as the pinned flag. This is also different from the behavior of getFile(path), which by * default will not use cached metadata. * * @param fid file id. * @return <code>TachyonFile</code> of the file id, or null if the file does not exist * @throws IOException if the underlying master RPC fails */ public synchronized TachyonFile getFile(long fid) throws IOException { return getFile(fid, true); }
/** * Gets <code>TachyonFile</code> based on the path. Does not utilize the file metadata cache. * * @param path file path. * @return TachyonFile of the path, or null if the file does not exist * @throws IOException if the underlying master RPC fails */ public synchronized TachyonFile getFile(TachyonURI path) throws IOException { validateUri(path); return getFile(path, false); }
mHadoopBufferSize = bufferSize; mStatistics = stats; mTachyonFile = mTFS.getFile(mFileId); if (mTachyonFile == null) { throw new FileNotFoundException("File " + hdfsPath + " with FID " + fileId
/** * Gets an existing partition. * * TODO(hy): Creating file here should be based on id. * * @param pId the partition id * @param cachedMetadata whether to use the file metadata cache * @return <code>TachyonFile</code> with the given partition * @throws IOException when the partition the path is invalid or points to a non-existing object */ public TachyonFile getPartition(int pId, boolean cachedMetadata) throws IOException { TachyonURI tUri = new TachyonURI(PathUtils.concatPath(mRawTable.getPath(), Constants.MASTER_COLUMN_FILE_PREFIX + mColumnIndex, pId)); return mTachyonFS.getFile(tUri, cachedMetadata); }
/** * {@inheritDoc} * * If the file does not exist in Tachyon, query it from HDFS. */ @Override public FileStatus getFileStatus(Path path) throws IOException { TachyonURI tPath = new TachyonURI(Utils.getPathWithoutScheme(path)); Path hdfsPath = Utils.getHDFSPath(tPath, mUnderFSAddress); LOG.info("getFileStatus(" + path + "): HDFS Path: " + hdfsPath + " TPath: " + mTachyonHeader + tPath); if (mStatistics != null) { mStatistics.incrementReadOps(1); } TachyonFile file; try { file = mTFS.getFile(tPath); } catch (IOException ioe) { LOG.info("File does not exist: " + path); throw new FileNotFoundException("File does not exist: " + path); } FileStatus ret = new FileStatus(file.length(), file.isDirectory(), file.getDiskReplication(), file.getBlockSizeByte(), file.getCreationTimeMs(), file.getCreationTimeMs(), null, null, null, new Path(mTachyonHeader + tPath)); return ret; }