@Override public TachyonFS createClient(String uri) throws IOException { if (!uri.startsWith(PREFIX)) uri = PREFIX + uri; return TachyonFS.get(uri); } }
client = ((PersistTachyon) Persist.I[Value.TACHYON]).createClient(PersistTachyon.PREFIX+serverUri); String rootFolder = pathComponents[1]; List<ClientFileInfo> filesOnTFS= client.listStatus(rootFolder); // do a recursive descend Futures fs = new Futures(); for (ClientFileInfo f : filesOnTFS ) { fillEmpty("Cannot access specified file(s) on tachyon FS, because " + e.getMessage()); } finally { if (client!=null) try { client.close(); } catch (TException _ ) {};
@Override protected InputStream open(long offset) throws IOException { TachyonFS tfs = (TachyonFS) (Persist.I[Value.TACHYON].createClient(clientURI)); InputStream is = tfs.getFile(fpath).getInStream(ReadType.NO_CACHE); is.skip(offset); return is; } }
if (mTFS.exist(path)) { if (overwrite && !mTFS.getFileStatus(-1, path).isFolder) { if (!mTFS.delete(path, false)) { throw new IOException("Failed to delete existing data " + cPath); long fileId = mTFS.createFile(path, blockSize); TachyonFile file = mTFS.getFile(fileId); file.setUFSConf(getConf());
/** * {@inheritDoc} * * Sets up a lazy connection to Tachyon through mTFS. */ @Override public void initialize(URI uri, Configuration conf) throws IOException { super.initialize(uri, conf); LOG.info("initialize(" + uri + ", " + conf + "). Connecting to Tachyon: " + uri.toString()); Utils.addS3Credentials(conf); setConf(conf); mTachyonHeader = getScheme() + "://" + uri.getHost() + ":" + uri.getPort(); // Set the statistics member. Use mStatistics instead of the parent class's variable. mStatistics = statistics; // Load TachyonConf if any and merge to the one in TachyonFS TachyonConf siteConf = ConfUtils.loadFromHadoopConfiguration(conf); if (siteConf != null) { mTachyonConf.merge(siteConf); } mTachyonConf.set(Constants.MASTER_HOSTNAME, uri.getHost()); mTachyonConf.set(Constants.MASTER_PORT, Integer.toString(uri.getPort())); mTachyonConf.set(Constants.ZOOKEEPER_ENABLED, Boolean.toString(isZookeeperMode())); mTFS = TachyonFS.get(mTachyonConf); mUri = URI.create(mTachyonHeader); mUnderFSAddress = mTFS.getUfsAddress(); LOG.info(mTachyonHeader + " " + mUri + " " + mUnderFSAddress); }
@Override public void close() throws IOException { synchronized (mClients) { for (TachyonFS fs : mClients) { fs.close(); } mClients.clear(); } } }
/** * Creates a <code>RawTable</code> and returns its id. * * @param path the RawTable's path * @param columns number of columns it has * @return the id if succeed, -1 otherwise * @throws IOException if the number of columns is invalid or the underlying master RPC fails */ public synchronized long createRawTable(TachyonURI path, int columns) throws IOException { return createRawTable(path, columns, ByteBuffer.allocate(0)); }
/** * Gets <code>TachyonFile</code> based on the path. Does not utilize the file metadata cache. * * @param path file path. * @return TachyonFile of the path, or null if the file does not exist * @throws IOException if the underlying master RPC fails */ public synchronized TachyonFile getFile(TachyonURI path) throws IOException { validateUri(path); return getFile(path, false); }
@Override public FSDataOutputStream append(Path cPath, int bufferSize, Progressable progress) throws IOException { LOG.info("append(" + cPath + ", " + bufferSize + ", " + progress + ")"); if (mStatistics != null) { mStatistics.incrementWriteOps(1); } TachyonURI path = new TachyonURI(Utils.getPathWithoutScheme(cPath)); long fileId = mTFS.getFileId(path); TachyonFile file = mTFS.getFile(fileId); if (file.length() > 0) { LOG.warn("This maybe an error."); } return new FSDataOutputStream(file.getOutStream(), mStatistics); }
@Override public FileStatus[] listStatus(Path path) throws IOException { TachyonURI tPath = new TachyonURI(Utils.getPathWithoutScheme(path)); Path hdfsPath = Utils.getHDFSPath(tPath, mUnderFSAddress); LOG.info("listStatus(" + path + "): HDFS Path: " + hdfsPath); if (mStatistics != null) { mStatistics.incrementReadOps(1); } if (!mTFS.exist(tPath)) { throw new FileNotFoundException("File does not exist: " + path); } List<FileInfo> files = mTFS.listStatus(tPath); FileStatus[] ret = new FileStatus[files.size()]; for (int k = 0; k < files.size(); k ++) { FileInfo info = files.get(k); // TODO(hy): Replicate 3 with the number of disk replications. ret[k] = new FileStatus(info.getLength(), info.isFolder, 3, info.getBlockSizeBytes(), info.getCreationTimeMs(), info.getCreationTimeMs(), null, null, null, new Path( mTachyonHeader + info.getPath())); } return ret; }
if (!tfs.exist(directoryName)) { LOG.debug("Loading ufs. Make dir if needed for '" + directoryName + "'."); tfs.mkdir(directoryName); TachyonURI tfsPath = buildTFSPath(directoryName, ufsAddrRootPath, ufsPath); LOG.debug("Loading ufs. tfs path = " + tfsPath + "."); if (tfs.exist(tfsPath)) { LOG.debug("File " + tfsPath + " already exists in Tachyon."); continue; long fileId = tfs.createFile(tfsPath, ufsPath); if (fileId == -1) { LOG.warn("Failed to create tachyon file: " + tfsPath); tachyonPath, ufsPath.getPath().substring(ufsAddrRootPath.getPath().length()))); LOG.debug("Loading ufs. ufs path is a directory. tfsPath = " + tfsPath + "."); if (!tfs.exist(tfsPath)) { LOG.debug("Loading ufs. ufs path is a directory. make dir = " + tfsPath + "."); tfs.mkdir(tfsPath);
/** * Identifies the number of column partitions. * * TODO(hy): Creating file here should be based on id. * * @return the number of column partitions * @throws IOException when any of the partition paths is invalid or points to a non-existing * object */ public int partitions() throws IOException { TachyonURI tUri = new TachyonURI(PathUtils.concatPath(mRawTable.getPath(), Constants.MASTER_COLUMN_FILE_PREFIX + mColumnIndex)); return mTachyonFS.listStatus(tUri).size(); } }
/** * Creates a <code>TachyonFS</code> handler for the given Tachyon configuration. * * @param tachyonConf The TachyonConf instance. * @return the corresponding TachyonFS handler */ public static synchronized TachyonFS get(TachyonConf tachyonConf) { Preconditions.checkArgument(tachyonConf != null, "TachyonConf cannot be null."); return new TachyonFS(tachyonConf); }
/** * Creates a new column partition. * * TODO(hy): Creating file here should be based on id. * * @param pId the partition id * @return whether operation succeeded * @throws IOException when the partition the path is invalid or points to an existing object */ public boolean createPartition(int pId) throws IOException { TachyonURI tUri = new TachyonURI(PathUtils.concatPath(getColumnPath(mRawTable.getPath(), mColumnIndex), pId)); return mTachyonFS.createFile(tUri) > 0; }
@Override public void close() throws IOException { try { super.close(); } finally { if (mTFS != null) { mTFS.close(); } } }
private void createRawTable(TachyonFS tachyonClient) throws IOException { ByteBuffer data = ByteBuffer.allocate(mMetadataLength * 4); data.order(ByteOrder.nativeOrder()); for (int k = -mMetadataLength; k < 0; k ++) { data.putInt(k); } data.flip(); mId = tachyonClient.createRawTable(mTablePath, 3, data); }
/** * Creates a <code>TachyonFS</code> handler for the given Tachyon URI. * * @param tachyonURI a Tachyon URI to indicate master address. e.g., tachyon://localhost:19998, * tachyon://localhost:19998/ab/c.txt * @return the corresponding TachyonFS handler * @see #get(tachyon.TachyonURI, tachyon.conf.TachyonConf) */ @Deprecated public static synchronized TachyonFS get(final TachyonURI tachyonURI) { return get(tachyonURI, ClientContext.getConf()); }
@Override public byte[] load(Value v) { Key k = v._key; // key for value if (k._kb[0] != Key.DVEC) throw H2O.unimpl(); // Load only from values stored in vector long skip = FileVec.chunkOffset(k); // Compute skip for this value long start_io_ms = System.currentTimeMillis(); final byte[] b = MemoryManager.malloc1(v._max); String[] keyComp = decodeKey(k); String clientUri = keyComp[0]; String fpath = keyComp[1]; TachyonFS tfs = null; InputStream is = null; try { tfs = (TachyonFS) (Persist.I[Value.TACHYON].createClient(clientUri)); long start_ns = System.nanoTime(); // Blocking i/o call timing - without counting repeats is = tfs.getFile(fpath).getInStream(ReadType.NO_CACHE); ByteStreams.skipFully(is, skip); ByteStreams.readFully(is, b); TimeLine.record_IOclose(start_ns, start_io_ms, 1/* read */, v._max, Value.TACHYON); return b; } catch (IOException e) { throw new RuntimeException(Log.err("File load failed: ", e)); } finally { if (is!=null) Utils.close(is); } }
if (sOldTfs.createRawTable(new TachyonURI(mWorkDir + mSuccessNum), 1) == -1) { break;
/** * Returns a {@link tachyon.client.TachyonFS} client. This client does not need to be closed * directly, but can be closed by calling {@link #close()} on this object. * * @param tachyonConf Tachyon configuration * @return a TachyonFS client * @throws IOException when the operation fails */ public TachyonFS getClient(TachyonConf tachyonConf) throws IOException { final TachyonFS fs = TachyonFS.get(new TachyonURI(mUriSuppliers.get()), tachyonConf); mClients.add(fs); return fs; }