/** * Returns a {@link FileSystem} client using a specific context. This client does not need to be * closed directly, but can be closed by calling {@link #close()} on this object. * * @param context the FileSystemContext to use * @return a {@link FileSystem} client */ public FileSystem getClient(FileSystemContext context) throws IOException { final FileSystem fs = FileSystem.Factory.get(context); mClients.add(fs); return fs; }
@Override public void run() { try { LOG.debug("Checking master is serving requests"); FileSystem fs = FileSystem.Factory.get(mConf); fs.exists(new AlluxioURI("/")); LOG.debug("Master is serving requests"); } catch (Throwable e) { throw new RuntimeException(e); } } }
/** * Reads a file. * * @param count the count to determine the filename * @throws Exception if it fails to read */ private static void readFile(CyclicBarrier barrier, AtomicLong runTime, int count, AlluxioConfiguration alluxioConf) throws Exception { FileSystem fileSystem = FileSystem.Factory.get(alluxioConf); byte[] buffer = new byte[(int) Math.min(sFileSize, 4 * Constants.MB)]; barrier.await(); long startTime = System.nanoTime(); try (FileInStream inStream = fileSystem.openFile(filename(count))) { while (inStream.read(buffer) != -1) { } } runTime.addAndGet(System.nanoTime() - startTime); }
FileSystemContext.create(new InstancedConfiguration(ConfigurationUtils.defaults())); FileSystem fs = FileSystem.Factory.get(fsContext); if (fs.exists(testDir)) { fs.delete(testDir, DeletePOptions.newBuilder().setRecursive(true).setUnchecked(true).build());
/** * Running this class will mount the file system according to * the options passed to this function {@link #parseOptions(String[], AlluxioConfiguration)}. * The user-space fuse application will stay on the foreground and keep * the file system mounted. The user can unmount the file system by * gracefully killing (SIGINT) the process. * * @param args arguments to run the command line */ public static void main(String[] args) { AlluxioConfiguration conf = new InstancedConfiguration(ConfigurationUtils.defaults()); final AlluxioFuseOptions opts = parseOptions(args, conf); if (opts == null) { System.exit(1); } final FileSystem tfs = FileSystem.Factory.get(conf); final AlluxioFuseFileSystem fs = new AlluxioFuseFileSystem(tfs, opts, conf); final List<String> fuseOpts = opts.getFuseOpts(); // Force direct_io in FUSE: writes and reads bypass the kernel page // cache and go directly to alluxio. This avoids extra memory copies // in the write path. fuseOpts.add("-odirect_io"); try { fs.mount(Paths.get(opts.getMountPoint()), true, opts.isDebug(), fuseOpts.toArray(new String[0])); } finally { fs.umount(); } }
/** * Writes a file. * * @param count the count to determine the filename * @throws Exception if it fails to write */ private static void writeFile(CyclicBarrier barrier, AtomicLong runtime, int count, AlluxioConfiguration alluxioConf) throws Exception { FileSystem fileSystem = FileSystem.Factory.get(alluxioConf); byte[] buffer = new byte[(int) Math.min(sFileSize, 4 * Constants.MB)]; Arrays.fill(buffer, (byte) 'a'); AlluxioURI path = filename(count); if (fileSystem.exists(path)) { fileSystem.delete(path); } barrier.await(); long startTime = System.nanoTime(); long bytesWritten = 0; try (FileOutStream outStream = fileSystem.createFile(path)) { while (bytesWritten < sFileSize) { outStream.write(buffer, 0, (int) Math.min(buffer.length, sFileSize - bytesWritten)); bytesWritten += buffer.length; } } runtime.addAndGet(System.nanoTime() - startTime); }
private static void basicAlluxioTests() throws Exception { LOG.info("Running tests"); FileSystem fs = FileSystem.Factory.get(sConf); int listSize = fs.listStatus(new AlluxioURI("/")).size(); if (listSize != 1) { throw new RuntimeException("Expected 1 path to exist at the root, but found " + listSize); } FileOutStream outStream = fs.createFile(new AlluxioURI("/test")); outStream.write("abc".getBytes()); outStream.close(); FileInStream inStream = fs.openFile(new AlluxioURI("/test")); String result = IOUtils.toString(inStream); if (!result.equals("abc")) { throw new RuntimeException("Expected abc but got " + result); } LOG.info("Tests passed"); }
public static FileSystem get(AlluxioConfiguration alluxioConf) { return get(FileSystemContext.create(alluxioConf)); }
public static FileSystem get(ClientContext ctx) { return get(FileSystemContext.create(ctx)); } }
@Override public void init() throws ServletException { super.init(); getServletContext().setAttribute(ALLUXIO_PROXY_SERVLET_RESOURCE_KEY, proxyProcess); getServletContext() .setAttribute(FILE_SYSTEM_SERVLET_RESOURCE_KEY, FileSystem.Factory.get(ServerConfiguration.global())); getServletContext().setAttribute(STREAM_CACHE_SERVLET_RESOURCE_KEY, new StreamCache(ServerConfiguration.getMs(PropertyKey.PROXY_STREAM_CACHE_TIMEOUT_MS))); } };
@Override public void init() throws ServletException { super.init(); getServletContext().setAttribute(ALLUXIO_MASTER_SERVLET_RESOURCE_KEY, masterProcess); getServletContext().setAttribute(ALLUXIO_FILESYSTEM_CLIENT_RESOURCE_KEY, FileSystem.Factory.get(ServerConfiguration.global())); } };
/** * Returns a {@link FileSystem} client. This client does not need to be * closed directly, but can be closed by calling {@link #close()} on this object. * * @return a {@link FileSystem} client */ public FileSystem getClient() throws IOException { final FileSystem fs = FileSystem.Factory.get(ServerConfiguration.global()); mClients.add(fs); return fs; }
/** * Constructs a new {@link RunOperation} object. * * @param alluxioConf Alluxio configuration */ public RunOperation(AlluxioConfiguration alluxioConf) { mFileSystem = FileSystem.Factory.get(alluxioConf); }
/** * @return a client for interacting with the cluster */ public synchronized FileSystem getFileSystemClient() { Preconditions.checkState(mState == State.STARTED, "must be in the started state to create an fs client, but state was %s", mState); return Factory.get(getFilesystemContext()); }
/** * Creates a new instance of {@link FileDataManager}. * * @param blockWorker the block worker handle * @param persistenceRateLimiter a per worker rate limiter to throttle async persistence * @param ufsManager the ufs manager */ public FileDataManager(BlockWorker blockWorker, RateLimiter persistenceRateLimiter, UfsManager ufsManager) { this(blockWorker, persistenceRateLimiter, ufsManager, () -> FileSystem.Factory.get(ServerConfiguration.global()), BufferUtils::fastCopy); }
protected AbstractFileSystemCommand(@Nullable FileSystemContext fsContext) { if (fsContext == null) { fsContext = FileSystemContext.create(new InstancedConfiguration(ConfigurationUtils.defaults())); } mFsContext = fsContext; mFileSystem = FileSystem.Factory.get(fsContext); }
@Override public Boolean call() throws Exception { FileSystem fs = FileSystem.Factory.get(mFsContext); writeFile(fs); return readFile(fs); }
@Override public Boolean call() throws Exception { FileSystem fs = FileSystem.Factory.get(mFsContext); writeFile(fs); return readFile(fs); }
@Override public void init() throws ServletException { super.init(); getServletContext().setAttribute(ALLUXIO_WORKER_SERVLET_RESOURCE_KEY, workerProcess); getServletContext().setAttribute(ALLUXIO_FILESYSTEM_CLIENT_RESOURCE_KEY, FileSystem.Factory.get(ServerConfiguration.global())); } };
@Override public Boolean call() throws Exception { FileSystem alluxioClient = FileSystem.Factory.get(mFsContext); write(alluxioClient); return read(alluxioClient); }