private List<FileBlockInfo> getFileBlocks(AlluxioURI path) throws IOException { try { return mFileSystem.getStatus(path).getFileBlockInfos(); } catch (AlluxioException e) { throw new IOException(e); } }
@Override protected void runPlainPath(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException { // TODO(calvin): Remove explicit state checking. boolean recursive = cl.hasOption("R"); if (!mFileSystem.exists(path)) { throw new FileDoesNotExistException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(path)); } if (!recursive && mFileSystem.getStatus(path).isFolder()) { throw new IOException( path.getPath() + " is a directory, to remove it, please use \"rm -R <path>\""); } boolean isAlluxioOnly = cl.hasOption(REMOVE_ALLUXIO_ONLY.getLongOpt()); DeletePOptions options = DeletePOptions.newBuilder().setRecursive(recursive).setAlluxioOnly(isAlluxioOnly) .setUnchecked(cl.hasOption(REMOVE_UNCHECKED_OPTION_CHAR)).build(); mFileSystem.delete(path, options); if (!isAlluxioOnly) { System.out.println(path + " has been removed"); } else { System.out.println(path + " has been removed from Alluxio space"); } }
private void applyOperation() throws IOException, AlluxioException { AlluxioURI uri = new AlluxioURI(String.format("%s/%s", mDir, UUID.randomUUID())); switch (mOperation) { case CreateEmptyFile: mFileSystem.createFile(uri).close(); break; case CreateAndDeleteEmptyFile: mFileSystem.createFile(uri).close(); mFileSystem.delete(uri); break; case CreateFile: try (FileOutStream file = mFileSystem.createFile(uri)) { file.write(mFiledata); } break; case ListStatus: mFileSystem.listStatus(new AlluxioURI(mDir)); break; default: throw new IllegalStateException("Unknown operation: " + mOperation); } } }
@Override public void apply(FileSystem fs) throws Exception { fs.createDirectory(DIR_SET); fs.setAcl(DIR_SET, SetAclAction.MODIFY, Arrays.asList(AclEntry.fromCliString(ACL_STRING))); fs.createDirectory(DIR_ADD_REMOVE); fs.setAcl(DIR_ADD_REMOVE, SetAclAction.MODIFY, Arrays.asList(AclEntry.fromCliString(ACL_STRING))); fs.setAcl(DIR_ADD_REMOVE, SetAclAction.REMOVE, Arrays.asList(AclEntry.fromCliString(ACL_STRING))); }
/** * Tells the mock to throw FileDoesNotExistException when the given path is queried. */ private void setPathToNotExist(String path) throws Exception { AlluxioURI uri = new AlluxioURI(path); when(mMockFileSystem.getStatus(uri)).thenThrow(new FileDoesNotExistException(uri)); when(mMockFileSystem.listStatus(uri)).thenThrow(new FileDoesNotExistException(uri)); } }
private boolean readFile(FileSystem fs) throws IOException, AlluxioException { boolean pass = true; for (int i = 0; i < mNumFiles; i++) { AlluxioURI filePath = new AlluxioURI(mFileFolder + "/part-" + i); LOG.debug("Reading data from {}", filePath); FileInStream is = fs.openFile(filePath); URIStatus status = fs.getStatus(filePath); ByteBuffer buf = ByteBuffer.allocate((int) status.getBlockSizeBytes()); is.read(buf.array()); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mNumFiles; k++) { pass = pass && (buf.getInt() == k); } is.close(); } return pass; }
AlluxioURI dstPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(dst)); try { mFileSystem.rename(srcPath, dstPath); } catch (FileDoesNotExistException e) { LOG.warn("rename failed: {}", e.getMessage()); URIStatus dstStatus; try { dstStatus = mFileSystem.getStatus(dstPath); } catch (IOException | AlluxioException e2) { LOG.warn("rename failed: {}", e.getMessage()); mFileSystem.rename(srcPath, dstPath); } catch (IOException | AlluxioException e2) { LOG.error("Failed to rename {} to {}", src, dst, e2);
private FileOutStream createFile(FileSystem fileSystem, AlluxioURI filePath, boolean deleteIfExists) throws IOException, AlluxioException { CreateFilePOptions options = CreateFilePOptions.newBuilder().setWriteType(mWriteType.toProto()) .setRecursive(true).build(); if (!fileSystem.exists(filePath)) { // file doesn't exist yet, so create it return fileSystem.createFile(filePath, options); } else if (deleteIfExists) { // file exists, so delete it and recreate fileSystem.delete(filePath); return fileSystem.createFile(filePath, options); } // file exists and deleteIfExists is false throw new FileAlreadyExistsException("File exists and deleteIfExists is false"); }
AlluxioURI testURI = new AlluxioURI(mWorkDir + mSuccessNum); if (ClientOpType.CREATE_FILE == mOpType) { sFileSystem.createFile(testURI, sCreateFileOptions).close(); } else if (ClientOpType.CREATE_DELETE_FILE == mOpType) { try { sFileSystem.createFile(testURI, sCreateFileOptions).close(); } catch (AlluxioException e) { sFileSystem.delete(testURI); } else if (ClientOpType.CREATE_RENAME_FILE == mOpType) { try { sFileSystem.createFile(testURI, sCreateFileOptions).close(); } catch (AlluxioException e) { sFileSystem.rename(testURI, new AlluxioURI(testURI + "-rename"));
private static void basicAlluxioTests() throws Exception { LOG.info("Running tests"); FileSystem fs = FileSystem.Factory.get(sConf); int listSize = fs.listStatus(new AlluxioURI("/")).size(); if (listSize != 1) { throw new RuntimeException("Expected 1 path to exist at the root, but found " + listSize); } FileOutStream outStream = fs.createFile(new AlluxioURI("/test")); outStream.write("abc".getBytes()); outStream.close(); FileInStream inStream = fs.openFile(new AlluxioURI("/test")); String result = IOUtils.toString(inStream); if (!result.equals("abc")) { throw new RuntimeException("Expected abc but got " + result); } LOG.info("Tests passed"); }
@Override public boolean setReplication(Path path, short replication) throws IOException { AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); try { if (!mFileSystem.exists(uri) || mFileSystem.getStatus(uri).isFolder()) { return false; } mFileSystem.setAttribute(uri, (SetAttributePOptions) SetAttributePOptions.newBuilder() .setReplicationMin(replication).build()); return true; } catch (AlluxioException e) { throw new IOException(e); } }
/** * Tests that the worker will delete the source directory if the directory contains nothing. */ @Test public void deleteEmptySourceDir() throws Exception { when(mMockFileSystem.listStatus(new AlluxioURI(TEST_DIR))) .thenReturn(Lists.<URIStatus>newArrayList()); runTask(TEST_DIR, TEST_SOURCE, TEST_DESTINATION, WriteType.THROUGH); verify(mMockFileSystem).delete(eq(new AlluxioURI(TEST_DIR)), any(DeletePOptions.class)); }
/** * @param command the move command to execute * @param writeType the write type to use for the moved file * @param fileSystem the Alluxio file system */ private static void move(MoveCommand command, WritePType writeType, FileSystem fileSystem) throws Exception { String source = command.getSource(); String destination = command.getDestination(); LOG.debug("Moving {} to {}", source, destination); CreateFilePOptions createOptions = CreateFilePOptions.newBuilder().setWriteType(writeType).build(); try (FileOutStream out = fileSystem.createFile(new AlluxioURI(destination), createOptions)) { try (FileInStream in = fileSystem.openFile(new AlluxioURI(source))) { IOUtils.copy(in, out); } catch (Throwable t) { try { out.cancel(); } catch (Throwable t2) { t.addSuppressed(t2); } throw t; } } fileSystem.delete(new AlluxioURI(source)); }
/** * Creates a directory in the Alluxio filesystem space. It will not throw any exception if the * destination directory already exists. * * @param dstPath the {@link AlluxioURI} of the destination directory which will be created */ private void createDstDir(AlluxioURI dstPath) throws AlluxioException, IOException { try { mFileSystem.createDirectory(dstPath); } catch (FileAlreadyExistsException e) { // it's fine if the directory already exists } URIStatus dstStatus = mFileSystem.getStatus(dstPath); if (!dstStatus.isFolder()) { throw new InvalidPathException(ExceptionMessage.DESTINATION_CANNOT_BE_FILE.getMessage()); } }
/** * Creates a new dir. * * @param path the path on the FS of the new dir * @param mode Dir creation flags (IGNORED) * @return 0 on success, a negative value on error */ @Override public int mkdir(String path, @mode_t long mode) { final AlluxioURI turi = mPathResolverCache.getUnchecked(path); LOG.trace("mkdir({}) [Alluxio: {}]", path, turi); try { mFileSystem.createDirectory(turi); } catch (Throwable t) { LOG.error("Failed to create directory {}", path, t); return AlluxioFuseUtils.getErrorCode(t); } return 0; }
@Override public void apply(FileSystem fs) throws Exception { try (FileOutStream out = fs.createFile(FILE, CreateFilePOptions.newBuilder() .setBlockSizeBytes(Constants.KB).setWriteType(WritePType.ASYNC_THROUGH).build())) { out.write("test".getBytes()); } // Nested file try (FileOutStream out = fs.createFile(NESTED_FILE, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setWriteType(WritePType.ASYNC_THROUGH).setRecursive(true).build())) { out.write("test".getBytes()); } CommonUtils.waitFor("files to be persisted", () -> { try { return fs.getStatus(FILE).isPersisted() && fs.getStatus(NESTED_FILE).isPersisted(); } catch (IOException | AlluxioException e) { throw new RuntimeException(e); } }); }
@Override protected void runPlainPath(AlluxioURI inputPath, CommandLine cl) throws AlluxioException, IOException { mFileSystem.createFile(inputPath).close(); System.out.println(inputPath + " has been created"); }
/** * Convenience internal method to remove files or non-empty directories. * * @param path The path to remove * @return 0 on success, a negative value on error */ private int rmInternal(String path) { final AlluxioURI turi = mPathResolverCache.getUnchecked(path); try { mFileSystem.delete(turi); } catch (Throwable t) { LOG.error("Failed to remove {}", path, t); return AlluxioFuseUtils.getErrorCode(t); } return 0; }
/** * Constructs a new stream for reading a file from HDFS. * * @param fs the file system * @param uri the Alluxio file URI * @param stats filesystem statistics */ public HdfsFileInputStream(FileSystem fs, AlluxioURI uri, Statistics stats) throws IOException { LOG.debug("HdfsFileInputStream({}, {})", uri, stats); mStatistics = stats; try { mInputStream = fs.openFile(uri, OpenFilePOptions.getDefaultInstance()); } catch (FileDoesNotExistException e) { // Transform the Alluxio exception to a Java exception to satisfy the HDFS API contract. throw new FileNotFoundException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(uri)); } catch (AlluxioException e) { throw new IOException(e); } }