@Override protected void runPlainPath(AlluxioURI inputPath, CommandLine cl) throws AlluxioException, IOException { mFileSystem.createFile(inputPath).close(); System.out.println(inputPath + " has been created"); }
/** * Creates a file at the given path. * * @param fs a filesystem client * @param path the file path * @param options create file options */ public static void createFile(FileSystem fs, AlluxioURI path, CreateFilePOptions options) throws Exception { try (FileOutStream out = fs.createFile(path, options)) { out.write("test".getBytes()); } }
private void writeFile(FileSystem fs) throws IOException, AlluxioException { for (int i = 0; i < mNumFiles; i++) { ByteBuffer buf = ByteBuffer.allocate(80); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mNumFiles; k++) { buf.putInt(k); } buf.flip(); AlluxioURI filePath = new AlluxioURI(mFileFolder + "/part-" + i); LOG.debug("Writing data to {}", filePath); OutputStream os = fs.createFile(filePath, CreateFilePOptions.newBuilder().setRecursive(true).build()); os.write(buf.array()); os.close(); } }
private void writeFile(FileSystem fileSystem) throws IOException, AlluxioException { ByteBuffer buf = ByteBuffer.allocate(NUMBERS * 4); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < NUMBERS; k++) { buf.putInt(k); } LOG.debug("Writing data..."); long startTimeMs = CommonUtils.getCurrentMs(); FileOutStream os = fileSystem.createFile(mFilePath, mWriteOptions); os.write(buf.array()); os.close(); LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "writeFile to file " + mFilePath)); }
/** * Copies a file in the Alluxio filesystem. * * @param srcPath the source {@link AlluxioURI} (has to be a file) * @param dstPath the destination path in the Alluxio filesystem */ private void copyFile(AlluxioURI srcPath, AlluxioURI dstPath) throws AlluxioException, IOException { try (Closer closer = Closer.create()) { OpenFilePOptions openFileOptions = OpenFilePOptions.getDefaultInstance(); FileInStream is = closer.register(mFileSystem.openFile(srcPath, openFileOptions)); FileOutStream os = closer.register(mFileSystem.createFile(dstPath)); try { IOUtils.copy(is, os); } catch (Exception e) { os.cancel(); throw e; } System.out.println(String.format(COPY_SUCCEED_MESSAGE, srcPath, dstPath)); } }
/** * Writes a file. * * @param count the count to determine the filename * @throws Exception if it fails to write */ private static void writeFile(CyclicBarrier barrier, AtomicLong runtime, int count, AlluxioConfiguration alluxioConf) throws Exception { FileSystem fileSystem = FileSystem.Factory.get(alluxioConf); byte[] buffer = new byte[(int) Math.min(sFileSize, 4 * Constants.MB)]; Arrays.fill(buffer, (byte) 'a'); AlluxioURI path = filename(count); if (fileSystem.exists(path)) { fileSystem.delete(path); } barrier.await(); long startTime = System.nanoTime(); long bytesWritten = 0; try (FileOutStream outStream = fileSystem.createFile(path)) { while (bytesWritten < sFileSize) { outStream.write(buffer, 0, (int) Math.min(buffer.length, sFileSize - bytesWritten)); bytesWritten += buffer.length; } } runtime.addAndGet(System.nanoTime() - startTime); }
private void applyOperation() throws IOException, AlluxioException { AlluxioURI uri = new AlluxioURI(String.format("%s/%s", mDir, UUID.randomUUID())); switch (mOperation) { case CreateEmptyFile: mFileSystem.createFile(uri).close(); break; case CreateAndDeleteEmptyFile: mFileSystem.createFile(uri).close(); mFileSystem.delete(uri); break; case CreateFile: try (FileOutStream file = mFileSystem.createFile(uri)) { file.write(mFiledata); } break; case ListStatus: mFileSystem.listStatus(new AlluxioURI(mDir)); break; default: throw new IllegalStateException("Unknown operation: " + mOperation); } } }
private static void basicAlluxioTests() throws Exception { LOG.info("Running tests"); FileSystem fs = FileSystem.Factory.get(sConf); int listSize = fs.listStatus(new AlluxioURI("/")).size(); if (listSize != 1) { throw new RuntimeException("Expected 1 path to exist at the root, but found " + listSize); } FileOutStream outStream = fs.createFile(new AlluxioURI("/test")); outStream.write("abc".getBytes()); outStream.close(); FileInStream inStream = fs.openFile(new AlluxioURI("/test")); String result = IOUtils.toString(inStream); if (!result.equals("abc")) { throw new RuntimeException("Expected abc but got " + result); } LOG.info("Tests passed"); }
/** * Creates a file at the given path. * * @param fs a filesystem client * @param path the file path */ public static void createFile(FileSystem fs, AlluxioURI path) throws Exception { try (FileOutStream out = fs.createFile(path, CreateFilePOptions.newBuilder() .setBlockSizeBytes(Constants.KB).setRecursive(true).build())) { out.write("test".getBytes()); } }
/** * Creates a simple file with {@code len} bytes. * * @param fs a {@link FileSystem} handler * @param fileURI URI of the file * @param options client options to create the file with * @param len file size */ public static void createByteFile(FileSystem fs, AlluxioURI fileURI, CreateFilePOptions options, int len) { try (FileOutStream os = fs.createFile(fileURI, options)) { byte[] arr = new byte[len]; for (int k = 0; k < len; k++) { arr[k] = (byte) k; } os.write(arr); } catch (IOException | AlluxioException e) { throw new RuntimeException(e); } }
/** * Writes a partition. */ public void writePartition() throws IOException, AlluxioException { if (sDebugMode) { mBuf.flip(); LOG.info(FormatUtils.byteBufferToString(mBuf)); } mBuf.flip(); for (int pId = mLeft; pId < mRight; pId++) { final long startTimeMs = System.currentTimeMillis(); FileOutStream os = mFileSystem.createFile(new AlluxioURI(sFileName + (pId + sBaseFileNumber)), CreateFilePOptions.newBuilder().setRecursive(true).build()); for (int k = 0; k < sBlocksPerFile; k++) { mBuf.putInt(0, k + mWorkerId); os.write(mBuf.array()); } os.close(); logPerIteration(startTimeMs, pId, "th WriteAlluxioFile @ Worker ", pId); } }
private FileOutStream createFile(FileSystem fileSystem, AlluxioURI filePath, boolean deleteIfExists) throws IOException, AlluxioException { CreateFilePOptions options = CreateFilePOptions.newBuilder().setWriteType(mWriteType.toProto()) .setRecursive(true).build(); if (!fileSystem.exists(filePath)) { // file doesn't exist yet, so create it return fileSystem.createFile(filePath, options); } else if (deleteIfExists) { // file exists, so delete it and recreate fileSystem.delete(filePath); return fileSystem.createFile(filePath, options); } // file exists and deleteIfExists is false throw new FileAlreadyExistsException("File exists and deleteIfExists is false"); }
FileOutStream os = mFileSystem.createFile(uri); synchronized (mOpenFiles) { mOpenFiles.add(new OpenFileEntry(mNextOpenFileId, path, null, os));
@Override public FSDataOutputStream append(Path path, int bufferSize, Progressable progress) throws IOException { LOG.debug("append({}, {}, {})", path, bufferSize, progress); if (mStatistics != null) { mStatistics.incrementWriteOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); try { if (mFileSystem.exists(uri)) { throw new IOException(ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(uri)); } return new FSDataOutputStream( mFileSystem.createFile(uri, CreateFilePOptions.newBuilder().setRecursive(true).build()), mStatistics); } catch (AlluxioException e) { throw new IOException(e); } }
@Override public void apply(FileSystem fs) throws Exception { try (FileOutStream out = fs.createFile(FILE, CreateFilePOptions.newBuilder() .setBlockSizeBytes(Constants.KB).setWriteType(WritePType.ASYNC_THROUGH).build())) { out.write("test".getBytes()); } // Nested file try (FileOutStream out = fs.createFile(NESTED_FILE, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setWriteType(WritePType.ASYNC_THROUGH).setRecursive(true).build())) { out.write("test".getBytes()); } CommonUtils.waitFor("files to be persisted", () -> { try { return fs.getStatus(FILE).isPersisted() && fs.getStatus(NESTED_FILE).isPersisted(); } catch (IOException | AlluxioException e) { throw new RuntimeException(e); } }); }
@Before public void before() throws Exception { AlluxioConfiguration conf = ConfigurationTestUtils.defaults(); mMockFileSystem = Mockito.mock(FileSystem.class); mMockFileSystemContext = PowerMockito.mock(FileSystemContext.class); when(mMockFileSystemContext.getClientContext()) .thenReturn(ClientContext.create(conf)); when(mMockFileSystemContext.getConf()) .thenReturn(conf); mMockInStream = new MockFileInStream(mMockFileSystemContext, TEST_SOURCE_CONTENTS, conf); when(mMockFileSystem.openFile(new AlluxioURI(TEST_SOURCE))).thenReturn(mMockInStream); mMockOutStream = new MockFileOutStream(mMockFileSystemContext); when(mMockFileSystem.createFile(eq(new AlluxioURI(TEST_DESTINATION)), any(CreateFilePOptions.class))).thenReturn(mMockOutStream); mMockUfsManager = Mockito.mock(UfsManager.class); }
/** * Tests that the worker writes with the specified write type. */ @Test @Ignore // TODO(ggezer) Fix. public void writeTypeTest() throws Exception { runTask(TEST_SOURCE, TEST_SOURCE, TEST_DESTINATION, WriteType.CACHE_THROUGH); verify(mMockFileSystem).createFile(eq(new AlluxioURI(TEST_DESTINATION)), Matchers .eq(CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH)).build()); runTask(TEST_SOURCE, TEST_SOURCE, TEST_DESTINATION, WriteType.MUST_CACHE); verify(mMockFileSystem).createFile(eq(new AlluxioURI(TEST_DESTINATION)), Matchers .eq(CreateFilePOptions.newBuilder().setWriteType(WritePType.MUST_CACHE)).build()); }
/** * Ensures that an exception is propagated correctly when creating a file system. */ @Test public void createException() throws Exception { doThrow(EXCEPTION).when(mFileSystemMasterClient) .createFile(any(AlluxioURI.class), any(CreateFilePOptions.class)); try { mFileSystem.createFile(new AlluxioURI("/"), CreateFilePOptions.getDefaultInstance()); fail(SHOULD_HAVE_PROPAGATED_MESSAGE); } catch (Exception e) { assertSame(EXCEPTION, e); } verifyFilesystemContextAcquiredAndReleased(); }
/** * @param command the move command to execute * @param writeType the write type to use for the moved file * @param fileSystem the Alluxio file system */ private static void move(MoveCommand command, WritePType writeType, FileSystem fileSystem) throws Exception { String source = command.getSource(); String destination = command.getDestination(); LOG.debug("Moving {} to {}", source, destination); CreateFilePOptions createOptions = CreateFilePOptions.newBuilder().setWriteType(writeType).build(); try (FileOutStream out = fileSystem.createFile(new AlluxioURI(destination), createOptions)) { try (FileInStream in = fileSystem.openFile(new AlluxioURI(source))) { IOUtils.copy(in, out); } catch (Throwable t) { try { out.cancel(); } catch (Throwable t2) { t.addSuppressed(t2); } throw t; } } fileSystem.delete(new AlluxioURI(source)); }
/** * Tests the creation of a file via the * {@link BaseFileSystem#createFile(AlluxioURI, CreateFilePOptions)} method. */ @Test public void createFile() throws Exception { doNothing().when(mFileSystemMasterClient) .createFile(any(AlluxioURI.class), any(CreateFilePOptions.class)); URIStatus status = new URIStatus(new FileInfo()); AlluxioURI file = new AlluxioURI("/file"); GetStatusPOptions getStatusOptions = GetStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).build(); when(mFileSystemMasterClient.getStatus(file, getStatusOptions)).thenReturn(status); FileOutStream out = mFileSystem.createFile(file, CreateFilePOptions.getDefaultInstance()); verify(mFileSystemMasterClient).createFile(file, CreateFilePOptions.getDefaultInstance()); assertEquals(out.mUri, file); verifyFilesystemContextAcquiredAndReleased(); }