@Override public void cancel() throws IOException { mCanceled = true; close(); }
@Override protected void runPlainPath(AlluxioURI inputPath, CommandLine cl) throws AlluxioException, IOException { mFileSystem.createFile(inputPath).close(); System.out.println(inputPath + " has been created"); }
/** * Closes the underlying open streams. */ @Override public void close() throws IOException { if (mIn != null) { mIn.close(); } if (mOut != null) { mOut.close(); } mOffset = -1; } }
private void writeFile(FileSystem fileSystem) throws IOException, AlluxioException { ByteBuffer buf = ByteBuffer.allocate(NUMBERS * 4); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < NUMBERS; k++) { buf.putInt(k); } LOG.debug("Writing data..."); long startTimeMs = CommonUtils.getCurrentMs(); FileOutStream os = fileSystem.createFile(mFilePath, mWriteOptions); os.write(buf.array()); os.close(); LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "writeFile to file " + mFilePath)); }
private static void basicAlluxioTests() throws Exception { LOG.info("Running tests"); FileSystem fs = FileSystem.Factory.get(sConf); int listSize = fs.listStatus(new AlluxioURI("/")).size(); if (listSize != 1) { throw new RuntimeException("Expected 1 path to exist at the root, but found " + listSize); } FileOutStream outStream = fs.createFile(new AlluxioURI("/test")); outStream.write("abc".getBytes()); outStream.close(); FileInStream inStream = fs.openFile(new AlluxioURI("/test")); String result = IOUtils.toString(inStream); if (!result.equals("abc")) { throw new RuntimeException("Expected abc but got " + result); } LOG.info("Tests passed"); }
/** * Writes a partition. */ public void writePartition() throws IOException, AlluxioException { if (sDebugMode) { mBuf.flip(); LOG.info(FormatUtils.byteBufferToString(mBuf)); } mBuf.flip(); for (int pId = mLeft; pId < mRight; pId++) { final long startTimeMs = System.currentTimeMillis(); FileOutStream os = mFileSystem.createFile(new AlluxioURI(sFileName + (pId + sBaseFileNumber)), CreateFilePOptions.newBuilder().setRecursive(true).build()); for (int k = 0; k < sBlocksPerFile; k++) { mBuf.putInt(0, k + mWorkerId); os.write(mBuf.array()); } os.close(); logPerIteration(startTimeMs, pId, "th WriteAlluxioFile @ Worker ", pId); } }
private void applyOperation() throws IOException, AlluxioException { AlluxioURI uri = new AlluxioURI(String.format("%s/%s", mDir, UUID.randomUUID())); switch (mOperation) { case CreateEmptyFile: mFileSystem.createFile(uri).close(); break; case CreateAndDeleteEmptyFile: mFileSystem.createFile(uri).close(); mFileSystem.delete(uri); break; case CreateFile: try (FileOutStream file = mFileSystem.createFile(uri)) { file.write(mFiledata); } break; case ListStatus: mFileSystem.listStatus(new AlluxioURI(mDir)); break; default: throw new IllegalStateException("Unknown operation: " + mOperation); } } }
AlluxioURI testURI = new AlluxioURI(mWorkDir + mSuccessNum); if (ClientOpType.CREATE_FILE == mOpType) { sFileSystem.createFile(testURI, sCreateFileOptions).close(); } else if (ClientOpType.CREATE_DELETE_FILE == mOpType) { try { sFileSystem.createFile(testURI, sCreateFileOptions).close(); } catch (AlluxioException e) { } else if (ClientOpType.CREATE_RENAME_FILE == mOpType) { try { sFileSystem.createFile(testURI, sCreateFileOptions).close(); } catch (AlluxioException e) {
os.close(); System.out.println("done"); } catch (Exception e) {
/** * Tests that many bytes, written one at a time, are written to the out streams correctly. */ @Test public void manyBytesWrite() throws IOException { int bytesToWrite = (int) ((BLOCK_LENGTH * 5) + (BLOCK_LENGTH / 2)); for (int i = 0; i < bytesToWrite; i++) { mTestStream.write(i); } mTestStream.close(); verifyIncreasingBytesWritten(bytesToWrite); }
/** * Tests that a single byte is written to the out stream correctly. */ @Test public void singleByteWrite() throws Exception { mTestStream.write(5); mTestStream.close(); Assert.assertArrayEquals(new byte[] {5}, mAlluxioOutStreamMap.get(0L).getWrittenData()); }
/** * Tests that {@link FileOutStream#close()} will close but not cancel the underlying out streams. * Also checks that {@link FileOutStream#close()} persists and completes the file. */ @Test public void close() throws Exception { mTestStream.write(BufferUtils.getIncreasingByteArray((int) (BLOCK_LENGTH * 1.5))); mTestStream.close(); for (long streamIndex = 0; streamIndex < 2; streamIndex++) { Assert.assertFalse(mAlluxioOutStreamMap.get(streamIndex).isCanceled()); Assert.assertTrue(mAlluxioOutStreamMap.get(streamIndex).isClosed()); } verify(mFileSystemMasterClient).completeFile(eq(FILE_NAME), any(CompleteFilePOptions.class)); }
/** * Tests that the async write invokes the expected client APIs. */ @Test public void asyncWrite() throws Exception { OutStreamOptions options = OutStreamOptions.defaults(sConf).setBlockSizeBytes(BLOCK_LENGTH) .setWriteType(WriteType.ASYNC_THROUGH); mTestStream = createTestStream(FILE_NAME, options); mTestStream.write(BufferUtils.getIncreasingByteArray((int) (BLOCK_LENGTH * 1.5))); mTestStream.close(); verify(mFileSystemMasterClient).completeFile(eq(FILE_NAME), any(CompleteFilePOptions.class)); verify(mFileSystemMasterClient).scheduleAsyncPersist(eq(FILE_NAME)); }
/** * Tests that write only writes a byte. */ @Test public void truncateWrite() throws IOException { // Only writes the lowest byte mTestStream.write(0x1fffff00); mTestStream.write(0x1fffff01); mTestStream.close(); verifyIncreasingBytesWritten(2); }
/** * Tests writing a buffer at an offset. */ @Test public void writeOffset() throws IOException { int bytesToWrite = (int) ((BLOCK_LENGTH * 5) + (BLOCK_LENGTH / 2)); int offset = (int) (BLOCK_LENGTH / 3); mTestStream.write(BufferUtils.getIncreasingByteArray(bytesToWrite + offset), offset, bytesToWrite); mTestStream.close(); verifyIncreasingBytesWritten(offset, bytesToWrite); }
/** * Tests that writing a buffer all at once will write bytes to the out streams correctly. */ @Test public void writeBuffer() throws IOException { int bytesToWrite = (int) ((BLOCK_LENGTH * 5) + (BLOCK_LENGTH / 2)); mTestStream.write(BufferUtils.getIncreasingByteArray(bytesToWrite)); mTestStream.close(); verifyIncreasingBytesWritten(bytesToWrite); }
@Override public void cancel() throws IOException { mCanceled = true; close(); }
private void writeFile(FileSystem fileSystem) throws IOException, AlluxioException { ByteBuffer buf = ByteBuffer.allocate(NUMBERS * 4); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < NUMBERS; k++) { buf.putInt(k); } LOG.debug("Writing data..."); long startTimeMs = CommonUtils.getCurrentMs(); FileOutStream os = fileSystem.createFile(mFilePath, mWriteOptions); os.write(buf.array()); os.close(); LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "writeFile to file " + mFilePath)); }
private void applyOperation() throws IOException, AlluxioException { AlluxioURI uri = new AlluxioURI(String.format("%s/%s", BASE_DIRECTORY, UUID.randomUUID())); switch (mOperation) { case CreateEmptyFile: mFileSystem.createFile(uri).close(); break; case CreateAndDeleteEmptyFile: mFileSystem.createFile(uri).close(); mFileSystem.delete(uri); break; default: throw new IllegalStateException("Unknown operation: " + mOperation); } } }
/** * Creates the files for this example. */ public static void createFiles() throws AlluxioException, IOException { final long startTimeMs = CommonUtils.getCurrentMs(); for (int k = 0; k < sFiles; k++) { sFileSystem.createFile(new AlluxioURI(sFileName + (k + sBaseFileNumber))).close(); LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "createFile")); } }