/** * Creates a file at the given path. * * @param fs a filesystem client * @param path the file path * @param options create file options */ public static void createFile(FileSystem fs, AlluxioURI path, CreateFilePOptions options) throws Exception { try (FileOutStream out = fs.createFile(path, options)) { out.write("test".getBytes()); } }
private void writeFile(FileSystem fileSystem) throws IOException, AlluxioException { ByteBuffer buf = ByteBuffer.allocate(NUMBERS * 4); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < NUMBERS; k++) { buf.putInt(k); } LOG.debug("Writing data..."); long startTimeMs = CommonUtils.getCurrentMs(); FileOutStream os = fileSystem.createFile(mFilePath, mWriteOptions); os.write(buf.array()); os.close(); LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "writeFile to file " + mFilePath)); }
private static void basicAlluxioTests() throws Exception { LOG.info("Running tests"); FileSystem fs = FileSystem.Factory.get(sConf); int listSize = fs.listStatus(new AlluxioURI("/")).size(); if (listSize != 1) { throw new RuntimeException("Expected 1 path to exist at the root, but found " + listSize); } FileOutStream outStream = fs.createFile(new AlluxioURI("/test")); outStream.write("abc".getBytes()); outStream.close(); FileInStream inStream = fs.openFile(new AlluxioURI("/test")); String result = IOUtils.toString(inStream); if (!result.equals("abc")) { throw new RuntimeException("Expected abc but got " + result); } LOG.info("Tests passed"); }
/** * Creates a file at the given path. * * @param fs a filesystem client * @param path the file path */ public static void createFile(FileSystem fs, AlluxioURI path) throws Exception { try (FileOutStream out = fs.createFile(path, CreateFilePOptions.newBuilder() .setBlockSizeBytes(Constants.KB).setRecursive(true).build())) { out.write("test".getBytes()); } }
/** * Creates a simple file with {@code len} bytes. * * @param fs a {@link FileSystem} handler * @param fileURI URI of the file * @param options client options to create the file with * @param len file size */ public static void createByteFile(FileSystem fs, AlluxioURI fileURI, CreateFilePOptions options, int len) { try (FileOutStream os = fs.createFile(fileURI, options)) { byte[] arr = new byte[len]; for (int k = 0; k < len; k++) { arr[k] = (byte) k; } os.write(arr); } catch (IOException | AlluxioException e) { throw new RuntimeException(e); } }
/** * Writes a partition. */ public void writePartition() throws IOException, AlluxioException { if (sDebugMode) { mBuf.flip(); LOG.info(FormatUtils.byteBufferToString(mBuf)); } mBuf.flip(); for (int pId = mLeft; pId < mRight; pId++) { final long startTimeMs = System.currentTimeMillis(); FileOutStream os = mFileSystem.createFile(new AlluxioURI(sFileName + (pId + sBaseFileNumber)), CreateFilePOptions.newBuilder().setRecursive(true).build()); for (int k = 0; k < sBlocksPerFile; k++) { mBuf.putInt(0, k + mWorkerId); os.write(mBuf.array()); } os.close(); logPerIteration(startTimeMs, pId, "th WriteAlluxioFile @ Worker ", pId); } }
/** * Tests that the correct exception is thrown when a buffer is written with invalid offset/length. */ @Test public void writeBadBufferOffset() throws IOException { try { mTestStream.write(new byte[10], 5, 6); Assert.fail("buffer write with invalid offset/length should fail"); } catch (IllegalArgumentException e) { Assert.assertEquals(String.format(PreconditionMessage.ERR_BUFFER_STATE.toString(), 10, 5, 6), e.getMessage()); } }
@Override public void apply(FileSystem fs) throws Exception { try (FileOutStream out = fs.createFile(FILE, CreateFilePOptions.newBuilder() .setBlockSizeBytes(Constants.KB).setWriteType(WritePType.ASYNC_THROUGH).build())) { out.write("test".getBytes()); } // Nested file try (FileOutStream out = fs.createFile(NESTED_FILE, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setWriteType(WritePType.ASYNC_THROUGH).setRecursive(true).build())) { out.write("test".getBytes()); } CommonUtils.waitFor("files to be persisted", () -> { try { return fs.getStatus(FILE).isPersisted() && fs.getStatus(NESTED_FILE).isPersisted(); } catch (IOException | AlluxioException e) { throw new RuntimeException(e); } }); }
/** * Tests that if an exception is thrown by the underlying out stream, and the user is using * {@link UnderStorageType#SYNC_PERSIST} for their under storage type, the error is recovered * from by writing the data to the under storage out stream. */ @Test public void cacheWriteExceptionSyncPersist() throws IOException { BlockOutStream stream = mock(BlockOutStream.class); when(mBlockStore.getOutStream(anyLong(), anyLong(), any(OutStreamOptions.class))) .thenReturn(stream); when(stream.remaining()).thenReturn(BLOCK_LENGTH); doThrow(new IOException("test error")).when(stream).write((byte) 7); mTestStream.write(7); mTestStream.write(8); Assert.assertArrayEquals(new byte[] {7, 8}, mUnderStorageOutputStream.getWrittenData()); // The cache stream is written to only once - the FileInStream gives up on it after it throws // the first exception. verify(stream, times(1)).write(anyByte()); }
/** * Tests that many bytes, written one at a time, are written to the out streams correctly. */ @Test public void manyBytesWrite() throws IOException { int bytesToWrite = (int) ((BLOCK_LENGTH * 5) + (BLOCK_LENGTH / 2)); for (int i = 0; i < bytesToWrite; i++) { mTestStream.write(i); } mTestStream.close(); verifyIncreasingBytesWritten(bytesToWrite); }
/** * Tests that writing a null buffer with offset/length information throws the correct exception. */ @Test public void writeNullBufferOffset() throws IOException { try { mTestStream.write(null, 0, 0); Assert.fail("writing null should fail"); } catch (IllegalArgumentException e) { Assert.assertEquals(PreconditionMessage.ERR_WRITE_BUFFER_NULL.toString(), e.getMessage()); } }
/** * Tests that a single byte is written to the out stream correctly. */ @Test public void singleByteWrite() throws Exception { mTestStream.write(5); mTestStream.close(); Assert.assertArrayEquals(new byte[] {5}, mAlluxioOutStreamMap.get(0L).getWrittenData()); }
/** * Tests that writing a null buffer throws the correct exception. */ @Test public void writeNullBuffer() throws IOException { try { mTestStream.write(null); Assert.fail("writing null should fail"); } catch (IllegalArgumentException e) { Assert.assertEquals(PreconditionMessage.ERR_WRITE_BUFFER_NULL.toString(), e.getMessage()); } }
/** * Tests that {@link FileOutStream#close()} will close but not cancel the underlying out streams. * Also checks that {@link FileOutStream#close()} persists and completes the file. */ @Test public void close() throws Exception { mTestStream.write(BufferUtils.getIncreasingByteArray((int) (BLOCK_LENGTH * 1.5))); mTestStream.close(); for (long streamIndex = 0; streamIndex < 2; streamIndex++) { Assert.assertFalse(mAlluxioOutStreamMap.get(streamIndex).isCanceled()); Assert.assertTrue(mAlluxioOutStreamMap.get(streamIndex).isClosed()); } verify(mFileSystemMasterClient).completeFile(eq(FILE_NAME), any(CompleteFilePOptions.class)); }
/** * Tests that {@link FileOutStream#cancel()} will cancel and close the underlying out streams, and * delete from the under file system when the delegation flag is set. Also makes sure that * cancel() doesn't persist or complete the file. */ @Test public void cancelWithDelegation() throws Exception { mTestStream.write(BufferUtils.getIncreasingByteArray((int) (BLOCK_LENGTH * 1.5))); mTestStream.cancel(); for (long streamIndex = 0; streamIndex < 2; streamIndex++) { Assert.assertTrue(mAlluxioOutStreamMap.get(streamIndex).isClosed()); Assert.assertTrue(mAlluxioOutStreamMap.get(streamIndex).isCanceled()); } // Don't complete the file if the stream was canceled verify(mFileSystemMasterClient, times(0)).completeFile(any(AlluxioURI.class), any(CompleteFilePOptions.class)); }
/** * Tests that the async write invokes the expected client APIs. */ @Test public void asyncWrite() throws Exception { OutStreamOptions options = OutStreamOptions.defaults(sConf).setBlockSizeBytes(BLOCK_LENGTH) .setWriteType(WriteType.ASYNC_THROUGH); mTestStream = createTestStream(FILE_NAME, options); mTestStream.write(BufferUtils.getIncreasingByteArray((int) (BLOCK_LENGTH * 1.5))); mTestStream.close(); verify(mFileSystemMasterClient).completeFile(eq(FILE_NAME), any(CompleteFilePOptions.class)); verify(mFileSystemMasterClient).scheduleAsyncPersist(eq(FILE_NAME)); }
/** * Tests writing a buffer at an offset. */ @Test public void writeOffset() throws IOException { int bytesToWrite = (int) ((BLOCK_LENGTH * 5) + (BLOCK_LENGTH / 2)); int offset = (int) (BLOCK_LENGTH / 3); mTestStream.write(BufferUtils.getIncreasingByteArray(bytesToWrite + offset), offset, bytesToWrite); mTestStream.close(); verifyIncreasingBytesWritten(offset, bytesToWrite); }
/** * Tests that writing a buffer all at once will write bytes to the out streams correctly. */ @Test public void writeBuffer() throws IOException { int bytesToWrite = (int) ((BLOCK_LENGTH * 5) + (BLOCK_LENGTH / 2)); mTestStream.write(BufferUtils.getIncreasingByteArray(bytesToWrite)); mTestStream.close(); verifyIncreasingBytesWritten(bytesToWrite); }
/** * Tests that the number of bytes written is correct when the stream is created with different * under storage types. */ @Test public void getBytesWrittenWithDifferentUnderStorageType() throws IOException { for (WriteType type : WriteType.values()) { OutStreamOptions options = OutStreamOptions.defaults(sConf).setBlockSizeBytes(BLOCK_LENGTH).setWriteType(type) .setUfsPath(FILE_NAME.getPath()); mTestStream = createTestStream(FILE_NAME, options); mTestStream.write(BufferUtils.getIncreasingByteArray((int) BLOCK_LENGTH)); mTestStream.flush(); Assert.assertEquals(BLOCK_LENGTH, mTestStream.getBytesWritten()); } }