private void writeFile(FileSystem fileSystem) throws IOException, AlluxioException { ByteBuffer buf = ByteBuffer.allocate(NUMBERS * 4); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < NUMBERS; k++) { buf.putInt(k); } LOG.debug("Writing data..."); long startTimeMs = CommonUtils.getCurrentMs(); FileOutStream os = fileSystem.createFile(mFilePath, mWriteOptions); os.write(buf.array()); os.close(); LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "writeFile to file " + mFilePath)); }
/** * Creates a {@link FileOutStream} for test. * * @param path the file path * @param options the set of options specific to this operation * @return a {@link FileOutStream} */ private FileOutStream createTestStream(AlluxioURI path, OutStreamOptions options) throws IOException { return new FileOutStream(path, options, mFileSystemContext); } }
private void writeInternal(int b) throws IOException { if (mShouldCacheCurrentBlock) { try { if (mCurrentBlockOutStream == null || mCurrentBlockOutStream.remaining() == 0) { getNextBlock(); } mCurrentBlockOutStream.write(b); } catch (IOException e) { handleCacheWriteException(e); } } if (mUnderStorageType.isSyncPersist()) { mUnderStorageOutputStream.write(b); Metrics.BYTES_WRITTEN_UFS.inc(); } mBytesWritten++; }
/** * Creates a file at the given path. * * @param fs a filesystem client * @param path the file path * @param options create file options */ public static void createFile(FileSystem fs, AlluxioURI path, CreateFilePOptions options) throws Exception { try (FileOutStream out = fs.createFile(path, options)) { out.write("test".getBytes()); } }
@Override public void cancel() throws IOException { mCanceled = true; close(); }
/** * Tests that the number of bytes written is correct when the stream is created with different * under storage types. */ @Test public void getBytesWrittenWithDifferentUnderStorageType() throws IOException { for (WriteType type : WriteType.values()) { OutStreamOptions options = OutStreamOptions.defaults(sConf).setBlockSizeBytes(BLOCK_LENGTH).setWriteType(type) .setUfsPath(FILE_NAME.getPath()); mTestStream = createTestStream(FILE_NAME, options); mTestStream.write(BufferUtils.getIncreasingByteArray((int) BLOCK_LENGTH)); mTestStream.flush(); Assert.assertEquals(BLOCK_LENGTH, mTestStream.getBytesWritten()); } }
if (!ufs.exists(tmpPath)) { updateUfsPath(); tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath); if (!ufs.exists(tmpPath)) { updateUfsPath(); tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath); handleCacheWriteException(e); scheduleAsyncPersist();
/** * Copies a file in the Alluxio filesystem. * * @param srcPath the source {@link AlluxioURI} (has to be a file) * @param dstPath the destination path in the Alluxio filesystem */ private void copyFile(AlluxioURI srcPath, AlluxioURI dstPath) throws AlluxioException, IOException { try (Closer closer = Closer.create()) { OpenFilePOptions openFileOptions = OpenFilePOptions.getDefaultInstance(); FileInStream is = closer.register(mFileSystem.openFile(srcPath, openFileOptions)); FileOutStream os = closer.register(mFileSystem.createFile(dstPath)); try { IOUtils.copy(is, os); } catch (Exception e) { os.cancel(); throw e; } System.out.println(String.format(COPY_SUCCEED_MESSAGE, srcPath, dstPath)); } }
/** * Flushes cached data on Alluxio. * * Called on explicit sync() operation or at close(). * * @param path The path on the FS of the file to close * @param fi FileInfo data struct kept by FUSE * @return 0 on success, a negative value on error */ @Override public int flush(String path, FuseFileInfo fi) { LOG.trace("flush({})", path); final long fd = fi.fh.get(); OpenFileEntry oe = mOpenFiles.getFirstByField(ID_INDEX, fd); if (oe == null) { LOG.error("Cannot find fd for {} in table", path); return -ErrorCodes.EBADFD(); } if (oe.getOut() != null) { try { oe.getOut().flush(); } catch (IOException e) { LOG.error("Failed to flush {}", path, e); return -ErrorCodes.EIO(); } } else { LOG.debug("Not flushing: {} was not open for writing", path); } return 0; }
private void getNextBlock() throws IOException { if (mCurrentBlockOutStream != null) { Preconditions.checkState(mCurrentBlockOutStream.remaining() <= 0, PreconditionMessage.ERR_BLOCK_REMAINING); mCurrentBlockOutStream.flush(); mPreviousBlockOutStreams.add(mCurrentBlockOutStream); } if (mAlluxioStorageType.isStore()) { mCurrentBlockOutStream = mBlockStore.getOutStream(getNextBlockId(), mBlockSize, mOptions); mShouldCacheCurrentBlock = true; } }
/** * Writes a file. * * @param count the count to determine the filename * @throws Exception if it fails to write */ private static void writeFile(CyclicBarrier barrier, AtomicLong runtime, int count, AlluxioConfiguration alluxioConf) throws Exception { FileSystem fileSystem = FileSystem.Factory.get(alluxioConf); byte[] buffer = new byte[(int) Math.min(sFileSize, 4 * Constants.MB)]; Arrays.fill(buffer, (byte) 'a'); AlluxioURI path = filename(count); if (fileSystem.exists(path)) { fileSystem.delete(path); } barrier.await(); long startTime = System.nanoTime(); long bytesWritten = 0; try (FileOutStream outStream = fileSystem.createFile(path)) { while (bytesWritten < sFileSize) { outStream.write(buffer, 0, (int) Math.min(buffer.length, sFileSize - bytesWritten)); bytesWritten += buffer.length; } } runtime.addAndGet(System.nanoTime() - startTime); }
@Override protected void runPlainPath(AlluxioURI inputPath, CommandLine cl) throws AlluxioException, IOException { mFileSystem.createFile(inputPath).close(); System.out.println(inputPath + " has been created"); }
while (tLen > 0) { if (mCurrentBlockOutStream == null || mCurrentBlockOutStream.remaining() == 0) { getNextBlock(); handleCacheWriteException(e);
/** * @param command the move command to execute * @param writeType the write type to use for the moved file * @param fileSystem the Alluxio file system */ private static void move(MoveCommand command, WritePType writeType, FileSystem fileSystem) throws Exception { String source = command.getSource(); String destination = command.getDestination(); LOG.debug("Moving {} to {}", source, destination); CreateFilePOptions createOptions = CreateFilePOptions.newBuilder().setWriteType(writeType).build(); try (FileOutStream out = fileSystem.createFile(new AlluxioURI(destination), createOptions)) { try (FileInStream in = fileSystem.openFile(new AlluxioURI(source))) { IOUtils.copy(in, out); } catch (Throwable t) { try { out.cancel(); } catch (Throwable t2) { t.addSuppressed(t2); } throw t; } } fileSystem.delete(new AlluxioURI(source)); }
/** * Tests that {@link FileOutStream#flush()} will flush the under store stream. */ @Test public void flush() throws IOException { Assert.assertFalse(mUnderStorageFlushed.get()); mTestStream.flush(); Assert.assertTrue(mUnderStorageFlushed.get()); }
private void getNextBlock() throws IOException { if (mCurrentBlockOutStream != null) { Preconditions.checkState(mCurrentBlockOutStream.remaining() <= 0, PreconditionMessage.ERR_BLOCK_REMAINING); mPreviousBlockOutStreams.add(mCurrentBlockOutStream); } if (mAlluxioStorageType.isStore()) { try { WorkerNetAddress address = mLocationPolicy .getWorkerForNextBlock(mContext.getAluxioBlockStore().getWorkerInfoList(), mBlockSize); mCurrentBlockOutStream = mContext.getAluxioBlockStore().getOutStream(getNextBlockId(), mBlockSize, address); mShouldCacheCurrentBlock = true; } catch (AlluxioException e) { throw new IOException(e); } } }
private static void basicAlluxioTests() throws Exception { LOG.info("Running tests"); FileSystem fs = FileSystem.Factory.get(sConf); int listSize = fs.listStatus(new AlluxioURI("/")).size(); if (listSize != 1) { throw new RuntimeException("Expected 1 path to exist at the root, but found " + listSize); } FileOutStream outStream = fs.createFile(new AlluxioURI("/test")); outStream.write("abc".getBytes()); outStream.close(); FileInStream inStream = fs.openFile(new AlluxioURI("/test")); String result = IOUtils.toString(inStream); if (!result.equals("abc")) { throw new RuntimeException("Expected abc but got " + result); } LOG.info("Tests passed"); }
/** * Creates a file at the given path. * * @param fs a filesystem client * @param path the file path */ public static void createFile(FileSystem fs, AlluxioURI path) throws Exception { try (FileOutStream out = fs.createFile(path, CreateFilePOptions.newBuilder() .setBlockSizeBytes(Constants.KB).setRecursive(true).build())) { out.write("test".getBytes()); } }
/** * Closes the underlying open streams. */ @Override public void close() throws IOException { if (mIn != null) { mIn.close(); } if (mOut != null) { mOut.close(); } mOffset = -1; } }