private boolean read(FileSystem alluxioClient) throws IOException, AlluxioException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(mReadType.toProto()).build(); boolean pass = true; long startTimeMs = CommonUtils.getCurrentMs(); try (DataInputStream input = new DataInputStream(alluxioClient.openFile(mFilePath, options))) { int length = input.readInt(); for (int i = 0; i < length; i++) { if (input.readInt() != i) { pass = false; break; } } } LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "readFile file " + mFilePath)); return pass; } }
OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); if (local) { if (!mFsContext.hasLocalWorker()) {
/** * Calculates the md5 checksum for a file. * * @param filePath The {@link AlluxioURI} path of the file calculate MD5 checksum on * @return A string representation of the file's MD5 checksum */ private String calculateChecksum(AlluxioURI filePath) throws AlluxioException, IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build(); try (FileInStream fis = mFileSystem.openFile(filePath, options)) { return DigestUtils.md5Hex(fis); } }
if (status.isCompleted()) { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build(); try (FileInStream is = fs.openFile(absolutePath, options)) { int len = (int) Math.min(5 * Constants.KB, status.getLength() - offset);
try (Closer closer = Closer.create()) { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build(); FileInStream in = closer.register(mFileSystem.openFile(uri, options)); AlluxioURI dstPath = new AlluxioURI(ufsPath);
.setFileReadLocationPolicy(LocalFirstPolicy.class.getCanonicalName()).build();
OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, readOptions, sConf), mContext);
/** * Creates with given {@link OpenFilePOptions} instance. * @param status URI status * @param options OpenFile options * @param alluxioConf Alluxio configuration */ public InStreamOptions(URIStatus status, OpenFilePOptions options, AlluxioConfiguration alluxioConf) { // Create OpenOptions builder with default options. OpenFilePOptions.Builder openOptionsBuilder = OpenFilePOptions.newBuilder() .setReadType(alluxioConf.getEnum(PropertyKey.USER_FILE_READ_TYPE_DEFAULT, ReadType.class) .toProto()) .setFileReadLocationPolicy( alluxioConf.get(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY)) .setHashingNumberOfShards(alluxioConf .getInt(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY_DETERMINISTIC_HASH_SHARDS)) .setMaxUfsReadConcurrency(alluxioConf .getInt(PropertyKey.USER_UFS_BLOCK_READ_CONCURRENCY_MAX)); // Merge default options with given options. OpenFilePOptions openOptions = openOptionsBuilder.mergeFrom(options).build(); mStatus = status; mProtoOptions = openOptions; CreateOptions blockLocationPolicyCreateOptions = CreateOptions.defaults().setLocationPolicyClassName(openOptions.getFileReadLocationPolicy()) .setDeterministicHashPolicyNumShards(openOptions.getHashingNumberOfShards()); mUfsReadLocationPolicy = BlockLocationPolicy.Factory.create(blockLocationPolicyCreateOptions, alluxioConf); }
/** * Tests seeking with incomplete block caching enabled. It seeks forward for more than a block. */ @Test public void longSeekForwardCachingPartiallyReadBlocks() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int seekAmount = (int) (BLOCK_LENGTH / 4 + BLOCK_LENGTH); int readAmount = (int) (BLOCK_LENGTH / 2); byte[] buffer = new byte[readAmount]; mTestStream.read(buffer); // Seek backward. mTestStream.seek(readAmount + seekAmount); // Block 0 is cached though it is not fully read. validatePartialCaching(0, readAmount); // Block 1 is being cached though its prefix it not read. validatePartialCaching(1, 0); mTestStream.close(); validatePartialCaching(1, 0); }
/** * Tests seeking with incomplete block caching enabled. It seeks forward within a block. */ @Test public void shortSeekForwardCachingPartiallyReadBlocks() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int seekAmount = (int) (BLOCK_LENGTH / 4); int readAmount = (int) (BLOCK_LENGTH * 2 - BLOCK_LENGTH / 2); byte[] buffer = new byte[readAmount]; mTestStream.read(buffer); // Seek backward. mTestStream.seek(readAmount + seekAmount); // Block 1 (till seek pos) is being cached. validatePartialCaching(1, (int) BLOCK_LENGTH / 2); // Seek forward many times. The prefix is always cached. for (int i = 0; i < seekAmount; i++) { mTestStream.seek(readAmount + seekAmount + i); validatePartialCaching(1, (int) BLOCK_LENGTH / 2); } }
/** * Tests reading and seeking with no local worker. Nothing should be cached. */ @Test public void testSeekWithNoLocalWorker() throws IOException { // Overrides the get local worker call PowerMockito.when(mContext.getLocalWorker()).thenReturn(null); OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int readAmount = (int) (BLOCK_LENGTH / 2); byte[] buffer = new byte[readAmount]; // read and seek several times mTestStream.read(buffer); assertEquals(readAmount, mInStreams.get(0).getBytesRead()); mTestStream.seek(BLOCK_LENGTH + BLOCK_LENGTH / 2); mTestStream.seek(0); // only reads the read amount, regardless of block source assertEquals(readAmount, mInStreams.get(0).getBytesRead()); assertEquals(0, mInStreams.get(1).getBytesRead()); }
/** * Tests seeking with incomplete block caching enabled. It seeks forward for more than a block * and then seek to the file beginning. */ @Test public void seekBackwardToFileBeginning() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int seekAmount = (int) (BLOCK_LENGTH / 4 + BLOCK_LENGTH); // Seek forward. mTestStream.seek(seekAmount); // Block 1 is partially cached though it is not fully read. validatePartialCaching(1, 0); // Seek backward. mTestStream.seek(0); // Block 1 is fully cached though it is not fully read. validatePartialCaching(1, 0); mTestStream.close(); // block 0 is cached validatePartialCaching(0, 0); }
@Test public void getInStreamUfs() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1"); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2"); BlockInfo info = new BlockInfo().setBlockId(0); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); OpenFilePOptions readOptions = OpenFilePOptions.newBuilder() .setFileReadLocationPolicy(MockFileWriteLocationPolicy.class.getTypeName()).build(); InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf); ((MockFileWriteLocationPolicy) options.getUfsReadLocationPolicy()) .setHosts(Arrays.asList(worker1, worker2)); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn( Arrays.asList(new WorkerInfo().setAddress(worker1), new WorkerInfo().setAddress(worker2))); // Location policy chooses worker1 first. assertEquals(worker1, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); // Location policy chooses worker2 second. assertEquals(worker2, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); }
/** * Tests seeking with incomplete block caching enabled. It seeks backward within 1 block. */ @Test public void shortSeekBackwardCachingPartiallyReadBlocks() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int seekAmount = (int) (BLOCK_LENGTH / 4); int readAmount = (int) (BLOCK_LENGTH * 2 - BLOCK_LENGTH / 2); byte[] buffer = new byte[readAmount]; mTestStream.read(buffer); // Seek backward. mTestStream.seek(readAmount - seekAmount); // Block 1 is cached though it is not fully read. validatePartialCaching(1, (int) BLOCK_LENGTH / 2); // Seek many times. It will cache block 1 only once. for (int i = 0; i <= seekAmount; i++) { mTestStream.seek(readAmount - seekAmount - i); } validatePartialCaching(1, (int) BLOCK_LENGTH / 2); }
/** * Tests seeking with incomplete block caching enabled. It seeks backward for more than a block. */ @Test public void longSeekBackwardCachingPartiallyReadBlocks() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int seekAmount = (int) (BLOCK_LENGTH / 4 + BLOCK_LENGTH); int readAmount = (int) (BLOCK_LENGTH * 3 - BLOCK_LENGTH / 2); byte[] buffer = new byte[readAmount]; mTestStream.read(buffer); // Seek backward. mTestStream.seek(readAmount - seekAmount); // Block 2 is cached though it is not fully read. validatePartialCaching(2, (int) BLOCK_LENGTH / 2); }
/** * Loads a file into Alluxio by reading it. * * @param fs a {@link FileSystem} * @param fileName the name of the file to load */ public static void loadFile(FileSystem fs, String fileName) { try (FileInStream is = fs.openFile(new AlluxioURI(fileName), OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE).build())) { IOUtils.copy(is, ByteStreams.nullOutputStream()); } catch (IOException | AlluxioException e) { throw new RuntimeException(e); } }
/** * Tests skipping backwards when the seek buffer size is smaller than block size. */ @Test public void seekBackwardSmallSeekBuffer() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int readAmount = (int) (BLOCK_LENGTH / 2); byte[] buffer = new byte[readAmount]; mTestStream.read(buffer); mTestStream.seek(readAmount - 1); validatePartialCaching(0, readAmount); }
/** * @param filePath the path for the files * @param readType the {@link ReadPType} * @param writeType the {@link WritePType} * @param fsContext the {@link FileSystemContext } to use for client operations */ public BasicOperations(AlluxioURI filePath, ReadType readType, WriteType writeType, FileSystemContext fsContext) { mFilePath = filePath; mReadOptions = OpenFilePOptions.newBuilder().setReadType(readType.toProto()).build(); mWriteOptions = CreateFilePOptions.newBuilder().setWriteType(writeType.toProto()) .setRecursive(true).build(); mFsContext = fsContext; }
/** * Converts a {@link CreateFilePOptions} object to an {@link OpenFilePOptions} object with a * matching Alluxio storage type. * * @param op a {@link CreateFilePOptions} object * @return an {@link OpenFilePOptions} object with a matching Alluxio storage type */ public static OpenFilePOptions toOpenFileOptions(CreateFilePOptions op) { if (WriteType.fromProto(op.getWriteType()).getAlluxioStorageType().isStore()) { return OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE).build(); } return OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build(); }
@Test public void seekAndClose() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int seekAmount = (int) (BLOCK_LENGTH / 2); mTestStream.seek(seekAmount); mTestStream.close(); // Block 0 is cached though it is not fully read. validatePartialCaching(0, 0); }