if (status.isCompleted()) { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build(); try (FileInStream is = fs.openFile(absolutePath, options)) { int len = (int) Math.min(5 * Constants.KB, status.getLength() - offset);
public Builder mergeFrom( com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException { alluxio.grpc.OpenFilePOptions parsedMessage = null; try { parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); } catch (com.google.protobuf.InvalidProtocolBufferException e) { parsedMessage = (alluxio.grpc.OpenFilePOptions) e.getUnfinishedMessage(); throw e.unwrapIOException(); } finally { if (parsedMessage != null) { mergeFrom(parsedMessage); } } return this; } private int bitField0_;
try (Closer closer = Closer.create()) { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build(); FileInStream in = closer.register(mFileSystem.openFile(uri, options)); AlluxioURI dstPath = new AlluxioURI(ufsPath);
OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE) .setFileReadLocationPolicy(LocalFirstPolicy.class.getCanonicalName()).build();
OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, readOptions, sConf), mContext);
OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); if (local) { if (!mFsContext.hasLocalWorker()) {
/** * Creates with given {@link OpenFilePOptions} instance. * @param status URI status * @param options OpenFile options * @param alluxioConf Alluxio configuration */ public InStreamOptions(URIStatus status, OpenFilePOptions options, AlluxioConfiguration alluxioConf) { // Create OpenOptions builder with default options. OpenFilePOptions.Builder openOptionsBuilder = OpenFilePOptions.newBuilder() .setReadType(alluxioConf.getEnum(PropertyKey.USER_FILE_READ_TYPE_DEFAULT, ReadType.class) .toProto()) .setFileReadLocationPolicy( alluxioConf.get(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY)) .setHashingNumberOfShards(alluxioConf .getInt(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY_DETERMINISTIC_HASH_SHARDS)) .setMaxUfsReadConcurrency(alluxioConf .getInt(PropertyKey.USER_UFS_BLOCK_READ_CONCURRENCY_MAX)); // Merge default options with given options. OpenFilePOptions openOptions = openOptionsBuilder.mergeFrom(options).build(); mStatus = status; mProtoOptions = openOptions; CreateOptions blockLocationPolicyCreateOptions = CreateOptions.defaults().setLocationPolicyClassName(openOptions.getFileReadLocationPolicy()) .setDeterministicHashPolicyNumShards(openOptions.getHashingNumberOfShards()); mUfsReadLocationPolicy = BlockLocationPolicy.Factory.create(blockLocationPolicyCreateOptions, alluxioConf); }
public Builder mergeFrom(alluxio.grpc.OpenFilePOptions other) { if (other == alluxio.grpc.OpenFilePOptions.getDefaultInstance()) return this; if (other.hasReadType()) { setReadType(other.getReadType()); } if (other.hasMaxUfsReadConcurrency()) { setMaxUfsReadConcurrency(other.getMaxUfsReadConcurrency()); } if (other.hasFileReadLocationPolicy()) { bitField0_ |= 0x00000004; fileReadLocationPolicy_ = other.fileReadLocationPolicy_; onChanged(); } if (other.hasHashingNumberOfShards()) { setHashingNumberOfShards(other.getHashingNumberOfShards()); } if (other.hasCommonOptions()) { mergeCommonOptions(other.getCommonOptions()); } this.mergeUnknownFields(other.unknownFields); onChanged(); return this; }
/** * Tests seeking with incomplete block caching enabled. It seeks forward within a block. */ @Test public void shortSeekForwardCachingPartiallyReadBlocks() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int seekAmount = (int) (BLOCK_LENGTH / 4); int readAmount = (int) (BLOCK_LENGTH * 2 - BLOCK_LENGTH / 2); byte[] buffer = new byte[readAmount]; mTestStream.read(buffer); // Seek backward. mTestStream.seek(readAmount + seekAmount); // Block 1 (till seek pos) is being cached. validatePartialCaching(1, (int) BLOCK_LENGTH / 2); // Seek forward many times. The prefix is always cached. for (int i = 0; i < seekAmount; i++) { mTestStream.seek(readAmount + seekAmount + i); validatePartialCaching(1, (int) BLOCK_LENGTH / 2); } }
/** * Tests seeking with incomplete block caching enabled. It seeks forward for more than a block. */ @Test public void longSeekForwardCachingPartiallyReadBlocks() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int seekAmount = (int) (BLOCK_LENGTH / 4 + BLOCK_LENGTH); int readAmount = (int) (BLOCK_LENGTH / 2); byte[] buffer = new byte[readAmount]; mTestStream.read(buffer); // Seek backward. mTestStream.seek(readAmount + seekAmount); // Block 0 is cached though it is not fully read. validatePartialCaching(0, readAmount); // Block 1 is being cached though its prefix it not read. validatePartialCaching(1, 0); mTestStream.close(); validatePartialCaching(1, 0); }
@Test public void getInStreamUfs() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1"); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2"); BlockInfo info = new BlockInfo().setBlockId(0); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); OpenFilePOptions readOptions = OpenFilePOptions.newBuilder() .setFileReadLocationPolicy(MockFileWriteLocationPolicy.class.getTypeName()).build(); InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf); ((MockFileWriteLocationPolicy) options.getUfsReadLocationPolicy()) .setHosts(Arrays.asList(worker1, worker2)); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn( Arrays.asList(new WorkerInfo().setAddress(worker1), new WorkerInfo().setAddress(worker2))); // Location policy chooses worker1 first. assertEquals(worker1, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); // Location policy chooses worker2 second. assertEquals(worker2, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); }
/** * Tests seeking with incomplete block caching enabled. It seeks backward within 1 block. */ @Test public void shortSeekBackwardCachingPartiallyReadBlocks() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int seekAmount = (int) (BLOCK_LENGTH / 4); int readAmount = (int) (BLOCK_LENGTH * 2 - BLOCK_LENGTH / 2); byte[] buffer = new byte[readAmount]; mTestStream.read(buffer); // Seek backward. mTestStream.seek(readAmount - seekAmount); // Block 1 is cached though it is not fully read. validatePartialCaching(1, (int) BLOCK_LENGTH / 2); // Seek many times. It will cache block 1 only once. for (int i = 0; i <= seekAmount; i++) { mTestStream.seek(readAmount - seekAmount - i); } validatePartialCaching(1, (int) BLOCK_LENGTH / 2); }
/** * Tests reading and seeking with no local worker. Nothing should be cached. */ @Test public void testSeekWithNoLocalWorker() throws IOException { // Overrides the get local worker call PowerMockito.when(mContext.getLocalWorker()).thenReturn(null); OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int readAmount = (int) (BLOCK_LENGTH / 2); byte[] buffer = new byte[readAmount]; // read and seek several times mTestStream.read(buffer); assertEquals(readAmount, mInStreams.get(0).getBytesRead()); mTestStream.seek(BLOCK_LENGTH + BLOCK_LENGTH / 2); mTestStream.seek(0); // only reads the read amount, regardless of block source assertEquals(readAmount, mInStreams.get(0).getBytesRead()); assertEquals(0, mInStreams.get(1).getBytesRead()); }
/** * Tests seeking with incomplete block caching enabled. It seeks forward for more than a block * and then seek to the file beginning. */ @Test public void seekBackwardToFileBeginning() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int seekAmount = (int) (BLOCK_LENGTH / 4 + BLOCK_LENGTH); // Seek forward. mTestStream.seek(seekAmount); // Block 1 is partially cached though it is not fully read. validatePartialCaching(1, 0); // Seek backward. mTestStream.seek(0); // Block 1 is fully cached though it is not fully read. validatePartialCaching(1, 0); mTestStream.close(); // block 0 is cached validatePartialCaching(0, 0); }
private boolean read(FileSystem alluxioClient) throws IOException, AlluxioException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(mReadType.toProto()).build(); boolean pass = true; long startTimeMs = CommonUtils.getCurrentMs(); try (DataInputStream input = new DataInputStream(alluxioClient.openFile(mFilePath, options))) { int length = input.readInt(); for (int i = 0; i < length; i++) { if (input.readInt() != i) { pass = false; break; } } } LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "readFile file " + mFilePath)); return pass; } }
/** * Tests seeking with incomplete block caching enabled. It seeks backward for more than a block. */ @Test public void longSeekBackwardCachingPartiallyReadBlocks() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int seekAmount = (int) (BLOCK_LENGTH / 4 + BLOCK_LENGTH); int readAmount = (int) (BLOCK_LENGTH * 3 - BLOCK_LENGTH / 2); byte[] buffer = new byte[readAmount]; mTestStream.read(buffer); // Seek backward. mTestStream.seek(readAmount - seekAmount); // Block 2 is cached though it is not fully read. validatePartialCaching(2, (int) BLOCK_LENGTH / 2); }
/** * @param filePath the path for the files * @param readType the {@link ReadPType} * @param writeType the {@link WritePType} * @param fsContext the {@link FileSystemContext } to use for client operations */ public BasicOperations(AlluxioURI filePath, ReadType readType, WriteType writeType, FileSystemContext fsContext) { mFilePath = filePath; mReadOptions = OpenFilePOptions.newBuilder().setReadType(readType.toProto()).build(); mWriteOptions = CreateFilePOptions.newBuilder().setWriteType(writeType.toProto()) .setRecursive(true).build(); mFsContext = fsContext; }
/** * Loads a file into Alluxio by reading it. * * @param fs a {@link FileSystem} * @param fileName the name of the file to load */ public static void loadFile(FileSystem fs, String fileName) { try (FileInStream is = fs.openFile(new AlluxioURI(fileName), OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE).build())) { IOUtils.copy(is, ByteStreams.nullOutputStream()); } catch (IOException | AlluxioException e) { throw new RuntimeException(e); } }
/** * Calculates the md5 checksum for a file. * * @param filePath The {@link AlluxioURI} path of the file calculate MD5 checksum on * @return A string representation of the file's MD5 checksum */ private String calculateChecksum(AlluxioURI filePath) throws AlluxioException, IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build(); try (FileInStream fis = mFileSystem.openFile(filePath, options)) { return DigestUtils.md5Hex(fis); } }
/** * Tests skipping backwards when the seek buffer size is smaller than block size. */ @Test public void seekBackwardSmallSeekBuffer() throws IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); mTestStream = new FileInStream(mStatus, new InStreamOptions(mStatus, options, sConf), mContext); int readAmount = (int) (BLOCK_LENGTH / 2); byte[] buffer = new byte[readAmount]; mTestStream.read(buffer); mTestStream.seek(readAmount - 1); validatePartialCaching(0, readAmount); }