public Builder newBuilderForType() { return newBuilder(); } public static Builder newBuilder() {
public Builder mergeFrom(alluxio.grpc.OpenFilePOptions other) { if (other == alluxio.grpc.OpenFilePOptions.getDefaultInstance()) return this; if (other.hasReadType()) { setReadType(other.getReadType()); } if (other.hasMaxUfsReadConcurrency()) { setMaxUfsReadConcurrency(other.getMaxUfsReadConcurrency()); } if (other.hasFileReadLocationPolicy()) { bitField0_ |= 0x00000004; fileReadLocationPolicy_ = other.fileReadLocationPolicy_; onChanged(); } if (other.hasHashingNumberOfShards()) { setHashingNumberOfShards(other.getHashingNumberOfShards()); } if (other.hasCommonOptions()) { mergeCommonOptions(other.getCommonOptions()); } this.mergeUnknownFields(other.unknownFields); onChanged(); return this; }
result = result && (hasReadType() == other.hasReadType()); if (hasReadType()) { result = result && readType_ == other.readType_; result = result && (hasMaxUfsReadConcurrency() == other.hasMaxUfsReadConcurrency()); if (hasMaxUfsReadConcurrency()) { result = result && (getMaxUfsReadConcurrency() == other.getMaxUfsReadConcurrency()); result = result && (hasFileReadLocationPolicy() == other.hasFileReadLocationPolicy()); if (hasFileReadLocationPolicy()) { result = result && getFileReadLocationPolicy() .equals(other.getFileReadLocationPolicy()); result = result && (hasHashingNumberOfShards() == other.hasHashingNumberOfShards()); if (hasHashingNumberOfShards()) { result = result && (getHashingNumberOfShards() == other.getHashingNumberOfShards()); result = result && (hasCommonOptions() == other.hasCommonOptions()); if (hasCommonOptions()) { result = result && getCommonOptions() .equals(other.getCommonOptions());
/** * <code>optional string fileReadLocationPolicy = 3;</code> */ public Builder clearFileReadLocationPolicy() { bitField0_ = (bitField0_ & ~0x00000004); fileReadLocationPolicy_ = getDefaultInstance().getFileReadLocationPolicy(); onChanged(); return this; } /**
/** * Creates with the default {@link OpenFilePOptions}. * * @param status the file to create the options for * @param alluxioConf Alluxio configuration */ public InStreamOptions(URIStatus status, AlluxioConfiguration alluxioConf) { this(status, OpenFilePOptions.getDefaultInstance(), alluxioConf); }
/** * Creates with given {@link OpenFilePOptions} instance. * @param status URI status * @param options OpenFile options * @param alluxioConf Alluxio configuration */ public InStreamOptions(URIStatus status, OpenFilePOptions options, AlluxioConfiguration alluxioConf) { // Create OpenOptions builder with default options. OpenFilePOptions.Builder openOptionsBuilder = OpenFilePOptions.newBuilder() .setReadType(alluxioConf.getEnum(PropertyKey.USER_FILE_READ_TYPE_DEFAULT, ReadType.class) .toProto()) .setFileReadLocationPolicy( alluxioConf.get(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY)) .setHashingNumberOfShards(alluxioConf .getInt(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY_DETERMINISTIC_HASH_SHARDS)) .setMaxUfsReadConcurrency(alluxioConf .getInt(PropertyKey.USER_UFS_BLOCK_READ_CONCURRENCY_MAX)); // Merge default options with given options. OpenFilePOptions openOptions = openOptionsBuilder.mergeFrom(options).build(); mStatus = status; mProtoOptions = openOptions; CreateOptions blockLocationPolicyCreateOptions = CreateOptions.defaults().setLocationPolicyClassName(openOptions.getFileReadLocationPolicy()) .setDeterministicHashPolicyNumShards(openOptions.getHashingNumberOfShards()); mUfsReadLocationPolicy = BlockLocationPolicy.Factory.create(blockLocationPolicyCreateOptions, alluxioConf); }
/** * @param blockId id of the block * @return a {@link Protocol.OpenUfsBlockOptions} based on the block id and options */ public Protocol.OpenUfsBlockOptions getOpenUfsBlockOptions(long blockId) { Preconditions.checkArgument(mStatus.getBlockIds().contains(blockId), "blockId"); boolean readFromUfs = mStatus.isPersisted(); // In case it is possible to fallback to read UFS blocks, also fill in the options. boolean storedAsUfsBlock = mStatus.getPersistenceState().equals("TO_BE_PERSISTED"); readFromUfs = readFromUfs || storedAsUfsBlock; if (!readFromUfs) { return Protocol.OpenUfsBlockOptions.getDefaultInstance(); } long blockStart = BlockId.getSequenceNumber(blockId) * mStatus.getBlockSizeBytes(); BlockInfo info = getBlockInfo(blockId); Protocol.OpenUfsBlockOptions openUfsBlockOptions = Protocol.OpenUfsBlockOptions.newBuilder() .setUfsPath(mStatus.getUfsPath()).setOffsetInFile(blockStart).setBlockSize(info.getLength()) .setMaxUfsReadConcurrency(mProtoOptions.getMaxUfsReadConcurrency()) .setNoCache(!ReadType.fromProto(mProtoOptions.getReadType()).isCache()) .setMountId(mStatus.getMountId()).build(); if (storedAsUfsBlock) { // On client-side, we do not have enough mount information to fill in the UFS file path. // Instead, we unset the ufsPath field and fill in a flag ufsBlock to indicate the UFS file // path can be derived from mount id and the block ID. Also because the entire file is only // one block, we set the offset in file to be zero. openUfsBlockOptions = openUfsBlockOptions.toBuilder().clearUfsPath().setBlockInUfsTier(true) .setOffsetInFile(0).build(); } return openUfsBlockOptions; }
throws IOException { URIStatus status = options.getStatus(); ReadType readType = ReadType.fromProto(options.getOptions().getReadType());
public alluxio.grpc.OpenFilePOptions getDefaultInstanceForType() { return alluxio.grpc.OpenFilePOptions.getDefaultInstance(); }
private void triggerAsyncCaching(BlockInStream stream) throws IOException { boolean cache = ReadType.fromProto(mOptions.getOptions().getReadType()).isCache(); boolean overReplicated = mStatus.getReplicationMax() > 0 && mStatus.getFileBlockInfos().get((int) (getPos() / mBlockSize))
/** * Calculates the md5 checksum for a file. * * @param filePath The {@link AlluxioURI} path of the file calculate MD5 checksum on * @return A string representation of the file's MD5 checksum */ private String calculateChecksum(AlluxioURI filePath) throws AlluxioException, IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build(); try (FileInStream fis = mFileSystem.openFile(filePath, options)) { return DigestUtils.md5Hex(fis); } }
@Override public FileInStream openFile(AlluxioURI path) throws FileDoesNotExistException, IOException, AlluxioException { return openFile(path, OpenFilePOptions.getDefaultInstance()); }
hash = (19 * hash) + getDescriptor().hashCode(); if (hasReadType()) { hash = (37 * hash) + READTYPE_FIELD_NUMBER; hash = (53 * hash) + readType_; if (hasMaxUfsReadConcurrency()) { hash = (37 * hash) + MAXUFSREADCONCURRENCY_FIELD_NUMBER; hash = (53 * hash) + getMaxUfsReadConcurrency(); if (hasFileReadLocationPolicy()) { hash = (37 * hash) + FILEREADLOCATIONPOLICY_FIELD_NUMBER; hash = (53 * hash) + getFileReadLocationPolicy().hashCode(); if (hasHashingNumberOfShards()) { hash = (37 * hash) + HASHINGNUMBEROFSHARDS_FIELD_NUMBER; hash = (53 * hash) + getHashingNumberOfShards(); if (hasCommonOptions()) { hash = (37 * hash) + COMMONOPTIONS_FIELD_NUMBER; hash = (53 * hash) + getCommonOptions().hashCode();
mDataTimeoutMs = conf.getMs(PropertyKey.USER_NETWORK_DATA_TIMEOUT_MS); boolean isPromote = ReadType.fromProto(options.getOptions().getReadType()).isPromote(); OpenLocalBlockRequest request = OpenLocalBlockRequest.newBuilder() .setBlockId(mBlockId).setPromote(isPromote).build();
private boolean read(FileSystem alluxioClient) throws IOException, AlluxioException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(mReadType.toProto()).build(); boolean pass = true; long startTimeMs = CommonUtils.getCurrentMs(); try (DataInputStream input = new DataInputStream(alluxioClient.openFile(mFilePath, options))) { int length = input.readInt(); for (int i = 0; i < length; i++) { if (input.readInt() != i) { pass = false; break; } } } LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "readFile file " + mFilePath)); return pass; } }
/** * Constructs a new stream for reading a file from HDFS. * * @param fs the file system * @param uri the Alluxio file URI * @param stats filesystem statistics */ public HdfsFileInputStream(FileSystem fs, AlluxioURI uri, Statistics stats) throws IOException { LOG.debug("HdfsFileInputStream({}, {})", uri, stats); mStatistics = stats; try { mInputStream = fs.openFile(uri, OpenFilePOptions.getDefaultInstance()); } catch (FileDoesNotExistException e) { // Transform the Alluxio exception to a Java exception to satisfy the HDFS API contract. throw new FileNotFoundException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(uri)); } catch (AlluxioException e) { throw new IOException(e); } }
OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build(); if (local) { if (!mFsContext.hasLocalWorker()) {
/** * Copies a file in the Alluxio filesystem. * * @param srcPath the source {@link AlluxioURI} (has to be a file) * @param dstPath the destination path in the Alluxio filesystem */ private void copyFile(AlluxioURI srcPath, AlluxioURI dstPath) throws AlluxioException, IOException { try (Closer closer = Closer.create()) { OpenFilePOptions openFileOptions = OpenFilePOptions.getDefaultInstance(); FileInStream is = closer.register(mFileSystem.openFile(srcPath, openFileOptions)); FileOutStream os = closer.register(mFileSystem.createFile(dstPath)); try { IOUtils.copy(is, os); } catch (Exception e) { os.cancel(); throw e; } System.out.println(String.format(COPY_SUCCEED_MESSAGE, srcPath, dstPath)); } }
/** * Loads a file into Alluxio by reading it. * * @param fs a {@link FileSystem} * @param fileName the name of the file to load */ public static void loadFile(FileSystem fs, String fileName) { try (FileInStream is = fs.openFile(new AlluxioURI(fileName), OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE).build())) { IOUtils.copy(is, ByteStreams.nullOutputStream()); } catch (IOException | AlluxioException e) { throw new RuntimeException(e); } }
OpenFilePOptions options = OpenFilePOptions.getDefaultInstance(); FileInStream is = closer.register(mFileSystem.openFile(srcPath, options)); FileOutputStream out = closer.register(new FileOutputStream(tmpDst));