/** * @param context the file system context * @param address the data server address * @param id the block or UFS ID * @param length the length of the block or file to write, set to Long.MAX_VALUE if unknown * @param type type of the write request * @param options the options of the output stream * @return an instance of {@link GrpcDataWriter} */ public static GrpcDataWriter create(FileSystemContext context, WorkerNetAddress address, long id, long length, RequestType type, OutStreamOptions options) throws IOException { AlluxioConfiguration conf = context.getConf(); long chunkSize = conf.getBytes(PropertyKey.USER_NETWORK_WRITER_CHUNK_SIZE_BYTES); BlockWorkerClient grpcClient = context.acquireBlockWorkerClient(address); try { return new GrpcDataWriter(context, address, id, length, chunkSize, type, options, grpcClient); } catch (Exception e) { context.releaseBlockWorkerClient(address, grpcClient); throw e; } }
@Before public void before() throws Exception { mClientContext = ClientContext.create(mConf); mContext = PowerMockito.mock(FileSystemContext.class); mAddress = Mockito.mock(WorkerNetAddress.class); mClient = mock(BlockWorkerClient.class); mRequestObserver = mock(ClientCallStreamObserver.class); PowerMockito.when(mContext.getClientContext()).thenReturn(mClientContext); PowerMockito.when(mContext.getConf()).thenReturn(mConf); PowerMockito.when(mContext.acquireBlockWorkerClient(mAddress)).thenReturn(mClient); PowerMockito.doNothing().when(mContext).releaseBlockWorkerClient(mAddress, mClient); PowerMockito.when(mClient.writeBlock(any(StreamObserver.class))).thenReturn(mRequestObserver); PowerMockito.when(mRequestObserver.isReady()).thenReturn(true); }
@Before public void before() throws Exception { mContext = PowerMockito.mock(FileSystemContext.class); mAddress = mock(WorkerNetAddress.class); mClient = mock(BlockWorkerClient.class); mRequestObserver = mock(ClientCallStreamObserver.class); PowerMockito.when(mContext.acquireBlockWorkerClient(mAddress)).thenReturn(mClient); PowerMockito.when(mContext.getClientContext()) .thenReturn(ClientContext.create(mConf)); PowerMockito.when(mContext.getConf()).thenReturn(mConf); PowerMockito.doNothing().when(mContext).releaseBlockWorkerClient(mAddress, mClient); PowerMockito.when(mClient.writeBlock(any(StreamObserver.class))).thenReturn(mRequestObserver); PowerMockito.when(mRequestObserver.isReady()).thenReturn(true); }
@Before public void before() throws Exception { mMasterClient = PowerMockito.mock(BlockMasterClient.class); mWorkerClient = PowerMockito.mock(BlockWorkerClient.class); mContext = PowerMockito.mock(FileSystemContext.class); when(mContext.acquireBlockMasterClientResource()) .thenReturn(new DummyCloseableResource<>(mMasterClient)); when(mContext.getClientContext()).thenReturn(ClientContext.create(sConf)); when(mContext.getConf()).thenReturn(sConf); mLocalAddr = new WorkerNetAddress().setHost(NetworkAddressUtils.getLocalHostName( (int) sConf.getMs(PropertyKey.NETWORK_HOST_RESOLUTION_TIMEOUT_MS))); mBlockStore = new AlluxioBlockStore(mContext, TieredIdentityFactory.fromString("node=" + WORKER_HOSTNAME_LOCAL, sConf)); when(mContext.acquireBlockWorkerClient(any(WorkerNetAddress.class))) .thenReturn(mWorkerClient); mStreamObserver = PowerMockito.mock(ClientCallStreamObserver.class); when(mWorkerClient.writeBlock(any(StreamObserver.class))) .thenReturn(mStreamObserver); when(mWorkerClient.openLocalBlock(any(StreamObserver.class))) .thenReturn(mStreamObserver); when(mStreamObserver.isReady()).thenReturn(true); }
BlockWorkerClient blockWorker = null; try { blockWorker = mFsContext.acquireBlockWorkerClient(localNetAddress); blockWorker.removeBlock(request); } catch (NotFoundException e) {
/** * Creates an instance of {@link GrpcDataReader}. * * @param context the file system context * @param address the data server address * @param readRequest the read request */ private GrpcDataReader(FileSystemContext context, WorkerNetAddress address, ReadRequest readRequest) throws IOException { mContext = context; mAddress = address; mPosToRead = readRequest.getOffset(); mReadRequest = readRequest; AlluxioConfiguration alluxioConf = context.getConf(); mReaderBufferSizeMessages = alluxioConf .getInt(PropertyKey.USER_NETWORK_READER_BUFFER_SIZE_MESSAGES); mDataTimeoutMs = alluxioConf.getMs(PropertyKey.USER_NETWORK_DATA_TIMEOUT_MS); mClient = mContext.acquireBlockWorkerClient(address); try { mStream = new GrpcBlockingStream<>(mClient::readBlock, mReaderBufferSizeMessages, address.toString()); mStream.send(mReadRequest, mDataTimeoutMs); } catch (Exception e) { mContext.releaseBlockWorkerClient(address, mClient); throw e; } }
@Before public void before() throws Exception { mContext = PowerMockito.mock(FileSystemContext.class); when(mContext.getClientContext()) .thenReturn(ClientContext.create(ConfigurationTestUtils.defaults())); when(mContext.getConf()).thenReturn(ConfigurationTestUtils.defaults()); mAddress = mock(WorkerNetAddress.class); ReadRequest readRequest = ReadRequest.newBuilder().setBlockId(BLOCK_ID).setChunkSize(CHUNK_SIZE).build(); mFactory = new GrpcDataReader.Factory(mContext, mAddress, readRequest); mClient = mock(BlockWorkerClient.class); mRequestObserver = mock(ClientCallStreamObserver.class); when(mContext.acquireBlockWorkerClient(mAddress)).thenReturn(mClient); PowerMockito.doNothing().when(mContext).releaseBlockWorkerClient(mAddress, mClient); when(mClient.readBlock(any(StreamObserver.class))).thenReturn(mRequestObserver); when(mRequestObserver.isReady()).thenReturn(true); }
when(mContext.acquireBlockWorkerClient(WORKER_NET_ADDRESS_LOCAL)) .thenThrow(new UnavailableException("failed to connect to " + WORKER_NET_ADDRESS_LOCAL.getHost()));
@Before public void before() throws Exception { BlockWorkerClient workerClient = PowerMockito.mock(BlockWorkerClient.class); ClientCallStreamObserver requestObserver = PowerMockito.mock(ClientCallStreamObserver.class); when(requestObserver.isReady()).thenReturn(true); when(workerClient.openLocalBlock(any(StreamObserver.class))) .thenAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { StreamObserver<OpenLocalBlockResponse> observer = invocation.getArgumentAt(0, StreamObserver.class); observer.onNext(OpenLocalBlockResponse.newBuilder().setPath("/tmp").build()); observer.onCompleted(); return requestObserver; } }); mMockContext = PowerMockito.mock(FileSystemContext.class); PowerMockito.when(mMockContext.acquireBlockWorkerClient(Matchers.any(WorkerNetAddress.class))) .thenReturn(workerClient); PowerMockito.when(mMockContext.getClientContext()).thenReturn(ClientContext.create(mConf)); PowerMockito.when(mMockContext.getConf()).thenReturn(mConf); PowerMockito.doNothing().when(mMockContext) .releaseBlockWorkerClient(Matchers.any(WorkerNetAddress.class), Matchers.any(BlockWorkerClient.class)); mInfo = new BlockInfo().setBlockId(1); mOptions = new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Collections .singletonList(1L))), mConf); }
long chunkSize = conf.getBytes(PropertyKey.USER_LOCAL_WRITER_CHUNK_SIZE_BYTES); final BlockWorkerClient blockWorker = context.acquireBlockWorkerClient(address); Closer closer = Closer.create(); try {
.setSourceHost(dataSource.getHost()).setSourcePort(dataSource.getDataPort()) .build(); BlockWorkerClient blockWorker = mContext.acquireBlockWorkerClient(worker); try { blockWorker.asyncCache(request);
.setBlockId(mBlockId).setPromote(isPromote).build(); mBlockWorker = context.acquireBlockWorkerClient(address); try { mStream = new GrpcBlockingStream<>(mBlockWorker::openLocalBlock, mReadBufferSize,