public NoOpRpcHandler() { streamManager = new OneForOneStreamManager(); }
@Override public ManagedBuffer openStream(String streamChunkId) { Pair<Long, Integer> streamChunkIdPair = parseStreamChunkId(streamChunkId); return getChunk(streamChunkIdPair.getLeft(), streamChunkIdPair.getRight()); }
@Override public void streamBeingSent(String streamId) { chunkBeingSent(parseStreamChunkId(streamId).getLeft()); }
@Override public void streamSent(String streamId) { chunkSent(OneForOneStreamManager.parseStreamChunkId(streamId).getLeft()); }
@Test public void managedBuffersAreFeedWhenConnectionIsClosed() throws Exception { OneForOneStreamManager manager = new OneForOneStreamManager(); List<ManagedBuffer> buffers = new ArrayList<>(); TestManagedBuffer buffer1 = Mockito.spy(new TestManagedBuffer(10)); TestManagedBuffer buffer2 = Mockito.spy(new TestManagedBuffer(20)); buffers.add(buffer1); buffers.add(buffer2); long streamId = manager.registerStream("appId", buffers.iterator()); Channel dummyChannel = Mockito.mock(Channel.class, Mockito.RETURNS_SMART_NULLS); manager.registerChannel(dummyChannel, streamId); manager.connectionTerminated(dummyChannel); Mockito.verify(buffer1, Mockito.times(1)).release(); Mockito.verify(buffer2, Mockito.times(1)).release(); } }
OpenBlocks msg = (OpenBlocks) msgObj; checkAuth(client, msg.appId); long streamId = streamManager.registerStream(client.getClientId(), new ManagedBufferIterator(msg.appId, msg.execId, msg.blockIds)); if (logger.isTraceEnabled()) {
managedBuffers.add(new TestManagedBuffer(30)); managedBuffers.add(new TestManagedBuffer(40)); long streamId = streamManager.registerStream("test-app", managedBuffers.iterator()); streamManager.registerChannel(channel, streamId); TransportClient reverseClient = mock(TransportClient.class); TransportRequestHandler requestHandler = new TransportRequestHandler(channel, reverseClient,
@Override public void onSuccess(ByteBuffer response) { try { streamHandle = (StreamHandle) BlockTransferMessage.Decoder.fromByteBuffer(response); logger.trace("Successfully opened blocks {}, preparing to fetch chunks.", streamHandle); // Immediately request all chunks -- we expect that the total size of the request is // reasonable due to higher level chunking in [[ShuffleBlockFetcherIterator]]. for (int i = 0; i < streamHandle.numChunks; i++) { if (downloadFileManager != null) { client.stream(OneForOneStreamManager.genStreamChunkId(streamHandle.streamId, i), new DownloadCallback(i)); } else { client.fetchChunk(streamHandle.streamId, i, chunkCallback); } } } catch (Exception e) { logger.error("Failed while starting block fetches after success", e); failRemainingBlocks(blockIds, e); } }
@Override public ManagedBuffer openStream(String streamChunkId) { String[] array = streamChunkId.split("_"); assert array.length == 2: "Stream id and chunk index should be specified when open stream for fetching block."; long streamId = Long.valueOf(array[0]); int chunkIndex = Integer.valueOf(array[1]); return getChunk(streamId, chunkIndex); }
@Test public void managedBuffersAreFeedWhenConnectionIsClosed() throws Exception { OneForOneStreamManager manager = new OneForOneStreamManager(); List<ManagedBuffer> buffers = new ArrayList<>(); TestManagedBuffer buffer1 = Mockito.spy(new TestManagedBuffer(10)); TestManagedBuffer buffer2 = Mockito.spy(new TestManagedBuffer(20)); buffers.add(buffer1); buffers.add(buffer2); long streamId = manager.registerStream("appId", buffers.iterator()); Channel dummyChannel = Mockito.mock(Channel.class, Mockito.RETURNS_SMART_NULLS); manager.registerChannel(dummyChannel, streamId); manager.connectionTerminated(dummyChannel); Mockito.verify(buffer1, Mockito.times(1)).release(); Mockito.verify(buffer2, Mockito.times(1)).release(); } }
OpenBlocks msg = (OpenBlocks) msgObj; checkAuth(client, msg.appId); long streamId = streamManager.registerStream(client.getClientId(), new ManagedBufferIterator(msg.appId, msg.execId, msg.blockIds)); if (logger.isTraceEnabled()) {
@Override public void streamSent(String streamId) { chunkSent(OneForOneStreamManager.parseStreamChunkId(streamId).getLeft()); }
@Override public void onSuccess(ByteBuffer response) { try { streamHandle = (StreamHandle) BlockTransferMessage.Decoder.fromByteBuffer(response); logger.trace("Successfully opened blocks {}, preparing to fetch chunks.", streamHandle); // Immediately request all chunks -- we expect that the total size of the request is // reasonable due to higher level chunking in [[ShuffleBlockFetcherIterator]]. for (int i = 0; i < streamHandle.numChunks; i++) { if (downloadFileManager != null) { client.stream(OneForOneStreamManager.genStreamChunkId(streamHandle.streamId, i), new DownloadCallback(i)); } else { client.fetchChunk(streamHandle.streamId, i, chunkCallback); } } } catch (Exception e) { logger.error("Failed while starting block fetches after success", e); failRemainingBlocks(blockIds, e); } }
@Override public StreamManager getStreamManager() { return new OneForOneStreamManager(); }
long streamId = streamManager.registerStream(client.getClientId(), iter); if (logger.isTraceEnabled()) { logger.trace("Registered streamId {} with {} buffers for client {} from host {}",
@Override public ManagedBuffer openStream(String streamChunkId) { Pair<Long, Integer> streamChunkIdPair = parseStreamChunkId(streamChunkId); return getChunk(streamChunkIdPair.getLeft(), streamChunkIdPair.getRight()); }
@Override public void streamBeingSent(String streamId) { chunkBeingSent(parseStreamChunkId(streamId).getLeft()); }
@Override public void onSuccess(ByteBuffer response) { try { streamHandle = (StreamHandle) BlockTransferMessage.Decoder.fromByteBuffer(response); logger.trace("Successfully opened blocks {}, preparing to fetch chunks.", streamHandle); // Immediately request all chunks -- we expect that the total size of the request is // reasonable due to higher level chunking in [[ShuffleBlockFetcherIterator]]. for (int i = 0; i < streamHandle.numChunks; i++) { if (tempShuffleFileManager != null) { client.stream(OneForOneStreamManager.genStreamChunkId(streamHandle.streamId, i), new DownloadCallback(i)); } else { client.fetchChunk(streamHandle.streamId, i, chunkCallback); } } } catch (Exception e) { logger.error("Failed while starting block fetches after success", e); failRemainingBlocks(blockIds, e); } }
public NoOpRpcHandler() { streamManager = new OneForOneStreamManager(); }
ArgumentCaptor<Iterator<ManagedBuffer>> stream = (ArgumentCaptor<Iterator<ManagedBuffer>>) (ArgumentCaptor<?>) ArgumentCaptor.forClass(Iterator.class); verify(streamManager, times(1)).registerStream(anyString(), stream.capture()); Iterator<ManagedBuffer> buffers = stream.getValue(); assertEquals(block0Marker, buffers.next());