@Before public void beforeEach() { streamManager = mock(OneForOneStreamManager.class); blockResolver = mock(ExternalShuffleBlockResolver.class); handler = new ExternalShuffleBlockHandler(streamManager, blockResolver); }
@After public void afterEach() { handler.applicationRemoved(APP_ID, false /* cleanupLocalDirs */); }
/** * Close the shuffle server to clean up any associated state. */ @Override protected void serviceStop() { try { if (shuffleServer != null) { shuffleServer.close(); } if (blockHandler != null) { blockHandler.close(); } if (db != null) { db.close(); } } catch (Exception e) { logger.error("Exception when stopping service", e); } }
@Test public void testRegisterExecutor() { RpcResponseCallback callback = mock(RpcResponseCallback.class); ExecutorShuffleInfo config = new ExecutorShuffleInfo(new String[] {"/a", "/b"}, 16, "sort"); ByteBuffer registerMessage = new RegisterExecutor("app0", "exec1", config).toByteBuffer(); handler.receive(client, registerMessage, callback); verify(blockResolver, times(1)).registerExecutor("app0", "exec1", config); verify(callback, times(1)).onSuccess(any(ByteBuffer.class)); verify(callback, never()).onFailure(any(Throwable.class)); // Verify register executor request latency metrics Timer registerExecutorRequestLatencyMillis = (Timer) ((ExternalShuffleBlockHandler) handler) .getAllMetrics() .getMetrics() .get("registerExecutorRequestLatencyMillis"); assertEquals(1, registerExecutorRequestLatencyMillis.getCount()); }
@Override public void receive(TransportClient client, ByteBuffer message, RpcResponseCallback callback) { BlockTransferMessage msgObj = BlockTransferMessage.Decoder.fromByteBuffer(message); handleMessage(msgObj, client, callback); }
try { OpenBlocks msg = (OpenBlocks) msgObj; checkAuth(client, msg.appId); long streamId = streamManager.registerStream(client.getClientId(), new ManagedBufferIterator(msg.appId, msg.execId, msg.blockIds)); try { RegisterExecutor msg = (RegisterExecutor) msgObj; checkAuth(client, msg.appId); blockManager.registerExecutor(msg.appId, msg.execId, msg.executorInfo); callback.onSuccess(ByteBuffer.wrap(new byte[0]));
@Test public void testRegisterExecutor() { RpcResponseCallback callback = mock(RpcResponseCallback.class); ExecutorShuffleInfo config = new ExecutorShuffleInfo(new String[] {"/a", "/b"}, 16, "sort"); ByteBuffer registerMessage = new RegisterExecutor("app0", "exec1", config).toByteBuffer(); handler.receive(client, registerMessage, callback); verify(blockResolver, times(1)).registerExecutor("app0", "exec1", config); verify(callback, times(1)).onSuccess(any(ByteBuffer.class)); verify(callback, never()).onFailure(any(Throwable.class)); // Verify register executor request latency metrics Timer registerExecutorRequestLatencyMillis = (Timer) ((ExternalShuffleBlockHandler) handler) .getAllMetrics() .getMetrics() .get("registerExecutorRequestLatencyMillis"); assertEquals(1, registerExecutorRequestLatencyMillis.getCount()); }
@Override public void receive(TransportClient client, ByteBuffer message, RpcResponseCallback callback) { BlockTransferMessage msgObj = BlockTransferMessage.Decoder.fromByteBuffer(message); handleMessage(msgObj, client, callback); }
try { OpenBlocks msg = (OpenBlocks) msgObj; checkAuth(client, msg.appId); long streamId = streamManager.registerStream(client.getClientId(), new ManagedBufferIterator(msg.appId, msg.execId, msg.blockIds)); try { RegisterExecutor msg = (RegisterExecutor) msgObj; checkAuth(client, msg.appId); blockManager.registerExecutor(msg.appId, msg.execId, msg.executorInfo); callback.onSuccess(ByteBuffer.wrap(new byte[0]));
@Before public void beforeEach() { streamManager = mock(OneForOneStreamManager.class); blockResolver = mock(ExternalShuffleBlockResolver.class); handler = new ExternalShuffleBlockHandler(streamManager, blockResolver); }
@After public void afterEach() { handler.applicationRemoved(APP_ID, false /* cleanupLocalDirs */); }
@Test public void testRegisterExecutor() { RpcResponseCallback callback = mock(RpcResponseCallback.class); ExecutorShuffleInfo config = new ExecutorShuffleInfo(new String[] {"/a", "/b"}, 16, "sort"); ByteBuffer registerMessage = new RegisterExecutor("app0", "exec1", config).toByteBuffer(); handler.receive(client, registerMessage, callback); verify(blockResolver, times(1)).registerExecutor("app0", "exec1", config); verify(callback, times(1)).onSuccess(any(ByteBuffer.class)); verify(callback, never()).onFailure(any(Throwable.class)); // Verify register executor request latency metrics Timer registerExecutorRequestLatencyMillis = (Timer) ((ExternalShuffleBlockHandler) handler) .getAllMetrics() .getMetrics() .get("registerExecutorRequestLatencyMillis"); assertEquals(1, registerExecutorRequestLatencyMillis.getCount()); }
@Override public void receive(TransportClient client, ByteBuffer message, RpcResponseCallback callback) { BlockTransferMessage msgObj = BlockTransferMessage.Decoder.fromByteBuffer(message); handleMessage(msgObj, client, callback); }
/** * Close the shuffle server to clean up any associated state. */ @Override protected void serviceStop() { try { if (shuffleServer != null) { shuffleServer.close(); } if (blockHandler != null) { blockHandler.close(); } if (db != null) { db.close(); } } catch (Exception e) { logger.error("Exception when stopping service", e); } }
@Before public void beforeEach() { streamManager = mock(OneForOneStreamManager.class); blockResolver = mock(ExternalShuffleBlockResolver.class); handler = new ExternalShuffleBlockHandler(streamManager, blockResolver); }
@After public void afterEach() { handler.applicationRemoved(APP_ID, false /* cleanupLocalDirs */); }
.getAllMetrics() .getMetrics() .get("openBlockRequestLatencyMillis"); .getAllMetrics() .getMetrics() .get("blockTransferRateBytes");
/** * Close the shuffle server to clean up any associated state. */ @Override protected void serviceStop() { try { if (shuffleServer != null) { shuffleServer.close(); } if (blockHandler != null) { blockHandler.close(); } if (db != null) { db.close(); } } catch (Exception e) { logger.error("Exception when stopping service", e); } }
blockHandler = new ExternalShuffleBlockHandler(transportConf, registeredExecutorFile);