public TajoIdProtos.TaskIdProto getProto() { return TajoIdProtos.TaskIdProto.newBuilder() .setExecutionBlockId(executionBlockId.getProto()) .setId(id) .build(); }
public TajoIdProtos.TaskIdProto getProto() { return TajoIdProtos.TaskIdProto.newBuilder() .setExecutionBlockId(executionBlockId.getProto()) .setId(id) .build(); }
/** * Sends stopping request to all worker */ protected void stopExecutionBlock() { // If there are still live tasks, try to kill the tasks. and send the shuffle report request List<TajoIdProtos.ExecutionBlockIdProto> ebIds = Lists.newArrayList(); if (!getContext().getQueryContext().getBool(SessionVars.DEBUG_ENABLED)) { List<ExecutionBlock> childs = getMasterPlan().getChilds(getId()); for (ExecutionBlock executionBlock : childs) { ebIds.add(executionBlock.getId().getProto()); } } StopExecutionBlockRequest.Builder stopRequest = StopExecutionBlockRequest.newBuilder(); ExecutionBlockListProto.Builder cleanupList = ExecutionBlockListProto.newBuilder(); cleanupList.addAllExecutionBlockId(Lists.newArrayList(ebIds)); stopRequest.setCleanupList(cleanupList.build()); stopRequest.setExecutionBlockId(getId().getProto()); sendStopExecutionBlockEvent(stopRequest.build()); }
/** * Sends stopping request to all worker */ protected void stopExecutionBlock() { // If there are still live tasks, try to kill the tasks. and send the shuffle report request List<TajoIdProtos.ExecutionBlockIdProto> ebIds = Lists.newArrayList(); if (!getContext().getQueryContext().getBool(SessionVars.DEBUG_ENABLED)) { List<ExecutionBlock> childs = getMasterPlan().getChilds(getId()); for (ExecutionBlock executionBlock : childs) { ebIds.add(executionBlock.getId().getProto()); } } StopExecutionBlockRequest.Builder stopRequest = StopExecutionBlockRequest.newBuilder(); ExecutionBlockListProto.Builder cleanupList = ExecutionBlockListProto.newBuilder(); cleanupList.addAllExecutionBlockId(Lists.newArrayList(ebIds)); stopRequest.setCleanupList(cleanupList.build()); stopRequest.setExecutionBlockId(getId().getProto()); sendStopExecutionBlockEvent(stopRequest.build()); }
public DataChannelProto getProto() { DataChannelProto.Builder builder = DataChannelProto.newBuilder(); builder.setSrcId(srcId.getProto()); builder.setTargetId(targetId.getProto()); if (transmitType != null) { builder.setTransmitType(transmitType); } builder.setShuffleType(shuffleType); if (schema != null) { builder.setSchema(schema.getProto()); } if (shuffleKeys != null) { for (Column column : shuffleKeys) { builder.addShuffleKeys(column.getProto()); } } if (numOutputs != null) { builder.setNumOutputs(numOutputs); } if(dataFormat != null){ builder.setDataFormat(dataFormat); } return builder.build(); }
public DataChannelProto getProto() { DataChannelProto.Builder builder = DataChannelProto.newBuilder(); builder.setSrcId(srcId.getProto()); builder.setTargetId(targetId.getProto()); if (transmitType != null) { builder.setTransmitType(transmitType); } builder.setShuffleType(shuffleType); if (schema != null) { builder.setSchema(schema.getProto()); } if (shuffleKeys != null) { for (Column column : shuffleKeys) { builder.addShuffleKeys(column.getProto()); } } if (numOutputs != null) { builder.setNumOutputs(numOutputs); } if(dataFormat != null){ builder.setDataFormat(dataFormat); } return builder.build(); }
@Override public FetchProto getProto() { FetchProto.Builder builder = FetchProto.newBuilder(); builder.setHost(host.getHost()); builder.setPort(host.getPort()); builder.setType(type); builder.setExecutionBlockId(executionBlockId.getProto()); builder.setPartitionId(partitionId); builder.setHasNext(hasNext); builder.setName(name); if (rangeParam != null) { builder.setRangeStart(ByteString.copyFrom(rangeParam.getStart())); builder.setRangeEnd(ByteString.copyFrom(rangeParam.getEnd())); builder.setRangeLastInclusive(rangeParam.isLastInclusive()); } Preconditions.checkArgument(taskIds.size() == attemptIds.size()); builder.addAllTaskId(taskIds); builder.addAllAttemptId(attemptIds); builder.setOffset(offset); builder.setLength(length); return builder.build(); }
@Override public FetchProto getProto() { FetchProto.Builder builder = FetchProto.newBuilder(); builder.setHost(host.getHost()); builder.setPort(host.getPort()); builder.setType(type); builder.setExecutionBlockId(executionBlockId.getProto()); builder.setPartitionId(partitionId); builder.setHasNext(hasNext); builder.setName(name); if (rangeParam != null) { builder.setRangeStart(ByteString.copyFrom(rangeParam.getStart())); builder.setRangeEnd(ByteString.copyFrom(rangeParam.getEnd())); builder.setRangeLastInclusive(rangeParam.isLastInclusive()); } Preconditions.checkArgument(taskIds.size() == attemptIds.size()); builder.addAllTaskId(taskIds); builder.addAllAttemptId(attemptIds); builder.setOffset(offset); builder.setLength(length); return builder.build(); }
protected ExecutionBlockContext createExecutionBlock(ExecutionBlockId executionBlockId, String queryMasterHostAndPort) { LOG.info("QueryMaster Address:" + queryMasterHostAndPort); AsyncRpcClient client = null; try { InetSocketAddress address = NetUtils.createSocketAddr(queryMasterHostAndPort); ExecutionBlockContextRequest.Builder request = ExecutionBlockContextRequest.newBuilder(); request.setExecutionBlockId(executionBlockId.getProto()) .setWorker(getWorkerContext().getConnectionInfo().getProto()); client = RpcClientManager.getInstance().newClient(address, QueryMasterProtocol.class, true, rpcParams); QueryMasterProtocol.QueryMasterProtocolService.Interface stub = client.getStub(); CallFuture<ExecutionBlockContextResponse> callback = new CallFuture<ExecutionBlockContextResponse>(); stub.getExecutionBlockContext(callback.getController(), request.build(), callback); ExecutionBlockContextResponse contextProto = callback.get(); ExecutionBlockContext context = new ExecutionBlockContext(getWorkerContext(), contextProto, client); context.init(); return context; } catch (Throwable e) { RpcClientManager.cleanup(client); LOG.fatal(e.getMessage(), e); throw new RuntimeException(e); } }
protected ExecutionBlockContext createExecutionBlock(ExecutionBlockId executionBlockId, String queryMasterHostAndPort) { LOG.info("QueryMaster Address:" + queryMasterHostAndPort); AsyncRpcClient client = null; try { InetSocketAddress address = NetUtils.createSocketAddr(queryMasterHostAndPort); ExecutionBlockContextRequest.Builder request = ExecutionBlockContextRequest.newBuilder(); request.setExecutionBlockId(executionBlockId.getProto()) .setWorker(getWorkerContext().getConnectionInfo().getProto()); client = RpcClientManager.getInstance().newClient(address, QueryMasterProtocol.class, true, rpcParams); QueryMasterProtocol.QueryMasterProtocolService.Interface stub = client.getStub(); CallFuture<ExecutionBlockContextResponse> callback = new CallFuture<>(); stub.getExecutionBlockContext(callback.getController(), request.build(), callback); ExecutionBlockContextResponse contextProto = callback.get(); ExecutionBlockContext context = new ExecutionBlockContext(getWorkerContext(), contextProto, client, pullServerService); context.init(); return context; } catch (Throwable e) { RpcClientManager.cleanup(client); LOG.fatal(e.getMessage(), e); throw new RuntimeException(e); } }
@Override public void run() { int complete = 0; while (true) { TaskAllocationProto task = totalTasks.poll(); if (task == null) break; BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); requestProto.addTaskRequest(task); requestProto.setExecutionBlockId(ebId.getProto()); CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>(); dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture)); try { BatchAllocationResponse proto = callFuture.get(); if (proto.getCancellationTaskCount() > 0) { totalTasks.addAll(proto.getCancellationTaskList()); totalCanceled.addAndGet(proto.getCancellationTaskCount()); } else { complete++; } } catch (Exception e) { fail(e.getMessage()); } } totalComplete.addAndGet(complete); } })
@Override protected ExecutionBlockContext createExecutionBlock(ExecutionBlockId executionBlockId, String queryMaster) { if (testEbCreateFailure) { throw new RuntimeException("Failure for test"); } try { ExecutionBlockContextResponse.Builder builder = ExecutionBlockContextResponse.newBuilder(); builder.setExecutionBlockId(executionBlockId.getProto()) .setPlanJson("test") .setQueryContext(new QueryContext(new TajoConf()).getProto()) .setQueryOutputPath("testpath") .setShuffleType(PlanProto.ShuffleType.HASH_SHUFFLE); return new MockExecutionBlockContext(getWorkerContext(), builder.build()); } catch (IOException e) { throw new RuntimeException(e); } }
@Test public void testTaskException() throws Exception { int requestSize = 1; QueryId qid = LocalTajoTestingUtility.newQueryId(); ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(qid, 1); CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>(); BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); requestProto.setExecutionBlockId(ebId.getProto()); assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource()); requestProto.addAllTaskRequest(MockNodeResourceManager.createTaskRequests(ebId, 10, requestSize)); taskExecutor.throwException.set(true); dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture)); //verify running task assertTrue(barrier.tryAcquire(3, TimeUnit.SECONDS)); assertEquals(1, taskExecutor.getRunningTasks()); assertTrue(barrier.tryAcquire(3, TimeUnit.SECONDS)); assertEquals(0, taskExecutor.getRunningTasks()); assertEquals(0, taskExecutor.completeTasks); //verify the released resources Thread.sleep(100); assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource()); }
@Test public void testResourceDeallocateWithEbCreateFailure() throws Exception { final int taskSize = 10; resourceManager.setTaskHandlerEvent(true); final ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0); final Queue<TaskAllocationProto> totalTasks = MockNodeResourceManager.createTaskRequests(ebId, taskMemory, taskSize); TaskAllocationProto task = totalTasks.poll(); BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); requestProto.addTaskRequest(task); requestProto.setExecutionBlockId(ebId.getProto()); CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>(); dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture)); assertTrue(callFuture.get().getCancellationTaskCount() == 0); Thread.sleep(2000); // wait for resource deallocation assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource()); } }
@Test(timeout = 10000) public void testExecutionBlockStop() throws Exception { QueryId qid = LocalTajoTestingUtility.newQueryId(); ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(qid, 1); TaskAllocationProto requestProto = MockNodeResourceManager.createTaskRequests(ebId, taskMemory, 1).poll(); taskDispatcher.getEventHandler().handle(new TaskStartEvent(requestProto.getTaskRequest(), new Allocation(new NodeResource(requestProto.getResource())))); assertTrue(barrier.tryAcquire(3, TimeUnit.SECONDS)); assertNotNull(taskManager.getExecutionBlockContext(ebId)); assertEquals(ebId, taskManager.getExecutionBlockContext(ebId).getExecutionBlockId()); ExecutionBlockListProto.Builder ebList = ExecutionBlockListProto.newBuilder(); taskDispatcher.getEventHandler().handle(new ExecutionBlockStopEvent(ebId.getProto(), ebList.build())); assertTrue(barrier.tryAcquire(3, TimeUnit.SECONDS)); assertNull(taskManager.getExecutionBlockContext(ebId)); } }
@Test public void testTaskRequest() throws Exception { int requestSize = 1; QueryId qid = LocalTajoTestingUtility.newQueryId(); ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(qid, 1); CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>(); BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); requestProto.setExecutionBlockId(ebId.getProto()); assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource()); requestProto.addAllTaskRequest(MockNodeResourceManager.createTaskRequests(ebId, 10, requestSize)); dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture)); //verify running task assertTrue(barrier.tryAcquire(3, TimeUnit.SECONDS)); assertEquals(1, taskExecutor.getRunningTasks()); assertTrue(barrier.tryAcquire(3, TimeUnit.SECONDS)); assertEquals(0, taskExecutor.getRunningTasks()); assertEquals(1, taskExecutor.completeTasks); //verify the released resources Thread.sleep(100); assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource()); }
@Test(timeout = 10000) public void testExecutionBlockStart() throws Exception { int requestSize = 1; QueryId qid = LocalTajoTestingUtility.newQueryId(); ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(qid, 1); CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>(); BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); requestProto.setExecutionBlockId(ebId.getProto()); assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource()); requestProto.addAllTaskRequest(MockNodeResourceManager.createTaskRequests(ebId, taskMemory, requestSize)); dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture)); assertTrue(barrier.tryAcquire(3, TimeUnit.SECONDS)); assertNotNull(taskManager.getExecutionBlockContext(ebId)); assertEquals(ebId, taskManager.getExecutionBlockContext(ebId).getExecutionBlockId()); }
@Test public void testNodeResourceCancellation() throws Exception { int requestSize = conf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES); int overSize = 10; resourceManager.setTaskHandlerEvent(false); //skip task execution CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>(); BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0); requestProto.setExecutionBlockId(ebId.getProto()); assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource()); requestProto.addAllTaskRequest( MockNodeResourceManager.createTaskRequests(ebId, taskMemory, requestSize + overSize)); dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture)); BatchAllocationResponse responseProto = callFuture.get(); assertEquals(overSize, responseProto.getCancellationTaskCount()); }
@Test public void testNodeResourceAllocateEvent() throws Exception { int requestSize = 4; resourceManager.setTaskHandlerEvent(false); //skip task execution CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>(); BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0); requestProto.setExecutionBlockId(ebId.getProto()); assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource()); requestProto.addAllTaskRequest(MockNodeResourceManager.createTaskRequests(ebId, taskMemory, requestSize)); dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture)); BatchAllocationResponse responseProto = callFuture.get(); assertNotEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource()); // allocated all assertEquals(0, responseProto.getCancellationTaskCount()); }
@Test public void testNodeResourceDeallocateEvent() throws Exception { int requestSize = 4; resourceManager.setTaskHandlerEvent(false); //skip task execution CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>(); BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0); requestProto.setExecutionBlockId(ebId.getProto()); assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource()); requestProto.addAllTaskRequest(MockNodeResourceManager.createTaskRequests(ebId, taskMemory, requestSize)); dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture)); BatchAllocationResponse responseProto = callFuture.get(); assertNotEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource()); assertEquals(0, responseProto.getCancellationTaskCount()); //deallocate for(TaskAllocationProto allocationRequestProto : requestProto.getTaskRequestList()) { // direct invoke handler for testing resourceManager.handle(new NodeResourceDeallocateEvent( allocationRequestProto.getResource(), NodeResourceEvent.ResourceType.TASK)); } assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource()); }