public String toString() { StringBuilder sb = new StringBuilder(); sb.append("[").append(srcId.getQueryId()).append("] "); sb.append(srcId.getId()).append(" => ").append(targetId.getId()); sb.append(" (type=").append(shuffleType); if (hasShuffleKeys()) { sb.append(", key="); sb.append(StringUtils.join(shuffleKeys)); sb.append(", num=").append(numOutputs); } sb.append(")"); return sb.toString(); } }
public ExecutionBlockStopEvent(TajoIdProtos.ExecutionBlockIdProto executionBlockId, ExecutionBlockListProto cleanupList) { super(EventType.EB_STOP); this.cleanupList = cleanupList; this.executionBlockId = new ExecutionBlockId(executionBlockId); }
@Override public int compareTo(TaskId taskId) { int result = executionBlockId.compareTo(taskId.executionBlockId); if (result == 0) { return id - taskId.id; } else { return result; } }
public static Path getBaseInputDir(ExecutionBlockId executionBlockId) { return TajoPullServerService.getBaseInputDir(executionBlockId.getQueryId().toString(), executionBlockId.toString()); }
if (executionBlockId.getId() > 1) { Bootstrap bootstrap = new Bootstrap() .group(NettyUtils.getSharedEventLoopGroup(NettyUtils.GROUP.FETCHER, 1)) ExecutionBlockId clearEbId = new ExecutionBlockId(executionBlockId.getQueryId(), executionBlockId.getId() - 1); HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.DELETE, "ebid=" + clearEbId.toString()); request.headers().set(Names.HOST, connInfo.getHost()); request.headers().set(Names.CONNECTION, Values.CLOSE);
public static List<URI> createFetchURL(int maxUrlLength, FetchProto fetch, boolean includeParts) { PullServerRequestURIBuilder builder = new PullServerRequestURIBuilder(fetch.getHost(), fetch.getPort(), maxUrlLength); ExecutionBlockId ebId = new ExecutionBlockId(fetch.getExecutionBlockId()); builder.setRequestType(PullServerConstants.CHUNK_REQUEST_PARAM_STRING) .setQueryId(ebId.getQueryId().toString()) .setEbId(ebId.getId()) .setPartId(fetch.getPartitionId()); if (fetch.getType() == HASH_SHUFFLE) { builder.setShuffleType(PullServerConstants.HASH_SHUFFLE_PARAM_STRING); } else if (fetch.getType() == RANGE_SHUFFLE) { builder.setShuffleType(PullServerConstants.RANGE_SHUFFLE_PARAM_STRING); builder.setStartKeyBase64(new String(org.apache.commons.codec.binary.Base64.encodeBase64(fetch.getRangeStart().toByteArray()))); builder.setEndKeyBase64(new String(org.apache.commons.codec.binary.Base64.encodeBase64(fetch.getRangeEnd().toByteArray()))); builder.setLastInclude(fetch.getRangeLastInclusive()); } else if (fetch.getType() == SCATTERED_HASH_SHUFFLE) { builder.setShuffleType(PullServerConstants.SCATTERED_HASH_SHUFFLE_PARAM_STRING); } if (fetch.getLength() >= 0) { builder.setOffset(fetch.getOffset()).setLength(fetch.getLength()); } if (includeParts) { builder.setTaskIds(fetch.getTaskIdList()); builder.setAttemptIds(fetch.getAttemptIdList()); } return builder.build(includeParts); }
public QueryCompletedEvent(final ExecutionBlockId executionBlockId, StageState finalState) { super(executionBlockId.getQueryId(), QueryEventType.QUERY_COMPLETED); this.executionBlockId = executionBlockId; this.finalState = finalState; }
public String toString() { StringBuilder sb = new StringBuilder(); for (ExecutionBlock block : ExecutionBlockCursor.this) { if (sb.length() > 0) { sb.append(','); } if (block == last) { sb.append('('); } sb.append(block.getId().getId()); if (block == last) { sb.append(')'); } } return sb.toString(); } }
public TajoIdProtos.TaskIdProto getProto() { return TajoIdProtos.TaskIdProto.newBuilder() .setExecutionBlockId(executionBlockId.getProto()) .setId(id) .build(); }
public static ScanNode buildInputExecutor(LogicalPlan plan, DataChannel channel) { Preconditions.checkArgument(channel.getSchema() != null, "Channel schema (" + channel.getSrcId().getId() + " -> " + channel.getTargetId().getId() + ") is not initialized"); TableMeta meta = new TableMeta(channel.getDataFormat(), new KeyValueSet()); TableDesc desc = new TableDesc( channel.getSrcId().toString(), channel.getSchema(), meta, StorageConstants.LOCAL_FS_URI); ScanNode scanNode = plan.createNode(ScanNode.class); scanNode.init(desc); return scanNode; }
@Test public void testResourceDeallocateWithEbCreateFailure() throws Exception { final int taskSize = 10; resourceManager.setTaskHandlerEvent(true); final ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0); final Queue<TaskAllocationProto> totalTasks = MockNodeResourceManager.createTaskRequests(ebId, taskMemory, taskSize); TaskAllocationProto task = totalTasks.poll(); BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); requestProto.addTaskRequest(task); requestProto.setExecutionBlockId(ebId.getProto()); CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>(); dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture)); assertTrue(callFuture.get().getCancellationTaskCount() == 0); Thread.sleep(2000); // wait for resource deallocation assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource()); } }
public String toString() { return executionBlockId.toString(); }
if (executionBlockId.getId() > 1) { Bootstrap bootstrap = new Bootstrap() .group(NettyUtils.getSharedEventLoopGroup(NettyUtils.GROUP.FETCHER, 1)) ExecutionBlockId clearEbId = new ExecutionBlockId(executionBlockId.getQueryId(), executionBlockId.getId() - 1); HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.DELETE, "ebid=" + clearEbId.toString()); request.headers().set(Names.HOST, connInfo.getHost()); request.headers().set(Names.CONNECTION, Values.CLOSE);
public QueryCompletedEvent(final ExecutionBlockId executionBlockId, StageState finalState) { super(executionBlockId.getQueryId(), QueryEventType.QUERY_COMPLETED); this.executionBlockId = executionBlockId; this.finalState = finalState; }
public static Path getBaseInputDir(ExecutionBlockId executionBlockId) { return PullServerUtil.getBaseInputDir(executionBlockId.getQueryId().toString(), executionBlockId.toString()); }
public String toString() { StringBuilder sb = new StringBuilder(); for (ExecutionBlock block : ExecutionBlockCursor.this) { if (sb.length() > 0) { sb.append(','); } if (block == last) { sb.append('('); } sb.append(block.getId().getId()); if (block == last) { sb.append(')'); } } return sb.toString(); } }
public TajoIdProtos.TaskIdProto getProto() { return TajoIdProtos.TaskIdProto.newBuilder() .setExecutionBlockId(executionBlockId.getProto()) .setId(id) .build(); }
public static ScanNode buildInputExecutor(LogicalPlan plan, DataChannel channel) { Preconditions.checkArgument(channel.getSchema() != null, "Channel schema (" + channel.getSrcId().getId() + " -> " + channel.getTargetId().getId() + ") is not initialized"); TableMeta meta = new TableMeta(channel.getDataFormat(), new KeyValueSet()); TableDesc desc = new TableDesc( channel.getSrcId().toString(), channel.getSchema(), meta, StorageConstants.LOCAL_FS_URI); ScanNode scanNode = plan.createNode(ScanNode.class); scanNode.init(desc); return scanNode; }
@Test public void testNodeResourceCancellation() throws Exception { int requestSize = conf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES); int overSize = 10; resourceManager.setTaskHandlerEvent(false); //skip task execution CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>(); BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0); requestProto.setExecutionBlockId(ebId.getProto()); assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource()); requestProto.addAllTaskRequest( MockNodeResourceManager.createTaskRequests(ebId, taskMemory, requestSize + overSize)); dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture)); BatchAllocationResponse responseProto = callFuture.get(); assertEquals(overSize, responseProto.getCancellationTaskCount()); }
public String toString() { return executionBlockId.toString(); }