@Benchmark @OperationsPerInvocation(SPLITS) public Object benchmark(BenchmarkData data) { List<RemoteTask> remoteTasks = ImmutableList.copyOf(data.getTaskMap().values()); Iterator<MockRemoteTaskFactory.MockRemoteTask> finishingTask = Iterators.cycle(data.getTaskMap().values()); Iterator<Split> splits = data.getSplits().iterator(); Set<Split> batch = new HashSet<>(); while (splits.hasNext() || !batch.isEmpty()) { Multimap<Node, Split> assignments = data.getNodeSelector().computeAssignments(batch, remoteTasks).getAssignments(); for (Node node : assignments.keySet()) { MockRemoteTaskFactory.MockRemoteTask remoteTask = data.getTaskMap().get(node); remoteTask.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder() .putAll(new PlanNodeId("sourceId"), assignments.get(node)) .build()); remoteTask.startSplits(MAX_SPLITS_PER_NODE); } if (assignments.size() == batch.size()) { batch.clear(); } else { batch.removeAll(assignments.values()); } while (batch.size() < SPLIT_BATCH_SIZE && splits.hasNext()) { batch.add(splits.next()); } finishingTask.next().finishSplits((int) Math.ceil(MAX_SPLITS_PER_NODE / 50.0)); } return remoteTasks; }
((MockRemoteTask) stage.getAllTasks().get(0)).clearSplits();
@Override public void addSplits(Multimap<PlanNodeId, Split> splits) { synchronized (this) { this.splits.putAll(splits); } partitionedSplitCountTracker.setPartitionedSplitCount(getPartitionedSplitCount()); updateSplitQueueSpace(); }
PartitionedSplitCountTracker partitionedSplitCountTracker) return new MockRemoteTask(taskId, fragment, node.getNodeIdentifier(), partition, executor, initialSplits, partitionedSplitCountTracker);
task++; MockRemoteTaskFactory.MockRemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, ImmutableList.copyOf(assignments.get(node)), nodeTaskMap.createPartitionedSplitCountTracker(node, taskId)); remoteTask.startSplits(20); nodeTaskMap.addTask(node, remoteTask); taskMap.put(node, remoteTask);
((MockRemoteTask) stage.getAllTasks().get(0)).clearSplits();
for (Node node : assignments.keySet()) { MockRemoteTaskFactory.MockRemoteTask remoteTask = data.getTaskMap().get(node); remoteTask.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder() .putAll(new PlanNodeId("sourceId"), assignments.get(node)) .build()); remoteTask.startSplits(MAX_SPLITS_PER_NODE); batch.add(splits.next()); finishingTask.next().finishSplits((int) Math.ceil(MAX_SPLITS_PER_NODE / 50.0));
public MockRemoteTask(TaskId taskId, PlanFragment fragment, String nodeId, int partition, Executor executor, Multimap<PlanNodeId, Split> initialSplits, PartitionedSplitCountTracker partitionedSplitCountTracker) { this.taskStateMachine = new TaskStateMachine(requireNonNull(taskId, "taskId is null"), requireNonNull(executor, "executor is null")); MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE)); MemoryPool memorySystemPool = new MemoryPool(new MemoryPoolId("testSystem"), new DataSize(1, GIGABYTE)); this.taskContext = new QueryContext(taskId.getQueryId(), new DataSize(1, MEGABYTE), memoryPool, memorySystemPool, executor).addTaskContext(taskStateMachine, TEST_SESSION, new DataSize(1, MEGABYTE), true, true); this.location = URI.create("fake://task/" + taskId); this.sharedBuffer = new SharedBuffer(taskId, TASK_INSTANCE_ID, executor, requireNonNull(new DataSize(1, BYTE), "maxBufferSize is null")); this.fragment = requireNonNull(fragment, "fragment is null"); this.nodeId = requireNonNull(nodeId, "nodeId is null"); checkArgument(partition >= 0, "partition is negative"); this.partition = partition; splits.putAll(initialSplits); this.partitionedSplitCountTracker = requireNonNull(partitionedSplitCountTracker, "partitionedSplitCountTracker is null"); partitionedSplitCountTracker.setPartitionedSplitCount(getPartitionedSplitCount()); }
public synchronized void clearSplits() { splits.clear(); partitionedSplitCountTracker.setPartitionedSplitCount(getPartitionedSplitCount()); runningDrivers = 0; }
@Override public void start() { taskStateMachine.addStateChangeListener(newValue -> { if (newValue.isDone()) { clearSplits(); } }); }
@Override public int getQueuedPartitionedSplitCount() { if (taskStateMachine.getState().isDone()) { return 0; } return getPartitionedSplitCount() - runningDrivers; } }
@Override public CompletableFuture<TaskInfo> getStateChange(TaskInfo taskInfo) { return taskStateMachine.getStateChange(taskInfo.getState()).thenApply(ignored -> getTaskInfo()); }
@Override public void addStateChangeListener(StateChangeListener<TaskInfo> stateChangeListener) { taskStateMachine.addStateChangeListener(newValue -> stateChangeListener.stateChanged(getTaskInfo())); }
@Override public void addSplits(Multimap<PlanNodeId, Split> splits) { synchronized (this) { this.splits.putAll(splits); } partitionedSplitCountTracker.setPartitionedSplitCount(getPartitionedSplitCount()); }
@Override public void abort() { taskStateMachine.abort(); clearSplits(); }
boolean summarizeTaskInfo) return new MockRemoteTask(taskId, fragment, node.getNodeIdentifier(), executor, scheduledExecutor, initialSplits, totalPartitions, partitionedSplitCountTracker);
task++; MockRemoteTaskFactory.MockRemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, ImmutableList.copyOf(assignments.get(node)), nodeTaskMap.createPartitionedSplitCountTracker(node, taskId)); remoteTask.startSplits(25); nodeTaskMap.addTask(node, remoteTask); taskMap.put(node, remoteTask);
splits.putAll(initialSplits); this.partitionedSplitCountTracker = requireNonNull(partitionedSplitCountTracker, "partitionedSplitCountTracker is null"); partitionedSplitCountTracker.setPartitionedSplitCount(getPartitionedSplitCount()); updateSplitQueueSpace();
@Override public void abort() { taskStateMachine.abort(); clearSplits(); }
@Override public void addStateChangeListener(StateChangeListener<TaskStatus> stateChangeListener) { taskStateMachine.addStateChangeListener(newValue -> stateChangeListener.stateChanged(getTaskStatus())); }