synchronized void updateExecutor( final String executorId, final BiFunction<ExecutorRepresenter, ExecutorState, Pair<ExecutorRepresenter, ExecutorState>> updater) { final Pair<ExecutorRepresenter, ExecutorState> pair = executors.get(executorId); if (pair == null) { throw new IllegalArgumentException("Unknown executor id " + executorId); } else { executors.put(executorId, updater.apply(pair.left(), pair.right())); } }
private Set<ExecutorRepresenter> getRunningExecutors() { return executors.values() .stream() .filter(pair -> pair.right().equals(ExecutorState.RUNNING)) .map(Pair::left) .collect(Collectors.toSet()); }
private Set<ExecutorRepresenter> getRunningExecutors() { return executors.values() .stream() .filter(pair -> pair.right().equals(ExecutorState.RUNNING)) .map(Pair::left) .collect(Collectors.toSet()); }
synchronized void updateExecutor( final String executorId, final BiFunction<ExecutorRepresenter, ExecutorState, Pair<ExecutorRepresenter, ExecutorState>> updater) { final Pair<ExecutorRepresenter, ExecutorState> pair = executors.get(executorId); if (pair == null) { throw new IllegalArgumentException("Unknown executor id " + executorId); } else { executors.put(executorId, updater.apply(pair.left(), pair.right())); } }
/** * Say a DoFn of this reader has 3 main inputs and 4 side inputs. * {@link org.apache.nemo.runtime.executor.datatransfer.InputWatermarkManager} guarantees that the watermark here * is the minimum of the all 7 input streams. * @param newWatermark to set. */ public void setCurrentWatermarkOfAllMainAndSideInputs(final long newWatermark) { if (curWatermark > newWatermark) { // Cannot go backwards in time. throw new IllegalStateException(curWatermark + " > " + newWatermark); } this.curWatermark = newWatermark; // TODO #282: Handle late data inMemorySideInputs.entrySet().removeIf(entry -> { return entry.getKey().right().maxTimestamp().getMillis() <= this.curWatermark; // Discard old sideinputs. }); } }
/** * @return the generated vertex. */ private OperatorVertex generateMetricAggregationVertex() { // Define a custom data aggregator for skew handling. // Here, the aggregator gathers key frequency data used in shuffle data repartitioning. final BiFunction<Object, Map<Object, Long>, Map<Object, Long>> dynOptDataAggregator = (BiFunction<Object, Map<Object, Long>, Map<Object, Long>> & Serializable) (element, aggregatedDynOptData) -> { final Object key = ((Pair<Object, Long>) element).left(); final Long count = ((Pair<Object, Long>) element).right(); final Map<Object, Long> aggregatedDynOptDataMap = (Map<Object, Long>) aggregatedDynOptData; if (aggregatedDynOptDataMap.containsKey(key)) { aggregatedDynOptDataMap.compute(key, (existingKey, accumulatedCount) -> accumulatedCount + count); } else { aggregatedDynOptDataMap.put(key, count); } return aggregatedDynOptData; }; final AggregateMetricTransform abt = new AggregateMetricTransform<Pair<Object, Long>, Map<Object, Long>>(new HashMap<>(), dynOptDataAggregator); return new OperatorVertex(abt); }
/** * Builds and returns the StateMachine. * * @return the StateMachine * @throws RuntimeException if an initial state was not set */ public StateMachine build() { if (initialState == null) { throw new RuntimeException("An initial state should be set"); } final Map<Enum, State> stateMap = new HashMap<>(); for (final Enum stateEnum : stateEnumSet) { stateMap.put(stateEnum, new State(stateEnum, stateDescriptionMap.get(stateEnum))); } for (final Enum stateEnum : stateEnumSet) { final State state = stateMap.get(stateEnum); if (transitionMap.containsKey(stateEnum)) { for (final Pair<Enum, String> transition : transitionMap.get(stateEnum)) { state.addTransition(new Transition(state, stateMap.get(transition.left()), transition.right())); } } } return new StateMachine(stateMap, initialState); } }
/** * @return the generated vertex. */ private OperatorVertex generateMetricAggregationVertex() { // Define a custom data aggregator for skew handling. // Here, the aggregator gathers key frequency data used in shuffle data repartitioning. final BiFunction<Object, Map<Object, Long>, Map<Object, Long>> dynOptDataAggregator = (BiFunction<Object, Map<Object, Long>, Map<Object, Long>> & Serializable) (element, aggregatedDynOptData) -> { final Object key = ((Pair<Object, Long>) element).left(); final Long count = ((Pair<Object, Long>) element).right(); final Map<Object, Long> aggregatedDynOptDataMap = (Map<Object, Long>) aggregatedDynOptData; if (aggregatedDynOptDataMap.containsKey(key)) { aggregatedDynOptDataMap.compute(key, (existingKey, accumulatedCount) -> accumulatedCount + count); } else { aggregatedDynOptDataMap.put(key, count); } return aggregatedDynOptData; }; final AggregateMetricTransform abt = new AggregateMetricTransform<Pair<Object, Long>, Map<Object, Long>>(new HashMap<>(), dynOptDataAggregator); return new OperatorVertex(abt); }
/** * Builds and returns the StateMachine. * * @return the StateMachine * @throws RuntimeException if an initial state was not set */ public StateMachine build() { if (initialState == null) { throw new RuntimeException("An initial state should be set"); } final Map<Enum, State> stateMap = new HashMap<>(); for (final Enum stateEnum : stateEnumSet) { stateMap.put(stateEnum, new State(stateEnum, stateDescriptionMap.get(stateEnum))); } for (final Enum stateEnum : stateEnumSet) { final State state = stateMap.get(stateEnum); if (transitionMap.containsKey(stateEnum)) { for (final Pair<Enum, String> transition : transitionMap.get(stateEnum)) { state.addTransition(new Transition(state, stateMap.get(transition.left()), transition.right())); } } } return new StateMachine(stateMap, initialState); } }
private Map<String, List<OutputWriter>> getExternalAdditionalOutputMap( final IRVertex irVertex, final List<StageEdge> outEdgesToChildrenTasks, final IntermediateDataIOFactory intermediateDataIOFactory) { // Add all inter-task additional tags to additional output map. final Map<String, List<OutputWriter>> map = new HashMap<>(); outEdgesToChildrenTasks .stream() .filter(edge -> edge.getSrcIRVertex().getId().equals(irVertex.getId())) .filter(edge -> edge.getPropertyValue(AdditionalOutputTagProperty.class).isPresent()) .map(edge -> Pair.of(edge.getPropertyValue(AdditionalOutputTagProperty.class).get(), intermediateDataIOFactory.createWriter(taskId, edge))) .forEach(pair -> { map.putIfAbsent(pair.left(), new ArrayList<>()); map.get(pair.left()).add(pair.right()); }); return map; }
private Map<String, List<OutputWriter>> getExternalAdditionalOutputMap( final IRVertex irVertex, final List<StageEdge> outEdgesToChildrenTasks, final IntermediateDataIOFactory intermediateDataIOFactory) { // Add all inter-task additional tags to additional output map. final Map<String, List<OutputWriter>> map = new HashMap<>(); outEdgesToChildrenTasks .stream() .filter(edge -> edge.getSrcIRVertex().getId().equals(irVertex.getId())) .filter(edge -> edge.getPropertyValue(AdditionalOutputTagProperty.class).isPresent()) .map(edge -> Pair.of(edge.getPropertyValue(AdditionalOutputTagProperty.class).get(), intermediateDataIOFactory.createWriter(taskId, edge))) .forEach(pair -> { map.putIfAbsent(pair.left(), new ArrayList<>()); map.get(pair.left()).add(pair.right()); }); return map; }
private Map<String, List<NextIntraTaskOperatorInfo>> getInternalAdditionalOutputMap( final IRVertex irVertex, final DAG<IRVertex, RuntimeEdge<IRVertex>> irVertexDag, final Map<Edge, Integer> edgeIndexMap, final Map<IRVertex, InputWatermarkManager> operatorWatermarkManagerMap) { // Add all intra-task additional tags to additional output map. final Map<String, List<NextIntraTaskOperatorInfo>> map = new HashMap<>(); irVertexDag.getOutgoingEdgesOf(irVertex.getId()) .stream() .filter(edge -> edge.getPropertyValue(AdditionalOutputTagProperty.class).isPresent()) .map(edge -> { final String outputTag = edge.getPropertyValue(AdditionalOutputTagProperty.class).get(); final int index = edgeIndexMap.get(edge); final OperatorVertex nextOperator = (OperatorVertex) edge.getDst(); final InputWatermarkManager inputWatermarkManager = operatorWatermarkManagerMap.get(nextOperator); return Pair.of(outputTag, new NextIntraTaskOperatorInfo(index, nextOperator, inputWatermarkManager)); }) .forEach(pair -> { map.putIfAbsent(pair.left(), new ArrayList<>()); map.get(pair.left()).add(pair.right()); }); return map; }
private Map<String, List<NextIntraTaskOperatorInfo>> getInternalAdditionalOutputMap( final IRVertex irVertex, final DAG<IRVertex, RuntimeEdge<IRVertex>> irVertexDag, final Map<Edge, Integer> edgeIndexMap, final Map<IRVertex, InputWatermarkManager> operatorWatermarkManagerMap) { // Add all intra-task additional tags to additional output map. final Map<String, List<NextIntraTaskOperatorInfo>> map = new HashMap<>(); irVertexDag.getOutgoingEdgesOf(irVertex.getId()) .stream() .filter(edge -> edge.getPropertyValue(AdditionalOutputTagProperty.class).isPresent()) .map(edge -> { final String outputTag = edge.getPropertyValue(AdditionalOutputTagProperty.class).get(); final int index = edgeIndexMap.get(edge); final OperatorVertex nextOperator = (OperatorVertex) edge.getDst(); final InputWatermarkManager inputWatermarkManager = operatorWatermarkManagerMap.get(nextOperator); return Pair.of(outputTag, new NextIntraTaskOperatorInfo(index, nextOperator, inputWatermarkManager)); }) .forEach(pair -> { map.putIfAbsent(pair.left(), new ArrayList<>()); map.get(pair.left()).add(pair.right()); }); return map; }
final List<KeyRange> keyRanges = calculateKeyRanges(metricData.right(), dstParallelism, partitioner); final Map<Integer, KeyRange> taskIdxToKeyRange = new HashMap<>(); for (int i = 0; i < dstParallelism; i++) {
final ScheduledExecutorService dagLoggingExecutor = executionResult.right(); try { planStateManager.waitUntilFinish();
final ScheduledExecutorService dagLoggingExecutor = executionResult.right(); try { planStateManager.waitUntilFinish();
this.sortedHarnesses = pair.right();
this.sortedHarnesses = pair.right();
@Override public PhysicalPlan apply(final PhysicalPlan originalPlan, final Pair<StageEdge, Map<Object, Long>> metricData) { final StageEdge targetEdge = metricData.left(); // Get number of evaluators of the next stage (number of blocks). final Integer dstParallelism = targetEdge.getDst().getPropertyValue(ParallelismProperty.class). orElseThrow(() -> new RuntimeException("No parallelism on a vertex")); // Calculate keyRanges. final List<KeyRange> keyRanges = calculateKeyRanges(metricData.right(), dstParallelism); final Map<Integer, KeyRange> taskIdxToKeyRange = new HashMap<>(); for (int i = 0; i < dstParallelism; i++) { taskIdxToKeyRange.put(i, keyRanges.get(i)); } // Overwrite the previously assigned key range in the physical DAG with the new range. final DAG<Stage, StageEdge> stageDAG = originalPlan.getStageDAG(); for (Stage stage : stageDAG.getVertices()) { List<StageEdge> stageEdges = stageDAG.getOutgoingEdgesOf(stage); for (StageEdge edge : stageEdges) { if (edge.equals(targetEdge)) { edge.setTaskIdxToKeyRange(taskIdxToKeyRange); } } } return new PhysicalPlan(originalPlan.getPlanId(), stageDAG); }