private Map<String, List<OutputWriter>> getExternalAdditionalOutputMap( final IRVertex irVertex, final List<StageEdge> outEdgesToChildrenTasks, final IntermediateDataIOFactory intermediateDataIOFactory) { // Add all inter-task additional tags to additional output map. final Map<String, List<OutputWriter>> map = new HashMap<>(); outEdgesToChildrenTasks .stream() .filter(edge -> edge.getSrcIRVertex().getId().equals(irVertex.getId())) .filter(edge -> edge.getPropertyValue(AdditionalOutputTagProperty.class).isPresent()) .map(edge -> Pair.of(edge.getPropertyValue(AdditionalOutputTagProperty.class).get(), intermediateDataIOFactory.createWriter(taskId, edge))) .forEach(pair -> { map.putIfAbsent(pair.left(), new ArrayList<>()); map.get(pair.left()).add(pair.right()); }); return map; }
/** * Static initializer of the Pair class. * @param left left element. * @param right right element. * @param <A> Type of the left element. * @param <B> Type of the right element. * @return the newly created Pair. */ public static <A, B> Pair<A, B> of(final A left, final B right) { return new Pair<>(left, right); } }
public void onTaskScheduled(final String edgeId, final long srcIndex) { final Pair<String, Long> keyPair = Pair.of(edgeId, srcIndex); if (null != runtimeEdgeSrcIndexToLock.put(keyPair, new ReentrantLock())) { throw new IllegalStateException(keyPair.toString()); } if (null != runtimeEdgeSrcIndexToCondition.put(keyPair, runtimeEdgeSrcIndexToLock.get(keyPair).newCondition())) { throw new IllegalStateException(keyPair.toString()); } }
synchronized void updateExecutor( final String executorId, final BiFunction<ExecutorRepresenter, ExecutorState, Pair<ExecutorRepresenter, ExecutorState>> updater) { final Pair<ExecutorRepresenter, ExecutorState> pair = executors.get(executorId); if (pair == null) { throw new IllegalArgumentException("Unknown executor id " + executorId); } else { executors.put(executorId, updater.apply(pair.left(), pair.right())); } }
/** * Gets a list of integer pair elements in range. * @param start value of the range (inclusive). * @param end value of the range (exclusive). * @return the list of elements. */ public static List getRangedNumList(final int start, final int end) { final List numList = new ArrayList<>(end - start); IntStream.range(start, end).forEach(number -> numList.add(Pair.of(number, number))); return numList; }
private Set<ExecutorRepresenter> getRunningExecutors() { return executors.values() .stream() .filter(pair -> pair.right().equals(ExecutorState.RUNNING)) .map(Pair::left) .collect(Collectors.toSet()); }
@Override public Object extractKey(final Object element) { if (element instanceof Pair) { return ((Pair) element).left(); } else { throw new IllegalStateException(element.toString()); } } }
synchronized void updateExecutor( final String executorId, final BiFunction<ExecutorRepresenter, ExecutorState, Pair<ExecutorRepresenter, ExecutorState>> updater) { final Pair<ExecutorRepresenter, ExecutorState> pair = executors.get(executorId); if (pair == null) { throw new IllegalArgumentException("Unknown executor id " + executorId); } else { executors.put(executorId, updater.apply(pair.left(), pair.right())); } }
/** * Gets a list of integer pair elements in range. * @param start value of the range (inclusive). * @param end value of the range (exclusive). * @return the list of elements. */ public static List getRangedNumList(final int start, final int end) { final List numList = new ArrayList<>(end - start); IntStream.range(start, end).forEach(number -> numList.add(Pair.of(number, number))); return numList; }
private Set<ExecutorRepresenter> getRunningExecutors() { return executors.values() .stream() .filter(pair -> pair.right().equals(ExecutorState.RUNNING)) .map(Pair::left) .collect(Collectors.toSet()); }
@Override public Object extractKey(final Object element) { if (element instanceof Pair) { return ((Pair) element).left(); } else { throw new IllegalStateException(element.toString()); } } }
private Map<String, List<OutputWriter>> getExternalAdditionalOutputMap( final IRVertex irVertex, final List<StageEdge> outEdgesToChildrenTasks, final IntermediateDataIOFactory intermediateDataIOFactory) { // Add all inter-task additional tags to additional output map. final Map<String, List<OutputWriter>> map = new HashMap<>(); outEdgesToChildrenTasks .stream() .filter(edge -> edge.getSrcIRVertex().getId().equals(irVertex.getId())) .filter(edge -> edge.getPropertyValue(AdditionalOutputTagProperty.class).isPresent()) .map(edge -> Pair.of(edge.getPropertyValue(AdditionalOutputTagProperty.class).get(), intermediateDataIOFactory.createWriter(taskId, edge))) .forEach(pair -> { map.putIfAbsent(pair.left(), new ArrayList<>()); map.get(pair.left()).add(pair.right()); }); return map; }
/** * @return the generated vertex. */ private OperatorVertex generateMetricAggregationVertex() { // Define a custom data aggregator for skew handling. // Here, the aggregator gathers key frequency data used in shuffle data repartitioning. final BiFunction<Object, Map<Object, Long>, Map<Object, Long>> dynOptDataAggregator = (BiFunction<Object, Map<Object, Long>, Map<Object, Long>> & Serializable) (element, aggregatedDynOptData) -> { final Object key = ((Pair<Object, Long>) element).left(); final Long count = ((Pair<Object, Long>) element).right(); final Map<Object, Long> aggregatedDynOptDataMap = (Map<Object, Long>) aggregatedDynOptData; if (aggregatedDynOptDataMap.containsKey(key)) { aggregatedDynOptDataMap.compute(key, (existingKey, accumulatedCount) -> accumulatedCount + count); } else { aggregatedDynOptDataMap.put(key, count); } return aggregatedDynOptData; }; final AggregateMetricTransform abt = new AggregateMetricTransform<Pair<Object, Long>, Map<Object, Long>>(new HashMap<>(), dynOptDataAggregator); return new OperatorVertex(abt); }
synchronized void registerExecutor(final ExecutorRepresenter executor) { final String executorId = executor.getExecutorId(); if (executors.containsKey(executorId)) { throw new IllegalArgumentException("Duplicate executor: " + executor.toString()); } else { executors.put(executorId, Pair.of(executor, ExecutorState.RUNNING)); } }
public void onTaskScheduled(final String edgeId, final long srcIndex) { final Pair<String, Long> keyPair = Pair.of(edgeId, srcIndex); if (null != runtimeEdgeSrcIndexToLock.put(keyPair, new ReentrantLock())) { throw new IllegalStateException(keyPair.toString()); } if (null != runtimeEdgeSrcIndexToCondition.put(keyPair, runtimeEdgeSrcIndexToLock.get(keyPair).newCondition())) { throw new IllegalStateException(keyPair.toString()); } }
/** * Say a DoFn of this reader has 3 main inputs and 4 side inputs. * {@link org.apache.nemo.runtime.executor.datatransfer.InputWatermarkManager} guarantees that the watermark here * is the minimum of the all 7 input streams. * @param newWatermark to set. */ public void setCurrentWatermarkOfAllMainAndSideInputs(final long newWatermark) { if (curWatermark > newWatermark) { // Cannot go backwards in time. throw new IllegalStateException(curWatermark + " > " + newWatermark); } this.curWatermark = newWatermark; // TODO #282: Handle late data inMemorySideInputs.entrySet().removeIf(entry -> { return entry.getKey().right().maxTimestamp().getMillis() <= this.curWatermark; // Discard old sideinputs. }); } }
/** * Static initializer of the Pair class. * @param left left element. * @param right right element. * @param <A> Type of the left element. * @param <B> Type of the right element. * @return the newly created Pair. */ public static <A, B> Pair<A, B> of(final A left, final B right) { return new Pair<>(left, right); } }
private Map<String, List<NextIntraTaskOperatorInfo>> getInternalAdditionalOutputMap( final IRVertex irVertex, final DAG<IRVertex, RuntimeEdge<IRVertex>> irVertexDag, final Map<Edge, Integer> edgeIndexMap, final Map<IRVertex, InputWatermarkManager> operatorWatermarkManagerMap) { // Add all intra-task additional tags to additional output map. final Map<String, List<NextIntraTaskOperatorInfo>> map = new HashMap<>(); irVertexDag.getOutgoingEdgesOf(irVertex.getId()) .stream() .filter(edge -> edge.getPropertyValue(AdditionalOutputTagProperty.class).isPresent()) .map(edge -> { final String outputTag = edge.getPropertyValue(AdditionalOutputTagProperty.class).get(); final int index = edgeIndexMap.get(edge); final OperatorVertex nextOperator = (OperatorVertex) edge.getDst(); final InputWatermarkManager inputWatermarkManager = operatorWatermarkManagerMap.get(nextOperator); return Pair.of(outputTag, new NextIntraTaskOperatorInfo(index, nextOperator, inputWatermarkManager)); }) .forEach(pair -> { map.putIfAbsent(pair.left(), new ArrayList<>()); map.get(pair.left()).add(pair.right()); }); return map; }
/** * @return the generated vertex. */ private OperatorVertex generateMetricAggregationVertex() { // Define a custom data aggregator for skew handling. // Here, the aggregator gathers key frequency data used in shuffle data repartitioning. final BiFunction<Object, Map<Object, Long>, Map<Object, Long>> dynOptDataAggregator = (BiFunction<Object, Map<Object, Long>, Map<Object, Long>> & Serializable) (element, aggregatedDynOptData) -> { final Object key = ((Pair<Object, Long>) element).left(); final Long count = ((Pair<Object, Long>) element).right(); final Map<Object, Long> aggregatedDynOptDataMap = (Map<Object, Long>) aggregatedDynOptData; if (aggregatedDynOptDataMap.containsKey(key)) { aggregatedDynOptDataMap.compute(key, (existingKey, accumulatedCount) -> accumulatedCount + count); } else { aggregatedDynOptDataMap.put(key, count); } return aggregatedDynOptData; }; final AggregateMetricTransform abt = new AggregateMetricTransform<Pair<Object, Long>, Map<Object, Long>>(new HashMap<>(), dynOptDataAggregator); return new OperatorVertex(abt); }
synchronized void registerExecutor(final ExecutorRepresenter executor) { final String executorId = executor.getExecutorId(); if (executors.containsKey(executorId)) { throw new IllegalArgumentException("Duplicate executor: " + executor.toString()); } else { executors.put(executorId, Pair.of(executor, ExecutorState.RUNNING)); } }