/** * @return id of irVertex. */ String getId() { return irVertex.getId(); }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { // On every vertex, if ResourceSlotProperty is not set, put it as true. dag.getVertices().stream() .filter(v -> !v.getPropertyValue(ResourceSlotProperty.class).isPresent()) .forEach(v -> v.setProperty(ResourceSlotProperty.of(true))); return dag; } }
/** * Static function to copy executionProperties from a vertex to the other. * @param thatVertex the edge to copy executionProperties to. */ public final void copyExecutionPropertiesTo(final IRVertex thatVertex) { this.getExecutionProperties().forEachProperties(thatVertex::setProperty); }
static int getSourceParallelism(final InputReader inputReader) { return inputReader.getSrcIrVertex().getPropertyValue(ParallelismProperty.class) .orElseThrow(() -> new IllegalStateException(inputReader.getSrcIrVertex().getId())); } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().stream() .filter(vertex -> dag.getIncomingEdgesOf(vertex.getId()) .stream() // TODO #198: Handle Un-cloneable Beam Sink Operators // only shuffle receivers (for now... as particular Beam sink operators fail when cloned) .anyMatch(edge -> edge.getPropertyValue(CommunicationPatternProperty.class) .orElseThrow(() -> new IllegalStateException()) .equals(CommunicationPatternProperty.Value.Shuffle)) ) .forEach(vertex -> vertex.setProperty( ClonedSchedulingProperty.of(new ClonedSchedulingProperty.CloneConf()))); // clone upfront, always return dag; } }
/** * Determines the range of {@link ScheduleGroupProperty} value that will prevent collision * with the existing {@link ScheduleGroupProperty}. * @param irVertexCollection collection of {@link IRVertex} * @return the minimum value for the {@link ScheduleGroupProperty} that won't collide with the existing values */ private int getNextScheudleGroup(final Collection<IRVertex> irVertexCollection) { int nextScheduleGroup = 0; for (final IRVertex irVertex : irVertexCollection) { final Optional<Integer> scheduleGroup = irVertex.getPropertyValue(ScheduleGroupProperty.class); if (scheduleGroup.isPresent()) { nextScheduleGroup = Math.max(scheduleGroup.get() + 1, nextScheduleGroup); } } return nextScheduleGroup; }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices() .forEach(v -> dag.getOutgoingEdgesOf(v).stream() .filter(edge -> edge.getPropertyValue(MetricCollectionProperty.class).isPresent()) .forEach(skewEdge -> { final IRVertex dstV = skewEdge.getDst(); dstV.setProperty(ResourceSkewedDataProperty.of(true)); dag.getDescendants(dstV.getId()).forEach(descendentV -> { descendentV.getExecutionProperties().put(ResourceSkewedDataProperty.of(true)); }); }) ); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { // Speculative execution policy. final double fractionToWaitFor = 0.00000001; // Aggressive final double medianTimeMultiplier = 1.00000001; // Aggressive // Apply the policy to ALL vertices dag.getVertices().forEach(vertex -> vertex.setProperty(ClonedSchedulingProperty.of( new ClonedSchedulingProperty.CloneConf(fractionToWaitFor, medianTimeMultiplier)))); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { // On every vertex that receive push edge, if ResourceSlotProperty is not set, put it as false. // For other vertices, if ResourceSlotProperty is not set, put it as true. dag.getVertices().stream() .filter(v -> !v.getPropertyValue(ResourceSlotProperty.class).isPresent()) .forEach(v -> { if (dag.getIncomingEdgesOf(v).stream().anyMatch( e -> e.getPropertyValue(DataFlowProperty.class) .orElseThrow(() -> new RuntimeException(String.format("DataFlowProperty for %s must be set", e.getId()))).equals(DataFlowProperty.Value.Push))) { v.setPropertyPermanently(ResourceSlotProperty.of(false)); } else { v.setPropertyPermanently(ResourceSlotProperty.of(true)); } }); return dag; } }
if (irVertex instanceof SourceVertex && !irVertex.getStagePartitioned()) { final SourceVertex sourceVertex = (SourceVertex) irVertex; try { final List<Readable> readables = sourceVertex.getReadables(stageParallelism); for (int i = 0; i < stageParallelism; i++) { vertexIdToReadables.get(i).put(irVertex.getId(), readables.get(i)); irVertex.setStagePartitioned();
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.topologicalDo(vertex -> { final List<IREdge> inEdges = dag.getIncomingEdgesOf(vertex); if (inEdges.isEmpty()) { vertex.setPropertyPermanently(ResourcePriorityProperty.of(ResourcePriorityProperty.TRANSIENT)); } else { if (hasM2M(inEdges) || allO2OFromReserved(inEdges)) { vertex.setPropertyPermanently(ResourcePriorityProperty.of(ResourcePriorityProperty.RESERVED)); } else { vertex.setPropertyPermanently(ResourcePriorityProperty.of(ResourcePriorityProperty.TRANSIENT)); } } }); return dag; }
final IRVertex newIrVertex = irVertex.getClone(); originalToNewIRVertex.putIfAbsent(irVertex, newIrVertex);
/** * Determines the range of {@link ScheduleGroupProperty} value that will prevent collision * with the existing {@link ScheduleGroupProperty}. * @param irVertexCollection collection of {@link IRVertex} * @return the minimum value for the {@link ScheduleGroupProperty} that won't collide with the existing values */ private int getNextScheudleGroup(final Collection<IRVertex> irVertexCollection) { int nextScheduleGroup = 0; for (final IRVertex irVertex : irVertexCollection) { final Optional<Integer> scheduleGroup = irVertex.getPropertyValue(ScheduleGroupProperty.class); if (scheduleGroup.isPresent()) { nextScheduleGroup = Math.max(scheduleGroup.get() + 1, nextScheduleGroup); } } return nextScheduleGroup; }
static int getSourceParallelism(final InputReader inputReader) { return inputReader.getSrcIrVertex().getPropertyValue(ParallelismProperty.class) .orElseThrow(() -> new IllegalStateException(inputReader.getSrcIrVertex().getId())); } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().stream() .filter(v -> v instanceof OperatorVertex && ((OperatorVertex) v).getTransform() instanceof MetricCollectTransform) .forEach(v -> v.setProperty(DynamicOptimizationProperty .of(DynamicOptimizationProperty.Value.DataSkewRuntimePass))); dag.getVertices().stream() .filter(v -> hasParentWithMetricCollectTransform(dag, v) && !v.getExecutionProperties().containsKey(ResourceSkewedDataProperty.class)) .forEach(childV -> { childV.getExecutionProperties().put(ResourceSkewedDataProperty.of(true)); dag.getDescendants(childV.getId()).forEach(descendentV -> { descendentV.getExecutionProperties().put(ResourceSkewedDataProperty.of(true)); }); }); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { // Speculative execution policy. final double fractionToWaitFor = 0.00000001; // Aggressive final double medianTimeMultiplier = 1.00000001; // Aggressive // Apply the policy to ALL vertices dag.getVertices().forEach(vertex -> vertex.setProperty(ClonedSchedulingProperty.of( new ClonedSchedulingProperty.CloneConf(fractionToWaitFor, medianTimeMultiplier)))); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { // On every vertex that receive push edge, if ResourceSlotProperty is not set, put it as false. // For other vertices, if ResourceSlotProperty is not set, put it as true. dag.getVertices().stream() .filter(v -> !v.getPropertyValue(ResourceSlotProperty.class).isPresent()) .forEach(v -> { if (dag.getIncomingEdgesOf(v).stream().anyMatch( e -> e.getPropertyValue(DataFlowProperty.class) .orElseThrow(() -> new RuntimeException(String.format("DataFlowProperty for %s must be set", e.getId()))).equals(DataFlowProperty.Value.Push))) { v.setPropertyPermanently(ResourceSlotProperty.of(false)); } else { v.setPropertyPermanently(ResourceSlotProperty.of(true)); } }); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().stream() .filter(vertex -> dag.getIncomingEdgesOf(vertex.getId()) .stream() // TODO #198: Handle Un-cloneable Beam Sink Operators // only shuffle receivers (for now... as particular Beam sink operators fail when cloned) .anyMatch(edge -> edge.getPropertyValue(CommunicationPatternProperty.class) .orElseThrow(() -> new IllegalStateException()) .equals(CommunicationPatternProperty.Value.Shuffle)) ) .forEach(vertex -> vertex.setProperty( ClonedSchedulingProperty.of(new ClonedSchedulingProperty.CloneConf()))); // clone upfront, always return dag; } }
if (irVertex instanceof SourceVertex && !irVertex.getStagePartitioned()) { final SourceVertex sourceVertex = (SourceVertex) irVertex; try { final List<Readable> readables = sourceVertex.getReadables(stageParallelism); for (int i = 0; i < stageParallelism; i++) { vertexIdToReadables.get(i).put(irVertex.getId(), readables.get(i)); irVertex.setStagePartitioned();
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.topologicalDo(vertex -> { final List<IREdge> inEdges = dag.getIncomingEdgesOf(vertex); if (inEdges.isEmpty()) { vertex.setPropertyPermanently(ResourcePriorityProperty.of(ResourcePriorityProperty.TRANSIENT)); } else { if (hasM2M(inEdges) || allO2OFromReserved(inEdges)) { vertex.setPropertyPermanently(ResourcePriorityProperty.of(ResourcePriorityProperty.RESERVED)); } else { vertex.setPropertyPermanently(ResourcePriorityProperty.of(ResourcePriorityProperty.TRANSIENT)); } } }); return dag; }