/** * @param vertex a vertex in a stage * @return set of stage-level properties for the stage */ public Set<VertexExecutionProperty> getStageProperties(final IRVertex vertex) { return vertex.getExecutionProperties().stream() .filter(p -> !ignoredPropertyKeys.contains(p.getClass())) .collect(Collectors.toSet()); } }
/** * @param vertex a vertex in a stage * @return set of stage-level properties for the stage */ public Set<VertexExecutionProperty> getStageProperties(final IRVertex vertex) { return vertex.getExecutionProperties().stream() .filter(p -> !ignoredPropertyKeys.contains(p.getClass())) .collect(Collectors.toSet()); } }
/** * Static function to copy executionProperties from a vertex to the other. * @param thatVertex the edge to copy executionProperties to. */ public final void copyExecutionPropertiesTo(final IRVertex thatVertex) { this.getExecutionProperties().forEachProperties(thatVertex::setProperty); }
/** * Static function to copy executionProperties from a vertex to the other. * @param thatVertex the edge to copy executionProperties to. */ public final void copyExecutionPropertiesTo(final IRVertex thatVertex) { this.getExecutionProperties().forEachProperties(thatVertex::setProperty); }
/** * Copy Constructor for IRVertex. * * @param that the source object for copying */ public IRVertex(final IRVertex that) { super(IdManager.newVertexId()); this.executionProperties = ExecutionPropertyMap.of(this); that.getExecutionProperties().forEachProperties(this::setProperty); this.stagePartitioned = that.stagePartitioned; }
/** * Copy Constructor for IRVertex. * * @param that the source object for copying */ public IRVertex(final IRVertex that) { super(IdManager.newVertexId()); this.executionProperties = ExecutionPropertyMap.of(this); that.getExecutionProperties().forEachProperties(this::setProperty); this.stagePartitioned = that.stagePartitioned; }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().stream() .filter(v -> v instanceof OperatorVertex && ((OperatorVertex) v).getTransform() instanceof MetricCollectTransform) .forEach(v -> v.setProperty(DynamicOptimizationProperty .of(DynamicOptimizationProperty.Value.DataSkewRuntimePass))); dag.getVertices().stream() .filter(v -> hasParentWithMetricCollectTransform(dag, v) && !v.getExecutionProperties().containsKey(ResourceSkewedDataProperty.class)) .forEach(childV -> { childV.getExecutionProperties().put(ResourceSkewedDataProperty.of(true)); dag.getDescendants(childV.getId()).forEach(descendentV -> { descendentV.getExecutionProperties().put(ResourceSkewedDataProperty.of(true)); }); }); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices() .forEach(v -> dag.getOutgoingEdgesOf(v).stream() .filter(edge -> edge.getPropertyValue(MetricCollectionProperty.class).isPresent()) .forEach(skewEdge -> { final IRVertex dstV = skewEdge.getDst(); dstV.setProperty(ResourceSkewedDataProperty.of(true)); dag.getDescendants(dstV.getId()).forEach(descendentV -> { descendentV.getExecutionProperties().put(ResourceSkewedDataProperty.of(true)); }); }) ); return dag; } }
/** * Checks if the reshaping pass hasn't modified execution properties. * It checks if all of its vertices and edges have the same execution properties as before (if it existed then). * @param before DAG before modification. * @param after DAG after modification. * @return true if there is no problem, false if there is a problem. */ private static Boolean checkReshapingPass(final DAG<IRVertex, IREdge> before, final DAG<IRVertex, IREdge> after) { final List<IRVertex> previousVertices = before.getVertices(); for (final IRVertex irVertex : after.getVertices()) { final Integer indexOfVertex = previousVertices.indexOf(irVertex); if (indexOfVertex >= 0) { final IRVertex previousVertexToCompare = previousVertices.get(indexOfVertex); if (!previousVertexToCompare.getExecutionProperties().equals(irVertex.getExecutionProperties())) { return false; } for (final IREdge irEdge : after.getIncomingEdgesOf(irVertex)) { final Integer indexOfEdge = before.getIncomingEdgesOf(previousVertexToCompare).indexOf(irEdge); if (indexOfEdge >= 0) { final IREdge previousIREdgeToCompare = before.getIncomingEdgesOf(previousVertexToCompare).get(indexOfEdge); if (!previousIREdgeToCompare.getExecutionProperties().equals(irEdge.getExecutionProperties())) { return false; } } } } } return true; }
/** * Checks if the reshaping pass hasn't modified execution properties. * It checks if all of its vertices and edges have the same execution properties as before (if it existed then). * @param before DAG before modification. * @param after DAG after modification. * @return true if there is no problem, false if there is a problem. */ private static Boolean checkReshapingPass(final DAG<IRVertex, IREdge> before, final DAG<IRVertex, IREdge> after) { final List<IRVertex> previousVertices = before.getVertices(); for (final IRVertex irVertex : after.getVertices()) { final Integer indexOfVertex = previousVertices.indexOf(irVertex); if (indexOfVertex >= 0) { final IRVertex previousVertexToCompare = previousVertices.get(indexOfVertex); if (!previousVertexToCompare.getExecutionProperties().equals(irVertex.getExecutionProperties())) { return false; } for (final IREdge irEdge : after.getIncomingEdgesOf(irVertex)) { final Integer indexOfEdge = before.getIncomingEdgesOf(previousVertexToCompare).indexOf(irEdge); if (indexOfEdge >= 0) { final IREdge previousIREdgeToCompare = before.getIncomingEdgesOf(previousVertexToCompare).get(indexOfEdge); if (!previousIREdgeToCompare.getExecutionProperties().equals(irEdge.getExecutionProperties())) { return false; } } } } } return true; }