/** * Helper method to check that all execution properties are correct and makes sense. */ private void executionPropertyCheck() { // DataSizeMetricCollection is not compatible with Push (All data have to be stored before the data collection) vertices.forEach(v -> incomingEdges.get(v).stream().filter(e -> e instanceof IREdge).map(e -> (IREdge) e) .filter(e -> e.getPropertyValue(MetricCollectionProperty.class).isPresent()) .filter(e -> !(e.getDst() instanceof OperatorVertex && ((OperatorVertex) e.getDst()).getTransform() instanceof AggregateMetricTransform)) .filter(e -> DataFlowProperty.Value.Push.equals(e.getPropertyValue(DataFlowProperty.class).get())) .forEach(e -> { throw new CompileTimeOptimizationException("DAG execution property check: " + "DataSizeMetricCollection edge is not compatible with push" + e.getId()); })); }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { // On every vertex that receive push edge, if ResourceSlotProperty is not set, put it as false. // For other vertices, if ResourceSlotProperty is not set, put it as true. dag.getVertices().stream() .filter(v -> !v.getPropertyValue(ResourceSlotProperty.class).isPresent()) .forEach(v -> { if (dag.getIncomingEdgesOf(v).stream().anyMatch( e -> e.getPropertyValue(DataFlowProperty.class) .orElseThrow(() -> new RuntimeException(String.format("DataFlowProperty for %s must be set", e.getId()))).equals(DataFlowProperty.Value.Push))) { v.setPropertyPermanently(ResourceSlotProperty.of(false)); } else { v.setPropertyPermanently(ResourceSlotProperty.of(true)); } }); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { // On every vertex that receive push edge, if ResourceSlotProperty is not set, put it as false. // For other vertices, if ResourceSlotProperty is not set, put it as true. dag.getVertices().stream() .filter(v -> !v.getPropertyValue(ResourceSlotProperty.class).isPresent()) .forEach(v -> { if (dag.getIncomingEdgesOf(v).stream().anyMatch( e -> e.getPropertyValue(DataFlowProperty.class) .orElseThrow(() -> new RuntimeException(String.format("DataFlowProperty for %s must be set", e.getId()))).equals(DataFlowProperty.Value.Push))) { v.setPropertyPermanently(ResourceSlotProperty.of(false)); } else { v.setPropertyPermanently(ResourceSlotProperty.of(true)); } }); return dag; } }
/** * Helper method to check that all execution properties are correct and makes sense. */ private void executionPropertyCheck() { // DataSizeMetricCollection is not compatible with Push (All data have to be stored before the data collection) vertices.forEach(v -> incomingEdges.get(v).stream().filter(e -> e instanceof IREdge).map(e -> (IREdge) e) .filter(e -> Optional.of(MetricCollectionProperty.Value.DataSkewRuntimePass) .equals(e.getPropertyValue(MetricCollectionProperty.class))) .filter(e -> DataFlowProperty.Value.Push.equals(e.getPropertyValue(DataFlowProperty.class).get())) .forEach(e -> { throw new CompileTimeOptimizationException("DAG execution property check: " + "DataSizeMetricCollection edge is not compatible with push" + e.getId()); })); }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.topologicalDo(irVertex -> dag.getIncomingEdgesOf(irVertex).forEach(irEdge -> { final DataFlowProperty.Value dataFlowModel = irEdge.getPropertyValue(DataFlowProperty.class).get(); if (DataFlowProperty.Value.Push.equals(dataFlowModel)) { irEdge.setPropertyPermanently(DataPersistenceProperty.of(DataPersistenceProperty.Value.Discard)); } })); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.topologicalDo(irVertex -> dag.getIncomingEdgesOf(irVertex).forEach(irEdge -> { final DataFlowProperty.Value dataFlowModel = irEdge.getPropertyValue(DataFlowProperty.class).get(); if (DataFlowProperty.Value.Push.equals(dataFlowModel)) { irEdge.setPropertyPermanently(DataPersistenceProperty.of(DataPersistenceProperty.Value.Discard)); } })); return dag; } }