/** * @return id of irVertex. */ String getId() { return irVertex.getId(); }
/** * @return id of irVertex. */ String getId() { return irVertex.getId(); }
public void setIRVertexPutOnHold(final IRVertex irVertex) { idOfVertexPutOnHold = irVertex.getId(); }
public void setIRVertexPutOnHold(final IRVertex irVertex) { idOfVertexPutOnHold = irVertex.getId(); }
private Optional<Readable> getSourceVertexReader(final IRVertex irVertex, final Map<String, Readable> irVertexIdToReadable) { if (irVertex instanceof SourceVertex) { final Readable readable = irVertexIdToReadable.get(irVertex.getId()); if (readable == null) { throw new IllegalStateException(irVertex.toString()); } return Optional.of(readable); } else { return Optional.empty(); } }
private Optional<Readable> getSourceVertexReader(final IRVertex irVertex, final Map<String, Readable> irVertexIdToReadable) { if (irVertex instanceof SourceVertex) { final Readable readable = irVertexIdToReadable.get(irVertex.getId()); if (readable == null) { throw new IllegalStateException(irVertex.toString()); } return Optional.of(readable); } else { return Optional.empty(); } }
private List<Edge> getAllIncomingEdges( final Task task, final DAG<IRVertex, RuntimeEdge<IRVertex>> irVertexDag, final IRVertex childVertex) { final List<Edge> edges = new ArrayList<>(); edges.addAll(irVertexDag.getIncomingEdgesOf(childVertex)); final List<StageEdge> taskEdges = task.getTaskIncomingEdges().stream() .filter(edge -> edge.getDstIRVertex().getId().equals(childVertex.getId())) .collect(Collectors.toList()); edges.addAll(taskEdges); return edges; }
private List<Edge> getAllIncomingEdges( final Task task, final DAG<IRVertex, RuntimeEdge<IRVertex>> irVertexDag, final IRVertex childVertex) { final List<Edge> edges = new ArrayList<>(); edges.addAll(irVertexDag.getIncomingEdgesOf(childVertex)); final List<StageEdge> taskEdges = task.getTaskIncomingEdges().stream() .filter(edge -> edge.getDstIRVertex().getId().equals(childVertex.getId())) .collect(Collectors.toList()); edges.addAll(taskEdges); return edges; }
@Override public ObjectNode getPropertiesAsJsonNode() { final ObjectNode node = JsonNodeFactory.instance.objectNode(); node.put("runtimeEdgeId", getId()); node.set("executionProperties", getExecutionProperties().asJsonNode()); node.put("externalSrcVertexId", srcVertex.getId()); node.put("externalDstVertexId", dstVertex.getId()); return node; }
@Override public ObjectNode getPropertiesAsJsonNode() { final ObjectNode node = JsonNodeFactory.instance.objectNode(); node.put("runtimeEdgeId", getId()); node.set("executionProperties", getExecutionProperties().asJsonNode()); node.put("externalSrcVertexId", srcVertex.getId()); node.put("externalDstVertexId", dstVertex.getId()); return node; }
/** * Convert the crossing edges to JSON. * @param map map of the crossing edges. * @return a string of JSON showing the crossing edges. */ private static ObjectNode crossingEdgesToJSON(final Map<IRVertex, Set<IREdge>> map) { final ObjectNode node = JsonNodeFactory.instance.objectNode(); map.forEach(((irVertex, irEdges) -> { final ArrayNode vertexNode = node.putArray(irVertex.getId()); irEdges.forEach(e -> vertexNode.add(e.getId())); })); return node; } }
/** * Convert the crossing edges to JSON. * @param map map of the crossing edges. * @return a string of JSON showing the crossing edges. */ private static ObjectNode crossingEdgesToJSON(final Map<IRVertex, Set<IREdge>> map) { final ObjectNode node = JsonNodeFactory.instance.objectNode(); map.forEach(((irVertex, irEdges) -> { final ArrayNode vertexNode = node.putArray(irVertex.getId()); irEdges.forEach(e -> vertexNode.add(e.getId())); })); return node; } }
private List<NextIntraTaskOperatorInfo> getInternalMainOutputs( final IRVertex irVertex, final DAG<IRVertex, RuntimeEdge<IRVertex>> irVertexDag, final Map<Edge, Integer> edgeIndexMap, final Map<IRVertex, InputWatermarkManager> operatorWatermarkManagerMap) { return irVertexDag.getOutgoingEdgesOf(irVertex.getId()) .stream() .filter(edge -> !edge.getPropertyValue(AdditionalOutputTagProperty.class).isPresent()) .map(edge -> { final int index = edgeIndexMap.get(edge); final OperatorVertex nextOperator = (OperatorVertex) edge.getDst(); final InputWatermarkManager inputWatermarkManager = operatorWatermarkManagerMap.get(nextOperator); return new NextIntraTaskOperatorInfo(index, nextOperator, inputWatermarkManager); }) .collect(Collectors.toList()); }
static int getSourceParallelism(final InputReader inputReader) { return inputReader.getSrcIrVertex().getPropertyValue(ParallelismProperty.class) .orElseThrow(() -> new IllegalStateException(inputReader.getSrcIrVertex().getId())); } }
static int getSourceParallelism(final InputReader inputReader) { return inputReader.getSrcIrVertex().getPropertyValue(ParallelismProperty.class) .orElseThrow(() -> new IllegalStateException(inputReader.getSrcIrVertex().getId())); } }
/** * @param dag that contains the {@code v}. * @param v to inspect. * @return whether or not the vertex has parent with MetricCollectTransform. */ private boolean hasParentWithMetricCollectTransform(final DAG<IRVertex, IREdge> dag, final IRVertex v) { List<IRVertex> parents = dag.getParents(v.getId()); for (IRVertex parent : parents) { if (parent instanceof OperatorVertex && ((OperatorVertex) v).getTransform() instanceof MetricCollectTransform) { return true; } } return false; }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().stream() .filter(vertex -> dag.getIncomingEdgesOf(vertex.getId()) .stream() // TODO #198: Handle Un-cloneable Beam Sink Operators // only shuffle receivers (for now... as particular Beam sink operators fail when cloned) .anyMatch(edge -> edge.getPropertyValue(CommunicationPatternProperty.class) .orElseThrow(() -> new IllegalStateException()) .equals(CommunicationPatternProperty.Value.Shuffle)) ) .forEach(vertex -> vertex.setProperty( ClonedSchedulingProperty.of(new ClonedSchedulingProperty.CloneConf()))); // clone upfront, always return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices() .forEach(v -> dag.getOutgoingEdgesOf(v).stream() .filter(edge -> edge.getPropertyValue(MetricCollectionProperty.class).isPresent()) .forEach(skewEdge -> { final IRVertex dstV = skewEdge.getDst(); dstV.setProperty(ResourceSkewedDataProperty.of(true)); dag.getDescendants(dstV.getId()).forEach(descendentV -> { descendentV.getExecutionProperties().put(ResourceSkewedDataProperty.of(true)); }); }) ); return dag; } }
/** * Static initializer for irVertex. * @param irVertex irVertex to keep the execution property of. * @return The corresponding ExecutionPropertyMap. */ public static ExecutionPropertyMap<VertexExecutionProperty> of(final IRVertex irVertex) { final ExecutionPropertyMap<VertexExecutionProperty> map = new ExecutionPropertyMap<>(irVertex.getId()); map.put(ParallelismProperty.of(1)); map.put(ResourcePriorityProperty.of(ResourcePriorityProperty.NONE)); return map; }
/** * Static initializer for irVertex. * @param irVertex irVertex to keep the execution property of. * @return The corresponding ExecutionPropertyMap. */ public static ExecutionPropertyMap<VertexExecutionProperty> of(final IRVertex irVertex) { final ExecutionPropertyMap<VertexExecutionProperty> map = new ExecutionPropertyMap<>(irVertex.getId()); map.put(ParallelismProperty.of(1)); map.put(ResourcePriorityProperty.of(ResourcePriorityProperty.NONE)); return map; }