/** * @param inEdges list of inEdges to the specific irVertex * @return true if and only if the irVertex has one OneToOne edge */ private static boolean isOneToOneEdge(final Collection<IREdge> inEdges) { return inEdges.size() == 1 && inEdges.iterator().next() .getPropertyValue(CommunicationPatternProperty.class).get() .equals(CommunicationPatternProperty.Value.OneToOne); }
/** * @param inEdges list of inEdges to the specific irVertex * @return true if and only if the irVertex has one OneToOne edge */ private static boolean isOneToOneEdge(final Collection<IREdge> inEdges) { return inEdges.size() == 1 && inEdges.iterator().next() .getPropertyValue(CommunicationPatternProperty.class).get() .equals(CommunicationPatternProperty.Value.OneToOne); }
/** * Checks whether the irEdges have M2M relationship. * @param irEdges irEdges to check. * @return whether of not any of them has M2M relationship. */ private boolean hasM2M(final List<IREdge> irEdges) { return irEdges.stream().anyMatch(edge -> edge.getPropertyValue(CommunicationPatternProperty.class).get() .equals(CommunicationPatternProperty.Value.Shuffle)); }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { final List<IREdge> inEdges = dag.getIncomingEdgesOf(vertex); inEdges.forEach(edge -> { if (edge.getPropertyValue(CommunicationPatternProperty.class).get() .equals(CommunicationPatternProperty.Value.Shuffle)) { edge.setPropertyPermanently(DataFlowProperty.of(DataFlowProperty.Value.Push)); // Push to the merger vertex. } else { edge.setPropertyPermanently(DataFlowProperty.of(DataFlowProperty.Value.Pull)); } }); }); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { final List<IREdge> inEdges = dag.getIncomingEdgesOf(vertex); inEdges.forEach(edge -> { if (edge.getPropertyValue(CommunicationPatternProperty.class).get() .equals(CommunicationPatternProperty.Value.Shuffle)) { dag.getOutgoingEdgesOf(edge.getDst()) .forEach(edgeFromRelay -> edgeFromRelay.setPropertyPermanently(EncoderProperty.of(BytesEncoderFactory.of()))); } }); }); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { final List<IREdge> inEdges = dag.getIncomingEdgesOf(vertex); inEdges.forEach(edge -> { if (edge.getPropertyValue(CommunicationPatternProperty.class).get() .equals(CommunicationPatternProperty.Value.Shuffle)) { edge.setPropertyPermanently(CompressionProperty.of(CompressionProperty.Value.LZ4)); dag.getOutgoingEdgesOf(edge.getDst()) .forEach(edgeFromRelay -> edgeFromRelay.setPropertyPermanently(CompressionProperty.of(CompressionProperty.Value.None))); } }); }); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().stream() .filter(vertex -> dag.getIncomingEdgesOf(vertex.getId()) .stream() // TODO #198: Handle Un-cloneable Beam Sink Operators // only shuffle receivers (for now... as particular Beam sink operators fail when cloned) .anyMatch(edge -> edge.getPropertyValue(CommunicationPatternProperty.class) .orElseThrow(() -> new IllegalStateException()) .equals(CommunicationPatternProperty.Value.Shuffle)) ) .forEach(vertex -> vertex.setProperty( ClonedSchedulingProperty.of(new ClonedSchedulingProperty.CloneConf()))); // clone upfront, always return dag; } }
@Override public List<CompletableFuture<DataUtil.IteratorWithNumBytes>> read() { final Optional<CommunicationPatternProperty.Value> comValue = runtimeEdge.getPropertyValue(CommunicationPatternProperty.class); if (comValue.get().equals(CommunicationPatternProperty.Value.OneToOne)) { return Collections.singletonList(readOneToOne()); } else if (comValue.get().equals(CommunicationPatternProperty.Value.BroadCast)) { return readBroadcast(); } else if (comValue.get().equals(CommunicationPatternProperty.Value.Shuffle)) { // If the dynamic optimization which detects data skew is enabled, read the data in the assigned range. // TODO #492: Modularize the data communication pattern. return readDataInRange(); } else { throw new UnsupportedCommPatternException(new Exception("Communication pattern not supported")); } }
public boolean hasSkewedData(final Task task) { final int taskIdx = RuntimeIdManager.getIndexFromTaskId(task.getTaskId()); for (StageEdge inEdge : task.getTaskIncomingEdges()) { if (CommunicationPatternProperty.Value.Shuffle .equals(inEdge.getDataCommunicationPattern())) { final Map<Integer, KeyRange> taskIdxToKeyRange = inEdge.getPropertyValue(DataSkewMetricProperty.class).get().getMetric(); final KeyRange hashRange = taskIdxToKeyRange.get(taskIdx); if (((HashRange) hashRange).isSkewed()) { return true; } } } return false; }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { final List<IREdge> inEdges = dag.getIncomingEdgesOf(vertex); inEdges.forEach(edge -> { if (edge.getPropertyValue(CommunicationPatternProperty.class).get() .equals(CommunicationPatternProperty.Value.Shuffle)) { edge.setPropertyPermanently(DecoderProperty.of(BytesDecoderFactory.of())); } }); }); return dag; } }
@Override public List<CompletableFuture<DataUtil.IteratorWithNumBytes>> read() { final Optional<CommunicationPatternProperty.Value> comValue = runtimeEdge.getPropertyValue(CommunicationPatternProperty.class); if (comValue.get().equals(CommunicationPatternProperty.Value.OneToOne)) { return Collections.singletonList(readOneToOne()); } else if (comValue.get().equals(CommunicationPatternProperty.Value.BroadCast)) { return readBroadcast(); } else if (comValue.get().equals(CommunicationPatternProperty.Value.Shuffle)) { // If the dynamic optimization which detects data skew is enabled, read the data in the assigned range. // TODO #492: Modularize the data communication pattern. return readDataInRange(); } else { throw new UnsupportedCommPatternException(new Exception("Communication pattern not supported")); } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { final List<IREdge> inEdges = dag.getIncomingEdgesOf(vertex); if (!inEdges.isEmpty()) { inEdges.forEach(edge -> { if (edge.getPropertyValue(CommunicationPatternProperty.class).get() .equals(CommunicationPatternProperty.Value.Shuffle)) { edge.setProperty(DataFlowProperty.of(DataFlowProperty.Value.Push)); } }); } }); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { final List<IREdge> inEdges = dag.getIncomingEdgesOf(vertex); inEdges.forEach(edge -> { if (edge.getPropertyValue(CommunicationPatternProperty.class).get() .equals(CommunicationPatternProperty.Value.Shuffle)) { edge.setPropertyPermanently(DecoderProperty.of(BytesDecoderFactory.of())); } }); }); return dag; } }
/** * Checks whether the irEdges are all from reserved containers. * @param irEdges irEdges to check. * @return whether of not they are from reserved containers. */ private boolean allO2OFromReserved(final List<IREdge> irEdges) { return irEdges.stream() .allMatch(edge -> CommunicationPatternProperty.Value.OneToOne.equals( edge.getPropertyValue(CommunicationPatternProperty.class).get()) && edge.getSrc().getPropertyValue(ResourcePriorityProperty.class).get().equals( ResourcePriorityProperty.RESERVED)); } }
private int getNumOfPipeToWait(final RuntimeEdge runtimeEdge) { final int dstParallelism = ((StageEdge) runtimeEdge).getDstIRVertex().getPropertyValue(ParallelismProperty.class) .orElseThrow(() -> new IllegalStateException()); final CommunicationPatternProperty.Value commPattern = ((StageEdge) runtimeEdge) .getPropertyValue(CommunicationPatternProperty.class) .orElseThrow(() -> new IllegalStateException()); return commPattern.equals(CommunicationPatternProperty.Value.OneToOne) ? 1 : dstParallelism; } }
/** * Checks whether the irEdges are all from reserved containers. * @param irEdges irEdges to check. * @return whether of not they are from reserved containers. */ private boolean allO2OFromReserved(final List<IREdge> irEdges) { return irEdges.stream() .allMatch(edge -> CommunicationPatternProperty.Value.OneToOne.equals( edge.getPropertyValue(CommunicationPatternProperty.class).get()) && edge.getSrc().getPropertyValue(ResourcePriorityProperty.class).get().equals( ResourcePriorityProperty.RESERVED)); } }
private List<ByteOutputContext> getPipeToWrite(final Object element) { final CommunicationPatternProperty.Value comm = (CommunicationPatternProperty.Value) runtimeEdge.getPropertyValue(CommunicationPatternProperty.class).get(); if (comm.equals(CommunicationPatternProperty.Value.OneToOne)) { return Collections.singletonList(pipes.get(0)); } else if (comm.equals(CommunicationPatternProperty.Value.BroadCast)) { return pipes; } else { return Collections.singletonList(pipes.get((int) partitioner.partition(element))); } } }
private List<ByteOutputContext> getPipeToWrite(final Object element) { final CommunicationPatternProperty.Value comm = (CommunicationPatternProperty.Value) runtimeEdge.getPropertyValue(CommunicationPatternProperty.class).get(); if (comm.equals(CommunicationPatternProperty.Value.OneToOne)) { return Collections.singletonList(pipes.get(0)); } else if (comm.equals(CommunicationPatternProperty.Value.BroadCast)) { return pipes; } else { return Collections.singletonList(pipes.get((int) partitioner.partition(element))); } } }
/** * Checks whether the irEdges have M2M relationship. * @param irEdges irEdges to check. * @return whether of not any of them has M2M relationship. */ private boolean hasM2M(final List<IREdge> irEdges) { return irEdges.stream().anyMatch(edge -> edge.getPropertyValue(CommunicationPatternProperty.class).get() .equals(CommunicationPatternProperty.Value.Shuffle)); }
private int getNumOfPipeToWait(final RuntimeEdge runtimeEdge) { final int dstParallelism = ((StageEdge) runtimeEdge).getDstIRVertex().getPropertyValue(ParallelismProperty.class) .orElseThrow(() -> new IllegalStateException()); final CommunicationPatternProperty.Value commPattern = ((StageEdge) runtimeEdge) .getPropertyValue(CommunicationPatternProperty.class) .orElseThrow(() -> new IllegalStateException()); return commPattern.equals(CommunicationPatternProperty.Value.OneToOne) ? 1 : dstParallelism; } }