@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { final List<IREdge> inEdges = dag.getIncomingEdgesOf(vertex); inEdges.forEach(edge -> { if (edge.getPropertyValue(CommunicationPatternProperty.class).get() .equals(CommunicationPatternProperty.Value.Shuffle)) { edge.setPropertyPermanently(DecoderProperty.of(BytesDecoderFactory.of())); } }); }); return dag; } }
/** * Static method exposing the constructor. * * @param value value of the new execution property. * @return the newly created execution property. */ public static DecoderProperty of(final DecoderFactory value) { return new DecoderProperty(value); } }
newIrEdge.setProperty(DecoderProperty.of(decoderProperty.get()));
/** * Static method exposing the constructor. * * @param value value of the new execution property. * @return the newly created execution property. */ public static DecoderProperty of(final DecoderFactory value) { return new DecoderProperty(value); } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { final List<IREdge> inEdges = dag.getIncomingEdgesOf(vertex); inEdges.forEach(edge -> { if (edge.getPropertyValue(CommunicationPatternProperty.class).get() .equals(CommunicationPatternProperty.Value.Shuffle)) { edge.setPropertyPermanently(DecoderProperty.of(BytesDecoderFactory.of())); } }); }); return dag; } }
/** * @param edge the original shuffle edge. * @param mcv the vertex with MetricCollectTransform. * @return the generated edge to {@code mcv}. */ private IREdge generateEdgeToMCV(final IREdge edge, final OperatorVertex mcv) { final IREdge newEdge = new IREdge(CommunicationPatternProperty.Value.OneToOne, edge.getSrc(), mcv); newEdge.setProperty(EncoderProperty.of(edge.getPropertyValue(EncoderProperty.class).get())); newEdge.setProperty(DecoderProperty.of(edge.getPropertyValue(DecoderProperty.class).get())); return newEdge; }
candidate.getKey(), edge.getDst()); newEdge.setProperty(EncoderProperty.of(edge.getPropertyValue(EncoderProperty.class).get())); newEdge.setProperty(DecoderProperty.of(edge.getPropertyValue(DecoderProperty.class).get())); edgesToAdd.add(newEdge); });
newIrEdge.setProperty(DecoderProperty.of(decoderProperty.get()));
candidate.getKey(), edge.getDst()); newEdge.setProperty(EncoderProperty.of(edge.getPropertyValue(EncoderProperty.class).get())); newEdge.setProperty(DecoderProperty.of(edge.getPropertyValue(DecoderProperty.class).get())); edgesToAdd.add(newEdge); });
/** * @param edge the original shuffle edge. * @param mcv the vertex with MetricCollectTransform. * @return the generated edge to {@code mcv}. */ private IREdge generateEdgeToMCV(final IREdge edge, final OperatorVertex mcv) { final IREdge newEdge = new IREdge(CommunicationPatternProperty.Value.OneToOne, edge.getSrc(), mcv); newEdge.setProperty(EncoderProperty.of(edge.getPropertyValue(EncoderProperty.class).get())); newEdge.setProperty(DecoderProperty.of(edge.getPropertyValue(DecoderProperty.class).get())); return newEdge; }
final DecoderFactory keyDecoderFactory = edge.getPropertyValue(KeyDecoderProperty.class).get(); newEdge.setProperty(EncoderProperty.of(PairEncoderFactory.of(keyEncoderFactory, LongEncoderFactory.of()))); newEdge.setProperty(DecoderProperty.of(PairDecoderFactory.of(keyDecoderFactory, LongDecoderFactory.of()))); } else { newEdge.setProperty(DecoderProperty.of(edge.getPropertyValue(DecoderProperty.class).get()));
iFileMergerVertex, v); newEdgeFromMerger.setProperty(EncoderProperty.of(edge.getPropertyValue(EncoderProperty.class).get())); newEdgeFromMerger.setProperty(DecoderProperty.of(edge.getPropertyValue(DecoderProperty.class).get())); builder.connectVertices(newEdgeToMerger); builder.connectVertices(newEdgeFromMerger);
iFileMergerVertex, v); newEdgeFromMerger.setProperty(EncoderProperty.of(edge.getPropertyValue(EncoderProperty.class).get())); newEdgeFromMerger.setProperty(DecoderProperty.of(edge.getPropertyValue(DecoderProperty.class).get())); builder.connectVertices(newEdgeToMerger); builder.connectVertices(newEdgeFromMerger);
EncoderProperty.of(PairEncoderFactory.of(keyEncoderFactory, LongEncoderFactory.of()))); newEdge.setPropertyPermanently( DecoderProperty.of(PairDecoderFactory.of(keyDecoderFactory, LongDecoderFactory.of()))); } else {
secondEdge.setProperty(DecoderProperty.of(new BeamDecoderFactory(sideInputElementCoder))); builder.connectVertices(secondEdge);
/** * @param edge IR edge to add. * @param elementCoder element coder. * @param windowCoder window coder. */ void addEdge(final IREdge edge, final Coder elementCoder, final Coder windowCoder) { edge.setProperty(KeyExtractorProperty.of(new BeamKeyExtractor())); if (elementCoder instanceof KvCoder) { Coder keyCoder = ((KvCoder) elementCoder).getKeyCoder(); edge.setProperty(KeyEncoderProperty.of(new BeamEncoderFactory(keyCoder))); edge.setProperty(KeyDecoderProperty.of(new BeamDecoderFactory(keyCoder))); } final WindowedValue.FullWindowedValueCoder coder = WindowedValue.getFullCoder(elementCoder, windowCoder); edge.setProperty(EncoderProperty.of(new BeamEncoderFactory<>(coder))); edge.setProperty(DecoderProperty.of(new BeamDecoderFactory<>(coder))); builder.connectVertices(edge); }
/** * Collect data by running the DAG. * * @param dag the DAG to execute. * @param loopVertexStack loop vertex stack. * @param lastVertex last vertex added to the dag. * @param serializer serializer for the edges. * @param <T> type of the return data. * @return the data collected. */ public static <T> List<T> collect(final DAG<IRVertex, IREdge> dag, final Stack<LoopVertex> loopVertexStack, final IRVertex lastVertex, final Serializer serializer) { final DAGBuilder<IRVertex, IREdge> builder = new DAGBuilder<>(dag); final IRVertex collectVertex = new OperatorVertex(new CollectTransform<>()); builder.addVertex(collectVertex, loopVertexStack); final IREdge newEdge = new IREdge(getEdgeCommunicationPattern(lastVertex, collectVertex), lastVertex, collectVertex); newEdge.setProperty(EncoderProperty.of(new SparkEncoderFactory(serializer))); newEdge.setProperty(DecoderProperty.of(new SparkDecoderFactory(serializer))); newEdge.setProperty(SPARK_KEY_EXTRACTOR_PROP); builder.connectVertices(newEdge); // launch DAG JobLauncher.launchDAG(builder.build(), SparkBroadcastVariables.getAll(), ""); return (List<T>) JobLauncher.getCollectedData(); }
map.put(DataFlowProperty.of(DataFlowProperty.Value.Pull)); map.put(EncoderProperty.of(EncoderFactory.DUMMY_ENCODER_FACTORY)); map.put(DecoderProperty.of(DecoderFactory.DUMMY_DECODER_FACTORY)); switch (commPattern) { case Shuffle:
shuffleEdgeBetweenT1AndT2.setProperty(KeyExtractorProperty.of(new DummyBeamKeyExtractor())); shuffleEdgeBetweenT1AndT2.setProperty(EncoderProperty.of(new EncoderFactory.DummyEncoderFactory())); shuffleEdgeBetweenT1AndT2.setProperty(DecoderProperty.of(new DecoderFactory.DummyDecoderFactory())); shuffleEdgeBetweenT3AndT4.setProperty(DecoderProperty.of(new DecoderFactory.DummyDecoderFactory()));
map.put(DataFlowProperty.of(DataFlowProperty.Value.Pull)); map.put(EncoderProperty.of(EncoderFactory.DUMMY_ENCODER_FACTORY)); map.put(DecoderProperty.of(DecoderFactory.DUMMY_DECODER_FACTORY)); switch (commPattern) { case Shuffle: