@Override public void emitWatermark(final Watermark watermark) { operatorVertex.getTransform().onWatermark(watermark); }
@Override public void emit(final O output) { nextOperatorVertex.getTransform().onData(output); }
@Override public void close() { ctxt.setSerializedData(Base64.getEncoder().encodeToString(SerializationUtils.serialize(list))); } }
private void closeTransform(final VertexHarness vertexHarness) { final IRVertex irVertex = vertexHarness.getIRVertex(); final Transform transform; if (irVertex instanceof OperatorVertex) { transform = ((OperatorVertex) irVertex).getTransform(); transform.close(); } vertexHarness.getContext().getSerializedData().ifPresent(data -> persistentConnectionToMasterMap.getMessageSender(MessageEnvironment.RUNTIME_MASTER_MESSAGE_LISTENER_ID).send( ControlMessage.Message.newBuilder() .setId(RuntimeIdManager.generateMessageId()) .setListenerId(MessageEnvironment.RUNTIME_MASTER_MESSAGE_LISTENER_ID) .setType(ControlMessage.MessageType.ExecutorDataCollected) .setDataCollected(ControlMessage.DataCollectMessage.newBuilder().setData(data).build()) .build())); }
private void prepareTransform(final VertexHarness vertexHarness) { final IRVertex irVertex = vertexHarness.getIRVertex(); final Transform transform; if (irVertex instanceof OperatorVertex) { transform = ((OperatorVertex) irVertex).getTransform(); transform.prepare(vertexHarness.getContext(), vertexHarness.getOutputCollector()); } }
/** * @return the generated vertex. */ private OperatorVertex generateMetricAggregationVertex() { // Define a custom data aggregator for skew handling. // Here, the aggregator gathers key frequency data used in shuffle data repartitioning. final BiFunction<Object, Map<Object, Long>, Map<Object, Long>> dynOptDataAggregator = (BiFunction<Object, Map<Object, Long>, Map<Object, Long>> & Serializable) (element, aggregatedDynOptData) -> { final Object key = ((Pair<Object, Long>) element).left(); final Long count = ((Pair<Object, Long>) element).right(); final Map<Object, Long> aggregatedDynOptDataMap = (Map<Object, Long>) aggregatedDynOptData; if (aggregatedDynOptDataMap.containsKey(key)) { aggregatedDynOptDataMap.compute(key, (existingKey, accumulatedCount) -> accumulatedCount + count); } else { aggregatedDynOptDataMap.put(key, count); } return aggregatedDynOptData; }; final AggregateMetricTransform abt = new AggregateMetricTransform<Pair<Object, Long>, Map<Object, Long>>(new HashMap<>(), dynOptDataAggregator); return new OperatorVertex(abt); }
= new MetricCollectTransform(new HashMap<>(), dynOptDataCollector, closer); return new OperatorVertex(mct);
final OperatorVertex iFileMergerVertex = new OperatorVertex(new RelayTransform());
private void closeTransform(final VertexHarness vertexHarness) { final IRVertex irVertex = vertexHarness.getIRVertex(); final Transform transform; if (irVertex instanceof OperatorVertex) { transform = ((OperatorVertex) irVertex).getTransform(); transform.close(); } vertexHarness.getContext().getSerializedData().ifPresent(data -> persistentConnectionToMasterMap.getMessageSender(MessageEnvironment.RUNTIME_MASTER_MESSAGE_LISTENER_ID).send( ControlMessage.Message.newBuilder() .setId(RuntimeIdManager.generateMessageId()) .setListenerId(MessageEnvironment.RUNTIME_MASTER_MESSAGE_LISTENER_ID) .setType(ControlMessage.MessageType.ExecutorDataCollected) .setDataCollected(ControlMessage.DataCollectMessage.newBuilder().setData(data).build()) .build())); }
private void emit(final OperatorVertex vertex, final O output) { vertex.getTransform().onData(output); }
@Override public void emitWatermark(final Watermark watermark) { operatorVertex.getTransform().onWatermark(watermark); }
private void prepareTransform(final VertexHarness vertexHarness) { final IRVertex irVertex = vertexHarness.getIRVertex(); final Transform transform; if (irVertex instanceof OperatorVertex) { transform = ((OperatorVertex) irVertex).getTransform(); transform.prepare(vertexHarness.getContext(), vertexHarness.getOutputCollector()); } }
/** * @return the generated vertex. */ private OperatorVertex generateMetricAggregationVertex() { // Define a custom data aggregator for skew handling. // Here, the aggregator gathers key frequency data used in shuffle data repartitioning. final BiFunction<Object, Map<Object, Long>, Map<Object, Long>> dynOptDataAggregator = (BiFunction<Object, Map<Object, Long>, Map<Object, Long>> & Serializable) (element, aggregatedDynOptData) -> { final Object key = ((Pair<Object, Long>) element).left(); final Long count = ((Pair<Object, Long>) element).right(); final Map<Object, Long> aggregatedDynOptDataMap = (Map<Object, Long>) aggregatedDynOptData; if (aggregatedDynOptDataMap.containsKey(key)) { aggregatedDynOptDataMap.compute(key, (existingKey, accumulatedCount) -> accumulatedCount + count); } else { aggregatedDynOptDataMap.put(key, count); } return aggregatedDynOptData; }; final AggregateMetricTransform abt = new AggregateMetricTransform<Pair<Object, Long>, Map<Object, Long>>(new HashMap<>(), dynOptDataAggregator); return new OperatorVertex(abt); }
= new MetricCollectTransform(new HashMap<>(), dynOptDataCollector, closer); return new OperatorVertex(mct);
final OperatorVertex iFileMergerVertex = new OperatorVertex(new RelayTransform());
@Override public void emit(final O output) { nextOperatorVertex.getTransform().onData(output); }
private void emit(final OperatorVertex vertex, final O output) { vertex.getTransform().onData(output); }