public void broadcastEvent(AbstractEvent event) throws IOException { recordWriter.broadcastEvent(event); }
@Override public void emit(T record) throws IOException, InterruptedException { checkErroneous(); super.emit(record); if (flushAlways) { flush(); } }
public void clearBuffers() { recordWriter.clearBuffers(); } }
private static <OUT> StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>> createStreamRecordWriter( StreamEdge edge, int outputIndex, Environment environment, String taskName, long bufferTimeout) { @SuppressWarnings("unchecked") StreamPartitioner<OUT> outputPartitioner = (StreamPartitioner<OUT>) edge.getPartitioner(); LOG.debug("Using partitioner {} for output {} of task {}", outputPartitioner, outputIndex, taskName); ResultPartitionWriter bufferWriter = environment.getWriter(outputIndex); // we initialize the partitioner here with the number of key groups (aka max. parallelism) if (outputPartitioner instanceof ConfigurableStreamPartitioner) { int numKeyGroups = bufferWriter.getNumTargetKeyGroups(); if (0 < numKeyGroups) { ((ConfigurableStreamPartitioner) outputPartitioner).configure(numKeyGroups); } } StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>> output = new StreamRecordWriter<>(bufferWriter, outputPartitioner, bufferTimeout, taskName); output.setMetricGroup(environment.getMetricGroup().getIOMetricGroup()); return output; } }
public void emitStreamStatus(StreamStatus streamStatus) { serializationDelegate.setInstance(streamStatus); try { recordWriter.broadcastEmit(serializationDelegate); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } }
@Override public void emit(T record) throws IOException, InterruptedException { checkErroneous(); super.emit(record); }
@Override public void emitLatencyMarker(LatencyMarker latencyMarker) { serializationDelegate.setInstance(latencyMarker); try { recordWriter.randomEmit(serializationDelegate); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } }
private <X> void pushToRecordWriter(StreamRecord<X> record) { serializationDelegate.setInstance(record); try { recordWriter.emit(serializationDelegate); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } }
@Override public void close() { recordWriter.close(); }
public void flush() throws IOException { recordWriter.flushAll(); }
private static <OUT> StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>> createStreamRecordWriter( StreamEdge edge, int outputIndex, Environment environment, String taskName, long bufferTimeout) { @SuppressWarnings("unchecked") StreamPartitioner<OUT> outputPartitioner = (StreamPartitioner<OUT>) edge.getPartitioner(); LOG.debug("Using partitioner {} for output {} of task {}", outputPartitioner, outputIndex, taskName); ResultPartitionWriter bufferWriter = environment.getWriter(outputIndex); // we initialize the partitioner here with the number of key groups (aka max. parallelism) if (outputPartitioner instanceof ConfigurableStreamPartitioner) { int numKeyGroups = bufferWriter.getNumTargetKeyGroups(); if (0 < numKeyGroups) { ((ConfigurableStreamPartitioner) outputPartitioner).configure(numKeyGroups); } } StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>> output = new StreamRecordWriter<>(bufferWriter, outputPartitioner, bufferTimeout, taskName); output.setMetricGroup(environment.getMetricGroup().getIOMetricGroup()); return output; } }
public void emitStreamStatus(StreamStatus streamStatus) { serializationDelegate.setInstance(streamStatus); try { recordWriter.broadcastEmit(serializationDelegate); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } }
@Override public void randomEmit(T record) throws IOException, InterruptedException { checkErroneous(); super.randomEmit(record); }
@Override public void emitLatencyMarker(LatencyMarker latencyMarker) { serializationDelegate.setInstance(latencyMarker); try { recordWriter.randomEmit(serializationDelegate); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } }
private <X> void pushToRecordWriter(StreamRecord<X> record) { serializationDelegate.setInstance(record); try { recordWriter.emit(serializationDelegate); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } }
@Override public void close() { recordWriter.close(); }
public void flush() throws IOException { recordWriter.flushAll(); }
private <T> RecordWriterOutput<T> createStreamOutput( StreamEdge edge, StreamConfig upStreamConfig, int outputIndex, Environment taskEnvironment, String taskName) { OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput TypeSerializer outSerializer = null; if (edge.getOutputTag() != null) { // side output outSerializer = upStreamConfig.getTypeSerializerSideOut( edge.getOutputTag(), taskEnvironment.getUserClassLoader()); } else { // main output outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader()); } @SuppressWarnings("unchecked") StreamPartitioner<T> outputPartitioner = (StreamPartitioner<T>) edge.getPartitioner(); LOG.debug("Using partitioner {} for output {} of task ", outputPartitioner, outputIndex, taskName); ResultPartitionWriter bufferWriter = taskEnvironment.getWriter(outputIndex); // we initialize the partitioner here with the number of key groups (aka max. parallelism) if (outputPartitioner instanceof ConfigurableStreamPartitioner) { int numKeyGroups = bufferWriter.getNumTargetKeyGroups(); if (0 < numKeyGroups) { ((ConfigurableStreamPartitioner) outputPartitioner).configure(numKeyGroups); } } StreamRecordWriter<SerializationDelegate<StreamRecord<T>>> output = new StreamRecordWriter<>(bufferWriter, outputPartitioner, upStreamConfig.getBufferTimeout()); output.setMetricGroup(taskEnvironment.getMetricGroup().getIOMetricGroup()); return new RecordWriterOutput<>(output, outSerializer, sideOutputTag, this); }
public void emitStreamStatus(StreamStatus streamStatus) { serializationDelegate.setInstance(streamStatus); try { recordWriter.broadcastEmit(serializationDelegate); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } }
@Override public void emit(T record) throws IOException, InterruptedException { checkErroneous(); super.emit(record); }