public void broadcastCheckpointBarrier(long id, long timestamp, CheckpointOptions checkpointOptions) throws IOException { CheckpointBarrier barrier = new CheckpointBarrier(id, timestamp, checkpointOptions); for (RecordWriterOutput<?> streamOutput : streamOutputs) { streamOutput.broadcastEvent(barrier); } }
/** * This method releases all resources of the record writer output. It stops the output * flushing thread (if there is one) and releases all buffers currently held by the output * serializers. * * <p>This method should never fail. */ public void releaseOutputs() { for (RecordWriterOutput<?> streamOutput : streamOutputs) { streamOutput.close(); } }
@Override public void toggleStreamStatus(StreamStatus status) { if (!status.equals(this.streamStatus)) { this.streamStatus = status; // try and forward the stream status change to all outgoing connections for (RecordWriterOutput<?> streamOutput : streamOutputs) { streamOutput.emitStreamStatus(status); } } }
synchronized (getCheckpointLock()) { for (RecordWriterOutput<OUT> output : outputs) { output.emitWatermark(new Watermark(Long.MAX_VALUE)); synchronized (getCheckpointLock()) { for (RecordWriterOutput<OUT> output : outputs) { output.collect(nextRecord);
@Override public void collect(StreamRecord<OUT> record) { if (this.outputTag != null) { // we are only responsible for emitting to the main input return; } pushToRecordWriter(record); }
/** * This method should be called before finishing the record emission, to make sure any data * that is still buffered will be sent. It also ensures that all data sending related * exceptions are recognized. * * @throws IOException Thrown, if the buffered data cannot be pushed into the output streams. */ public void flushOutputs() throws IOException { for (RecordWriterOutput<?> streamOutput : getStreamOutputs()) { streamOutput.flush(); } }
private RecordWriterOutput<OUT> createStreamOutput( RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter, StreamEdge edge, StreamConfig upStreamConfig, Environment taskEnvironment) { OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput TypeSerializer outSerializer = null; if (edge.getOutputTag() != null) { // side output outSerializer = upStreamConfig.getTypeSerializerSideOut( edge.getOutputTag(), taskEnvironment.getUserClassLoader()); } else { // main output outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader()); } return new RecordWriterOutput<>(recordWriter, outSerializer, sideOutputTag, this); }
/** * This method releases all resources of the record writer output. It stops the output * flushing thread (if there is one) and releases all buffers currently held by the output * serializers. * * <p>This method should never fail. */ public void releaseOutputs() { try { for (RecordWriterOutput<?> streamOutput : streamOutputs) { streamOutput.close(); } } finally { // make sure that we release the buffers in any case for (RecordWriterOutput<?> output : streamOutputs) { output.clearBuffers(); } } }
output.emitWatermark(new Watermark(Long.MAX_VALUE)); output.collect(nextRecord);
@Override public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) { if (this.outputTag == null || !this.outputTag.equals(outputTag)) { // we are only responsible for emitting to the side-output specified by our // OutputTag. return; } pushToRecordWriter(record); }
/** * This method should be called before finishing the record emission, to make sure any data * that is still buffered will be sent. It also ensures that all data sending related * exceptions are recognized. * * @throws IOException Thrown, if the buffered data cannot be pushed into the output streams. */ public void flushOutputs() throws IOException { for (RecordWriterOutput<?> streamOutput : getStreamOutputs()) { streamOutput.flush(); } }
private RecordWriterOutput<OUT> createStreamOutput( StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>> streamRecordWriter, StreamEdge edge, StreamConfig upStreamConfig, Environment taskEnvironment) { OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput TypeSerializer outSerializer = null; if (edge.getOutputTag() != null) { // side output outSerializer = upStreamConfig.getTypeSerializerSideOut( edge.getOutputTag(), taskEnvironment.getUserClassLoader()); } else { // main output outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader()); } return new RecordWriterOutput<>(streamRecordWriter, outSerializer, sideOutputTag, this); }
for (RecordWriterOutput<?> output : this.streamOutputs) { if (output != null) { output.close(); output.clearBuffers();
output.emitWatermark(new Watermark(Long.MAX_VALUE)); output.collect(nextRecord);
@Override public void collect(StreamRecord<OUT> record) { if (this.outputTag != null) { // we are only responsible for emitting to the main input return; } pushToRecordWriter(record); }
public void broadcastCheckpointCancelMarker(long id) throws IOException { CancelCheckpointMarker barrier = new CancelCheckpointMarker(id); for (RecordWriterOutput<?> streamOutput : streamOutputs) { streamOutput.broadcastEvent(barrier); } }
for (RecordWriterOutput<?> output : this.streamOutputs) { if (output != null) { output.close();
@Override public void toggleStreamStatus(StreamStatus status) { if (!status.equals(this.streamStatus)) { this.streamStatus = status; // try and forward the stream status change to all outgoing connections for (RecordWriterOutput<?> streamOutput : streamOutputs) { streamOutput.emitStreamStatus(status); } } }
/** * This method should be called before finishing the record emission, to make sure any data * that is still buffered will be sent. It also ensures that all data sending related * exceptions are recognized. * * @throws IOException Thrown, if the buffered data cannot be pushed into the output streams. */ public void flushOutputs() throws IOException { for (RecordWriterOutput<?> streamOutput : getStreamOutputs()) { streamOutput.flush(); } }
private RecordWriterOutput<OUT> createStreamOutput( StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>> streamRecordWriter, StreamEdge edge, StreamConfig upStreamConfig, Environment taskEnvironment) { OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput TypeSerializer outSerializer = null; if (edge.getOutputTag() != null) { // side output outSerializer = upStreamConfig.getTypeSerializerSideOut( edge.getOutputTag(), taskEnvironment.getUserClassLoader()); } else { // main output outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader()); } return new RecordWriterOutput<>(streamRecordWriter, outSerializer, sideOutputTag, this); }