numRecordsIn = ((OperatorMetricGroup) streamOperator.getMetricGroup()).getIOMetricGroup().getNumRecordsInCounter(); } catch (Exception e) { LOG.warn("An exception occurred during the metrics setup.", e); streamOperator.processLatencyMarker(recordOrMark.asLatencyMarker()); synchronized (lock) { numRecordsIn.inc(); streamOperator.setKeyContextElement1(record); streamOperator.processElement(record);
public void processWatermark(Watermark mark) throws Exception { currentWatermark = mark.getTimestamp(); oneInputOperator.processWatermark(mark); }
chainedOperator.setup(containingTask, operatorConfig, chainedOperatorOutput); chainedOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, currentOperatorOutput.getWatermarkGauge()::getValue); chainedOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_OUTPUT_WATERMARK, chainedOperatorOutput.getWatermarkGauge()::getValue);
protected <X> void pushToOperator(StreamRecord<X> record) { try { // we know that the given outputTag matches our OutputTag so the record // must be of the type that our operator expects. @SuppressWarnings("unchecked") StreamRecord<T> castRecord = (StreamRecord<T>) record; numRecordsIn.inc(); operator.setKeyContextElement1(castRecord); operator.processElement(castRecord); } catch (Exception e) { throw new ExceptionInChainedOperatorException(e); } }
@Override public void init() throws Exception { final String iterationId = getConfiguration().getIterationId(); if (iterationId == null || iterationId.length() == 0) { throw new Exception("Missing iteration ID in the task configuration"); } final String brokerID = StreamIterationHead.createBrokerIdString(getEnvironment().getJobID(), iterationId, getEnvironment().getTaskInfo().getIndexOfThisSubtask()); final long iterationWaitTime = getConfiguration().getIterationWaitTime(); LOG.info("Iteration tail {} trying to acquire feedback queue under {}", getName(), brokerID); @SuppressWarnings("unchecked") BlockingQueue<StreamRecord<IN>> dataChannel = (BlockingQueue<StreamRecord<IN>>) BlockingQueueBroker.INSTANCE.get(brokerID); LOG.info("Iteration tail {} acquired feedback queue {}", getName(), brokerID); this.headOperator = new RecordPusher<>(); this.headOperator.setup(this, getConfiguration(), new IterationTailOutput<>(dataChannel, iterationWaitTime)); // call super.init() last because that needs this.headOperator to be set up super.init(); }
public ChainingOutput( OneInputStreamOperator<T, ?> operator, StreamStatusProvider streamStatusProvider, @Nullable OutputTag<T> outputTag) { this.operator = operator; { Counter tmpNumRecordsIn; try { OperatorIOMetricGroup ioMetricGroup = ((OperatorMetricGroup) operator.getMetricGroup()).getIOMetricGroup(); tmpNumRecordsIn = ioMetricGroup.getNumRecordsInCounter(); } catch (Exception e) { LOG.warn("An exception occurred during the metrics setup.", e); tmpNumRecordsIn = new SimpleCounter(); } numRecordsIn = tmpNumRecordsIn; } this.streamStatusProvider = streamStatusProvider; this.outputTag = outputTag; }
@Override public final void setChainingStrategy(ChainingStrategy strategy) { operator.setChainingStrategy(strategy); } }
@Override public void close() { try { operator.close(); } catch (Exception e) { throw new ExceptionInChainedOperatorException(e); } }
@Override public void emitLatencyMarker(LatencyMarker latencyMarker) { try { operator.processLatencyMarker(latencyMarker); } catch (Exception e) { throw new ExceptionInChainedOperatorException(e); } }
public void processElements(Collection<StreamRecord<IN>> elements) throws Exception { for (StreamRecord<IN> element: elements) { operator.setKeyContextElement1(element); oneInputOperator.processElement(element); } }
@Override protected <X> void pushToOperator(StreamRecord<X> record) { try { // we know that the given outputTag matches our OutputTag so the record // must be of the type that our operator (and Serializer) expects. @SuppressWarnings("unchecked") StreamRecord<T> castRecord = (StreamRecord<T>) record; numRecordsIn.inc(); StreamRecord<T> copy = castRecord.copy(serializer.copy(castRecord.getValue())); operator.setKeyContextElement1(copy); operator.processElement(copy); } catch (ClassCastException e) { if (outputTag != null) { // Enrich error message ClassCastException replace = new ClassCastException( String.format( "%s. Failed to push OutputTag with id '%s' to operator. " + "This can occur when multiple OutputTags with different types " + "but identical names are being used.", e.getMessage(), outputTag.getId())); throw new ExceptionInChainedOperatorException(replace); } else { throw new ExceptionInChainedOperatorException(e); } } catch (Exception e) { throw new ExceptionInChainedOperatorException(e); } } }
@SafeVarargs private static <T, OP extends StreamOperator<T>> OperatorChain<T, OP> setupOperatorChain( OneInputStreamOperator<T, T>... operators) { checkNotNull(operators); checkArgument(operators.length > 0); try (MockEnvironment env = MockEnvironment.builder().build()) { final StreamTask<?, ?> containingTask = new OneInputStreamTask<T, OneInputStreamOperator<T, T>>(env); final StreamStatusProvider statusProvider = mock(StreamStatusProvider.class); final StreamConfig cfg = new StreamConfig(new Configuration()); final StreamOperator<?>[] ops = new StreamOperator<?>[operators.length]; // initial output goes to nowhere @SuppressWarnings({"unchecked", "rawtypes"}) WatermarkGaugeExposingOutput<StreamRecord<T>> lastWriter = new BroadcastingOutputCollector<>( new Output[0], statusProvider); // build the reverse operators array for (int i = 0; i < ops.length; i++) { OneInputStreamOperator<T, T> op = operators[ops.length - i - 1]; op.setup(containingTask, cfg, lastWriter); lastWriter = new ChainingOutput<>(op, statusProvider, null); ops[i] = op; } @SuppressWarnings("unchecked") final OP head = (OP) operators[0]; return new OperatorChain<>( ops, new RecordWriterOutput<?>[0], lastWriter, head); } }
@Override public void init() throws Exception { StreamConfig configuration = getConfiguration(); TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator, getEnvironment().getMetricGroup().getIOMetricGroup(), inputWatermarkGauge); } headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge); // wrap watermark gauge since registered metrics must be unique getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge::getValue); }
@Override public final void setChainingStrategy(ChainingStrategy strategy) { operator.setChainingStrategy(strategy); } }
@Override public void close() { try { operator.close(); } catch (Exception e) { throw new ExceptionInChainedOperatorException(e); } }
@Override public void emitLatencyMarker(LatencyMarker latencyMarker) { try { operator.processLatencyMarker(latencyMarker); } catch (Exception e) { throw new ExceptionInChainedOperatorException(e); } }
public void processElement(StreamRecord<IN> element) throws Exception { operator.setKeyContextElement1(element); oneInputOperator.processElement(element); }
numRecordsIn = ((OperatorMetricGroup) streamOperator.getMetricGroup()).getIOMetricGroup().getNumRecordsInCounter(); } catch (Exception e) { LOG.warn("An exception occurred during the metrics setup.", e); streamOperator.processLatencyMarker(recordOrMark.asLatencyMarker()); synchronized (lock) { numRecordsIn.inc(); streamOperator.setKeyContextElement1(record); streamOperator.processElement(record);
protected <X> void pushToOperator(StreamRecord<X> record) { try { // we know that the given outputTag matches our OutputTag so the record // must be of the type that our operator expects. @SuppressWarnings("unchecked") StreamRecord<T> castRecord = (StreamRecord<T>) record; numRecordsIn.inc(); operator.setKeyContextElement1(castRecord); operator.processElement(castRecord); } catch (Exception e) { throw new ExceptionInChainedOperatorException(e); } }
chainedOperator.setup(containingTask, operatorConfig, chainedOperatorOutput); chainedOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, currentOperatorOutput.getWatermarkGauge()::getValue); chainedOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_OUTPUT_WATERMARK, chainedOperatorOutput.getWatermarkGauge()::getValue);