@Override public void init() throws Exception { final String iterationId = getConfiguration().getIterationId(); if (iterationId == null || iterationId.length() == 0) { throw new Exception("Missing iteration ID in the task configuration"); } final String brokerID = StreamIterationHead.createBrokerIdString(getEnvironment().getJobID(), iterationId, getEnvironment().getTaskInfo().getIndexOfThisSubtask()); final long iterationWaitTime = getConfiguration().getIterationWaitTime(); LOG.info("Iteration tail {} trying to acquire feedback queue under {}", getName(), brokerID); @SuppressWarnings("unchecked") BlockingQueue<StreamRecord<IN>> dataChannel = (BlockingQueue<StreamRecord<IN>>) BlockingQueueBroker.INSTANCE.get(brokerID); LOG.info("Iteration tail {} acquired feedback queue {}", getName(), brokerID); this.headOperator = new RecordPusher<>(); this.headOperator.setup(this, getConfiguration(), new IterationTailOutput<>(dataChannel, iterationWaitTime)); // call super.init() last because that needs this.headOperator to be set up super.init(); }
chainedOperator.setup(containingTask, operatorConfig, chainedOperatorOutput);
@SafeVarargs private static <T, OP extends StreamOperator<T>> OperatorChain<T, OP> setupOperatorChain( OneInputStreamOperator<T, T>... operators) { checkNotNull(operators); checkArgument(operators.length > 0); try (MockEnvironment env = MockEnvironment.builder().build()) { final StreamTask<?, ?> containingTask = new OneInputStreamTask<T, OneInputStreamOperator<T, T>>(env); final StreamStatusProvider statusProvider = mock(StreamStatusProvider.class); final StreamConfig cfg = new StreamConfig(new Configuration()); final StreamOperator<?>[] ops = new StreamOperator<?>[operators.length]; // initial output goes to nowhere @SuppressWarnings({"unchecked", "rawtypes"}) WatermarkGaugeExposingOutput<StreamRecord<T>> lastWriter = new BroadcastingOutputCollector<>( new Output[0], statusProvider); // build the reverse operators array for (int i = 0; i < ops.length; i++) { OneInputStreamOperator<T, T> op = operators[ops.length - i - 1]; op.setup(containingTask, cfg, lastWriter); lastWriter = new ChainingOutput<>(op, statusProvider, null); ops[i] = op; } @SuppressWarnings("unchecked") final OP head = (OP) operators[0]; return new OperatorChain<>( ops, new RecordWriterOutput<?>[0], lastWriter, head); } }
private <IN, OUT> Output<StreamRecord<IN>> createChainedOperator( StreamTask<?, ?> containingTask, StreamConfig operatorConfig, Map<Integer, StreamConfig> chainedConfigs, ClassLoader userCodeClassloader, Map<StreamEdge, RecordWriterOutput<?>> streamOutputs, List<StreamOperator<?>> allOperators, OutputTag<IN> outputTag) { // create the output that the operator writes to first. this may recursively create more operators Output<StreamRecord<OUT>> output = createOutputCollector( containingTask, operatorConfig, chainedConfigs, userCodeClassloader, streamOutputs, allOperators); // now create the operator and give it the output collector to write its output to OneInputStreamOperator<IN, OUT> chainedOperator = operatorConfig.getStreamOperator(userCodeClassloader); chainedOperator.setup(containingTask, operatorConfig, output); allOperators.add(chainedOperator); if (containingTask.getExecutionConfig().isObjectReuseEnabled()) { return new ChainingOutput<>(chainedOperator, this, outputTag); } else { TypeSerializer<IN> inSerializer = operatorConfig.getTypeSerializerIn1(userCodeClassloader); return new CopyingChainingOutput<>(chainedOperator, inSerializer, outputTag, this); } }
@Override public void init() throws Exception { final String iterationId = getConfiguration().getIterationId(); if (iterationId == null || iterationId.length() == 0) { throw new Exception("Missing iteration ID in the task configuration"); } final String brokerID = StreamIterationHead.createBrokerIdString(getEnvironment().getJobID(), iterationId, getEnvironment().getTaskInfo().getIndexOfThisSubtask()); final long iterationWaitTime = getConfiguration().getIterationWaitTime(); LOG.info("Iteration tail {} trying to acquire feedback queue under {}", getName(), brokerID); @SuppressWarnings("unchecked") BlockingQueue<StreamRecord<IN>> dataChannel = (BlockingQueue<StreamRecord<IN>>) BlockingQueueBroker.INSTANCE.get(brokerID); LOG.info("Iteration tail {} acquired feedback queue {}", getName(), brokerID); this.headOperator = new RecordPusher<>(); this.headOperator.setup(this, getConfiguration(), new IterationTailOutput<>(dataChannel, iterationWaitTime)); // call super.init() last because that needs this.headOperator to be set up super.init(); }
@Override public void init() throws Exception { final String iterationId = getConfiguration().getIterationId(); if (iterationId == null || iterationId.length() == 0) { throw new Exception("Missing iteration ID in the task configuration"); } final String brokerID = StreamIterationHead.createBrokerIdString(getEnvironment().getJobID(), iterationId, getEnvironment().getTaskInfo().getIndexOfThisSubtask()); final long iterationWaitTime = getConfiguration().getIterationWaitTime(); LOG.info("Iteration tail {} trying to acquire feedback queue under {}", getName(), brokerID); @SuppressWarnings("unchecked") BlockingQueue<StreamRecord<IN>> dataChannel = (BlockingQueue<StreamRecord<IN>>) BlockingQueueBroker.INSTANCE.get(brokerID); LOG.info("Iteration tail {} acquired feedback queue {}", getName(), brokerID); this.headOperator = new RecordPusher<>(); this.headOperator.setup(this, getConfiguration(), new IterationTailOutput<>(dataChannel, iterationWaitTime)); // call super.init() last because that needs this.headOperator to be set up super.init(); }
@Override public void init() throws Exception { final String iterationId = getConfiguration().getIterationId(); if (iterationId == null || iterationId.length() == 0) { throw new Exception("Missing iteration ID in the task configuration"); } final String brokerID = StreamIterationHead.createBrokerIdString(getEnvironment().getJobID(), iterationId, getEnvironment().getTaskInfo().getIndexOfThisSubtask()); final long iterationWaitTime = getConfiguration().getIterationWaitTime(); LOG.info("Iteration tail {} trying to acquire feedback queue under {}", getName(), brokerID); @SuppressWarnings("unchecked") BlockingQueue<StreamRecord<IN>> dataChannel = (BlockingQueue<StreamRecord<IN>>) BlockingQueueBroker.INSTANCE.get(brokerID); LOG.info("Iteration tail {} acquired feedback queue {}", getName(), brokerID); this.headOperator = new RecordPusher<>(); this.headOperator.setup(this, getConfiguration(), new IterationTailOutput<>(dataChannel, iterationWaitTime)); // call super.init() last because that needs this.headOperator to be set up super.init(); }
chainedOperator.setup(containingTask, operatorConfig, chainedOperatorOutput);
chainedOperator.setup(containingTask, operatorConfig, chainedOperatorOutput);