@Override public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) { super.setup(containingTask, config, output); this.checkpointingLock = getContainingTask().getCheckpointLock(); this.inStreamElementSerializer = new StreamElementSerializer<>( getOperatorConfig().<IN>getTypeSerializerIn1(getUserCodeClassloader())); // create the operators executor for the complete operations of the queue entries this.executor = Executors.newSingleThreadExecutor(); switch (outputMode) { case ORDERED: queue = new OrderedStreamElementQueue( capacity, executor, this); break; case UNORDERED: queue = new UnorderedStreamElementQueue( capacity, executor, this); break; default: throw new IllegalStateException("Unknown async mode: " + outputMode + '.'); } }
ClassLoader userClassLoader = getUserCodeClassLoader(); TypeSerializer<IN1> inputDeserializer1 = configuration.getTypeSerializerIn1(userClassLoader); TypeSerializer<IN2> inputDeserializer2 = configuration.getTypeSerializerIn2(userClassLoader);
TypeSerializer<IN> inSerializer = operatorConfig.getTypeSerializerIn1(userCodeClassloader); currentOperatorOutput = new CopyingChainingOutput<>(chainedOperator, inSerializer, outputTag, this);
@Override public void init() throws Exception { StreamConfig configuration = getConfiguration(); TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator, getEnvironment().getMetricGroup().getIOMetricGroup(), inputWatermarkGauge); } headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge); // wrap watermark gauge since registered metrics must be unique getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge::getValue); }
@Override public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) { super.setup(containingTask, config, output); this.checkpointingLock = getContainingTask().getCheckpointLock(); this.inStreamElementSerializer = new StreamElementSerializer<>( getOperatorConfig().<IN>getTypeSerializerIn1(getUserCodeClassloader())); // create the operators executor for the complete operations of the queue entries this.executor = Executors.newSingleThreadExecutor(); switch (outputMode) { case ORDERED: queue = new OrderedStreamElementQueue( capacity, executor, this); break; case UNORDERED: queue = new UnorderedStreamElementQueue( capacity, executor, this); break; default: throw new IllegalStateException("Unknown async mode: " + outputMode + '.'); } }
@Override public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) { super.setup(containingTask, config, output); this.checkpointingLock = getContainingTask().getCheckpointLock(); this.inStreamElementSerializer = new StreamElementSerializer<>( getOperatorConfig().<IN>getTypeSerializerIn1(getUserCodeClassloader())); // create the operators executor for the complete operations of the queue entries this.executor = Executors.newSingleThreadExecutor(); switch (outputMode) { case ORDERED: queue = new OrderedStreamElementQueue( capacity, executor, this); break; case UNORDERED: queue = new UnorderedStreamElementQueue( capacity, executor, this); break; default: throw new IllegalStateException("Unknown async mode: " + outputMode + '.'); } }
@Override public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) { super.setup(containingTask, config, output); this.checkpointingLock = getContainingTask().getCheckpointLock(); this.inStreamElementSerializer = new StreamElementSerializer<>( getOperatorConfig().<IN>getTypeSerializerIn1(getUserCodeClassloader())); // create the operators executor for the complete operations of the queue entries this.executor = Executors.newSingleThreadExecutor(); switch (outputMode) { case ORDERED: queue = new OrderedStreamElementQueue( capacity, executor, this); break; case UNORDERED: queue = new UnorderedStreamElementQueue( capacity, executor, this); break; default: throw new IllegalStateException("Unknown async mode: " + outputMode + '.'); } }
private <IN, OUT> Output<StreamRecord<IN>> createChainedOperator( StreamTask<?, ?> containingTask, StreamConfig operatorConfig, Map<Integer, StreamConfig> chainedConfigs, ClassLoader userCodeClassloader, Map<StreamEdge, RecordWriterOutput<?>> streamOutputs, List<StreamOperator<?>> allOperators, OutputTag<IN> outputTag) { // create the output that the operator writes to first. this may recursively create more operators Output<StreamRecord<OUT>> output = createOutputCollector( containingTask, operatorConfig, chainedConfigs, userCodeClassloader, streamOutputs, allOperators); // now create the operator and give it the output collector to write its output to OneInputStreamOperator<IN, OUT> chainedOperator = operatorConfig.getStreamOperator(userCodeClassloader); chainedOperator.setup(containingTask, operatorConfig, output); allOperators.add(chainedOperator); if (containingTask.getExecutionConfig().isObjectReuseEnabled()) { return new ChainingOutput<>(chainedOperator, this, outputTag); } else { TypeSerializer<IN> inSerializer = operatorConfig.getTypeSerializerIn1(userCodeClassloader); return new CopyingChainingOutput<>(chainedOperator, inSerializer, outputTag, this); } }
@Override public void open() throws Exception { super.open(); inputSer = (AbstractRowSerializer) getOperatorConfig().getTypeSerializerIn1(getUserCodeClassloader()); comparator = (RecordComparator) CodeGenUtils.compile( getContainingTask().getUserCodeClassLoader(), gSorter.comparator().name(), gSorter.comparator().code()).newInstance(); comparator.init(gSorter.serializers(), gSorter.comparators()); gSorter = null; // reverse the comparision. heap = new PriorityQueue<>((int) limitEnd, new Comparator<BaseRow>() { @Override public int compare(BaseRow o1, BaseRow o2) { return comparator.compare(o2, o1); } }); this.collector = new StreamRecordCollector<>(output); }
ClassLoader userClassLoader = getUserCodeClassLoader(); TypeSerializer<IN1> inputDeserializer1 = configuration.getTypeSerializerIn1(userClassLoader); TypeSerializer<IN2> inputDeserializer2 = configuration.getTypeSerializerIn2(userClassLoader);
TypeSerializer<IN> inSerializer = operatorConfig.getTypeSerializerIn1(userCodeClassloader); currentOperatorOutput = new CopyingChainingOutput<>(chainedOperator, inSerializer, outputTag, this);
TypeSerializer<IN> inSerializer = operatorConfig.getTypeSerializerIn1(userCodeClassloader); currentOperatorOutput = new CopyingChainingOutput<>(chainedOperator, inSerializer, outputTag, this);
@Override public void open() throws Exception { super.open(); isFinished1 = false; isFinished2 = false; collector = new StreamRecordCollector<>(output); serializer1 = (AbstractRowSerializer) getOperatorConfig().getTypeSerializerIn1(getUserCodeClassloader()); serializer2 = (AbstractRowSerializer) getOperatorConfig().getTypeSerializerIn2(getUserCodeClassloader()); leftNullRow = new GenericRow(serializer1.getNumFields()); rightNullRow = new GenericRow(serializer2.getNumFields()); joinedRow = new JoinedRow(); CookedClasses classes = cookGeneratedClasses(getContainingTask().getUserCodeClassLoader()); condFunc = classes.condFuncClass.newInstance(); keyComparator = classes.keyComparatorClass.newInstance(); keyComparator.init(keyGSorter.serializers(), keyGSorter.comparators()); memManager = getContainingTask().getEnvironment().getMemoryManager(); ioManager = getContainingTask().getEnvironment().getIOManager(); Projection<BaseRow, BinaryRow> projection1 = classes.projectionClass1.newInstance(); Projection<BaseRow, BinaryRow> projection2 = classes.projectionClass2.newInstance(); int pageNum1 = (int) (leftBufferMemory / memManager.getPageSize()); List<MemorySegment> mem1 = memManager.allocatePages(getContainingTask(), pageNum1); buffer1 = new WrappedBuffer(mem1, serializer1, projection1); int pageNum2 = (int) (rightBufferMemory / memManager.getPageSize()); List<MemorySegment> mem2 = memManager.allocatePages(getContainingTask(), pageNum2); buffer2 = new WrappedBuffer(mem2, serializer2, projection2); initGauge(); initJoin(); }
@Override public void open() throws Exception { super.open(); LOG.info("Opening SortOperator"); cookGeneratedClasses(getContainingTask().getUserCodeClassLoader()); TypeSerializer<BaseRow> inputSerializer = getOperatorConfig().getTypeSerializerIn1(getUserCodeClassloader()); this.binarySerializer = new BinaryRowSerializer(((AbstractRowSerializer) inputSerializer).getTypes()); MemoryManager memManager = this.getContainingTask().getEnvironment().getMemoryManager(); IOManager ioManager = this.getContainingTask().getEnvironment().getIOManager(); NormalizedKeyComputer computer = computerClass.newInstance(); RecordComparator comparator = comparatorClass.newInstance(); computer.init(gSorter.serializers(), gSorter.comparators()); comparator.init(gSorter.serializers(), gSorter.comparators()); this.sorter = new BinaryExternalSorter(this.getContainingTask(), memManager, reservedMemorySize, maxMemorySize, perRequestMemorySize, ioManager, inputSerializer, binarySerializer, computer, comparator, getSqlConf()); this.sorter.startThreads(); gSorter = null; collector = new StreamRecordCollector<>(output); //register the the metrics. getMetricGroup().gauge("memoryUsedSizeInBytes", (Gauge<Long>) sorter::getUsedMemoryInBytes); getMetricGroup().gauge("numSpillFiles", (Gauge<Long>) sorter::getNumSpillFiles); getMetricGroup().gauge("spillInBytes", (Gauge<Long>) sorter::getSpillInBytes); }
@Override public void init() throws Exception { StreamConfig configuration = getConfiguration(); TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator); // make sure that stream tasks report their I/O statistics inputProcessor.setMetricGroup(getEnvironment().getMetricGroup().getIOMetricGroup()); } }
.getTypeSerializerIn1(getUserCodeClassloader()); final AbstractRowSerializer probeSerializer = (AbstractRowSerializer) getOperatorConfig() .getTypeSerializerIn2(getUserCodeClassloader());
@Override public void open() throws Exception { super.open(); isFinished1 = false; isFinished2 = false; classes = cookGeneratedClasses(getContainingTask().getUserCodeClassLoader()); Collector<BaseRow> collector = new StreamRecordCollector<>(output); inputSerializer1 = (AbstractRowSerializer) getOperatorConfig() .getTypeSerializerIn1(getUserCodeClassloader()); serializer1 = new BinaryRowSerializer(inputSerializer1.getTypes()); inputSerializer2 = (AbstractRowSerializer) getOperatorConfig() .getTypeSerializerIn2(getUserCodeClassloader()); serializer2 = new BinaryRowSerializer(inputSerializer2.getTypes()); memManager = this.getContainingTask().getEnvironment().getMemoryManager(); ioManager = this.getContainingTask().getEnvironment().getIOManager(); JoinConditionFunction condFunc = classes.condFuncClass.newInstance(); leftNullRow = new GenericRow(serializer1.getNumFields()); rightNullRow = new GenericRow(serializer2.getNumFields()); JoinedRow joinedRow = new JoinedRow(); helper = new SortMergeJoinHelper(collector, condFunc, leftNullRow, rightNullRow, joinedRow); initSorter(); probeBuffer = newBuffer(probeBufferMemory, leftNeedsSort ? inputSerializer2 : inputSerializer1); initGauge(); }
@Override public void init() throws Exception { StreamConfig configuration = getConfiguration(); TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator, getEnvironment().getMetricGroup().getIOMetricGroup(), inputWatermarkGauge); } headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge); // wrap watermark gauge since registered metrics must be unique getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge::getValue); }
@Override public void init() throws Exception { StreamConfig configuration = getConfiguration(); TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator, getEnvironment().getMetricGroup().getIOMetricGroup(), inputWatermarkGauge); } headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge); // wrap watermark gauge since registered metrics must be unique getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge::getValue); }
this.inputSerializer1 = getOperatorConfig().getTypeSerializerIn1(getUserCodeClassloader()); this.serializer1 = new BinaryRowSerializer(((AbstractRowSerializer) inputSerializer1).getTypes());