private void tryShutdownTimerService() { if (timerService != null && !timerService.isTerminated()) { try { final long timeoutMs = getEnvironment().getTaskManagerInfo().getConfiguration(). getLong(TaskManagerOptions.TASK_CANCELLATION_TIMEOUT_TIMERS); if (!timerService.shutdownServiceUninterruptible(timeoutMs)) { LOG.warn("Timer service shutdown exceeded time limit of {} ms while waiting for pending " + "timers. Will continue with shutdown procedure.", timeoutMs); } } catch (Throwable t) { // catch and log the exception to not replace the original exception LOG.error("Could not shut down timer service", t); } } }
private StateBackend createStateBackend() throws Exception { final StateBackend fromApplication = configuration.getStateBackend(getUserCodeClassLoader()); return StateBackendLoader.fromApplicationOrConfigOrDefault( fromApplication, getEnvironment().getTaskManagerInfo().getConfiguration(), getUserCodeClassLoader(), LOG); }
getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator,
Configuration taskManagerConfig = environment.getTaskManagerInfo().getConfiguration(); int historySize = taskManagerConfig.getInteger(MetricOptions.LATENCY_HISTORY_SIZE); if (historySize <= 0) {
@Override public void init() throws Exception { StreamConfig configuration = getConfiguration(); TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator, getEnvironment().getMetricGroup().getIOMetricGroup(), inputWatermarkGauge); } headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge); // wrap watermark gauge since registered metrics must be unique getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge::getValue); }
final Configuration configuration = this.getContainingTask().getEnvironment().getTaskManagerInfo().getConfiguration(); final long latencyTrackingInterval = getExecutionConfig().isLatencyTrackingConfigured() ? getExecutionConfig().getLatencyTrackingInterval()
private void tryShutdownTimerService() { if (timerService != null && !timerService.isTerminated()) { try { final long timeoutMs = getEnvironment().getTaskManagerInfo().getConfiguration(). getLong(TaskManagerOptions.TASK_CANCELLATION_TIMEOUT_TIMERS); if (!timerService.shutdownServiceUninterruptible(timeoutMs)) { LOG.warn("Timer service shutdown exceeded time limit of {} ms while waiting for pending " + "timers. Will continue with shutdown procedure.", timeoutMs); } } catch (Throwable t) { // catch and log the exception to not replace the original exception LOG.error("Could not shut down timer service", t); } } }
private void tryShutdownTimerService() { if (timerService != null && !timerService.isTerminated()) { try { final long timeoutMs = getEnvironment().getTaskManagerInfo().getConfiguration(). getLong(TaskManagerOptions.TASK_CANCELLATION_TIMEOUT_TIMERS); if (!timerService.shutdownServiceUninterruptible(timeoutMs)) { LOG.warn("Timer service shutdown exceeded time limit of {} ms while waiting for pending " + "timers. Will continue with shutdown procedure.", timeoutMs); } } catch (Throwable t) { // catch and log the exception to not replace the original exception LOG.error("Could not shut down timer service", t); } } }
private StateBackend createStateBackend() throws Exception { final StateBackend fromJob = configuration.getStateBackend(getUserCodeClassLoader()); if (fromJob != null) { // backend has been configured on the environment LOG.info("Using user-defined state backend: {}.", fromJob); return fromJob; } else { return AbstractStateBackend.loadStateBackendFromConfigOrCreateDefault( getEnvironment().getTaskManagerInfo().getConfiguration(), getUserCodeClassLoader(), LOG); } }
private StateBackend createStateBackend() throws Exception { final StateBackend fromApplication = configuration.getStateBackend(getUserCodeClassLoader()); return StateBackendLoader.fromApplicationOrConfigOrDefault( fromApplication, getEnvironment().getTaskManagerInfo().getConfiguration(), getUserCodeClassLoader(), LOG); }
private StateBackend createStateBackend() throws Exception { final StateBackend fromApplication = configuration.getStateBackend(getUserCodeClassLoader()); return StateBackendLoader.fromApplicationOrConfigOrDefault( fromApplication, getEnvironment().getTaskManagerInfo().getConfiguration(), getUserCodeClassLoader(), LOG); }
getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator);
getEnvironment().getInputGate(0), getEnvironment().getTaskManagerInfo().getTmpDirectories(), getEnvironment().getTaskManagerInfo().getConfiguration()); } else if (groupSize > 1){ new UnionInputGate(getEnvironment().getAllInputGates()), getEnvironment().getTaskManagerInfo().getTmpDirectories(), getEnvironment().getTaskManagerInfo().getConfiguration()); } else { throw new Exception("Illegal input group size in task configuration: " + groupSize);
getEnvironment().getInputGate(currentReaderOffset), getEnvironment().getTaskManagerInfo().getTmpDirectories(), getEnvironment().getTaskManagerInfo().getConfiguration()); } else if (groupSize > 1){ new UnionInputGate(readers), getEnvironment().getTaskManagerInfo().getTmpDirectories(), getEnvironment().getTaskManagerInfo().getConfiguration()); } else { throw new Exception("Illegal input group size in task configuration: " + groupSize);
@Override public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) { this.container = containingTask; this.config = config; this.metrics = container.getEnvironment().getMetricGroup().addOperator(config.getOperatorName()); this.output = new CountingOutput(output, ((OperatorMetricGroup) this.metrics).getIOMetricGroup().getNumRecordsOutCounter()); if (config.isChainStart()) { ((OperatorMetricGroup) this.metrics).getIOMetricGroup().reuseInputMetricsForTask(); } if (config.isChainEnd()) { ((OperatorMetricGroup) this.metrics).getIOMetricGroup().reuseOutputMetricsForTask(); } Configuration taskManagerConfig = container.getEnvironment().getTaskManagerInfo().getConfiguration(); int historySize = taskManagerConfig.getInteger(MetricOptions.LATENCY_HISTORY_SIZE); if (historySize <= 0) { LOG.warn("{} has been set to a value equal or below 0: {}. Using default.", MetricOptions.LATENCY_HISTORY_SIZE, historySize); historySize = MetricOptions.LATENCY_HISTORY_SIZE.defaultValue(); } latencyGauge = this.metrics.gauge("latency", new LatencyGauge(historySize)); this.runtimeContext = new StreamingRuntimeContext(this, container.getEnvironment(), container.getAccumulatorMap()); stateKeySelector1 = config.getStatePartitioner(0, getUserCodeClassloader()); stateKeySelector2 = config.getStatePartitioner(1, getUserCodeClassloader()); }
@Override public void init() throws Exception { StreamConfig configuration = getConfiguration(); TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator); // make sure that stream tasks report their I/O statistics inputProcessor.setMetricGroup(getEnvironment().getMetricGroup().getIOMetricGroup()); } }
@Override public void init() throws Exception { StreamConfig configuration = getConfiguration(); TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator, getEnvironment().getMetricGroup().getIOMetricGroup(), inputWatermarkGauge); } headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge); // wrap watermark gauge since registered metrics must be unique getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge::getValue); }
@Override public void init() throws Exception { StreamConfig configuration = getConfiguration(); TypeSerializer<IN> inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { InputGate[] inputGates = getEnvironment().getAllInputGates(); inputProcessor = new StreamInputProcessor<>( inputGates, inSerializer, this, configuration.getCheckpointMode(), getCheckpointLock(), getEnvironment().getIOManager(), getEnvironment().getTaskManagerInfo().getConfiguration(), getStreamStatusMaintainer(), this.headOperator, getEnvironment().getMetricGroup().getIOMetricGroup(), inputWatermarkGauge); } headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge); // wrap watermark gauge since registered metrics must be unique getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge::getValue); }
final Configuration configuration = this.getContainingTask().getEnvironment().getTaskManagerInfo().getConfiguration(); final long latencyTrackingInterval = getExecutionConfig().isLatencyTrackingConfigured() ? getExecutionConfig().getLatencyTrackingInterval()
final Configuration configuration = this.getContainingTask().getEnvironment().getTaskManagerInfo().getConfiguration(); final long latencyTrackingInterval = getExecutionConfig().isLatencyTrackingConfigured() ? getExecutionConfig().getLatencyTrackingInterval()