/** * Returns the buffer timeout of the job. * * @return buffer timeout (in milliseconds) */ public long getBufferTimeout() { return streamConfig.getBufferTimeout(); }
public boolean isFlushAlwaysEnabled() { return getBufferTimeout() == 0; }
builder.append("\nOperator: Missing"); builder.append("\nBuffer timeout: ").append(getBufferTimeout()); builder.append("\nState Monitoring: ").append(isCheckpointingEnabled()); if (isChainStart() && getChainedOutputs(cl).size() > 0) {
@VisibleForTesting public static <OUT> List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> createRecordWriters( StreamConfig configuration, Environment environment) { List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> recordWriters = new ArrayList<>(); List<StreamEdge> outEdgesInOrder = configuration.getOutEdgesInOrder(environment.getUserClassLoader()); Map<Integer, StreamConfig> chainedConfigs = configuration.getTransitiveChainedTaskConfigsWithSelf(environment.getUserClassLoader()); for (int i = 0; i < outEdgesInOrder.size(); i++) { StreamEdge edge = outEdgesInOrder.get(i); recordWriters.add( createRecordWriter( edge, i, environment, environment.getTaskInfo().getTaskName(), chainedConfigs.get(edge.getSourceId()).getBufferTimeout())); } return recordWriters; }
/** * Returns the buffer timeout of the job. * * @return buffer timeout (in milliseconds) */ public long getBufferTimeout() { return streamConfig.getBufferTimeout(); }
/** * Returns the buffer timeout of the job. * * @return buffer timeout (in milliseconds) */ public long getBufferTimeout() { return streamConfig.getBufferTimeout(); }
/** * Returns the buffer timeout of the job. * * @return buffer timeout (in milliseconds) */ public long getBufferTimeout() { return streamConfig.getBufferTimeout(); }
public boolean isFlushAlwaysEnabled() { return getBufferTimeout() == 0; }
public boolean isFlushAlwaysEnabled() { return getBufferTimeout() == 0; }
builder.append("\nOperator: Missing"); builder.append("\nBuffer timeout: ").append(getBufferTimeout()); builder.append("\nState Monitoring: ").append(isCheckpointingEnabled()); if (isChainStart() && getChainedOutputs(cl).size() > 0) {
builder.append("\nOperator: Missing"); builder.append("\nBuffer timeout: ").append(getBufferTimeout()); builder.append("\nState Monitoring: ").append(isCheckpointingEnabled()); if (isChainStart() && getChainedOutputs(cl).size() > 0) {
builder.append("\nOperator: Missing"); builder.append("\nBuffer timeout: ").append(getBufferTimeout()); builder.append("\nState Monitoring: ").append(isCheckpointingEnabled()); if (isChainStart() && getChainedOutputs(cl).size() > 0) {
@VisibleForTesting public static <OUT> List<StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>> createStreamRecordWriters( StreamConfig configuration, Environment environment) { List<StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>> streamRecordWriters = new ArrayList<>(); List<StreamEdge> outEdgesInOrder = configuration.getOutEdgesInOrder(environment.getUserClassLoader()); Map<Integer, StreamConfig> chainedConfigs = configuration.getTransitiveChainedTaskConfigsWithSelf(environment.getUserClassLoader()); for (int i = 0; i < outEdgesInOrder.size(); i++) { StreamEdge edge = outEdgesInOrder.get(i); streamRecordWriters.add( createStreamRecordWriter( edge, i, environment, environment.getTaskInfo().getTaskName(), chainedConfigs.get(edge.getSourceId()).getBufferTimeout())); } return streamRecordWriters; }
@VisibleForTesting public static <OUT> List<StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>> createStreamRecordWriters( StreamConfig configuration, Environment environment) { List<StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>> streamRecordWriters = new ArrayList<>(); List<StreamEdge> outEdgesInOrder = configuration.getOutEdgesInOrder(environment.getUserClassLoader()); Map<Integer, StreamConfig> chainedConfigs = configuration.getTransitiveChainedTaskConfigsWithSelf(environment.getUserClassLoader()); for (int i = 0; i < outEdgesInOrder.size(); i++) { StreamEdge edge = outEdgesInOrder.get(i); streamRecordWriters.add( createStreamRecordWriter( edge, i, environment, environment.getTaskInfo().getTaskName(), chainedConfigs.get(edge.getSourceId()).getBufferTimeout())); } return streamRecordWriters; }
private <T> RecordWriterOutput<T> createStreamOutput( StreamEdge edge, StreamConfig upStreamConfig, int outputIndex, Environment taskEnvironment, String taskName) { OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput TypeSerializer outSerializer = null; if (edge.getOutputTag() != null) { // side output outSerializer = upStreamConfig.getTypeSerializerSideOut( edge.getOutputTag(), taskEnvironment.getUserClassLoader()); } else { // main output outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader()); } @SuppressWarnings("unchecked") StreamPartitioner<T> outputPartitioner = (StreamPartitioner<T>) edge.getPartitioner(); LOG.debug("Using partitioner {} for output {} of task ", outputPartitioner, outputIndex, taskName); ResultPartitionWriter bufferWriter = taskEnvironment.getWriter(outputIndex); // we initialize the partitioner here with the number of key groups (aka max. parallelism) if (outputPartitioner instanceof ConfigurableStreamPartitioner) { int numKeyGroups = bufferWriter.getNumTargetKeyGroups(); if (0 < numKeyGroups) { ((ConfigurableStreamPartitioner) outputPartitioner).configure(numKeyGroups); } } StreamRecordWriter<SerializationDelegate<StreamRecord<T>>> output = new StreamRecordWriter<>(bufferWriter, outputPartitioner, upStreamConfig.getBufferTimeout()); output.setMetricGroup(taskEnvironment.getMetricGroup().getIOMetricGroup()); return new RecordWriterOutput<>(output, outSerializer, sideOutputTag, this); }