How to use

getConfig

method

in

org.apache.flink.streaming.api.environment.StreamExecutionEnvironment

public ExecutionConfig getExecutionConfig() {
  return environment.getConfig();
}

public StreamGraph(StreamExecutionEnvironment environment) {
  this.environment = environment;
  this.executionConfig = environment.getConfig();
  this.checkpointConfig = environment.getCheckpointConfig();
  // create an empty new stream graph.
  clear();
}

/**
 * Sets the time characteristic for all streams create from this environment, e.g., processing
 * time, event time, or ingestion time.
 *
 * <p>If you set the characteristic to IngestionTime of EventTime this will set a default
 * watermark update interval of 200 ms. If this is not applicable for your application
 * you should change it using {@link ExecutionConfig#setAutoWatermarkInterval(long)}.
 *
 * @param characteristic The time characteristic.
 */
@PublicEvolving
public void setStreamTimeCharacteristic(TimeCharacteristic characteristic) {
  this.timeCharacteristic = Preconditions.checkNotNull(characteristic);
  if (characteristic == TimeCharacteristic.ProcessingTime) {
    getConfig().setAutoWatermarkInterval(0);
  } else {
    getConfig().setAutoWatermarkInterval(200);
  }
}

public static StreamExecutionEnvironment prepareExecutionEnv(ParameterTool parameterTool)
  throws Exception {
  if (parameterTool.getNumberOfParameters() < 5) {
    System.out.println("Missing parameters!\n" +
      "Usage: Kafka --input-topic <topic> --output-topic <topic> " +
      "--bootstrap.servers <kafka brokers> " +
      "--zookeeper.connect <zk quorum> --group.id <some id>");
    throw new Exception("Missing parameters!\n" +
      "Usage: Kafka --input-topic <topic> --output-topic <topic> " +
      "--bootstrap.servers <kafka brokers> " +
      "--zookeeper.connect <zk quorum> --group.id <some id>");
  }
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000));
  env.enableCheckpointing(5000); // create a checkpoint every 5 seconds
  env.getConfig().setGlobalJobParameters(parameterTool); // make parameters available in the web interface
  env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
  return env;
}

PythonStreamExecutionEnvironment(StreamExecutionEnvironment env, Path tmpLocalDir, String scriptName) {
  this.env = env;
  this.pythonTmpCachePath = tmpLocalDir;
  env.getConfig().setGlobalJobParameters(new PythonJobParameters(scriptName));
  registerJythonSerializers(this.env);
}

/**
 * Returns a "closure-cleaned" version of the given function. Cleans only if closure cleaning
 * is not disabled in the {@link org.apache.flink.api.common.ExecutionConfig}
 */
@Internal
public <F> F clean(F f) {
  if (getConfig().isClosureCleanerEnabled()) {
    ClosureCleaner.clean(f, true);
  }
  ClosureCleaner.ensureSerializable(f);
  return f;
}

@Test
public void testMultiChainingWithObjectReuse() throws Exception {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().enableObjectReuse();
  testMultiChaining(env);
}

@Test
public void testMultiChainingWithSplitWithoutObjectReuse() throws Exception {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableObjectReuse();
  testMultiChainingWithSplit(env);
}

@Test
public void testMultiChainingWithSplitWithObjectReuse() throws Exception {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().enableObjectReuse();
  testMultiChainingWithSplit(env);
}

@Test
public void testMultiChainingWithoutObjectReuse() throws Exception {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableObjectReuse();
  testMultiChaining(env);
}

  @Test
  public void testOperatorChainWithObjectReuseAndNoOutputOperators() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().enableObjectReuse();
    DataStream<Integer> input = env.fromElements(1, 2, 3);
    input.flatMap(new FlatMapFunction<Integer, Integer>() {
      @Override
      public void flatMap(Integer value, Collector<Integer> out) throws Exception {
        out.collect(value << 1);
      }
    });
    env.execute();
  }
}

/**
 * Creates a streaming JobGraph from the StreamEnvironment.
 */
private JobGraph createJobGraph(
  int parallelism,
  int numberOfRetries,
  long restartDelay) {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setParallelism(parallelism);
  env.disableOperatorChaining();
  env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(numberOfRetries, restartDelay));
  env.getConfig().disableSysoutLogging();
  DataStream<Integer> stream = env
    .addSource(new InfiniteTestSource())
    .shuffle()
    .map(new StatefulCounter());
  stream.addSink(new DiscardingSink<>());
  return env.getStreamGraph().getJobGraph();
}

  public static void main(String[] args) throws Exception {
    final ParameterTool pt = ParameterTool.fromArgs(args);
    final String checkpointDir = pt.getRequired("checkpoint.dir");

    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(new FsStateBackend(checkpointDir));
    env.setRestartStrategy(RestartStrategies.noRestart());
    env.enableCheckpointing(1000L);
    env.getConfig().disableGenericTypes();

    env.addSource(new MySource()).uid("my-source")
        .keyBy(anInt -> 0)
        .map(new MyStatefulFunction()).uid("my-map")
        .addSink(new DiscardingSink<>()).uid("my-sink");
    env.execute();
  }
}

public static void main(String[] args) throws Exception {
  ParameterTool params = ParameterTool.fromArgs(args);
  String outputPath = params.getRequired("outputPath");
  int recordsPerSecond = params.getInt("recordsPerSecond", 10);
  int duration = params.getInt("durationInSecond", 60);
  int offset = params.getInt("offsetInSecond", 0);
  StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment();
  sEnv.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
  sEnv.enableCheckpointing(4000);
  sEnv.getConfig().setAutoWatermarkInterval(1000);
  // execute a simple pass through program.
  PeriodicSourceGenerator generator = new PeriodicSourceGenerator(
    recordsPerSecond, duration, offset);
  DataStream<Tuple> rows = sEnv.addSource(generator);
  DataStream<Tuple> result = rows
    .keyBy(1)
    .timeWindow(Time.seconds(5))
    .sum(0);
  result.writeAsText(outputPath + "/result.txt", FileSystem.WriteMode.OVERWRITE)
    .setParallelism(1);
  sEnv.execute();
}

/**
 * This verifies that an event time source works when setting stream time characteristic to
 * processing time. In this case, the watermarks should just be swallowed.
 */
@Test
public void testEventTimeSourceWithProcessingTime() throws Exception {
  StreamExecutionEnvironment env =
      StreamExecutionEnvironment.getExecutionEnvironment();
  env.setParallelism(2);
  env.getConfig().disableSysoutLogging();
  env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
  DataStream<Integer> source1 = env.addSource(new MyTimestampSource(0, 10));
  source1
    .map(new IdentityMap())
    .transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(false));
  env.execute();
  // verify that we don't get any watermarks, the source is used as watermark source in
  // other tests, so it normally emits watermarks
  Assert.assertTrue(CustomOperator.finalWatermarks[0].size() == 0);
}

/**
 * Tests that the KeyGroupStreamPartitioner are properly set up with the correct value of
 * maximum parallelism.
 */
@Test
public void testSetupOfKeyGroupPartitioner() {
  int maxParallelism = 42;
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().setMaxParallelism(maxParallelism);
  DataStream<Integer> source = env.fromElements(1, 2, 3);
  DataStream<Integer> keyedResult = source.keyBy(value -> value).map(new NoOpIntMap());
  keyedResult.addSink(new DiscardingSink<>());
  StreamGraph graph = env.getStreamGraph();
  StreamNode keyedResultNode = graph.getStreamNode(keyedResult.getId());
  StreamPartitioner<?> streamPartitioner = keyedResultNode.getInEdges().get(0).getPartitioner();
}

  /**
   * Checks that in a streaming use case where checkpointing is enabled and the number
   * of execution retries is set to 42 and the delay to 1337, fixed delay restarting is used.
   */
  @Test
  public void testFixedRestartingWhenCheckpointingAndExplicitExecutionRetriesNonZero() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.enableCheckpointing(500);
    env.setNumberOfExecutionRetries(42);
    env.getConfig().setExecutionRetryDelay(1337);

    env.fromElements(1).print();

    StreamGraph graph = env.getStreamGraph();
    JobGraph jobGraph = graph.getJobGraph();

    RestartStrategies.RestartStrategyConfiguration restartStrategy =
      jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

    Assert.assertNotNull(restartStrategy);
    Assert.assertTrue(restartStrategy instanceof RestartStrategies.FixedDelayRestartStrategyConfiguration);
    Assert.assertEquals(42, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getRestartAttempts());
    Assert.assertEquals(1337, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getDelayBetweenAttemptsInterval().toMilliseconds());
  }
}

private static void runPartitioningProgram(int parallelism) throws Exception {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setParallelism(parallelism);
  env.getConfig().enableObjectReuse();
  env.setBufferTimeout(5L);
  env.enableCheckpointing(1000, CheckpointingMode.AT_LEAST_ONCE);
  env
    .addSource(new TimeStampingSource())
    .map(new IdMapper<Tuple2<Long, Long>>())
    .keyBy(0)
    .addSink(new TimestampingSink());
  env.execute("Partitioning Program");
}

/**
 * These check whether timestamps are properly ignored when they are disabled.
 */
@Test
public void testDisabledTimestamps() throws Exception {
  final int numElements = 10;
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
  env.setParallelism(PARALLELISM);
  env.getConfig().disableSysoutLogging();
  DataStream<Integer> source1 = env.addSource(new MyNonWatermarkingSource(numElements));
  DataStream<Integer> source2 = env.addSource(new MyNonWatermarkingSource(numElements));
  source1
      .map(new IdentityMap())
      .connect(source2).map(new IdentityCoMap())
      .transform("Custom Operator", BasicTypeInfo.INT_TYPE_INFO, new DisabledTimestampCheckingOperator())
      .addSink(new DiscardingSink<Integer>());
  env.execute();
}

/**
 * These check whether timestamps are properly assigned at the sources and handled in
 * network transmission and between chained operators when timestamps are enabled.
 */
@Test
public void testTimestampHandling() throws Exception {
  final int numElements = 10;
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
  env.setParallelism(PARALLELISM);
  env.getConfig().disableSysoutLogging();
  DataStream<Integer> source1 = env.addSource(new MyTimestampSource(0L, numElements));
  DataStream<Integer> source2 = env.addSource(new MyTimestampSource(0L, numElements));
  source1
      .map(new IdentityMap())
      .connect(source2).map(new IdentityCoMap())
      .transform("Custom Operator", BasicTypeInfo.INT_TYPE_INFO, new TimestampCheckingOperator())
      .addSink(new DiscardingSink<Integer>());
  env.execute();
}