private static void checkLimit(int value, ConfigOption<Integer> option) { if (value < -1) { throw new IllegalConfigurationException("Invalid value for '" + option.key() + "': " + value); } }
private static void checkTimeout(long timeout, ConfigOption<Long> option) { if (timeout < 0) { throw new IllegalConfigurationException("Invalid value for '" + option.key() + "': " + timeout); } } }
private static boolean isOriginalJobVariant(final ParameterTool pt) { switch (pt.get(TEST_JOB_VARIANT.key())) { case TEST_JOB_VARIANT_ORIGINAL: return true; case TEST_JOB_VARIANT_UPGRADED: return false; default: throw new IllegalArgumentException(String.format("'--test.job.variant' can be either '%s' or '%s'", TEST_JOB_VARIANT_ORIGINAL, TEST_JOB_VARIANT_UPGRADED)); } }
@Override public void setBoolean(ConfigOption<Boolean> key, boolean value) { this.backingConfig.setBoolean(prefix + key.key(), value); }
@Override public void setInteger(ConfigOption<Integer> key, int value) { this.backingConfig.setInteger(prefix + key.key(), value); }
@Override public void setString(ConfigOption<String> key, String value) { this.backingConfig.setString(prefix + key.key(), value); }
private void loggingFallback(FallbackKey fallbackKey, ConfigOption<?> configOption) { if (fallbackKey.isDeprecated()) { LOG.warn("Config uses deprecated configuration key '{}' instead of proper key '{}'", fallbackKey.getKey(), configOption.key()); } else { LOG.info("Config uses fallback configuration key '{}' instead of key '{}'", fallbackKey.getKey(), configOption.key()); } }
@Override public void open(MetricConfig config) { boolean filterLabelValueCharacters = config.getBoolean( FILTER_LABEL_VALUE_CHARACTER.key(), FILTER_LABEL_VALUE_CHARACTER.defaultValue()); if (!filterLabelValueCharacters) { labelValueCharactersFilter = input -> input; } }
/** * Adds the given value to the configuration object. * The main key of the config option will be used to map the value. * * @param key * the option specifying the key to be added * @param value * the value of the key/value pair to be added */ @PublicEvolving public void setBoolean(ConfigOption<Boolean> key, boolean value) { setValueInternal(key.key(), value); }
static MapFunction<Event, Event> createFailureMapper(ParameterTool pt) { return new FailureMapper<>( pt.getLong( TEST_SIMULATE_FAILURE_NUM_RECORDS.key(), TEST_SIMULATE_FAILURE_NUM_RECORDS.defaultValue()), pt.getLong( TEST_SIMULATE_FAILURE_NUM_CHECKPOINTS.key(), TEST_SIMULATE_FAILURE_NUM_CHECKPOINTS.defaultValue()), pt.getInt( TEST_SIMULATE_FAILURE_MAX_FAILURES.key(), TEST_SIMULATE_FAILURE_MAX_FAILURES.defaultValue())); }
static FlatMapFunction<Event, String> createSemanticsCheckMapper(ParameterTool pt) { String semantics = pt.get(TEST_SEMANTICS.key(), TEST_SEMANTICS.defaultValue()); SemanticsCheckMapper.ValidatorFunction validatorFunction; if (semantics.equalsIgnoreCase("exactly-once")) { validatorFunction = SemanticsCheckMapper.ValidatorFunction.exactlyOnce(); } else if (semantics.equalsIgnoreCase("at-least-once")) { validatorFunction = SemanticsCheckMapper.ValidatorFunction.atLeastOnce(); } else { throw new IllegalArgumentException("Unknown semantics requested: " + semantics); } return new SemanticsCheckMapper(validatorFunction); }
static TtlTestConfig fromArgs(ParameterTool pt) { int keySpace = pt.getInt(UPDATE_GENERATOR_SRC_KEYSPACE.key(), UPDATE_GENERATOR_SRC_KEYSPACE.defaultValue()); long sleepAfterElements = pt.getLong(UPDATE_GENERATOR_SRC_SLEEP_AFTER_ELEMENTS.key(), UPDATE_GENERATOR_SRC_SLEEP_AFTER_ELEMENTS.defaultValue()); long sleepTime = pt.getLong(UPDATE_GENERATOR_SRC_SLEEP_TIME.key(), UPDATE_GENERATOR_SRC_SLEEP_TIME.defaultValue()); Time ttl = Time.milliseconds(pt.getLong(STATE_TTL_VERIFIER_TTL_MILLI.key(), STATE_TTL_VERIFIER_TTL_MILLI.defaultValue())); long reportStatAfterUpdatesNum = pt.getLong(REPORT_STAT_AFTER_UPDATES_NUM.key(), REPORT_STAT_AFTER_UPDATES_NUM.defaultValue()); return new TtlTestConfig(keySpace, sleepAfterElements, sleepTime, ttl, reportStatAfterUpdatesNum); } }
static FlatMapFunction<Tuple2<Integer, List<Event>>, String> createSlidingWindowCheckMapper(ParameterTool pt) { return new SlidingWindowCheckMapper(pt.getInt( TEST_SLIDE_FACTOR.key(), TEST_SLIDE_FACTOR.defaultValue() )); } }
private static YarnConfigOptions.UserJarInclusion getUserJarInclusionMode(org.apache.flink.configuration.Configuration config) { String configuredUserJarInclusion = config.getString(YarnConfigOptions.CLASSPATH_INCLUDE_USER_JAR); try { return YarnConfigOptions.UserJarInclusion.valueOf(configuredUserJarInclusion.toUpperCase()); } catch (IllegalArgumentException e) { LOG.warn("Configuration parameter {} was configured with an invalid value {}. Falling back to default ({}).", YarnConfigOptions.CLASSPATH_INCLUDE_USER_JAR.key(), configuredUserJarInclusion, YarnConfigOptions.CLASSPATH_INCLUDE_USER_JAR.defaultValue()); return YarnConfigOptions.UserJarInclusion.valueOf(YarnConfigOptions.CLASSPATH_INCLUDE_USER_JAR.defaultValue()); } }
static SlidingEventTimeWindows createSlidingWindow(ParameterTool pt) { long slideSize = pt.getLong( TEST_SLIDE_SIZE.key(), TEST_SLIDE_SIZE.defaultValue()); long slideFactor = pt.getInt( TEST_SLIDE_FACTOR.key(), TEST_SLIDE_FACTOR.defaultValue() ); return SlidingEventTimeWindows.of(Time.milliseconds(slideSize * slideFactor), Time.milliseconds(slideSize)); }
public static void main(String[] args) throws Exception { final ParameterTool pt = ParameterTool.fromArgs(args); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); setupEnvironment(env, pt); final int numStates = pt.getInt(NUM_LIST_STATES_PER_OP.key(), NUM_LIST_STATES_PER_OP.defaultValue()); final int numPartitionsPerState = pt.getInt(NUM_PARTITIONS_PER_LIST_STATE.key(), NUM_PARTITIONS_PER_LIST_STATE.defaultValue()); Preconditions.checkState(env.getCheckpointInterval() > 0L, "Checkpointing must be enabled for this test!"); env.addSource(new SimpleEndlessSourceWithBloatedState(numStates, numPartitionsPerState)).setParallelism(env.getParallelism()) .addSink(new DiscardingSink<>()).setParallelism(1); env.execute("HeavyDeploymentStressTestProgram"); }
private void testLocalRecoveryInternal(String failoverStrategyValue) throws Exception { final Configuration configuration = new Configuration(); configuration.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, true); configuration.setString(EXECUTION_FAILOVER_STRATEGY.key(), failoverStrategyValue); executeSchedulingTest(configuration); }
@Test(expected = IllegalArgumentException.class) public void illegalArgument() { Configuration conf = new Configuration(); conf.setString(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO.key(), "1.1"); Assert.assertEquals(0, Utils.calculateHeapSize(4000, conf)); }
@Test(expected = IllegalArgumentException.class) public void tooMuchCutoff() { Configuration conf = new Configuration(); conf.setString(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO.key(), "6000"); Assert.assertEquals(0, Utils.calculateHeapSize(4000, conf)); }
@Test(expected = IllegalArgumentException.class) public void illegalArgumentNegative() { Configuration conf = new Configuration(); conf.setString(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO.key(), "-0.01"); Assert.assertEquals(0, Utils.calculateHeapSize(4000, conf)); }