private void configureWindowInterval(Map<String, String> configs, Config config, Collection<OperatorSpec> reachableOperators) { if (!reachableOperators.stream().anyMatch(op -> op.getOpCode() == OperatorSpec.OpCode.WINDOW || op.getOpCode() == OperatorSpec.OpCode.JOIN)) { return; } // set triggering interval if a window or join is defined. Only applies to high-level applications long triggerInterval = computeTriggerInterval(reachableOperators); LOG.info("Using triggering interval: {}", triggerInterval); configs.put(TaskConfig.WINDOW_MS(), String.valueOf(triggerInterval)); }
private void configureWindowInterval(Map<String, String> configs, Config config, Collection<OperatorSpec> reachableOperators) { if (!reachableOperators.stream().anyMatch(op -> op.getOpCode() == OperatorSpec.OpCode.WINDOW || op.getOpCode() == OperatorSpec.OpCode.JOIN)) { return; } // set triggering interval if a window or join is defined. Only applies to high-level applications long triggerInterval = computeTriggerInterval(reachableOperators); LOG.info("Using triggering interval: {}", triggerInterval); configs.put(TaskConfig.WINDOW_MS(), String.valueOf(triggerInterval)); }
private void configureWindowInterval(Map<String, String> configs, Config config, Collection<OperatorSpec> reachableOperators) { if (!reachableOperators.stream().anyMatch(op -> op.getOpCode() == OperatorSpec.OpCode.WINDOW || op.getOpCode() == OperatorSpec.OpCode.JOIN)) { return; } // set triggering interval if a window or join is defined. Only applies to high-level applications long triggerInterval = computeTriggerInterval(reachableOperators); LOG.info("Using triggering interval: {}", triggerInterval); configs.put(TaskConfig.WINDOW_MS(), String.valueOf(triggerInterval)); }
private void configureWindowInterval(Map<String, String> configs, Config config, Collection<OperatorSpec> reachableOperators) { if (!reachableOperators.stream().anyMatch(op -> op.getOpCode() == OperatorSpec.OpCode.WINDOW || op.getOpCode() == OperatorSpec.OpCode.JOIN)) { return; } // set triggering interval if a window or join is defined. Only applies to high-level applications long triggerInterval = computeTriggerInterval(reachableOperators); LOG.info("Using triggering interval: {}", triggerInterval); configs.put(TaskConfig.WINDOW_MS(), String.valueOf(triggerInterval)); }
private void configureWindowInterval(Map<String, String> configs, Config config, Collection<OperatorSpec> reachableOperators) { if (!reachableOperators.stream().anyMatch(op -> op.getOpCode() == OperatorSpec.OpCode.WINDOW || op.getOpCode() == OperatorSpec.OpCode.JOIN)) { return; } // set triggering interval if a window or join is defined. Only applies to high-level applications long triggerInterval = computeTriggerInterval(reachableOperators); LOG.info("Using triggering interval: {}", triggerInterval); configs.put(TaskConfig.WINDOW_MS(), String.valueOf(triggerInterval)); }
@Test public void testTriggerIntervalWhenWindowMsIsConfigured() { Map<String, String> map = new HashMap<>(config); map.put(TaskConfig.WINDOW_MS(), "2000"); map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS(), String.valueOf(DEFAULT_PARTITIONS)); Config cfg = new MapConfig(map); ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager); StreamApplicationDescriptorImpl graphSpec = createSimpleGraph(); ExecutionPlan plan = planner.plan(graphSpec); List<JobConfig> jobConfigs = plan.getJobConfigs(); assertEquals(1, jobConfigs.size()); assertEquals("2000", jobConfigs.get(0).get(TaskConfig.WINDOW_MS())); }
@Test public void testTriggerIntervalForStatelessOperators() { Map<String, String> map = new HashMap<>(config); map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS(), String.valueOf(DEFAULT_PARTITIONS)); Config cfg = new MapConfig(map); ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager); StreamApplicationDescriptorImpl graphSpec = createSimpleGraph(); ExecutionPlan plan = planner.plan(graphSpec); List<JobConfig> jobConfigs = plan.getJobConfigs(); assertEquals(1, jobConfigs.size()); assertFalse(jobConfigs.get(0).containsKey(TaskConfig.WINDOW_MS())); }
@Test public void testTriggerIntervalWithNoWindowMs() { Map<String, String> map = new HashMap<>(config); map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS(), String.valueOf(DEFAULT_PARTITIONS)); Config cfg = new MapConfig(map); ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager); StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithJoinAndWindow(); ExecutionPlan plan = planner.plan(graphSpec); List<JobConfig> jobConfigs = plan.getJobConfigs(); assertEquals(1, jobConfigs.size()); // GCD of 8, 16, 1600 and 252 is 4 assertEquals("4", jobConfigs.get(0).get(TaskConfig.WINDOW_MS())); }
@Test public void testTriggerIntervalForWindowsAndJoins() { Map<String, String> map = new HashMap<>(config); map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS(), String.valueOf(DEFAULT_PARTITIONS)); Config cfg = new MapConfig(map); ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager); StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithJoinAndWindow(); ExecutionPlan plan = planner.plan(graphSpec); List<JobConfig> jobConfigs = plan.getJobConfigs(); assertEquals(1, jobConfigs.size()); // GCD of 8, 16, 1600 and 252 is 4 assertEquals("4", jobConfigs.get(0).get(TaskConfig.WINDOW_MS())); }
@Test public void testConfigureSerdesWithRepartitionJoinApplication() { mockStreamAppDesc = new StreamApplicationDescriptorImpl(getRepartitionJoinStreamApplication(), mockConfig); configureJobNode(mockStreamAppDesc); // create the JobGraphConfigureGenerator and generate the jobConfig for the jobNode JobNodeConfigurationGenerator configureGenerator = new JobNodeConfigurationGenerator(); JobConfig jobConfig = configureGenerator.generateJobConfig(mockJobNode, "testJobGraphJson"); // Verify the results Config expectedJobConfig = getExpectedJobConfig(mockConfig, mockJobNode.getInEdges()); validateJobConfig(expectedJobConfig, jobConfig); // additional, check the computed window.ms for join assertEquals("3600000", jobConfig.get(TaskConfig.WINDOW_MS())); Map<String, Serde> deserializedSerdes = validateAndGetDeserializedSerdes(jobConfig, 5); validateStreamConfigures(jobConfig, deserializedSerdes); validateJoinStoreConfigures(jobConfig, deserializedSerdes); }