@Override public S withDefaultSystem(SystemDescriptor<?> defaultSystemDescriptor) { Preconditions.checkNotNull(defaultSystemDescriptor, "Provided defaultSystemDescriptor must not be null."); Preconditions.checkState(getInputStreamIds().isEmpty() && getOutputStreamIds().isEmpty(), "Default system must be set before creating any input or output streams."); addSystemDescriptor(defaultSystemDescriptor); defaultSystemDescriptorOptional = Optional.of(defaultSystemDescriptor); return (S) this; }
final void addTableDescriptor(TableDescriptor tableDescriptor) { String tableId = tableDescriptor.getTableId(); Preconditions.checkState(StringUtils.isNotBlank(tableId) && ID_PATTERN.matcher(tableId).matches(), String.format("tableId: %s must confirm to pattern: %s", tableId, ID_PATTERN.toString())); Preconditions.checkState(!tableDescriptors.containsKey(tableId) || tableDescriptors.get(tableId) == tableDescriptor, String.format("Cannot add multiple table descriptors with the same tableId: %s", tableId)); if (tableDescriptor instanceof HybridTableDescriptor) { List<? extends TableDescriptor> tableDescs = ((HybridTableDescriptor) tableDescriptor).getTableDescriptors(); tableDescs.forEach(td -> addTableDescriptor(td)); } tableDescriptors.put(tableId, tableDescriptor); }
boolean isLegacyTaskApplication() { return LegacyTaskApplication.class.isAssignableFrom(appDesc.getAppClass()); }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
@VisibleForTesting StreamProcessor createStreamProcessor(Config config, ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc, StreamProcessor.StreamProcessorLifecycleListenerFactory listenerFactory) { TaskFactory taskFactory = TaskFactoryUtil.getTaskFactory(appDesc); Map<String, MetricsReporter> reporters = new HashMap<>(); // TODO: the null processorId has to be fixed after SAMZA-1835 appDesc.getMetricsReporterFactories().forEach((name, factory) -> reporters.put(name, factory.getMetricsReporter(name, null, config))); return new StreamProcessor(config, reporters, taskFactory, appDesc.getApplicationContainerContextFactory(), appDesc.getApplicationTaskContextFactory(), listenerFactory, null); }
JobGraph jobGraph = new JobGraph(config, appDesc); Set<StreamSpec> sourceStreams = getStreamSpecs(appDesc.getInputStreamIds(), streamConfig); Set<StreamSpec> sinkStreams = getStreamSpecs(appDesc.getOutputStreamIds(), streamConfig); Set<StreamSpec> outputStreams = Sets.difference(sinkStreams, intermediateStreams); Set<TableSpec> tables = appDesc.getTableDescriptors().stream() .map(tableDescriptor -> ((BaseTableDescriptor) tableDescriptor).getTableSpec()).collect(Collectors.toSet()); if (!LegacyTaskApplication.class.isAssignableFrom(appDesc.getAppClass())) {
private Map<String, String> getGeneratedConfig(String runId) { Map<String, String> generatedConfig = new HashMap<>(); if (StringUtils.isNoneEmpty(runId)) { generatedConfig.put(ApplicationConfig.APP_RUN_ID, runId); } StreamConfig streamConfig = new StreamConfig(userConfig); Set<String> inputStreamIds = new HashSet<>(appDesc.getInputStreamIds()); inputStreamIds.removeAll(appDesc.getOutputStreamIds()); // exclude intermediate streams ApplicationConfig.ApplicationMode mode = inputStreamIds.stream().allMatch(streamConfig::getIsBounded) ? ApplicationConfig.ApplicationMode.BATCH : ApplicationConfig.ApplicationMode.STREAM; generatedConfig.put(ApplicationConfig.APP_MODE, mode.name()); Map<String, String> systemStreamConfigs = generateSystemStreamConfigs(appDesc); generatedConfig.putAll(systemStreamConfigs); // adding app.class in the configuration, unless it is LegacyTaskApplication if (!LegacyTaskApplication.class.getName().equals(appDesc.getAppClass().getName())) { generatedConfig.put(ApplicationConfig.APP_CLASS, appDesc.getAppClass().getName()); } return generatedConfig; }
taskFactory, JobContextImpl.fromConfigWithDefaults(config), Option.apply(appDesc.getApplicationContainerContextFactory().orElse(null)), Option.apply(appDesc.getApplicationTaskContextFactory().orElse(null))); ProcessorLifecycleListener listener = appDesc.getProcessorLifecycleListenerFactory() .createInstance(new ProcessorContext() { }, config);
JobPlanner(ApplicationDescriptorImpl<? extends ApplicationDescriptor> descriptor) { this.appDesc = descriptor; this.userConfig = descriptor.getConfig(); }
InputOperatorSpec getInputOperator(String inputStreamId) { if (!inEdges.containsKey(inputStreamId)) { return null; } return appDesc.getInputOperators().get(inputStreamId); }
final void addInputDescriptor(InputDescriptor inputDescriptor) { String streamId = inputDescriptor.getStreamId(); Preconditions.checkState(!inputDescriptors.containsKey(streamId) || inputDescriptors.get(streamId) == inputDescriptor, String.format("Cannot add multiple input descriptors with the same streamId: %s", streamId)); inputDescriptors.put(streamId, inputDescriptor); addSystemDescriptor(inputDescriptor.getSystemDescriptor()); }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
JobGraph jobGraph = new JobGraph(config, appDesc); Set<StreamSpec> sourceStreams = getStreamSpecs(appDesc.getInputStreamIds(), streamConfig); Set<StreamSpec> sinkStreams = getStreamSpecs(appDesc.getOutputStreamIds(), streamConfig); Set<StreamSpec> outputStreams = Sets.difference(sinkStreams, intermediateStreams); Set<TableSpec> tables = appDesc.getTableDescriptors().stream() .map(tableDescriptor -> ((BaseTableDescriptor) tableDescriptor).getTableSpec()).collect(Collectors.toSet()); if (!LegacyTaskApplication.class.isAssignableFrom(appDesc.getAppClass())) {
@VisibleForTesting StreamProcessor createStreamProcessor(Config config, ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc, StreamProcessor.StreamProcessorLifecycleListenerFactory listenerFactory) { TaskFactory taskFactory = TaskFactoryUtil.getTaskFactory(appDesc); Map<String, MetricsReporter> reporters = new HashMap<>(); // TODO: the null processorId has to be fixed after SAMZA-1835 appDesc.getMetricsReporterFactories().forEach((name, factory) -> reporters.put(name, factory.getMetricsReporter(name, null, config))); return new StreamProcessor(config, reporters, taskFactory, appDesc.getApplicationContainerContextFactory(), appDesc.getApplicationTaskContextFactory(), listenerFactory, null); }
private Map<String, String> getGeneratedConfig(String runId) { Map<String, String> generatedConfig = new HashMap<>(); if (StringUtils.isNoneEmpty(runId)) { generatedConfig.put(ApplicationConfig.APP_RUN_ID, runId); } StreamConfig streamConfig = new StreamConfig(userConfig); Set<String> inputStreamIds = new HashSet<>(appDesc.getInputStreamIds()); inputStreamIds.removeAll(appDesc.getOutputStreamIds()); // exclude intermediate streams ApplicationConfig.ApplicationMode mode = inputStreamIds.stream().allMatch(streamConfig::getIsBounded) ? ApplicationConfig.ApplicationMode.BATCH : ApplicationConfig.ApplicationMode.STREAM; generatedConfig.put(ApplicationConfig.APP_MODE, mode.name()); Map<String, String> systemStreamConfigs = generateSystemStreamConfigs(appDesc); generatedConfig.putAll(systemStreamConfigs); // adding app.class in the configuration, unless it is LegacyTaskApplication if (!LegacyTaskApplication.class.getName().equals(appDesc.getAppClass().getName())) { generatedConfig.put(ApplicationConfig.APP_CLASS, appDesc.getAppClass().getName()); } return generatedConfig; }
taskFactory, JobContextImpl.fromConfigWithDefaults(config), Option.apply(appDesc.getApplicationContainerContextFactory().orElse(null)), Option.apply(appDesc.getApplicationTaskContextFactory().orElse(null))); ProcessorLifecycleListener listener = appDesc.getProcessorLifecycleListenerFactory() .createInstance(new ProcessorContext() { }, config);
JobPlanner(ApplicationDescriptorImpl<? extends ApplicationDescriptor> descriptor) { this.appDesc = descriptor; this.userConfig = descriptor.getConfig(); }
InputOperatorSpec getInputOperator(String inputStreamId) { if (!inEdges.containsKey(inputStreamId)) { return null; } return appDesc.getInputOperators().get(inputStreamId); }
final void addOutputDescriptor(OutputDescriptor outputDescriptor) { String streamId = outputDescriptor.getStreamId(); Preconditions.checkState(!outputDescriptors.containsKey(streamId) || outputDescriptors.get(streamId) == outputDescriptor, String.format("Cannot add an output descriptor multiple times with the same streamId: %s", streamId)); outputDescriptors.put(streamId, outputDescriptor); addSystemDescriptor(outputDescriptor.getSystemDescriptor()); }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }