public AzureConfig(Config config) { super(config); ApplicationConfig appConfig = new ApplicationConfig(config); //Remove all non-alphanumeric characters from id as table name does not allow them. String id = appConfig.getGlobalAppId().replaceAll("[^A-Za-z0-9]", ""); containerName = "samzacontainer" + id; blobName = "samzablob" + id; tableName = "samzatable" + id; }
/** * returns full application id * @return full app id */ public String getGlobalAppId() { return String.format("app-%s-%s", getAppName(), getAppId()); }
/** * Create a unique stream name if it's batch mode and has a valid run.id. * @param stream physical name of the stream * @param config {@link Config} object * @return stream name created */ public static String createUniqueNameForBatch(String stream, Config config) { ApplicationConfig appConfig = new ApplicationConfig(config); if (appConfig.getAppMode() == ApplicationConfig.ApplicationMode.BATCH && appConfig.getRunId() != null) { return stream + "-" + appConfig.getRunId(); } else { return stream; } } }
private String createProcessorId(Config config) { // TODO: This check to be removed after 0.13+ ApplicationConfig appConfig = new ApplicationConfig(config); if (appConfig.getProcessorId() != null) { return appConfig.getProcessorId(); } else if (StringUtils.isNotBlank(appConfig.getAppProcessorIdGeneratorClass())) { ProcessorIdGenerator idGenerator = Util.getObj(appConfig.getAppProcessorIdGeneratorClass(), ProcessorIdGenerator.class); return idGenerator.generateProcessorId(config); } else { throw new ConfigException(String .format("Expected either %s or %s to be configured", ApplicationConfig.PROCESSOR_ID, ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS)); } }
private void validateConfig() { ApplicationConfig appConfig = new ApplicationConfig(config); ClusterManagerConfig clusterConfig = new ClusterManagerConfig(config); // currently we don't support host-affinity in batch mode if (appConfig.getAppMode() == ApplicationConfig.ApplicationMode.BATCH && clusterConfig.getHostAffinityEnabled()) { throw new SamzaException(String.format("Host affinity is not supported in batch mode. Please configure %s=false.", ClusterManagerConfig.JOB_HOST_AFFINITY_ENABLED)); } }
/** * Creates the {@link SamzaApplication} object from the task or application class name specified in {@code config} * * @param config the configuration of the application * @return the {@link SamzaApplication} object */ public static SamzaApplication fromConfig(Config config) { String appClassName = new ApplicationConfig(config).getAppClass(); if (StringUtils.isNotBlank(appClassName)) { // app.class is configured try { Class<SamzaApplication> appClass = (Class<SamzaApplication>) Class.forName(appClassName); if (StreamApplication.class.isAssignableFrom(appClass) || TaskApplication.class.isAssignableFrom(appClass)) { return appClass.newInstance(); } } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { throw new ConfigException(String.format("Loading app.class %s failed. The user application has to implement " + "StreamApplication or TaskApplication.", appClassName), e); } } // no app.class defined. It has to be a legacy application with task.class configuration Option<String> taskClassOption = new TaskConfig(config).getTaskClass(); if (!taskClassOption.isDefined() || !StringUtils.isNotBlank(taskClassOption.getOrElse(null))) { // no task.class defined either. This is wrong. throw new ConfigException("Legacy task applications must set a non-empty task.class in configuration."); } return new LegacyTaskApplication(taskClassOption.get()); } }
ApplicationConfig appConfig = new ApplicationConfig(prevConfig); LOGGER.info("run.id from previous run is {}", appConfig.getRunId());
public String getProcessorId() { return get(PROCESSOR_ID, null); }
/** * Returns the config for this application * @return {@link ApplicationConfig} */ @Override public ApplicationConfig getApplicationConfig() { return new ApplicationConfig(config); }
/** * Generates a unique logical identifier for the stream processor using the provided {@param appConfig}. * 1. If the processorId is defined in the configuration, then returns the value defined in the configuration. * 2. Else if the {@linkplain ProcessorIdGenerator} class is defined the configuration, then uses the {@linkplain ProcessorIdGenerator} * to generate the unique processorId. * 3. Else throws the {@see ConfigException} back to the caller. * @param appConfig the configuration of the samza application. * @throws ConfigException if neither processor.id nor app.processor-id-generator.class is defined in the configuration. * @return the generated processor identifier. */ @VisibleForTesting static String createProcessorId(ApplicationConfig appConfig) { if (StringUtils.isNotBlank(appConfig.getProcessorId())) { return appConfig.getProcessorId(); } else if (StringUtils.isNotBlank(appConfig.getAppProcessorIdGeneratorClass())) { ProcessorIdGenerator idGenerator = Util.getObj(appConfig.getAppProcessorIdGeneratorClass(), ProcessorIdGenerator.class); return idGenerator.generateProcessorId(appConfig); } else { throw new ConfigException(String.format("Expected either %s or %s to be configured", ApplicationConfig.PROCESSOR_ID, ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS)); } }
if (plan.getApplicationConfig().getAppMode() == ApplicationConfig.ApplicationMode.BATCH) { streamManager.clearStreamsFromPreviousRun(getConfigFromPrevRun());
@Test(expected = ConfigException.class) public void testCreateProcessorIdShouldThrowExceptionWhenProcessorIdAndGeneratorAreNotDefined() { ApplicationConfig mockConfig = Mockito.mock(ApplicationConfig.class); Mockito.when(mockConfig.getProcessorId()).thenReturn(null); LocalApplicationRunner.createProcessorId(mockConfig); }
private String createProcessorId(Config config) { // TODO: This check to be removed after 0.13+ ApplicationConfig appConfig = new ApplicationConfig(config); if (appConfig.getProcessorId() != null) { return appConfig.getProcessorId(); } else if (appConfig.getAppProcessorIdGeneratorClass() != null) { ProcessorIdGenerator idGenerator = Util.getObj(appConfig.getAppProcessorIdGeneratorClass(), ProcessorIdGenerator.class); return idGenerator.generateProcessorId(config); } else { throw new ConfigException(String .format("Expected either %s or %s to be configured", ApplicationConfig.PROCESSOR_ID, ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS)); } } }
private void validateConfig() { ApplicationConfig appConfig = new ApplicationConfig(config); ClusterManagerConfig clusterConfig = new ClusterManagerConfig(config); // currently we don't support host-affinity in batch mode if (appConfig.getAppMode() == ApplicationConfig.ApplicationMode.BATCH && clusterConfig.getHostAffinityEnabled()) { throw new SamzaException(String.format("Host affinity is not supported in batch mode. Please configure %s=false.", ClusterManagerConfig.CLUSTER_MANAGER_HOST_AFFINITY_ENABLED)); } }
/** * Creates the {@link SamzaApplication} object from the task or application class name specified in {@code config} * * @param config the configuration of the application * @return the {@link SamzaApplication} object */ public static SamzaApplication fromConfig(Config config) { String appClassName = new ApplicationConfig(config).getAppClass(); if (StringUtils.isNotBlank(appClassName)) { // app.class is configured try { Class<SamzaApplication> appClass = (Class<SamzaApplication>) Class.forName(appClassName); if (StreamApplication.class.isAssignableFrom(appClass) || TaskApplication.class.isAssignableFrom(appClass)) { return appClass.newInstance(); } } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { throw new ConfigException(String.format("Loading app.class %s failed. The user application has to implement " + "StreamApplication or TaskApplication.", appClassName), e); } } // no app.class defined. It has to be a legacy application with task.class configuration Option<String> taskClassOption = new TaskConfig(config).getTaskClass(); if (!taskClassOption.isDefined() || !StringUtils.isNotBlank(taskClassOption.getOrElse(null))) { // no task.class defined either. This is wrong. throw new ConfigException("Legacy task applications must set a non-empty task.class in configuration."); } return new LegacyTaskApplication(taskClassOption.get()); } }
ApplicationConfig appConfig = new ApplicationConfig(prevConfig); LOGGER.info("run.id from previous run is {}", appConfig.getRunId());
public String getAppClass() { return get(APP_CLASS, null); }
/** * Returns the config for this application * @return {@link ApplicationConfig} */ @Override public ApplicationConfig getApplicationConfig() { return new ApplicationConfig(config); }
if (plan.getApplicationConfig().getAppMode() == ApplicationConfig.ApplicationMode.BATCH) { streamManager.clearStreamsFromPreviousRun(getConfigFromPrevRun());
public static String getJobCoordinationZkPath(Config config) { JobConfig jobConfig = new JobConfig(config); String appId = new ApplicationConfig(config).getGlobalAppId(); String jobName = jobConfig.getName().isDefined() ? jobConfig.getName().get() : DEFAULT_JOB_NAME; String jobId = jobConfig.getJobId(); return String.format(JOB_COORDINATOR_ZK_PATH_FORMAT, appId, jobName, jobId); } }