public CoordinatorStreamWriter(Config config) { coordinatorStreamSystemProducer = new CoordinatorStreamSystemProducer(config, new MetricsRegistryMap()); }
public CoordinatorStreamWriter(Config config) { coordinatorStreamSystemProducer = new CoordinatorStreamSystemProducer(config, new MetricsRegistryMap()); }
public CoordinatorStreamWriter(Config config) { coordinatorStreamSystemProducer = new CoordinatorStreamSystemProducer(config, new MetricsRegistryMap()); }
public CoordinatorStreamWriter(Config config) { coordinatorStreamSystemProducer = new CoordinatorStreamSystemProducer(config, new MetricsRegistryMap()); }
public CoordinatorStreamWriter(Config config) { coordinatorStreamSystemProducer = new CoordinatorStreamSystemProducer(config, new MetricsRegistryMap()); }
/** * get the SystemConsumers for the stores */ private HashMap<String, SystemConsumer> getStoreConsumers() { HashMap<String, SystemConsumer> storeConsumers = new HashMap<>(); Map<String, SystemFactory> systemFactories = new JavaSystemConfig(jobConfig).getSystemFactories(); for (Entry<String, SystemStream> entry : changeLogSystemStreams.entrySet()) { String storeSystem = entry.getValue().getSystem(); if (!systemFactories.containsKey(storeSystem)) { throw new SamzaException("Changelog system " + storeSystem + " for store " + entry.getKey() + " does not exist in the config."); } storeConsumers.put(entry.getKey(), systemFactories.get(storeSystem).getConsumer(storeSystem, jobConfig, new MetricsRegistryMap())); } return storeConsumers; }
/** * get the SystemConsumers for the stores */ private HashMap<String, SystemConsumer> getStoreConsumers() { HashMap<String, SystemConsumer> storeConsumers = new HashMap<>(); Map<String, SystemFactory> systemFactories = new JavaSystemConfig(jobConfig).getSystemFactories(); for (Entry<String, SystemStream> entry : changeLogSystemStreams.entrySet()) { String storeSystem = entry.getValue().getSystem(); if (!systemFactories.containsKey(storeSystem)) { throw new SamzaException("Changelog system " + storeSystem + " for store " + entry.getKey() + " does not exist in the config."); } storeConsumers.put(entry.getKey(), systemFactories.get(storeSystem).getConsumer(storeSystem, jobConfig, new MetricsRegistryMap())); } return storeConsumers; }
/** * get the SystemConsumers for the stores */ private HashMap<String, SystemConsumer> getStoreConsumers() { HashMap<String, SystemConsumer> storeConsumers = new HashMap<>(); Map<String, SystemFactory> systemFactories = new JavaSystemConfig(jobConfig).getSystemFactories(); for (Entry<String, SystemStream> entry : changeLogSystemStreams.entrySet()) { String storeSystem = entry.getValue().getSystem(); if (!systemFactories.containsKey(storeSystem)) { throw new SamzaException("Changelog system " + storeSystem + " for store " + entry.getKey() + " does not exist in the config."); } storeConsumers.put(entry.getKey(), systemFactories.get(storeSystem).getConsumer(storeSystem, jobConfig, new MetricsRegistryMap())); } return storeConsumers; }
/** * get the SystemConsumers for the stores */ private HashMap<String, SystemConsumer> getStoreConsumers() { HashMap<String, SystemConsumer> storeConsumers = new HashMap<>(); Map<String, SystemFactory> systemFactories = new JavaSystemConfig(jobConfig).getSystemFactories(); for (Entry<String, SystemStream> entry : changeLogSystemStreams.entrySet()) { String storeSystem = entry.getValue().getSystem(); if (!systemFactories.containsKey(storeSystem)) { throw new SamzaException("Changelog system " + storeSystem + " for store " + entry.getKey() + " does not exist in the config."); } storeConsumers.put(entry.getKey(), systemFactories.get(storeSystem).getConsumer(storeSystem, jobConfig, new MetricsRegistryMap())); } return storeConsumers; }
private Config getConfigFromPrevRun() { CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(userConfig, new MetricsRegistryMap()); consumer.register(); consumer.start(); consumer.bootstrap(); consumer.stop(); Config cfg = consumer.getConfig(); LOG.info("Previous config is: " + cfg.toString()); return cfg; } }
private Config getConfigFromPrevRun() { CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(userConfig, new MetricsRegistryMap()); consumer.register(); consumer.start(); consumer.bootstrap(); consumer.stop(); Config cfg = consumer.getConfig(); LOG.info("Previous config is: " + cfg.toString()); return cfg; } }
private Config getConfigFromPrevRun() { CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(userConfig, new MetricsRegistryMap()); consumer.register(); consumer.start(); consumer.bootstrap(); consumer.stop(); Config cfg = consumer.getConfig(); LOG.info("Previous config is: " + cfg.toString()); return cfg; } }
private Config getConfigFromPrevRun() { CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(userConfig, new MetricsRegistryMap()); consumer.register(); consumer.start(); consumer.bootstrap(); consumer.stop(); Config cfg = consumer.getConfig(); LOG.info("Previous config is: " + cfg.toString()); return cfg; } }
private Config getConfigFromPrevRun() { CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(userConfig, new MetricsRegistryMap()); consumer.register(); consumer.start(); consumer.bootstrap(); consumer.stop(); Config cfg = consumer.getConfig(); LOG.info("Previous config is: " + cfg.toString()); return cfg; } }
/** * Builds list of {@link Task} from job model in coordinator stream. * @param consumer system consumer associated with a job's coordinator stream. * @return list of {@link Task} constructed from job model in coordinator stream. */ protected List<Task> readTasksFromCoordinatorStream(CoordinatorStreamSystemConsumer consumer) { LocalityManager localityManager = new LocalityManager(consumer.getConfig(), new MetricsRegistryMap()); Map<String, Map<String, String>> containerIdToHostMapping = localityManager.readContainerLocality(); TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(consumer.getConfig(), new MetricsRegistryMap()); Map<String, String> taskNameToContainerIdMapping = taskAssignmentManager.readTaskAssignment(); StorageConfig storageConfig = new StorageConfig(consumer.getConfig()); List<String> storeNames = JavaConverters.seqAsJavaListConverter(storageConfig.getStoreNames()).asJava(); return taskNameToContainerIdMapping.entrySet() .stream() .map(entry -> { String hostName = containerIdToHostMapping.get(entry.getValue()).get(SetContainerHostMapping.HOST_KEY); return new Task(hostName, entry.getKey(), entry.getValue(), new ArrayList<>(), storeNames); }).collect(Collectors.toList()); } }
/** * Instantiates an {@link ZkJobCoordinator} using the {@link Config}. * * @param config zookeeper configurations required for instantiating {@link ZkJobCoordinator} * @return An instance of {@link ZkJobCoordinator} */ @Override public JobCoordinator getJobCoordinator(Config config) { // TODO: Separate JC related configs into a "ZkJobCoordinatorConfig" MetricsRegistry metricsRegistry = new MetricsRegistryMap(); String jobCoordinatorZkBasePath = getJobCoordinationZkPath(config); ZkUtils zkUtils = getZkUtils(config, metricsRegistry, jobCoordinatorZkBasePath); LOG.debug("Creating ZkJobCoordinator with config: {}.", config); return new ZkJobCoordinator(config, metricsRegistry, zkUtils); }
/** * Instantiates an {@link ZkJobCoordinator} using the {@link Config}. * * @param config zookeeper configurations required for instantiating {@link ZkJobCoordinator} * @return An instance of {@link ZkJobCoordinator} */ @Override public JobCoordinator getJobCoordinator(Config config) { // TODO: Separate JC related configs into a "ZkJobCoordinatorConfig" MetricsRegistry metricsRegistry = new MetricsRegistryMap(); String jobCoordinatorZkBasePath = getJobCoordinationZkPath(config); ZkUtils zkUtils = getZkUtils(config, metricsRegistry, jobCoordinatorZkBasePath); LOG.debug("Creating ZkJobCoordinator with config: {}.", config); return new ZkJobCoordinator(config, metricsRegistry, zkUtils); }
/** * Instantiates an {@link ZkJobCoordinator} using the {@link Config}. * * @param config zookeeper configurations required for instantiating {@link ZkJobCoordinator} * @return An instance of {@link ZkJobCoordinator} */ @Override public JobCoordinator getJobCoordinator(Config config) { // TODO: Separate JC related configs into a "ZkJobCoordinatorConfig" MetricsRegistry metricsRegistry = new MetricsRegistryMap(); String jobCoordinatorZkBasePath = getJobCoordinationZkPath(config); ZkUtils zkUtils = getZkUtils(config, metricsRegistry, jobCoordinatorZkBasePath); LOG.debug("Creating ZkJobCoordinator with config: {}.", config); return new ZkJobCoordinator(config, metricsRegistry, zkUtils); }
/** * Instantiates an {@link ZkJobCoordinator} using the {@link Config}. * * @param config zookeeper configurations required for instantiating {@link ZkJobCoordinator} * @return An instance of {@link ZkJobCoordinator} */ @Override public JobCoordinator getJobCoordinator(Config config) { // TODO: Separate JC related configs into a "ZkJobCoordinatorConfig" MetricsRegistry metricsRegistry = new MetricsRegistryMap(); String jobCoordinatorZkBasePath = getJobCoordinationZkPath(config); ZkUtils zkUtils = getZkUtils(config, metricsRegistry, jobCoordinatorZkBasePath); LOG.debug("Creating ZkJobCoordinator with config: {}.", config); return new ZkJobCoordinator(config, metricsRegistry, zkUtils); }
@Before public void setup() { TaskInstanceMetrics metrics = new TaskInstanceMetrics("Partition 0", new MetricsRegistryMap()); listener = new TaskCallbackListener() { @Override public void onComplete(TaskCallback callback) { } @Override public void onFailure(TaskCallback callback, Throwable t) { } }; callbackManager = new TaskCallbackManager(listener, null, -1, 2, () -> System.nanoTime()); }