public StreamOperatorTask(OperatorSpecGraph specGraph) { this(specGraph, SystemClock.instance()); }
public StreamOperatorTask(OperatorSpecGraph specGraph) { this(specGraph, SystemClock.instance()); }
public StreamOperatorTask(OperatorSpecGraph specGraph) { this(specGraph, SystemClock.instance()); }
public StreamOperatorTask(OperatorSpecGraph specGraph) { this(specGraph, SystemClock.instance()); }
public StreamOperatorTask(OperatorSpecGraph specGraph) { this(specGraph, SystemClock.instance()); }
@Override public JobModel getJobModel() { SystemAdmins systemAdmins = new SystemAdmins(config); StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 5000, SystemClock.instance()); systemAdmins.start(); try { String containerId = Integer.toString(config.getInt(JobConfig.PROCESSOR_ID())); GrouperMetadata grouperMetadata = new GrouperMetadataImpl(ImmutableMap.of(String.valueOf(containerId), locationId), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); return JobModelManager.readJobModel(this.config, Collections.emptyMap(), streamMetadataCache, grouperMetadata); } finally { systemAdmins.stop(); } }
@Override public JobModel getJobModel() { SystemAdmins systemAdmins = new SystemAdmins(config); StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 5000, SystemClock.instance()); systemAdmins.start(); String containerId = Integer.toString(config.getInt(JobConfig.PROCESSOR_ID())); /** TODO: Locality Manager seems to be required in JC for reading locality info and grouping tasks intelligently and also, in SamzaContainer for writing locality info to the coordinator stream. This closely couples together TaskNameGrouper with the LocalityManager! Hence, groupers should be a property of the jobcoordinator (job.coordinator.task.grouper, instead of task.systemstreampartition.grouper) */ JobModel jobModel = JobModelManager.readJobModel(this.config, Collections.emptyMap(), null, streamMetadataCache, Collections.singletonList(containerId)); systemAdmins.stop(); return jobModel; }
@Override public JobModel getJobModel() { SystemAdmins systemAdmins = new SystemAdmins(config); StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 5000, SystemClock.instance()); systemAdmins.start(); String containerId = Integer.toString(config.getInt(JobConfig.PROCESSOR_ID())); /** TODO: Locality Manager seems to be required in JC for reading locality info and grouping tasks intelligently and also, in SamzaContainer for writing locality info to the coordinator stream. This closely couples together TaskNameGrouper with the LocalityManager! Hence, groupers should be a property of the jobcoordinator (job.coordinator.task.grouper, instead of task.systemstreampartition.grouper) */ JobModel jobModel = JobModelManager.readJobModel(this.config, Collections.emptyMap(), null, streamMetadataCache, Collections.singletonList(containerId)); systemAdmins.stop(); return jobModel; }
@Override public JobModel getJobModel() { SystemAdmins systemAdmins = new SystemAdmins(config); StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 5000, SystemClock.instance()); systemAdmins.start(); String containerId = Integer.toString(config.getInt(JobConfig.PROCESSOR_ID())); /** TODO: Locality Manager seems to be required in JC for reading locality info and grouping tasks intelligently and also, in SamzaContainer for writing locality info to the coordinator stream. This closely couples together TaskNameGrouper with the LocalityManager! Hence, groupers should be a property of the jobcoordinator (job.coordinator.task.grouper, instead of task.systemstreampartition.grouper) */ JobModel jobModel = JobModelManager.readJobModel(this.config, Collections.emptyMap(), null, streamMetadataCache, Collections.singletonList(containerId)); systemAdmins.stop(); return jobModel; }
@Override public JobModel getJobModel() { SystemAdmins systemAdmins = new SystemAdmins(config); StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 5000, SystemClock.instance()); systemAdmins.start(); String containerId = Integer.toString(config.getInt(JobConfig.PROCESSOR_ID())); /** TODO: Locality Manager seems to be required in JC for reading locality info and grouping tasks intelligently and also, in SamzaContainer for writing locality info to the coordinator stream. This closely couples together TaskNameGrouper with the LocalityManager! Hence, groupers should be a property of the jobcoordinator (job.coordinator.task.grouper, instead of task.systemstreampartition.grouper) */ JobModel jobModel = JobModelManager.readJobModel(this.config, Collections.emptyMap(), null, streamMetadataCache, Collections.singletonList(containerId)); systemAdmins.stop(); return jobModel; }
private Optional<StreamPartitionCountMonitor> getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty."); } return Optional.of(new StreamPartitionCountMonitor(inputStreamsToMonitor, streamMetadata, metrics, new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted if (hasDurableStores) { log.error("Input topic partition count changed in a job with durable state. Failing the job."); state.status = SamzaApplicationState.SamzaAppStatus.FAILED; } coordinatorException = new PartitionChangeException("Input topic partition count changes detected."); })); }
@Override public void start() { LOG.info("Starting Azure job coordinator."); // The systemAdmins should be started before streamMetadataCache can be used. And it should be stopped when this coordinator is stopped. systemAdmins = new SystemAdmins(config); systemAdmins.start(); streamMetadataCache = new StreamMetadataCache(systemAdmins, METADATA_CACHE_TTL_MS, SystemClock.instance()); table.addProcessorEntity(INITIAL_STATE, processorId, false); // Start scheduler for heartbeating LOG.info("Starting scheduler for heartbeating."); heartbeat.scheduleTask(); azureLeaderElector.tryBecomeLeader(); // Start scheduler to check for job model version upgrades LOG.info("Starting scheduler to check for job model version upgrades."); versionUpgrade.setStateChangeListener(createJMVersionUpgradeListener()); versionUpgrade.scheduleTask(); // Start scheduler to check for leader liveness LOG.info("Starting scheduler to check for leader liveness."); leaderAlive.setStateChangeListener(createLeaderLivenessListener()); leaderAlive.scheduleTask(); }
private StreamPartitionCountMonitor getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty."); } return new StreamPartitionCountMonitor( inputStreamsToMonitor, streamMetadata, metrics, new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted if (hasDurableStores) { log.error("Input topic partition count changed in a job with durable state. Failing the job."); state.status = SamzaApplicationState.SamzaAppStatus.FAILED; } coordinatorException = new PartitionChangeException("Input topic partition count changes detected."); }); }
@VisibleForTesting StreamPartitionCountMonitor getPartitionCountMonitor() { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); return new StreamPartitionCountMonitor( inputStreamsToMonitor, streamMetadata, metrics.getMetricsRegistry(), new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { if (leaderElector.amILeader()) { debounceTimer.scheduleAfterDebounceTime(ON_PROCESSOR_CHANGE, 0, this::doOnProcessorChange); } }); }
/** * create one TaskStorageManager for each task. Add all of them to the * List<TaskStorageManager> */ @SuppressWarnings({"unchecked", "rawtypes"}) private void getContainerStorageManagers() { Clock clock = SystemClock.instance(); StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 5000, clock); // don't worry about prefetching for this; looks like the tool doesn't flush to offset files anyways Map<String, SystemFactory> systemFactories = new JavaSystemConfig(jobConfig).getSystemFactories(); for (ContainerModel containerModel : containers.values()) { ContainerContext containerContext = new ContainerContextImpl(containerModel, new MetricsRegistryMap()); ContainerStorageManager containerStorageManager = new ContainerStorageManager(containerModel, streamMetadataCache, systemAdmins, changeLogSystemStreams, storageEngineFactories, systemFactories, this.getSerdes(), jobConfig, new HashMap<>(), new SamzaContainerMetrics(containerModel.getId(), new MetricsRegistryMap()), JobContextImpl.fromConfigWithDefaults(jobConfig), containerContext, new HashMap<>(), storeBaseDir, storeBaseDir, maxPartitionNumber, new SystemClock()); this.containerStorageManagers.put(containerModel.getId(), containerStorageManager); } } }
private StreamPartitionCountMonitor getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty."); } return new StreamPartitionCountMonitor( inputStreamsToMonitor, streamMetadata, metrics, new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted if (hasDurableStores) { log.error("Input topic partition count changed in a job with durable state. Failing the job."); state.status = SamzaApplicationState.SamzaAppStatus.FAILED; } coordinatorException = new PartitionChangeException("Input topic partition count changes detected."); }); }
private StreamPartitionCountMonitor getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty."); } return new StreamPartitionCountMonitor( inputStreamsToMonitor, streamMetadata, metrics, new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted if (hasDurableStores) { log.error("Input topic partition count changed in a job with durable state. Failing the job."); state.status = SamzaApplicationState.SamzaAppStatus.FAILED; } coordinatorException = new PartitionChangeException("Input topic partition count changes detected."); }); }
private StreamPartitionCountMonitor getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty."); } return new StreamPartitionCountMonitor( inputStreamsToMonitor, streamMetadata, metrics, new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted if (hasDurableStores) { log.error("Input topic partition count changed in a job with durable state. Failing the job."); state.status = SamzaApplicationState.SamzaAppStatus.FAILED; } coordinatorException = new PartitionChangeException("Input topic partition count changes detected."); }); }
ZkJobCoordinator(Config config, MetricsRegistry metricsRegistry, ZkUtils zkUtils) { this.config = config; this.metrics = new ZkJobCoordinatorMetrics(metricsRegistry); this.processorId = createProcessorId(config); this.zkUtils = zkUtils; // setup a listener for a session state change // we are mostly interested in "session closed" and "new session created" events zkUtils.getZkClient().subscribeStateChanges(new ZkSessionStateChangedListener()); leaderElector = new ZkLeaderElector(processorId, zkUtils); leaderElector.setLeaderElectorListener(new LeaderElectorListenerImpl()); this.debounceTimeMs = new JobConfig(config).getDebounceTimeMs(); this.reporters = MetricsReporterLoader.getMetricsReporters(new MetricsConfig(config), processorId); debounceTimer = new ScheduleAfterDebounceTime(processorId); debounceTimer.setScheduledTaskCallback(throwable -> { LOG.error("Received exception in debounce timer! Stopping the job coordinator", throwable); stop(); }); this.barrier = new ZkBarrierForVersionUpgrade(zkUtils.getKeyBuilder().getJobModelVersionBarrierPrefix(), zkUtils, new ZkBarrierListenerImpl(), debounceTimer); systemAdmins = new SystemAdmins(config); streamMetadataCache = new StreamMetadataCache(systemAdmins, METADATA_CACHE_TTL_MS, SystemClock.instance()); }
ZkJobCoordinator(Config config, MetricsRegistry metricsRegistry, ZkUtils zkUtils) { this.config = config; this.metrics = new ZkJobCoordinatorMetrics(metricsRegistry); this.processorId = createProcessorId(config); this.zkUtils = zkUtils; // setup a listener for a session state change // we are mostly interested in "session closed" and "new session created" events zkUtils.getZkClient().subscribeStateChanges(new ZkSessionStateChangedListener()); leaderElector = new ZkLeaderElector(processorId, zkUtils); leaderElector.setLeaderElectorListener(new LeaderElectorListenerImpl()); this.debounceTimeMs = new JobConfig(config).getDebounceTimeMs(); this.reporters = MetricsReporterLoader.getMetricsReporters(new MetricsConfig(config), processorId); debounceTimer = new ScheduleAfterDebounceTime(processorId); debounceTimer.setScheduledTaskCallback(throwable -> { LOG.error("Received exception in debounce timer! Stopping the job coordinator", throwable); stop(); }); this.barrier = new ZkBarrierForVersionUpgrade(zkUtils.getKeyBuilder().getJobModelVersionBarrierPrefix(), zkUtils, new ZkBarrierListenerImpl(), debounceTimer); systemAdmins = new SystemAdmins(config); streamMetadataCache = new StreamMetadataCache(systemAdmins, METADATA_CACHE_TTL_MS, SystemClock.instance()); }