@Override public void onAddSpec(Spec addedSpec) { topologySpecMap.put(addedSpec.getUri(), (TopologySpec) addedSpec); }
@Override public void onUpdateSpec(Spec updatedSpec) { topologySpecMap.put(updatedSpec.getUri(), (TopologySpec) updatedSpec); }
@Override public Future<?> updateSpec(Spec updatedSpec) { if (!provisionedSpecs.containsKey(updatedSpec.getUri())) { throw new RuntimeException("Spec not found: " + updatedSpec.getUri()); } provisionedSpecs.put(updatedSpec.getUri(), updatedSpec); log.info(String.format("Updated Spec: %s with Uri: %s for execution on this executor.", updatedSpec, updatedSpec.getUri())); return new CompletedFuture(Boolean.TRUE, null); }
@Override public void addSpec(Spec spec) throws IOException { Preconditions.checkArgument(null != spec, "Spec should not be null"); log.info(String.format("Adding Spec with URI: %s in FSSpecStore: %s", spec.getUri(), this.fsSpecStoreDirPath)); Path specPath = getPathForURI(this.fsSpecStoreDirPath, spec.getUri(), spec.getVersion()); writeSpecToFile(specPath, spec); }
@Override public Spec updateSpec(Spec spec) throws IOException, SpecNotFoundException { Preconditions.checkArgument(null != spec, "Spec should not be null"); log.info(String.format("Updating Spec with URI: %s in FSSpecStore: %s", spec.getUri(), this.fsSpecStoreDirPath)); Path specPath = getPathForURI(this.fsSpecStoreDirPath, spec.getUri(), spec.getVersion()); writeSpecToFile(specPath, spec); return spec; }
@Override public Collection<Spec> getAllVersionsOfSpec(URI specUri) throws IOException, SpecNotFoundException { Preconditions.checkArgument(null != specUri, "Spec URI should not be null"); Collection<Spec> specs = getSpecs(); Collection<Spec> filteredSpecs = Lists.newArrayList(); for (Spec spec : specs) { if (spec.getUri().equals(specUri)) { filteredSpecs.add(spec); } } if (filteredSpecs.size() == 0) { throw new SpecNotFoundException(specUri); } return filteredSpecs; }
@Override public boolean deleteSpec(Spec spec) throws IOException { Preconditions.checkArgument(null != spec, "Spec should not be null"); return deleteSpec(spec.getUri(), spec.getVersion()); }
@Override public void remove(URI uri) { try { Preconditions.checkState(state() == Service.State.RUNNING, String.format("%s is not running.", this.getClass().getName())); Preconditions.checkNotNull(uri); log.info(String.format("Removing TopologySpec with URI: %s", uri)); Spec spec = specStore.getSpec(uri); this.listeners.onDeleteSpec(spec.getUri(), spec.getVersion()); specStore.deleteSpec(uri); } catch (IOException | SpecNotFoundException e) { throw new RuntimeException("Cannot delete Spec from Spec store for URI: " + uri, e); } }
@Override public void remove(URI uri) { try { Preconditions.checkState(state() == State.RUNNING, String.format("%s is not running.", this.getClass().getName())); Preconditions.checkNotNull(uri); log.info(String.format("Removing FlowSpec with URI: %s", uri)); Spec spec = specStore.getSpec(uri); this.listeners.onDeleteSpec(spec.getUri(), spec.getVersion()); specStore.deleteSpec(uri); } catch (IOException | SpecNotFoundException e) { throw new RuntimeException("Cannot delete Spec from Spec store for URI: " + uri, e); } }
private void submitTrackingEvent(Spec spec, String operType) { submitTrackingEvent(spec.getUri(), spec.getVersion(), operType); }
if (!isActive && helixManager.isPresent()) { _log.info("Scheduler running in slave mode, forward Spec add via Helix message to master: " + addedSpec); HelixUtils.sendUserDefinedMessage(ServiceConfigKeys.HELIX_FLOWSPEC_ADD, addedSpec.getUri().toString(), UUID.randomUUID().toString(), InstanceType.CONTROLLER, helixManager.get(), _log); return; Properties jobConfig = new Properties(); jobConfig.putAll(this.properties); jobConfig.setProperty(ConfigurationKeys.JOB_NAME_KEY, addedSpec.getUri().toString()); this.scheduledFlowSpecs.put(addedSpec.getUri().toString(), addedSpec);
/** {@inheritDoc} */ @Override public void onUpdateSpec(Spec updatedSpec) { _log.info("Spec changed: " + updatedSpec); if (!(updatedSpec instanceof TopologySpec)) { return; } try { onDeleteSpec(updatedSpec.getUri(), updatedSpec.getVersion()); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } try { onAddSpec(updatedSpec); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } }
@Override public void put(Spec spec) { try { Preconditions.checkState(state() == Service.State.RUNNING, String.format("%s is not running.", this.getClass().getName())); Preconditions.checkNotNull(spec); log.info(String.format("Adding TopologySpec with URI: %s and Config: %s", spec.getUri(), ((TopologySpec) spec).getConfigAsProperties())); if (specStore.exists(spec.getUri())) { specStore.updateSpec(spec); this.listeners.onUpdateSpec(spec); } else { specStore.addSpec(spec); this.listeners.onAddSpec(spec); } } catch (IOException | SpecNotFoundException e) { throw new RuntimeException("Cannot add Spec to Spec store: " + spec, e); } }
@Override public void put(Spec spec) { try { Preconditions.checkState(state() == State.RUNNING, String.format("%s is not running.", this.getClass().getName())); Preconditions.checkNotNull(spec); log.info(String.format("Adding FlowSpec with URI: %s and Config: %s", spec.getUri(), ((FlowSpec) spec).getConfigAsProperties())); if (specStore.exists(spec.getUri())) { specStore.updateSpec(spec); this.listeners.onUpdateSpec(spec); } else { specStore.addSpec(spec); this.listeners.onAddSpec(spec); } } catch (IOException | SpecNotFoundException e) { throw new RuntimeException("Cannot add Spec to Spec store: " + spec, e); } }
/** {@inheritDoc} */ @Override public void onUpdateSpec(Spec updatedSpec) { if (this.helixManager.isPresent() && !this.helixManager.get().isConnected()) { // Specs in store will be notified when Scheduler is added as listener to FlowCatalog, so ignore // .. Specs if in cluster mode and Helix is not yet initialized _log.info("System not yet initialized. Skipping Spec Update: " + updatedSpec); return; } _log.info("Spec changed: " + updatedSpec); if (!(updatedSpec instanceof FlowSpec)) { return; } if (!isActive && helixManager.isPresent()) { _log.info("Scheduler running in slave mode, forward Spec update via Helix message to master: " + updatedSpec); HelixUtils.sendUserDefinedMessage(ServiceConfigKeys.HELIX_FLOWSPEC_UPDATE, updatedSpec.getUri().toString(), UUID.randomUUID().toString(), InstanceType.CONTROLLER, helixManager.get(), _log); return; } try { onDeleteSpec(updatedSpec.getUri(), updatedSpec.getVersion()); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } try { onAddSpec(updatedSpec); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } }
/*** * TODO: Change cluster code to handle Spec. Right now all job properties are needed to be in config and template is not honored * TODO: Materialized JobSpec and make use of ResolvedJobSpec * @throws ExecutionException * @throws InterruptedException */ private void fetchJobSpecs() throws ExecutionException, InterruptedException { List<Pair<SpecExecutorInstance.Verb, Spec>> changesSpecs = (List<Pair<SpecExecutorInstance.Verb, Spec>>) this.specExecutorInstanceConsumer.changedSpecs().get(); for (Pair<SpecExecutorInstance.Verb, Spec> entry : changesSpecs) { SpecExecutorInstance.Verb verb = entry.getKey(); if (verb.equals(SpecExecutorInstance.Verb.ADD)) { // Handle addition JobSpec jobSpec = (JobSpec) entry.getValue(); postNewJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); jobSpecs.put(entry.getValue().getUri(), (JobSpec) entry.getValue()); } else if (verb.equals(SpecExecutorInstanceConsumer.Verb.UPDATE)) { // Handle update JobSpec jobSpec = (JobSpec) entry.getValue(); postUpdateJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); jobSpecs.put(entry.getValue().getUri(), (JobSpec) entry.getValue()); } else if (verb.equals(SpecExecutorInstanceConsumer.Verb.DELETE)) { // Handle delete Spec anonymousSpec = (Spec) entry.getValue(); postDeleteJobConfigArrival(anonymousSpec.getUri().toString(), new Properties()); jobSpecs.remove(entry.getValue().getUri()); } } }
public synchronized void setActive(boolean isActive) { if (this.isActive == isActive) { // No-op if already in correct state return; } // Since we are going to change status to isActive=true, schedule all flows if (isActive) { if (this.flowCatalog.isPresent()) { Collection<Spec> specs = this.flowCatalog.get().getSpecs(); for (Spec spec : specs) { onAddSpec(spec); } } } // Since we are going to change status to isActive=false, unschedule all flows else { for (Spec spec : this.scheduledFlowSpecs.values()) { onDeleteSpec(spec.getUri(), spec.getVersion()); } } // Change status after invoking addition / removal of specs, or else they will use isActive // .. to exhibit behavior for updated iActive value this.isActive = isActive; }
private void fetchJobSpecs() throws ExecutionException, InterruptedException { List<Pair<SpecExecutorInstance.Verb, Spec>> changesSpecs = (List<Pair<SpecExecutorInstance.Verb, Spec>>) this.specExecutorInstanceConsumer.changedSpecs().get(); // propagate thread interruption so that caller will exit from loop if (Thread.interrupted()) { throw new InterruptedException(); } for (Pair<SpecExecutorInstance.Verb, Spec> entry : changesSpecs) { SpecExecutorInstance.Verb verb = entry.getKey(); if (verb.equals(SpecExecutorInstance.Verb.ADD)) { // Handle addition JobSpec jobSpec = (JobSpec) entry.getValue(); postNewJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); } else if (verb.equals(SpecExecutorInstanceConsumer.Verb.UPDATE)) { // Handle update JobSpec jobSpec = (JobSpec) entry.getValue(); postUpdateJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); } else if (verb.equals(SpecExecutorInstanceConsumer.Verb.DELETE)) { // Handle delete Spec anonymousSpec = (Spec) entry.getValue(); postDeleteJobConfigArrival(anonymousSpec.getUri().toString(), new Properties()); } } }