@Override public synchronized void onUpdateSpec(Spec updatedSpec) { topologySpecMap.put(updatedSpec.getUri(), (TopologySpec) updatedSpec); }
@Override public boolean deleteSpec(Spec spec) throws IOException { Preconditions.checkArgument(null != spec, "Spec should not be null"); return deleteSpec(spec.getUri(), spec.getVersion()); }
@Override public Spec getSpec(URI specUri) throws SpecNotFoundException { Preconditions.checkArgument(null != specUri, "Spec URI should not be null"); Collection<Spec> specs = getAllVersionsOfSpec(specUri); Spec highestVersionSpec = null; for (Spec spec : specs) { if (null == highestVersionSpec) { highestVersionSpec = spec; } else if (null != spec.getVersion() && spec.getVersion().compareTo(spec.getVersion()) > 0) { highestVersionSpec = spec; } } if (null == highestVersionSpec) { throw new SpecNotFoundException(specUri); } return highestVersionSpec; }
@Override public Spec getSpec(URI specUri) throws SpecNotFoundException { Preconditions.checkArgument(null != specUri, "Spec URI should not be null"); Collection<Spec> specs = getAllVersionsOfSpec(specUri); Spec highestVersionSpec = null; for (Spec spec : specs) { if (null == highestVersionSpec) { highestVersionSpec = spec; } else if (null != spec.getVersion() && spec.getVersion().compareTo(spec.getVersion()) > 0) { highestVersionSpec = spec; } } if (null == highestVersionSpec) { throw new SpecNotFoundException(specUri); } return highestVersionSpec; }
@Override public Future<?> updateSpec(Spec updatedSpec) { if (!provisionedSpecs.containsKey(updatedSpec.getUri())) { throw new RuntimeException("Spec not found: " + updatedSpec.getUri()); } provisionedSpecs.put(updatedSpec.getUri(), updatedSpec); log.info(String.format("Updated Spec: %s with Uri: %s for execution on this executor.", updatedSpec, updatedSpec.getUri())); return new CompletedFuture(Boolean.TRUE, null); }
private void submitTrackingEvent(Spec spec, String operType) { submitTrackingEvent(spec.getUri(), spec.getVersion(), operType); }
@Override public void addSpec(Spec spec) throws IOException { Preconditions.checkArgument(null != spec, "Spec should not be null"); log.info(String.format("Adding Spec with URI: %s in FSSpecStore: %s", spec.getUri(), this.fsSpecStoreDirPath)); Path specPath = getPathForURI(this.fsSpecStoreDirPath, spec.getUri(), spec.getVersion()); writeSpecToFile(specPath, spec); }
@Override public synchronized void onAddSpec(Spec addedSpec) { TopologySpec spec = (TopologySpec) addedSpec; log.info ("Loading topology {}", spec.toLongString()); for (Map.Entry entry: spec.getConfigAsProperties().entrySet()) { log.info ("topo: {} --> {}", entry.getKey(), entry.getValue()); } topologySpecMap.put(addedSpec.getUri(), (TopologySpec) addedSpec); }
/** {@inheritDoc} */ @Override public void onUpdateSpec(Spec updatedSpec) { _log.info("Spec changed: " + updatedSpec); if (!(updatedSpec instanceof TopologySpec)) { return; } try { onDeleteSpec(updatedSpec.getUri(), updatedSpec.getVersion()); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } try { onAddSpec(updatedSpec); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } }
@Override public void put(Spec spec) { try { Preconditions.checkState(state() == Service.State.RUNNING, String.format("%s is not running.", this.getClass().getName())); Preconditions.checkNotNull(spec); log.info(String.format("Adding TopologySpec with URI: %s and Config: %s", spec.getUri(), ((TopologySpec) spec).getConfigAsProperties())); specStore.addSpec(spec); this.listeners.onAddSpec(spec); } catch (IOException e) { throw new RuntimeException("Cannot add Spec to Spec store: " + spec, e); } }
/** {@inheritDoc} */ @Override public void onUpdateSpec(Spec updatedSpec) { if (this.helixManager.isPresent() && !this.helixManager.get().isConnected()) { // Specs in store will be notified when Scheduler is added as listener to FlowCatalog, so ignore // .. Specs if in cluster mode and Helix is not yet initialized _log.info("System not yet initialized. Skipping Spec Update: " + updatedSpec); return; } _log.info("Spec changed: " + updatedSpec); if (!(updatedSpec instanceof FlowSpec)) { return; } try { onDeleteSpec(updatedSpec.getUri(), updatedSpec.getVersion()); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } try { onAddSpec(updatedSpec); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } }
public synchronized void setActive(boolean isActive) { if (this.isActive == isActive) { // No-op if already in correct state return; } // Since we are going to change status to isActive=true, schedule all flows if (isActive) { // Need to set active=true first; otherwise in the onAddSpec(), node will forward specs to active node, which is itself. this.isActive = isActive; if (this.flowCatalog.isPresent()) { Collection<Spec> specs = this.flowCatalog.get().getSpecsWithTimeUpdate(); for (Spec spec : specs) { //Disable FLOW_RUN_IMMEDIATELY on service startup or leadership change if (spec instanceof FlowSpec) { Spec modifiedSpec = disableFlowRunImmediatelyOnStart((FlowSpec) spec); onAddSpec(modifiedSpec); } else { onAddSpec(spec); } } } } else { // Since we are going to change status to isActive=false, unschedule all flows for (Spec spec : this.scheduledFlowSpecs.values()) { onDeleteSpec(spec.getUri(), spec.getVersion()); } // Need to set active=false at the end; otherwise in the onDeleteSpec(), node will forward specs to active node, which is itself. this.isActive = isActive; } }
@Override public boolean deleteSpec(Spec spec) throws IOException { Preconditions.checkArgument(null != spec, "Spec should not be null"); return deleteSpec(spec.getUri(), spec.getVersion()); }
public void put(Spec spec, boolean triggerListener) { try { Preconditions.checkState(state() == State.RUNNING, String.format("%s is not running.", this.getClass().getName())); Preconditions.checkNotNull(spec); long startTime = System.currentTimeMillis(); log.info(String.format("Adding FlowSpec with URI: %s and Config: %s", spec.getUri(), ((FlowSpec) spec).getConfigAsProperties())); specStore.addSpec(spec); metrics.updatePutSpecTime(startTime); if (triggerListener) { this.listeners.onAddSpec(spec); } } catch (IOException e) { throw new RuntimeException("Cannot add Spec to Spec store: " + spec, e); } }
private void submitTrackingEvent(Spec spec, String operType) { submitTrackingEvent(spec.getUri(), spec.getVersion(), operType); }
/*** * TODO: Change cluster code to handle Spec. Right now all job properties are needed to be in config and template is not honored * TODO: Materialized JobSpec and make use of ResolvedJobSpec * @throws ExecutionException * @throws InterruptedException */ private void fetchJobSpecs() throws ExecutionException, InterruptedException { List<Pair<SpecExecutor.Verb, Spec>> changesSpecs = (List<Pair<SpecExecutor.Verb, Spec>>) this._specConsumer.changedSpecs().get(); for (Pair<SpecExecutor.Verb, Spec> entry : changesSpecs) { SpecExecutor.Verb verb = entry.getKey(); if (verb.equals(SpecExecutor.Verb.ADD)) { // Handle addition JobSpec jobSpec = (JobSpec) entry.getValue(); postNewJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); jobSpecs.put(entry.getValue().getUri(), (JobSpec) entry.getValue()); } else if (verb.equals(SpecExecutor.Verb.UPDATE)) { // Handle update JobSpec jobSpec = (JobSpec) entry.getValue(); postUpdateJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); jobSpecs.put(entry.getValue().getUri(), (JobSpec) entry.getValue()); } else if (verb.equals(SpecExecutor.Verb.DELETE)) { // Handle delete Spec anonymousSpec = (Spec) entry.getValue(); postDeleteJobConfigArrival(anonymousSpec.getUri().toString(), new Properties()); jobSpecs.remove(entry.getValue().getUri()); } } }
@Override public void addSpec(Spec spec) throws IOException { Preconditions.checkArgument(null != spec, "Spec should not be null"); log.info(String.format("Adding Spec with URI: %s in FSSpecStore: %s", spec.getUri(), this.fsSpecStoreDirPath)); Path specPath = getPathForURI(this.fsSpecStoreDirPath, spec.getUri(), spec.getVersion()); writeSpecToFile(specPath, spec); }