@Override public synchronized void onUpdateSpec(Spec updatedSpec) { topologySpecMap.put(updatedSpec.getUri(), (TopologySpec) updatedSpec); }
@Override public Future<?> updateSpec(Spec updatedSpec) { if (!provisionedSpecs.containsKey(updatedSpec.getUri())) { throw new RuntimeException("Spec not found: " + updatedSpec.getUri()); } provisionedSpecs.put(updatedSpec.getUri(), updatedSpec); log.info(String.format("Updated Spec: %s with Uri: %s for execution on this executor.", updatedSpec, updatedSpec.getUri())); return new CompletedFuture(Boolean.TRUE, null); }
@Override public synchronized void onAddSpec(Spec addedSpec) { TopologySpec spec = (TopologySpec) addedSpec; log.info ("Loading topology {}", spec.toLongString()); for (Map.Entry entry: spec.getConfigAsProperties().entrySet()) { log.info ("topo: {} --> {}", entry.getKey(), entry.getValue()); } topologySpecMap.put(addedSpec.getUri(), (TopologySpec) addedSpec); }
@Override public void addSpec(Spec spec) throws IOException { Preconditions.checkArgument(null != spec, "Spec should not be null"); log.info(String.format("Adding Spec with URI: %s in FSSpecStore: %s", spec.getUri(), this.fsSpecStoreDirPath)); Path specPath = getPathForURI(this.fsSpecStoreDirPath, spec.getUri(), spec.getVersion()); writeSpecToFile(specPath, spec); }
@Override public void put(Spec spec) { try { Preconditions.checkState(state() == Service.State.RUNNING, String.format("%s is not running.", this.getClass().getName())); Preconditions.checkNotNull(spec); log.info(String.format("Adding TopologySpec with URI: %s and Config: %s", spec.getUri(), ((TopologySpec) spec).getConfigAsProperties())); specStore.addSpec(spec); this.listeners.onAddSpec(spec); } catch (IOException e) { throw new RuntimeException("Cannot add Spec to Spec store: " + spec, e); } }
@Override public boolean deleteSpec(Spec spec) throws IOException { Preconditions.checkArgument(null != spec, "Spec should not be null"); return deleteSpec(spec.getUri(), spec.getVersion()); }
public void put(Spec spec, boolean triggerListener) { try { Preconditions.checkState(state() == State.RUNNING, String.format("%s is not running.", this.getClass().getName())); Preconditions.checkNotNull(spec); long startTime = System.currentTimeMillis(); log.info(String.format("Adding FlowSpec with URI: %s and Config: %s", spec.getUri(), ((FlowSpec) spec).getConfigAsProperties())); specStore.addSpec(spec); metrics.updatePutSpecTime(startTime); if (triggerListener) { this.listeners.onAddSpec(spec); } } catch (IOException e) { throw new RuntimeException("Cannot add Spec to Spec store: " + spec, e); } }
/** {@inheritDoc} */ @Override public void onUpdateSpec(Spec updatedSpec) { _log.info("Spec changed: " + updatedSpec); if (!(updatedSpec instanceof TopologySpec)) { return; } try { onDeleteSpec(updatedSpec.getUri(), updatedSpec.getVersion()); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } try { onAddSpec(updatedSpec); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } }
private void submitTrackingEvent(Spec spec, String operType) { submitTrackingEvent(spec.getUri(), spec.getVersion(), operType); }
/** {@inheritDoc} */ @Override public void onUpdateSpec(Spec updatedSpec) { if (this.helixManager.isPresent() && !this.helixManager.get().isConnected()) { // Specs in store will be notified when Scheduler is added as listener to FlowCatalog, so ignore // .. Specs if in cluster mode and Helix is not yet initialized _log.info("System not yet initialized. Skipping Spec Update: " + updatedSpec); return; } _log.info("Spec changed: " + updatedSpec); if (!(updatedSpec instanceof FlowSpec)) { return; } try { onDeleteSpec(updatedSpec.getUri(), updatedSpec.getVersion()); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } try { onAddSpec(updatedSpec); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } }
@Test (dependsOnMethods = "testUpdateSpec") public void testDeleteSpec() throws Exception { // delete needs to be on a job spec that exists to get notification String deletedSpecUriString = "/foo/bar/addedSpec"; WriteResponse writeResponse = (WriteResponse) _seip.deleteSpec(new URI(deletedSpecUriString)).get(); log.info("WriteResponse: " + writeResponse); List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get(); Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production"); Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0); Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.DELETE), "Verb did not match"); Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(deletedSpecUriString), "Expected URI did not match"); Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec"); }
private void fetchJobSpecs() throws ExecutionException, InterruptedException { List<Pair<SpecExecutor.Verb, Spec>> changesSpecs = (List<Pair<SpecExecutor.Verb, Spec>>) this.specConsumer.changedSpecs().get(); // propagate thread interruption so that caller will exit from loop if (Thread.interrupted()) { throw new InterruptedException(); } for (Pair<SpecExecutor.Verb, Spec> entry : changesSpecs) { SpecExecutor.Verb verb = entry.getKey(); if (verb.equals(SpecExecutor.Verb.ADD)) { // Handle addition JobSpec jobSpec = (JobSpec) entry.getValue(); postNewJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); } else if (verb.equals(SpecExecutor.Verb.UPDATE)) { // Handle update JobSpec jobSpec = (JobSpec) entry.getValue(); postUpdateJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); } else if (verb.equals(SpecExecutor.Verb.DELETE)) { // Handle delete Spec anonymousSpec = (Spec) entry.getValue(); postDeleteJobConfigArrival(anonymousSpec.getUri().toString(), new Properties()); } } }
@Test (dependsOnMethods = "testAddSpec") public void testUpdateSpec() throws Exception { // update is only treated as an update for existing job specs String updatedSpecUriString = "/foo/bar/addedSpec"; Spec spec = initJobSpec(updatedSpecUriString); WriteResponse writeResponse = (WriteResponse) _seip.updateSpec(spec).get(); log.info("WriteResponse: " + writeResponse); List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get(); Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production"); Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0); Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.UPDATE), "Verb did not match"); Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(updatedSpecUriString), "Expected URI did not match"); Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec"); }
@Test (dependsOnMethods = "testUpdateSpec") public void testDeleteSpec() throws Exception { String deletedSpecUriString = "/foo/bar/deletedSpec"; WriteResponse writeResponse = (WriteResponse) _seip.deleteSpec(new URI(deletedSpecUriString)).get(); log.info("WriteResponse: " + writeResponse); try { Thread.sleep(1000); } catch(InterruptedException ex) { Thread.currentThread().interrupt(); } List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get(); Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production"); Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0); Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.DELETE), "Verb did not match"); Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(deletedSpecUriString), "Expected URI did not match"); Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec"); }
@Test (dependsOnMethods = "testAddSpec") public void testUpdateSpec() throws Exception { String updatedSpecUriString = "/foo/bar/updatedSpec"; Spec spec = initJobSpec(updatedSpecUriString); WriteResponse writeResponse = (WriteResponse) _seip.updateSpec(spec).get(); log.info("WriteResponse: " + writeResponse); try { Thread.sleep(1000); } catch(InterruptedException ex) { Thread.currentThread().interrupt(); } List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get(); Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production"); Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0); Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.UPDATE), "Verb did not match"); Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(updatedSpecUriString), "Expected URI did not match"); Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec"); }
public void remove(Spec spec, Properties headers) { // TODO: Evolve logic to cache and reuse previously compiled JobSpecs // .. this will work for Identity compiler but not always for multi-hop. // Note: Current logic assumes compilation is consistent between all executions if (spec instanceof FlowSpec) { Dag<JobExecutionPlan> jobExecutionPlanDag = specCompiler.compileFlow(spec); if (jobExecutionPlanDag.isEmpty()) { _log.warn("Cannot determine an executor to delete Spec: " + spec); return; } // Delete all compiled JobSpecs on their respective Executor for (Dag.DagNode<JobExecutionPlan> dagNode: jobExecutionPlanDag.getNodes()) { JobExecutionPlan jobExecutionPlan = dagNode.getValue(); // Delete this spec on selected executor SpecProducer producer = null; try { producer = jobExecutionPlan.getSpecExecutor().getProducer().get(); Spec jobSpec = jobExecutionPlan.getJobSpec(); _log.info(String.format("Going to delete JobSpec: %s on Executor: %s", jobSpec, producer)); producer.deleteSpec(jobSpec.getUri(), headers); } catch (Exception e) { _log.error("Cannot successfully delete spec: " + jobExecutionPlan.getJobSpec() + " on executor: " + producer + " for flow: " + spec, e); } } } else { throw new RuntimeException("Spec not of type FlowSpec, cannot delete: " + spec); } }
public synchronized void setActive(boolean isActive) { if (this.isActive == isActive) { // No-op if already in correct state return; } // Since we are going to change status to isActive=true, schedule all flows if (isActive) { // Need to set active=true first; otherwise in the onAddSpec(), node will forward specs to active node, which is itself. this.isActive = isActive; if (this.flowCatalog.isPresent()) { Collection<Spec> specs = this.flowCatalog.get().getSpecsWithTimeUpdate(); for (Spec spec : specs) { //Disable FLOW_RUN_IMMEDIATELY on service startup or leadership change if (spec instanceof FlowSpec) { Spec modifiedSpec = disableFlowRunImmediatelyOnStart((FlowSpec) spec); onAddSpec(modifiedSpec); } else { onAddSpec(spec); } } } } else { // Since we are going to change status to isActive=false, unschedule all flows for (Spec spec : this.scheduledFlowSpecs.values()) { onDeleteSpec(spec.getUri(), spec.getVersion()); } // Need to set active=false at the end; otherwise in the onDeleteSpec(), node will forward specs to active node, which is itself. this.isActive = isActive; } }