for (Either<JobSpec, URI> parsedMessage : parsedCollection) { JobSpec jobSpec = ((Either.Left<JobSpec, URI>)parsedMessage).getLeft(); if (jobSpec.getMetadata().get(JobSpec.VERB_KEY).equalsIgnoreCase(SpecExecutor.Verb.DELETE.name())) { this.removedSpecs.inc(); URI jobSpecUri = jobSpec.getUri();
Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.ADD), "Verb did not match"); Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(addedSpecUriString), "Expected URI did not match"); Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
SpecExecutor.Verb verb = SpecExecutor.Verb.valueOf(verbName);
for (Either<JobSpec, URI> parsedMessage : parsedCollection) { JobSpec jobSpec = ((Either.Left<JobSpec, URI>)parsedMessage).getLeft(); if (jobSpec.getMetadata().get(JobSpec.VERB_KEY).equalsIgnoreCase(SpecExecutor.Verb.DELETE.name())) { this.removedSpecs.inc(); URI jobSpecUri = jobSpec.getUri();
/*** * TODO: Change cluster code to handle Spec. Right now all job properties are needed to be in config and template is not honored * TODO: Materialized JobSpec and make use of ResolvedJobSpec * @throws ExecutionException * @throws InterruptedException */ private void fetchJobSpecs() throws ExecutionException, InterruptedException { List<Pair<SpecExecutor.Verb, Spec>> changesSpecs = (List<Pair<SpecExecutor.Verb, Spec>>) this._specConsumer.changedSpecs().get(); for (Pair<SpecExecutor.Verb, Spec> entry : changesSpecs) { SpecExecutor.Verb verb = entry.getKey(); if (verb.equals(SpecExecutor.Verb.ADD)) { // Handle addition JobSpec jobSpec = (JobSpec) entry.getValue(); postNewJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); jobSpecs.put(entry.getValue().getUri(), (JobSpec) entry.getValue()); } else if (verb.equals(SpecExecutor.Verb.UPDATE)) { // Handle update JobSpec jobSpec = (JobSpec) entry.getValue(); postUpdateJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); jobSpecs.put(entry.getValue().getUri(), (JobSpec) entry.getValue()); } else if (verb.equals(SpecExecutor.Verb.DELETE)) { // Handle delete Spec anonymousSpec = (Spec) entry.getValue(); postDeleteJobConfigArrival(anonymousSpec.getUri().toString(), new Properties()); jobSpecs.remove(entry.getValue().getUri()); } } }
private void fetchJobSpecs() throws ExecutionException, InterruptedException { List<Pair<SpecExecutor.Verb, Spec>> changesSpecs = (List<Pair<SpecExecutor.Verb, Spec>>) this.specConsumer.changedSpecs().get(); // propagate thread interruption so that caller will exit from loop if (Thread.interrupted()) { throw new InterruptedException(); } for (Pair<SpecExecutor.Verb, Spec> entry : changesSpecs) { SpecExecutor.Verb verb = entry.getKey(); if (verb.equals(SpecExecutor.Verb.ADD)) { // Handle addition JobSpec jobSpec = (JobSpec) entry.getValue(); postNewJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); } else if (verb.equals(SpecExecutor.Verb.UPDATE)) { // Handle update JobSpec jobSpec = (JobSpec) entry.getValue(); postUpdateJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); } else if (verb.equals(SpecExecutor.Verb.DELETE)) { // Handle delete Spec anonymousSpec = (Spec) entry.getValue(); postDeleteJobConfigArrival(anonymousSpec.getUri().toString(), new Properties()); } } }
private AvroJobSpec convertToAvroJobSpec(Spec spec, SpecExecutor.Verb verb) { if (spec instanceof JobSpec) { JobSpec jobSpec = (JobSpec) spec; AvroJobSpec.Builder avroJobSpecBuilder = AvroJobSpec.newBuilder(); avroJobSpecBuilder.setUri(jobSpec.getUri().toString()).setVersion(jobSpec.getVersion()) .setDescription(jobSpec.getDescription()).setProperties(Maps.fromProperties(jobSpec.getConfigAsProperties())) .setMetadata(ImmutableMap.of(VERB_KEY, verb.name())); if (jobSpec.getTemplateURI().isPresent()) { avroJobSpecBuilder.setTemplateUri(jobSpec.getTemplateURI().get().toString()); } return avroJobSpecBuilder.build(); } else { throw new RuntimeException("Unsupported spec type " + spec.getClass()); } } }
public Map getDefaultMetadata() { log.warn("Job Spec Verb is not provided, using type 'UNKNOWN'."); return ImmutableMap.of(VERB_KEY, SpecExecutor.Verb.UNKNOWN.name()); }
@Override public Future<?> deleteSpec(URI deletedSpecURI, Properties headers) { AvroJobSpec avroJobSpec = AvroJobSpec.newBuilder().setUri(deletedSpecURI.toString()) .setMetadata(ImmutableMap.of(VERB_KEY, SpecExecutor.Verb.DELETE.name())) .setProperties(Maps.fromProperties(headers)).build(); log.info("Deleting Spec: " + deletedSpecURI + " using Kafka."); return getKafkaProducer().write(_serializer.serializeRecord(avroJobSpec), WriteCallback.EMPTY); }
SpecExecutor.Verb verb = SpecExecutor.Verb.valueOf(verbName);
@Test public void testAddSpec() throws Exception { _closer = Closer.create(); _properties = new Properties(); // Properties for Producer _properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, TOPIC); _properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", _kafkaBrokers); _properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX+"value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer"); // Properties for Consumer _properties.setProperty(ConfigurationKeys.KAFKA_BROKERS, _kafkaBrokers); _properties.setProperty(SimpleKafkaSpecExecutor.SPEC_KAFKA_TOPICS_KEY, TOPIC); // SEI Producer _seip = _closer.register(new SimpleKafkaSpecProducer(ConfigUtils.propertiesToConfig(_properties))); String addedSpecUriString = "/foo/bar/addedSpec"; Spec spec = initJobSpec(addedSpecUriString); WriteResponse writeResponse = (WriteResponse) _seip.addSpec(spec).get(); log.info("WriteResponse: " + writeResponse); try { Thread.sleep(1000); } catch(InterruptedException ex) { Thread.currentThread().interrupt(); } _seic = _closer.register(new SimpleKafkaSpecConsumer(ConfigUtils.propertiesToConfig(_properties))); List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get(); Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production"); Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0); Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.ADD), "Verb did not match"); Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(addedSpecUriString), "Expected URI did not match"); Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec"); }
/*** * TODO: Change cluster code to handle Spec. Right now all job properties are needed to be in config and template is not honored * TODO: Materialized JobSpec and make use of ResolvedJobSpec * @throws ExecutionException * @throws InterruptedException */ private void fetchJobSpecs() throws ExecutionException, InterruptedException { List<Pair<SpecExecutor.Verb, Spec>> changesSpecs = (List<Pair<SpecExecutor.Verb, Spec>>) this._specConsumer.changedSpecs().get(); for (Pair<SpecExecutor.Verb, Spec> entry : changesSpecs) { SpecExecutor.Verb verb = entry.getKey(); if (verb.equals(SpecExecutor.Verb.ADD)) { // Handle addition JobSpec jobSpec = (JobSpec) entry.getValue(); postNewJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); jobSpecs.put(entry.getValue().getUri(), (JobSpec) entry.getValue()); } else if (verb.equals(SpecExecutor.Verb.UPDATE)) { // Handle update JobSpec jobSpec = (JobSpec) entry.getValue(); postUpdateJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); jobSpecs.put(entry.getValue().getUri(), (JobSpec) entry.getValue()); } else if (verb.equals(SpecExecutor.Verb.DELETE)) { // Handle delete Spec anonymousSpec = (Spec) entry.getValue(); postDeleteJobConfigArrival(anonymousSpec.getUri().toString(), new Properties()); jobSpecs.remove(entry.getValue().getUri()); } } }
private void fetchJobSpecs() throws ExecutionException, InterruptedException { List<Pair<SpecExecutor.Verb, Spec>> changesSpecs = (List<Pair<SpecExecutor.Verb, Spec>>) this.specConsumer.changedSpecs().get(); // propagate thread interruption so that caller will exit from loop if (Thread.interrupted()) { throw new InterruptedException(); } for (Pair<SpecExecutor.Verb, Spec> entry : changesSpecs) { SpecExecutor.Verb verb = entry.getKey(); if (verb.equals(SpecExecutor.Verb.ADD)) { // Handle addition JobSpec jobSpec = (JobSpec) entry.getValue(); postNewJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); } else if (verb.equals(SpecExecutor.Verb.UPDATE)) { // Handle update JobSpec jobSpec = (JobSpec) entry.getValue(); postUpdateJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); } else if (verb.equals(SpecExecutor.Verb.DELETE)) { // Handle delete Spec anonymousSpec = (Spec) entry.getValue(); postDeleteJobConfigArrival(anonymousSpec.getUri().toString(), new Properties()); } } }
@Test (dependsOnMethods = "testUpdateSpec") public void testDeleteSpec() throws Exception { String deletedSpecUriString = "/foo/bar/deletedSpec"; WriteResponse writeResponse = (WriteResponse) _seip.deleteSpec(new URI(deletedSpecUriString)).get(); log.info("WriteResponse: " + writeResponse); try { Thread.sleep(1000); } catch(InterruptedException ex) { Thread.currentThread().interrupt(); } List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get(); Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production"); Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0); Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.DELETE), "Verb did not match"); Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(deletedSpecUriString), "Expected URI did not match"); Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec"); }
@Test (dependsOnMethods = "testAddSpec") public void testUpdateSpec() throws Exception { String updatedSpecUriString = "/foo/bar/updatedSpec"; Spec spec = initJobSpec(updatedSpecUriString); WriteResponse writeResponse = (WriteResponse) _seip.updateSpec(spec).get(); log.info("WriteResponse: " + writeResponse); try { Thread.sleep(1000); } catch(InterruptedException ex) { Thread.currentThread().interrupt(); } List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get(); Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production"); Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0); Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.UPDATE), "Verb did not match"); Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(updatedSpecUriString), "Expected URI did not match"); Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec"); }
private AvroJobSpec convertToAvroJobSpec(Spec spec, SpecExecutor.Verb verb) { if (spec instanceof JobSpec) { JobSpec jobSpec = (JobSpec) spec; AvroJobSpec.Builder avroJobSpecBuilder = AvroJobSpec.newBuilder(); avroJobSpecBuilder.setUri(jobSpec.getUri().toString()).setVersion(jobSpec.getVersion()) .setDescription(jobSpec.getDescription()).setProperties(Maps.fromProperties(jobSpec.getConfigAsProperties())) .setMetadata(ImmutableMap.of(VERB_KEY, verb.name())); if (jobSpec.getTemplateURI().isPresent()) { avroJobSpecBuilder.setTemplateUri(jobSpec.getTemplateURI().get().toString()); } return avroJobSpecBuilder.build(); } else { throw new RuntimeException("Unsupported spec type " + spec.getClass()); } } }
public Map getDefaultMetadata() { log.warn("Job Spec Verb is not provided, using type 'UNKNOWN'."); return ImmutableMap.of(VERB_KEY, SpecExecutor.Verb.UNKNOWN.name()); }
@Test (dependsOnMethods = "testUpdateSpec") public void testDeleteSpec() throws Exception { // delete needs to be on a job spec that exists to get notification String deletedSpecUriString = "/foo/bar/addedSpec"; WriteResponse writeResponse = (WriteResponse) _seip.deleteSpec(new URI(deletedSpecUriString)).get(); log.info("WriteResponse: " + writeResponse); List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get(); Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production"); Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0); Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.DELETE), "Verb did not match"); Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(deletedSpecUriString), "Expected URI did not match"); Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec"); }
@Test (dependsOnMethods = "testAddSpec") public void testUpdateSpec() throws Exception { // update is only treated as an update for existing job specs String updatedSpecUriString = "/foo/bar/addedSpec"; Spec spec = initJobSpec(updatedSpecUriString); WriteResponse writeResponse = (WriteResponse) _seip.updateSpec(spec).get(); log.info("WriteResponse: " + writeResponse); List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get(); Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production"); Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0); Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.UPDATE), "Verb did not match"); Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(updatedSpecUriString), "Expected URI did not match"); Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec"); }
@Override public Future<?> deleteSpec(URI deletedSpecURI, Properties headers) { AvroJobSpec avroJobSpec = AvroJobSpec.newBuilder().setUri(deletedSpecURI.toString()) .setMetadata(ImmutableMap.of(VERB_KEY, SpecExecutor.Verb.DELETE.name())) .setProperties(Maps.fromProperties(headers)).build(); log.info("Deleting Spec: " + deletedSpecURI + " using Kafka."); return getKafkaProducer().write(_serializer.serializeRecord(avroJobSpec), WriteCallback.EMPTY); }