@Override public boolean apply(JobSpec input) { Preconditions.checkNotNull(input); boolean res = true; if (this.uriPredicate.isPresent()) { res &= this.uriPredicate.get().apply(input.getUri()); } if (res && this.versionPredicate.isPresent()) { res &= this.versionPredicate.get().apply(input.getVersion()); } return res; }
public String toShortString() { return getUri().toString() + "/" + getVersion(); }
public String toLongString() { return getUri().toString() + "/" + getVersion() + "[" + getDescription() + "]"; }
@Override public void remove(URI uri) { Preconditions.checkState(state() == State.RUNNING, String.format("%s is not running.", this.getClass().getName())); Preconditions.checkNotNull(uri); JobSpec jobSpec = doRemove(uri); if (null != jobSpec) { this.listeners.onDeleteJob(jobSpec.getUri(), jobSpec.getVersion()); } }
private void submitTrackingEvent(JobSpec job, String operType) { submitTrackingEvent(job.getUri(), job.getVersion(), operType); }
/** * Allow user to programmatically delete a new JobSpec. * This method is designed to be reentrant. * @param jobURI The relative Path that specified by user, need to make it into complete path. */ @Override public synchronized void remove(URI jobURI) { Preconditions.checkState(state() == State.RUNNING, String.format("%s is not running.", this.getClass().getName())); try { long startTime = System.currentTimeMillis(); JobSpec jobSpec = getJobSpec(jobURI); Path jobSpecPath = getPathForURI(this.jobConfDirPath, jobURI); if (fs.exists(jobSpecPath)) { fs.delete(jobSpecPath, false); this.mutableMetrics.updateRemoveJobTime(startTime); this.listeners.onDeleteJob(jobURI, jobSpec.getVersion()); } else { LOGGER.warn("No file with URI:" + jobSpecPath + " is found. Deletion failed."); } } catch (IOException e) { throw new RuntimeException("When removing a JobConf. file, issues unexpected happen:" + e.getMessage()); } catch (SpecNotFoundException e) { LOGGER.warn("No file with URI:" + jobURI + " is found. Deletion failed."); } } }
.put(ImmutableFSJobCatalog.VERSION_KEY_IN_JOBSPEC, jobSpec.getVersion());
public static JobExecutionUpdatable createFromJobSpec(JobSpec jobSpec) { return new JobExecutionUpdatable(jobSpec.getUri(), jobSpec.getVersion(), System.currentTimeMillis(), JobLauncherUtils.newJobId(JobState.getJobNameFromProps(jobSpec.getConfigAsProperties()))); } }
String uri = (jobSpec.getUri() != null) ? jobSpec.getUri().toString() : null; jobSpecJson.addProperty(SerializationConstants.JOB_SPEC_URI_KEY, uri); jobSpecJson.addProperty(SerializationConstants.JOB_SPEC_VERSION_KEY, jobSpec.getVersion()); jobSpecJson.addProperty(SerializationConstants.JOB_SPEC_DESCRIPTION_KEY, jobSpec.getDescription()); String jobSpecTemplateURI = (jobSpec.getTemplateURI().isPresent()) ? jobSpec.getTemplateURI().get().toString() : null;
private AvroJobSpec convertToAvroJobSpec(Spec spec, SpecExecutor.Verb verb) { if (spec instanceof JobSpec) { JobSpec jobSpec = (JobSpec) spec; AvroJobSpec.Builder avroJobSpecBuilder = AvroJobSpec.newBuilder(); avroJobSpecBuilder.setUri(jobSpec.getUri().toString()).setVersion(jobSpec.getVersion()) .setDescription(jobSpec.getDescription()).setProperties(Maps.fromProperties(jobSpec.getConfigAsProperties())) .setMetadata(ImmutableMap.of(VERB_KEY, verb.name())); if (jobSpec.getTemplateURI().isPresent()) { avroJobSpecBuilder.setTemplateUri(jobSpec.getTemplateURI().get().toString()); } return avroJobSpecBuilder.build(); } else { throw new RuntimeException("Unsupported spec type " + spec.getClass()); } } }
/** * Resolve the job spec using classpath templates as well as any templates available in the input {@link JobCatalog}. */ public ResolvedJobSpec(JobSpec other, JobCatalog catalog) throws SpecNotFoundException, JobTemplate.TemplateException { super(other.getUri(), other.getVersion(), other.getDescription(), resolveConfig(other, catalog), ConfigUtils.configToProperties(resolveConfig(other, catalog)), other.getTemplateURI(), other.getMetadata()); this.originalJobSpec = other; }
monitor.awaitExactlyNSpecs(1); Assert.assertTrue(monitor.getJobSpecs().containsKey(new URI("job1"))); Assert.assertEquals(monitor.getJobSpecs().get(new URI("job1")).getVersion(), "1"); Assert.assertEquals(monitor.getJobSpecs().get(new URI("job2")).getVersion(), "1"); monitor.awaitExactlyNSpecs(2); Assert.assertTrue(monitor.getJobSpecs().containsKey(new URI("job1"))); Assert.assertEquals(monitor.getJobSpecs().get(new URI("job1")).getVersion(), "2"); Assert.assertTrue(monitor.getJobSpecs().containsKey(new URI("job2"))); Assert.assertEquals(monitor.getJobSpecs().get(new URI("job2")).getVersion(), "2");
.onDeleteJob(Mockito.eq(js2.getUri()), Mockito.eq(js2.getVersion())); ll.onAddJob(js2); ll.onUpdateJob(js1_2); ll.onDeleteJob(js2.getUri(), js2.getVersion()); Mockito.verify(l1).onDeleteJob(Mockito.eq(js2.getUri()), Mockito.eq(js2.getVersion())); Mockito.verify(l2).onDeleteJob(Mockito.eq(js2.getUri()), Mockito.eq(js2.getVersion())); Mockito.verify(l3).onDeleteJob(Mockito.eq(js2.getUri()), Mockito.eq(js2.getVersion()));
Assert.assertEquals(js1Deserialized.getVersion(), js1.getVersion()); Assert.assertNotNull(js1Deserialized.getDescription()); Assert.assertTrue(js1Deserialized.getDescription().contains(js1.getDescription())); Assert.assertEquals(js2Deserialized.getVersion(), js2.getVersion()); Assert.assertEquals(js2Deserialized.getDescription(), js2.getDescription()); Assert.assertEquals(js2Deserialized.getConfig().getString("a1"), "a_value");
testListener.onDeleteJob(js1_1.getUri(), js1_1.getVersion()); testListener.onUpdateJob(js1_1);; testListener.onStatusChange(jss1_1, RunningState.PENDING, RunningState.RUNNING); testListener.onDeleteJob(js1_2.getUri(), js1_2.getVersion()); testListener.onUpdateJob(js1_2); testListener.onStatusChange(jss1_2, RunningState.RUNNING, RunningState.SUCCESSFUL); eq(js1_2.getVersion())); verify(mockListener).onUpdateJob(eq(js1_2)); verify(mockListener).onStatusChange(eq(jss1_2), eq(RunningState.RUNNING), verify(mockListener, never()).onDeleteJob(eq(js1_1.getUri()), eq(js1_1.getVersion())); verify(mockListener, never()).onUpdateJob(eq(js1_1)); verify(mockListener, never()).onStatusChange(eq(jss1_1), eq(RunningState.RUNNING),
JobSpec js1_1_notified = specs.get(js1_1.getUri()); Assert.assertTrue(ConfigUtils.verifySubset(js1_1_notified.getConfig(), js1_1.getConfig())); Assert.assertEquals(js1_1.getVersion(), js1_1_notified.getVersion()); JobSpec js1_2_notified = specs.get(js1_2.getUri()); Assert.assertTrue(ConfigUtils.verifySubset(js1_2_notified.getConfig(), js1_2.getConfig())); Assert.assertEquals(js1_2.getVersion(), js1_2_notified.getVersion()); JobSpec js2_notified = specs.get(js2.getUri()); Assert.assertTrue(ConfigUtils.verifySubset(js2_notified.getConfig(), js2.getConfig())); Assert.assertEquals(js2.getVersion(), js2_notified.getVersion()); JobSpec js3_notified = specs.get(js3.getUri()); Assert.assertTrue(ConfigUtils.verifySubset(js3_notified.getConfig(), js3.getConfig())); Assert.assertEquals(js3.getVersion(), js3_notified.getVersion()); ResolvedJobSpec js3_resolved = new ResolvedJobSpec(js3_notified, cat); Assert.assertEquals(js3_resolved.getConfig().getString("param1"), "value1");
Assert.assertEquals(js1.getVersion(), "1"); Assert.assertNotNull(js1.getDescription()); Assert.assertTrue(js1.getDescription().contains("test:job")); Assert.assertEquals(js2.getVersion(), "2"); Assert.assertEquals(js2.getDescription(), "A test job"); Assert.assertEquals(js2.getConfig().getString("a1"), "a_value"); Assert.assertEquals(js3.getVersion(), "3"); Assert.assertEquals(js3.getDescription(), "A test job"); Assert.assertEquals(js3.getConfigAsProperties().getProperty("a1"), "some_string");
Mockito.verify(l).onDeleteJob(Mockito.eq(js1_2.getUri()), Mockito.eq(js1_2.getVersion()));
@Test public void testCallbacks() throws Exception { InMemoryJobCatalog cat = new InMemoryJobCatalog(); cat.startAsync(); cat.awaitRunning(1, TimeUnit.SECONDS); JobCatalogListener l = Mockito.mock(JobCatalogListener.class); JobSpec js1_1 = JobSpec.builder("test:job1").withVersion("1").build(); JobSpec js1_2 = JobSpec.builder("test:job1").withVersion("2").build(); JobSpec js1_3 = JobSpec.builder("test:job1").withVersion("3").build(); JobSpec js2 = JobSpec.builder("test:job2").withVersion("1").build(); cat.put(js1_1); cat.addListener(l); cat.put(js1_2); cat.put(js2); cat.put(js1_3); cat.remove(js2.getUri()); cat.remove(new URI("test:dummy_job")); cat.removeListener(l); cat.remove(js1_3.getUri()); Mockito.verify(l).onAddJob(Mockito.eq(js1_1)); Mockito.verify(l).onUpdateJob(Mockito.eq(js1_2)); Mockito.verify(l).onAddJob(Mockito.eq(js2)); Mockito.verify(l).onUpdateJob(Mockito.eq(js1_3)); Mockito.verify(l).onDeleteJob(Mockito.eq(js2.getUri()), Mockito.eq(js2.getVersion())); Mockito.verifyNoMoreInteractions(l); cat.stopAsync(); cat.awaitTerminated(1, TimeUnit.SECONDS); }
JobExecution jex1 = jled.getJobExecution(); Assert.assertEquals(jex1.getJobSpecURI(), jobSpec1.getUri()); Assert.assertEquals(jex1.getJobSpecVersion(), jobSpec1.getVersion()); Assert.assertEquals(jex2.getJobSpecVersion(), jobSpec2.getVersion()); Assert.assertTrue(jex2.getLaunchTimeMillis() >= jex1.getLaunchTimeMillis());