private void recordProvisionAndStart(ProgramRunId programRunId, AppMetadataStore metadataStoreDataset) { metadataStoreDataset.recordProgramProvisioning(programRunId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID); metadataStoreDataset.recordProgramProvisioned(programRunId, 0, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet())); metadataStoreDataset.recordProgramStart(programRunId, null, ImmutableMap.of(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet())); }
private void setStart(ProgramRunId id, Map<String, String> runtimeArgs, Map<String, String> systemArgs, ArtifactId artifactId) { if (!systemArgs.containsKey(SystemArguments.PROFILE_NAME)) { systemArgs = ImmutableMap.<String, String>builder() .putAll(systemArgs) .put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()) .build(); } store.setProvisioning(id, runtimeArgs, systemArgs, AppFabricTestHelper.createSourceId(++sourceId), artifactId); store.setProvisioned(id, 0, AppFabricTestHelper.createSourceId(++sourceId)); store.setStart(id, null, systemArgs, AppFabricTestHelper.createSourceId(++sourceId)); }
private void setStartAndRunning(Store store, ProgramRunId id, Map<String, String> runtimeArgs, Map<String, String> systemArgs) { if (!systemArgs.containsKey(SystemArguments.PROFILE_NAME)) { systemArgs = ImmutableMap.<String, String>builder() .putAll(systemArgs) .put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()) .build(); } long startTime = RunIds.getTime(id.getRun(), TimeUnit.SECONDS); store.setProvisioning(id, runtimeArgs, systemArgs, AppFabricTestHelper.createSourceId(++sourceId), ARTIFACT_ID); store.setProvisioned(id, 0, AppFabricTestHelper.createSourceId(++sourceId)); store.setStart(id, null, systemArgs, AppFabricTestHelper.createSourceId(++sourceId)); store.setRunning(id, startTime + 1, null, AppFabricTestHelper.createSourceId(++sourceId)); }
private void setStartAndRunning(Store store, ProgramRunId id, Map<String, String> runtimeArgs, Map<String, String> systemArgs) { if (!systemArgs.containsKey(SystemArguments.PROFILE_NAME)) { systemArgs = ImmutableMap.<String, String>builder() .putAll(systemArgs) .put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()) .build(); } long startTime = RunIds.getTime(id.getRun(), TimeUnit.SECONDS); store.setProvisioning(id, runtimeArgs, systemArgs, AppFabricTestHelper.createSourceId(++sourceId), ARTIFACT_ID); store.setProvisioned(id, 0, AppFabricTestHelper.createSourceId(++sourceId)); store.setStart(id, null, systemArgs, AppFabricTestHelper.createSourceId(++sourceId)); store.setRunning(id, startTime + 1, null, AppFabricTestHelper.createSourceId(++sourceId)); }
private void writeStopRecord(ProgramRunId run, long stopTimeInMillis) { store.setStop(run, TimeUnit.MILLISECONDS.toSeconds(stopTimeInMillis), ProgramRunStatus.COMPLETED, AppFabricTestHelper.createSourceId(++sourceId)); Assert.assertNotNull(store.getRun(run)); }
private void setStartAndRunning(Store store, ProgramId id, String pid, Map<String, String> runtimeArgs, Map<String, String> systemArgs, ArtifactId artifactId) { if (!systemArgs.containsKey(SystemArguments.PROFILE_NAME)) { systemArgs = ImmutableMap.<String, String>builder() .putAll(systemArgs) .put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()) .build(); } long startTime = RunIds.getTime(pid, TimeUnit.SECONDS); store.setProvisioning(id.run(pid), runtimeArgs, systemArgs, AppFabricTestHelper.createSourceId(++sourceId), artifactId); store.setProvisioned(id.run(pid), 0, AppFabricTestHelper.createSourceId(++sourceId)); store.setStart(id.run(pid), null, systemArgs, AppFabricTestHelper.createSourceId(++sourceId)); store.setRunning(id.run(pid), startTime + 1, null, AppFabricTestHelper.createSourceId(++sourceId)); }
/** Adds runs which have workflows associated with them * * @param store store instance * @param workflowName name of the workflow * @param workflowRunId run ID associated with all program runs * @param runs list ofo runs to be added */ private void addWorkflowRuns(Store store, String workflowName, String workflowRunId, ProgramRunId... runs) { Map<String, String> workflowIDMap = new HashMap<>(); Map<String, String> emptyMap = ImmutableMap.of(); workflowIDMap.put(ProgramOptionConstants.WORKFLOW_NAME, workflowName); workflowIDMap.put(ProgramOptionConstants.WORKFLOW_NODE_ID, "workflowNodeId"); workflowIDMap.put(ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId); workflowIDMap.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()); for (ProgramRunId run : runs) { ArtifactId artifactId = run.getNamespaceId().artifact("testArtifact", "1.0").toApiArtifactId(); store.setProvisioning(run, emptyMap, workflowIDMap, AppFabricTestHelper.createSourceId(++sourceId), artifactId); store.setProvisioned(run, 0, AppFabricTestHelper.createSourceId(++sourceId)); store.setStart(run, null, workflowIDMap, AppFabricTestHelper.createSourceId(++sourceId)); store.setRunning(run, RunIds.getTime(run.getRun(), TimeUnit.SECONDS) + 1, null, AppFabricTestHelper.createSourceId(++sourceId)); } }
private void writeSuspendedRecord(ProgramRunId run) { store.setSuspend(run, AppFabricTestHelper.createSourceId(++sourceId), -1); Assert.assertNotNull(store.getRun(run)); }
private void assertPersistedStatus(final AppMetadataStore metadataStoreDataset, TransactionExecutor txnl, final long startSourceId, final long runningSourceId, final long killedSourceId, final ProgramRunStatus expectedRunStatus) throws Exception { // Add some run records ApplicationId application = NamespaceId.DEFAULT.app("app"); final ProgramId program = application.program(ProgramType.WORKFLOW, "program"); final AtomicReference<RunRecordMeta> resultRecord = new AtomicReference<>(); final RunId runId = RunIds.generate(runIdTime.incrementAndGet()); final ProgramRunId programRunId = program.run(runId); txnl.execute(() -> { metadataStoreDataset.recordProgramProvisioning(programRunId, null, SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(startSourceId), ARTIFACT_ID); metadataStoreDataset.recordProgramProvisioned(programRunId, 0, AppFabricTestHelper.createSourceId(startSourceId + 1)); metadataStoreDataset.recordProgramStart(programRunId, null, ImmutableMap.of(), AppFabricTestHelper.createSourceId(startSourceId + 2)); metadataStoreDataset.recordProgramRunning(programRunId, RunIds.getTime(runId, TimeUnit.SECONDS), null, AppFabricTestHelper.createSourceId(runningSourceId)); metadataStoreDataset.recordProgramStop(programRunId, RunIds.getTime(runId, TimeUnit.SECONDS), ProgramRunStatus.KILLED, null, AppFabricTestHelper.createSourceId(killedSourceId)); resultRecord.set(metadataStoreDataset.getRun(programRunId)); }); Assert.assertEquals(expectedRunStatus, resultRecord.get().getStatus()); }
private void setStartAndRunning(ProgramRunId id, Map<String, String> runtimeArgs, Map<String, String> systemArgs, ArtifactId artifactId) { long startTime = RunIds.getTime(id.getRun(), TimeUnit.SECONDS); setStart(id, runtimeArgs, systemArgs, artifactId); store.setRunning(id, startTime + 1, null, AppFabricTestHelper.createSourceId(++sourceId)); }
@Test public void testPendingToCompletedIsIgnored() throws Exception { AppMetadataStore metadataStoreDataset = getMetadataStore("testPendingToCompletedIgnored"); TransactionExecutor txnl = getTxExecutor(metadataStoreDataset); ApplicationId application = NamespaceId.DEFAULT.app("app"); ProgramId program = application.program(ProgramType.WORKFLOW, "program"); RunId runId1 = RunIds.generate(); ProgramRunId programRunId = program.run(runId1); txnl.execute(() -> { metadataStoreDataset.recordProgramProvisioning(programRunId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID); metadataStoreDataset.recordProgramStop(programRunId, 0, ProgramRunStatus.COMPLETED, null, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet())); RunRecordMeta runRecordMeta = metadataStoreDataset.getRun(programRunId); Assert.assertEquals(ProgramRunStatus.PENDING, runRecordMeta.getStatus()); }); }
metadataStoreDataset.recordProgramRunning( programRunId, RunIds.getTime(runId, TimeUnit.SECONDS) + 1, null, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet())); metadataStoreDataset.recordProgramStop( programRunId, RunIds.getTime(runId, TimeUnit.SECONDS), STOP_STATUSES.get(j % STOP_STATUSES.size()), null, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet())); });
int sourceId = 0; metadataStoreDataset.recordProgramProvisioning(programRunId, Collections.emptyMap(), systemArgs, AppFabricTestHelper.createSourceId(++sourceId), artifactId); metadataStoreDataset.recordProgramProvisioned(programRunId, 3, AppFabricTestHelper.createSourceId(++sourceId)); metadataStoreDataset.recordProgramStart(programRunId, null, systemArgs, AppFabricTestHelper.createSourceId(++sourceId)); metadataStoreDataset.recordProgramRunning(programRunId, startTime + 60000, null, AppFabricTestHelper.createSourceId(++sourceId)); });
@Test public void testProfileInRunRecord() throws Exception { AppMetadataStore store = getMetadataStore("testProfileInRunRecord"); TransactionExecutor txnl = getTxExecutor(store); ProgramRunId runId = NamespaceId.DEFAULT.app("myApp").workflow("myProgram").run(RunIds.generate()); ProfileId profileId = NamespaceId.DEFAULT.profile("MyProfile"); txnl.execute(() -> { long startSourceId = 1L; store.recordProgramProvisioning(runId, null, Collections.singletonMap(SystemArguments.PROFILE_NAME, profileId.getScopedName()), AppFabricTestHelper.createSourceId(startSourceId), ARTIFACT_ID); // the profile id should be there after the provisioning stage RunRecordMeta run = store.getRun(runId); Assert.assertNotNull(run); Assert.assertEquals(profileId, run.getProfileId()); store.recordProgramProvisioned(runId, 0, AppFabricTestHelper.createSourceId(startSourceId + 1)); store.recordProgramStart(runId, null, ImmutableMap.of(), AppFabricTestHelper.createSourceId(startSourceId + 2)); store.recordProgramRunning(runId, RunIds.getTime(runId.getRun(), TimeUnit.SECONDS), null, AppFabricTestHelper.createSourceId(startSourceId + 3)); store.recordProgramStop(runId, RunIds.getTime(runId.getRun(), TimeUnit.SECONDS), ProgramRunStatus.KILLED, null, AppFabricTestHelper.createSourceId(startSourceId + 4)); run = store.getRun(runId); Assert.assertNotNull(run); Assert.assertEquals(profileId, run.getProfileId()); }); }
AppFabricTestHelper.createSourceId(++sourceId)); assertSatisfied(false, concurrencyConstraint.check(schedule, constraintContext)); store.setSuspend(pid3, AppFabricTestHelper.createSourceId(++sourceId), -1); assertSatisfied(false, concurrencyConstraint.check(schedule, constraintContext)); store.setResume(pid3, AppFabricTestHelper.createSourceId(++sourceId), -1); assertSatisfied(false, concurrencyConstraint.check(schedule, constraintContext)); AppFabricTestHelper.createSourceId(++sourceId)); assertSatisfied(true, concurrencyConstraint.check(schedule, constraintContext)); AppFabricTestHelper.createSourceId(++sourceId)); assertSatisfied(true, concurrencyConstraint.check(schedule, constraintContext));
@Test public void testConcurrentStopStart() { // Two programs that start/stop at same time // Should have two run history. ProgramId programId = new ProgramId("account1", "concurrentApp", ProgramType.MAPREDUCE, "concurrentMR"); long now = System.currentTimeMillis(); long nowSecs = TimeUnit.MILLISECONDS.toSeconds(now); RunId run1 = RunIds.generate(now - 10000); ArtifactId artifactId = programId.getNamespaceId().artifact("testArtifact", "1.0").toApiArtifactId(); setStartAndRunning(programId.run(run1.getId()), artifactId); RunId run2 = RunIds.generate(now - 10000); setStartAndRunning(programId.run(run2.getId()), artifactId); store.setStop(programId.run(run1.getId()), nowSecs, ProgramController.State.COMPLETED.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId)); store.setStop(programId.run(run2.getId()), nowSecs, ProgramController.State.COMPLETED.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId)); Map<ProgramRunId, RunRecordMeta> historymap = store.getRuns(programId, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, Integer.MAX_VALUE); Assert.assertEquals(2, historymap.size()); }
@Test public void testDeleteSuspendedWorkflow() { NamespaceId namespaceId = new NamespaceId("namespace1"); // Test delete application ApplicationId appId1 = namespaceId.app("app1"); ProgramId programId1 = appId1.workflow("pgm1"); ArtifactId artifactId = namespaceId.artifact("testArtifact", "1.0").toApiArtifactId(); RunId run1 = RunIds.generate(); setStartAndRunning(programId1.run(run1.getId()), artifactId); store.setSuspend(programId1.run(run1.getId()), AppFabricTestHelper.createSourceId(++sourceId), -1); store.removeApplication(appId1); Assert.assertTrue(store.getRuns(programId1, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, Integer.MAX_VALUE).isEmpty()); // Test delete namespace ProgramId programId2 = namespaceId.app("app2").workflow("pgm2"); RunId run2 = RunIds.generate(); setStartAndRunning(programId2.run(run2.getId()), artifactId); store.setSuspend(programId2.run(run2.getId()), AppFabricTestHelper.createSourceId(++sourceId), -1); store.removeAll(namespaceId); nsStore.delete(namespaceId); Assert.assertTrue(store.getRuns(programId2, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, Integer.MAX_VALUE).isEmpty()); }
@Test public void testRunsLimit() { ApplicationSpecification spec = Specifications.from(new AllProgramsApp()); ApplicationId appId = new ApplicationId("testRunsLimit", spec.getName()); store.addApplication(appId, spec); ProgramId mapreduceProgramId = new ApplicationId("testRunsLimit", spec.getName()) .mr(AllProgramsApp.NoOpMR.class.getSimpleName()); ArtifactId artifactId = appId.getNamespaceId().artifact("testArtifact", "1.0").toApiArtifactId(); Assert.assertNotNull(store.getApplication(appId)); long now = System.currentTimeMillis(); ProgramRunId programRunId = mapreduceProgramId.run(RunIds.generate(now - 3000)); setStartAndRunning(programRunId, artifactId); store.setStop(programRunId, now - 100, ProgramController.State.COMPLETED.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId)); setStartAndRunning(mapreduceProgramId.run(RunIds.generate(now - 2000)), artifactId); // even though there's two separate run records (one that's complete and one that's active), only one should be // returned by the query, because the limit parameter of 1 is being passed in. Map<ProgramRunId, RunRecordMeta> historymap = store.getRuns(mapreduceProgramId, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, 1); Assert.assertEquals(1, historymap.size()); }
store.recordProgramRunning( runId, RunIds.getTime(runId.getRun(), TimeUnit.SECONDS) + 10, null, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet())); store.recordProgramStop(runId, RunIds.getTime(runId.getRun(), TimeUnit.SECONDS) + 20, ProgramRunStatus.COMPLETED, null, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet())); });
setStartAndRunning(mapreduceProgramRunId1, artifactId); store.setStop(mapreduceProgramRunId1, now, ProgramController.State.COMPLETED.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId)); AppFabricTestHelper.createSourceId(++sourceId)); AppFabricTestHelper.createSourceId(++sourceId));