public void provisioning(ProgramRunId programRunId, ProgramOptions programOptions, ProgramDescriptor programDescriptor, String userId) { publish(ImmutableMap.<String, String>builder() .put(ProgramOptionConstants.PROGRAM_RUN_ID, GSON.toJson(programRunId)) .put(ProgramOptionConstants.PROGRAM_DESCRIPTOR, GSON.toJson(programDescriptor)) .put(ProgramOptionConstants.USER_ID, userId) .put(ProgramOptionConstants.CLUSTER_STATUS, ProgramRunClusterStatus.PROVISIONING.name()) .put(ProgramOptionConstants.DEBUG_ENABLED, String.valueOf(programOptions.isDebug())) .put(ProgramOptionConstants.USER_OVERRIDES, GSON.toJson(programOptions.getUserArguments().asMap())) .put(ProgramOptionConstants.SYSTEM_OVERRIDES, GSON.toJson(programOptions.getArguments().asMap())) .put(ProgramOptionConstants.ARTIFACT_ID, GSON.toJson(programDescriptor.getArtifactId().toApiArtifactId())) .build()); }
public void provisioning(ProgramRunId programRunId, ProgramOptions programOptions, ProgramDescriptor programDescriptor, String userId) { publish(ImmutableMap.<String, String>builder() .put(ProgramOptionConstants.PROGRAM_RUN_ID, GSON.toJson(programRunId)) .put(ProgramOptionConstants.PROGRAM_DESCRIPTOR, GSON.toJson(programDescriptor)) .put(ProgramOptionConstants.USER_ID, userId) .put(ProgramOptionConstants.CLUSTER_STATUS, ProgramRunClusterStatus.PROVISIONING.name()) .put(ProgramOptionConstants.DEBUG_ENABLED, String.valueOf(programOptions.isDebug())) .put(ProgramOptionConstants.USER_OVERRIDES, GSON.toJson(programOptions.getUserArguments().asMap())) .put(ProgramOptionConstants.SYSTEM_OVERRIDES, GSON.toJson(programOptions.getArguments().asMap())) .put(ProgramOptionConstants.ARTIFACT_ID, GSON.toJson(programDescriptor.getArtifactId().toApiArtifactId())) .build()); }
GSON.toJson(programDescriptor.getArtifactId().toApiArtifactId()));
GSON.toJson(programDescriptor.getArtifactId().toApiArtifactId()));
ApplicationSpecification dummyAppSpec1 = new DefaultApplicationSpecification(APP1_ID.getApplication(), "dummy app", null, ARTIFACT_ID1.toApiArtifactId(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), ApplicationSpecification dummyAppSpec2 = new DefaultApplicationSpecification(APP2_ID.getApplication(), "dummy app", null, ARTIFACT_ID2.toApiArtifactId(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(),
private TaskFields createTaskInfo(ProvisionerInfo provisionerInfo) { ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").workflow("wf").run(RunIds.generate()); Map<String, String> systemArgs = new HashMap<>(); Map<String, String> userArgs = new HashMap<>(); Profile profile = new Profile(ProfileId.NATIVE.getProfile(), "label", "desc", provisionerInfo); SystemArguments.addProfileArgs(systemArgs, profile); ProgramOptions programOptions = new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments(userArgs)); ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId(); ApplicationSpecification appSpec = new DefaultApplicationSpecification( "name", "1.0.0", "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); ProgramDescriptor programDescriptor = new ProgramDescriptor(programRunId.getParent(), appSpec); return new TaskFields(programDescriptor, programOptions, programRunId); }
private void addRuns(Store store, ProgramRunId... runs) { for (ProgramRunId run : runs) { ArtifactId artifactId = run.getNamespaceId().artifact("testArtifact", "1.0").toApiArtifactId(); setStartAndRunning(store, run.getParent(), run.getEntityName(), artifactId); } }
/** Adds runs which have workflows associated with them * * @param store store instance * @param workflowName name of the workflow * @param workflowRunId run ID associated with all program runs * @param runs list ofo runs to be added */ private void addWorkflowRuns(Store store, String workflowName, String workflowRunId, ProgramRunId... runs) { Map<String, String> workflowIDMap = new HashMap<>(); Map<String, String> emptyMap = ImmutableMap.of(); workflowIDMap.put(ProgramOptionConstants.WORKFLOW_NAME, workflowName); workflowIDMap.put(ProgramOptionConstants.WORKFLOW_NODE_ID, "workflowNodeId"); workflowIDMap.put(ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId); workflowIDMap.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()); for (ProgramRunId run : runs) { ArtifactId artifactId = run.getNamespaceId().artifact("testArtifact", "1.0").toApiArtifactId(); store.setProvisioning(run, emptyMap, workflowIDMap, AppFabricTestHelper.createSourceId(++sourceId), artifactId); store.setProvisioned(run, 0, AppFabricTestHelper.createSourceId(++sourceId)); store.setStart(run, null, workflowIDMap, AppFabricTestHelper.createSourceId(++sourceId)); store.setRunning(run, RunIds.getTime(run.getRun(), TimeUnit.SECONDS) + 1, null, AppFabricTestHelper.createSourceId(++sourceId)); } }
@Test public void testAppSpecNotRequiredToWriteState() throws Exception { ProgramId programId = NamespaceId.DEFAULT.app("someapp").program(ProgramType.SERVICE, "s"); Map<String, String> systemArguments = new HashMap<>(); systemArguments.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString()); systemArguments.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()); ProgramOptions programOptions = new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments()); ProgramRunId runId = programId.run(RunIds.generate()); ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId(); ApplicationSpecification appSpec = new DefaultApplicationSpecification( "name", "1.0.0", "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); ProgramDescriptor programDescriptor = new ProgramDescriptor(programId, appSpec); programStateWriter.start(runId, programOptions, null, programDescriptor); Tasks.waitFor(ProgramRunStatus.STARTING, () -> txnl.execute(() -> { RunRecordMeta meta = metadataStoreDataset.getRun(runId); if (meta == null) { return null; } Assert.assertEquals(artifactId, meta.getArtifactId()); return meta.getStatus(); }), 10, TimeUnit.SECONDS); programStateWriter.completed(runId); }
expectedRunRecords.add( new DashboardProgramRunRecord(sched1.getProgramId().getNamespace(), ArtifactSummary.from(ARTIFACT_ID1.toApiArtifactId()), new DashboardProgramRunRecord.ApplicationNameVersion( sched1.getProgramId().getApplication(), sched1.getProgramId().getVersion()),
ArtifactId artifactId = appId.getNamespaceId().artifact("testArtifact", "1.0").toApiArtifactId();
new BasicArguments()); ProgramRunId runId = programId.run(RunIds.generate()); ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
/** * setup and return mock program properties on runrecord builder but use passed namespaceId and runId */ private RunRecordMeta.Builder getMockRunRecordMeta(NamespaceId namespaceId, RunId runId) { ProgramId programId = namespaceId.app("someapp").program(ProgramType.SERVICE, "s"); RunRecordMeta.Builder runRecordBuilder = RunRecordMeta.builder(); runRecordBuilder.setArtifactId(ARTIFACT_ID.toApiArtifactId()); runRecordBuilder.setPrincipal("userA"); runRecordBuilder.setProgramRunId(programId.run(runId)); runRecordBuilder.setSourceId(SOURCE_ID); runRecordBuilder.setStartTime(RunIds.getTime(runId, TimeUnit.SECONDS)); return runRecordBuilder; }
ApplicationSpecification spec = Specifications.from(new AllProgramsApp()); ApplicationId appId = NamespaceId.DEFAULT.app(spec.getName()); ArtifactId testArtifact = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId(); ProgramId workflowId = appId.workflow(AllProgramsApp.NoOpWorkflow.NAME); ProgramId serviceId = appId.service(AllProgramsApp.NoOpService.NAME);
ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId(); RunId runId = RunIds.generate(System.currentTimeMillis()); ProgramRunId programRunId = programId.run(runId.getId());
@Test public void testConcurrentStopStart() { // Two programs that start/stop at same time // Should have two run history. ProgramId programId = new ProgramId("account1", "concurrentApp", ProgramType.MAPREDUCE, "concurrentMR"); long now = System.currentTimeMillis(); long nowSecs = TimeUnit.MILLISECONDS.toSeconds(now); RunId run1 = RunIds.generate(now - 10000); ArtifactId artifactId = programId.getNamespaceId().artifact("testArtifact", "1.0").toApiArtifactId(); setStartAndRunning(programId.run(run1.getId()), artifactId); RunId run2 = RunIds.generate(now - 10000); setStartAndRunning(programId.run(run2.getId()), artifactId); store.setStop(programId.run(run1.getId()), nowSecs, ProgramController.State.COMPLETED.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId)); store.setStop(programId.run(run2.getId()), nowSecs, ProgramController.State.COMPLETED.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId)); Map<ProgramRunId, RunRecordMeta> historymap = store.getRuns(programId, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, Integer.MAX_VALUE); Assert.assertEquals(2, historymap.size()); }
@Test public void testRunsLimit() { ApplicationSpecification spec = Specifications.from(new AllProgramsApp()); ApplicationId appId = new ApplicationId("testRunsLimit", spec.getName()); store.addApplication(appId, spec); ProgramId mapreduceProgramId = new ApplicationId("testRunsLimit", spec.getName()) .mr(AllProgramsApp.NoOpMR.class.getSimpleName()); ArtifactId artifactId = appId.getNamespaceId().artifact("testArtifact", "1.0").toApiArtifactId(); Assert.assertNotNull(store.getApplication(appId)); long now = System.currentTimeMillis(); ProgramRunId programRunId = mapreduceProgramId.run(RunIds.generate(now - 3000)); setStartAndRunning(programRunId, artifactId); store.setStop(programRunId, now - 100, ProgramController.State.COMPLETED.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId)); setStartAndRunning(mapreduceProgramId.run(RunIds.generate(now - 2000)), artifactId); // even though there's two separate run records (one that's complete and one that's active), only one should be // returned by the query, because the limit parameter of 1 is being passed in. Map<ProgramRunId, RunRecordMeta> historymap = store.getRuns(mapreduceProgramId, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, 1); Assert.assertEquals(1, historymap.size()); }
@Test public void testDeleteSuspendedWorkflow() { NamespaceId namespaceId = new NamespaceId("namespace1"); // Test delete application ApplicationId appId1 = namespaceId.app("app1"); ProgramId programId1 = appId1.workflow("pgm1"); ArtifactId artifactId = namespaceId.artifact("testArtifact", "1.0").toApiArtifactId(); RunId run1 = RunIds.generate(); setStartAndRunning(programId1.run(run1.getId()), artifactId); store.setSuspend(programId1.run(run1.getId()), AppFabricTestHelper.createSourceId(++sourceId), -1); store.removeApplication(appId1); Assert.assertTrue(store.getRuns(programId1, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, Integer.MAX_VALUE).isEmpty()); // Test delete namespace ProgramId programId2 = namespaceId.app("app2").workflow("pgm2"); RunId run2 = RunIds.generate(); setStartAndRunning(programId2.run(run2.getId()), artifactId); store.setSuspend(programId2.run(run2.getId()), AppFabricTestHelper.createSourceId(++sourceId), -1); store.removeAll(namespaceId); nsStore.delete(namespaceId); Assert.assertTrue(store.getRuns(programId2, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, Integer.MAX_VALUE).isEmpty()); }
ArtifactId artifactId = appId1.getNamespaceId().artifact("testArtifact", "1.0").toApiArtifactId();
@Test public void testRunningInRangeSimple() { NamespaceId ns = new NamespaceId("d"); ProgramRunId run1 = ns.app("a1").program(ProgramType.SERVICE, "f1").run(RunIds.generate(20000).getId()); ProgramRunId run2 = ns.app("a2").program(ProgramType.MAPREDUCE, "f2").run(RunIds.generate(10000).getId()); ProgramRunId run3 = ns.app("a3").program(ProgramType.WORKER, "f3").run(RunIds.generate(40000).getId()); ProgramRunId run4 = ns.app("a4").program(ProgramType.SERVICE, "f4").run(RunIds.generate(70000).getId()); ProgramRunId run5 = ns.app("a5").program(ProgramType.SPARK, "f5").run(RunIds.generate(30000).getId()); ProgramRunId run6 = ns.app("a6").program(ProgramType.WORKFLOW, "f6").run(RunIds.generate(60000).getId()); ArtifactId artifactId = ns.artifact("testArtifact", "1.0").toApiArtifactId(); writeStartRecord(run1, artifactId); writeStartRecord(run2, artifactId); writeStartRecord(run3, artifactId); writeStartRecord(run4, artifactId); writeStartRecord(run5, artifactId); writeStartRecord(run6, artifactId); Assert.assertEquals(runsToTime(run1, run2), runIdsToTime(store.getRunningInRange(1, 30))); Assert.assertEquals(runsToTime(run1, run2, run5, run3), runIdsToTime(store.getRunningInRange(30, 50))); Assert.assertEquals(runsToTime(run1, run2, run3, run4, run5, run6), runIdsToTime(store.getRunningInRange(1, 71))); Assert.assertEquals(runsToTime(run1, run2, run3, run4, run5, run6), runIdsToTime(store.getRunningInRange(50, 71))); Assert.assertEquals(ImmutableSet.of(), runIdsToTime(store.getRunningInRange(1, 10))); writeStopRecord(run1, 45000); writeStopRecord(run3, 55000); writeSuspendedRecord(run5); Assert.assertEquals(runsToTime(run2, run3, run4, run5, run6), runIdsToTime(store.getRunningInRange(50, 71))); }