@Override public WorkflowManager getWorkflowManager(String workflowName) { WorkflowId programId = application.workflow(workflowName); return new RemoteWorkflowManager(programId, clientConfig, restClient, this); }
private void updateAppProfileMetadata(ApplicationId applicationId, ApplicationSpecification appSpec) { for (String name : appSpec.getWorkflows().keySet()) { WorkflowId programId = applicationId.workflow(name); updateProgramProfileMetadata(programId); } }
/** * Delete the specified application version without performing checks that its programs are stopped. * * @param appId the id of the application to delete * @param spec the spec of the application to delete */ private void deleteAppVersion(ApplicationId appId, ApplicationSpecification spec) { //Delete the schedules scheduler.deleteSchedules(appId); for (WorkflowSpecification workflowSpec : spec.getWorkflows().values()) { scheduler.modifySchedulesTriggeredByDeletedProgram(appId.workflow(workflowSpec.getName())); } store.removeApplication(appId); }
/** * Delete the specified application version without performing checks that its programs are stopped. * * @param appId the id of the application to delete * @param spec the spec of the application to delete */ private void deleteAppVersion(ApplicationId appId, ApplicationSpecification spec) { //Delete the schedules scheduler.deleteSchedules(appId); for (WorkflowSpecification workflowSpec : spec.getWorkflows().values()) { scheduler.modifySchedulesTriggeredByDeletedProgram(appId.workflow(workflowSpec.getName())); } store.removeApplication(appId); }
private ProgramSchedule toProgramSchedule(ApplicationId appId, ScheduleCreationSpec scheduleCreationSpec) { ProgramId programId = appId.workflow(scheduleCreationSpec.getProgramName()); Trigger trigger = scheduleCreationSpec.getTrigger(); return new ProgramSchedule(scheduleCreationSpec.getName(), scheduleCreationSpec.getDescription(), programId, scheduleCreationSpec.getProperties(), trigger, scheduleCreationSpec.getConstraints(), scheduleCreationSpec.getTimeoutMillis()); } }
@Override public void perform(Arguments arguments, PrintStream printStream) throws Exception { String[] programIdParts = arguments.get(elementType.getArgumentName().toString()).split("\\."); if (programIdParts.length < 2) { throw new CommandInputError(this); } ProgramRunId programRunId = cliConfig.getCurrentNamespace().app(programIdParts[0]).workflow(programIdParts[1]) .run(arguments.get(ArgumentName.RUN_ID.toString())); Table table = getWorkflowNodeStates(programRunId); cliConfig.getTableRenderer().render(cliConfig, printStream, table); }
private ProgramSchedule toProgramSchedule(ApplicationId appId, ScheduleCreationSpec scheduleCreationSpec) { ProgramId programId = appId.workflow(scheduleCreationSpec.getProgramName()); Trigger trigger = scheduleCreationSpec.getTrigger(); return new ProgramSchedule(scheduleCreationSpec.getName(), scheduleCreationSpec.getDescription(), programId, scheduleCreationSpec.getProperties(), trigger, scheduleCreationSpec.getConstraints(), scheduleCreationSpec.getTimeoutMillis()); } }
@Override public void perform(Arguments arguments, PrintStream printStream) throws Exception { String[] programIdParts = arguments.get(elementType.getArgumentName().toString()).split("\\."); if (programIdParts.length < 2) { throw new CommandInputError(this); } ProgramRunId programRunId = cliConfig.getCurrentNamespace().app(programIdParts[0]).workflow(programIdParts[1]) .run(arguments.get(ArgumentName.RUN_ID.toString())); Table table = getWorkflowLocalDatasets(programRunId); cliConfig.getTableRenderer().render(cliConfig, printStream, table); }
private WorkflowToken getWorkflowToken(String namespaceId, String appName, String workflow, String runId) throws NotFoundException { ApplicationId appId = new ApplicationId(namespaceId, appName); ApplicationSpecification appSpec = store.getApplication(appId); if (appSpec == null) { throw new NotFoundException(appId); } WorkflowId workflowId = appId.workflow(workflow); if (!appSpec.getWorkflows().containsKey(workflow)) { throw new NotFoundException(workflowId); } if (store.getRun(workflowId.run(runId)) == null) { throw new NotFoundException(workflowId.run(runId)); } return store.getWorkflowToken(workflowId, runId); }
private WorkflowToken getWorkflowToken(String namespaceId, String appName, String workflow, String runId) throws NotFoundException { ApplicationId appId = new ApplicationId(namespaceId, appName); ApplicationSpecification appSpec = store.getApplication(appId); if (appSpec == null) { throw new NotFoundException(appId); } WorkflowId workflowId = appId.workflow(workflow); if (!appSpec.getWorkflows().containsKey(workflow)) { throw new NotFoundException(workflowId); } if (store.getRun(workflowId.run(runId)) == null) { throw new NotFoundException(workflowId.run(runId)); } return store.getWorkflowToken(workflowId, runId); }
@Override public void perform(Arguments arguments, PrintStream printStream) throws Exception { String[] programIdParts = arguments.get(elementType.getArgumentName().toString()).split("\\."); if (programIdParts.length < 2) { throw new CommandInputError(this); } ProgramRunId programRunId = cliConfig.getCurrentNamespace().app(programIdParts[0]).workflow(programIdParts[1]) .run(arguments.get(ArgumentName.RUN_ID.toString())); Table table = getWorkflowLocalDatasets(programRunId); cliConfig.getTableRenderer().render(cliConfig, printStream, table); }
@Override public void perform(Arguments arguments, PrintStream printStream) throws Exception { String[] programIdParts = arguments.get(elementType.getArgumentName().toString()).split("\\."); if (programIdParts.length < 2) { throw new CommandInputError(this); } ProgramRunId programRunId = cliConfig.getCurrentNamespace().app(programIdParts[0]).workflow(programIdParts[1]) .run(arguments.get(ArgumentName.RUN_ID.toString())); Table table = getWorkflowNodeStates(programRunId); cliConfig.getTableRenderer().render(cliConfig, printStream, table); }
private TaskFields createTaskInfo(ProvisionerInfo provisionerInfo) { ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").workflow("wf").run(RunIds.generate()); Map<String, String> systemArgs = new HashMap<>(); Map<String, String> userArgs = new HashMap<>(); Profile profile = new Profile(ProfileId.NATIVE.getProfile(), "label", "desc", provisionerInfo); SystemArguments.addProfileArgs(systemArgs, profile); ProgramOptions programOptions = new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments(userArgs)); ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId(); ApplicationSpecification appSpec = new DefaultApplicationSpecification( "name", "1.0.0", "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); ProgramDescriptor programDescriptor = new ProgramDescriptor(programRunId.getParent(), appSpec); return new TaskFields(programDescriptor, programOptions, programRunId); }
private LoggingContext createLoggingContext(ProgramId programId, RunId runId, @Nullable WorkflowProgramInfo workflowProgramInfo) { if (workflowProgramInfo == null) { return new MapReduceLoggingContext(programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } ProgramId workflowProramId = programId.getParent().workflow(workflowProgramInfo.getName()); return new WorkflowProgramLoggingContext(workflowProramId.getNamespace(), workflowProramId.getApplication(), workflowProramId.getProgram(), workflowProgramInfo.getRunId().getId(), ProgramType.MAPREDUCE, programId.getProgram(), runId.getId()); }
private LoggingContext createLoggingContext(ProgramId programId, RunId runId, @Nullable WorkflowProgramInfo workflowProgramInfo) { if (workflowProgramInfo == null) { return new MapReduceLoggingContext(programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } ProgramId workflowProramId = programId.getParent().workflow(workflowProgramInfo.getName()); return new WorkflowProgramLoggingContext(workflowProramId.getNamespace(), workflowProramId.getApplication(), workflowProramId.getProgram(), workflowProgramInfo.getRunId().getId(), ProgramType.MAPREDUCE, programId.getProgram(), runId.getId()); }
private LoggingContext createLoggingContext(ProgramId programId, RunId runId, @Nullable WorkflowProgramInfo workflowProgramInfo) { if (workflowProgramInfo == null) { return new SparkLoggingContext(programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } ProgramId workflowProramId = Ids.namespace(programId.getNamespace()).app(programId.getApplication()) .workflow(workflowProgramInfo.getName()); return new WorkflowProgramLoggingContext(workflowProramId.getNamespace(), workflowProramId.getApplication(), workflowProramId.getProgram(), workflowProgramInfo.getRunId().getId(), ProgramType.SPARK, programId.getProgram(), runId.getId()); }
private LoggingContext createLoggingContext(ProgramId programId, RunId runId, @Nullable WorkflowProgramInfo workflowProgramInfo) { if (workflowProgramInfo == null) { return new SparkLoggingContext(programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } ProgramId workflowProramId = Ids.namespace(programId.getNamespace()).app(programId.getApplication()) .workflow(workflowProgramInfo.getName()); return new WorkflowProgramLoggingContext(workflowProramId.getNamespace(), workflowProramId.getApplication(), workflowProramId.getProgram(), workflowProgramInfo.getRunId().getId(), ProgramType.SPARK, programId.getProgram(), runId.getId()); }
private LoggingContext createLoggingContext(ProgramId programId, RunId runId, @Nullable WorkflowProgramInfo workflowProgramInfo) { if (workflowProgramInfo == null) { return new SparkLoggingContext(programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } ProgramId workflowProramId = Ids.namespace(programId.getNamespace()).app(programId.getApplication()) .workflow(workflowProgramInfo.getName()); return new WorkflowProgramLoggingContext(workflowProramId.getNamespace(), workflowProramId.getApplication(), workflowProramId.getProgram(), workflowProgramInfo.getRunId().getId(), ProgramType.SPARK, programId.getProgram(), runId.getId()); }
@SuppressWarnings("unchecked") private ApplicationWithPrograms createAppWithWorkflow(ArtifactId artifactId, ApplicationId appId, String workflowName) throws IOException { LocationFactory locationFactory = new LocalLocationFactory(TEMP_FOLDER.newFolder()); AbstractApplication app = new WorkflowAppWithFork(); ApplicationSpecification appSpec = Specifications.from(app); Location workflowJar = AppJarHelper.createDeploymentJar(locationFactory, WorkflowAppWithFork.class); ApplicationDeployable appDeployable = new ApplicationDeployable(artifactId, workflowJar, appId, appSpec, null, ApplicationDeployScope.USER); return new ApplicationWithPrograms(appDeployable, ImmutableList.of(new ProgramDescriptor(appId.workflow(workflowName), appSpec))); }
@Test public void testGetProgramProfile() { ProfileId profileId = NamespaceId.DEFAULT.profile("p"); Map<String, String> args = Collections.singletonMap(SystemArguments.PROFILE_NAME, profileId.getScopedName()); ApplicationId appId = NamespaceId.DEFAULT.app("a"); ProgramId mrId = appId.mr("mr"); ProgramId serviceId = appId.service("serv"); ProgramId sparkId = appId.spark("spark"); ProgramId workerId = appId.worker("worker"); ProgramId workflowID = appId.workflow("wf"); Assert.assertEquals(profileId, SystemArguments.getProfileIdForProgram(mrId, args)); Assert.assertEquals(ProfileId.NATIVE, SystemArguments.getProfileIdForProgram(serviceId, args)); Assert.assertEquals(profileId, SystemArguments.getProfileIdForProgram(sparkId, args)); Assert.assertEquals(ProfileId.NATIVE, SystemArguments.getProfileIdForProgram(workerId, args)); Assert.assertEquals(profileId, SystemArguments.getProfileIdForProgram(workflowID, args)); }