@Override public Map<String, WorkflowSpecification> getWorkflows() { return delegate.getWorkflows(); }
@Override public Map<String, WorkflowSpecification> getWorkflows() { return delegate.getWorkflows(); }
private void updateAppProfileMetadata(ApplicationId applicationId, ApplicationSpecification appSpec) { for (String name : appSpec.getWorkflows().keySet()) { WorkflowId programId = applicationId.workflow(name); updateProgramProfileMetadata(programId); } }
private Iterable<ProgramSpecification> getProgramSpecs(ApplicationId appId) { ApplicationSpecification appSpec = store.getApplication(appId); return Iterables.concat(appSpec.getMapReduce().values(), appSpec.getServices().values(), appSpec.getSpark().values(), appSpec.getWorkers().values(), appSpec.getWorkflows().values()); }
private Iterable<ProgramSpecification> getProgramSpecs(ApplicationId appId) { ApplicationSpecification appSpec = store.getApplication(appId); return Iterables.concat(appSpec.getFlows().values(), appSpec.getMapReduce().values(), appSpec.getServices().values(), appSpec.getSpark().values(), appSpec.getWorkers().values(), appSpec.getWorkflows().values()); }
/** * Creates a new instance based on the given {@link WorkflowProgramInfo}. */ public static NameMappedDatasetFramework createFromWorkflowProgramInfo(DatasetFramework datasetFramework, WorkflowProgramInfo info, ApplicationSpecification appSpec) { Set<String> localDatasets = appSpec.getWorkflows().get(info.getName()).getLocalDatasetSpecs().keySet(); return new NameMappedDatasetFramework(datasetFramework, localDatasets, info.getRunId().getId()); }
private Set<ProgramId> getAllPrograms(ApplicationId appId, ApplicationSpecification appSpec) { Set<ProgramId> result = new HashSet<>(); result.addAll(getProgramsWithType(appId, ProgramType.MAPREDUCE, appSpec.getMapReduce())); result.addAll(getProgramsWithType(appId, ProgramType.WORKFLOW, appSpec.getWorkflows())); result.addAll(getProgramsWithType(appId, ProgramType.SERVICE, appSpec.getServices())); result.addAll(getProgramsWithType(appId, ProgramType.SPARK, appSpec.getSpark())); result.addAll(getProgramsWithType(appId, ProgramType.WORKER, appSpec.getWorkers())); return result; }
/** * Creates a new instance based on the given {@link WorkflowProgramInfo}. */ public static NameMappedDatasetFramework createFromWorkflowProgramInfo(DatasetFramework datasetFramework, WorkflowProgramInfo info, ApplicationSpecification appSpec) { Set<String> localDatasets = appSpec.getWorkflows().get(info.getName()).getLocalDatasetSpecs().keySet(); return new NameMappedDatasetFramework(datasetFramework, localDatasets, info.getRunId().getId()); }
private Set<ProgramId> getAllPrograms(ApplicationId appId, ApplicationSpecification appSpec) { Set<ProgramId> result = new HashSet<>(); result.addAll(getProgramsWithType(appId, ProgramType.FLOW, appSpec.getFlows())); result.addAll(getProgramsWithType(appId, ProgramType.MAPREDUCE, appSpec.getMapReduce())); result.addAll(getProgramsWithType(appId, ProgramType.WORKFLOW, appSpec.getWorkflows())); result.addAll(getProgramsWithType(appId, ProgramType.SERVICE, appSpec.getServices())); result.addAll(getProgramsWithType(appId, ProgramType.SPARK, appSpec.getSpark())); result.addAll(getProgramsWithType(appId, ProgramType.WORKER, appSpec.getWorkers())); return result; }
private void addPrograms(ImmutableMap.Builder<String, String> properties) { addPrograms(ProgramType.MAPREDUCE, appSpec.getMapReduce().values(), properties); addPrograms(ProgramType.SERVICE, appSpec.getServices().values(), properties); addPrograms(ProgramType.SPARK, appSpec.getSpark().values(), properties); addPrograms(ProgramType.WORKER, appSpec.getWorkers().values(), properties); addPrograms(ProgramType.WORKFLOW, appSpec.getWorkflows().values(), properties); }
private void addPrograms(ImmutableMap.Builder<String, String> properties) { addPrograms(ProgramType.FLOW, appSpec.getFlows().values(), properties); addPrograms(ProgramType.MAPREDUCE, appSpec.getMapReduce().values(), properties); addPrograms(ProgramType.SERVICE, appSpec.getServices().values(), properties); addPrograms(ProgramType.SPARK, appSpec.getSpark().values(), properties); addPrograms(ProgramType.WORKER, appSpec.getWorkers().values(), properties); addPrograms(ProgramType.WORKFLOW, appSpec.getWorkflows().values(), properties); }
private boolean programExists(ProgramId id, ApplicationSpecification appSpec) { switch (id.getType()) { case MAPREDUCE: return appSpec.getMapReduce().containsKey(id.getProgram()); case SERVICE: return appSpec.getServices().containsKey(id.getProgram()); case SPARK: return appSpec.getSpark().containsKey(id.getProgram()); case WORKER: return appSpec.getWorkers().containsKey(id.getProgram()); case WORKFLOW: return appSpec.getWorkflows().containsKey(id.getProgram()); default: throw new IllegalArgumentException("Unexpected ProgramType " + id.getType()); } }
private boolean programExists(ProgramId id, ApplicationSpecification appSpec) { switch (id.getType()) { case FLOW: return appSpec.getFlows().containsKey(id.getProgram()); case MAPREDUCE: return appSpec.getMapReduce().containsKey(id.getProgram()); case SERVICE: return appSpec.getServices().containsKey(id.getProgram()); case SPARK: return appSpec.getSpark().containsKey(id.getProgram()); case WORKER: return appSpec.getWorkers().containsKey(id.getProgram()); case WORKFLOW: return appSpec.getWorkflows().containsKey(id.getProgram()); default: throw new IllegalArgumentException("Unexpected ProgramType " + id.getType()); } }
/** * Delete the specified application version without performing checks that its programs are stopped. * * @param appId the id of the application to delete * @param spec the spec of the application to delete */ private void deleteAppVersion(ApplicationId appId, ApplicationSpecification spec) { //Delete the schedules scheduler.deleteSchedules(appId); for (WorkflowSpecification workflowSpec : spec.getWorkflows().values()) { scheduler.modifySchedulesTriggeredByDeletedProgram(appId.workflow(workflowSpec.getName())); } store.removeApplication(appId); }
/** * Delete the specified application version without performing checks that its programs are stopped. * * @param appId the id of the application to delete * @param spec the spec of the application to delete */ private void deleteAppVersion(ApplicationId appId, ApplicationSpecification spec) { //Delete the schedules scheduler.deleteSchedules(appId); for (WorkflowSpecification workflowSpec : spec.getWorkflows().values()) { scheduler.modifySchedulesTriggeredByDeletedProgram(appId.workflow(workflowSpec.getName())); } store.removeApplication(appId); }
@Override public void process(ApplicationWithPrograms input) { // use current time as creation time for app and all programs creationTime = String.valueOf(System.currentTimeMillis()); // add system metadata for apps ApplicationId appId = input.getApplicationId(); ApplicationSpecification appSpec = input.getSpecification(); new AppSystemMetadataWriter(metadataPublisher, appId, appSpec, creationTime).write(); // add system metadata for programs writeProgramSystemMetadata(appId, ProgramType.MAPREDUCE, appSpec.getMapReduce().values()); writeProgramSystemMetadata(appId, ProgramType.SERVICE, appSpec.getServices().values()); writeProgramSystemMetadata(appId, ProgramType.SPARK, appSpec.getSpark().values()); writeProgramSystemMetadata(appId, ProgramType.WORKER, appSpec.getWorkers().values()); writeProgramSystemMetadata(appId, ProgramType.WORKFLOW, appSpec.getWorkflows().values()); // Emit input to the next stage emit(input); }
@Override protected void validateOptions(Program program, ProgramOptions options) { super.validateOptions(program, options); // Extract and verify parameters ApplicationSpecification appSpec = program.getApplicationSpecification(); Preconditions.checkNotNull(appSpec, "Missing application specification."); ProgramType processorType = program.getType(); Preconditions.checkNotNull(processorType, "Missing processor type."); Preconditions.checkArgument(processorType == ProgramType.WORKFLOW, "Only WORKFLOW process type is supported."); WorkflowSpecification spec = appSpec.getWorkflows().get(program.getName()); Preconditions.checkNotNull(spec, "Missing WorkflowSpecification for %s", program.getName()); for (WorkflowNode node : spec.getNodes()) { if (node.getType().equals(WorkflowNodeType.ACTION)) { SystemArguments.validateTransactionTimeout(options.getUserArguments().asMap(), cConf, "action", node.getNodeId()); } } }
@Override public void process(ApplicationWithPrograms input) throws Exception { // add system metadata for apps ApplicationId appId = input.getApplicationId(); ApplicationSpecification appSpec = input.getSpecification(); // only update creation time if this is a new app Map<String, String> properties = metadataStore.getProperties(MetadataScope.SYSTEM, appId.toMetadataEntity()); SystemMetadataWriter appSystemMetadataWriter = new AppSystemMetadataWriter(metadataStore, appId, appSpec, !properties.isEmpty()); appSystemMetadataWriter.write(); // add system metadata for programs writeProgramSystemMetadata(appId, ProgramType.FLOW, appSpec.getFlows().values()); writeProgramSystemMetadata(appId, ProgramType.MAPREDUCE, appSpec.getMapReduce().values()); writeProgramSystemMetadata(appId, ProgramType.SERVICE, appSpec.getServices().values()); writeProgramSystemMetadata(appId, ProgramType.SPARK, appSpec.getSpark().values()); writeProgramSystemMetadata(appId, ProgramType.WORKER, appSpec.getWorkers().values()); writeProgramSystemMetadata(appId, ProgramType.WORKFLOW, appSpec.getWorkflows().values()); // Emit input to the next stage emit(input); }
private WorkflowToken getWorkflowToken(String namespaceId, String appName, String workflow, String runId) throws NotFoundException { ApplicationId appId = new ApplicationId(namespaceId, appName); ApplicationSpecification appSpec = store.getApplication(appId); if (appSpec == null) { throw new NotFoundException(appId); } WorkflowId workflowId = appId.workflow(workflow); if (!appSpec.getWorkflows().containsKey(workflow)) { throw new NotFoundException(workflowId); } if (store.getRun(workflowId.run(runId)) == null) { throw new NotFoundException(workflowId.run(runId)); } return store.getWorkflowToken(workflowId, runId); }
private WorkflowToken getWorkflowToken(String namespaceId, String appName, String workflow, String runId) throws NotFoundException { ApplicationId appId = new ApplicationId(namespaceId, appName); ApplicationSpecification appSpec = store.getApplication(appId); if (appSpec == null) { throw new NotFoundException(appId); } WorkflowId workflowId = appId.workflow(workflow); if (!appSpec.getWorkflows().containsKey(workflow)) { throw new NotFoundException(workflowId); } if (store.getRun(workflowId.run(runId)) == null) { throw new NotFoundException(workflowId.run(runId)); } return store.getWorkflowToken(workflowId, runId); }