@Override public Map<String, WorkerSpecification> getWorkers() { return delegate.getWorkers(); }
@Override public Map<String, WorkerSpecification> getWorkers() { return delegate.getWorkers(); }
private Iterable<ProgramSpecification> getProgramSpecs(ApplicationId appId) { ApplicationSpecification appSpec = store.getApplication(appId); return Iterables.concat(appSpec.getMapReduce().values(), appSpec.getServices().values(), appSpec.getSpark().values(), appSpec.getWorkers().values(), appSpec.getWorkflows().values()); }
@Override protected void validateOptions(Program program, ProgramOptions options) { super.validateOptions(program, options); ApplicationSpecification appSpec = program.getApplicationSpecification(); Preconditions.checkNotNull(appSpec, "Missing application specification."); ProgramType processorType = program.getType(); Preconditions.checkNotNull(processorType, "Missing processor type."); Preconditions.checkArgument(processorType == ProgramType.WORKER, "Only WORKER process type is supported."); WorkerSpecification workerSpec = appSpec.getWorkers().get(program.getName()); Preconditions.checkNotNull(workerSpec, "Missing WorkerSpecification for %s", program.getName()); }
private Iterable<ProgramSpecification> getProgramSpecs(ApplicationId appId) { ApplicationSpecification appSpec = store.getApplication(appId); return Iterables.concat(appSpec.getFlows().values(), appSpec.getMapReduce().values(), appSpec.getServices().values(), appSpec.getSpark().values(), appSpec.getWorkers().values(), appSpec.getWorkflows().values()); }
@Override protected void validateOptions(Program program, ProgramOptions options) { super.validateOptions(program, options); ApplicationSpecification appSpec = program.getApplicationSpecification(); Preconditions.checkNotNull(appSpec, "Missing application specification."); ProgramType processorType = program.getType(); Preconditions.checkNotNull(processorType, "Missing processor type."); Preconditions.checkArgument(processorType == ProgramType.WORKER, "Only WORKER process type is supported."); WorkerSpecification workerSpec = appSpec.getWorkers().get(program.getName()); Preconditions.checkNotNull(workerSpec, "Missing WorkerSpecification for %s", program.getName()); }
private static WorkerSpecification getWorkerSpecOrFail(ProgramId id, ApplicationSpecification appSpec) { WorkerSpecification workerSpecification = appSpec.getWorkers().get(id.getProgram()); if (workerSpecification == null) { throw new NoSuchElementException("no such worker @ namespace id: " + id.getNamespaceId() + ", app id: " + id.getApplication() + ", worker id: " + id.getProgram()); } return workerSpecification; }
private static WorkerSpecification getWorkerSpecOrFail(ProgramId id, ApplicationSpecification appSpec) { WorkerSpecification workerSpecification = appSpec.getWorkers().get(id.getProgram()); if (workerSpecification == null) { throw new NoSuchElementException("no such worker @ namespace id: " + id.getNamespaceId() + ", app id: " + id.getApplication() + ", worker id: " + id.getProgram()); } return workerSpecification; }
private Set<ProgramId> getAllPrograms(ApplicationId appId, ApplicationSpecification appSpec) { Set<ProgramId> result = new HashSet<>(); result.addAll(getProgramsWithType(appId, ProgramType.MAPREDUCE, appSpec.getMapReduce())); result.addAll(getProgramsWithType(appId, ProgramType.WORKFLOW, appSpec.getWorkflows())); result.addAll(getProgramsWithType(appId, ProgramType.SERVICE, appSpec.getServices())); result.addAll(getProgramsWithType(appId, ProgramType.SPARK, appSpec.getSpark())); result.addAll(getProgramsWithType(appId, ProgramType.WORKER, appSpec.getWorkers())); return result; }
private void addPrograms(ImmutableMap.Builder<String, String> properties) { addPrograms(ProgramType.MAPREDUCE, appSpec.getMapReduce().values(), properties); addPrograms(ProgramType.SERVICE, appSpec.getServices().values(), properties); addPrograms(ProgramType.SPARK, appSpec.getSpark().values(), properties); addPrograms(ProgramType.WORKER, appSpec.getWorkers().values(), properties); addPrograms(ProgramType.WORKFLOW, appSpec.getWorkflows().values(), properties); }
private Set<ProgramId> getAllPrograms(ApplicationId appId, ApplicationSpecification appSpec) { Set<ProgramId> result = new HashSet<>(); result.addAll(getProgramsWithType(appId, ProgramType.FLOW, appSpec.getFlows())); result.addAll(getProgramsWithType(appId, ProgramType.MAPREDUCE, appSpec.getMapReduce())); result.addAll(getProgramsWithType(appId, ProgramType.WORKFLOW, appSpec.getWorkflows())); result.addAll(getProgramsWithType(appId, ProgramType.SERVICE, appSpec.getServices())); result.addAll(getProgramsWithType(appId, ProgramType.SPARK, appSpec.getSpark())); result.addAll(getProgramsWithType(appId, ProgramType.WORKER, appSpec.getWorkers())); return result; }
private void addPrograms(ImmutableMap.Builder<String, String> properties) { addPrograms(ProgramType.FLOW, appSpec.getFlows().values(), properties); addPrograms(ProgramType.MAPREDUCE, appSpec.getMapReduce().values(), properties); addPrograms(ProgramType.SERVICE, appSpec.getServices().values(), properties); addPrograms(ProgramType.SPARK, appSpec.getSpark().values(), properties); addPrograms(ProgramType.WORKER, appSpec.getWorkers().values(), properties); addPrograms(ProgramType.WORKFLOW, appSpec.getWorkflows().values(), properties); }
private boolean programExists(ProgramId id, ApplicationSpecification appSpec) { switch (id.getType()) { case MAPREDUCE: return appSpec.getMapReduce().containsKey(id.getProgram()); case SERVICE: return appSpec.getServices().containsKey(id.getProgram()); case SPARK: return appSpec.getSpark().containsKey(id.getProgram()); case WORKER: return appSpec.getWorkers().containsKey(id.getProgram()); case WORKFLOW: return appSpec.getWorkflows().containsKey(id.getProgram()); default: throw new IllegalArgumentException("Unexpected ProgramType " + id.getType()); } }
private boolean programExists(ProgramId id, ApplicationSpecification appSpec) { switch (id.getType()) { case FLOW: return appSpec.getFlows().containsKey(id.getProgram()); case MAPREDUCE: return appSpec.getMapReduce().containsKey(id.getProgram()); case SERVICE: return appSpec.getServices().containsKey(id.getProgram()); case SPARK: return appSpec.getSpark().containsKey(id.getProgram()); case WORKER: return appSpec.getWorkers().containsKey(id.getProgram()); case WORKFLOW: return appSpec.getWorkflows().containsKey(id.getProgram()); default: throw new IllegalArgumentException("Unexpected ProgramType " + id.getType()); } }
@Override public void process(ApplicationWithPrograms input) { // use current time as creation time for app and all programs creationTime = String.valueOf(System.currentTimeMillis()); // add system metadata for apps ApplicationId appId = input.getApplicationId(); ApplicationSpecification appSpec = input.getSpecification(); new AppSystemMetadataWriter(metadataPublisher, appId, appSpec, creationTime).write(); // add system metadata for programs writeProgramSystemMetadata(appId, ProgramType.MAPREDUCE, appSpec.getMapReduce().values()); writeProgramSystemMetadata(appId, ProgramType.SERVICE, appSpec.getServices().values()); writeProgramSystemMetadata(appId, ProgramType.SPARK, appSpec.getSpark().values()); writeProgramSystemMetadata(appId, ProgramType.WORKER, appSpec.getWorkers().values()); writeProgramSystemMetadata(appId, ProgramType.WORKFLOW, appSpec.getWorkflows().values()); // Emit input to the next stage emit(input); }
@Override protected void setupLaunchConfig(ProgramLaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) { ApplicationSpecification appSpec = program.getApplicationSpecification(); WorkerSpecification workerSpec = appSpec.getWorkers().get(program.getName()); String instances = options.getArguments().getOption(ProgramOptionConstants.INSTANCES, String.valueOf(workerSpec.getInstances())); launchConfig.addRunnable(workerSpec.getName(), new WorkerTwillRunnable(workerSpec.getName()), Integer.parseInt(instances), options.getUserArguments().asMap(), workerSpec.getResources()); } }
@Override protected void setupLaunchConfig(ProgramLaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) { ApplicationSpecification appSpec = program.getApplicationSpecification(); WorkerSpecification workerSpec = appSpec.getWorkers().get(program.getName()); String instances = options.getArguments().getOption(ProgramOptionConstants.INSTANCES, String.valueOf(workerSpec.getInstances())); launchConfig.addRunnable(workerSpec.getName(), new WorkerTwillRunnable(workerSpec.getName()), Integer.parseInt(instances), options.getUserArguments().asMap(), workerSpec.getResources()); } }
@Override public ProgramController run(Program program, ProgramOptions options) { // Extract and verify parameters ApplicationSpecification appSpec = program.getApplicationSpecification(); Preconditions.checkNotNull(appSpec, "Missing application specification."); ProgramType type = program.getType(); Preconditions.checkNotNull(type, "Missing processor type."); Preconditions.checkArgument(type == ProgramType.WORKER, "Only WORKER process type is supported."); WorkerSpecification workerSpec = appSpec.getWorkers().get(program.getName()); Preconditions.checkNotNull(workerSpec, "Missing WorkerSpecification for %s", program.getName()); String instances = options.getArguments().getOption(ProgramOptionConstants.INSTANCES, String.valueOf(workerSpec.getInstances())); WorkerSpecification newWorkerSpec = new WorkerSpecification(workerSpec.getClassName(), workerSpec.getName(), workerSpec.getDescription(), workerSpec.getProperties(), workerSpec.getDatasets(), workerSpec.getResources(), Integer.valueOf(instances), workerSpec.getPlugins()); return startAll(program, options, newWorkerSpec.getInstances()); }
@Override public ProgramController run(Program program, ProgramOptions options) { // Extract and verify parameters ApplicationSpecification appSpec = program.getApplicationSpecification(); Preconditions.checkNotNull(appSpec, "Missing application specification."); ProgramType type = program.getType(); Preconditions.checkNotNull(type, "Missing processor type."); Preconditions.checkArgument(type == ProgramType.WORKER, "Only WORKER process type is supported."); WorkerSpecification workerSpec = appSpec.getWorkers().get(program.getName()); Preconditions.checkNotNull(workerSpec, "Missing WorkerSpecification for %s", program.getName()); String instances = options.getArguments().getOption(ProgramOptionConstants.INSTANCES, String.valueOf(workerSpec.getInstances())); WorkerSpecification newWorkerSpec = new WorkerSpecification(workerSpec.getClassName(), workerSpec.getName(), workerSpec.getDescription(), workerSpec.getProperties(), workerSpec.getDatasets(), workerSpec.getResources(), Integer.valueOf(instances), workerSpec.getPlugins()); return startAll(program, options, newWorkerSpec.getInstances()); }
@Test public void testWorkerInstances() { ApplicationSpecification spec = Specifications.from(new AppWithWorker()); ApplicationId appId = NamespaceId.DEFAULT.app(spec.getName()); store.addApplication(appId, spec); ProgramId programId = appId.worker(AppWithWorker.WORKER); int instancesFromSpec = spec.getWorkers().get(AppWithWorker.WORKER).getInstances(); Assert.assertEquals(1, instancesFromSpec); int instances = store.getWorkerInstances(programId); Assert.assertEquals(instancesFromSpec, instances); store.setWorkerInstances(programId, 9); instances = store.getWorkerInstances(programId); Assert.assertEquals(9, instances); }