/** * Lists programs of a type belonging to an application. * * @param app the application * @param programType type of the programs to list * @return list of {@link ProgramRecord}s * @throws ApplicationNotFoundException if the application with the given ID was not found * @throws IOException if a network error occurred * @throws UnauthenticatedException if the request is not authorized successfully in the gateway server */ public List<ProgramRecord> listPrograms(ApplicationId app, ProgramType programType) throws ApplicationNotFoundException, IOException, UnauthenticatedException, UnauthorizedException { Preconditions.checkArgument(programType.isListable()); List<ProgramRecord> programs = Lists.newArrayList(); for (ProgramRecord program : listPrograms(app)) { if (programType.equals(program.getType())) { programs.add(program); } } return programs; }
@Override public Map<RunId, RuntimeInfo> list(ProgramType type) { Map<RunId, RuntimeInfo> result = new HashMap<>(); Lock lock = runtimeInfosLock.readLock(); lock.lock(); try { result.putAll(runtimeInfos.row(type)); } finally { lock.unlock(); } // Add any missing RuntimeInfo from the remote twill runner if (remoteTwillRunnerService == null) { return Collections.unmodifiableMap(result); } for (TwillRunner.LiveInfo liveInfo : remoteTwillRunnerService.lookupLive()) { ProgramId programId = TwillAppNames.fromTwillAppName(liveInfo.getApplicationName(), false); if (programId == null || !programId.getType().equals(type)) { continue; } for (TwillController controller : liveInfo.getControllers()) { // For remote twill runner, the twill run id and cdap run id are the same RunId runId = controller.getRunId(); if (result.computeIfAbsent(runId, rid -> createRuntimeInfo(programId, runId, controller)) == null) { LOG.warn("Unable to create runtime info for program {} with run id {}", programId, runId); } } } return Collections.unmodifiableMap(result); }
@Override public Map<RunId, RuntimeInfo> list(ProgramType type) { Map<RunId, RuntimeInfo> result = new HashMap<>(); Lock lock = runtimeInfosLock.readLock(); lock.lock(); try { result.putAll(runtimeInfos.row(type)); } finally { lock.unlock(); } // Add any missing RuntimeInfo from the remote twill runner if (remoteTwillRunnerService == null) { return Collections.unmodifiableMap(result); } for (TwillRunner.LiveInfo liveInfo : remoteTwillRunnerService.lookupLive()) { ProgramId programId = TwillAppNames.fromTwillAppName(liveInfo.getApplicationName(), false); if (programId == null || !programId.getType().equals(type)) { continue; } for (TwillController controller : liveInfo.getControllers()) { // For remote twill runner, the twill run id and cdap run id are the same RunId runId = controller.getRunId(); if (result.computeIfAbsent(runId, rid -> createRuntimeInfo(programId, runId, controller)) == null) { LOG.warn("Unable to create runtime info for program {} with run id {}", programId, runId); } } } return Collections.unmodifiableMap(result); }
/** * @return The workflow token if the program is a workflow, {@code null} otherwise. */ @Nullable public WorkflowToken getWorkflowToken(ProgramRunId programRunId) { ProgramId programId = programRunId.getParent(); if (!programId.getType().equals(ProgramType.WORKFLOW)) { return null; } return store.getWorkflowToken(new WorkflowId(programId.getParent(), programId.getProgram()), programRunId.getRun()); }
/** * @return The workflow token if the program is a workflow, {@code null} otherwise. */ @Nullable public WorkflowToken getWorkflowToken(ProgramRunId programRunId) { ProgramId programId = programRunId.getParent(); if (!programId.getType().equals(ProgramType.WORKFLOW)) { return null; } return store.getWorkflowToken(new WorkflowId(programId.getParent(), programId.getProgram()), programRunId.getRun()); }
Preconditions.checkArgument(ProgramType.MAPREDUCE.equals(runId.getProgram().getType()));
continue; if (!type.equals(programId.getType())) { continue;
Preconditions.checkArgument(ProgramType.MAPREDUCE.equals(runId.getProgram().getType()));
continue; if (!type.equals(programId.getType())) { continue;
Containers.ContainerType containerType = ProgramType.FLOW.equals(program.getType()) ? FLOWLET : Containers.ContainerType.valueOf(program.getType().name());
/** * @param runId for which information will be returned. * @return a {@link MRJobInfo} containing information about a particular MapReduce program run. * @throws IOException if there is failure to communicate through the JobClient. * @throws NotFoundException if a Job with the given runId is not found. */ @Override public MRJobInfo getMRJobInfo(Id.Run runId) throws IOException, NotFoundException { Preconditions.checkArgument(ProgramType.MAPREDUCE.equals(runId.getProgram().getType())); JobClient jobClient = new JobClient(hConf); JobStatus[] jobs = jobClient.getAllJobs(); JobStatus thisJob = findJobForRunId(jobs, runId.toEntityId()); RunningJob runningJob = jobClient.getJob(thisJob.getJobID()); if (runningJob == null) { throw new IllegalStateException(String.format("JobClient returned null for RunId: '%s', JobId: '%s'", runId, thisJob.getJobID())); } Counters counters = runningJob.getCounters(); TaskReport[] mapTaskReports = jobClient.getMapTaskReports(thisJob.getJobID()); TaskReport[] reduceTaskReports = jobClient.getReduceTaskReports(thisJob.getJobID()); return new MRJobInfo(runningJob.mapProgress(), runningJob.reduceProgress(), groupToMap(counters.getGroup(TaskCounter.class.getName())), toMRTaskInfos(mapTaskReports), toMRTaskInfos(reduceTaskReports), true); }
/** * @param runId for which information will be returned. * @return a {@link MRJobInfo} containing information about a particular MapReduce program run. * @throws IOException if there is failure to communicate through the JobClient. * @throws NotFoundException if a Job with the given runId is not found. */ @Override public MRJobInfo getMRJobInfo(Id.Run runId) throws IOException, NotFoundException { Preconditions.checkArgument(ProgramType.MAPREDUCE.equals(runId.getProgram().getType())); JobClient jobClient = new JobClient(hConf); JobStatus[] jobs = jobClient.getAllJobs(); JobStatus thisJob = findJobForRunId(jobs, runId.toEntityId()); RunningJob runningJob = jobClient.getJob(thisJob.getJobID()); if (runningJob == null) { throw new IllegalStateException(String.format("JobClient returned null for RunId: '%s', JobId: '%s'", runId, thisJob.getJobID())); } Counters counters = runningJob.getCounters(); TaskReport[] mapTaskReports = jobClient.getMapTaskReports(thisJob.getJobID()); TaskReport[] reduceTaskReports = jobClient.getReduceTaskReports(thisJob.getJobID()); return new MRJobInfo(runningJob.mapProgress(), runningJob.reduceProgress(), groupToMap(counters.getGroup(TaskCounter.class.getName())), toMRTaskInfos(mapTaskReports), toMRTaskInfos(reduceTaskReports), true); }
@Override public void process(ApplicationDeployable appSpec) throws Exception { List<ProgramSpecification> deletedSpecs = store.getDeletedProgramSpecifications(appSpec.getApplicationId(), appSpec.getSpecification()); // TODO: this should also delete logs and run records (or not?), and do it for all program types [CDAP-2187] List<String> deletedFlows = Lists.newArrayList(); for (ProgramSpecification spec : deletedSpecs) { //call the deleted spec ProgramType type = ProgramTypes.fromSpecification(spec); final ProgramId programId = appSpec.getApplicationId().program(type, spec.getName()); programTerminator.stop(programId); programScheduler.deleteSchedules(programId); programScheduler.modifySchedulesTriggeredByDeletedProgram(programId); // drop all queues and stream states of a deleted flow if (ProgramType.FLOW.equals(type)) { FlowUtils.clearDeletedFlow(impersonator, queueAdmin, streamConsumerFactory, programId, (FlowSpecification) spec); deletedFlows.add(programId.getEntityName()); } // Remove metadata for the deleted program metadataStore.removeMetadata(programId.toMetadataEntity()); } if (!deletedFlows.isEmpty()) { deleteMetrics(appSpec.getApplicationId(), deletedFlows); } emit(appSpec); }
checkStarted(); for (ProgramSchedule schedule: schedules) { if (!schedule.getProgramId().getType().equals(ProgramType.WORKFLOW)) { throw new BadRequestException(String.format( "Cannot schedule program %s of type %s: Only workflows can be scheduled",
checkStarted(); for (ProgramSchedule schedule: schedules) { if (!schedule.getProgramId().getType().equals(ProgramType.WORKFLOW)) { throw new BadRequestException(String.format( "Cannot schedule program %s of type %s: Only workflows can be scheduled",
if (programId.getType().equals(ProgramType.WORKFLOW)) { updateProgramProfileMetadata(programId);
if (!ProgramType.CUSTOM_ACTION.equals(type)) { Assert.assertTrue(programLifecycleService.list(NamespaceId.DEFAULT, type).isEmpty()); if (!ProgramType.CUSTOM_ACTION.equals(type)) { Assert.assertFalse(programLifecycleService.list(NamespaceId.DEFAULT, type).isEmpty()); SecurityRequestContext.setUserId("bob");
if (programDescriptor.getProgramId().getType().equals(ProgramType.MAPREDUCE)) { continue; if (programDescriptor.getProgramId().getType().equals(ProgramType.MAPREDUCE)) { controller = AppFabricTestHelper.submit(app, programDescriptor.getSpecification().getClassName(), new BasicArguments(), TEMP_FOLDER_SUPPLIER);