public DistributedMapReduceTaskContextProvider(CConfiguration cConf, Configuration hConf, MapReduceClassLoader mapReduceClassLoader) { super(createInjector(cConf, hConf), mapReduceClassLoader); MapReduceContextConfig mapReduceContextConfig = new MapReduceContextConfig(hConf); Injector injector = getInjector(); Deque<Service> coreServices = new LinkedList<>(); coreServices.add(injector.getInstance(ZKClientService.class)); coreServices.add(injector.getInstance(MetricsCollectionService.class)); if (ProgramRunners.getClusterMode(mapReduceContextConfig.getProgramOptions()) == ClusterMode.ON_PREMISE) { coreServices.add(injector.getInstance(KafkaClientService.class)); } this.coreServices = coreServices; this.logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class); this.mapReduceContextConfig = new MapReduceContextConfig(hConf); }
public DistributedMapReduceTaskContextProvider(CConfiguration cConf, Configuration hConf, MapReduceClassLoader mapReduceClassLoader) { super(createInjector(cConf, hConf), mapReduceClassLoader); MapReduceContextConfig mapReduceContextConfig = new MapReduceContextConfig(hConf); Injector injector = getInjector(); Deque<Service> coreServices = new LinkedList<>(); coreServices.add(injector.getInstance(ZKClientService.class)); coreServices.add(injector.getInstance(MetricsCollectionService.class)); if (ProgramRunners.getClusterMode(mapReduceContextConfig.getProgramOptions()) == ClusterMode.ON_PREMISE) { coreServices.add(injector.getInstance(KafkaClientService.class)); } this.coreServices = coreServices; this.logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class); this.mapReduceContextConfig = new MapReduceContextConfig(hConf); }
private static Injector createInjector(CConfiguration cConf, Configuration hConf) { MapReduceContextConfig mapReduceContextConfig = new MapReduceContextConfig(hConf); // principal will be null if running on a kerberos distributed cluster ProgramOptions programOptions = mapReduceContextConfig.getProgramOptions(); Arguments systemArgs = programOptions.getArguments(); String runId = systemArgs.getOption(ProgramOptionConstants.RUN_ID); return Guice.createInjector( new DistributedProgramContainerModule(cConf, hConf, mapReduceContextConfig.getProgramId().run(runId), systemArgs) ); } }
private static Injector createInjector(CConfiguration cConf, Configuration hConf) { MapReduceContextConfig mapReduceContextConfig = new MapReduceContextConfig(hConf); // principal will be null if running on a kerberos distributed cluster ProgramOptions programOptions = mapReduceContextConfig.getProgramOptions(); Arguments systemArgs = programOptions.getArguments(); String runId = systemArgs.getOption(ProgramOptionConstants.RUN_ID); return Guice.createInjector( new DistributedProgramContainerModule(cConf, hConf, mapReduceContextConfig.getProgramId().run(runId), systemArgs) ); } }
@Override protected void startUp() throws Exception { super.startUp(); try { for (Service service : coreServices) { service.startAndWait(); } logAppenderInitializer.initialize(); ProgramOptions programOptions = mapReduceContextConfig.getProgramOptions(); SystemArguments.setLogLevel(programOptions.getUserArguments(), logAppenderInitializer); } catch (Exception e) { // Try our best to stop services. Chain stop guarantees it will stop everything, even some of them failed. try { shutDown(); } catch (Exception stopEx) { e.addSuppressed(stopEx); } throw e; } }
@Override protected void startUp() throws Exception { super.startUp(); try { for (Service service : coreServices) { service.startAndWait(); } logAppenderInitializer.initialize(); ProgramOptions programOptions = mapReduceContextConfig.getProgramOptions(); SystemArguments.setLogLevel(programOptions.getUserArguments(), logAppenderInitializer); } catch (Exception e) { // Try our best to stop services. Chain stop guarantees it will stop everything, even some of them failed. try { shutDown(); } catch (Exception stopEx) { e.addSuppressed(stopEx); } throw e; } }
LOG.info("Setting up for MapReduce job: namespaceId={}, applicationId={}, program={}, runid={}", programId.getNamespace(), programId.getApplication(), programId.getProgram(), ProgramRunners.getRunId(contextConfig.getProgramOptions())); SystemArguments.getRetryStrategy(contextConfig.getProgramOptions().getUserArguments().asMap(), contextConfig.getProgramId().getType(), cConf); this.txClient = new RetryingLongTransactionSystemClient(injector.getInstance(TransactionSystemClient.class), if (ProgramRunners.getClusterMode(contextConfig.getProgramOptions()) == ClusterMode.ON_PREMISE) { this.transaction = txClient.startLong();
LOG.info("Setting up for MapReduce job: namespaceId={}, applicationId={}, program={}, runid={}", programId.getNamespace(), programId.getApplication(), programId.getProgram(), ProgramRunners.getRunId(contextConfig.getProgramOptions())); SystemArguments.getRetryStrategy(contextConfig.getProgramOptions().getUserArguments().asMap(), contextConfig.getProgramId().getType(), cConf); this.txClient = new RetryingLongTransactionSystemClient(injector.getInstance(TransactionSystemClient.class), if (ProgramRunners.getClusterMode(contextConfig.getProgramOptions()) == ClusterMode.ON_PREMISE) { this.transaction = txClient.startLong();
ProgramRunId programRunId = program.getId().run(ProgramRunners.getRunId(contextConfig.getProgramOptions())); ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programRunId)); MapReduceMetrics.TaskType taskType = null; String taskId = null; ProgramOptions options = contextConfig.getProgramOptions(); RuntimeArguments.extractScope( "task", taskType.toString().toLowerCase(), contextConfig.getProgramOptions().getUserArguments().asMap())), options.isDebug());
ProgramRunId programRunId = program.getId().run(ProgramRunners.getRunId(contextConfig.getProgramOptions())); ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programRunId)); MapReduceMetrics.TaskType taskType = null; String taskId = null; ProgramOptions options = contextConfig.getProgramOptions(); RuntimeArguments.extractScope( "task", taskType.toString().toLowerCase(), contextConfig.getProgramOptions().getUserArguments().asMap())), options.isDebug());
/** * Creates logging context for MapReduce program. If the program is started * by Workflow an instance of {@link WorkflowProgramLoggingContext} is returned, * otherwise an instance of {@link MapReduceLoggingContext} is returned. */ private LoggingContext createMapReduceLoggingContext() { MapReduceContextConfig contextConfig = new MapReduceContextConfig(parameters.getHConf()); ProgramId programId = contextConfig.getProgramId(); RunId runId = ProgramRunners.getRunId(contextConfig.getProgramOptions()); WorkflowProgramInfo workflowProgramInfo = contextConfig.getWorkflowProgramInfo(); if (workflowProgramInfo == null) { return new MapReduceLoggingContext(programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } String workflowId = workflowProgramInfo.getName(); String workflowRunId = workflowProgramInfo.getRunId().getId(); return new WorkflowProgramLoggingContext(programId.getNamespace(), programId.getApplication(), workflowId, workflowRunId, ProgramType.MAPREDUCE, programId.getProgram(), runId.getId()); }
/** * Creates logging context for MapReduce program. If the program is started * by Workflow an instance of {@link WorkflowProgramLoggingContext} is returned, * otherwise an instance of {@link MapReduceLoggingContext} is returned. */ private LoggingContext createMapReduceLoggingContext() { MapReduceContextConfig contextConfig = new MapReduceContextConfig(parameters.getHConf()); ProgramId programId = contextConfig.getProgramId(); RunId runId = ProgramRunners.getRunId(contextConfig.getProgramOptions()); WorkflowProgramInfo workflowProgramInfo = contextConfig.getWorkflowProgramInfo(); if (workflowProgramInfo == null) { return new MapReduceLoggingContext(programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } String workflowId = workflowProgramInfo.getName(); String workflowRunId = workflowProgramInfo.getRunId().getId(); return new WorkflowProgramLoggingContext(programId.getNamespace(), programId.getApplication(), workflowId, workflowRunId, ProgramType.MAPREDUCE, programId.getProgram(), runId.getId()); }