@Override public int getInstanceId() { return twillContext.getInstanceId(); }
@Override public int getInstanceId() { return twillContext.getInstanceId(); }
MetricsProcessorModule(TwillContext twillContext) { this.instanceId = twillContext.getInstanceId(); }
/** * Creates program arguments. It includes all configurations from the specification, excluding hConf and cConf. */ private Arguments createProgramArguments(TwillContext context, Map<String, String> configs) { Map<String, String> args = ImmutableMap.<String, String>builder() .put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(context.getInstanceId())) .put(ProgramOptionConstants.INSTANCES, Integer.toString(context.getInstanceCount())) .put(ProgramOptionConstants.RUN_ID, context.getApplicationRunId().getId()) .putAll(Maps.filterKeys(configs, Predicates.not(Predicates.in(ImmutableSet.of("hConf", "cConf"))))) .build(); return new BasicArguments(args); }
/** * Returns a set of extra system arguments that will be available through the {@link ProgramOptions#getArguments()} * for the program execution. */ protected Map<String, String> getExtraSystemArguments() { Map<String, String> args = new HashMap<>(); args.put(ProgramOptionConstants.INSTANCE_ID, context == null ? "0" : Integer.toString(context.getInstanceId())); args.put(ProgramOptionConstants.INSTANCES, context == null ? "1" : Integer.toString(context.getInstanceCount())); args.put(ProgramOptionConstants.TWILL_RUN_ID, context.getApplicationRunId().getId()); args.put(ProgramOptionConstants.HOST, context.getHost().getCanonicalHostName()); return args; }
/** * Returns a set of extra system arguments that will be available through the {@link ProgramOptions#getArguments()} * for the program execution. */ protected Map<String, String> getExtraSystemArguments() { Map<String, String> args = new HashMap<>(); args.put(ProgramOptionConstants.INSTANCE_ID, context == null ? "0" : Integer.toString(context.getInstanceId())); args.put(ProgramOptionConstants.INSTANCES, context == null ? "1" : Integer.toString(context.getInstanceCount())); args.put(ProgramOptionConstants.TWILL_RUN_ID, context.getApplicationRunId().getId()); args.put(ProgramOptionConstants.HOST, context.getHost().getCanonicalHostName()); return args; }
/** * Returns the metric context. A metric context is of the form {flowY}.{flowletZ}. */ private String getMetricContext(Program program, TwillContext context) { String metricContext = program.getName(); metricContext += "." + context.getSpecification().getName() + "." + context.getInstanceId(); return metricContext; } }
@Override public void handleRequest(BufferedReader reader, PrintWriter writer) throws IOException { String line = reader.readLine(); writer.println(getContext().getInstanceId() + line); writer.flush(); } }
@Override public void run() { LOG.info("Failing instance {}", getContext().getInstanceId()); throw new RuntimeException("FAIL early FAIL often"); } }
@Override public void handleCommand(Command command) throws Exception { if (command.getCommand().equals("kill" + getContext().getInstanceId())) { killed = true; running = false; serverSocket.close(); } }
@Override public void run() { if (getContext().getInstanceId() > 1) { LOG.info("Failing instance {}", getContext().getInstanceId()); throw new RuntimeException("FAIL early FAIL often"); } else { LOG.info("Instance {} is running", getContext().getInstanceId()); while (true) { try { Thread.sleep(100); } catch (InterruptedException e) { return; } } } } }
@Override protected Injector doInit(TwillContext context) { // Set the hostname of the machine so that cConf can be used to start internal services getCConfiguration().set(Constants.Metrics.ADDRESS, context.getHost().getCanonicalHostName()); LOG.info("{} Setting host name to {}", name, context.getHost().getCanonicalHostName()); String txClientId = String.format("cdap.service.%s.%d", Constants.Service.METRICS, context.getInstanceId()); injector = createGuiceInjector(getCConfiguration(), getConfiguration(), txClientId); injector.getInstance(LogAppenderInitializer.class).initialize(); LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.METRICS)); return injector; }
@Override protected Injector doInit(TwillContext context) { getCConfiguration().set(Constants.Transaction.Container.ADDRESS, context.getHost().getCanonicalHostName()); // Set the hostname of the machine so that cConf can be used to start internal services LOG.info("{} Setting host name to {}", name, context.getHost().getCanonicalHostName()); String txClientId = String.format("cdap.service.%s.%d", Constants.Service.TRANSACTION, context.getInstanceId()); injector = createGuiceInjector(getCConfiguration(), getConfiguration(), txClientId); injector.getInstance(LogAppenderInitializer.class).initialize(); LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.TRANSACTION)); return injector; }
@Override protected Injector doInit(TwillContext context) { getCConfiguration().set(Constants.MetricsProcessor.ADDRESS, context.getHost().getCanonicalHostName()); // Set the hostname of the machine so that cConf can be used to start internal services LOG.info("{} Setting host name to {}", name, context.getHost().getCanonicalHostName()); String txClientId = String.format("cdap.service.%s.%d", Constants.Service.METRICS_PROCESSOR, context.getInstanceId()); injector = createGuiceInjector(getCConfiguration(), getConfiguration(), txClientId, context); injector.getInstance(LogAppenderInitializer.class).initialize(); LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.METRICS_PROCESSOR)); return injector; }
@Override protected Injector doInit(TwillContext context) { CConfiguration cConf = getCConfiguration(); Configuration hConf = getConfiguration(); // Set the host name to the one provided by Twill cConf.set(Constants.Dataset.Executor.ADDRESS, context.getHost().getHostName()); String txClientId = String.format("cdap.service.%s.%d", Constants.Service.DATASET_EXECUTOR, context.getInstanceId()); injector = createInjector(cConf, hConf, txClientId); injector.getInstance(LogAppenderInitializer.class).initialize(); LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.DATASET_EXECUTOR)); return injector; }
System.setProperty(MetricNames.METRICS_ID_PROP, "oracle-" + context.getInstanceId());
@Override protected Injector doInit(TwillContext context) { CConfiguration cConf = getCConfiguration(); cConf.set(Constants.MessagingSystem.HTTP_SERVER_BIND_ADDRESS, context.getHost().getHostName()); cConf.setInt(Constants.MessagingSystem.CONTAINER_INSTANCE_ID, context.getInstanceId()); injector = createInjector(cConf, getConfiguration()); injector.getInstance(LogAppenderInitializer.class).initialize(); LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.MESSAGING_SERVICE)); return injector; }
@Override public void initialize(TwillContext context) { super.initialize(context); running = true; try { serverSocket = new ServerSocket(0); LOG.info("Server started: " + serverSocket.getLocalSocketAddress() + ", id: " + context.getInstanceId() + ", count: " + context.getInstanceCount()); // Announce with service names as specified in app arguments and runnable arguments final List<Cancellable> cancellables = new ArrayList<>(); for (String[] args : new String[][] {context.getApplicationArguments(), context.getArguments()}) { if (args.length > 0) { cancellables.add(context.announce(args[0], serverSocket.getLocalPort())); } } canceller = new Cancellable() { @Override public void cancel() { for (Cancellable c : cancellables) { c.cancel(); } } }; } catch (IOException e) { throw Throwables.propagate(e); } }
@Override protected Injector doInit(TwillContext context) { setupHive(); CConfiguration cConf = getCConfiguration(); Configuration hConf = getConfiguration(); addResource(hConf, "yarn-site.xml"); addResource(hConf, "mapred-site.xml"); addResource(hConf, "hive-site.xml"); addResource(hConf, "tez-site.xml"); // Set the host name to the one provided by Twill cConf.set(Constants.Explore.SERVER_ADDRESS, context.getHost().getHostName()); String txClientId = String.format("cdap.service.%s.%d", Constants.Service.EXPLORE_HTTP_USER_SERVICE, context.getInstanceId()); // NOTE: twill client will try to load all the classes present here - including hive classes but it // will fail since Hive classes are not in master classpath, and ignore those classes silently injector = createInjector(cConf, hConf, txClientId); injector.getInstance(LogAppenderInitializer.class).initialize(); LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.EXPLORE_HTTP_USER_SERVICE)); return injector; }
@VisibleForTesting static Injector createGuiceInjector(CConfiguration cConf, Configuration hConf, TwillContext twillContext) { String txClientId = String.format("cdap.service.%s.%d", Constants.Service.LOGSAVER, twillContext.getInstanceId()); return Guice.createInjector( new ConfigModule(cConf, hConf), new IOModule(), new ZKClientModule(), new ZKDiscoveryModule(), new KafkaClientModule(), new MetricsClientRuntimeModule().getDistributedModules(), new DFSLocationModule(), new NamespaceQueryAdminModule(), new DataFabricModules(txClientId).getDistributedModules(), new DataSetsModules().getDistributedModules(), new DistributedLogFrameworkModule(twillContext), new KafkaLogAppenderModule(), new AuditModule().getDistributedModules(), new AuthorizationEnforcementModule().getDistributedModules(), new AuthenticationContextModules().getMasterModule(), new MessagingClientModule(), new AbstractModule() { @Override protected void configure() { bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class); bind(UGIProvider.class).to(RemoteUGIProvider.class).in(Scopes.SINGLETON); } } ); } }