public NMContext(NMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInNM nmTokenSecretManager, LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager, NMStateStoreService stateStore, boolean isDistSchedulingEnabled, Configuration conf) { if (YarnConfiguration.timelineServiceV2Enabled(conf)) { this.registeringCollectors = new ConcurrentHashMap<>(); this.knownCollectors = new ConcurrentHashMap<>(); } this.containerTokenSecretManager = containerTokenSecretManager; this.nmTokenSecretManager = nmTokenSecretManager; this.dirsHandler = dirsHandler; this.aclsManager = aclsManager; this.nodeHealthStatus.setIsNodeHealthy(true); this.nodeHealthStatus.setHealthReport("Healthy"); this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis()); this.stateStore = stateStore; this.logAggregationReportForApps = new ConcurrentLinkedQueue< LogAggregationReport>(); this.isDistSchedulingEnabled = isDistSchedulingEnabled; this.conf = conf; }
private FlowContext getFlowContext(ContainerLaunchContext launchContext, ApplicationId applicationID) { FlowContext flowContext = null; if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { String flowName = launchContext.getEnvironment() .get(TimelineUtils.FLOW_NAME_TAG_PREFIX); String flowVersion = launchContext.getEnvironment() .get(TimelineUtils.FLOW_VERSION_TAG_PREFIX); String flowRunIdStr = launchContext.getEnvironment() .get(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX); long flowRunId = 0L; if (flowRunIdStr != null && !flowRunIdStr.isEmpty()) { flowRunId = Long.parseLong(flowRunIdStr); } flowContext = new FlowContext(flowName, flowVersion, flowRunId); if (LOG.isDebugEnabled()) { LOG.debug( "Flow context: " + flowContext + " created for an application " + applicationID); } } return flowContext; }
protected SystemMetricsPublisher createSystemMetricsPublisher() { List<SystemMetricsPublisher> publishers = new ArrayList<SystemMetricsPublisher>(); if (YarnConfiguration.timelineServiceV1Enabled(conf)) { SystemMetricsPublisher publisherV1 = new TimelineServiceV1Publisher(); publishers.add(publisherV1); } if (YarnConfiguration.timelineServiceV2Enabled(conf)) { // we're dealing with the v.2.x publisher LOG.info("system metrics publisher with the timeline service V2 is " + "configured"); SystemMetricsPublisher publisherV2 = new TimelineServiceV2Publisher( rmContext.getRMTimelineCollectorManager()); publishers.add(publisherV2); } if (publishers.isEmpty()) { LOG.info("TimelineServicePublisher is not configured"); SystemMetricsPublisher noopPublisher = new NoOpSystemMetricPublisher(); publishers.add(noopPublisher); } for (SystemMetricsPublisher publisher : publishers) { addIfService(publisher); } SystemMetricsPublisher combinedPublisher = new CombinedSystemMetricsPublisher(publishers); return combinedPublisher; }
protected void serviceInit(Configuration conf) throws Exception { if (!YarnConfiguration.timelineServiceV2Enabled(conf)) { throw new IOException("Timeline V2 client is not properly configured. " + "Either timeline service is not enabled or version is not set to"
public RunningAppContext(Configuration config, TaskAttemptFinishingMonitor taskAttemptFinishingMonitor) { this.conf = config; this.clientToAMTokenSecretManager = new ClientToAMTokenSecretManager(appAttemptID, null); this.taskAttemptFinishingMonitor = taskAttemptFinishingMonitor; if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, MRJobConfig.DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA) && YarnConfiguration.timelineServiceEnabled(conf)) { if (YarnConfiguration.timelineServiceV2Enabled(conf)) { // create new version TimelineClient timelineV2Client = TimelineV2Client.createTimelineClient( appAttemptID.getApplicationId()); } else { timelineClient = TimelineClient.createTimelineClient(); } } }
public ApplicationImpl(Dispatcher dispatcher, String user, FlowContext flowContext, ApplicationId appId, Credentials credentials, Context context, long recoveredLogInitedTime) { this.dispatcher = dispatcher; this.user = user; this.appId = appId; this.credentials = credentials; this.aclsManager = context.getApplicationACLsManager(); Configuration conf = context.getConf(); if (YarnConfiguration.timelineServiceV2Enabled(conf)) { if (flowContext == null) { throw new IllegalArgumentException("flow context cannot be null"); } this.flowContext = flowContext; if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) { context.getNMTimelinePublisher().createTimelineClient(appId); } } this.context = context; this.appStateStore = context.getNMStateStore(); ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); readLock = lock.readLock(); writeLock = lock.writeLock(); stateMachine = stateMachineFactory.make(this); setAppLogInitedTimestamp(recoveredLogInitedTime); }
@Override public Void run() throws Exception { if (YarnConfiguration.timelineServiceEnabled(conf)) { timelineServiceV1Enabled = YarnConfiguration.timelineServiceV1Enabled(conf); timelineServiceV2Enabled = YarnConfiguration.timelineServiceV2Enabled(conf); // Creating the Timeline Client if (timelineServiceV1Enabled) { timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf); timelineClient.start(); LOG.info("Timeline service V1 client is enabled"); } if (timelineServiceV2Enabled) { timelineV2Client = TimelineV2Client.createTimelineClient( appAttemptID.getApplicationId()); timelineV2Client.init(conf); timelineV2Client.start(); LOG.info("Timeline service V2 client is enabled"); } } else { timelineClient = null; timelineV2Client = null; LOG.warn("Timeline service is not enabled"); } return null; } });
if (YarnConfiguration.timelineServiceEnabled(conf)) { boolean timelineServiceV2Enabled = YarnConfiguration.timelineServiceV2Enabled(conf); if(timelineServiceV2Enabled) { timelineV2Client =
if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
private void setFlowContext(ContainerLaunchContext container) { if (YarnConfiguration.timelineServiceV2Enabled(conf)) { Map<String, String> environment = container.getEnvironment(); ApplicationId applicationId =
@Test(timeout = 10000) public void testTimelineServiceConfiguration() throws Exception { Configuration config = new Configuration(false); config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); config.set(YarnConfiguration.TIMELINE_SERVICE_VERSIONS, "2.0,1.5"); config.set(YarnConfiguration.TIMELINE_SERVICE_VERSION, "2.0"); Assert.assertTrue(YarnConfiguration.timelineServiceV2Enabled(config)); Assert.assertTrue(YarnConfiguration.timelineServiceV15Enabled(config)); Assert.assertTrue(YarnConfiguration.timelineServiceV1Enabled(config)); config.set(YarnConfiguration.TIMELINE_SERVICE_VERSIONS, "2.0,1"); config.set(YarnConfiguration.TIMELINE_SERVICE_VERSION, "1.5"); Assert.assertTrue(YarnConfiguration.timelineServiceV2Enabled(config)); Assert.assertFalse(YarnConfiguration.timelineServiceV15Enabled(config)); Assert.assertTrue(YarnConfiguration.timelineServiceV1Enabled(config)); config.set(YarnConfiguration.TIMELINE_SERVICE_VERSIONS, "2.0"); config.set(YarnConfiguration.TIMELINE_SERVICE_VERSION, "1.5"); Assert.assertTrue(YarnConfiguration.timelineServiceV2Enabled(config)); Assert.assertFalse(YarnConfiguration.timelineServiceV15Enabled(config)); Assert.assertFalse(YarnConfiguration.timelineServiceV1Enabled(config)); }
if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { ((RMAppImpl) rmApp).removeCollectorData();
if (YarnConfiguration.timelineServiceV2Enabled(conf) && YarnConfiguration.systemMetricsPublisherEnabled(conf)) { LOG.info("YARN system metrics publishing service is enabled");
if (YarnConfiguration.timelineServiceV2Enabled(conf)) { Assert.assertTrue(enableV2); publisherV2 = new TimelineServiceV2Publisher(
if (YarnConfiguration.timelineServiceV2Enabled(context.getConf())) { updateTimelineCollectorData(response);
YarnConfiguration.timelineServiceV2Enabled(getConfig()); if (timelineV2Enabled) {
if (YarnConfiguration.timelineServiceV2Enabled( getRmContext().getYarnConfiguration())) { CollectorInfo collectorInfo = app.getCollectorInfo();
if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
if (YarnConfiguration.timelineServiceV2Enabled(conf)) { this.nmCollectorService = createNMCollectorService(context); addService(nmCollectorService);
if (YarnConfiguration.timelineServiceV2Enabled(this.conf)) { RMTimelineCollectorManager timelineCollectorManager = createRMTimelineCollectorManager();