/** * Build the environment map from a role option map, finding all entries * beginning with "env.", adding them to a map of (prefix-removed) * env vars * @param roleOpts role options. This can be null, meaning the * role is undefined * @return a possibly empty map of environment variables. */ public static Map<String, String> buildEnvMap(Map<String, String> roleOpts) { return buildEnvMap(roleOpts, null); }
/** * Release a tag associated with a container * * @param component * @param containerId */ public void releaseTag(String component, String containerId) { if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) { ConcurrentHashMap<String, String> compTags = allTags.get(component); if (compTags != null) { synchronized (compTags) { for (String key : compTags.keySet()) { if (compTags.get(key).equals(containerId)) { compTags.put(key, FREE); } } } } } }
private String getSubjectDN(String hostname, String containerId, String appName) { return String.format("/CN=%s%s%s", SliderUtils.extractDomainNameFromFQDN(hostname), containerId != null ? "/OU=" + containerId : "", appName != null ? "/OU=" + appName : ""); } }
/** * Get slider dependency parent dir in HDFS * * @return the parent dir path of slider.tar.gz in HDFS */ public Path getDependencyPath() { String parentDir = (SliderUtils.isHdp()) ? SliderKeys.SLIDER_DEPENDENCY_HDP_PARENT_DIR + SliderKeys.SLIDER_DEPENDENCY_DIR : SliderKeys.SLIDER_DEPENDENCY_DIR; Path dependencyPath = new Path(String.format(parentDir, SliderUtils.getSliderVersion())); return dependencyPath; }
/** * Retrieve the version of the current Slider install * * @return the version string of the Slider release */ public static String getSliderVersion() { if (isHdp()) { return getHdpVersion(); } else { Properties props = SliderVersionInfo.loadVersionProperties(); return props.getProperty(SliderVersionInfo.APP_VERSION); } }
/** * Loads all dependency jars from the default path * @param providerResources map of provider resources to add these entries to * @param sliderFileSystem target filesystem * @param tempPath path in the cluster FS for temp files * @param libDir relative directory to place resources * @param libLocalSrcDir explicitly supplied local libs dir * @throws IOException * @throws SliderException */ public static void addAllDependencyJars(Map<String, LocalResource> providerResources, SliderFileSystem sliderFileSystem, Path tempPath, String libDir, String libLocalSrcDir) throws IOException, SliderException { String libSrcToUse = libLocalSrcDir; if (SliderUtils.isSet(libLocalSrcDir)) { File file = new File(libLocalSrcDir); if (!file.exists() || !file.isDirectory()) { throw new BadCommandArgumentsException("Supplied lib src dir %s is not valid", libLocalSrcDir); } } SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath, libDir, libSrcToUse); }
throws YarnException, IOException{ deployedClusterName = clustername; validateClusterName(clustername); verifyNoLiveClusters(clustername, "Launch"); Configuration config = getConfig(); lookupZKQuorum(); boolean clusterSecure = isHadoopClusterSecure(config); getApplicationDefinitionPath(appOperations)); Path localConfDirPath = createLocalPath(confDir); remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR); log.debug("Slider configuration directory is {}; remote to be {}", localConfDirPath, remoteConfPath); copyDirectory(config, localConfDirPath, remoteConfPath, null); sliderFileSystem.submitDirectory(remoteConfPath, relativeConfDir); mergeMaps(localResources, submittedConfDir); copyDirectory(config, snapshotConfPath, generatedConfDirPath, clusterPerms); extractImagePath(sliderFileSystem, internalOptions); if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) { log.debug("Registered image path {}", imagePath); buildEnvMap(sliderAMResourceComponent)); ClasspathConstructor classpath = buildClasspath(relativeConfDir,
String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition .getAppConfOperations(), roleGroup); launcher.putEnv(SliderUtils.buildEnvMap(appComponent, getStandardTokenMap(getAmState().getAppConfSnapshot(), roleName, roleGroup))); String appHome = instanceDefinition.getAppConfOperations(). getGlobalOptions().get(AgentKeys.PACKAGE_PATH); if (SliderUtils.isSet(appHome)) { scriptPath = new File(appHome, AgentKeys.AGENT_MAIN_SCRIPT).getPath(); String agentImage = instanceDefinition.getInternalOperations(). get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); if (SliderUtils.isUnset(agentImage)) { agentImagePath = new Path(new Path(new Path(instanceDefinition.getInternalOperations().get(InternalKeys.INTERNAL_TMP_DIR), if (SliderUtils.isSet(agentConf)) { LocalResource agentConfRes = fileSystem.createAmResource(fileSystem .getFileSystem().resolvePath(new Path(agentConf)), if (SliderUtils.isHadoopClusterSecure(getConfig())) { localizeServiceKeytabs(launcher, instanceDefinition, fileSystem); if (SliderUtils.isSet(debugCmd)) { operation.add(ARG_DEBUG); operation.add(debugCmd);
log.info("Conf dir {} does not exist.", confDir); File parentFile = confDir.getParentFile(); log.info("Parent dir {}:\n{}", parentFile, SliderUtils.listDir(parentFile)); initAndAddService(sliderAMProvider); InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf); log.info("RM is at {}", rmSchedulerAddress); yarnRPC = YarnRPC.create(serviceConf); InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(serviceConf); if (!SliderUtils.isAddressDefined(clientRpcAddress)) { SliderUtils.mandatoryEnvVariable(ApplicationConstants.Environment.CONTAINER_ID.name())); appAttemptID = appMasterContainerID.getApplicationAttemptId(); if (SliderUtils.isUnset(appMasterProxiedUrl)) { log.warn("Proxied URL is not set in application report"); appMasterProxiedUrl = appMasterTrackingUrl;
launcher.putEnv(SliderUtils.buildEnvMap(appComponent)); if (SliderUtils.isSet(heap)) { String adjustedHeap = SliderUtils.translateTrailingHeapUnit(heap); launcher.setEnv(HBASE_HEAPSIZE, adjustedHeap); if (SliderUtils.isSet(gcOpts)) { launcher.setEnv(HBASE_GC_OPTS, gcOpts);
@Override public void validate() throws BadCommandArgumentsException, UsageException { super.validate(); if (keytab != null && SliderUtils.isUnset(principal)) { throw new UsageException("Missing argument " + ARG_PRINCIPAL); } if (keytab == null && SliderUtils.isSet(principal)) { throw new UsageException("Missing argument " + ARG_KEYTAB); } } }
launcher.putEnv(SliderUtils.buildEnvMap(appComponent)); Map<String, String> env = SliderUtils.buildEnvMap(appComponent); launcher.setEnv(ACCUMULO_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR); ConfTreeOperations appConf = if (SliderUtils.isSet(heap)) {
private void createSummaryMetainfoFile(Path srcFile, Path destFile, boolean overwrite) throws IOException { FileSystem srcFs = srcFile.getFileSystem(getConfig()); try (InputStream inputStreamJson = SliderUtils .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.json"); InputStream inputStreamXml = SliderUtils .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.xml");) { InputStream inputStream = null; Path summaryFileInFs = null; if (inputStreamJson != null) { inputStream = inputStreamJson; summaryFileInFs = new Path(destFile.getParent(), destFile.getName() + ".metainfo.json"); log.info("Found JSON metainfo file in package"); } else if (inputStreamXml != null) { inputStream = inputStreamXml; summaryFileInFs = new Path(destFile.getParent(), destFile.getName() + ".metainfo.xml"); log.info("Found XML metainfo file in package"); } if (inputStream != null) { try (FSDataOutputStream dataOutputStream = sliderFileSystem .getFileSystem().create(summaryFileInFs, overwrite)) { log.info("Creating summary metainfo file"); IOUtils.copy(inputStream, dataOutputStream); } } } }
/** * Get the URL log for a container * @param c container * @return the URL or "" if it cannot be determined */ public static String getLogsURLForContainer(Container c) { if (c==null) { return null; } String user = null; try { user = SliderUtils.getCurrentUser().getShortUserName(); } catch (IOException ignored) { } String completedLogsUrl = ""; String url = logServerURL; if (user != null && SliderUtils.isSet(url)) { completedLogsUrl = url + "/" + c.getNodeId() + "/" + c.getId() + "/ctx/" + user; } log.info("Completed log link = {}", completedLogsUrl); return completedLogsUrl; }
/** * Submit a JAR containing and map it * @param providerResources provider map to build up * @param sliderFileSystem remote fs * @param libDir lib directory * @param srcPath copy jars from * @throws IOException, SliderException trouble copying to HDFS */ public static void putAllJars(Map<String, LocalResource> providerResources, SliderFileSystem sliderFileSystem, Path tempPath, String libDir, String srcPath) throws IOException, SliderException { log.info("Loading all dependencies from {}", srcPath); if (SliderUtils.isSet(srcPath)) { File srcFolder = new File(srcPath); FilenameFilter jarFilter = createJarFilter(); File[] listOfJars = srcFolder.listFiles(jarFilter); if (listOfJars == null || listOfJars.length == 0) { return; } for (File jarFile : listOfJars) { LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName()); providerResources.put(libDir + "/" + jarFile.getName(), res); } } }
sortApplicationsByMostRecent(instances); Map<String, ApplicationReport> reportMap = buildApplicationReportMap(instances, min, max); log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}", persistentInstances.size(), String details = instanceDetailsToString(name, report, containers, version, components, verbose); print(details);
@Override //AMRMClientAsync public void onError(Throwable e) { if (e instanceof InvalidResourceRequestException) { // stop the cluster LOG_YARN.error("AMRMClientAsync.onError() received {}", e, e); ActionStopSlider stopSlider = new ActionStopSlider("stop", EXIT_EXCEPTION_THROWN, FinalApplicationStatus.FAILED, SliderUtils.extractFirstLine(e.getLocalizedMessage())); stopSlider.setExitReason(SliderExitReason.APP_ERROR); signalAMComplete(stopSlider); } else if (e instanceof InvalidApplicationMasterRequestException) { // halt the AM LOG_YARN.error("AMRMClientAsync.onError() received {}", e, e); queue(new ActionHalt(EXIT_EXCEPTION_THROWN, SliderUtils.extractFirstLine(e.getLocalizedMessage()))); } else { // ignore and log LOG_YARN.info("Ignoring AMRMClientAsync.onError() received {}", e); } }
/** * Verify that a file exists * @param file file * @param errorlog log for output on an error * @throws FileNotFoundException */ public static void verifyFileExists(File file, Logger errorlog) throws FileNotFoundException { if (!file.exists()) { errorlog.warn("contents of {}: {}", file, listDir(file.getParentFile())); throw new FileNotFoundException(file.toString()); } if (!file.isFile()) { throw new FileNotFoundException("Not a file: " + file.toString()); } }
@Test public void testApplicationReportComparison() { List<ApplicationReport> instances = getApplicationReports(); SliderUtils.sortApplicationsByMostRecent(instances); Assert.assertEquals(1000, instances.get(0).getStartTime()); Assert.assertEquals(1000, instances.get(1).getStartTime()); Assert.assertEquals(1000, instances.get(2).getStartTime()); Assert.assertEquals(1000, instances.get(3).getStartTime()); instances = getApplicationReports(); SliderUtils.sortApplicationReport(instances); Assert.assertEquals(1000, instances.get(0).getStartTime()); Assert.assertEquals(1000, instances.get(1).getStartTime()); Assert.assertEquals(1000, instances.get(2).getStartTime()); Assert.assertEquals(1000, instances.get(3).getStartTime()); Assert.assertTrue(instances.get(0).getYarnApplicationState() == YarnApplicationState.ACCEPTED || instances.get(0).getYarnApplicationState() == YarnApplicationState.RUNNING); Assert.assertTrue(instances.get(1).getYarnApplicationState() == YarnApplicationState.ACCEPTED || instances.get(1).getYarnApplicationState() == YarnApplicationState.RUNNING); Assert.assertTrue(instances.get(2).getYarnApplicationState() == YarnApplicationState.ACCEPTED || instances.get(2).getYarnApplicationState() == YarnApplicationState.RUNNING); Assert.assertTrue(instances.get(3).getYarnApplicationState() == YarnApplicationState.KILLED); }
SliderUtils.sortApplicationsByMostRecent(rmInstances); Map<String, ApplicationReport> reportMap = SliderUtils.buildApplicationReportMap(rmInstances, minAppState, maxAppState); log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}",