public void refresh() throws IOException { HostDetails hostDetails = current.get(); refresh(hostDetails.includesFile, hostDetails.excludesFile); }
public void refresh(String includesFile, String excludesFile) throws IOException { LOG.info("Refreshing hosts (include/exclude) list"); HostDetails oldDetails = current.get(); Set<String> newIncludes = oldDetails.includes; Map<String, Integer> newExcludes = oldDetails.excludes; if (includesFile != null && !includesFile.isEmpty()) { newIncludes = new HashSet<>(); readFileToSet("included", includesFile, newIncludes); newIncludes = Collections.unmodifiableSet(newIncludes); } if (excludesFile != null && !excludesFile.isEmpty()) { newExcludes = new HashMap<>(); readFileToMap("excluded", excludesFile, newExcludes); newExcludes = Collections.unmodifiableMap(newExcludes); } HostDetails newDetails = new HostDetails(includesFile, newIncludes, excludesFile, newExcludes); current.set(newDetails); }
public static void readFileToSet(String type, String filename, Set<String> set) throws IOException { File file = new File(filename); FileInputStream fis = new FileInputStream(file); readFileToSetWithFileInputStream(type, filename, fis, set); }
public static void readFileToMapWithFileInputStream(String type, String filename, InputStream inputStream, Map<String, Integer> map) throws IOException { // The input file could be either simple text or XML. boolean xmlInput = filename.toLowerCase().endsWith(".xml"); if (xmlInput) { readXmlFileToMapWithFileInputStream(type, filename, inputStream, map); } else { HashSet<String> nodes = new HashSet<String>(); readFileToSetWithFileInputStream(type, filename, inputStream, nodes); for (String node : nodes) { map.put(node, null); } } }
@Private public void refresh(InputStream inFileInputStream, InputStream exFileInputStream) throws IOException { LOG.info("Refreshing hosts (include/exclude) list"); HostDetails oldDetails = current.get(); Set<String> newIncludes = oldDetails.includes; Map<String, Integer> newExcludes = oldDetails.excludes; if (inFileInputStream != null) { newIncludes = new HashSet<>(); readFileToSetWithFileInputStream("included", oldDetails.includesFile, inFileInputStream, newIncludes); newIncludes = Collections.unmodifiableSet(newIncludes); } if (exFileInputStream != null) { newExcludes = new HashMap<>(); readFileToMapWithFileInputStream("excluded", oldDetails.excludesFile, exFileInputStream, newExcludes); newExcludes = Collections.unmodifiableMap(newExcludes); } HostDetails newDetails = new HostDetails( oldDetails.includesFile, newIncludes, oldDetails.excludesFile, newExcludes); current.set(newDetails); }
@Test public void testHostFileReaderWithCommentsOnly() throws Exception { FileWriter efw = new FileWriter(excludesFile); FileWriter ifw = new FileWriter(includesFile); efw.write("#DFS-Hosts-excluded\n"); efw.close(); ifw.write("#Hosts-in-DFS\n"); ifw.close(); HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile); int includesLen = hfp.getHosts().size(); int excludesLen = hfp.getExcludedHosts().size(); assertEquals(0, includesLen); assertEquals(0, excludesLen); assertFalse(hfp.getHosts().contains("somehost5")); assertFalse(hfp.getExcludedHosts().contains("somehost5")); }
private void printConfiguredHosts() { if (!LOG.isDebugEnabled()) { return; } LOG.debug("hostsReader: in=" + conf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH) + " out=" + conf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH)); for (String include : hostsReader.getHosts()) { LOG.debug("include: " + include); } for (String exclude : hostsReader.getExcludedHosts()) { LOG.debug("exclude: " + exclude); } }
private synchronized void refreshHosts() throws IOException { // Reread the config to get HOSTS and HOSTS_EXCLUDE filenames. // Update the file names and refresh internal includes and excludes list LOG.info("Refreshing hosts information"); Configuration conf = new Configuration(); hostsReader.updateFileNames(conf.get(JTConfig.JT_HOSTS_FILENAME,""), conf.get(JTConfig.JT_HOSTS_EXCLUDE_FILENAME, "")); hostsReader.refresh(); Set<String> excludeSet = new HashSet<String>(); for(Map.Entry<String, TaskTracker> eSet : taskTrackers.entrySet()) { String trackerName = eSet.getKey(); TaskTrackerStatus status = eSet.getValue().getStatus(); // Check if not include i.e not in host list or in hosts list but excluded if (!inHostsList(status) || inExcludedHostsList(status)) { excludeSet.add(status.getHost()); // add to rejected trackers } } decommissionNodes(excludeSet); }
/** * Returns a set of excluded nodes. */ Collection<String> getExcludedNodes() { return hostsReader.getExcludedHosts(); }
private static int processHostList(String[] args, int i, String type, Set<String> nodes) { Preconditions.checkArgument(++i < args.length, "List of %s nodes | -f <filename> is missing: args=%s", type, Arrays.toString(args)); if ("-f".equalsIgnoreCase(args[i])) { Preconditions.checkArgument(++i < args.length, "File containing %s nodes is not specified: args=%s", type, Arrays.toString(args)); final String filename = args[i]; try { HostsFileReader.readFileToSet(type, filename, nodes); } catch (IOException e) { throw new IllegalArgumentException( "Failed to read " + type + " node list from file: " + filename); } } else { final String[] addresses = StringUtils.getTrimmedStrings(args[i]); nodes.addAll(Arrays.asList(addresses)); } return i; }
private void refreshHostsReader( Configuration yarnConf, boolean graceful, Integer timeout) throws IOException, YarnException { // resolve the default timeout to the decommission timeout that is // configured at this moment if (null == timeout) { timeout = readDecommissioningTimeout(yarnConf); } if (null == yarnConf) { yarnConf = new YarnConfiguration(); } includesFile = yarnConf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH); excludesFile = yarnConf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH); LOG.info("refreshNodes excludesFile " + excludesFile); hostsReader.refresh(includesFile, excludesFile); printConfiguredHosts(); LOG.info("hostsReader include:{" + StringUtils.join(",", hostsReader.getHosts()) + "} exclude:{" + StringUtils.join(",", hostsReader.getExcludedHosts()) + "}"); handleExcludeNodeList(graceful, timeout); }
@Test public void testRefreshHostFileReaderWithNonexistentFile() throws Exception { FileWriter efw = new FileWriter(excludesFile); FileWriter ifw = new FileWriter(includesFile); efw.close(); ifw.close(); HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile); assertTrue(INCLUDES_FILE.delete()); try { hfp.refresh(); Assert.fail("Should throw FileNotFoundException"); } catch (FileNotFoundException ex) { // Exception as expected } }
private synchronized void refreshHosts() throws IOException { // Reread the config to get mapred.hosts and mapred.hosts.exclude filenames. // Update the file names and refresh internal includes and excludes list LOG.info("Refreshing hosts information"); Configuration conf = new Configuration(); hostsReader.updateFileNames(conf.get("mapred.hosts",""), conf.get("mapred.hosts.exclude", "")); hostsReader.refresh(); Set<String> excludeSet = new HashSet<String>(); for(Map.Entry<String, TaskTracker> eSet : taskTrackers.entrySet()) { String trackerName = eSet.getKey(); TaskTrackerStatus status = eSet.getValue().getStatus(); // Check if not include i.e not in host list or in hosts list but excluded if (!inHostsList(status) || inExcludedHostsList(status)) { excludeSet.add(status.getHost()); // add to rejected trackers } } decommissionNodes(excludeSet); int totalExcluded = hostsReader.getExcludedHosts().size(); getInstrumentation().setDecommissionedTrackers(totalExcluded); }
public synchronized void updateFileNames(String includesFile, String excludesFile) { setIncludesFile(includesFile); setExcludesFile(excludesFile); } }
conf = new Configuration(); hostsReader.updateFileNames(conf.get("dfs.hosts", ""), conf.get("dfs.hosts.exclude", "")); Set<String> includes = hostsReader.getNewIncludes(); Set<String> excludes = hostsReader.getNewExcludes(); Set<String> prevIncludes = hostsReader.getHosts(); Set<String> prevExcludes = hostsReader.getExcludedHosts(); hostsReader.switchFiles(includes, excludes); try { replicator.hostsUpdated(); hostsReader.switchFiles(prevIncludes, prevExcludes); throw e;
private HostsFileReader createHostsFileReader(String includesFile, String excludesFile) throws IOException, YarnException { HostsFileReader hostsReader = new HostsFileReader(includesFile, (includesFile == null || includesFile.isEmpty()) ? null : this.rmContext.getConfigurationProvider() .getConfigurationInputStream(this.conf, includesFile), excludesFile, (excludesFile == null || excludesFile.isEmpty()) ? null : this.rmContext.getConfigurationProvider() .getConfigurationInputStream(this.conf, excludesFile)); return hostsReader; }
/** * Return if the specified tasktracker is in the hosts list, * if one was configured. If none was configured, then this * returns true. */ private boolean inHostsList(TaskTrackerStatus status) { Set<String> hostsList = hostsReader.getHosts(); return (hostsList.isEmpty() || hostsList.contains(status.getHost())); }
replthread.start(); this.hostsReader = new HostsFileReader(conf.get("dfs.hosts",""), conf.get("dfs.hosts.exclude","")); this.dnthread = new Daemon(new DecommissionManager(this).new Monitor( dnsToSwitchMapping.resolve(new ArrayList<String>(hostsReader.getHosts()));
try { updateFileNames(includeFiles, excludeFiles); Set<String> newIncludes = new HashSet<String>(); Set<String> newExcludes = new HashSet<String>(); boolean switchExcludes = false; if (includeFiles != null && !includeFiles.isEmpty()) { readFileToSet("included", includeFiles, newIncludes); switchIncludes = true; readFileToSet("excluded", excludeFiles, newExcludes); switchExcludes = true;
overreplthread.start(); this.hostsReader = new HostsFileReader(conf.get("dfs.hosts", ""), conf.get("dfs.hosts.exclude", "")); this.dnthread = new Daemon(new DecommissionManager(this).new Monitor( dnsToSwitchMapping.resolve(new ArrayList<String>(hostsReader.getHostNames()));