/** * Create a <code>GuaguaOptionsParser</code> to parse given options as well as generic Hadoop options. * * The resulting <code>CommandLine</code> object can be obtained by {@link #getCommandLine()}. * * @param conf * the configuration to modify * @param options * options built by the caller * @param args * User-specified arguments * @throws IOException */ public GuaguaOptionsParser(Configuration conf, Options options, String[] args) throws IOException { parseGeneralOptions(options, conf, args); this.conf = conf; }
private static void printUsage() { GuaguaOptionsParser.printGenericCommandUsage(System.out); System.out.println("For detailed invalid parameter, please check:"); }
/** * Parse the user-specified options, get the generic options, and modify configuration accordingly * * @param conf * Configuration to be modified * @param args * User-specified arguments * @return Command-specific arguments */ private String[] parseGeneralOptions(Options opts, Configuration conf, String[] args) throws IOException { opts = buildGeneralOptions(opts); CommandLineParser parser = new GnuParser(); try { commandLine = parser.parse(opts, args, true); processGeneralOptions(conf, commandLine); return commandLine.getArgs(); } catch (ParseException e) { LOG.warn("options parsing failed: {}", e.getMessage()); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("general options are: ", opts); } return args; }
private static GuaguaOptionsParser parseOpts(String[] args, Configuration conf) throws IOException, ClassNotFoundException { GuaguaOptionsParser parser = new GuaguaOptionsParser(conf, args); // we have use InputSplit while it is in mr jar but mr jar in newest yarn is not in container classpath String mrJar = HDPUtils.findContainingJar(InputSplit.class); conf.set(GuaguaYarnConstants.GUAGUA_YARN_APP_LIB_JAR, conf.get("tmpjars") + "," + mrJar); String jar = findContainingJar(Class.forName(conf.get(GuaguaConstants.MASTER_COMPUTABLE_CLASS, GuaguaYarnClient.class.getName()))); if(jar != null) { conf.set(GuaguaYarnConstants.GUAGUA_YARN_APP_JAR, jar); } // with a bug in hdp 2.2.4, we have to set hdp version String hdpVersion = HDPUtils.getHdpVersionForHDP224(); if(hdpVersion != null && hdpVersion.length() != 0) { conf.set("hdp.version", hdpVersion); } CommandLine cmdLine = parser.getCommandLine(); checkInputSetting(conf, cmdLine); checkZkServerSetting(conf, cmdLine); checkWorkerClassSetting(conf, cmdLine); checkMasterClassName(conf, cmdLine); checkIterationCountSetting(conf, cmdLine); checkResultClassSetting(conf, cmdLine); checkAppName(conf, cmdLine); return parser; }
conf.set("tmpjars", validateFiles(line.getOptionValue("libjars"), conf)); URL[] libjars = getLibJars(conf); if(libjars != null && libjars.length > 0) { conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader())); conf.set("tmpfiles", validateFiles(line.getOptionValue("files"), conf)); conf.set("tmparchives", validateFiles(line.getOptionValue("archives"), conf));
public static void main(String[] args) { if(args.length == 0 || (args.length == 1 && (args[0].equals("h") || args[0].equals("-h") || args[0].equals("-help") || args[0] .equals("help")))) { GuaguaOptionsParser.printGenericCommandUsage(System.out); System.exit(0); } long startTime = System.currentTimeMillis(); int result = 0; try { GuaguaYarnClient client = new GuaguaYarnClient(); client.setStartTime(startTime); GuaguaOptionsParser parser = parseOpts(args, client.getConf()); LOG.info("Initializing client."); String[] remainingArgs = parser.getRemainingArgs(); if(!client.init(remainingArgs)) { System.exit(-1); } result = client.run(remainingArgs); } catch (Throwable t) { LOG.error("Error running yarn client", t); System.exit(1); } if(result == 0) { LOG.info("Application completed successfully"); } else { LOG.error("Application failed, please check the diagnosis info."); } System.exit(result); }
/** * Parse the user-specified options, get the generic options, and modify configuration accordingly * * @param conf * Configuration to be modified * @param args * User-specified arguments * @return Command-specific arguments */ private String[] parseGeneralOptions(Options opts, Configuration conf, String[] args) throws IOException { opts = buildGeneralOptions(opts); CommandLineParser parser = new GnuParser(); try { commandLine = parser.parse(opts, args, true); processGeneralOptions(conf, commandLine); return commandLine.getArgs(); } catch (ParseException e) { LOG.warn("options parsing failed: {}", e.getMessage()); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("general options are: ", opts); } return args; }
private static GuaguaOptionsParser parseOpts(String[] args, Configuration conf) throws IOException, ClassNotFoundException { GuaguaOptionsParser parser = new GuaguaOptionsParser(conf, args); // we have use InputSplit while it is in mr jar but mr jar in newest yarn is not in container classpath String mrJar = HDPUtils.findContainingJar(InputSplit.class); conf.set(GuaguaYarnConstants.GUAGUA_YARN_APP_LIB_JAR, conf.get("tmpjars") + "," + mrJar); String jar = findContainingJar(Class.forName(conf.get(GuaguaConstants.MASTER_COMPUTABLE_CLASS, GuaguaYarnClient.class.getName()))); if(jar != null) { conf.set(GuaguaYarnConstants.GUAGUA_YARN_APP_JAR, jar); } // with a bug in hdp 2.2.4, we have to set hdp version String hdpVersion = HDPUtils.getHdpVersionForHDP224(); if(hdpVersion != null && hdpVersion.length() != 0) { conf.set("hdp.version", hdpVersion); } CommandLine cmdLine = parser.getCommandLine(); checkInputSetting(conf, cmdLine); checkZkServerSetting(conf, cmdLine); checkWorkerClassSetting(conf, cmdLine); checkMasterClassName(conf, cmdLine); checkIterationCountSetting(conf, cmdLine); checkResultClassSetting(conf, cmdLine); checkAppName(conf, cmdLine); return parser; }
conf.set("tmpjars", validateFiles(line.getOptionValue("libjars"), conf)); URL[] libjars = getLibJars(conf); if(libjars != null && libjars.length > 0) { conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader())); conf.set("tmpfiles", validateFiles(line.getOptionValue("files"), conf)); conf.set("tmparchives", validateFiles(line.getOptionValue("archives"), conf));
public static void main(String[] args) { if(args.length == 0 || (args.length == 1 && (args[0].equals("h") || args[0].equals("-h") || args[0].equals("-help") || args[0] .equals("help")))) { GuaguaOptionsParser.printGenericCommandUsage(System.out); System.exit(0); } long startTime = System.currentTimeMillis(); int result = 0; try { GuaguaYarnClient client = new GuaguaYarnClient(); client.setStartTime(startTime); GuaguaOptionsParser parser = parseOpts(args, client.getConf()); LOG.info("Initializing client."); String[] remainingArgs = parser.getRemainingArgs(); if(!client.init(remainingArgs)) { System.exit(-1); } result = client.run(remainingArgs); } catch (Throwable t) { LOG.error("Error running yarn client", t); System.exit(1); } if(result == 0) { LOG.info("Application completed successfully"); } else { LOG.error("Application failed, please check the diagnosis info."); } System.exit(result); }
/** * Parse the user-specified options, get the generic options, and modify * configuration accordingly * * @param conf * Configuration to be modified * @param args * User-specified arguments * @return Command-specific arguments */ private String[] parseGeneralOptions(Options opts, Configuration conf, String[] args) throws IOException { opts = buildGeneralOptions(opts); CommandLineParser parser = new GnuParser(); try { commandLine = parser.parse(opts, args, true); processGeneralOptions(conf, commandLine); return commandLine.getArgs(); } catch (ParseException e) { LOG.warn("options parsing failed: {}", e.getMessage()); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("general options are: ", opts); } return args; }
conf.setInt(GuaguaMapReduceConstants.MAPREDUCE_TASK_TIMEOUT, conf.getInt(GuaguaMapReduceConstants.MAPREDUCE_TASK_TIMEOUT, 1800000)); GuaguaOptionsParser parser = new GuaguaOptionsParser(conf, args); HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("yarn-site.xml"), conf); CommandLine cmdLine = parser.getCommandLine(); checkInputSetting(conf, cmdLine); checkZkServerSetting(conf, cmdLine);
conf.set("tmpjars", validateFiles(line.getOptionValue("libjars"), conf)); URL[] libjars = getLibJars(conf); if(libjars != null && libjars.length > 0) { conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader())); conf.set("tmpfiles", validateFiles(line.getOptionValue("files"), conf)); conf.set("tmparchives", validateFiles(line.getOptionValue("archives"), conf));
private static void printUsage() { GuaguaOptionsParser.printGenericCommandUsage(System.out); System.out.println("For detailed invalid parameter, please check:"); }
/** * Create a <code>GuaguaOptionsParser</code> to parse given options as well as generic Hadoop options. * * The resulting <code>CommandLine</code> object can be obtained by {@link #getCommandLine()}. * * @param conf * the configuration to modify * @param options * options built by the caller * @param args * User-specified arguments * @throws IOException */ public GuaguaOptionsParser(Configuration conf, Options options, String[] args) throws IOException { parseGeneralOptions(options, conf, args); this.conf = conf; }
/** * Parse the user-specified options, get the generic options, and modify * configuration accordingly * * @param conf * Configuration to be modified * @param args * User-specified arguments * @return Command-specific arguments */ private String[] parseGeneralOptions(Options opts, Configuration conf, String[] args) throws IOException { opts = buildGeneralOptions(opts); CommandLineParser parser = new GnuParser(); try { commandLine = parser.parse(opts, args, true); processGeneralOptions(conf, commandLine); return commandLine.getArgs(); } catch (ParseException e) { LOG.warn("options parsing failed: {}", e.getMessage()); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("general options are: ", opts); } return args; }
conf.setInt(GuaguaMapReduceConstants.MAPREDUCE_TASK_TIMEOUT, conf.getInt(GuaguaMapReduceConstants.MAPREDUCE_TASK_TIMEOUT, 1800000)); GuaguaOptionsParser parser = new GuaguaOptionsParser(conf, args); HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("yarn-site.xml"), conf); CommandLine cmdLine = parser.getCommandLine(); checkInputSetting(conf, cmdLine); checkZkServerSetting(conf, cmdLine);
conf.set("tmpjars", validateFiles(line.getOptionValue("libjars"), conf)); URL[] libjars = getLibJars(conf); if(libjars != null && libjars.length > 0) { conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader())); conf.set("tmpfiles", validateFiles(line.getOptionValue("files"), conf)); conf.set("tmparchives", validateFiles(line.getOptionValue("archives"), conf));
private static void printUsage() { GuaguaOptionsParser.printGenericCommandUsage(System.out); System.out.println("For detailed invalid parameter, please check:"); }
/** * Create a <code>GuaguaOptionsParser</code> to parse given options as well as generic Hadoop options. * * The resulting <code>CommandLine</code> object can be obtained by {@link #getCommandLine()}. * * @param conf * the configuration to modify * @param options * options built by the caller * @param args * User-specified arguments * @throws IOException */ public GuaguaOptionsParser(Configuration conf, Options options, String[] args) throws IOException { parseGeneralOptions(options, conf, args); this.conf = conf; }