@Override public int run(String[] args) throws Exception { if (args.length != 2) { throw new IllegalArgumentException( "run: Must have 2 arguments <output path> <# of workers>"); } GiraphJob job = new GiraphJob(getConf(), getClass().getName()); job.getConfiguration().setComputationClass(SimpleComputation.class); job.getConfiguration().setVertexInputFormatClass( SimpleSuperstepVertexInputFormat.class); job.getConfiguration().setWorkerContextClass(EmitterWorkerContext.class); job.getConfiguration().set( SimpleVertexWithWorkerContext.OUTPUTDIR, args[0]); job.getConfiguration().setWorkerConfiguration(Integer.parseInt(args[1]), Integer.parseInt(args[1]), 100.0f); if (job.run(true)) { return 0; } else { return -1; } }
/** * Populate internal Hadoop Job (and Giraph IO Formats) with Hadoop-specific * configuration/setup metadata, propagating exceptions to calling code. * @param job the GiraphJob object to help populate Giraph IO Format data. * @param cmd the CommandLine for parsing Hadoop MR-specific args. */ private void prepareHadoopMRJob(final GiraphJob job, final CommandLine cmd) throws Exception { if (cmd.hasOption("vof") || cmd.hasOption("eof")) { if (cmd.hasOption("op")) { FileOutputFormat.setOutputPath(job.getInternalJob(), new Path(cmd.getOptionValue("op"))); } } if (cmd.hasOption("cf")) { DistributedCache.addCacheFile(new URI(cmd.getOptionValue("cf")), job.getConfiguration()); } }
GiraphJob bspJob = new GiraphJob(getConf(), getClass().getName()); bspJob.getConfiguration().setComputationClass( SimpleCheckpointComputation.class); bspJob.getConfiguration().setVertexInputFormatClass( GeneratedVertexInputFormat.class); bspJob.getConfiguration().setVertexOutputFormatClass( IdWithValueTextOutputFormat.class); bspJob.getConfiguration().setWorkerContextClass( SimpleCheckpointVertexWorkerContext.class); bspJob.getConfiguration().setMasterComputeClass( SimpleCheckpointVertexMasterCompute.class); int minWorkers = Integer.parseInt(cmd.getOptionValue('w')); int maxWorkers = Integer.parseInt(cmd.getOptionValue('w')); bspJob.getConfiguration().setWorkerConfiguration( minWorkers, maxWorkers, 100.0f); FileOutputFormat.setOutputPath(bspJob.getInternalJob(), new Path(cmd.getOptionValue('o'))); boolean verbose = false; Integer.parseInt(cmd.getOptionValue('s'))); if (bspJob.run(verbose)) { return 0; } else {
@Override /** * Drives a job run configured for "Giraph on Hadoop MR cluster" * @param args the command line arguments * @return job run exit code */ public int run(String[] args) throws Exception { if (null == getConf()) { // for YARN profile conf = new Configuration(); } GiraphConfiguration giraphConf = new GiraphConfiguration(getConf()); CommandLine cmd = ConfigurationUtils.parseArgs(giraphConf, args); if (null == cmd) { return 0; // user requested help/info printout, don't run a job. } // set up job for various platforms final String vertexClassName = args[0]; final String jobName = "Giraph: " + vertexClassName; GiraphJob job = new GiraphJob(giraphConf, jobName); prepareHadoopMRJob(job, cmd); // run the job, collect results if (LOG.isDebugEnabled()) { LOG.debug("Attempting to run Vertex: " + vertexClassName); } boolean verbose = !cmd.hasOption('q'); return job.run(verbose) ? 0 : -1; }
/** * Run the ZooKeeper in-process and the job. * * @param zookeeperConfig Quorum peer configuration * @param giraphJob Giraph job to run * @return True if successful, false otherwise */ private static boolean runZooKeeperAndJob( final ZookeeperConfig zookeeperConfig, GiraphJob giraphJob) throws IOException { final InProcessZooKeeperRunner.ZooKeeperServerRunner zookeeper = new InProcessZooKeeperRunner.ZooKeeperServerRunner(); int port = zookeeper.start(zookeeperConfig); LOG.info("Started test zookeeper on port " + port); GiraphConstants.ZOOKEEPER_LIST.set(giraphJob.getConfiguration(), "localhost:" + port); try { return giraphJob.run(true); } catch (InterruptedException | ClassNotFoundException | IOException e) { LOG.error("runZooKeeperAndJob: Got exception on running", e); } finally { zookeeper.stop(); } return false; }
GiraphJob job = new GiraphJob(conf, conf.getComputationName()); Job internalJob = job.getInternalJob();
protected GiraphJob getJob(GiraphConfiguration conf, String jobName) throws IOException { return new GiraphJob(conf, jobName); }
@Override public int run(String[] args) throws Exception { if (null == getConf()) { conf = new Configuration(); } GiraphConfiguration giraphConf = new GiraphConfiguration(getConf()); YamlConfiguration yamlConfig = new YamlConfiguration(args); yamlConfig.load(); yamlConfig.populateGiraphConfiguration(giraphConf); // set up job for various platforms final String arabesqueComputationName = giraphConf.get(io.arabesque.conf.Configuration.CONF_COMPUTATION_CLASS); final String jobName = "Arabesque: " + arabesqueComputationName; // run the job, collect results if (LOG.isDebugEnabled()) { LOG.debug("Attempting to run computation: " + arabesqueComputationName); } GiraphJob job = getJob(giraphConf, jobName); boolean verbose = yamlConfig.getBoolean("verbose"); return job.run(verbose) ? 0 : -1; }
throws IOException, InterruptedException, ClassNotFoundException { setIntConfIfDefault("mapreduce.job.counters.limit", 512); setIntConfIfDefault("mapred.job.map.memory.mb", 1024); setIntConfIfDefault("mapred.job.reduce.memory.mb", 0); checkLocalJobRunnerConfiguration(conf);
/** * Run the ZooKeeper in-process and the job. * * @param zookeeperConfig Quorum peer configuration * @param giraphJob Giraph job to run * @return True if successful, false otherwise */ private static boolean runZooKeeperAndJob( final ZookeeperConfig zookeeperConfig, GiraphJob giraphJob) throws IOException { final InProcessZooKeeperRunner.ZooKeeperServerRunner zookeeper = new InProcessZooKeeperRunner.ZooKeeperServerRunner(); int port = zookeeper.start(zookeeperConfig); LOG.info("Started test zookeeper on port " + port); GiraphConstants.ZOOKEEPER_LIST.set(giraphJob.getConfiguration(), "localhost:" + port); try { return giraphJob.run(true); } catch (InterruptedException | ClassNotFoundException | IOException e) { LOG.error("runZooKeeperAndJob: Got exception on running", e); } finally { zookeeper.stop(); } return false; }
GiraphJob job = new GiraphJob(conf, conf.getComputationName()); Job internalJob = job.getInternalJob(); if (conf.hasVertexInputFormat()) { GiraphFileInputFormat.setVertexInputPath(internalJob.getConfiguration(), new Path(edgeInputFile.toString())); FileOutputFormat.setOutputPath(job.getInternalJob(), new Path(outputDir.toString()));
GiraphJob job = new GiraphJob(conf, conf.getComputationName());
GiraphJob job = new GiraphJob(getConf(), getClass().getName()); int workers = Integer.parseInt(BenchmarkOption.WORKERS.getOptionValue(cmd)); GiraphConfiguration giraphConf = job.getConfiguration(); giraphConf.addWorkerObserverClass(LogVersions.class); giraphConf.addMasterObserverClass(LogVersions.class); isVerbose = true; if (job.run(isVerbose)) { return 0; } else {