/** * Populate internal Hadoop Job (and Giraph IO Formats) with Hadoop-specific * configuration/setup metadata, propagating exceptions to calling code. * @param job the GiraphJob object to help populate Giraph IO Format data. * @param cmd the CommandLine for parsing Hadoop MR-specific args. */ private void prepareHadoopMRJob(final GiraphJob job, final CommandLine cmd) throws Exception { if (cmd.hasOption("vof") || cmd.hasOption("eof")) { if (cmd.hasOption("op")) { FileOutputFormat.setOutputPath(job.getInternalJob(), new Path(cmd.getOptionValue("op"))); } } if (cmd.hasOption("cf")) { DistributedCache.addCacheFile(new URI(cmd.getOptionValue("cf")), job.getConfiguration()); } }
@Override public int run(String[] args) throws Exception { if (args.length != 2) { throw new IllegalArgumentException( "run: Must have 2 arguments <output path> <# of workers>"); } GiraphJob job = new GiraphJob(getConf(), getClass().getName()); job.getConfiguration().setComputationClass(SimpleComputation.class); job.getConfiguration().setVertexInputFormatClass( SimpleSuperstepVertexInputFormat.class); job.getConfiguration().setWorkerContextClass(EmitterWorkerContext.class); job.getConfiguration().set( SimpleVertexWithWorkerContext.OUTPUTDIR, args[0]); job.getConfiguration().setWorkerConfiguration(Integer.parseInt(args[1]), Integer.parseInt(args[1]), 100.0f); if (job.run(true)) { return 0; } else { return -1; } }
bspJob.getConfiguration().setComputationClass( SimpleCheckpointComputation.class); bspJob.getConfiguration().setVertexInputFormatClass( GeneratedVertexInputFormat.class); bspJob.getConfiguration().setVertexOutputFormatClass( IdWithValueTextOutputFormat.class); bspJob.getConfiguration().setWorkerContextClass( SimpleCheckpointVertexWorkerContext.class); bspJob.getConfiguration().setMasterComputeClass( SimpleCheckpointVertexMasterCompute.class); int minWorkers = Integer.parseInt(cmd.getOptionValue('w')); int maxWorkers = Integer.parseInt(cmd.getOptionValue('w')); bspJob.getConfiguration().setWorkerConfiguration( minWorkers, maxWorkers, 100.0f);
/** * Run the ZooKeeper in-process and the job. * * @param zookeeperConfig Quorum peer configuration * @param giraphJob Giraph job to run * @return True if successful, false otherwise */ private static boolean runZooKeeperAndJob( final ZookeeperConfig zookeeperConfig, GiraphJob giraphJob) throws IOException { final InProcessZooKeeperRunner.ZooKeeperServerRunner zookeeper = new InProcessZooKeeperRunner.ZooKeeperServerRunner(); int port = zookeeper.start(zookeeperConfig); LOG.info("Started test zookeeper on port " + port); GiraphConstants.ZOOKEEPER_LIST.set(giraphJob.getConfiguration(), "localhost:" + port); try { return giraphJob.run(true); } catch (InterruptedException | ClassNotFoundException | IOException e) { LOG.error("runZooKeeperAndJob: Got exception on running", e); } finally { zookeeper.stop(); } return false; }
int workers = Integer.parseInt(BenchmarkOption.WORKERS.getOptionValue(cmd)); GiraphConfiguration giraphConf = job.getConfiguration(); giraphConf.addWorkerObserverClass(LogVersions.class); giraphConf.addMasterObserverClass(LogVersions.class);
/** * Run the ZooKeeper in-process and the job. * * @param zookeeperConfig Quorum peer configuration * @param giraphJob Giraph job to run * @return True if successful, false otherwise */ private static boolean runZooKeeperAndJob( final ZookeeperConfig zookeeperConfig, GiraphJob giraphJob) throws IOException { final InProcessZooKeeperRunner.ZooKeeperServerRunner zookeeper = new InProcessZooKeeperRunner.ZooKeeperServerRunner(); int port = zookeeper.start(zookeeperConfig); LOG.info("Started test zookeeper on port " + port); GiraphConstants.ZOOKEEPER_LIST.set(giraphJob.getConfiguration(), "localhost:" + port); try { return giraphJob.run(true); } catch (InterruptedException | ClassNotFoundException | IOException e) { LOG.error("runZooKeeperAndJob: Got exception on running", e); } finally { zookeeper.stop(); } return false; }