/** * Helper to deal with computation class. * * @param conf Configuration * @param cmd CommandLine * @param computationClassName Name of computation * @throws ClassNotFoundException error finding class */ private static void handleComputationClass(GiraphConfiguration conf, CommandLine cmd, String computationClassName) throws ClassNotFoundException { if (computationClassName.endsWith("py")) { handleJythonComputation(conf, cmd, computationClassName); } else { conf.setComputationClass( (Class<? extends Computation>) Class.forName(computationClassName)); } }
@Test public void testNumberOfVertices() throws Exception { GiraphConfiguration conf = new GiraphConfiguration(); conf.setComputationClass(GiraphHelloWorld.class); conf.setVertexInputFormatClass(IntIntNullTextInputFormat.class); conf.setVertexOutputFormatClass(AdjacencyListTextVertexOutputFormat.class); Iterable<String> results = InternalVertexRunner.run(conf, graphSeed); } }
bspJob.getConfiguration().setComputationClass( SimpleCheckpointComputation.class); bspJob.getConfiguration().setVertexInputFormatClass(
@Override public int run(String[] args) throws Exception { if (args.length != 2) { throw new IllegalArgumentException( "run: Must have 2 arguments <output path> <# of workers>"); } GiraphJob job = new GiraphJob(getConf(), getClass().getName()); job.getConfiguration().setComputationClass(SimpleComputation.class); job.getConfiguration().setVertexInputFormatClass( SimpleSuperstepVertexInputFormat.class); job.getConfiguration().setWorkerContextClass(EmitterWorkerContext.class); job.getConfiguration().set( SimpleVertexWithWorkerContext.OUTPUTDIR, args[0]); job.getConfiguration().setWorkerConfiguration(Integer.parseInt(args[1]), Integer.parseInt(args[1]), 100.0f); if (job.run(true)) { return 0; } else { return -1; } }
conf.setComputationClass(BlockComputation.class); conf.setWorkerContextClass(BlockWorkerContext.class);
private void addPrivateConfiguration(GiraphConfiguration configuration) { // Set currently fixed classes. TODO: Allow configuration at some point perhaps configuration.setComputationClass(io.arabesque.computation.ExecutionEngine.class); configuration.setMasterComputeClass(io.arabesque.computation.MasterExecutionEngine.class); configuration.setVertexInputFormatClass(ArabesqueInputFormat.class); // Calculate partition count based on # of workers and # of threads if no count was provided if (GiraphConstants.USER_PARTITION_COUNT.get(configuration) == -1) { int numWorkers = configuration.getMinWorkers(); int numComputeThreads = configuration.getNumComputeThreads(); GiraphConstants.USER_PARTITION_COUNT.set(configuration, numWorkers * numComputeThreads); } // Set our default worker context class instead of Giraph's default if (GiraphConstants.WORKER_CONTEXT_CLASS.get(configuration).equals(DefaultWorkerContext.class)) { configuration.setWorkerContextClass(WorkerContext.class); } }
@Override protected void prepareConfiguration(GiraphConfiguration conf, CommandLine cmd) { conf.setComputationClass(ReducersBenchmarkComputation.class); conf.setMasterComputeClass(ReducersBenchmarkMasterCompute.class); conf.setVertexInputFormatClass(PseudoRandomVertexInputFormat.class); conf.setWorkerContextClass(ReducersBenchmarkWorkerContext.class); conf.setLong(PseudoRandomInputFormatConstants.AGGREGATE_VERTICES, BenchmarkOption.VERTICES.getOptionLongValue(cmd)); conf.setLong(PseudoRandomInputFormatConstants.EDGES_PER_VERTEX, 1); conf.setInt(REDUCERS_NUM, REDUCERS.getOptionIntValue(cmd)); conf.setInt("workers", conf.getInt(GiraphConstants.MAX_WORKERS, -1)); }
@Override protected void prepareConfiguration(GiraphConfiguration conf, CommandLine cmd) { conf.setComputationClass(ShortestPathsComputation.class); if (EDGES_CLASS.getOptionIntValue(cmd, 1) == 1) { conf.setOutEdgesClass(ArrayListEdges.class); } else { conf.setOutEdgesClass(HashMapEdges.class); } LOG.info("Using class " + GiraphConstants.COMPUTATION_CLASS.get(conf)); conf.setVertexInputFormatClass(PseudoRandomVertexInputFormat.class); if (!NO_COMBINER.optionTurnedOn(cmd)) { conf.setMessageCombinerClass(MinimumDoubleMessageCombiner.class); } conf.setLong(PseudoRandomInputFormatConstants.AGGREGATE_VERTICES, BenchmarkOption.VERTICES.getOptionLongValue(cmd)); conf.setLong(PseudoRandomInputFormatConstants.EDGES_PER_VERTEX, BenchmarkOption.EDGES_PER_VERTEX.getOptionLongValue(cmd)); }
@Override protected void prepareConfiguration(GiraphConfiguration conf, CommandLine cmd) { conf.setComputationClass(AggregatorsBenchmarkComputation.class); conf.setMasterComputeClass(AggregatorsBenchmarkMasterCompute.class); conf.setVertexInputFormatClass(PseudoRandomVertexInputFormat.class); conf.setWorkerContextClass(AggregatorsBenchmarkWorkerContext.class); conf.setLong(PseudoRandomInputFormatConstants.AGGREGATE_VERTICES, BenchmarkOption.VERTICES.getOptionLongValue(cmd)); conf.setLong(PseudoRandomInputFormatConstants.EDGES_PER_VERTEX, 1); conf.setInt(AGGREGATORS_NUM, AGGREGATORS.getOptionIntValue(cmd)); conf.setInt("workers", conf.getInt(GiraphConstants.MAX_WORKERS, -1)); MasterLoggingAggregator.setUseMasterLoggingAggregator(true, conf); }
JythonUtils.init(conf, "PageRank"); } else { conf.setComputationClass(PageRankComputation.class);
configuration.setComputationClass(WeightedPageRankComputation.class); int edgesClassOption = EDGES_CLASS.getOptionIntValue(cmd, 1); switch (edgesClassOption) {
@Override protected void prepareConfiguration(GiraphConfiguration conf, CommandLine cmd) { conf.setComputationClass(RandomMessageComputation.class); conf.setVertexInputFormatClass(PseudoRandomVertexInputFormat.class); conf.setWorkerContextClass(RandomMessageBenchmarkWorkerContext.class); conf.setMasterComputeClass(RandomMessageBenchmarkMasterCompute.class); conf.setLong(PseudoRandomInputFormatConstants.AGGREGATE_VERTICES, BenchmarkOption.VERTICES.getOptionLongValue(cmd)); conf.setLong(PseudoRandomInputFormatConstants.EDGES_PER_VERTEX, BenchmarkOption.EDGES_PER_VERTEX.getOptionLongValue(cmd)); conf.setInt(SUPERSTEP_COUNT, BenchmarkOption.SUPERSTEPS.getOptionIntValue(cmd)); conf.setInt(RandomMessageBenchmark.NUM_BYTES_PER_MESSAGE, BYTES_PER_MESSAGE.getOptionIntValue(cmd)); conf.setInt(RandomMessageBenchmark.NUM_MESSAGES_PER_EDGE, MESSAGES_PER_EDGE.getOptionIntValue(cmd)); if (FLUSH_THREADS.optionTurnedOn(cmd)) { conf.setInt(GiraphConstants.MSG_NUM_FLUSH_THREADS, FLUSH_THREADS.getOptionIntValue(cmd)); } }