/** * Utilize an existing ZooKeeper service. If this is not set, ZooKeeper * will be dynamically started by Giraph for this job. * * @param serverList Comma separated list of servers and ports * (i.e. zk1:2221,zk2:2221) */ public final void setZooKeeperConfiguration(String serverList) { ZOOKEEPER_LIST.set(this, serverList); }
/** * Populate jar list for Pure YARN jobs. * * @param jarList a comma-separated list of jar names */ public void setYarnLibJars(String jarList) { GIRAPH_YARN_LIBJARS.set(this, jarList); }
/** * Set the vertex output format path * * @param path path where the verteces will be written */ public final void setVertexOutputFormatSubdir(String path) { VERTEX_OUTPUT_FORMAT_SUBDIR.set(this, path); }
/** * Set the edge output format path * * @param path path where the edges will be written */ public final void setEdgeOutputFormatSubdir(String path) { EDGE_OUTPUT_FORMAT_SUBDIR.set(this, path); }
/** * Set the ZooKeeper list to the provided list. This method is used when the * ZooKeeper is started internally and will set the zkIsExternal option to * false as well. * * @param zkList list of strings, comma separated of zookeeper servers */ public void setZookeeperList(String zkList) { ZOOKEEPER_LIST.set(this, zkList); ZOOKEEPER_IS_EXTERNAL.set(this, false); }
/** * Sets up the Configuration for using Jython * * @param conf Configuration to se * @param klassName Class name of Jython Computation */ public static void init(Configuration conf, String klassName) { COMPUTATION_LANGUAGE.set(conf, Language.JYTHON); COMPUTATION_FACTORY_CLASS.set(conf, JythonComputationFactory.class); JythonOptions.JYTHON_COMPUTATION_CLASS_NAME.set(conf, klassName); }
public static <OD> void addOutputDesc(OD outputDesc, String confOption, GiraphConfiguration conf) { GiraphConstants.HADOOP_OUTPUT_FORMAT_CLASS.set(conf, BlockOutputFormat.class); String currentOutputs = OUTPUT_CONF_OPTIONS.get(conf); if (!currentOutputs.isEmpty()) { currentOutputs = currentOutputs + ","; } OUTPUT_CONF_OPTIONS.set(conf, currentOutputs + confOption); ConfigurationObjectUtils.setObjectKryo(outputDesc, confOption, conf); }
/** * Use this factory in the {@link org.apache.hadoop.conf.Configuration} * * @param conf {@link org.apache.hadoop.conf.Configuration} * @param jythonClassName Name of Jython class implementing value type */ public void useThisFactory(Configuration conf, String jythonClassName) { if (LOG.isInfoEnabled()) { LOG.info("useThisFactory: Setting up Jython factory for " + getGraphType() + " reading " + " using Jython type " + jythonClassName); } ClassConfOption factoryOption = getGraphType().factoryClassOption(); factoryOption.set(conf, getClass()); jythonClassNameOption().set(conf, jythonClassName); } }
/** * Run the ZooKeeper in-process and the job. * * @param zookeeperConfig Quorum peer configuration * @param giraphJob Giraph job to run * @return True if successful, false otherwise */ private static boolean runZooKeeperAndJob( final ZookeeperConfig zookeeperConfig, GiraphJob giraphJob) throws IOException { final InProcessZooKeeperRunner.ZooKeeperServerRunner zookeeper = new InProcessZooKeeperRunner.ZooKeeperServerRunner(); int port = zookeeper.start(zookeeperConfig); LOG.info("Started test zookeeper on port " + port); GiraphConstants.ZOOKEEPER_LIST.set(giraphJob.getConfiguration(), "localhost:" + port); try { return giraphJob.run(true); } catch (InterruptedException | ClassNotFoundException | IOException e) { LOG.error("runZooKeeperAndJob: Got exception on running", e); } finally { zookeeper.stop(); } return false; }
/** * Run the ZooKeeper in-process and the job. * * @param zookeeperConfig Quorum peer configuration * @param giraphJob Giraph job to run * @return True if successful, false otherwise */ private static boolean runZooKeeperAndJob( final ZookeeperConfig zookeeperConfig, GiraphJob giraphJob) throws IOException { final InProcessZooKeeperRunner.ZooKeeperServerRunner zookeeper = new InProcessZooKeeperRunner.ZooKeeperServerRunner(); int port = zookeeper.start(zookeeperConfig); LOG.info("Started test zookeeper on port " + port); GiraphConstants.ZOOKEEPER_LIST.set(giraphJob.getConfiguration(), "localhost:" + port); try { return giraphJob.run(true); } catch (InterruptedException | ClassNotFoundException | IOException e) { LOG.error("runZooKeeperAndJob: Got exception on running", e); } finally { zookeeper.stop(); } return false; }
/** * Create and start the Thrift server. * * @param conf Giraph conf to set the host and ports for. * @param services Services to start */ public ClientThriftServer(GiraphConfiguration conf, List<?> services) { checkNotNull(conf, "conf is null"); checkNotNull(services, "services is null"); ThriftServiceProcessor processor = new ThriftServiceProcessor(new ThriftCodecManager(), new ArrayList<ThriftEventHandler>(), services); clientThriftServer = new ThriftServer(processor, new ThriftServerConfig()); clientThriftServer.start(); try { CLIENT_THRIFT_SERVER_HOST.set( conf, conf.getLocalHostname()); } catch (UnknownHostException e) { throw new IllegalStateException("Unable to get host information", e); } CLIENT_THRIFT_SERVER_PORT.set(conf, clientThriftServer.getPort()); }
String restartFrom = retryChecker.shouldRestartCheckpoint(submittedJob); if (restartFrom != null) { GiraphConstants.RESTART_JOB_ID.set(conf, restartFrom); continue;
List<String> javaOpts = getMemoryJavaOpts(conf); javaOpts.addAll(getGcJavaOpts(conf)); MAPRED_JAVA_JOB_OPTIONS.set(conf, StringUtils.join(javaOpts, " "));