/** * Populate jar list for Pure YARN jobs. * * @param jarList a comma-separated list of jar names */ public void setYarnLibJars(String jarList) { GIRAPH_YARN_LIBJARS.set(this, jarList); }
/** * Jars required in "Pure YARN" jobs (names only, no paths) should * be listed here in full, including Giraph framework jar(s). * * @return the comma-separated list of jar names for export to cluster. */ public String getYarnLibJars() { return GIRAPH_YARN_LIBJARS.get(this); }
/** * Get list of values for key * * @param conf Configuration * @return list of values for key */ public List<String> getList(Configuration conf) { return Lists.newArrayList(getArray(conf)); }
/** * Path to the checkpoint's root (including job id) * @param conf Immutable configuration of the job * @param jobId job ID * @return checkpoint's root */ public static String getCheckpointBasePath(Configuration conf, String jobId) { return CHECKPOINT_DIRECTORY.getWithDefault(conf, CHECKPOINT_DIRECTORY.getDefaultValue() + "/" + jobId); }
public static <OD> void addOutputDesc(OD outputDesc, String confOption, GiraphConfiguration conf) { GiraphConstants.HADOOP_OUTPUT_FORMAT_CLASS.set(conf, BlockOutputFormat.class); String currentOutputs = OUTPUT_CONF_OPTIONS.get(conf); if (!currentOutputs.isEmpty()) { currentOutputs = currentOutputs + ","; } OUTPUT_CONF_OPTIONS.set(conf, currentOutputs + confOption); ConfigurationObjectUtils.setObjectKryo(outputDesc, confOption, conf); }
/** * Lookup value * * @param conf Configuration * @return value for key, or defaultValue */ public String get(Configuration conf) { return conf.get(getKey(), defaultValue); }
@Override public void checkConfiguration(ImmutableClassesGiraphConfiguration conf) { if (SCRIPTS_TO_LOAD.isDefaultValue(conf)) { throw new IllegalStateException("checkConfiguration: " + SCRIPTS_TO_LOAD.getKey() + " not set in configuration"); } if (!JythonOptions.JYTHON_COMPUTATION_CLASS_NAME.contains(conf)) { throw new IllegalStateException("checkConfiguration: " + JythonOptions.JYTHON_COMPUTATION_CLASS_NAME.getKey() + " not set in configuration"); } }
/** * Generate the final ZooKeeper coordination directory on HDFS * * @return directory path with job id */ private String getFinalZooKeeperPath() { return ZOOKEEPER_MANAGER_DIRECTORY.getDefaultValue() + "/" + jobId; }
if (GiraphConstants.METRICS_DIRECTORY.isDefaultValue(conf)) { aggregatedMetrics.print(superstep, System.err); } else {
/** * Constructor * * @param graphType GraphType */ public JythonGraphTypeOptions(GraphType graphType) { this.graphType = graphType; jythonClassNameOption = new StrConfOption("giraph.jython." + graphType.dotString() + ".class.name", null, "Name of class in Jython implementing " + graphType.spaceString()); needsWrapperOption = new BooleanConfOption("giraph.jython." + graphType.dotString() + ".needs.wrapper", false, "Whether the " + graphType.spaceString() + " jython type needs a wrapper"); languageOption = EnumConfOption.create("giraph." + graphType.dotString() + ".language", Language.class, Language.JAVA, "Language " + graphType.spaceString() + " is implemented in"); }
jobId = conf.getJobId(); baseDirectory = new Path(ZOOKEEPER_MANAGER_DIRECTORY.getWithDefault(conf, getFinalZooKeeperPath())); taskDirectory = new Path(baseDirectory, } else { zkDirDefault = System.getProperty("user.dir") + "/" + ZOOKEEPER_MANAGER_DIRECTORY.getDefaultValue();
/** * Get array of values for key * * @param conf Configuration * @return array of values for key */ public String[] getArray(Configuration conf) { return conf.getStrings(getKey(), defaultValue); }
/** * Set the vertex output format path * * @param path path where the verteces will be written */ public final void setVertexOutputFormatSubdir(String path) { VERTEX_OUTPUT_FORMAT_SUBDIR.set(this, path); }
@Override public String computationName(GiraphConfiguration conf) { return JythonOptions.JYTHON_COMPUTATION_CLASS_NAME.get(conf); } }
/** * Lookup value with user defined defaultValue * * @param conf Configuration * @param defaultVal default value to use * @return value for key, or defaultVal passed in */ public String getWithDefault(Configuration conf, String defaultVal) { return conf.get(getKey(), defaultVal); }
ImmutableClassesGiraphConfiguration<?, ?, ?> conf) { String[] userPaths = GiraphConstants.PARTITIONS_DIRECTORY.getArray(conf); this.numDisks = userPaths.length; if (!GiraphConstants.NUM_OUT_OF_CORE_THREADS.isDefaultValue(conf) ||
/** * Set the edge output format path * * @param path path where the edges will be written */ public final void setEdgeOutputFormatSubdir(String path) { EDGE_OUTPUT_FORMAT_SUBDIR.set(this, path); }
/** * Get the ZooKeeper list. * * @return ZooKeeper list of strings, comma separated or null if none set. */ public String getZookeeperList() { return ZOOKEEPER_LIST.get(this); }
/** * Set value for key * * @param conf Configuration * @param value to set */ public void set(Configuration conf, String value) { conf.set(getKey(), value); }
/** * Utilize an existing ZooKeeper service. If this is not set, ZooKeeper * will be dynamically started by Giraph for this job. * * @param serverList Comma separated list of servers and ports * (i.e. zk1:2221,zk2:2221) */ public final void setZooKeeperConfiguration(String serverList) { ZOOKEEPER_LIST.set(this, serverList); }