public Vertex<I, V, E> createVertex() { return getConf().createVertex(); }
@Override public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException { ImmutableClassesGiraphConfiguration conf = new ImmutableClassesGiraphConfiguration(context.getConfiguration()); if (!conf.hasVertexOutputFormat() && !conf.hasEdgeOutputFormat()) { LOG.warn("checkOutputSpecs: ImmutableOutputCommiter" + " will not check anything"); return; } if (conf.hasVertexOutputFormat()) { conf.createWrappedVertexOutputFormat().checkOutputSpecs(context); } if (conf.hasEdgeOutputFormat()) { conf.createWrappedEdgeOutputFormat().checkOutputSpecs(context); } }
/** * Create a reusable edge. * * @return Instantiated reusable edge. */ public ReusableEdge<I, E> createReusableEdge() { if (isEdgeValueNullWritable()) { return (ReusableEdge<I, E>) EdgeFactory.createReusable(createVertexId()); } else { return EdgeFactory.createReusable(createVertexId(), createEdgeValue()); } }
/** * Constructor requiring classes * * @param conf Should have vertex and edge classes set. */ public TestGraph(GiraphConfiguration conf) { this.conf = new ImmutableClassesGiraphConfiguration(conf); vertexValueCombiner = this.conf.createVertexValueCombiner(); }
/** * Reset the empty Vertex to an initial state. */ private void resetEmptyVertex() { vertex = configuration.createVertex(); I id = configuration.createVertexId(); V value = configuration.createVertexValue(); OutEdges<I, E> edges = configuration.createOutEdges(); vertex.initialize(id, value, edges); }
@Override public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException { beta = getConf().getFloat( BETA, 0.0f); aggregateVertices = getConf().getLong( AGGREGATE_VERTICES, 0); BspInputSplit bspInputSplit = (BspInputSplit) inputSplit; long extraVertices = aggregateVertices % bspInputSplit.getNumSplits(); totalSplitVertices = aggregateVertices / bspInputSplit.getNumSplits(); if (bspInputSplit.getSplitIndex() < extraVertices) { ++totalSplitVertices; } startingVertexId = bspInputSplit.getSplitIndex() * (aggregateVertices / bspInputSplit.getNumSplits()) + Math.min(bspInputSplit.getSplitIndex(), extraVertices); edgesPerVertex = getConf().getInt( EDGES_PER_VERTEX, 0); if (getConf().reuseEdgeObjects()) { reusableEdge = getConf().createReusableEdge(); } int seed = getConf().getInt(SEED, -1); if (seed != -1) { rnd = new Random(seed); } else { rnd = new Random(); } }
@Override public void initialize(int partitionId, Progressable progressable) { super.initialize(partitionId, progressable); vertexMap = new MapMaker().concurrencyLevel( getConf().getNettyServerExecutionConcurrency()).makeMap(); representativeVertex = getConf().createVertex(); representativeVertex.initialize( getConf().createVertexId(), getConf().createVertexValue(), getConf().createOutEdges()); representativeCombinerVertex = getConf().createVertex(); representativeCombinerVertex.initialize( getConf().createVertexId(), getConf().createVertexValue(), getConf().createOutEdges()); useUnsafeSerialization = getConf().useUnsafeSerialization(); }
getCurrentVertex() throws IOException, InterruptedException { Vertex<LongWritable, DoubleWritable, DoubleWritable> vertex = getConf().createVertex(); long vertexId = startingVertexId + verticesRead; OutEdges<LongWritable, DoubleWritable> edges = getConf().createOutEdges(); edges.initialize(edgesPerVertex); destVertices.clear(); if (destVertexId != vertexId) { Edge<LongWritable, DoubleWritable> edge = (reusableEdge == null) ? getConf().createEdge() : reusableEdge; edge.getTargetVertexId().set( rnd.nextFloat() < beta ? getRandomDestination() : destVertexId);
/** * Add an edge to an existing vertex * * @param vertexId Edge origin * @param edgePair The edge * @return this */ public TestGraph<I, V, E> addEdge(I vertexId, Entry<I, E> edgePair) { if (!vertices.containsKey(vertexId)) { Vertex<I, V, E> v = conf.createVertex(); v.initialize(vertexId, conf.createVertexValue()); vertices.put(vertexId, v); } vertices.get(vertexId) .addEdge(EdgeFactory.create(edgePair.getKey(), edgePair.getValue())); return this; }
@Override public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException, InterruptedException { aggregateVertices = getConf().getLong( PseudoRandomInputFormatConstants.AGGREGATE_VERTICES, 0); if (aggregateVertices <= 0) { " instead of " + BspInputSplit.class); edgesPerVertex = getConf().getInt( PseudoRandomInputFormatConstants.EDGES_PER_VERTEX, 0); if (edgesPerVertex <= 0) { PseudoRandomInputFormatConstants.EDGES_PER_VERTEX + " <= 0"); float minLocalEdgesRatio = getConf().getFloat( PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO, PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO_DEFAULT);
this.graphTaskManager = graphTaskManager; this.conf = graphTaskManager.getConf(); this.jobId = conf.getJobId(); this.taskPartition = conf.getTaskPartition(); this.restartedSuperstep = conf.getLong( GiraphConstants.RESTART_SUPERSTEP, UNSET_SUPERSTEP); try { this.hostname = conf.getLocalHostname(); } catch (UnknownHostException e) { throw new RuntimeException(e); this.graphPartitionerFactory = conf.createGraphPartitioner(); conf.getZooKeeperSessionTimeout(), conf.getZookeeperOpsMaxAttempts(), conf.getZookeeperOpsRetryWaitMsecs(), this, context);
/** * Creates a new Vertex object, without adding it into the graph. * * This function is safe to call from multiple threads at the same time, * and then synchronize only on actual addition of Vertex to the graph * itself. */ public Vertex<I, V, E> makeVertex( Number vertexId, V vertexValue, Supplier<E> edgeSupplier, Number... edges) { Vertex<I, V, E> vertex = getConf().createVertex(); List<Edge<I, E>> edgesList = new ArrayList<>(); for (Number edge: edges) { edgesList.add( EdgeFactory.create(numberToVertexId.apply(edge), edgeSupplier != null ? edgeSupplier.get() : getConf().createEdgeValue())); } vertex.initialize( numberToVertexId.apply(vertexId), vertexValue != null ? vertexValue : getConf().createVertexValue(), edgesList); return vertex; }
public Integer getInteger(String key, Integer defaultValue) { return giraphConfiguration.getInt(key, defaultValue); }
/** * Check if we should use thread local aggregators. * * @param conf Giraph configuration * @return True iff we should use thread local aggregators */ public static boolean useThreadLocalAggregators(ImmutableClassesGiraphConfiguration conf) { return conf.getBoolean(USE_THREAD_LOCAL_AGGREGATORS, USE_THREAD_LOCAL_AGGREGATORS_DEFAULT); }
@Override public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException { int aggregateVertices = getConf().getInt( PseudoRandomInputFormatConstants.AGGREGATE_VERTICES, 0); BspInputSplit bspInputSplit = (BspInputSplit) inputSplit; int extraVertices = aggregateVertices % bspInputSplit.getNumSplits(); totalSplitVertices = aggregateVertices / bspInputSplit.getNumSplits(); if (bspInputSplit.getSplitIndex() < extraVertices) { ++totalSplitVertices; } startingVertexId = bspInputSplit.getSplitIndex() * (aggregateVertices / bspInputSplit.getNumSplits()) + Math.min(bspInputSplit.getSplitIndex(), extraVertices); edgesPerVertex = getConf().getInt( PseudoRandomInputFormatConstants.EDGES_PER_VERTEX, 0); rand = new Random(bspInputSplit.getSplitIndex()); if (getConf().reuseEdgeObjects()) { reusableEdge = getConf().createReusableEdge(); } localEdgesHelper = new PseudoRandomIntNullLocalEdgesHelper( aggregateVertices, getConf()); }
@Override public void readFields(DataInput in) throws IOException { int numEdges = in.readInt(); initialize(numEdges); for (int i = 0; i < numEdges; ++i) { I targetVertexId = getConf().createVertexId(); targetVertexId.readFields(in); E edgeValue = getConf().createEdgeValue(); edgeValue.readFields(in); edgeMap.put(targetVertexId, edgeValue); } }
@Override @SuppressWarnings("rawtypes") public void initialize(Context context, long attempt) throws IOException { frequency = getConf().getInt(FREQUENCY, NEVER); String filename = getConf().get(FILENAME, DEFAULT_FILENAME); if (frequency != NEVER) { Path p = new Path(filename + "_" + attempt); FileSystem fs = FileSystem.get(getConf()); if (fs.exists(p)) { throw new RuntimeException("aggregatorWriter file already" + " exists: " + p.getName()); } output = fs.create(p); } }
Partition<I, V, E> partition = conf.createPartition(partitionId, context); OutOfCoreDataAccessor dataAccessor = oocEngine.getDataAccessor(); index.addIndex(DataIndex.TypeIndexEntry.PARTITION_VERTICES); long numVertices = dataInput.readLong(); for (long i = 0; i < numVertices; ++i) { Vertex<I, V, E> vertex = conf.createVertex(); readVertexData(dataInput, vertex); partition.putVertex(vertex); if (!conf.isStaticGraph() || oocEngine.getSuperstep() == BspService.INPUT_SUPERSTEP) { shouldDeleteEdges = true;
this.context = context; this.conf = configuration; taskPartition = conf.getTaskPartition(); jobId = conf.getJobId(); baseDirectory = new Path(ZOOKEEPER_MANAGER_DIRECTORY.getWithDefault(conf, (new ComputationDoneName(taskPartition)).getName()); pollMsecs = GiraphConstants.ZOOKEEPER_SERVERLIST_POLL_MSECS.get(conf); String jobLocalDir = conf.get("job.local.dir"); String zkDirDefault; if (jobLocalDir != null) { // for non-local jobs ZOOKEEPER_MANAGER_DIRECTORY.getDefaultValue(); zkDir = conf.get(GiraphConstants.ZOOKEEPER_DIR, zkDirDefault); config = new ZookeeperConfig(); zkBasePort = GiraphConstants.ZOOKEEPER_SERVER_PORT.get(conf); myHostname = conf.getLocalHostname(); fs = FileSystem.get(conf);
/** * Constructor to execute the validation test, throws * unchecked exception to end job run on failure. * * @param conf the Configuration for this run. */ public GiraphConfigurationValidator(Configuration conf) { this.conf = new ImmutableClassesGiraphConfiguration(conf); }