/** * Constructor. * * @param conf Configuration * @param oocEngine Out-of-core engine */ DiskBackedDataStore(ImmutableClassesGiraphConfiguration conf, OutOfCoreEngine oocEngine) { this.minBufferSizeToOffload = MINIMUM_BUFFER_SIZE_TO_FLUSH.get(conf); this.oocEngine = oocEngine; }
/** * Set the number of input split threads * * @param numInputSplitsThreads Number of input split threads to use */ public void setNumInputSplitsThreads(int numInputSplitsThreads) { NUM_INPUT_THREADS.set(this, numInputSplitsThreads); }
private void addPrivateConfiguration(GiraphConfiguration configuration) { // Set currently fixed classes. TODO: Allow configuration at some point perhaps configuration.setComputationClass(io.arabesque.computation.ExecutionEngine.class); configuration.setMasterComputeClass(io.arabesque.computation.MasterExecutionEngine.class); configuration.setVertexInputFormatClass(ArabesqueInputFormat.class); // Calculate partition count based on # of workers and # of threads if no count was provided if (GiraphConstants.USER_PARTITION_COUNT.get(configuration) == -1) { int numWorkers = configuration.getMinWorkers(); int numComputeThreads = configuration.getNumComputeThreads(); GiraphConstants.USER_PARTITION_COUNT.set(configuration, numWorkers * numComputeThreads); } // Set our default worker context class instead of Giraph's default if (GiraphConstants.WORKER_CONTEXT_CLASS.get(configuration).equals(DefaultWorkerContext.class)) { configuration.setWorkerContextClass(WorkerContext.class); } }
int userPartitionCount = USER_PARTITION_COUNT.get(conf); int partitionCount; if (userPartitionCount == USER_PARTITION_COUNT.getDefaultValue()) { float multiplier = GiraphConstants.PARTITION_COUNT_MULTIPLIER.get(conf); partitionCount = Math.max( (int) (multiplier * availableWorkerCount * availableWorkerCount), 1); int minPartitionsPerComputeThread = MIN_PARTITIONS_PER_COMPUTE_THREAD.get(conf); int totalComputeThreads = NUM_COMPUTE_THREADS.get(conf) * availableWorkerCount; partitionCount = Math.max(partitionCount, minPartitionsPerComputeThread * totalComputeThreads);
sendBufferSize = GiraphConstants.SERVER_SEND_BUFFER_SIZE.get(conf); receiveBufferSize = GiraphConstants.SERVER_RECEIVE_BUFFER_SIZE.get(conf); maxPoolSize = GiraphConstants.NETTY_SERVER_THREADS.get(conf); tcpBacklog = conf.getInt(GiraphConstants.TCP_BACKLOG.getKey(), conf.getInt(GiraphConstants.MAX_WORKERS, GiraphConstants.TCP_BACKLOG.getDefaultValue()));
if (!GiraphConstants.NUM_OUT_OF_CORE_THREADS.isDefaultValue(conf) || GiraphConstants.NUM_OUT_OF_CORE_THREADS.get(conf) != numDisks) { LOG.warn("LocalDiskDataAccessor: with this data accessor, number of " + "out-of-core threads is only specified by the number of " + ptr++; final int diskBufferSize = OOC_DISK_BUFFER_SIZE.get(conf); this.perThreadBuffers = new byte[numDisks][diskBufferSize];
int workers = conf.getInt(GiraphConstants.MIN_WORKERS, -1); Preconditions.checkArgument(workers > 0, "Number of workers not set"); int cores = MAPPER_CORES.get(conf); Integer.toString(cores)); GiraphConstants.NUM_OUTPUT_THREADS.setIfUnset(conf, cores); GiraphConstants.NUM_INPUT_THREADS.setIfUnset( conf, threadsDuringCommunication); GiraphConstants.NUM_COMPUTE_THREADS.setIfUnset( conf, threadsDuringCommunication); GiraphConstants.NETTY_SERVER_THREADS.setIfUnset( conf, threadsDuringCommunication); GiraphConstants.CHANNELS_PER_SERVER.setIfUnset(conf, Math.max(1, 2 * threadsDuringCommunication / workers)); StaticFlowControl.MAX_NUMBER_OF_OPEN_REQUESTS.setIfUnset(conf, 100); GiraphConstants.MIN_PARTITIONS_PER_COMPUTE_THREAD.setIfUnset(conf, 3);
/** * Set value * * @param conf Configuration * @param value to set */ public void set(Configuration conf, int value) { conf.setInt(getKey(), value); }
GiraphConstants.MAX_NUMBER_OF_SUPERSTEPS.getDefaultValue() && (getSuperstep() == maxNumberOfSupersteps - 1)) { if (LOG.isInfoEnabled()) {
public int run(String[] args) { final GiraphConfiguration giraphConf = new GiraphConfiguration(getConf()); final int zkPort = ZOOKEEPER_SERVER_PORT.get(giraphConf); final String zkBasePath = giraphConf.get( GiraphConstants.BASE_ZNODE_KEY, "") + BspService.BASE_DIR; ZooKeeperExt zooKeeper = new ZooKeeperExt( formatZkServerList(zkServerList, zkPort), GiraphConstants.ZOOKEEPER_SESSION_TIMEOUT.getDefaultValue(), GiraphConstants.ZOOKEEPER_OPS_MAX_ATTEMPTS.getDefaultValue(), GiraphConstants.ZOOKEEPER_SERVERLIST_POLL_MSECS.getDefaultValue(), this); doZooKeeperCleanup(zooKeeper, zkBasePath);
/** * Lookup value * * @param conf Configuration * @return value for key, or default value if not set */ public int get(Configuration conf) { return conf.getInt(getKey(), defaultValue); }
/** * Get the maximum number of supersteps of this application. After this * many supersteps are executed, the application will shutdown. * * @return Maximum number of supersteps */ public int getMaxNumberOfSupersteps() { return MAX_NUMBER_OF_SUPERSTEPS.get(this); }
/** * Set the number of threads to use for writing output in the end of the * application. Will be used only if {#vertexOutputFormatThreadSafe} is true. * * @param numOutputThreads Number of output threads */ public void setNumOutputThreads(int numOutputThreads) { NUM_OUTPUT_THREADS.set(this, numOutputThreads); }
LOG.info("run: Since checkpointing is disabled (default), " + "do not allow any task retries (setting " + GiraphConstants.MAX_TASK_ATTEMPTS.getKey() + " = 1, " + "old value = " + oldMaxTaskAttempts + ")");
/** * Constructor * * @param conf configuration * @param oocEngine out-of-core engine */ public FixedPartitionsOracle(ImmutableClassesGiraphConfiguration conf, OutOfCoreEngine oocEngine) { this.maxPartitionsInMemory = GiraphConstants.MAX_PARTITIONS_IN_MEMORY.get(conf); this.oocEngine = oocEngine; }
/** * Set heap size for Giraph tasks in our job run, assuming * the job will run on the "pure YARN" profile. * * @param heapMb the heap size for all tasks */ public void setYarnTaskHeapMb(int heapMb) { GIRAPH_YARN_TASK_HEAP_MB.set(this, heapMb); }
/** * Get the checkpoint frequeuncy of how many supersteps to wait * before checkpointing * * @return Checkpoint frequency (0 means never) */ public int getCheckpointFrequency() { return CHECKPOINT_FREQUENCY.get(this); }
/** * Set the number of compute threads * * @param numComputeThreads Number of compute threads to use */ public void setNumComputeThreads(int numComputeThreads) { NUM_COMPUTE_THREADS.set(this, numComputeThreads); }
/** * Get the maximum timeout (in milliseconds) for waiting for all tasks * to complete after the job is done. * * @return Wait task done timeout in milliseconds. */ public int getWaitTaskDoneTimeoutMs() { return WAIT_TASK_DONE_TIMEOUT_MS.get(this); }