public final float getMinPercentResponded() { return MIN_PERCENT_RESPONDED.get(this); }
INPUT_SPLIT_SAMPLE_PERCENT.get(getConfiguration()); if (samplePercent != INPUT_SPLIT_SAMPLE_PERCENT.getDefaultValue()) { int lastIndex = (int) (samplePercent * splits.size() / 100f);
@Override public boolean isDefaultValue(Configuration conf) { return Float.compare(get(conf), defaultValue) == 0; }
/** * Constructor * * @param config configuration */ public ExtendedByteArrayOutputBuffer( ImmutableClassesGiraphConfiguration<?, ?, ?> config) { this.config = config; maxBufSize = CAPACITY_OF_DATAOUT_IN_BUFFER.get(config); threshold = (int) (FILLING_THRESHOLD_OF_DATAOUT_IN_BUFFER.get(config) * maxBufSize); }
/** * Constructor * * @param conf Giraph configuration * @param serviceWorker Service worker */ public SendEdgeCache(ImmutableClassesGiraphConfiguration conf, CentralizedServiceWorker<?, ?, ?> serviceWorker) { super(conf, serviceWorker, MAX_EDGE_REQUEST_SIZE.get(conf), ADDITIONAL_EDGE_REQUEST_SIZE.get(conf)); }
/** * Constructor. * * @param conf Giraph configuration * @param serviceWorker Service worker */ public SendPartitionCache(ImmutableClassesGiraphConfiguration<I, V, E> conf, CentralizedServiceWorker<?, ?, ?> serviceWorker) { super(conf, serviceWorker, MAX_VERTEX_REQUEST_SIZE.get(conf), ADDITIONAL_VERTEX_REQUEST_SIZE.get(conf)); }
/** * Constructor * * @param conf Giraph configuration * @param serviceWorker Service worker * @param processor NettyWorkerClientRequestProcessor * @param maxMsgSize Max message size sent to a worker */ public SendMessageCache(ImmutableClassesGiraphConfiguration conf, CentralizedServiceWorker<?, ?, ?> serviceWorker, NettyWorkerClientRequestProcessor<I, ?, ?> processor, int maxMsgSize) { super(conf, serviceWorker, MAX_MSG_REQUEST_SIZE.get(conf), ADDITIONAL_MSG_REQUEST_SIZE.get(conf)); maxMessagesSizePerWorker = maxMsgSize; clientProcessor = processor; }
/** * Constructor * * @param conf configuration * @param oocEngine out-of-core engine */ public SimpleGCMonitoringOracle(ImmutableClassesGiraphConfiguration conf, OutOfCoreEngine oocEngine) { this.optimalMemoryPressure = OPTIMAL_MEMORY_PRESSURE.get(conf); this.oocEngine = oocEngine; this.lastGCObservation = new GCObservation(-1, 0, 0); for (IOCommand.IOCommandType type : IOCommand.IOCommandType.values()) { commandOccurrences.put(type, new AtomicInteger(0)); } }
/** * Get garbage collection java opts to use * * @param conf Configuration * @return Java opts */ public static List<String> getGcJavaOpts(Configuration conf) { List<String> gcJavaOpts = new ArrayList<>(); if (USE_G1_COLLECTOR.get(conf)) { gcJavaOpts.add("-XX:+UseG1GC"); gcJavaOpts.add("-XX:MaxGCPauseMillis=500"); } else { int newGenMemoryGb = Math.max(1, (int) (MAPPER_MEMORY.get(conf) * NEW_GEN_MEMORY_FRACTION.get(conf))); // Use parallel gc collector gcJavaOpts.add("-XX:+UseParallelGC"); gcJavaOpts.add("-XX:+UseParallelOldGC"); // Fix new size generation gcJavaOpts.add("-XX:NewSize=" + newGenMemoryGb + "g"); gcJavaOpts.add("-XX:MaxNewSize=" + newGenMemoryGb + "g"); } return gcJavaOpts; } }
/** * Constructor * * @param conf configuration * @param nettyClient netty client */ public StaticFlowControl(ImmutableClassesGiraphConfiguration conf, NettyClient nettyClient) { this.nettyClient = nettyClient; maxNumberOfOpenRequests = MAX_NUMBER_OF_OPEN_REQUESTS.get(conf); numberOfRequestsToProceed = (int) (maxNumberOfOpenRequests * (1 - FRACTION_OF_REQUESTS_TO_CLOSE_BEFORE_PROCEEDING.get(conf))); if (LOG.isInfoEnabled()) { LOG.info("StaticFlowControl: Limit number of open requests to " + maxNumberOfOpenRequests + " and proceed when <= " + numberOfRequestsToProceed); } waitingRequestMsecs = WAITING_REQUEST_MSECS.get(conf); GiraphMetrics.get().addSuperstepResetObserver(this); }
this.failMemoryPressure = FAIL_MEMORY_PRESSURE.get(conf); this.emergencyMemoryPressure = EMERGENCY_MEMORY_PRESSURE.get(conf); this.highMemoryPressure = HIGH_MEMORY_PRESSURE.get(conf); this.optimalMemoryPressure = OPTIMAL_MEMORY_PRESSURE.get(conf); this.lowMemoryPressure = LOW_MEMORY_PRESSURE.get(conf); this.checkMemoryInterval = CHECK_MEMORY_INTERVAL.get(conf); this.lastGCCallInterval = LAST_GC_CALL_INTERVAL.get(conf);
int partitionCount; if (userPartitionCount == USER_PARTITION_COUNT.getDefaultValue()) { float multiplier = GiraphConstants.PARTITION_COUNT_MULTIPLIER.get(conf); partitionCount = Math.max( (int) (multiplier * availableWorkerCount * availableWorkerCount), 1);
@Override public void modifyGraph(NumericTestGraph<I, V, E> graph) { GiraphConfiguration conf = graph.getConf(); int numPartitions = NUM_COMMUNITIES.get(conf); int numVertices = NUM_VERTICES.get(conf); int numEdgesPerVertex = NUM_EDGES_PER_VERTEX.get(conf); int communitySize = numVertices / numPartitions; float actualLocalityRatio = ACTUAL_LOCALITY_RATIO.get(conf); Random random = new Random(42); for (int i = 0; i < numVertices; ++i) { for (int e = 0; e < numEdgesPerVertex / 2; ++e) { boolean localEdge = random.nextFloat() < actualLocalityRatio; int community = i / communitySize; int j; do { if (localEdge) { j = community * communitySize + random.nextInt(communitySize); } else { j = random.nextInt(numVertices); } } while (j == i); graph.addSymmetricEdge( i, j, edgeSupplier != null ? edgeSupplier.get() : null); } } } }
FREE_MEMORY_FRACTION_FOR_GC.get(conf); Thread thread = new Thread(new Runnable() { @Override
receiveBufferSize = CLIENT_RECEIVE_BUFFER_SIZE.get(conf); this.requestSizeWarningThreshold = GiraphConstants.REQUEST_SIZE_WARNING_THRESHOLD.get(conf);
(int) (cores * CORES_FRACTION_DURING_COMMUNICATION.get(conf)));
normalFreeMemoryFraction = NORMAL_FREE_MEMORY_FRACTION.get(conf); for (WorkerProgress workerProgress : workerProgresses) { if (workerProgress.getCurrentSuperstep() > currentSuperstep) {
minWorkers = conf.getMinWorkers(); maxNumberOfSupersteps = conf.getMaxNumberOfSupersteps(); minPercentResponded = GiraphConstants.MIN_PERCENT_RESPONDED.get(conf); eventWaitMsecs = conf.getEventWaitMsecs(); maxSuperstepWaitMsecs = conf.getMaxMasterSuperstepWaitMsecs();