public void closeAllWriters() { final Queue<BlockOutputWriter> allWriters = new ConcurrentLinkedQueue<>(); for (Queue<BlockOutputWriter> blockOutputWriters : freeWriters.values()) { allWriters.addAll(blockOutputWriters); } if (allWriters.isEmpty()) { return; } // Closing writers can take time - use multiple threads and call progress CallableFactory<Void> callableFactory = new CallableFactory<Void>() { @Override public Callable<Void> newCallable(int callableId) { return new Callable<Void>() { @Override public Void call() throws Exception { BlockOutputWriter writer = allWriters.poll(); while (writer != null) { writer.close(); writer = allWriters.poll(); } return null; } }; } }; ProgressableUtils.getResultsWithNCallables(callableFactory, Math.min(GiraphConstants.NUM_OUTPUT_THREADS.get(conf), allWriters.size()), "close-writers-%d", progressable); } }
/** * Stop the server. */ public void stop() { if (LOG.isInfoEnabled()) { LOG.info("stop: Halting netty server"); } ProgressableUtils.awaitChannelGroupFuture(accepted.close(), progressable); if (LOG.isInfoEnabled()) { LOG.info("stop: Start releasing resources"); } bossGroup.shutdownGracefully(); workerGroup.shutdownGracefully(); ProgressableUtils.awaitTerminationFuture(bossGroup, progressable); ProgressableUtils.awaitTerminationFuture(workerGroup, progressable); if (useExecutionGroup) { executionGroup.shutdownGracefully(); ProgressableUtils.awaitTerminationFuture(executionGroup, progressable); } if (LOG.isInfoEnabled()) { LOG.info("stop: Netty server halted"); } }
/** * Get desired number of elements when they become available, * reporting progress while waiting * * @param elementCount How many elements to wait for * @param progressable Progressable to report progress * @return List of elements acquired */ public List<T> getElements(int elementCount, Progressable progressable) { ProgressableUtils.awaitSemaphorePermits( semaphore, elementCount, progressable); Preconditions.checkState(elements.size() == elementCount); List<T> ret = new ArrayList<>(elements); elements.clear(); return ret; } }
/** * Wait forever for waitable to finish. Periodically reports progress. * * @param waitable Waitable which we wait for * @param progressable Progressable for reporting progress (Job context) * @param <T> Result type * @return Result of waitable */ private static <T> T waitForever(Waitable<T> waitable, Progressable progressable) { return waitForever(waitable, progressable, DEFUALT_MSEC_PERIOD); }
while (reconnectFailures < maxConnectionFailures) { ChannelFuture connectionFuture = bootstrap.connect(remoteServer); ProgressableUtils.awaitChannelFuture(connectionFuture, context); if (connectionFuture.isSuccess()) { if (LOG.isInfoEnabled()) {
/** * Wait forever for waitable to finish. Periodically reports progress. * * @param waitable Waitable which we wait for * @param progressable Progressable for reporting progress (Job context) * @param msecsPeriod How often to report progress * @param <T> Result type * @return Result of waitable */ private static <T> T waitForever(Waitable<T> waitable, Progressable progressable, int msecsPeriod) { while (true) { waitFor(waitable, progressable, msecsPeriod, msecsPeriod); if (waitable.isFinished()) { try { return waitable.getResult(); } catch (ExecutionException e) { throw new IllegalStateException("waitForever: " + "ExecutionException occurred while waiting for " + waitable, e); } catch (InterruptedException e) { throw new IllegalStateException("waitForever: " + "InterruptedException occurred while waiting for " + waitable, e); } } } }
ProgressableUtils.awaitTerminationFuture(workerGroup, context); if (executionGroup != null) { ProgressableUtils.awaitTerminationFuture(executionGroup, context);
/** * Wait for {@link ChannelGroupFuture} to finish, while periodically * reporting progress. * * @param future ChannelGroupFuture * @param progressable Progressable for reporting progress (Job context) */ public static void awaitChannelGroupFuture(ChannelGroupFuture future, Progressable progressable) { waitForever(new ChannelGroupFutureWaitable(future), progressable); }
context.progress(); ChannelFuture future = waitingConnection.future; ProgressableUtils.awaitChannelFuture(future, context); if (!future.isSuccess() || !future.channel().isOpen()) {
ProgressableUtils.getResultsWithNCallables(callableFactory, Math.min(configuration.getNumOutputThreads(), availableVertexWriters.size()), "close-writers-%d", context);
/** * Wait for {@link ChannelFuture} to finish, while periodically * reporting progress. * * @param future ChannelFuture * @param progressable Progressable for reporting progress (Job context) */ public static void awaitChannelFuture(ChannelFuture future, Progressable progressable) { waitForever(new ChannelFutureWaitable(future), progressable); }
ProgressableUtils.getResultsWithNCallables(callableFactory, numThreads, "move-edges-%d", progressable);
/** * Wait for executor tasks to terminate, while periodically reporting * progress. * * @param executor Executor which we are waiting for * @param progressable Progressable for reporting progress (Job context) */ public static void awaitExecutorTermination(ExecutorService executor, Progressable progressable) { waitForever(new ExecutorServiceWaitable(executor), progressable); }
ProgressableUtils.getResultsWithNCallables(callableFactory, numThreads, "load-vertices-%d", getContext());
/** * Wait for executor tasks to terminate, while periodically reporting * progress. * * @param executor Executor which we are waiting for * @param progressable Progressable for reporting progress (Job context) * @param msecsPeriod How often to report progress */ public static void awaitExecutorTermination(ExecutorService executor, Progressable progressable, int msecsPeriod) { waitForever(new ExecutorServiceWaitable(executor), progressable, msecsPeriod); }
ProgressableUtils.getResultsWithNCallables(callableFactory, numThreads, "checkpoint-vertices-%d", getContext());
/** * Wait for the result of the future to be ready, while periodically * reporting progress. * * @param <T> Type of the return value of the future * @param future Future * @param progressable Progressable for reporting progress (Job context) * @return Computed result of the future. */ public static <T> T getFutureResult(Future<T> future, Progressable progressable) { return waitForever(new FutureWaitable<T>(future), progressable); }
ProgressableUtils.getResultsWithNCallables(inputSplitsCallableFactory, numThreads, "load-%d", getContext()); for (VertexEdgeCount result : results) {
/** * Wait to acquire enough permits from {@link Semaphore}, while periodically * reporting progress. * * @param semaphore Semaphore * @param permits How many permits to acquire * @param progressable Progressable for reporting progress (Job context) */ public static void awaitSemaphorePermits(final Semaphore semaphore, int permits, Progressable progressable) { while (true) { waitForever(new SemaphoreWaitable(semaphore, permits), progressable); // Verify permits were not taken by another thread, // if they were keep looping if (semaphore.tryAcquire(permits)) { return; } } }
ProgressableUtils.getResultsWithNCallables(callableFactory, numThreads, "compute-%d", context);