private void drainQueue() throws Exception { for (HttpUrl url; (url = queue.take()) != null; ) { if (!fetchedUrls.add(url)) { continue; } Thread currentThread = Thread.currentThread(); String originalName = currentThread.getName(); currentThread.setName("Crawler " + url.toString()); try { fetch(url); } catch (IOException e) { System.out.printf("XXX: %s %s%n", url, e); } finally { currentThread.setName(originalName); } } }
public static void main(String[] args) throws IOException { if (args.length != 2) { System.out.println("Usage: Crawler <cache dir> <root>"); return; } int threadCount = 20; long cacheByteCount = 1024L * 1024L * 100L; Cache cache = new Cache(new File(args[0]), cacheByteCount); OkHttpClient client = new OkHttpClient.Builder() .cache(cache) .build(); Crawler crawler = new Crawler(client); crawler.queue.add(HttpUrl.get(args[1])); crawler.parallelDrainQueue(threadCount); } }