public void start() { spider.start(); }
public static void main(String[] args) throws Exception { Spider zhihuSpider = Spider.create(new ZhihuPageProcessor()) .addUrl("http://my.oschina.net/flashsword/blog"); Spider githubSpider = Spider.create(new GithubRepoPageProcessor()) .addUrl("https://github.com/code4craft"); SpiderMonitor.instance().register(zhihuSpider); SpiderMonitor.instance().register(githubSpider); zhihuSpider.start(); githubSpider.start(); } }
public static void main(String[] args) throws IOException, JMException { //Just for benchmark Spider thread = OOSpider.create(Site.me().setSleepTime(0), new PageModelPipeline() { @Override public void process(Object o, Task task) { } }, Kr36NewsModel.class).thread(20).addUrl("http://www.36kr.com/"); thread.start(); SpiderMonitor spiderMonitor = SpiderMonitor.instance(); spiderMonitor.register(thread); }
public static void main(String[] args) throws IOException, JMException { //Just for benchmark Spider thread = OOSpider.create(Site.me().setSleepTime(0), new PageModelPipeline() { @Override public void process(Object o, Task task) { } }, Kr36NewsModel.class).thread(20).addUrl("http://www.36kr.com/"); thread.start(); SpiderMonitor spiderMonitor = SpiderMonitor.instance(); spiderMonitor.register(thread); }