public boolean equals(Object o){ if (o instanceof LocalPage){ LocalPage input = (LocalPage)o; return (input.getLanguage().equals(this.getLanguage()) && input.getLocalId() == this.getLocalId() ); } else { return false; } }
@Override public void call(LocalPage page) throws Exception { int views = allViews.get(page.getLocalId()); LocalPage cat = catDao.getClosestCategory(page, topLevelCategories, true); if (cat != null) { if (articleCounts.containsKey(cat)) { articleCounts.put(cat, articleCounts.get(cat) + 1); viewCounts.put(cat, viewCounts.get(cat) + views); } else { articleCounts.put(cat, 1); viewCounts.put(cat, views); } if (numPages.incrementAndGet() % 10000 == 0) { System.err.println("doing page " + numPages.get()); } } } });
@Override public void call(LocalPage page) throws Exception { int views = allViews.get(page.getLocalId()); LocalPage cat = catDao.getClosestCategory(page, topLevelCategories, true); if (cat != null) { if (articleCounts.containsKey(cat)) { articleCounts.put(cat, articleCounts.get(cat) + 1); viewCounts.put(cat, viewCounts.get(cat) + views); } else { articleCounts.put(cat, 1); viewCounts.put(cat, views); } if (numPages.incrementAndGet() % 10000 == 0) { System.err.println("doing page " + numPages.get()); } } } });
/** * * @param localCategory the category * @return a collection of the ids of all members in the given category * @throws DaoException */ public Collection<Integer> getCategoryMemberIds(LocalPage localCategory) throws DaoException { return getCategoryMemberIds(localCategory.getLanguage(), localCategory.getLocalId()); }
/** * * @param localPage the page * @return a collection of categoryids of all the categories this article belongs to * @throws DaoException */ @Override public Collection<Integer> getCategoryIds(LocalPage localPage) throws DaoException { return getCategoryIds(localPage.getLanguage(), localPage.getLocalId()); }
private Map pageJson(LocalPage p) { if (p == null) { return null; } Map json = new HashMap(); json.put("articleId", p.getLocalId()); json.put("title", p.getTitle().getCanonicalTitle()); json.put("lang", p.getLanguage().getLangCode()); return json; }
@Override public Collection<Integer> getCategoryIds(LocalPage localArticle) throws DaoException { return getCategoryIds(localArticle.getLanguage(), localArticle.getLocalId()); }
@Override public int getUnivPageId(LocalPage localPage) throws DaoException { return getUnivPageId(localPage.getLanguage(), localPage.getLocalId()); }
@Override public Collection<Integer> getCategoryMemberIds(LocalPage localCategory) throws DaoException { return getCategoryMemberIds(localCategory.getLanguage(), localCategory.getLocalId()); }
public LocalCategoryMember(LocalPage localCategory, LocalPage localArticle) throws WikiBrainException { if (!localArticle.getLanguage().equals(localCategory.getLanguage())) { throw new WikiBrainException("Language Mismatch"); } this.categoryId = localCategory.getLocalId(); this.articleId = localArticle.getLocalId(); this.language = localCategory.getLanguage(); }
/** * @return, for example "/w/en/1000/Hercule_Poirot" */ public String getCompactUrl() { String escapedTitle = getTitle().getCanonicalTitle().replace(" ", "_"); escapedTitle = escapedTitle.replaceAll("\\s+", ""); return "/w/" + getLanguage().getLangCode() + "/" + getLocalId() + "/" + escapedTitle; }
@Override public LinkedHashMap<String, Float> describe(Language language, LocalPage page, int maxPhrases) throws DaoException { LinkedHashMap<String, Float> result = new LinkedHashMap<String, Float>(); PrunedCounts<String> counts = phraseDao.getPageCounts(language, page.getLocalId(), maxPhrases); if (counts == null) { return null; } for (String phrase : counts.keySet()) { result.put(phrase, (float)1.0 * counts.get(phrase) / counts.getTotal()); if (result.size() >= maxPhrases) { break; } } return result; }
private String makeMentionUrl(LocalPage page) { return "/w/" + language.getLangCode() + "/" + page.getLocalId() + "/" + page.getTitle().getCanonicalTitle().replaceAll(" ", "_"); }
public TIntSet getRedirects(LocalPage localPage) throws DaoException { List<Integer> redirects = getRedirectsFromId(localPage.getLanguage(), localPage.getLocalId()); return new TIntHashSet(redirects); }
/** * Builds a lucene text field for page based on the specified text field elements * * @param page * @param elements * @return * @throws DaoException */ public TextField buildTextField(LocalPage page, TextFieldElements elements) throws DaoException { return buildTextField( page, rawPageDao.getById(page.getLanguage(), page.getLocalId()), elements); }
public static void main(String args[]) throws ConfigurationException, DaoException { // The following ten-line dance to get an env is awkward and repeated over and over. // Figure out a good way to consolidate it. Env env = EnvBuilder.envFromArgs(args); Configurator configurator = env.getConfigurator(); LocalPageDao lpDao = configurator.get(LocalPageDao.class); LocalLinkDao sqlDao = configurator.get(LocalLinkDao.class, "sql"); Language simple = env.getLanguages().getDefaultLanguage(); LocalPage page = lpDao.getByTitle(new Title("List of Soundgarden band members", simple), NameSpace.ARTICLE); System.out.println("page is " + page); DaoFilter filter = new DaoFilter().setSourceIds(page.getLocalId()).setLanguages(simple); for (LocalLink link : sqlDao.get(filter)) { System.out.println("link is: " + link); } } }
public static void main(String args[]) throws ConfigurationException, DaoException { // The following ten-line dance to get an env is awkward and repeated over and over. // Figure out a good way to consolidate it. Env env = EnvBuilder.envFromArgs(args); Configurator configurator = env.getConfigurator(); LocalPageDao lpDao = configurator.get(LocalPageDao.class); LocalLinkDao sqlDao = configurator.get(LocalLinkDao.class, "sql"); Language simple = env.getLanguages().getDefaultLanguage(); LocalPage page = lpDao.getByTitle(new Title("List of Soundgarden band members", simple), NameSpace.ARTICLE); System.out.println("page is " + page); DaoFilter filter = new DaoFilter().setSourceIds(page.getLocalId()).setLanguages(simple); for (LocalLink link : sqlDao.get(filter)) { System.out.println("link is: " + link); } } }
private TIntSet getAllPageIds() throws IOException { TIntSet validIds;DaoFilter filter = new DaoFilter() .setLanguages(getLanguage()) .setDisambig(false) .setRedirect(false) .setNameSpaces(NameSpace.ARTICLE); validIds = new TIntHashSet(); try { for (LocalPage page : (Iterable<LocalPage>)getLocalPageDao().get(filter)) { validIds.add(page.getLocalId()); } } catch (DaoException e) { throw new IOException(e); } return validIds; }
@Override public void save(LocalPage page) throws DaoException { insert( page.getLanguage().getId(), page.getLocalId(), page.getTitle().getCanonicalTitle(), page.getNameSpace().getArbitraryId(), page.isRedirect(), page.isDisambig() ); }
@Override public TIntSet getRedirects(LocalPage localPage) throws DaoException { DSLContext context = getJooq(); try { Result<Record> result = context.select(). from(Tables.REDIRECT). where(Tables.REDIRECT.DEST_PAGE_ID.eq(localPage.getLocalId())). and(Tables.REDIRECT.LANG_ID.eq(localPage.getLanguage().getId())). fetch(); TIntSet ids = new TIntHashSet(); for (Record record : result){ ids.add(record.getValue(Tables.REDIRECT.SRC_PAGE_ID)); } return ids; } finally { freeJooq(context); } }