private void save(File file, String json) throws WpParseException, DaoException { if (!json.contains("{")) { return; } json = json.trim(); if (json.endsWith(",")) { json = json.substring(0, json.length()-1); } if (counter.incrementAndGet() % 100000 == 0) { LOG.info("processing wikidata entity " + counter.get()); } WikidataEntity entity = wdParser.parse(json); // check if others use prune's boolean? entity.prune(languages); if (keepEntity(entity)) { wikidataDao.save(entity); } }
LanguageSet langs = conf.get(LanguageSet.class); WikidataDumpLoader loader = new WikidataDumpLoader(wdDao, metaDao, upDao, langs); loader.setKeepAllLabeledEntities(true); loader.load(path);
LanguageSet langs = conf.get(LanguageSet.class); WikidataDumpLoader loader = new WikidataDumpLoader(wdDao, metaDao, upDao, langs); loader.setKeepAllLabeledEntities(true); loader.load(path);
private void save(File file, String json) throws WpParseException, DaoException { if (!json.contains("{")) { return; } json = json.trim(); if (json.endsWith(",")) { json = json.substring(0, json.length()-1); } if (counter.incrementAndGet() % 100000 == 0) { LOG.info("processing wikidata entity " + counter.get()); } WikidataEntity entity = wdParser.parse(json); // check if others use prune's boolean? entity.prune(languages); if (keepEntity(entity)) { wikidataDao.save(entity); } }