private void fillBuff() { if (buff != null) { return; } // try to queue up the next article while (buff == null && iterImpl.hasNext()) { String line = iterImpl.next(); if (line.trim().equals("[") || line.trim().equals("]")) { continue; } try { if (line.endsWith(",")) { line = line.substring(0, line.length()-1); } if (!line.trim().isEmpty()) { buff = wdParser.parse(line); } } catch (Exception e) { LOG.warn("parsing of " + file + " failed for line '" + line + "':", e); } } }
private void fillBuff() { if (buff != null) { return; } // try to queue up the next article while (buff == null && iterImpl.hasNext()) { String line = iterImpl.next(); if (line.trim().equals("[") || line.trim().equals("]")) { continue; } try { if (line.endsWith(",")) { line = line.substring(0, line.length()-1); } if (!line.trim().isEmpty()) { buff = wdParser.parse(line); } } catch (Exception e) { LOG.warn("parsing of " + file + " failed for line '" + line + "':", e); } } }
private void save(File file, String json) throws WpParseException, DaoException { if (!json.contains("{")) { return; } json = json.trim(); if (json.endsWith(",")) { json = json.substring(0, json.length()-1); } if (counter.incrementAndGet() % 100000 == 0) { LOG.info("processing wikidata entity " + counter.get()); } WikidataEntity entity = wdParser.parse(json); // check if others use prune's boolean? entity.prune(languages); if (keepEntity(entity)) { wikidataDao.save(entity); } }
private void save(File file, String json) throws WpParseException, DaoException { if (!json.contains("{")) { return; } json = json.trim(); if (json.endsWith(",")) { json = json.substring(0, json.length()-1); } if (counter.incrementAndGet() % 100000 == 0) { LOG.info("processing wikidata entity " + counter.get()); } WikidataEntity entity = wdParser.parse(json); // check if others use prune's boolean? entity.prune(languages); if (keepEntity(entity)) { wikidataDao.save(entity); } }