@Override public void call(WikidataStatement statement) throws Exception { try { if (storeStatement(savedConcepts, langs, statement)) { matches.incrementAndGet(); } } catch (Exception e) { LOG.error("storage of statement failed: " + statement.toString(), e); miDao.incrementErrorsQuietly(Geometry.class); } if (count.incrementAndGet() % 10000 == 0){ LOG.info("Matched " + matches + " out of " + count + " statements from " + this.getClass().getName()); } } }, Integer.MAX_VALUE);
@Override public double score(LocalId candidate, Map<String, String> row, Geometry geometry) throws DaoException { for (WikidataStatement st : wikidataDao.getStatements(candidate.asLocalPage())) { if (st.getProperty() == null || st.getProperty().getId() != INSTANCE_OF_PROPERTY) { continue; } if (st.getValue().getType() == WikidataValue.Type.ITEM) { String name = getPropertyName(st.getValue().getIntValue()); if (name != null && instanceOfKeywords.contains(name.toLowerCase())) { return 1.0; } } } return 0.0; }
@Override public LocalWikidataStatement getLocalStatement(Language language, WikidataStatement statement) throws DaoException { language = getRealLang(language ); String item = getLabel(language, statement.getItem().getType(), statement.getItem().getId()); String prop = getLabel(language, statement.getProperty().getType(), statement.getProperty().getId()); String value = null; WikidataValue wdv = statement.getValue(); if (wdv.getType() == WikidataValue.Type.ITEM) { value = getLabel(language, WikidataEntity.Type.ITEM, wdv.getItemValue()); } else if (wdv.getValue() == null) { value = "unknown"; } else { value = wdv.getValue().toString(); } String full = item + " " + prop + " " + value; return new LocalWikidataStatement(language, statement, full, item, prop, value); }
private boolean storeStatement(TIntSet savedConcepts, LanguageSet langs, WikidataStatement statement) throws DaoException { UniversalPage uPage = wdDao.getUniversalPage(statement.getItem().getId()); if (uPage == null || !uPage.isInLanguageSet(langs, false)){ return false; } int itemId = statement.getItem().getId(); Geometry g = WikiBrainSpatialUtils.jsonToGeometry(statement.getValue().getJsonValue().getAsJsonObject()); if (g == null) { return false; } if (savedConcepts.contains(itemId)) { return false; } savedConcepts.add(itemId); spatialDao.saveGeometry(itemId, Layers.WIKIDATA, RefSys.EARTH, g); miDao.incrementRecords(Geometry.class); return true; } }
@Override public Set<Integer> conceptsWithValue(String propertyName, WikidataValue value) throws DaoException { Set<Integer> concepts = new HashSet<Integer>(); for (WikidataStatement st : getByValue(propertyName, value)) { if (st.getItem().getType() == WikidataEntity.Type.ITEM) { concepts.add(st.getItem().getId()); } } return concepts; }
public Map<String, List<WikidataStatement>> getStatementsInLanguage(Language language) { Map<String, List<WikidataStatement>> inLang = new HashMap<String, List<WikidataStatement>>(); for (WikidataStatement s : statements) { String label = s.getProperty().getLabels().get(language); if (label != null) { if (!inLang.containsKey(label)) { inLang.put(label, new ArrayList<WikidataStatement>()); } inLang.get(label).add(s); } } return inLang; }
protected WikidataStatement buildStatement(Record record) throws DaoException { if (record == null) { return null; } WikidataEntity item = new WikidataEntity( WikidataEntity.Type.getByCode(record.getValue(Tables.WIKIDATA_STATEMENT.ENTITY_TYPE).charAt(0)), record.getValue(Tables.WIKIDATA_STATEMENT.ENTITY_ID) ); WikidataEntity prop = getProperty(record.getValue(Tables.WIKIDATA_STATEMENT.PROP_ID)); Short rankOrdinal = record.getValue(Tables.WIKIDATA_STATEMENT.RANK); JsonElement json = new JsonParser().parse(record.getValue(Tables.WIKIDATA_STATEMENT.VAL_STR)); WikidataValue val; try { val = parser.jsonToValue( record.getValue(Tables.WIKIDATA_STATEMENT.VAL_TYPE), json); } catch (WpParseException e) { throw new DaoException(e); } WikidataStatement stmt = new WikidataStatement( record.getValue(Tables.WIKIDATA_STATEMENT.ID), item, prop, val, WikidataStatement.Rank.values()[rankOrdinal] ); return stmt; }
@Override public LocalWikidataStatement getLocalStatement(Language language, WikidataStatement statement) throws DaoException { language = getRealLang(language ); String item = getLabel(language, statement.getItem().getType(), statement.getItem().getId()); String prop = getLabel(language, statement.getProperty().getType(), statement.getProperty().getId()); String value = null; WikidataValue wdv = statement.getValue(); if (wdv.getType() == WikidataValue.Type.ITEM) { value = getLabel(language, WikidataEntity.Type.ITEM, wdv.getItemValue()); } else if (wdv.getValue() == null) { value = "unknown"; } else { value = wdv.getValue().toString(); } String full = item + " " + prop + " " + value; return new LocalWikidataStatement(language, statement, full, item, prop, value); }
@Override public double score(LocalId candidate, Map<String, String> row, Geometry geometry) throws DaoException { if (!row.containsKey(columnName)) { throw new IllegalArgumentException("No column with name " + columnName + " in dbx"); } String rowValue = row.get(columnName).toString(); for (WikidataStatement st : wikidataDao.getStatements(candidate.asLocalPage())) { if (st.getProperty() == null || st.getProperty().getId() != propertyId) { continue; } String candidateVal = null; if (st.getValue().getType() == WikidataValue.Type.ITEM) { candidateVal = String.valueOf(st.getValue().getItemValue()); } else if (st.getValue().getType() == WikidataValue.Type.INT) { candidateVal = String.valueOf(st.getValue().getIntValue()); } else if (st.getValue().getType() == WikidataValue.Type.STRING) { candidateVal = st.getValue().getStringValue(); } else { LOG.warn("Unexpected type for property " + st.getProperty() + ": " + st.getValue()); } if (candidateVal != null && candidateVal.equalsIgnoreCase(rowValue)) { return 1.0; } } return 0.0; } }
private boolean storeStatement(TIntSet savedConcepts, LanguageSet langs, WikidataStatement statement) throws DaoException { UniversalPage uPage = wdDao.getUniversalPage(statement.getItem().getId()); if (uPage == null || !uPage.isInLanguageSet(langs, false)){ return false; } int itemId = statement.getItem().getId(); Geometry g = WikiBrainSpatialUtils.jsonToGeometry(statement.getValue().getJsonValue().getAsJsonObject()); if (g == null) { return false; } if (savedConcepts.contains(itemId)) { return false; } savedConcepts.add(itemId); spatialDao.saveGeometry(itemId, Layers.WIKIDATA, RefSys.EARTH, g); miDao.incrementRecords(Geometry.class); return true; } }
@Override public Set<Integer> conceptsWithValue(String propertyName, WikidataValue value) throws DaoException { Set<Integer> concepts = new HashSet<Integer>(); for (WikidataStatement st : getByValue(propertyName, value)) { if (st.getItem().getType() == WikidataEntity.Type.ITEM) { concepts.add(st.getItem().getId()); } } return concepts; }
public Map<String, List<WikidataStatement>> getStatementsInLanguage(Language language) { Map<String, List<WikidataStatement>> inLang = new HashMap<String, List<WikidataStatement>>(); for (WikidataStatement s : statements) { String label = s.getProperty().getLabels().get(language); if (label != null) { if (!inLang.containsKey(label)) { inLang.put(label, new ArrayList<WikidataStatement>()); } inLang.get(label).add(s); } } return inLang; }
protected WikidataStatement buildStatement(Record record) throws DaoException { if (record == null) { return null; } WikidataEntity item = new WikidataEntity( WikidataEntity.Type.getByCode(record.getValue(Tables.WIKIDATA_STATEMENT.ENTITY_TYPE).charAt(0)), record.getValue(Tables.WIKIDATA_STATEMENT.ENTITY_ID) ); WikidataEntity prop = getProperty(record.getValue(Tables.WIKIDATA_STATEMENT.PROP_ID)); Short rankOrdinal = record.getValue(Tables.WIKIDATA_STATEMENT.RANK); JsonElement json = new JsonParser().parse(record.getValue(Tables.WIKIDATA_STATEMENT.VAL_STR)); WikidataValue val; try { val = parser.jsonToValue( record.getValue(Tables.WIKIDATA_STATEMENT.VAL_TYPE), json); } catch (WpParseException e) { throw new DaoException(e); } WikidataStatement stmt = new WikidataStatement( record.getValue(Tables.WIKIDATA_STATEMENT.ID), item, prop, val, WikidataStatement.Rank.values()[rankOrdinal] ); return stmt; }
@Override public double score(LocalId candidate, Map<String, String> row, Geometry geometry) throws DaoException { for (WikidataStatement st : wikidataDao.getStatements(candidate.asLocalPage())) { if (st.getProperty() == null || st.getProperty().getId() != INSTANCE_OF_PROPERTY) { continue; } if (st.getValue().getType() == WikidataValue.Type.ITEM) { String name = getPropertyName(st.getValue().getIntValue()); if (name != null && instanceOfKeywords.contains(name.toLowerCase())) { return 1.0; } } } return 0.0; }
int countryId = st.getItem().getId(); WikidataEntity country = wd.getItem(countryId); Geometry geometry = sd.getGeometry(countryId, "wikidata", "earth");
return new WikidataStatement(uuid, item, prop, value, rank);
@Override public void call(WikidataStatement statement) throws Exception { try { if (storeStatement(savedConcepts, langs, statement)) { matches.incrementAndGet(); } } catch (Exception e) { LOG.error("storage of statement failed: " + statement.toString(), e); miDao.incrementErrorsQuietly(Geometry.class); } if (count.incrementAndGet() % 10000 == 0){ LOG.info("Matched " + matches + " out of " + count + " statements from " + this.getClass().getName()); } } }, Integer.MAX_VALUE);
@Override public double score(LocalId candidate, Map<String, String> row, Geometry geometry) throws DaoException { if (!row.containsKey(columnName)) { throw new IllegalArgumentException("No column with name " + columnName + " in dbx"); } String rowValue = row.get(columnName).toString(); for (WikidataStatement st : wikidataDao.getStatements(candidate.asLocalPage())) { if (st.getProperty() == null || st.getProperty().getId() != propertyId) { continue; } String candidateVal = null; if (st.getValue().getType() == WikidataValue.Type.ITEM) { candidateVal = String.valueOf(st.getValue().getItemValue()); } else if (st.getValue().getType() == WikidataValue.Type.INT) { candidateVal = String.valueOf(st.getValue().getIntValue()); } else if (st.getValue().getType() == WikidataValue.Type.STRING) { candidateVal = st.getValue().getStringValue(); } else { LOG.warn("Unexpected type for property " + st.getProperty() + ": " + st.getValue()); } if (candidateVal != null && candidateVal.equalsIgnoreCase(rowValue)) { return 1.0; } } return 0.0; } }