public LanguageSet getLanguageSet() { return new LanguageSet(searchers.keySet()); }
public AbstractUniversalEntity(int algorithmId, Multimap<Language, T> localEntities) { this.algorithmId = algorithmId; this.localEntities = localEntities; List<Language> langs = new ArrayList<Language>(localEntities.keySet()); Collections.sort(langs); this.languages = new LanguageSet(langs); }
public EnvBuilder setLanguages(String langCodes) { return setLanguages(new LanguageSet(langCodes)); }
public List<File> getFiles(Language language, FileMatcher ... matchers) { return getFiles(new LanguageSet(language), matchers); }
public static void main(String[] args) throws Exception { Env env = EnvBuilder.envFromArgs(args); Configurator conf = env.getConfigurator(); KNNEvaluator evaluator = new KNNEvaluator(env, new LanguageSet("simple")); SpatialDataDao sdDao = conf.get(SpatialDataDao.class); Set<Integer> originSet = new HashSet<Integer>(); originSet.add(36091);originSet.add(956);originSet.add(64);originSet.add(258);originSet.add(60);originSet.add(65);originSet.add(90);originSet.add(84);originSet.add(1490); evaluator.evaluate(originSet, 100, 5, 1, 30, "test-topo.csv"); }
public static void main(String[] args) throws Exception { Env env = EnvBuilder.envFromArgs(args); Configurator conf = env.getConfigurator(); KNNEvaluator evaluator = new KNNEvaluator(env, new LanguageSet("simple")); SpatialDataDao sdDao = conf.get(SpatialDataDao.class); Set<Integer> originSet = new HashSet<Integer>(); originSet.add(36091);originSet.add(956);originSet.add(64);originSet.add(258);originSet.add(60);originSet.add(65);originSet.add(90);originSet.add(84);originSet.add(1490); evaluator.evaluate(originSet, 100, 5, 1, 30, "test-topo.csv"); }
public static LanguageSet getLanguageSet(byte[] truncated) { Set<Language> languages = new HashSet<Language>(); boolean extra = false; for (byte b : truncated) { if (extra) { languages.add(Language.getById(b+128+255)); extra = false; } else if (b == -128) { extra = true; } else { languages.add(Language.getById(b + 128)); } } return new LanguageSet(languages); }
/** * Load specified tagged geometries to concepts * @param geometries * @throws DaoException */ public void retrieveLocations(Map<Integer, Geometry> geometries) throws DaoException { LOG.info(String.format("Found %d total geometries, now loading geometries", geometries.size())); // Build up list of concepts in all languages for (Integer conceptId : geometries.keySet()){ UniversalPage concept = upDao.getById(conceptId); if (concept != null && concept.hasAllLanguages(new LanguageSet(langs))) { concepts.add(concept); Geometry g1 = geometries.get(conceptId); locations.put(conceptId, g1.getCentroid()); if (concepts.size() % 1000 == 0) { LOG.info(String.format("Loaded %d geometries with articles in %s...", concepts.size(), langs)); } } } LOG.info(String.format("Finish loading %d geometries with articles in %s", concepts.size(), langs)); }
/** * Load specified tagged geometries to concepts * @param geometries * @throws DaoException */ public void retrieveLocations(Map<Integer, Geometry> geometries) throws DaoException { LOG.info(String.format("Found %d total geometries, now loading geometries", geometries.size())); // Build up list of concepts in all languages for (Integer conceptId : geometries.keySet()){ UniversalPage concept = upDao.getById(conceptId); if (concept != null && concept.hasAllLanguages(new LanguageSet(langs))) { concepts.add(concept); Geometry g1 = geometries.get(conceptId); locations.put(conceptId, g1.getCentroid()); if (concepts.size() % 1000 == 0) { LOG.info(String.format("Loaded %d geometries with articles in %s...", concepts.size(), langs)); } } } LOG.info(String.format("Finish loading %d geometries with articles in %s", concepts.size(), langs)); }
/** * Load all locations from all language editions of Wikipedia to concepts * * @throws DaoException */ public void retrieveAllLocations() throws DaoException { // Get all known concept geometries Map<Integer, Geometry> geometries = sdDao.getAllGeometriesInLayer("wikidata", "earth"); LOG.info(String.format("Found %d total geometries, now loading geometries", geometries.size())); // Build up list of concepts in all languages for (Integer conceptId : geometries.keySet()){ UniversalPage concept = upDao.getById(conceptId); if (concept != null && concept.hasAllLanguages(new LanguageSet(langs))) { concepts.add(concept); Geometry g1 = geometries.get(conceptId); locations.put(conceptId, g1.getCentroid()); if (concepts.size() % 1000 == 0) { LOG.info(String.format("Loaded %d geometries with articles in %s...", concepts.size(), langs)); } } } LOG.info(String.format("Found %d geometries with articles in %s", concepts.size(), langs)); }
@Override public PhraseAnalyzer get(String name, Config config, Map<String, String> runtimeParams) throws ConfigurationException { if (!config.getString("type").equals("olucene")) { return null; } LocalPageDao localPageDao = getConfigurator().get(LocalPageDao.class, config.getString("localPageDao")); LuceneSearcher searcher = new LuceneSearcher( new LanguageSet("simple"), getConfigurator().get(LuceneOptions.class)); return new LucenePhraseAnalyzer(localPageDao, searcher); }
/** * Load all locations from all language editions of Wikipedia to concepts * * @throws DaoException */ public void retrieveAllLocations() throws DaoException { // Get all known concept geometries Map<Integer, Geometry> geometries = sdDao.getAllGeometriesInLayer("wikidata", "earth"); LOG.info(String.format("Found %d total geometries, now loading geometries", geometries.size())); // Build up list of concepts in all languages for (Integer conceptId : geometries.keySet()){ UniversalPage concept = upDao.getById(conceptId); if (concept != null && concept.hasAllLanguages(new LanguageSet(langs))) { concepts.add(concept); Geometry g1 = geometries.get(conceptId); locations.put(conceptId, g1.getCentroid()); if (concepts.size() % 1000 == 0) { LOG.info(String.format("Loaded %d geometries with articles in %s...", concepts.size(), langs)); } } } LOG.info(String.format("Found %d geometries with articles in %s", concepts.size(), langs)); }
public static void main(String[] args) throws Exception { Env env = EnvBuilder.envFromArgs(args); Configurator conf = env.getConfigurator(); ToblersLawEvaluator evaluator = new ToblersLawEvaluator(env, new LanguageSet("simple")); SpatialDataDao sdDao = conf.get(SpatialDataDao.class); Map<Integer, Geometry> allGeometries = sdDao.getAllGeometriesInLayer("wikidata", "earth"); Map<Integer, Geometry> geometryMap = new HashMap<Integer, Geometry>(); int counter = 0; for(Integer id: allGeometries.keySet()){ geometryMap.put(id, allGeometries.get(id)); counter ++; if(counter >= 100) break; } //evaluator.retrieveLocations(allGeometries); evaluator.retrieveAllLocations(); //evaluator.evaluateAll(new File("testTFL.csv")); evaluator.evaluateSample(new File("testTFL-2.csv"), 1000000); }
public static void main(String[] args) throws Exception { Env env = EnvBuilder.envFromArgs(args); Configurator conf = env.getConfigurator(); ToblersLawEvaluator evaluator = new ToblersLawEvaluator(env, new LanguageSet("simple")); SpatialDataDao sdDao = conf.get(SpatialDataDao.class); Map<Integer, Geometry> allGeometries = sdDao.getAllGeometriesInLayer("wikidata", "earth"); Map<Integer, Geometry> geometryMap = new HashMap<Integer, Geometry>(); int counter = 0; for(Integer id: allGeometries.keySet()){ geometryMap.put(id, allGeometries.get(id)); counter ++; if(counter >= 100) break; } //evaluator.retrieveLocations(allGeometries); evaluator.retrieveAllLocations(); //evaluator.evaluateAll(new File("testTFL.csv")); evaluator.evaluateSample(new File("testTFL-2.csv"), 1000000); }
public UniversalWord2VecMain(Env env, Language lang) throws ConfigurationException, DaoException { this.env = env; this.lang = lang; UniversalPageDao univDao = env.getConfigurator().get(UniversalPageDao.class); this.pageDao = env.getComponent(LocalPageDao.class); Map<Language, TIntIntMap> allConcepts = univDao.getAllLocalToUnivIdsMap(new LanguageSet(lang)); this.concepts = allConcepts.containsKey(lang) ? allConcepts.get(lang) : new TIntIntHashMap(); }
public static void main(String[] args) throws Exception { Env env = EnvBuilder.envFromArgs(args); Configurator conf = env.getConfigurator(); TopoEvaluator evaluator = new TopoEvaluator(env, new LanguageSet("simple")); SpatialDataDao sdDao = conf.get(SpatialDataDao.class); //Map<Integer, Geometry> allGeometries = sdDao.getAllGeometriesInLayer("wikidata", "earth"); //Map<Integer, Geometry> geometryMap = new HashMap<Integer, Geometry>(); Set<String> subLayers = Sets.newHashSet(); subLayers.add("wikidata"); SpatialContainmentDao scDao = conf.get(SpatialContainmentDao.class); TIntSet containedItemIds = scDao.getContainedItemIds(30, "country", RefSys.EARTH, subLayers, SpatialContainmentDao.ContainmentOperationType.CONTAINMENT); LinkedList<Integer> itemIdList = new LinkedList<Integer>(); int[] itemIds = containedItemIds.toArray(); for(Integer k : itemIds){ itemIdList.add(k); } Map<Integer, Geometry> geometryMap = sdDao.getBulkGeometriesInLayer(itemIdList, "wikidata", "earth"); evaluator.retrieveLocations(geometryMap, "wikidata", "states"); //evaluator.retrieveAllLocations("wikidata", "country"); evaluator.evaluateSample(new File("TopoEval.csv"), 500000); }
public static void main(String[] args) throws Exception { Env env = EnvBuilder.envFromArgs(args); Configurator conf = env.getConfigurator(); TopoEvaluator evaluator = new TopoEvaluator(env, new LanguageSet("simple")); SpatialDataDao sdDao = conf.get(SpatialDataDao.class); //Map<Integer, Geometry> allGeometries = sdDao.getAllGeometriesInLayer("wikidata", "earth"); //Map<Integer, Geometry> geometryMap = new HashMap<Integer, Geometry>(); Set<String> subLayers = Sets.newHashSet(); subLayers.add("wikidata"); SpatialContainmentDao scDao = conf.get(SpatialContainmentDao.class); TIntSet containedItemIds = scDao.getContainedItemIds(30, "country", RefSys.EARTH, subLayers, SpatialContainmentDao.ContainmentOperationType.CONTAINMENT); LinkedList<Integer> itemIdList = new LinkedList<Integer>(); int[] itemIds = containedItemIds.toArray(); for(Integer k : itemIds){ itemIdList.add(k); } Map<Integer, Geometry> geometryMap = sdDao.getBulkGeometriesInLayer(itemIdList, "wikidata", "earth"); evaluator.retrieveLocations(geometryMap, "wikidata", "states"); //evaluator.retrieveAllLocations("wikidata", "country"); evaluator.evaluateSample(new File("TopoEval.csv"), 500000); }
public List<StageDiagnostic> getAll() throws ParseException { List<StageDiagnostic> result = new ArrayList<StageDiagnostic>(); ParseQuery<ParseObject> query = ParseQuery.getQuery("StageDiagnostic"); for (ParseObject object : query.find()) { result.add(new StageDiagnostic( object.getLong("runId"), object.getString("stage"), new LanguageSet(object.getString("langs")), object.getDouble("elapsed"), object.getDouble("singleCoreSpeed"), object.getDouble("multiCoreSpeed"), object.getDouble("megabytes") )); } return result; }
public List<StageDiagnostic> getAll() throws ParseException { List<StageDiagnostic> result = new ArrayList<StageDiagnostic>(); ParseQuery<ParseObject> query = ParseQuery.getQuery("StageDiagnostic"); for (ParseObject object : query.find()) { result.add(new StageDiagnostic( object.getLong("runId"), object.getString("stage"), new LanguageSet(object.getString("langs")), object.getDouble("elapsed"), object.getDouble("singleCoreSpeed"), object.getDouble("multiCoreSpeed"), object.getDouble("megabytes") )); } return result; }
@Override public void save(UniversalLink item) throws DaoException { try { int sourceId = item.getSourceId(); int destId = item.getDestId(); LanguageSet languages = item.getLanguageSet(); String key = sourceId + "_" + destId + "_" + item.getAlgorithmId(); byte[] temp = objectDb.get(key); if (temp != null) { languages = new LanguageSet(Sets.union(LanguageSet.getLanguageSet(temp).getLanguages(), languages.getLanguages())); } objectDb.put(key, languages.toByteArray()); } catch (IOException e) { throw new DaoException(e); } catch (ClassNotFoundException e) { throw new DaoException(e); } }