@Override public void beginLoad() throws DaoException { super.beginLoad(); try { path = File.createTempFile("univ-links", "odb"); if (path.isFile()) { path.delete(); } path.mkdirs(); objectDb = new ObjectDb<byte[]>(path, true); } catch (IOException e) { throw new DaoException(e); } }
@Override public void endLoad() throws DaoException { for (Pair<String, byte[]> pair : objectDb) { String[] ids = pair.getKey().split("_"); insert( Integer.valueOf(ids[0]), Integer.valueOf(ids[1]), Integer.valueOf(ids[2]), pair.getValue() ); } objectDb.close(); path.delete(); super.endLoad(); }
@Override public void clear() throws DaoException { super.clear(); cache.remove(LOADED_CACHE_KEY); }
public static void main(String args[]) throws ConfigurationException, DaoException { Env env = new EnvBuilder() .setConfigFile("../psql.conf") .build(); AbstractSqlDao dao = (AbstractSqlDao) env.getConfigurator().get( LocalPageDao.class, "sql"); dao.setFetchSize(1000); System.out.println("about to start reads"); long before = System.currentTimeMillis(); int i = 0; for (Object obj : dao.get(new DaoFilter())) { if (i++ % 10000 == 0) { System.out.println("read entity " + i); } } long after = System.currentTimeMillis(); System.out.println("elapsed is " + (after - before)); } }
@Override public void clear() throws DaoException { executeSqlScriptWithSuffix("-drop-indexes.sql"); executeSqlScriptWithSuffix("-drop-tables.sql"); }
/** * Closes this iterable, disabling all functionality. */ public void close() { usedUp = true; finished = true; // while (iterator.hasNext()) { // iterator.next(); // } if (!result.isClosed()) { result.close(); } AbstractSqlDao.quietlyCloseConn(conn); }
/** * Executes the appropriate sql script with a particular suffix (.e.g. "-drop-tables.sql"). * @param suffix * @throws DaoException */ protected void executeSqlScriptWithSuffix(String suffix) throws DaoException { executeSqlResource(sqlScriptPrefix + suffix); }
public static void main(String args[]) throws ConfigurationException, DaoException { Env env = new EnvBuilder() .setConfigFile("../psql.conf") .build(); AbstractSqlDao dao = (AbstractSqlDao) env.getConfigurator().get( LocalPageDao.class, "sql"); dao.setFetchSize(1000); System.out.println("about to start reads"); long before = System.currentTimeMillis(); int i = 0; for (Object obj : dao.get(new DaoFilter())) { if (i++ % 10000 == 0) { System.out.println("read entity " + i); } } long after = System.currentTimeMillis(); System.out.println("elapsed is " + (after - before)); } }
@Override public void beginLoad() throws DaoException { executeSqlScriptWithSuffix("-drop-indexes.sql"); executeSqlScriptWithSuffix("-create-tables.sql"); if (fields != null) { loader = new FastLoader(wpDs , fields); } }
JDBCUtils.safeClose(statement); AbstractSqlDao.quietlyCloseConn(cnx);
@Override public void endLoad() throws DaoException { sync(); super.endLoad(); wpDs.optimize(Tables.META_INFO); }
@Override public void beginLoad() throws DaoException { super.beginLoad(); if (labelLoader == null) { labelLoader = new FastLoader(wpDs, new TableField[] { WIKIDATA_ENTITY_LABELS.ENTITY_TYPE, WIKIDATA_ENTITY_LABELS.ENTITY_ID, WIKIDATA_ENTITY_LABELS.LANG_ID, WIKIDATA_ENTITY_LABELS.LABEL, }); } if (descLoader == null) { descLoader = new FastLoader(wpDs, new TableField[] { WIKIDATA_ENTITY_DESCRIPTIONS.ENTITY_TYPE, WIKIDATA_ENTITY_DESCRIPTIONS.ENTITY_ID, WIKIDATA_ENTITY_DESCRIPTIONS.LANG_ID, WIKIDATA_ENTITY_DESCRIPTIONS.DESCRIPTION, }); } if (aliasLoader == null) { aliasLoader = new FastLoader(wpDs, new TableField[] { WIKIDATA_ENTITY_ALIASES.ENTITY_TYPE, WIKIDATA_ENTITY_ALIASES.ENTITY_ID, WIKIDATA_ENTITY_ALIASES.LANG_ID, WIKIDATA_ENTITY_ALIASES.ALIAS }); } properties = new HashMap<Integer, WikidataEntity>(); }
@Override public void endLoad() throws DaoException { if (loader != null) { loader.endLoad(); } LOG.info("creating indexes in {}-create-indexes.sql (this can take some time)", sqlScriptPrefix); executeSqlScriptWithSuffix("-create-indexes.sql"); if (fields != null && fields.length > 0) { wpDs.optimize(fields[0].getTable()); } }
throw new DaoException("SQL Dao Failed. Check if the table exists / if the desired information has been parsed and stored in the database\n" + e.toString()); } finally { quietlyCloseConn(conn);
@Override public void clear() throws DaoException { super.clear(); cache.remove(LOADED_CACHE_KEY); }
@Override public void endLoad() throws DaoException { if (labelLoader != null) labelLoader.endLoad(); if (descLoader != null) descLoader.endLoad(); if (aliasLoader != null) aliasLoader.endLoad(); labelLoader = null; descLoader = null; aliasLoader = null; super.endLoad(); if (cache != null) { cache.put("wikidata-properties", properties); } wpDs.optimize(WIKIDATA_ENTITY_LABELS); wpDs.optimize(WIKIDATA_ENTITY_ALIASES); wpDs.optimize(WIKIDATA_ENTITY_DESCRIPTIONS); wpDs.optimize(WIKIDATA_STATEMENT); }
@Override public void beginLoad() throws DaoException { super.beginLoad(); if (labelLoader == null) { labelLoader = new FastLoader(wpDs, new TableField[] { WIKIDATA_ENTITY_LABELS.ENTITY_TYPE, WIKIDATA_ENTITY_LABELS.ENTITY_ID, WIKIDATA_ENTITY_LABELS.LANG_ID, WIKIDATA_ENTITY_LABELS.LABEL, }); } if (descLoader == null) { descLoader = new FastLoader(wpDs, new TableField[] { WIKIDATA_ENTITY_DESCRIPTIONS.ENTITY_TYPE, WIKIDATA_ENTITY_DESCRIPTIONS.ENTITY_ID, WIKIDATA_ENTITY_DESCRIPTIONS.LANG_ID, WIKIDATA_ENTITY_DESCRIPTIONS.DESCRIPTION, }); } if (aliasLoader == null) { aliasLoader = new FastLoader(wpDs, new TableField[] { WIKIDATA_ENTITY_ALIASES.ENTITY_TYPE, WIKIDATA_ENTITY_ALIASES.ENTITY_ID, WIKIDATA_ENTITY_ALIASES.LANG_ID, WIKIDATA_ENTITY_ALIASES.ALIAS }); } properties = new HashMap<Integer, WikidataEntity>(); }
@Override public void endLoad() throws DaoException { if (labelLoader != null) labelLoader.endLoad(); if (descLoader != null) descLoader.endLoad(); if (aliasLoader != null) aliasLoader.endLoad(); labelLoader = null; descLoader = null; aliasLoader = null; super.endLoad(); if (cache != null) { cache.put("wikidata-properties", properties); } wpDs.optimize(WIKIDATA_ENTITY_LABELS); wpDs.optimize(WIKIDATA_ENTITY_ALIASES); wpDs.optimize(WIKIDATA_ENTITY_DESCRIPTIONS); wpDs.optimize(WIKIDATA_STATEMENT); }