boolean rtIndexing, Comparator<String> versionComparator,boolean skipBadRecord) this(new DefaultDirectoryManager(idxDir), interpreter, indexReaderDecorator, analyzer, similarity, batchSize, batchDelay, rtIndexing, versionComparator,skipBadRecord);
boolean rtIndexing, Comparator<String> versionComparator,boolean skipBadRecord) this(new DefaultDirectoryManager(idxDir), interpreter, indexReaderDecorator, docIdMapperFactory, analyzer, similarity, batchSize, batchDelay, rtIndexing, versionComparator,skipBadRecord);
int batchSize, long batchDelay, boolean rtIndexing, Comparator<String> versionComparator, boolean skipBadRecord) { this(new DefaultDirectoryManager(idxDir), interpreter, indexReaderDecorator, analyzer, similarity, batchSize, batchDelay, rtIndexing, versionComparator, skipBadRecord);
int batchSize, long batchDelay, boolean rtIndexing, Comparator<String> versionComparator, boolean skipBadRecord) { this(new DefaultDirectoryManager(idxDir), interpreter, indexReaderDecorator, analyzer, similarity, batchSize, batchDelay, rtIndexing, versionComparator, skipBadRecord);
Analyzer analyzer, Similarity similarity, int batchSize, long batchDelay, boolean rtIndexing, Comparator<String> versionComparator, boolean skipBadRecord) { this(new DefaultDirectoryManager(idxDir), interpreter, indexReaderDecorator, docIdMapperFactory, analyzer, similarity, batchSize, batchDelay, rtIndexing, versionComparator, skipBadRecord);
Analyzer analyzer, Similarity similarity, int batchSize, long batchDelay, boolean rtIndexing, Comparator<String> versionComparator, boolean skipBadRecord) { this(new DefaultDirectoryManager(idxDir), interpreter, indexReaderDecorator, docIdMapperFactory, analyzer, similarity, batchSize, batchDelay, rtIndexing, versionComparator, skipBadRecord);
public Pair(File zoieOneRoot, DIRECTORY_MODE dirMode, IndexCopier indexCopier, ZoieIndexableInterpreter<D> interpreter, IndexReaderDecorator<R> decorator, ZoieConfig zoieConfig, Zoie<R, D> zoieTwo) { _zoieOneRoot = zoieOneRoot; _indexCopier = indexCopier; _interpreter = interpreter; _decorator = decorator; _zoieConfig = zoieConfig; _zoieTwo = zoieTwo; _activeReaders = new HashMap<IndexReader, ZoieRef>(); // Initialize zoieOne: if (_zoieOneRoot == null) { throw new IllegalArgumentException("zoieOneRoot cannot be null."); } if (_zoieOneRoot.exists()) { File[] files = _zoieOneRoot.listFiles(); Arrays.sort(files); for (int i = files.length - 1; i >= 0; --i) { if (new File(files[i], COMMIT_FILE).exists()) { log.info("Found latest zoieOne index: " + files[i].getAbsolutePath()); DirectoryManager dirMgr = new DefaultDirectoryManager(files[i], dirMode); _zoieOne = new ZoieSystem<R, D>(dirMgr, _interpreter, _decorator, _zoieConfig); break; } } } }
public Pair(File zoieOneRoot, DIRECTORY_MODE dirMode, IndexCopier indexCopier, ZoieIndexableInterpreter<D> interpreter, IndexReaderDecorator<R> decorator, ZoieConfig zoieConfig, Zoie<R, D> zoieTwo) { _zoieOneRoot = zoieOneRoot; _indexCopier = indexCopier; _interpreter = interpreter; _decorator = decorator; _zoieConfig = zoieConfig; _zoieTwo = zoieTwo; _activeReaders = new HashMap<IndexReader, ZoieRef>(); // Initialize zoieOne: if (_zoieOneRoot == null) { throw new IllegalArgumentException("zoieOneRoot cannot be null."); } if (_zoieOneRoot.exists()) { File[] files = _zoieOneRoot.listFiles(); Arrays.sort(files); for (int i = files.length - 1; i >= 0; --i) { if (new File(files[i], COMMIT_FILE).exists()) { log.info("Found latest zoieOne index: " + files[i].getAbsolutePath()); DirectoryManager dirMgr = new DefaultDirectoryManager(files[i], dirMode); _zoieOne = new ZoieSystem<R, D>(dirMgr, _interpreter, _decorator, _zoieConfig); break; } } } }
/** * @return true if the current index accepting updates is changed. * This method should be paired with clearRecentlyChanged() to clear the flag. * @see proj.zoie.hourglass.impl.HourglassDirectoryManagerFactory#clearRecentlyChanged() */ public boolean updateDirectoryManager() { Calendar now = Calendar.getInstance(); now.setTimeInMillis(System.currentTimeMillis()); if (now.before(_nextUpdateTime)) return false; String folderName; folderName = _scheduler.getFolderName(_nextUpdateTime); _location = new File(_root, folderName); try { log.info("rolling forward with new path: " + _location.getCanonicalPath()); } catch (IOException e) { log.error(e); } _currentDirMgr = new DefaultDirectoryManager(_location, _mode); isRecentlyChanged = true; setNextUpdateTime(); return isRecentlyChanged; }
/** * @return true if the current index accepting updates is changed. * This method should be paired with clearRecentlyChanged() to clear the flag. * @see proj.zoie.hourglass.impl.HourglassDirectoryManagerFactory#clearRecentlyChanged() */ public boolean updateDirectoryManager() { Calendar now = Calendar.getInstance(); now.setTimeInMillis(System.currentTimeMillis()); if (now.before(_nextUpdateTime)) return false; String folderName; folderName = _scheduler.getFolderName(_nextUpdateTime); _location = new File(_root, folderName); try { log.info("rolling forward with new path: " + _location.getCanonicalPath()); } catch (IOException e) { log.error(e); } _currentDirMgr = new DefaultDirectoryManager(_location, _mode); isRecentlyChanged = true; setNextUpdateTime(); return isRecentlyChanged; }
DirectoryManager dirMgr = new DefaultDirectoryManager(files[i], dirMode); _zoieOne = new ZoieSystem(dirMgr, _interpreter, _decorator, _zoieConfig); break;
/** * @return true if the current index accepting updates is changed. * This method should be paired with clearRecentlyChanged() to clear the flag. * @see proj.zoie.hourglass.impl.HourglassDirectoryManagerFactory#clearRecentlyChanged() */ public boolean updateDirectoryManager() { Calendar now = Calendar.getInstance(); now.setTimeInMillis(System.currentTimeMillis()); if (now.before(_nextUpdateTime)) return false; String folderName; folderName = _scheduler.getFolderName(_nextUpdateTime); _location = new File(_root, folderName); try { log.info("rolling forward with new path: " + _location.getCanonicalPath()); } catch (IOException e) { log.error(e); } _currentDirMgr = new DefaultDirectoryManager(_location, _mode); isRecentlyChanged = true; setNextUpdateTime(); return isRecentlyChanged; } public boolean isRecentlyChanged()
protected List<ZoieSystem<R, D>> loadArchiveZoies() { List<ZoieSystem<R, D>> archives = new ArrayList<ZoieSystem<R, D>>(); long t0 = System.currentTimeMillis(); List<File> dirs = _dirMgrFactory.getAllArchivedDirs(); for (File dir : dirs) { try { DirectoryManager dirMgr = new DefaultDirectoryManager(dir, _dirMgrFactory.getMode()); ZoieSystem<R, D> zoie = new ZoieSystem<R, D>(dirMgr, _interpreter, _decorator, _zConfig); zoie.start(); archives.add(zoie); } catch (Exception e) { log.error("Load index: " + dir + " failed.", e); } } log.info("load " + dirs.size() + " archived indices of " + getSizeBytes() + " bytes in " + (System.currentTimeMillis() - t0) + "ms"); return archives; }
protected List<ZoieSystem<R, D>> loadArchiveZoies() { List<ZoieSystem<R, D>> archives = new ArrayList<ZoieSystem<R, D>>(); long t0 = System.currentTimeMillis(); List<File> dirs = _dirMgrFactory.getAllArchivedDirs(); for (File dir : dirs) { try { DirectoryManager dirMgr = new DefaultDirectoryManager(dir, _dirMgrFactory.getMode()); ZoieSystem<R, D> zoie = new ZoieSystem<R, D>(dirMgr, _interpreter, _decorator, _zConfig); zoie.start(); archives.add(zoie); } catch (Exception e) { log.error("Load index: " + dir + " failed.", e); } } log.info("load " + dirs.size() + " archived indices of " + getSizeBytes() + " bytes in " + (System.currentTimeMillis() - t0) + "ms"); return archives; }
protected List<ZoieSystem<R, D>> loadArchiveZoies() { List<ZoieSystem<R, D>> archives = new ArrayList<ZoieSystem<R, D>>(); long t0 = System.currentTimeMillis(); List<File> dirs = _dirMgrFactory.getAllArchivedDirs(); for(File dir : dirs) { IndexReader reader; try { DirectoryManager dirMgr = new DefaultDirectoryManager(dir, _dirMgrFactory.getMode()); ZoieSystem<R, D> zoie = new ZoieSystem<R, D>(dirMgr, _interpreter, _decorator, _zConfig); zoie.start(); archives.add(zoie); } catch (Exception e) { log.error("Load index: " + dir + " failed.", e); } } log.info("load "+dirs.size()+" archived indices of " + getSizeBytes() +" bytes in " + (System.currentTimeMillis() - t0) + "ms"); return archives; }
DirectoryManager dirMgr = new DefaultDirectoryManager(partDir, _dirMode); ZoieSystem<BoboSegmentReader, T> zoie = new ZoieSystem<BoboSegmentReader, T>(dirMgr, _interpreter, _indexReaderDecorator, _zoieConfig);
static PerfTestHandler buildZoieHandler(File idxDir, Configuration topConf, Configuration conf) throws Exception { ZoieConfig zoieConfig = new ZoieConfig(); zoieConfig.setAnalyzer(new StandardAnalyzer(Version.LUCENE_43)); zoieConfig.setBatchSize(100000); zoieConfig.setBatchDelay(10000); zoieConfig.setMaxBatchSize(100000); zoieConfig.setRtIndexing(true); zoieConfig.setVersionComparator(ZoiePerfVersion.COMPARATOR); zoieConfig.setReadercachefactory(SimpleReaderCache.FACTORY); String modeConf = topConf.getString("perf.directory.type", "file"); DIRECTORY_MODE mode = modeMap.get(modeConf); if (mode == null) mode = DIRECTORY_MODE.SIMPLE; DirectoryManager dirMgr = new DefaultDirectoryManager(idxDir, mode); IndexReaderDecorator<IndexReader> indexReaderDecorator = new DefaultIndexReaderDecorator(); File queryFile = new File(topConf.getString("perf.query.file")); if (!queryFile.exists()) { throw new ConfigurationException(queryFile.getAbsolutePath() + " does not exist!"); } ZoieSystem<IndexReader, String> zoieSystem = new ZoieSystem<IndexReader, String>(dirMgr, interpreter, indexReaderDecorator, zoieConfig); SearchQueryHandler queryHandler = new SearchQueryHandler(queryFile, zoieSystem); return new PerfTestHandler(zoieSystem, queryHandler); }
/** * Creates a new ZoieSystem. * * @param idxDir * index directory, mandatory. * @param interpreter * data interpreter, mandatory. * @param indexReaderDecorator * index reader decorator,optional. If not specified, * {@link proj.zoie.impl.indexing.DefaultIndexReaderDecorator} is * used. * @param zoieConfig * configuration object */ @SuppressWarnings({ "deprecation", "unchecked" }) public ZoieSystem(File idxDir, ZoieIndexableInterpreter<D> interpreter, IndexReaderDecorator<R> indexReaderDecorator, ZoieConfig zoieConfig) { this(new DefaultDirectoryManager(idxDir), interpreter, indexReaderDecorator, zoieConfig .getDocidMapperFactory(), zoieConfig.getAnalyzer(), zoieConfig.getSimilarity(), zoieConfig .getBatchSize(), zoieConfig.getBatchDelay(), zoieConfig.isRtIndexing(), zoieConfig .getMaxBatchSize(), zoieConfig.getReadercachefactory(), (RAMIndexFactory<R>) zoieConfig .getRamIndexFactory(), zoieConfig.getVersionComparator(), zoieConfig.isSkipBadRecord()); readercache.setFreshness(zoieConfig.getFreshness()); }
/** * Creates a new ZoieSystem. * * @param idxDir * index directory, mandatory. * @param interpreter * data interpreter, mandatory. * @param indexReaderDecorator * index reader decorator,optional. If not specified, * {@link proj.zoie.impl.indexing.DefaultIndexReaderDecorator} is * used. * @param zoieConfig * configuration object */ @SuppressWarnings({ "deprecation", "unchecked" }) public ZoieSystem(File idxDir, ZoieIndexableInterpreter<D> interpreter, IndexReaderDecorator<R> indexReaderDecorator, ZoieConfig zoieConfig) { this(new DefaultDirectoryManager(idxDir), interpreter, indexReaderDecorator, zoieConfig .getDocidMapperFactory(), zoieConfig.getAnalyzer(), zoieConfig.getSimilarity(), zoieConfig .getBatchSize(), zoieConfig.getBatchDelay(), zoieConfig.isRtIndexing(), zoieConfig .getMaxBatchSize(), zoieConfig.getReadercachefactory(), (RAMIndexFactory<R>) zoieConfig .getRamIndexFactory(), zoieConfig.getVersionComparator(), zoieConfig.isSkipBadRecord()); readercache.setFreshness(zoieConfig.getFreshness()); }
/** * Creates a new ZoieSystem. * * @param idxDir * index directory, mandatory. * @param interpreter * data interpreter, mandatory. * @param indexReaderDecorator * index reader decorator,optional. If not specified, * {@link proj.zoie.impl.indexing.DefaultIndexReaderDecorator} is * used. * @param zoieConfig * configuration object */ public ZoieSystem(File idxDir, ZoieIndexableInterpreter<D> interpreter, IndexReaderDecorator<R> indexReaderDecorator, ZoieConfig zoieConfig) { this(new DefaultDirectoryManager(idxDir), interpreter, indexReaderDecorator, zoieConfig.getDocidMapperFactory(), zoieConfig.getAnalyzer(), zoieConfig.getSimilarity(), zoieConfig.getBatchSize(), zoieConfig .getBatchDelay(), zoieConfig.isRtIndexing(), zoieConfig .getMaxBatchSize(), zoieConfig.getReadercachefactory(), (RAMIndexFactory<R>)zoieConfig.getRamIndexFactory(),zoieConfig.getVersionComparator(), zoieConfig.isSkipBadRecord()); readercache.setFreshness(zoieConfig.getFreshness()); }