public static void doIndexInsert(Set<Key> keys, Index index) throws JasDBStorageException { LOG.trace("Inserting {} keys into index: {}", keys.size(), index.getKeyInfo().getKeyName()); StatRecord indexInsert = StatisticsMonitor.createRecord("bag:indexInsert"); for(Key key : keys) { index.insertIntoIndex(key); } indexInsert.stop(); }
@Override public void shutdownIndexes() throws JasDBStorageException { LOG.info("Shutting down {} indexes", indexes.size()); for(String bagName : indexes.keySet()) { for(Index index : indexes.get(bagName).values()) { LOG.debug("Closing index: {} on bag: {}", index.getKeyInfo().getKeyName(), bagName); index.close(); } } indexes.clear(); }
@Override public void flush() throws JasDBStorageException { index.flushIndex(); }
private RestEntity loadIndexes(String instanceId, String bagName) throws JasDBException { IndexManager indexManager = getIndexManager(instanceId); StatRecord getIndexCounter = StatisticsMonitor.createRecord("getIndexes"); Map<String, Index> indexes = indexManager.getIndexes(bagName); getIndexCounter.stop(); List<IndexEntry> indexEntries = new ArrayList<>(indexes.size()); for(Index index : indexes.values()) { KeyInfo keyInfo = index.getKeyInfo(); IndexEntry entry = new IndexEntry(keyInfo.getKeyName(), keyInfo.keyAsHeader(), keyInfo.valueAsHeader(), index.hasUniqueConstraint(), index.getIndexType()); entry.setMemorySize(index.getMemoryManager().getTotalMemoryUsage()); indexEntries.add(entry); } return new IndexCollection(indexEntries); }
} else { index.insertIntoIndex(key); index.removeFromIndex(removedKey); index.removeFromIndex(key);
remainingFields.removeAll(bestIndexMatch.getKeyInfo().getKeyFields()); for(SearchCondition condition : conditions) { StatRecord record = StatisticsMonitor.createRecord("bag:search:indexcondition"); IndexSearchResultIteratorCollection indexResults = bestIndexMatch.searchIndex(condition, limit); record.stop();
@Override public void removeIndex(String bagName, String keyName) throws JasDBStorageException { Index index = getIndex(bagName, keyName); if(index != null) { Map<String, Index> bagIndexes = indexes.get(bagName); bagIndexes.remove(keyName); KeyInfo keyInfo = index.getKeyInfo(); IndexDefinition definition = new IndexDefinition(keyInfo.getKeyName(), keyInfo.keyAsHeader(), keyInfo.valueAsHeader(), index.getIndexType()); metadataStore.removeBagIndex(instanceId, bagName, definition); index.removeIndex(); } else { throw new JasDBStorageException("Could not remove index, does not exist"); } }
private Optional<Long> getRecordPointer(UUIDKey documentId) { try { IndexSearchResultIterator resultIterator = index.searchIndex(new EqualsCondition(documentId), Index.NO_SEARCH_LIMIT); if (!resultIterator.isEmpty()) { Key key = resultIterator.next(); LongKey longKey = (LongKey) key.getKey(keyInfo.getKeyNameMapper(), "RECORD_POINTER"); return Optional.of(longKey.getKey()); } else { throw new RecordNotFoundException("Unable to read record: " + documentId + ", could not be found"); } } catch(JasDBStorageException e) { LOG.error("Unable lookup data record for document: {}", documentId); } return Optional.empty(); }
public static void bulkInsertIndex(Index index, KeyInfo keyInfo, Iterator<IndexableItem> indexableItems) throws JasDBStorageException { KeyFactory keyFactory = keyInfo.getKeyFactory(); long counter = 0; int interval = 0; long start = System.currentTimeMillis(); while(indexableItems.hasNext()) { IndexableItem indexableItem = indexableItems.next(); if(keyFactory.isMultiValueKey(indexableItem)) { Set<Key> keys = keyFactory.createMultivalueKeys(indexableItem); for(Key key : keys) { index.insertIntoIndex(key); } } else { if(KeyUtil.isAnyDataPresent(indexableItem, index)) { Key key = keyFactory.createKey(indexableItem); index.insertIntoIndex(key); } } interval++; counter++; if(interval >= REPORT_INTERVAL) { LOG.info("Index: {} rebuild at: {} items", index, counter); interval = 0; } } long end = System.currentTimeMillis(); LOG.info("Finished rebuild for: {} items in: {}", counter, (end - start)); } }
index.insertIntoIndex(insertKey); } catch(JasDBStorageException e) { index.updateKey(key, insertKey);
public static boolean isAnyDataPresent(IndexableItem sEntity, Index index) { for(String indexField : index.getKeyInfo().getKeyFields()) { if(sEntity.hasValue(indexField)) { return true; } } return false; }
private void initializeIndex(Index index) throws JasDBStorageException { KeyInfo keyInfo = index.getKeyInfo(); IndexDefinition definition = new IndexDefinition(keyInfo.getKeyName(), keyInfo.keyAsHeader(), keyInfo.valueAsHeader(), index.getIndexType()); if(metadataStore.containsIndex(instanceId, bagName, definition)) { try { indexRebuilder.submit(new IndexScanAndRecovery(index, getRecordWriter().readAllRecords(), true)).get(); } catch(ExecutionException | InterruptedException e) { throw new JasDBStorageException("Unable to initialize index, index rebuild failed", e); } } else { throw new JasDBStorageException("Cannot initialize index, does not exist in store"); } }
@Override public void removeRecord(UUIDKey documentId) throws JasDBStorageException { index.removeFromIndex(documentId); }
@Override public void closeWriter() throws JasDBStorageException { this.index.close(); }
@Override public void openWriter() throws JasDBStorageException { this.keyInfo = new KeyInfoImpl(new SimpleIndexField("__ID", new UUIDKeyType()), new SimpleIndexField("data", new DataKeyType())); try { this.index = new BTreeIndex(recordStorageLocation, keyInfo); this.index.openIndex(); } catch(OverlappingFileLockException e) { throw new RecordStoreInUseException("Record datastore: " + recordStorageLocation + " is already in use, cannot be opened"); } }
@Override public void updateRecord(UUIDKey documentId, ClonableDataStream dataStream) throws JasDBStorageException { Key documentKey = documentId.cloneKey(); documentKey.addKey(keyInfo.getKeyNameMapper(), "data", new DataKey(dataStream)); index.updateKey(documentId, documentKey); } }
private Index createInStore(String bagName, KeyInfo keyInfo) throws JasDBStorageException { if(!indexes.containsKey(bagName)) { loadIndexes(bagName); } Map<String, Index> bagIndexes = this.indexes.get(bagName); if(bagIndexes != null && !bagIndexes.containsKey(keyInfo.getKeyName())) { File indexFile = createIndexFile(bagName, keyInfo.getKeyName(), false); try { Index index = new BTreeIndex(indexFile, keyInfo); configureIndex(IndexTypes.BTREE, index); IndexDefinition definition = new IndexDefinition(keyInfo.getKeyName(), keyInfo.keyAsHeader(), keyInfo.valueAsHeader(), index.getIndexType()); metadataStore.addBagIndex(instanceId, bagName, definition); bagIndexes.put(keyInfo.getKeyName(), index); return index; } catch(ConfigurationException e) { throw new JasDBStorageException("Unable to create index, configuration error", e); } } else if(bagIndexes != null){ return bagIndexes.get(keyInfo.getKeyName()); } else { return null; } }
Set<Key> keys = keyFactory.createMultivalueKeys(indexableItem); for(Key key : keys) { if(index.searchIndex(new EqualsCondition(key), Index.NO_SEARCH_LIMIT).isEmpty()) { isInIndex = false; break; isInIndex = !index.searchIndex(new EqualsCondition(keyFactory.createKey(indexableItem)), Index.NO_SEARCH_LIMIT).isEmpty();
@Override public void writeRecord(UUIDKey documentId, ClonableDataStream dataStream) throws JasDBStorageException { Key documentKey = documentId.cloneKey(); documentKey.addKey(keyInfo.getKeyNameMapper(), "data", new DataKey(dataStream)); StatRecord insertDataRecordIndex = StatisticsMonitor.createRecord("insert:data:record"); index.insertIntoIndex(documentKey); insertDataRecordIndex.stop(); }
private synchronized void loadIndexes(final String bagName) throws JasDBStorageException { LOG.debug("Loading indexes for bag: {}", bagName); if(!indexes.containsKey(bagName)) { Bag bag = metadataStore.getBag(instanceId, bagName); if(bag != null) { Set<IndexDefinition> indexDefinitions = new HashSet<>(bag.getIndexDefinitions()); LOG.info("Found {} potential indexes for bag: {}", indexDefinitions.size(), bagName); Map<String, Index> bagIndexes = new HashMap<>(); for(IndexDefinition indexDefinition : indexDefinitions) { Index index = loadIndex(bagName, indexDefinition); bagIndexes.put(index.getKeyInfo().getKeyName(), index); } this.indexes.put(bagName, bagIndexes); } } }