@Override public void ensureIndex(IndexField indexField, boolean isUnique, IndexField... valueFields) throws JasDBStorageException { BagConnector connector = RemoteConnectorFactory.createConnector(nodeInformation, BagConnector.class); KeyInfo keyInfo = new KeyInfoImpl(indexField, valueFields); connector.createIndex(context, instance, meta.getName(), new IndexDefinition(keyInfo.getKeyName(), keyInfo.keyAsHeader(), keyInfo.valueAsHeader(), -1), isUnique); }
@Override public RecordIterator readAllRecords(int limit) throws JasDBStorageException { IndexIterator indexIterator = index.getIndexIterator(); return new BtreeRecordIteratorImpl(indexIterator, keyInfo.getKeyNameMapper(), limit); }
protected DataBlock writeKey(TreeBlock treeBlock, Key key, DataBlock dataBlock) throws JasDBStorageException { return keyInfo.getKeyFactory().writeKey(key, dataBlock); }
private Set<SearchCondition> getSearchConditions(BlockOperation blockOperation, Index index) { KeyInfo keyInfo = index.getKeyInfo(); return blockOperation.getConditions(keyInfo.getKeyNameMapper(), keyInfo.getKeyFields()); } }
public static IndexHeader createHeader(DataBlock headerBlock, int pageSize, long count, KeyInfo keyInfo) throws JasDBStorageException, IOException { String headerInfo = keyInfo.keyAsHeader(); byte[] headerInfoBytes = headerInfo.getBytes(); String valueHeaderInfo = keyInfo.valueAsHeader(); byte[] valueHeaderInfoBytes = valueHeaderInfo.getBytes(); DataBlockHeader dataBlockHeader = headerBlock.getHeader(); dataBlockHeader.putInt(VERSION_HEADER_INDEX, INDEX_VERSION); dataBlockHeader.putInt(TYPE_HEADER_INDEX, BTREE_INDEX_TYPE_ID); dataBlockHeader.putInt(PAGE_SIZE_INDEX, pageSize); dataBlockHeader.putLong(COUNT_INDEX, count); headerBlock.reset(); headerBlock.writeBytes(headerInfoBytes); headerBlock.writeBytes(valueHeaderInfoBytes); headerBlock.flush(); return new BtreeIndexHeader(INDEX_VERSION, pageSize, headerBlock.size(), keyInfo, count); }
@Override public String getName() { return keyInfo.getKeyName(); }
private static boolean isIndexable(KeyInfo keyInfo, IndexableItem item) { for(String field : keyInfo.getKeyFields()) { if(!item.hasValue(field)) { return false; } } return true; }
@RequestMapping(value = "/Instances({instanceId})/Bags({bagName})/Indexes", method = POST, consumes = "application/json", produces = "application/json") public RestEntity writeEntry(@PathVariable String instanceId, @PathVariable String bagName, @RequestBody IndexEntry indexEntry) throws RestException { if(StringUtils.hasText(instanceId) && StringUtils.hasText(bagName)) { try { StorageService storageService = storageServiceFactory.getStorageService(instanceId, bagName); KeyInfo keyInfo = new KeyInfoImpl(indexEntry.getKeyHeader(), indexEntry.getValueHeader()); List<IndexField> indexFields = keyInfo.getIndexKeyFields(); List<IndexField> indexFieldValues = keyInfo.getIndexValueFields(); IndexField[] indexFieldValueArray = indexFieldValues.toArray(new IndexField[indexFieldValues.size()]); if (indexFields.size() > SINGLE_KEY_FIELD) { CompositeIndexField compositeIndexField = new SimpleCompositeIndexField(indexFields.toArray(new IndexField[indexFields.size()])); storageService.ensureIndex(compositeIndexField, indexEntry.isUniqueConstraint(), indexFieldValueArray); } else if (indexFields.size() == SINGLE_KEY_FIELD) { storageService.ensureIndex(indexFields.get(0), indexEntry.isUniqueConstraint(), indexFieldValueArray); } else { throw new RestException("Unable to create index, no key fields specified"); } return indexEntry; } catch (JasDBStorageException e) { throw new RestException("Unable to write index data: " + e.getMessage()); } } else { throw new RestException("No InstanceId or Bag Name where specified when creating index"); } }
public static IndexHeader loadAndValidateHeader(DataBlock dataBlock, KeyInfo keyInfo) throws JasDBStorageException, IOException { DataBlockHeader dataBlockHeader = dataBlock.getHeader(); int version = dataBlockHeader.getInt(VERSION_HEADER_INDEX); int type = dataBlockHeader.getInt(TYPE_HEADER_INDEX); int pageSize = dataBlockHeader.getInt(PAGE_SIZE_INDEX); long count = dataBlockHeader.getLong(COUNT_INDEX); DataBlockResult<byte[]> keyResult = dataBlock.loadBytes(0); DataBlockResult<byte[]> keyValueResult = dataBlock.loadBytes(keyResult.getNextOffset()); if(Arrays.binarySearch(supportedVersions, version) == -1) { throw new JasDBStorageException("Version of found index is not supported by this version"); } else if(type != BTREE_INDEX_TYPE_ID) { throw new JasDBStorageException("Index type is not supported or invalid"); } else if(!keyInfo.keyAsHeader().equals(new String(keyResult.getValue()))) { throw new JasDBStorageException("Key information in index does not match specification on the storage metadata"); } else if(!keyInfo.valueAsHeader().equals(new String(keyValueResult.getValue()))) { throw new JasDBStorageException("Key information in index does not match specification on the storage metadata"); } else { return new BtreeIndexHeader(version, pageSize, dataBlock.size(), keyInfo, count); } }
@Override public void shutdownIndexes() throws JasDBStorageException { LOG.info("Shutting down {} indexes", indexes.size()); for(String bagName : indexes.keySet()) { for(Index index : indexes.get(bagName).values()) { LOG.debug("Closing index: {} on bag: {}", index.getKeyInfo().getKeyName(), bagName); index.close(); } } indexes.clear(); }
public static boolean isAnyDataPresent(IndexableItem sEntity, Index index) { for(String indexField : index.getKeyInfo().getKeyFields()) { if(sEntity.hasValue(indexField)) { return true; } } return false; }
private Index createInStore(String bagName, KeyInfo keyInfo) throws JasDBStorageException { if(!indexes.containsKey(bagName)) { loadIndexes(bagName); } Map<String, Index> bagIndexes = this.indexes.get(bagName); if(bagIndexes != null && !bagIndexes.containsKey(keyInfo.getKeyName())) { File indexFile = createIndexFile(bagName, keyInfo.getKeyName(), false); try { Index index = new BTreeIndex(indexFile, keyInfo); configureIndex(IndexTypes.BTREE, index); IndexDefinition definition = new IndexDefinition(keyInfo.getKeyName(), keyInfo.keyAsHeader(), keyInfo.valueAsHeader(), index.getIndexType()); metadataStore.addBagIndex(instanceId, bagName, definition); bagIndexes.put(keyInfo.getKeyName(), index); return index; } catch(ConfigurationException e) { throw new JasDBStorageException("Unable to create index, configuration error", e); } } else if(bagIndexes != null){ return bagIndexes.get(keyInfo.getKeyName()); } else { return null; } }
private IndexSearchResultIteratorCollection doLeaveSearch(LeaveBlock leaveBlock, Key desiredKey) { List<Key> results = new ArrayList<>(1); if(leaveBlock.contains(desiredKey)) { results.add(leaveBlock.getKey(desiredKey)); } return new IndexSearchResultIteratorImpl(results, keyInfo.getKeyNameMapper().clone()); }
private synchronized void loadIndexes(final String bagName) throws JasDBStorageException { LOG.debug("Loading indexes for bag: {}", bagName); if(!indexes.containsKey(bagName)) { Bag bag = metadataStore.getBag(instanceId, bagName); if(bag != null) { Set<IndexDefinition> indexDefinitions = new HashSet<>(bag.getIndexDefinitions()); LOG.info("Found {} potential indexes for bag: {}", indexDefinitions.size(), bagName); Map<String, Index> bagIndexes = new HashMap<>(); for(IndexDefinition indexDefinition : indexDefinitions) { Index index = loadIndex(bagName, indexDefinition); bagIndexes.put(index.getKeyInfo().getKeyName(), index); } this.indexes.put(bagName, bagIndexes); } } }
protected KeyLoadResult loadKeyResult(TreeBlock treeBlock, int offset, DataBlock dataBlock) throws JasDBStorageException { return keyInfo.getKeyFactory().loadKey(offset, dataBlock); }
private SearchCondition handleEqualsToRange(EqualsCondition equalsCondition) { int nrOfKeys = keyInfo.getKeyFields().size(); if(nrOfKeys > 1) { //composite keys always use range search return new RangeCondition(equalsCondition.getKey(), true, equalsCondition.getKey(), true); } else { return equalsCondition; } }
@Override public void ensureIndex(CompositeIndexField queryFields, boolean isUnique, IndexField... valueFields) throws JasDBStorageException { BagConnector connector = RemoteConnectorFactory.createConnector(nodeInformation, BagConnector.class); KeyInfo keyInfo = new KeyInfoImpl(queryFields.getIndexFields(), Lists.newArrayList(valueFields)); connector.createIndex(context, instance, meta.getName(), new IndexDefinition(keyInfo.getKeyName(), keyInfo.keyAsHeader(), keyInfo.valueAsHeader(), -1), isUnique); }
private Optional<Long> getRecordPointer(UUIDKey documentId) { try { IndexSearchResultIterator resultIterator = index.searchIndex(new EqualsCondition(documentId), Index.NO_SEARCH_LIMIT); if (!resultIterator.isEmpty()) { Key key = resultIterator.next(); LongKey longKey = (LongKey) key.getKey(keyInfo.getKeyNameMapper(), "RECORD_POINTER"); return Optional.of(longKey.getKey()); } else { throw new RecordNotFoundException("Unable to read record: " + documentId + ", could not be found"); } } catch(JasDBStorageException e) { LOG.error("Unable lookup data record for document: {}", documentId); } return Optional.empty(); }
public static void doIndexInsert(Set<Key> keys, Index index) throws JasDBStorageException { LOG.trace("Inserting {} keys into index: {}", keys.size(), index.getKeyInfo().getKeyName()); StatRecord indexInsert = StatisticsMonitor.createRecord("bag:indexInsert"); for(Key key : keys) { index.insertIntoIndex(key); } indexInsert.stop(); }
public static void bulkInsertIndex(Index index, KeyInfo keyInfo, Iterator<IndexableItem> indexableItems) throws JasDBStorageException { KeyFactory keyFactory = keyInfo.getKeyFactory(); long counter = 0; int interval = 0; long start = System.currentTimeMillis(); while(indexableItems.hasNext()) { IndexableItem indexableItem = indexableItems.next(); if(keyFactory.isMultiValueKey(indexableItem)) { Set<Key> keys = keyFactory.createMultivalueKeys(indexableItem); for(Key key : keys) { index.insertIntoIndex(key); } } else { if(KeyUtil.isAnyDataPresent(indexableItem, index)) { Key key = keyFactory.createKey(indexableItem); index.insertIntoIndex(key); } } interval++; counter++; if(interval >= REPORT_INTERVAL) { LOG.info("Index: {} rebuild at: {} items", index, counter); interval = 0; } } long end = System.currentTimeMillis(); LOG.info("Finished rebuild for: {} items in: {}", counter, (end - start)); } }