/** * Creates a new index in the default repository. * * @param indexDef definition of the index to be created * @throws InterruptedException * @throws IOException * * @throws IndexExistsException if an index with the same name already exists * @throws IndexNotFoundException very unlikely to occur here * @deprecated Use the version of this method where a repository name is supplied */ @Deprecated public synchronized Index getIndex(IndexDefinition indexDef) throws IOException, InterruptedException, IndexNotFoundException { return getIndex(RepoAndTableUtil.DEFAULT_REPOSITORY, indexDef); }
/** * Delete a DerefMap. This will delete the corresponding hbase tables. * * @param indexName name of the index to delete * @param hbaseConfiguration hbase configuration * @throws IOException * @throws IndexNotFoundException if the index doesn't exist (maybe it was already deleted?) */ public static void delete(final String indexName, final Configuration hbaseConfiguration) throws IOException, IndexNotFoundException { final IndexManager manager = new IndexManager(hbaseConfiguration); manager.deleteIndex(forwardIndexName(indexName)); manager.deleteIndex(backwardIndexName(indexName)); }
/** * Private constructor. Clients should use static factory methods {@link #delete(String, * org.apache.hadoop.conf.Configuration)} and {@link #create(String, Configuration, HBaseTableFactory, * IdGenerator)} */ private DerefMapHbaseImpl(final String owningRepoName, final String indexName, final Configuration hbaseConfiguration, final HBaseTableFactory tableFactory, final IdGenerator idGenerator) throws IndexNotFoundException, IOException, InterruptedException { this.serializationUtil = new DerefMapSerializationUtil(idGenerator); final IndexManager indexManager = new IndexManager(hbaseConfiguration, tableFactory); IndexDefinition forwardIndexDef = new IndexDefinition(forwardIndexName(indexName)); // For the record ID we use a variable length byte array field of which the first two bytes are fixed length // The first byte is actually the record identifier byte. // The second byte really is the first byte of the record id. We put this in the fixed length part // (safely because a record id should at least be a single byte long) because this prevents BCD encoding // on the first byte, thus making it easier to configure table splitting based on the original input. forwardIndexDef.addVariableLengthByteField("dependant_recordid", 2); forwardIndexDef.addByteField("dependant_vtag", DerefMapSerializationUtil.SCHEMA_ID_BYTE_LENGTH); forwardDerefIndex = indexManager.getIndex(owningRepoName, forwardIndexDef); IndexDefinition backwardIndexDef = new IndexDefinition(backwardIndexName(indexName)); // Same remark as in the forward index. backwardIndexDef.addVariableLengthByteField("dependency_masterrecordid", 2); backwardIndexDef.addByteField("dependant_vtag", DerefMapSerializationUtil.SCHEMA_ID_BYTE_LENGTH); backwardIndexDef.addVariableLengthByteField("variant_properties_pattern"); backwardDerefIndex = indexManager.getIndex(owningRepoName, backwardIndexDef); }
byte[] jsonData = serialize(indexDef); return instantiateIndex(indexDef.getName(), table);
@PostConstruct public void start() throws InterruptedException, KeeperException, IOException, IndexNotFoundException, RepositoryException { if (linkIndexEnabled) { // assure the subscription exists sepModel.addSubscriptionSilent("LinkIndexUpdater"); } else { // assure the subscription doesn't exist sepModel.removeSubscriptionSilent("LinkIndexUpdater"); } if (linkIndexEnabled) { IndexManager indexManager = new IndexManager(hbaseConf, tableFactory); LinkIndex linkIndex = new LinkIndex(indexManager, /* TODO multiple repositories */ repositoryManager); LinkIndexUpdater linkIndexUpdater = new LinkIndexUpdater(repositoryManager, linkIndex); sepConsumer = new SepConsumer("LinkIndexUpdater", 0L, linkIndexUpdater, threads, hostName, new ZooKeeperItfAdapter(zk), hbaseConf, new LilyPayloadExtractor()); sepConsumer.start(); } }
/** * Retrieves an Index. * * @throws IndexNotFoundException if the index does not exist */ public Index getIndex(String name) throws IOException, IndexNotFoundException { HTableInterface table; try { table = new LocalHTable(hbaseConf, name); } catch (RuntimeException e) { if (e.getCause() != null && e.getCause() instanceof TableNotFoundException) { throw new IndexNotFoundException(name); } else { throw e; } } catch (TableNotFoundException e) { throw new IndexNotFoundException(name); } return instantiateIndex(name, table); }
private Index instantiateIndex(String name, HTableInterface table) throws IOException, IndexNotFoundException { byte[] jsonData; try { jsonData = table.getTableDescriptor().getValue(INDEX_META_KEY); } catch (RuntimeException e) { if (e.getCause() != null && e.getCause() instanceof TableNotFoundException) { throw new IndexNotFoundException(name); } else { throw e; } } catch (TableNotFoundException e) { throw new IndexNotFoundException(name); } if (jsonData == null) { throw new IOException("Not a valid index table: " + name); } IndexDefinition indexDef = deserialize(name, jsonData); return new Index(table, indexDef); }
public LinkIndex(final IndexManager indexManager, RepositoryManager repositoryManager) throws IndexNotFoundException, IOException, InterruptedException { metrics = new LinkIndexMetrics("linkIndex"); this.repositoryManager = repositoryManager; // About the structure of these indexes: // - the vtag comes after the recordid because this way we can delete all // entries for a record without having to know the vtags under which they occur // - the sourcefield will often by optional in queries, that's why it comes last final int schemaIdByteLength = 16; // see SchemaIdImpl { IndexDefinition indexDef = new IndexDefinition("links-forward"); // For the record ID we use a variable length byte array field of which the first two bytes are fixed length // The first byte is actually the record identifier byte. // The second byte really is the first byte of the record id. We put this in the fixed length part // (safely because a record id should at least be a single byte long) because this prevents BCD encoding // on the first byte, thus making it easier to configure table splitting based on the original input. indexDef.addVariableLengthByteField("source", 2); indexDef.addByteField("vtag", schemaIdByteLength); indexDef.addByteField("sourcefield", schemaIdByteLength); forwardIndex = indexManager.getIndex(indexDef); } { IndexDefinition indexDef = new IndexDefinition("links-backward"); // Same remark as in the forwardIndex. indexDef.addVariableLengthByteField("target", 2); indexDef.addByteField("vtag", schemaIdByteLength); indexDef.addByteField("sourcefield", schemaIdByteLength); backwardIndex = indexManager.getIndex(indexDef); } }