private Index instantiateIndex(String name, HTableInterface table) throws IOException, IndexNotFoundException { byte[] jsonData; try { jsonData = table.getTableDescriptor().getValue(INDEX_META_KEY); } catch (RuntimeException e) { if (e.getCause() != null && e.getCause() instanceof TableNotFoundException) { throw new IndexNotFoundException(name); } else { throw e; } } catch (TableNotFoundException e) { throw new IndexNotFoundException(name); } if (jsonData == null) { throw new IOException("Not a valid index table: " + name); } IndexDefinition indexDef = deserialize(name, jsonData); return new Index(table, indexDef); }
bkwdEntries.add(bkwdEntry); forwardIndex.addEntries(fwdEntries); backwardIndex.addEntries(bkwdEntries); fwdEntries.add(fwdEntry); backwardIndex.removeEntries(bkwdEntries); forwardIndex.removeEntries(fwdEntries);
createBackwardEntry(removed.getDependency(), parentRecordId, parentVtagId, null, removed.getMoreDimensionedVariants()); backwardDerefIndex.removeEntry(backwardEntry); forwardDerefIndex.addEntry(fwdEntry); createBackwardEntry(added.getDependency(), parentRecordId, parentVtagId, fields, added.getMoreDimensionedVariants()); backwardDerefIndex.addEntry(backwardEntry);
/** * Find the set of record ids (and corresponding version tags) on which a given record (in a given version tag) * depends. * * @param parentRecordId record id of the record to find dependencies for * @param vtag vtag of the record to find dependencies for * @return the record ids and vtags on which the given record depends */ Set<DependencyEntry> findDependencies(AbsoluteRecordId parentRecordId, SchemaId vtag) throws IOException { final Query query = new Query(); query.addEqualsCondition("dependant_recordid", parentRecordId.toBytes()); query.addEqualsCondition("dependant_vtag", vtag.getBytes()); final Set<DependencyEntry> result; final QueryResult queryResult = forwardDerefIndex.performQuery(query); if (queryResult.next() != null) { final byte[] serializedEntries = queryResult.getData(DEPENDENCIES_KEY); result = this.serializationUtil.deserializeDependenciesForward(serializedEntries); if (queryResult.next() != null) { throw new IllegalStateException( "Expected only a single matching entry in " + forwardDerefIndex.getDefinition().getName()); } } else { result = new HashSet<DependencyEntry>(); } // Not closed in finally block: avoid HBase contact when there could be connection problems. Closer.close(queryResult); return result; }
public Set<AbsoluteRecordId> getAbsoluteReferrers(AbsoluteRecordId record, SchemaId vtag, SchemaId sourceField) throws LinkIndexException, InterruptedException { long before = System.currentTimeMillis(); try { Query query = new Query(); query.addEqualsCondition("target", record.toBytes()); if (vtag != null) { query.addEqualsCondition("vtag", vtag.getBytes()); } if (sourceField != null) { query.addEqualsCondition("sourcefield", sourceField.getBytes()); } Set<AbsoluteRecordId> result = Sets.newHashSet(); QueryResult qr = backwardIndex.performQuery(query); byte[] id; while ((id = qr.next()) != null) { result.add(getIdGenerator().absoluteFromBytes(id)); } Closer.close( qr); // Not closed in finally block: avoid HBase contact when there could be connection problems. return result; } catch (IOException e) { throw new LinkIndexException("Error getting referrers for record '" + record + "', vtag '" + vtag + "', field '" + sourceField + "'", e); } finally { metrics.report(Action.GET_REFERRERS, System.currentTimeMillis() - before); } }
private IndexEntry createForwardEntry(AbsoluteRecordId parentRecordId, SchemaId parentVtagId, Collection<DependencyEntry> newDependencies) throws IOException { final IndexEntry fwdEntry = new IndexEntry(forwardDerefIndex.getDefinition()); fwdEntry.addField("dependant_recordid", parentRecordId.toBytes()); fwdEntry.addField("dependant_vtag", parentVtagId.getBytes()); // we do not really use the identifier... all we are interested in is in the data of the entry fwdEntry.setIdentifier(DUMMY_IDENTIFIER); // the data contains the dependencies of the dependant (master record ids and vtags) fwdEntry.addData(DEPENDENCIES_KEY, this.serializationUtil.serializeDependenciesForward(newDependencies)); return fwdEntry; }
public QueryResult performQuery(Query query) throws IOException { validateQuery(query); if (eqCond != null) { checkQueryValueType(fieldDef, eqCond.getValue()); final RowKey key = fieldDef.asRowKey(); key.setTermination(Termination.MUST); checkQueryValueType(fieldDef, fromValue); fromKeyComponents.add(fromValue); fromKeyStructBuilder.add(fieldDef.asRowKeyWithoutTermination()); checkQueryValueType(fieldDef, toValue); toKeyComponents.add(toValue); toKeyStructBuilder.add(fieldDef.asRowKeyWithoutTermination());
private Put createAddEntryPut(IndexEntry entry) throws IOException { byte[] indexKey = buildRowKey(entry); Put put = new Put(indexKey); Map<ByteArrayKey, byte[]> data = entry.getData(); if (data.size() > 0) { for (Map.Entry<ByteArrayKey, byte[]> item : data.entrySet()) { put.add(IndexDefinition.DATA_FAMILY, item.getKey().getKey(), item.getValue()); } } else { // HBase does not allow to create a row without columns, so add a dummy column put.add(IndexDefinition.DATA_FAMILY, DUMMY_QUALIFIER, DUMMY_VALUE); } return put; }
/** * Adds multiple entries to the index. Uses one HBase put so is more efficient * than adding each entry individually. */ public void addEntries(List<IndexEntry> entries) throws IOException { List<Put> puts = new ArrayList<Put>(); for (IndexEntry entry : entries) { entry.validate(); Put put = createAddEntryPut(entry); puts.add(put); } htable.put(puts); }
entries.add(entry); backwardIndex.removeEntries(entries); entries.add(entry); forwardIndex.removeEntries(entries); } catch (LinkIndexException e) { throw new LinkIndexException("Error deleting links for record '" + sourceRecord + "', vtag '" + vtag + "'",
public Set<AbsoluteRecordId> getForwardLinks(AbsoluteRecordId record, SchemaId vtag, SchemaId sourceField) throws LinkIndexException, InterruptedException { long before = System.currentTimeMillis(); try { Query query = new Query(); query.addEqualsCondition("source", record.toBytes()); if (vtag != null) { query.addEqualsCondition("vtag", vtag.getBytes()); } if (sourceField != null) { query.addEqualsCondition("sourcefield", sourceField.getBytes()); } Set<AbsoluteRecordId> result = new HashSet<AbsoluteRecordId>(); QueryResult qr = forwardIndex.performQuery(query); byte[] id; while ((id = qr.next()) != null) { result.add(getIdGenerator().absoluteFromBytes(id)); } Closer.close( qr); // Not closed in finally block: avoid HBase contact when there could be connection problems. return result; } catch (IOException e) { throw new LinkIndexException("Error getting forward links for record '" + record + "', vtag '" + vtag + "', field '" + sourceField + "'", e); } finally { metrics.report(Action.GET_FW_LINKS, System.currentTimeMillis() - before); } }
private IndexEntry createBackwardIndexEntry(SchemaId vtag, AbsoluteRecordId target, SchemaId sourceField) { IndexEntry entry = new IndexEntry(backwardIndex.getDefinition()); entry.addField("vtag", vtag.getBytes()); entry.addField("target", target.toBytes()); entry.addField("sourcefield", sourceField.getBytes()); entry.addData(SOURCE_FIELD_KEY, sourceField.getBytes()); return entry; }
public void removeEntries(List<IndexEntry> entries) throws IOException { ArgumentValidator.notNull(entries, "entries"); List<Delete> deletes = new ArrayList<Delete>(); for (IndexEntry entry : entries) { entry.validate(); byte[] indexKey = buildRowKey(entry); Delete delete = new Delete(indexKey); deletes.add(delete); } htable.delete(deletes); }
/** * Adds an entry to this index. See {@link IndexEntry} for more information. * * @param entry the values to be part of the index key, should correspond to the fields * defined in the {@link IndexDefinition} */ public void addEntry(IndexEntry entry) throws IOException { ArgumentValidator.notNull(entry, "entry"); entry.validate(); Put put = createAddEntryPut(entry); htable.put(put); }
entries.add(entry); backwardIndex.removeEntries(entries); entries.add(entry); forwardIndex.removeEntries(entries); } catch (LinkIndexException e) { throw new LinkIndexException("Error deleting links for record '" + sourceRecord + "'", e);
public Set<FieldedLink> getFieldedForwardLinks(AbsoluteRecordId record, SchemaId vtag) throws LinkIndexException, InterruptedException { long before = System.currentTimeMillis(); try { Query query = new Query(); query.addEqualsCondition("source", record.toBytes()); if (vtag != null) { query.addEqualsCondition("vtag", vtag.getBytes()); } Set<FieldedLink> result = new HashSet<FieldedLink>(); QueryResult qr = forwardIndex.performQuery(query); byte[] id; while ((id = qr.next()) != null) { SchemaId sourceField = getIdGenerator().getSchemaId(qr.getData(SOURCE_FIELD_KEY)); result.add(new FieldedLink(getIdGenerator().absoluteFromBytes(id), sourceField)); } Closer.close( qr); // Not closed in finally block: avoid HBase contact when there could be connection problems. return result; } catch (IOException e) { throw new LinkIndexException("Error getting forward links for record '" + record + "', vtag '" + vtag + "'", e); } finally { metrics.report(Action.GET_FW_LINKS, System.currentTimeMillis() - before); } }
private IndexEntry createForwardIndexEntry(SchemaId vtag, AbsoluteRecordId source, SchemaId sourceField) { IndexEntry entry = new IndexEntry(forwardIndex.getDefinition()); entry.addField("vtag", vtag.getBytes()); entry.addField("source", source.toBytes()); entry.addField("sourcefield", sourceField.getBytes()); entry.addData(SOURCE_FIELD_KEY, sourceField.getBytes()); entry.addData(VTAG_KEY, vtag.getBytes()); return entry; }
/** * Removes an entry from the index. The contents of the supplied * entry and the identifier should exactly match those supplied * when creating the index entry. */ public void removeEntry(IndexEntry entry) throws IOException { ArgumentValidator.notNull(entry, "entry"); entry.validate(); byte[] indexKey = buildRowKey(entry); Delete delete = new Delete(indexKey); htable.delete(delete); }
public Set<FieldedLink> getFieldedReferrers(RecordId record, SchemaId vtag) throws LinkIndexException, InterruptedException { long before = System.currentTimeMillis(); try { Query query = new Query(); query.addEqualsCondition("target", record.toBytes()); if (vtag != null) { query.addEqualsCondition("vtag", vtag.getBytes()); } Set<FieldedLink> result = new HashSet<FieldedLink>(); QueryResult qr = backwardIndex.performQuery(query); byte[] id; while ((id = qr.next()) != null) { SchemaId sourceField = getIdGenerator().getSchemaId(qr.getData(SOURCE_FIELD_KEY)); result.add(new FieldedLink(getIdGenerator().absoluteFromBytes(id), sourceField)); } Closer.close( qr); // Not closed in finally block: avoid HBase contact when there could be connection problems. return result; } catch (IOException e) { throw new LinkIndexException("Error getting referrers for record '" + record + "', vtag '" + vtag + "'", e); } finally { metrics.report(Action.GET_FIELDED_REFERRERS, System.currentTimeMillis() - before); } }
private IndexEntry createBackwardEntry(AbsoluteRecordId parentRecordId, AbsoluteRecordId dependantRecordId, SchemaId dependantVtagId, Set<SchemaId> fields, Set<String> moreDimensionedVariantProperties) throws IOException { final byte[] serializedVariantPropertiesPattern = this.serializationUtil.serializeVariantPropertiesPattern( this.serializationUtil.createVariantPropertiesPattern(parentRecordId.getRecordId().getVariantProperties(), moreDimensionedVariantProperties)); final IndexEntry bwdEntry = new IndexEntry(backwardDerefIndex.getDefinition()); bwdEntry.addField("dependency_masterrecordid", parentRecordId.getRecordId().getMaster().toBytes()); bwdEntry.addField("dependant_vtag", dependantVtagId.getBytes()); bwdEntry.addField("variant_properties_pattern", serializedVariantPropertiesPattern); // the identifier is the dependant which depends on the dependency bwdEntry.setIdentifier(dependantRecordId.toBytes()); // the fields which the dependant uses of the dependency (null if used for deleting the entry) if (fields != null) { bwdEntry.addData(FIELDS_KEY, this.serializationUtil.serializeFields(fields)); } return bwdEntry; }