@Override public void postCreate(String repositoryName) throws Exception { log.info("Performing repository post-creation actions for repository " + repositoryName); LilyHBaseSchema.getRecordTable(tableFactory, repositoryName, LilyHBaseSchema.Table.RECORD.name, false); }
private static void verifyIsRecordTable(HTableDescriptor htableDescriptor) { if (!isRecordTableDescriptor(htableDescriptor)) { throw new IllegalArgumentException(htableDescriptor.getNameAsString() + " is not a valid record table"); } }
public static HTableInterface getRecordTable(HBaseTableFactory tableFactory, String repositoryName, String tableName) throws IOException, InterruptedException { HTableInterface recordTable = tableFactory.getTable(createRecordTableDescriptor(repositoryName, tableName)); verifyIsRecordTable(recordTable.getTableDescriptor()); return recordTable; }
public BlobIncubatorMonitor(ZooKeeperItf zk, HBaseTableFactory tableFactory, TableManager tableManager, BlobManager blobManager, TypeManager typeManager, long minimalAge, long monitorDelay, long runDelay) throws IOException, InterruptedException { this.zk = zk; this.blobManager = blobManager; this.typeManager = typeManager; this.minimalAge = minimalAge; this.monitorDelay = monitorDelay; this.runDelay = runDelay; this.blobIncubatorTable = LilyHBaseSchema.getBlobIncubatorTable(tableFactory, false); this.tableFactory = tableFactory; this.tableManager = tableManager; }
public HBaseTypeManager(IdGenerator idGenerator, Configuration configuration, ZooKeeperItf zooKeeper, HBaseTableFactory hbaseTableFactory) throws IOException, InterruptedException, KeeperException, RepositoryException { super(zooKeeper); schemaCache = new LocalSchemaCache(zooKeeper, this); log = LogFactory.getLog(getClass()); this.idGenerator = idGenerator; this.typeTable = LilyHBaseSchema.getTypeTable(hbaseTableFactory); registerDefaultValueTypes(); schemaCache.start(); // The 'last' vtag should always exist in the system (at least, for everything index-related). Therefore we // create it here. try { FieldType fieldType = newFieldType(getValueType("LONG"), VersionTag.LAST, Scope.NON_VERSIONED); createFieldType(fieldType); } catch (FieldTypeExistsException e) { // ok } catch (ConcurrentUpdateTypeException e) { // ok, another lily-server is starting up and doing the same thing } }
public BlobManagerImpl(HBaseTableFactory hbaseTableFactory, BlobStoreAccessFactory blobStoreAccessFactory, boolean clientMode) throws IOException, InterruptedException { blobIncubatorTable = LilyHBaseSchema.getBlobIncubatorTable(hbaseTableFactory, clientMode); registry = new BlobStoreAccessRegistry(this); registry.setBlobStoreAccessFactory(blobStoreAccessFactory); }
private EventPublisher createEventPublisher(String repositoryName, String tableName) throws IOException, InterruptedException { HTableInterface recordTable = LilyHBaseSchema.getRecordTable(tableFactory, repositoryName, tableName); return new LilyHBaseEventPublisher(recordTable); } }
public Map<String, byte[]> getDefaultTimestampReusingTables() { Map<String, byte[]> defaultTables = Maps.newHashMap(DEFAULT_TIMESTAMP_REUSING_TABLES); try { HBaseAdmin hbaseAdmin = new HBaseAdmin(conf); HTableDescriptor[] descriptors = hbaseAdmin.listTables(); hbaseAdmin.close(); if (descriptors != null) { for (HTableDescriptor descriptor : descriptors) { if (LilyHBaseSchema.isRecordTableDescriptor(descriptor)) { defaultTables.put(descriptor.getNameAsString(), Bytes.toBytes("data")); } } } } catch (Exception e) { throw new RuntimeException("Error listing repository tables", e); } return Collections.unmodifiableMap(defaultTables); }
public static HTableInterface getRecordTable(HBaseTableFactory tableFactory, String repositoryName, String tableName, byte[][] splitKeys) throws IOException, InterruptedException { HTableInterface recordTable = tableFactory.getTable(createRecordTableDescriptor(repositoryName, tableName), splitKeys); verifyIsRecordTable(recordTable.getTableDescriptor()); return recordTable; }
@Override public RepositoryTable createTable(TableCreateDescriptor descriptor) throws InterruptedException, IOException { if (!RepoAndTableUtil.isValidTableName(descriptor.getName())) { throw new IllegalArgumentException(String.format("'%s' is not a valid table name. " + RepoAndTableUtil.VALID_NAME_EXPLANATION, descriptor.getName())); } if (tableExists(descriptor.getName())) { throw new IllegalArgumentException(String.format("Table '%s' already exists", descriptor.getName())); } LilyHBaseSchema.getRecordTable(tableFactory, repositoryName, descriptor.getName(), descriptor.getSplitKeys()); return new RepositoryTableImpl(repositoryName, descriptor.getName()); }
@Override public void dropTable(String tableName) throws InterruptedException, IOException { if (Table.RECORD.name.equals(tableName)) { throw new IllegalArgumentException("Can't delete the default record table"); } HBaseAdmin hbaseAdmin = new HBaseAdmin(configuration); String hbaseTableName = RepoAndTableUtil.getHBaseTableName(repositoryName, tableName); try { if (hbaseAdmin.tableExists(hbaseTableName) && LilyHBaseSchema.isRecordTableDescriptor(hbaseAdmin.getTableDescriptor(Bytes.toBytes(hbaseTableName)))) { hbaseAdmin.disableTable(hbaseTableName); hbaseAdmin.deleteTable(hbaseTableName); } else { throw new IllegalArgumentException( String.format("Table '%s' is not a valid record table (HBase table name: '%s')", tableName, hbaseTableName)); } } finally { hbaseAdmin.close(); } }
public static HTableInterface getRecordTable(HBaseTableFactory tableFactory, String repositoryName, String tableName, boolean clientMode) throws IOException, InterruptedException { HTableInterface recordTable = tableFactory.getTable(createRecordTableDescriptor(repositoryName, tableName), !clientMode); verifyIsRecordTable(recordTable.getTableDescriptor()); return recordTable; }
/** * Factory method for creation of a {@code BulkIngester} that operates on a non-default repository table. * * @param zkConnString connection string for ZooKeeper * @param timeout ZooKeeper session timeout * @param tableName name of the repository table to write to */ public static BulkIngester newBulkIngester(String zkConnString, int timeout, String repositoryName, String tableName, boolean bulkMode) { try { ZooKeeperItf zk = ZkUtil.connect(zkConnString, timeout); // we need a lily client for non bulk access LilyClient lilyClient = new LilyClient(zk); // we need an HBaseRepository for bulk access Configuration conf = HBaseConfiguration.create(); conf.set("hbase.zookeeper.quorum", zkConnString); HBaseTableFactory hbaseTableFactory = new HBaseTableFactoryImpl(conf); HBaseRepository hbaseRepository = createHBaseRepository(repositoryName, tableName, zk, conf, hbaseTableFactory); return new BulkIngester( lilyClient, hbaseRepository, LilyHBaseSchema.getRecordTable(hbaseTableFactory, hbaseRepository.getRepositoryName(), hbaseRepository.getTableName()), bulkMode); } catch (Exception e) { ExceptionUtil.handleInterrupt(e); throw new RuntimeException(e); } }
@Override public List<RepositoryTable> getTables() throws InterruptedException, IOException { HBaseAdmin hbaseAdmin = new HBaseAdmin(configuration); List<RepositoryTable> recordTables = Lists.newArrayList(); try { for (HTableDescriptor tableDescriptor : hbaseAdmin.listTables()) { if (LilyHBaseSchema.isRecordTableDescriptor(tableDescriptor) && RepoAndTableUtil.belongsToRepository(tableDescriptor.getNameAsString(), repositoryName)) { String name = RepoAndTableUtil.extractLilyTableName(repositoryName, tableDescriptor.getNameAsString()); recordTables.add(new RepositoryTableImpl(repositoryName, name)); } } } finally { hbaseAdmin.close(); } return recordTables; }
@Override protected Repository createRepository(RepoTableKey key) throws InterruptedException, RepositoryException { try { TableManager tableManager = new RemoteTableManager(key.getRepositoryName(), transceiver, avroConverter); HTableInterface nonAuthRecordTable = LilyHBaseSchema.getRecordTable(tableFactory, key.getRepositoryName(), key.getTableName(), true); HTableInterface recordTable = wrapWithAuthorization(nonAuthRecordTable); Repository repo = new RemoteRepository(key, transceiver, avroConverter, this, blobManager, recordTable, nonAuthRecordTable, tableManager, getRecordFactory()); if ("true".equals(System.getProperty("lilyclient.trace"))) { repo = TracingRepository.wrap(repo); } return repo; } catch (org.apache.hadoop.hbase.TableNotFoundException e) { throw new TableNotFoundException(key.getRepositoryName(), key.getTableName()); } catch (IOException e) { throw new RepositoryException(e); } }
private Result getBlobUsage(byte[] blobKey, SchemaId recordId, SchemaId fieldId) throws FieldTypeNotFoundException, TypeException, InterruptedException, IOException, RepositoryException { FieldTypeImpl fieldType = (FieldTypeImpl)typeManager.getFieldTypeById(fieldId); ValueType valueType = fieldType.getValueType(); Get get = new Get(recordId.getBytes()); get.addColumn(RecordCf.DATA.bytes, fieldType.getQualifier()); byte[] valueToCompare = Bytes.toBytes(valueType.getNestingLevel()); valueToCompare = Bytes.add(valueToCompare, blobKey); WritableByteArrayComparable valueComparator = new ContainsValueComparator(valueToCompare); Filter filter = new SingleColumnValueFilter(RecordCf.DATA.bytes, fieldType.getQualifier(), CompareOp.EQUAL, valueComparator); get.setFilter(filter); for (RepositoryTable repoTable : tableManager.getTables()) { HTableInterface recordTable = LilyHBaseSchema.getRecordTable(tableFactory, repoTable.getRepositoryName(), repoTable.getName()); Result result = recordTable.get(get); if (result != null && !result.isEmpty()) { return result; } } return null; } }
@Override protected Repository createRepository(RepoTableKey key) throws InterruptedException, RepositoryException { TableManager tableManager = new TableManagerImpl(key.getRepositoryName(), hbaseConf, hbaseTableFactory); try { HTableInterface nonAuthHTable = LilyHBaseSchema.getRecordTable(hbaseTableFactory, key.getRepositoryName(), key.getTableName(), true); HTableInterface htable = wrapWithAuthorization(nonAuthHTable); return new HBaseRepository(key, this, htable, nonAuthHTable, blobManager, tableManager, getRecordFactory()); } catch (org.apache.hadoop.hbase.TableNotFoundException e) { throw new TableNotFoundException(key.getRepositoryName(), key.getTableName()); } catch (IOException e) { throw new RepositoryException(e); } }