public TableId createHTableId(NamespaceId namespace, String tableName) throws IOException { return TableId.from(getHBaseNamespace(namespace), tableName); }
public TableId createHTableId(NamespaceId namespace, String tableName) throws IOException { return TableId.from(getHBaseNamespace(namespace), tableName); }
/** * Returns a map of HBase to CDAP namespace. This is required when we want to report metrics for HBase tables where * it is run a separate service and reads the table metrics and reports it. There we need to translate the hbase * namespace to cdap namespace for metrics to make sense from CDAP perspective. This is also used during upgrade * step where we want to construct the correct {@link DatasetAdmin} for each dataset. * * @return map of hbase namespace to cdap namespace * @throws IOException if there was an error getting the {@link NamespaceMeta} of all the namespaces */ public Map<String, String> getHBaseToCDAPNamespaceMap() throws IOException { Map<String, String> reverseMap = new HashMap<>(); if (namespaceQueryAdmin == null) { throw new IOException("NamespaceQueryAdmin is not set and a reverseLookupMap was requested."); } try { List<NamespaceMeta> namespaceMetas = namespaceQueryAdmin.list(); for (NamespaceMeta namespaceMeta : namespaceMetas) { String hbaseNamespace = getHBaseNamespace(namespaceMeta); reverseMap.put(hbaseNamespace, namespaceMeta.getName()); } } catch (Exception ex) { throw new IOException("NamespaceQueryAdmin lookup to list all NamespaceMetas failed", ex); } return ImmutableMap.copyOf(reverseMap); }
/** * Returns a map of HBase to CDAP namespace. This is required when we want to report metrics for HBase tables where * it is run a separate service and reads the table metrics and reports it. There we need to translate the hbase * namespace to cdap namespace for metrics to make sense from CDAP perspective. This is also used during upgrade * step where we want to construct the correct {@link DatasetAdmin} for each dataset. * * @return map of hbase namespace to cdap namespace * @throws IOException if there was an error getting the {@link NamespaceMeta} of all the namespaces */ public Map<String, String> getHBaseToCDAPNamespaceMap() throws IOException { Map<String, String> reverseMap = new HashMap<>(); if (namespaceQueryAdmin == null) { throw new IOException("NamespaceQueryAdmin is not set and a reverseLookupMap was requested."); } try { List<NamespaceMeta> namespaceMetas = namespaceQueryAdmin.list(); for (NamespaceMeta namespaceMeta : namespaceMetas) { String hbaseNamespace = getHBaseNamespace(namespaceMeta); reverseMap.put(hbaseNamespace, namespaceMeta.getName()); } } catch (Exception ex) { throw new IOException("NamespaceQueryAdmin lookup to list all NamespaceMetas failed", ex); } return ImmutableMap.copyOf(reverseMap); }
/** * Creates HBase namespace for the cdap system namespace. */ private void createSystemHBaseNamespace() { HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get(); try (HBaseDDLExecutor ddlExecutor = new HBaseDDLExecutorFactory(cConf, hConf).get()) { ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(NamespaceId.SYSTEM)); } catch (IOException e) { throw Throwables.propagate(e); } }
public String getHBaseNamespace(NamespaceId namespaceId) throws IOException { // Convert CDAP Namespace to HBase namespace if (NamespaceId.SYSTEM.equals(namespaceId) || NamespaceId.CDAP.equals(namespaceId) || NamespaceId.DEFAULT.equals(namespaceId)) { return toCDAPManagedHBaseNamespace(namespaceId); } if (namespaceQueryAdmin == null) { throw new IOException(String.format("NamespaceQueryAdmin is not set and a non-reserved namespace " + "lookup is requested. Namespace %s", namespaceId.getNamespace())); } try { return getHBaseNamespace(namespaceQueryAdmin.get(namespaceId)); } catch (Exception ex) { throw new IOException(String.format("NamespaceQueryAdmin lookup to get NamespaceMeta failed. " + "Can't find mapping for %s", namespaceId.getNamespace()), ex); } }
private Map<TableId, Future<?>> upgradeQueues(final NamespaceMeta namespaceMeta, ExecutorService executor, final HBaseAdmin admin) throws Exception { String hbaseNamespace = tableUtil.getHBaseNamespace(namespaceMeta); List<TableId> tableIds = tableUtil.listTablesInNamespace(admin, hbaseNamespace); List<TableId> stateStoreTableIds = Lists.newArrayList();
public String getHBaseNamespace(NamespaceId namespaceId) throws IOException { // Convert CDAP Namespace to HBase namespace if (NamespaceId.SYSTEM.equals(namespaceId) || NamespaceId.CDAP.equals(namespaceId) || NamespaceId.DEFAULT.equals(namespaceId)) { return toCDAPManagedHBaseNamespace(namespaceId); } if (namespaceQueryAdmin == null) { throw new IOException(String.format("NamespaceQueryAdmin is not set and a non-reserved namespace " + "lookup is requested. Namespace %s", namespaceId.getNamespace())); } try { return getHBaseNamespace(namespaceQueryAdmin.get(namespaceId)); } catch (Exception ex) { throw new IOException(String.format("NamespaceQueryAdmin lookup to get NamespaceMeta failed. " + "Can't find mapping for %s", namespaceId.getNamespace()), ex); } }
private Map<String, Future<?>> upgradeUserTables(final NamespaceMeta namespaceMeta, final ExecutorService executor, final HBaseDDLExecutor ddlExecutor) throws Exception { Map<String, Future<?>> futures = new HashMap<>(); String hBaseNamespace = hBaseTableUtil.getHBaseNamespace(namespaceMeta); try (HBaseAdmin hAdmin = new HBaseAdmin(hConf)) { for (final HTableDescriptor desc : hAdmin.listTableDescriptorsByNamespace(HTableNameConverter.encodeHBaseEntity(hBaseNamespace))) { Callable<Void> callable = new Callable<Void>() { @Override public Void call() throws Exception { if (isCDAPUserTable(desc)) { upgradeUserTable(desc); } else if (isStreamOrQueueTable(desc.getNameAsString())) { updateTableDesc(desc, ddlExecutor); } return null; } }; Future<?> future = executor.submit(ImpersonationUtils.createImpersonatingCallable(impersonator, namespaceMeta, callable)); futures.put(desc.getNameAsString(), future); } } return futures; }
@SuppressWarnings("ConstantConditions") @Override public void delete(NamespaceId namespaceId) throws IOException, ExploreException, SQLException { // delete namespace directory from filesystem super.delete(namespaceId); if (NamespaceId.DEFAULT.equals(namespaceId)) { return; } // delete HBase namespace NamespaceConfig namespaceConfig; try { namespaceConfig = namespaceQueryAdmin.get(namespaceId).getConfig(); } catch (Exception ex) { throw new IOException("Could not fetch custom HBase mapping.", ex); } if (!Strings.isNullOrEmpty(namespaceConfig.getHbaseNamespace())) { // custom namespace mapping is set for HBase, hence don't do anything during delete since the lifecycle of the // namespace will be managed by the user LOG.debug("Custom HBase mapping {} was found while deleting {}. Hence skipping deletion of HBase namespace", namespaceConfig.getHbaseNamespace(), namespaceId); return; } // delete HBase namespace String namespace = tableUtil.getHBaseNamespace(namespaceId); try (HBaseDDLExecutor executor = hBaseDDLExecutorFactory.get()) { executor.deleteNamespaceIfExists(namespace); } } }
@SuppressWarnings("ConstantConditions") @Override public void delete(NamespaceId namespaceId) throws IOException, ExploreException, SQLException { // delete namespace directory from filesystem super.delete(namespaceId); if (NamespaceId.DEFAULT.equals(namespaceId)) { return; } // delete HBase namespace NamespaceConfig namespaceConfig; try { namespaceConfig = namespaceQueryAdmin.get(namespaceId).getConfig(); } catch (Exception ex) { throw new IOException("Could not fetch custom HBase mapping.", ex); } if (!Strings.isNullOrEmpty(namespaceConfig.getHbaseNamespace())) { // custom namespace mapping is set for HBase, hence don't do anything during delete since the lifecycle of the // namespace will be managed by the user LOG.debug("Custom HBase mapping {} was found while deleting {}. Hence skipping deletion of HBase namespace", namespaceConfig.getHbaseNamespace(), namespaceId); return; } // delete HBase namespace String namespace = tableUtil.getHBaseNamespace(namespaceId); try (HBaseDDLExecutor executor = hBaseDDLExecutorFactory.get()) { executor.deleteNamespaceIfExists(namespace); } } }
private void deleteNamespace(String namespace) throws IOException { String hbaseNamespace = getTableUtil().getHBaseNamespace(new NamespaceId(namespace)); ddlExecutor.deleteNamespaceIfExists(hbaseNamespace); }
private TableId getTableId(String namespace, String tableName) throws IOException { HBaseTableUtil tableUtil = getTableUtil(); List<TableId> tableIds = tableUtil.listTablesInNamespace(hAdmin, tableUtil.getHBaseNamespace(new NamespaceId(namespace))); for (TableId tId : tableIds) { if (tId.getTableName().endsWith(tableName)) { return tId; } } return null; }
private void createNamespace(String namespace) throws IOException { String hbaseNamespace = getTableUtil().getHBaseNamespace(new NamespaceId(namespace)); ddlExecutor.createNamespaceIfNotExists(hbaseNamespace); }
@Override public void dropAllInNamespace(NamespaceId namespaceId) throws Exception { Set<QueueConstants.QueueType> queueTypes = EnumSet.of(QueueConstants.QueueType.QUEUE, QueueConstants.QueueType.SHARDED_QUEUE); try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) { for (QueueConstants.QueueType queueType : queueTypes) { // Note: The trailing "." is crucial, since otherwise nsId could match nsId1, nsIdx etc // It's important to keep config table enabled while disabling and dropping queue tables. final String queueTableNamePrefix = String.format("%s.%s.", NamespaceId.SYSTEM.getNamespace(), queueType); final String hbaseNamespace = tableUtil.getHBaseNamespace(namespaceId); final TableId configTableId = TableId.from(hbaseNamespace, getConfigTableName()); tableUtil.deleteAllInNamespace(ddlExecutor, hbaseNamespace, hConf, new Predicate<TableId>() { @Override public boolean apply(TableId tableId) { // It's a bit hacky here since we know how the Dataset System names tables return (tableId.getTableName().startsWith(queueTableNamePrefix)) && !tableId.equals(configTableId); } }); } } // Delete the state store in the namespace DatasetId id = getStateStoreId(namespaceId.getEntityName()); if (datasetFramework.hasInstance(id)) { datasetFramework.deleteInstance(id); } }
String hbaseNamespace = tableUtil.getHBaseNamespace(namespaceMeta);
String hbaseNamespace = tableUtil.getHBaseNamespace(namespaceMeta);
tableUtil.listTablesInNamespace(hAdmin, tableUtil.getHBaseNamespace(new NamespaceId("foo")))); Assert.assertEquals(fooNamespaceHTableIds, retrievedTableIds); tableUtil.deleteAllInNamespace(ddlExecutor, tableUtil.getHBaseNamespace(new NamespaceId("foo")), hAdmin.getConfiguration()); Assert.assertEquals(1, hAdmin.listTables().length);
tableUtil.deleteAllInNamespace(ddlExecutor, tableUtil.getHBaseNamespace(new NamespaceId("foonamespace")), hAdmin.getConfiguration(), new Predicate<TableId>() {
@Test public void testCustomNamespaceMap() throws Exception { final String tableName = "mytable"; Assert.assertEquals(HBASE_NS, getTableUtil().getHBaseNamespace(new NamespaceId(CDAP_NS))); Assert.assertFalse(getTableUtil().hasNamespace(hAdmin, HBASE_NS)); createNamespace(CDAP_NS); // Check if namespace was created - in actual run, custom namespaces' lifecycle is managed by the user and hence // we won't create the namespace Assert.assertTrue(getTableUtil().hasNamespace(hAdmin, HBASE_NS)); TableId tableId = TableId.from(CDAP_NS, tableName); create(tableId); List<TableId> actualTableIds = getTableUtil().listTablesInNamespace(hAdmin, HBASE_NS); Assert.assertEquals(1, actualTableIds.size()); getTableUtil().deleteAllInNamespace(ddlExecutor, HBASE_NS, hAdmin.getConfiguration()); actualTableIds = getTableUtil().listTablesInNamespace(hAdmin, HBASE_NS); Assert.assertTrue(actualTableIds.isEmpty()); deleteNamespace(CDAP_NS); Assert.assertFalse(getTableUtil().hasNamespace(hAdmin, HBASE_NS)); }