@VisibleForTesting AtlasEntityWithExtInfo createOrUpdateTopic(String topic) throws Exception { String topicQualifiedName = getTopicQualifiedName(clusterName, topic); AtlasEntityWithExtInfo topicEntity = findTopicEntityInAtlas(topicQualifiedName); if (topicEntity == null) { System.out.println("Adding Kafka topic " + topic); LOG.info("Importing Kafka topic: {}", topicQualifiedName); AtlasEntity entity = getTopicEntity(topic, null); topicEntity = createEntityInAtlas(new AtlasEntityWithExtInfo(entity)); } else { System.out.println("Updating Kafka topic " + topic); LOG.info("Kafka topic {} already exists in Atlas. Updating it..", topicQualifiedName); AtlasEntity entity = getTopicEntity(topic, topicEntity.getEntity()); topicEntity.setEntity(entity); topicEntity = updateEntityInAtlas(topicEntity); } return topicEntity; }
protected List<AtlasEntityWithExtInfo> createOrUpdateColumnFamilies(String nameSpace, String tableName, String owner, HColumnDescriptor[] hcdts , AtlasEntity tableEntity) throws Exception { List<AtlasEntityWithExtInfo > ret = new ArrayList<>(); if (hcdts != null) { AtlasObjectId tableId = AtlasTypeUtil.getAtlasObjectId(tableEntity); for (HColumnDescriptor columnFamilyDescriptor : hcdts) { String cfName = columnFamilyDescriptor.getNameAsString(); String cfQualifiedName = getColumnFamilyQualifiedName(clusterName, nameSpace, tableName, cfName); AtlasEntityWithExtInfo cfEntity = findColumnFamiltyEntityInAtlas(cfQualifiedName); if (cfEntity == null) { LOG.info("Importing Column-family: " + cfQualifiedName); AtlasEntity entity = getColumnFamilyEntity(nameSpace, tableName, owner, columnFamilyDescriptor, tableId, null); cfEntity = createEntityInAtlas(new AtlasEntityWithExtInfo(entity)); } else { LOG.info("ColumnFamily already present in Atlas. Updating it..: " + cfQualifiedName); AtlasEntity entity = getColumnFamilyEntity(nameSpace, tableName, owner, columnFamilyDescriptor, tableId, cfEntity.getEntity()); cfEntity.setEntity(entity); cfEntity = updateEntityInAtlas(cfEntity); } ret.add(cfEntity); } } return ret; }
protected List<AtlasEntityWithExtInfo> createOrUpdateColumnFamilies(String nameSpace, String tableName, String owner, HColumnDescriptor[] hcdts , AtlasEntity tableEntity) throws Exception { List<AtlasEntityWithExtInfo > ret = new ArrayList<>(); if (hcdts != null) { AtlasObjectId tableId = AtlasTypeUtil.getAtlasObjectId(tableEntity); for (HColumnDescriptor columnFamilyDescriptor : hcdts) { String cfName = columnFamilyDescriptor.getNameAsString(); String cfQualifiedName = getColumnFamilyQualifiedName(clusterName, nameSpace, tableName, cfName); AtlasEntityWithExtInfo cfEntity = findColumnFamiltyEntityInAtlas(cfQualifiedName); if (cfEntity == null) { LOG.info("Importing Column-family: " + cfQualifiedName); AtlasEntity entity = getColumnFamilyEntity(nameSpace, tableName, owner, columnFamilyDescriptor, tableId, null); cfEntity = createEntityInAtlas(new AtlasEntityWithExtInfo(entity)); } else { LOG.info("ColumnFamily already present in Atlas. Updating it..: " + cfQualifiedName); AtlasEntity entity = getColumnFamilyEntity(nameSpace, tableName, owner, columnFamilyDescriptor, tableId, cfEntity.getEntity()); cfEntity.setEntity(entity); cfEntity = updateEntityInAtlas(cfEntity); } ret.add(cfEntity); } } return ret; }
protected AtlasEntityWithExtInfo createOrUpdateTable(String nameSpace, String tableName, AtlasEntity nameSapceEntity, HTableDescriptor htd, HColumnDescriptor[] hcdts) throws Exception { String owner = htd.getOwnerString(); String tblQualifiedName = getTableQualifiedName(clusterName, nameSpace, tableName); AtlasEntityWithExtInfo ret = findTableEntityInAtlas(tblQualifiedName); if (ret == null) { LOG.info("Importing Table: " + tblQualifiedName); AtlasEntity entity = getTableEntity(nameSpace, tableName, owner, nameSapceEntity, htd, null); ret = createEntityInAtlas(new AtlasEntityWithExtInfo(entity)); } else { LOG.info("Table already present in Atlas. Updating it..: " + tblQualifiedName); AtlasEntity entity = getTableEntity(nameSpace, tableName, owner, nameSapceEntity, htd, ret.getEntity()); ret.setEntity(entity); ret = updateEntityInAtlas(ret); } AtlasEntity tableEntity = ret.getEntity(); if (tableEntity != null) { List<AtlasEntityWithExtInfo> cfEntities = createOrUpdateColumnFamilies(nameSpace, tableName, owner, hcdts, tableEntity); List<AtlasObjectId> cfIDs = new ArrayList<>(); if (CollectionUtils.isNotEmpty(cfEntities)) { for (AtlasEntityWithExtInfo cfEntity : cfEntities) { cfIDs.add(AtlasTypeUtil.getAtlasObjectId(cfEntity.getEntity())); } } tableEntity.setAttribute(COLUMN_FAMILIES, cfIDs); } return ret; }
protected AtlasEntityWithExtInfo createOrUpdateTable(String nameSpace, String tableName, AtlasEntity nameSapceEntity, HTableDescriptor htd, HColumnDescriptor[] hcdts) throws Exception { String owner = htd.getOwnerString(); String tblQualifiedName = getTableQualifiedName(clusterName, nameSpace, tableName); AtlasEntityWithExtInfo ret = findTableEntityInAtlas(tblQualifiedName); if (ret == null) { LOG.info("Importing Table: " + tblQualifiedName); AtlasEntity entity = getTableEntity(nameSpace, tableName, owner, nameSapceEntity, htd, null); ret = createEntityInAtlas(new AtlasEntityWithExtInfo(entity)); } else { LOG.info("Table already present in Atlas. Updating it..: " + tblQualifiedName); AtlasEntity entity = getTableEntity(nameSpace, tableName, owner, nameSapceEntity, htd, ret.getEntity()); ret.setEntity(entity); ret = updateEntityInAtlas(ret); } AtlasEntity tableEntity = ret.getEntity(); if (tableEntity != null) { List<AtlasEntityWithExtInfo> cfEntities = createOrUpdateColumnFamilies(nameSpace, tableName, owner, hcdts, tableEntity); List<AtlasObjectId> cfIDs = new ArrayList<>(); if (CollectionUtils.isNotEmpty(cfEntities)) { for (AtlasEntityWithExtInfo cfEntity : cfEntities) { cfIDs.add(AtlasTypeUtil.getAtlasObjectId(cfEntity.getEntity())); } } tableEntity.setAttribute(COLUMN_FAMILIES, cfIDs); } return ret; }
protected AtlasEntityWithExtInfo createOrUpdateNameSpace(NamespaceDescriptor namespaceDescriptor) throws Exception { String nsName = namespaceDescriptor.getName(); String nsQualifiedName = getNameSpaceQualifiedName(clusterName, nsName); AtlasEntityWithExtInfo nsEntity = findNameSpaceEntityInAtlas(nsQualifiedName); if (nsEntity == null) { LOG.info("Importing NameSpace: " + nsQualifiedName); AtlasEntity entity = getNameSpaceEntity(nsName, null); nsEntity = createEntityInAtlas(new AtlasEntityWithExtInfo(entity)); } else { LOG.info("NameSpace already present in Atlas. Updating it..: " + nsQualifiedName); AtlasEntity entity = getNameSpaceEntity(nsName, nsEntity.getEntity()); nsEntity.setEntity(entity); nsEntity = updateEntityInAtlas(nsEntity); } return nsEntity; }
protected AtlasEntityWithExtInfo createOrUpdateNameSpace(NamespaceDescriptor namespaceDescriptor) throws Exception { String nsName = namespaceDescriptor.getName(); String nsQualifiedName = getNameSpaceQualifiedName(clusterName, nsName); AtlasEntityWithExtInfo nsEntity = findNameSpaceEntityInAtlas(nsQualifiedName); if (nsEntity == null) { LOG.info("Importing NameSpace: " + nsQualifiedName); AtlasEntity entity = getNameSpaceEntity(nsName, null); nsEntity = createEntityInAtlas(new AtlasEntityWithExtInfo(entity)); } else { LOG.info("NameSpace already present in Atlas. Updating it..: " + nsQualifiedName); AtlasEntity entity = getNameSpaceEntity(nsName, nsEntity.getEntity()); nsEntity.setEntity(entity); nsEntity = updateEntityInAtlas(nsEntity); } return nsEntity; }
protected AtlasEntityWithExtInfo toTableEntity(Table table) throws Exception { AtlasEntityWithExtInfo ret = new AtlasEntityWithExtInfo(); AtlasEntity entity = toTableEntity(table, ret); if (entity != null) { ret.setEntity(entity); } else { ret = null; } return ret; }