/** * This is the client-side hook that storm fires when a topology is added. * * @param topologyInfo topology info * @param stormConf configuration * @param stormTopology a storm topology */ @Override public void notify(TopologyInfo topologyInfo, Map stormConf, StormTopology stormTopology) { LOG.info("Collecting metadata for a new storm topology: {}", topologyInfo.get_name()); try { String user = getUser(topologyInfo.get_owner(), null); AtlasEntity topology = createTopologyInstance(topologyInfo, stormConf); AtlasEntitiesWithExtInfo entity = new AtlasEntitiesWithExtInfo(topology); addTopologyDataSets(stormTopology, topologyInfo.get_owner(), stormConf, topology, entity); // create the graph for the topology List<AtlasEntity> graphNodes = createTopologyGraph(stormTopology, stormTopology.get_spouts(), stormTopology.get_bolts()); if (CollectionUtils.isNotEmpty(graphNodes)) { // add the connection from topology to the graph topology.setAttribute("nodes", AtlasTypeUtil.getAtlasObjectIds(graphNodes)); for (AtlasEntity graphNode : graphNodes) { entity.addReferredEntity(graphNode); } } List<HookNotification> hookNotifications = Collections.singletonList(new EntityCreateRequestV2(user, entity)); notifyEntities(hookNotifications, null); } catch (Exception e) { throw new RuntimeException("Atlas hook is unable to process the topology.", e); } }
private void createOrUpdateTableInstance(HBaseOperationContext hbaseOperationContext) { AtlasEntity nameSpace = buildNameSpace(hbaseOperationContext); AtlasEntity table = buildTable(hbaseOperationContext, nameSpace); List<AtlasEntity> columnFamilies = buildColumnFamilies(hbaseOperationContext, nameSpace, table); table.setAttribute(ATTR_COLUMNFAMILIES, AtlasTypeUtil.getAtlasObjectIds(columnFamilies)); AtlasEntitiesWithExtInfo entities = new AtlasEntitiesWithExtInfo(table); entities.addReferredEntity(nameSpace); if (CollectionUtils.isNotEmpty(columnFamilies)) { for (AtlasEntity columnFamily : columnFamilies) { entities.addReferredEntity(columnFamily); } } switch (hbaseOperationContext.getOperation()) { case CREATE_TABLE: LOG.info("Create Table {}", table.getAttribute(REFERENCEABLE_ATTRIBUTE_NAME)); hbaseOperationContext.addMessage(new EntityCreateRequestV2(hbaseOperationContext.getUser(), entities)); break; case ALTER_TABLE: LOG.info("Modify Table {}", table.getAttribute(REFERENCEABLE_ATTRIBUTE_NAME)); hbaseOperationContext.addMessage(new EntityUpdateRequestV2(hbaseOperationContext.getUser(), entities)); break; } }
private void createOrUpdateTableInstance(HBaseOperationContext hbaseOperationContext) { AtlasEntity nameSpace = buildNameSpace(hbaseOperationContext); AtlasEntity table = buildTable(hbaseOperationContext, nameSpace); List<AtlasEntity> columnFamilies = buildColumnFamilies(hbaseOperationContext, nameSpace, table); table.setAttribute(ATTR_COLUMNFAMILIES, AtlasTypeUtil.getAtlasObjectIds(columnFamilies)); AtlasEntitiesWithExtInfo entities = new AtlasEntitiesWithExtInfo(table); entities.addReferredEntity(nameSpace); if (CollectionUtils.isNotEmpty(columnFamilies)) { for (AtlasEntity columnFamily : columnFamilies) { entities.addReferredEntity(columnFamily); } } switch (hbaseOperationContext.getOperation()) { case CREATE_TABLE: LOG.info("Create Table {}", table.getAttribute(REFERENCEABLE_ATTRIBUTE_NAME)); hbaseOperationContext.addMessage(new EntityCreateRequestV2(hbaseOperationContext.getUser(), entities)); break; case ALTER_TABLE: LOG.info("Modify Table {}", table.getAttribute(REFERENCEABLE_ATTRIBUTE_NAME)); hbaseOperationContext.addMessage(new EntityUpdateRequestV2(hbaseOperationContext.getUser(), entities)); break; } }
complexEntitiesInfo.addReferredEntity(e0Array); complexEntitiesInfo.addReferredEntity(e1Array); complexEntitiesInfo.addReferredEntity(e2Array); complexEntitiesInfo.addReferredEntity(e3Array); complexEntity.setAttribute("listOfEntities", entityList); complexEntitiesInfo.getReferredEntities().clear(); complexEntitiesInfo.addReferredEntity(e3Array_duplicate);
put("key22", getAtlasObjectId(e2MapValue)); }}; complexEntity.setAttribute("mapOfEntities", entityMap); complexEntitiesInfo.addReferredEntity(e0MapValue); complexEntitiesInfo.addReferredEntity(e1MapValue); complexEntitiesInfo.addReferredEntity(e2MapValue); entityMap.put("key33", getAtlasObjectId(e3MapValue)); complexEntity.setAttribute("mapOfEntities", entityMap); complexEntitiesInfo.addReferredEntity(e3MapValue); entityMap.remove("key00"); complexEntity.setAttribute("mapOfEntities", entityMap); complexEntitiesInfo.addReferredEntity(e3MapValue); entityMap.put("key33", getAtlasObjectId(e3MapValue)); complexEntity.setAttribute("mapOfEntities", entityMap); complexEntitiesInfo.addReferredEntity(e1MapValueEdit); entityMap.put("key33", getAtlasObjectId(e3MapValue)); complexEntity.setAttribute("mapOfEntities", entityMap); complexEntitiesInfo.addReferredEntity(e3MapValue); entityMap.clear(); complexEntity.setAttribute("mapOfEntities", entityMap); complexEntitiesInfo.addReferredEntity(e3MapValue);
entitiesInfo.addReferredEntity(columnEntity1); entitiesInfo.addReferredEntity(columnEntity2); entitiesInfo.addReferredEntity(columnEntity3); AtlasTypeUtil.getAtlasObjectId(col3))); AtlasEntitiesWithExtInfo tableInfo = new AtlasEntitiesWithExtInfo(tableEntity1); tableInfo.addReferredEntity(col1.getGuid(), col1); tableInfo.addReferredEntity(col2.getGuid(), col2); tableInfo.addReferredEntity(col3.getGuid(), col3);
entitiesInfo.addReferredEntity(dbEntity.getEntity()); entitiesInfo.addReferredEntity(col1); entitiesInfo.addReferredEntity(col2); tableEntity.setAttribute(TestUtilsV2.COLUMNS_ATTR_NAME, columns); entitiesInfo.addReferredEntity(col3); entitiesInfo.addReferredEntity(col4);
complexEntitiesInfo.addReferredEntity(e0Array); complexEntitiesInfo.addReferredEntity(e1Array); complexEntitiesInfo.addReferredEntity(e2Array); entityList.add(getAtlasObjectId(e3Array)); complexEntity.setAttribute("listOfEntities", entityList); complexEntitiesInfo.addReferredEntity(e3Array); entityList.add(getAtlasObjectId(e3Array_duplicate)); complexEntity.setAttribute("listOfEntities", entityList); complexEntitiesInfo.addReferredEntity(e3Array_duplicate);
entitiesInfo.addReferredEntity(columnEntity1); entitiesInfo.addReferredEntity(columnEntity2); entitiesInfo.addReferredEntity(columnEntity3); AtlasTypeUtil.getAtlasObjectId(col3))); AtlasEntitiesWithExtInfo tableInfo = new AtlasEntitiesWithExtInfo(tableEntity1); tableInfo.addReferredEntity(col1.getGuid(), col1); tableInfo.addReferredEntity(col2.getGuid(), col2); tableInfo.addReferredEntity(col3.getGuid(), col3);
entitiesInfo.addReferredEntity(dbEntity.getEntity()); entitiesInfo.addReferredEntity(col1); entitiesInfo.addReferredEntity(col2); tableEntity.setAttribute(TestUtilsV2.COLUMNS_ATTR_NAME, columns); entitiesInfo.addReferredEntity(col3); entitiesInfo.addReferredEntity(col4);
ret.addReferredEntity(hdfsPathEntity);
@Test(dependsOnMethods = "testCreate") public void testMapOfPrimitivesUpdate() throws Exception { AtlasEntity tableEntity = new AtlasEntity(tblEntity.getEntity()); AtlasEntitiesWithExtInfo entitiesInfo = new AtlasEntitiesWithExtInfo(tableEntity); entitiesInfo.addReferredEntity(tableEntity); //Add a new entry Map<String, String> paramsMap = (Map<String, String>) tableEntity.getAttribute("parametersMap"); paramsMap.put("newParam", "value"); init(); EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false); validateMutationResponse(response, EntityMutations.EntityOperation.UPDATE, 1); AtlasEntityHeader updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE); validateEntity(entitiesInfo, getEntityFromStore(updatedTable)); //Remove an entry paramsMap.remove("key1"); tableEntity.setAttribute("parametersMap", paramsMap); init(); response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false); validateMutationResponse(response, EntityMutations.EntityOperation.UPDATE, 1); updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE); validateEntity(entitiesInfo, getEntityFromStore(updatedTable)); }
protected AtlasEntity toReferencedHBaseTable(Table table, AtlasEntitiesWithExtInfo entities) { AtlasEntity ret = null; HBaseTableInfo hBaseTableInfo = new HBaseTableInfo(table); String hbaseNameSpace = hBaseTableInfo.getHbaseNameSpace(); String hbaseTableName = hBaseTableInfo.getHbaseTableName(); if (hbaseTableName != null) { AtlasEntity nsEntity = new AtlasEntity(HBASE_TYPE_NAMESPACE); nsEntity.setAttribute(ATTRIBUTE_NAME, hbaseNameSpace); nsEntity.setAttribute(ATTRIBUTE_CLUSTER_NAME, getClusterName()); nsEntity.setAttribute(ATTRIBUTE_QUALIFIED_NAME, getHBaseNameSpaceQualifiedName(getClusterName(), hbaseNameSpace)); ret = new AtlasEntity(HBASE_TYPE_TABLE); ret.setAttribute(ATTRIBUTE_NAME, hbaseTableName); ret.setAttribute(ATTRIBUTE_URI, hbaseTableName); ret.setAttribute(ATTRIBUTE_NAMESPACE, getObjectId(nsEntity)); ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, getHBaseTableQualifiedName(getClusterName(), hbaseNameSpace, hbaseTableName)); entities.addReferredEntity(nsEntity); entities.addEntity(ret); } return ret; }
@Override public void publish(SqoopJobDataPublisher.Data data) throws AtlasHookException { try { Configuration atlasProperties = ApplicationProperties.get(); String clusterName = atlasProperties.getString(ATLAS_CLUSTER_NAME, DEFAULT_CLUSTER_NAME); AtlasEntity entDbStore = toSqoopDBStoreEntity(data); AtlasEntity entHiveDb = toHiveDatabaseEntity(clusterName, data.getHiveDB()); AtlasEntity entHiveTable = data.getHiveTable() != null ? toHiveTableEntity(entHiveDb, data.getHiveTable()) : null; AtlasEntity entProcess = toSqoopProcessEntity(entDbStore, entHiveDb, entHiveTable, data, clusterName); AtlasEntitiesWithExtInfo entities = new AtlasEntitiesWithExtInfo(entProcess); entities.addReferredEntity(entDbStore); entities.addReferredEntity(entHiveDb); if (entHiveTable != null) { entities.addReferredEntity(entHiveTable); } HookNotification message = new EntityCreateRequestV2(AtlasHook.getUser(), entities); atlasHook.sendNotification(message); } catch(Exception e) { LOG.error("SqoopHook.publish() failed", e); throw new AtlasHookException("SqoopHook.publish() failed.", e); } }
@Test(dependsOnMethods = "testCreate") public void testMapOfPrimitivesUpdate() throws Exception { AtlasEntity tableEntity = new AtlasEntity(tblEntity.getEntity()); AtlasEntitiesWithExtInfo entitiesInfo = new AtlasEntitiesWithExtInfo(tableEntity); entitiesInfo.addReferredEntity(tableEntity); //Add a new entry Map<String, String> paramsMap = (Map<String, String>) tableEntity.getAttribute("parametersMap"); paramsMap.put("newParam", "value"); init(); EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false); validateMutationResponse(response, EntityMutations.EntityOperation.UPDATE, 1); AtlasEntityHeader updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE); validateEntity(entitiesInfo, getEntityFromStore(updatedTable)); //Remove an entry paramsMap.remove("key1"); init(); response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false); validateMutationResponse(response, EntityMutations.EntityOperation.UPDATE, 1); updatedTable = response.getFirstUpdatedEntityByTypeName(TABLE_TYPE); validateEntity(entitiesInfo, getEntityFromStore(updatedTable)); }
private void createOrUpdateColumnFamilyInstance(HBaseOperationContext hbaseOperationContext) { AtlasEntity nameSpace = buildNameSpace(hbaseOperationContext); AtlasEntity table = buildTable(hbaseOperationContext, nameSpace); AtlasEntity columnFamily = buildColumnFamily(hbaseOperationContext, hbaseOperationContext.gethColumnDescriptor(), nameSpace, table); AtlasEntitiesWithExtInfo entities = new AtlasEntitiesWithExtInfo(columnFamily); entities.addReferredEntity(nameSpace); entities.addReferredEntity(table); switch (hbaseOperationContext.getOperation()) { case CREATE_COLUMN_FAMILY: LOG.info("Create ColumnFamily {}", columnFamily.getAttribute(REFERENCEABLE_ATTRIBUTE_NAME)); hbaseOperationContext.addMessage(new EntityCreateRequestV2(hbaseOperationContext.getUser(), entities)); break; case ALTER_COLUMN_FAMILY: LOG.info("Alter ColumnFamily {}", columnFamily.getAttribute(REFERENCEABLE_ATTRIBUTE_NAME)); hbaseOperationContext.addMessage(new EntityUpdateRequestV2(hbaseOperationContext.getUser(), entities)); break; } }
private void createOrUpdateColumnFamilyInstance(HBaseOperationContext hbaseOperationContext) { AtlasEntity nameSpace = buildNameSpace(hbaseOperationContext); AtlasEntity table = buildTable(hbaseOperationContext, nameSpace); AtlasEntity columnFamily = buildColumnFamily(hbaseOperationContext, hbaseOperationContext.gethColumnDescriptor(), nameSpace, table); AtlasEntitiesWithExtInfo entities = new AtlasEntitiesWithExtInfo(columnFamily); entities.addReferredEntity(nameSpace); entities.addReferredEntity(table); switch (hbaseOperationContext.getOperation()) { case CREATE_COLUMN_FAMILY: LOG.info("Create ColumnFamily {}", columnFamily.getAttribute(REFERENCEABLE_ATTRIBUTE_NAME)); hbaseOperationContext.addMessage(new EntityCreateRequestV2(hbaseOperationContext.getUser(), entities)); break; case ALTER_COLUMN_FAMILY: LOG.info("Alter ColumnFamily {}", columnFamily.getAttribute(REFERENCEABLE_ATTRIBUTE_NAME)); hbaseOperationContext.addMessage(new EntityUpdateRequestV2(hbaseOperationContext.getUser(), entities)); break; } }
public void addReferredEntity(AtlasEntity entity) { if (entities == null) { entities = new AtlasEntitiesWithExtInfo(); } entities.addReferredEntity(entity); }
public void addReferredEntity(AtlasEntity entity) { if (entities == null) { entities = new AtlasEntitiesWithExtInfo(); } entities.addReferredEntity(entity); }
protected void addProcessedEntities(AtlasEntitiesWithExtInfo entitiesWithExtInfo) { for (AtlasEntity entity : context.getEntities()) { entitiesWithExtInfo.addReferredEntity(entity); } entitiesWithExtInfo.compact(); context.addToKnownEntities(entitiesWithExtInfo.getEntities()); if (entitiesWithExtInfo.getReferredEntities() != null) { context.addToKnownEntities(entitiesWithExtInfo.getReferredEntities().values()); } }