private void recordAddedEntityGuids(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) { guids.add(entityWithExtInfo.getEntity().getGuid()); if(entityWithExtInfo.getReferredEntities() != null) { guids.addAll(entityWithExtInfo.getReferredEntities().keySet()); } }
private void processEntity(String guid, ExportContext context) throws AtlasBaseException { if (LOG.isDebugEnabled()) { LOG.debug("==> processEntity({})", guid); } if (!context.guidsProcessed.contains(guid)) { TraversalDirection direction = context.guidDirection.get(guid); AtlasEntityWithExtInfo entityWithExtInfo = entityGraphRetriever.toAtlasEntityWithExtInfo(guid); if(!context.lineageProcessed.contains(guid)) { context.result.getData().getEntityCreationOrder().add(entityWithExtInfo.getEntity().getGuid()); } addEntity(entityWithExtInfo, context); addTypes(entityWithExtInfo.getEntity(), context); context.guidsProcessed.add(entityWithExtInfo.getEntity().getGuid()); getConntedEntitiesBasedOnOption(entityWithExtInfo.getEntity(), context, direction); if(entityWithExtInfo.getReferredEntities() != null) { for (AtlasEntity e : entityWithExtInfo.getReferredEntities().values()) { addTypes(e, context); getConntedEntitiesBasedOnOption(e, context, direction); } context.guidsProcessed.addAll(entityWithExtInfo.getReferredEntities().keySet()); } } if (LOG.isDebugEnabled()) { LOG.debug("<== processEntity({})", guid); } }
private String[] getExtEntityExpectedValues(AtlasEntityWithExtInfo entityWithExtInfo) { String[] ret = new String[entityWithExtInfo.getReferredEntities().size()]; for (int i = 0; i < ret.length; i++) { String attrValue = (String) entityWithExtInfo.getReferredEntities().get(Integer.toString(i)).getAttribute(qualifiedName); ret[i] = attrValue.replace(lowerCaseCL1, lowerCaseCL2); } return ret; }
@Override @GraphTransaction public SchemaDetails getSchemaForHiveTableByGuid(final String guid) throws AtlasBaseException { if (StringUtils.isEmpty(guid)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST); } SchemaDetails ret = new SchemaDetails(); AtlasEntityType hive_column = atlasTypeRegistry.getEntityTypeByName("hive_column"); ret.setDataType(AtlasTypeUtil.toClassTypeDefinition(hive_column)); AtlasEntityWithExtInfo entityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(guid); AtlasEntity entity = entityWithExtInfo.getEntity(); AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(atlasTypeRegistry, AtlasPrivilege.ENTITY_READ, new AtlasEntityHeader(entity)), "read entity schema: guid=", guid); Map<String, AtlasEntity> referredEntities = entityWithExtInfo.getReferredEntities(); List<String> columnIds = getColumnIds(entity); if (MapUtils.isNotEmpty(referredEntities)) { List<Map<String, Object>> rows = referredEntities.entrySet() .stream() .filter(e -> isColumn(columnIds, e)) .map(e -> AtlasTypeUtil.toMap(e.getValue())) .collect(Collectors.toList()); ret.setRows(rows); } return ret; }
@Test(dependsOnMethods = "atT1_NewClassificationAttachedToTable_ReturnsChangedTable") public void atT2_NewClassificationAttachedToColumn_ReturnsChangedColumn() throws AtlasBaseException { final int expectedEntityCount = 1; AtlasEntity.AtlasEntityWithExtInfo tableEntity = entityStore.getById(TABLE_GUID); long preExportTableEntityTimestamp = tableEntity.getEntity().getUpdateTime().getTime(); entityStore.addClassifications(COLUMN_GUID_HIGH, ImmutableList.of(typeRegistry.getClassificationTypeByName("T1").createDefaultValue())); ZipSource source = runExportWithParameters(exportService, getIncrementalRequest(nextTimestamp)); AtlasEntity.AtlasEntityWithExtInfo entities = getEntities(source, expectedEntityCount); for (Map.Entry<String, AtlasEntity> entry : entities.getReferredEntities().entrySet()) { AtlasEntity entity = entry.getValue(); assertNotNull(entity.getGuid()); break; } long postUpdateTableEntityTimestamp = tableEntity.getEntity().getUpdateTime().getTime(); assertEquals(preExportTableEntityTimestamp, postUpdateTableEntityTimestamp); nextTimestamp = updateTimesampForNextIncrementalExport(source); }
@Test public void testCreateWithDuplicateGuids() throws Exception { init(); AtlasEntityWithExtInfo tblEntity2 = TestUtilsV2.createTableEntityDuplicatesV2(dbEntity.getEntity()); EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(tblEntity2), false); List<AtlasEntityHeader> createdEntities = response.getCreatedEntities(); assertTrue(CollectionUtils.isNotEmpty(createdEntities)); assertEquals(createdEntities.size(), 2); String tableGuid = createdEntities.get(0).getGuid(); AtlasEntityWithExtInfo tableEntity = entityStore.getById(tableGuid); assertEquals(tableEntity.getReferredEntities().size(), 1); List<AtlasObjectId> columns = (List<AtlasObjectId>) tableEntity.getEntity().getAttribute("columns"); assertEquals(columns.size(), 1); Set<AtlasObjectId> uniqueColumns = new HashSet<>(columns); assertTrue(uniqueColumns.size() == 1); }
@Test(dependsOnMethods = "atT0_ReturnsAllEntities") public void atT1_NewClassificationAttachedToTable_ReturnsChangedTable() throws AtlasBaseException { final int expectedEntityCount = 1; AtlasClassificationType ct = createNewClassification(); entityStore.addClassifications(TABLE_GUID, ImmutableList.of(ct.createDefaultValue())); AtlasExportRequest request = getIncrementalRequest(nextTimestamp); ZipSource source = runExportWithParameters(exportService, request); AtlasEntity.AtlasEntityWithExtInfo entities = getEntities(source, expectedEntityCount); AtlasEntity entity = null; for (Map.Entry<String, AtlasEntity> entry : entities.getReferredEntities().entrySet()) { entity = entry.getValue(); assertNotNull(entity); break; } nextTimestamp = updateTimesampForNextIncrementalExport(source); assertEquals(entity.getGuid(),TABLE_GUID); }
@Test public void exportWithoutLineage() { final int expectedEntityCount = 3; AtlasExportRequest request = getRequest(); ZipSource source = runExportWithParameters(exportService, request); AtlasEntity.AtlasEntityWithExtInfo entities = ZipFileResourceTestUtils.getEntities(source, expectedEntityCount); int count = 0; for (Map.Entry<String, AtlasEntity> entry : entities.getReferredEntities().entrySet()) { assertNotNull(entry.getValue()); if(entry.getValue().getTypeName().equals("hive_process")) { fail("Process entities should not be part of export!"); } count++; } assertEquals(count, expectedEntityCount); }
/** * Attribute passed by name is updated with the value passed. * @param entityWithExtInfo Entity to be updated * @param propertyName attribute name * @param objectId Value to be set for attribute */ private void updateAttribute(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo, String propertyName, AtlasObjectId objectId) { String value = EntityGraphMapper.getSoftRefFormattedValue(objectId); updateAttribute(entityWithExtInfo.getEntity(), propertyName, value); for (AtlasEntity e : entityWithExtInfo.getReferredEntities().values()) { updateAttribute(e, propertyName, value); } }
@Test public void atT0_ReturnsAllEntities() throws AtlasBaseException { final int expectedEntityCount = 2; AtlasExportRequest request = getIncrementalRequest(0); ZipSource source = runExportWithParameters(exportService, request); AtlasEntity.AtlasEntityWithExtInfo entities = getEntities(source, expectedEntityCount); int count = 0; for (Map.Entry<String, AtlasEntity> entry : entities.getReferredEntities().entrySet()) { assertNotNull(entry.getValue()); count++; } nextTimestamp = updateTimesampForNextIncrementalExport(source); assertEquals(count, expectedEntityCount); }
@Test public void transformEntityWithExtInfo() throws AtlasBaseException { addColumnTransform(transform); AtlasEntityWithExtInfo entityWithExtInfo = getAtlasEntityWithExtInfo(); AtlasEntity entity = entityWithExtInfo.getEntity(); String attrValue = (String) entity.getAttribute(qualifiedName); String[] expectedValues = getExtEntityExpectedValues(entityWithExtInfo); transform.apply(entityWithExtInfo); assertEquals(entityWithExtInfo.getEntity().getAttribute(qualifiedName), applyDefaultTransform(attrValue)); for (int i = 0; i < expectedValues.length; i++) { assertEquals(entityWithExtInfo.getReferredEntities().get(Integer.toString(i)).getAttribute(qualifiedName), expectedValues[i]); } }
private void addEntity(AtlasEntityWithExtInfo entity, ExportContext context) throws AtlasBaseException { if(context.sink.hasEntity(entity.getEntity().getGuid())) { return; } context.sink.add(entity); context.result.incrementMeticsCounter(String.format("entity:%s", entity.getEntity().getTypeName())); if(entity.getReferredEntities() != null) { for (AtlasEntity e: entity.getReferredEntities().values()) { context.result.incrementMeticsCounter(String.format("entity:%s", e.getTypeName())); } } context.result.incrementMeticsCounter("entity:withExtInfo"); context.reportProgress(); }
public AtlasEntity.AtlasEntityWithExtInfo apply(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) throws AtlasBaseException { if (entityWithExtInfo != null) { apply(entityWithExtInfo.getEntity()); if(MapUtils.isNotEmpty(entityWithExtInfo.getReferredEntities())) { for (AtlasEntity e : entityWithExtInfo.getReferredEntities().values()) { apply(e); } } } return entityWithExtInfo; }
private String[] getExtEntityExpectedValues(AtlasEntityWithExtInfo entityWithExtInfo) { String[] ret = new String[entityWithExtInfo.getReferredEntities().size()]; for (int i = 0; i < ret.length; i++) { String attrValue = (String) entityWithExtInfo.getReferredEntities().get(Integer.toString(i)).getAttribute(qualifiedName); ret[i] = attrValue.replace(lowerCaseCL1, lowerCaseCL2); } return ret; }
private void assertTag(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo, String tagName) { if(entityWithExtInfo.getReferredEntities() == null || entityWithExtInfo.getReferredEntities().size() == 0) { return; } for (AtlasEntity entity : entityWithExtInfo.getReferredEntities().values()) { assertTag(entity, tagName); } }
public Referenceable getReferenceable(AtlasEntity.AtlasEntityWithExtInfo entity) throws AtlasBaseException { AtlasFormatConverter.ConverterContext ctx = new AtlasFormatConverter.ConverterContext(); ctx.addEntity(entity.getEntity()); for(Map.Entry<String, AtlasEntity> entry : entity.getReferredEntities().entrySet()) { ctx.addEntity(entry.getValue()); } return getReferenceable(entity.getEntity(), ctx); }
private void clearRelationshipAttributes(AtlasEntityWithExtInfo entity) { if (entity != null) { clearRelationshipAttributes(entity.getEntity()); if (entity.getReferredEntities() != null) { clearRelationshipAttributes(entity.getReferredEntities().values()); } } }
private void clearRelationshipAttributes(AtlasEntityWithExtInfo entity) { if (entity != null) { clearRelationshipAttributes(entity.getEntity()); if (entity.getReferredEntities() != null) { clearRelationshipAttributes(entity.getReferredEntities().values()); } } }
private void clearRelationshipAttributes(AtlasEntityWithExtInfo entity) { if (entity != null) { clearRelationshipAttributes(entity.getEntity()); if (entity.getReferredEntities() != null) { clearRelationshipAttributes(entity.getReferredEntities().values()); } } }