AtlasTypeDefGraphStoreV2 typeDefStore) throws AtlasBaseException { AtlasAttributeDef ret = new AtlasAttributeDef(); ret.setName((String) attribInfo.get("name")); ret.setTypeName((String) attribInfo.get("dataType")); ret.setIsUnique((Boolean) attribInfo.get("isUnique")); ret.setIsIndexable((Boolean) attribInfo.get("isIndexable")); ret.setIncludeInNotification((Boolean) attribInfo.get("includeInNotification")); ret.setDefaultValue((String) attribInfo.get("defaultValue")); ret.setDescription((String) attribInfo.get("description")); ret.setOptions(AtlasType.fromJson((String) attribInfo.get("options"), Map.class)); ret.addConstraint(new AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF)); ret.addConstraint(new AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{ put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, reverseAttributeName); ret.setIsOptional(true); ret.setValuesMinCount(0); } else { ret.setIsOptional(false); ret.setValuesMinCount(minCount.intValue()); ret.setCardinality(AtlasAttributeDef.Cardinality.SINGLE); ret.setValuesMaxCount(1); } else {
attribInfo.put("name", attributeDef.getName()); attribInfo.put("dataType", attributeDef.getTypeName()); attribInfo.put("isUnique", attributeDef.getIsUnique()); attribInfo.put("isIndexable", attributeDef.getIsIndexable()); attribInfo.put("includeInNotification", attributeDef.getIncludeInNotification()); attribInfo.put("isComposite", attribute.isOwnedRef()); attribInfo.put("reverseAttributeName", attribute.getInverseRefAttributeName()); attribInfo.put("defaultValue", attributeDef.getDefaultValue()); attribInfo.put("description", attributeDef.getDescription()); if(attributeDef.getOptions() != null) { attribInfo.put("options", AtlasType.toJson(attributeDef.getOptions())); if (attributeDef.getCardinality() == AtlasAttributeDef.Cardinality.SINGLE) { lower = attributeDef.getIsOptional() ? 0 : 1; upper = 1; } else { if(attributeDef.getIsOptional()) { lower = 0; } else { lower = attributeDef.getValuesMinCount() < 1 ? 1 : attributeDef.getValuesMinCount(); upper = attributeDef.getValuesMaxCount() < 2 ? Integer.MAX_VALUE : attributeDef.getValuesMaxCount(); multiplicity.put("lower", lower); multiplicity.put("upper", upper); multiplicity.put("isUnique", AtlasAttributeDef.Cardinality.SET.equals(attributeDef.getCardinality()));
assertTrue(createdTypesDef.getClassificationDefs().get(0).getAttributeDefs().get(0).getIsIndexable()); assertTrue(getBackFromCache.getAttributeDefs().get(0).getIsIndexable()); assertEquals(guid, getBackFromCache.getGuid()); assertNotNull(getBackFromCache.getCreatedBy()); testTypesDefFromJson.getClassificationDefs().get(0).getAttributeDefs().get(0).setIsIndexable(false); AtlasTypesDef updatedTypesDef = typeDefStore.updateTypesDef(testTypesDefFromJson); assertEquals("test_classification_11", updatedTypesDef.getClassificationDefs().get(0).getName()); assertFalse(updatedTypesDef.getClassificationDefs().get(0).getAttributeDefs().get(0).getIsIndexable()); assertEquals(guid, updatedTypesDef.getClassificationDefs().get(0).getGuid()); assertEquals(createdTime, updatedTypesDef.getClassificationDefs().get(0).getCreateTime()); assertFalse(getBackFromCache.getAttributeDefs().get(0).getIsIndexable()); assertEquals(guid, getBackFromCache.getGuid()); assertEquals(createdTime, getBackFromCache.getCreateTime()); getBackFromCache = typeDefStore.getClassificationDefByGuid(guid); assertEquals("test_classification_11", getBackFromCache.getName()); assertFalse(getBackFromCache.getAttributeDefs().get(0).getIsIndexable()); assertEquals(guid, getBackFromCache.getGuid()); assertEquals(createdTime, getBackFromCache.getCreateTime());
private void createIndexForAttribute(AtlasGraphManagement management, String typeName, AtlasAttributeDef attributeDef) { final String propertyName = AtlasGraphUtilsV2.encodePropertyKey(typeName + "." + attributeDef.getName()); AtlasCardinality cardinality = toAtlasCardinality(attributeDef.getCardinality()); boolean isUnique = attributeDef.getIsUnique(); boolean isIndexable = attributeDef.getIsIndexable(); String attribTypeName = attributeDef.getTypeName(); boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName); boolean isArrayType = isArrayType(attribTypeName);
public static AtlasAttributeDef toAtlasAttributeDef(final AttributeDefinition attrDefinition) { AtlasAttributeDef ret = new AtlasAttributeDef(); ret.setIsIndexable(attrDefinition.isIndexable); ret.setIsUnique(attrDefinition.isUnique); if (attrDefinition.isComposite) { ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_OWNED_REF));
AtlasTypeDefGraphStoreV2 typeDefStore) throws AtlasBaseException { AtlasAttributeDef ret = new AtlasAttributeDef(); ret.setName((String) attribInfo.get("name")); ret.setTypeName((String) attribInfo.get("dataType")); ret.setIsUnique((Boolean) attribInfo.get("isUnique")); ret.setIsIndexable((Boolean) attribInfo.get("isIndexable")); ret.setIncludeInNotification((Boolean) attribInfo.get("includeInNotification")); ret.setDefaultValue((String) attribInfo.get("defaultValue")); ret.setDescription((String) attribInfo.get("description")); ret.addConstraint(new AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF)); ret.addConstraint(new AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{ put(AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE, reverseAttributeName); ret.setIsOptional(true); ret.setValuesMinCount(0); } else { ret.setIsOptional(false); ret.setValuesMinCount(minCount.intValue()); ret.setCardinality(AtlasAttributeDef.Cardinality.SINGLE); ret.setValuesMaxCount(1); } else { if (isUnique == null || isUnique == Boolean.FALSE) { ret.setCardinality(AtlasAttributeDef.Cardinality.LIST);
public static AtlasAttributeDef toAtlasAttributeDef(final AttributeDefinition attrDefinition) { AtlasAttributeDef ret = new AtlasAttributeDef(attrDefinition.getName(), attrDefinition.getDataTypeName()); ret.setIsIndexable(attrDefinition.getIsIndexable()); ret.setIsUnique(attrDefinition.getIsUnique()); if (attrDefinition.getIsComposite()) { ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_OWNED_REF)); ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_INVERSE_REF, new HashMap<String, Object>() {{ put(CONSTRAINT_PARAM_ATTRIBUTE, attrDefinition.getReverseAttributeName()); ret.setIsOptional(true); ret.setValuesMinCount(0); } else { ret.setIsOptional(false); ret.setValuesMinCount(minCount); ret.setCardinality(Cardinality.SINGLE); ret.setValuesMaxCount(1); } else { if (!isUnique) { ret.setCardinality(Cardinality.LIST); } else { ret.setCardinality(Cardinality.SET); ret.setValuesMaxCount(maxCount);
private Object mapPrimitiveValue(AttributeMutationContext ctx) { boolean isIndexableStrAttr = ctx.getAttributeDef().getIsIndexable() && ctx.getAttrType() instanceof AtlasBuiltInTypes.AtlasStringType;
switch (field.dataType().getTypeCategory()) { case PRIMITIVE: AtlasCardinality cardinality = getCardinality(field.multiplicity); createIndexes(management, propertyName, getPrimitiveClass(field.dataType()), field.isUnique, cardinality, false, field.isIndexable); cardinality = getCardinality(field.multiplicity); createIndexes(management, propertyName, String.class, field.isUnique, cardinality, false, field.isIndexable); break;
if (attribute.getAttributeDef().getIsUnique() || attribute.getAttributeDef().getIncludeInNotification()) { Object attrValue = entity.getAttribute(attribute.getName());
Map<String, Object> attribInfo = new HashMap<>(); attribInfo.put("name", attributeDef.getName()); attribInfo.put("dataType", attributeDef.getTypeName()); attribInfo.put("isUnique", attributeDef.getIsUnique()); attribInfo.put("isIndexable", attributeDef.getIsIndexable()); attribInfo.put("includeInNotification", attributeDef.getIncludeInNotification()); attribInfo.put("isComposite", attribute.isOwnedRef()); attribInfo.put("reverseAttributeName", attribute.getInverseRefAttributeName()); attribInfo.put("defaultValue", attributeDef.getDefaultValue()); attribInfo.put("description", attributeDef.getDescription()); if (attributeDef.getCardinality() == AtlasAttributeDef.Cardinality.SINGLE) { lower = attributeDef.getIsOptional() ? 0 : 1; upper = 1; } else { if(attributeDef.getIsOptional()) { lower = 0; } else { lower = attributeDef.getValuesMinCount() < 1 ? 1 : attributeDef.getValuesMinCount(); upper = attributeDef.getValuesMaxCount() < 2 ? Integer.MAX_VALUE : attributeDef.getValuesMaxCount(); multiplicity.put("lower", lower); multiplicity.put("upper", upper); multiplicity.put("isUnique", AtlasAttributeDef.Cardinality.SET.equals(attributeDef.getCardinality()));
public static AtlasEntityDef createPrimitiveEntityDef() { AtlasEntityDef newtestEntityDef = new AtlasEntityDef("newtest"); AtlasAttributeDef attrName = new AtlasAttributeDef("name", AtlasBaseTypeDef.ATLAS_TYPE_STRING); AtlasAttributeDef attrDescription = new AtlasAttributeDef("description", AtlasBaseTypeDef.ATLAS_TYPE_STRING); attrDescription.setIsOptional(false); AtlasAttributeDef attrcheck = new AtlasAttributeDef("check", AtlasBaseTypeDef.ATLAS_TYPE_STRING); attrcheck.setIsOptional(true); AtlasAttributeDef attrSourceCode = new AtlasAttributeDef("sourcecode", AtlasBaseTypeDef.ATLAS_TYPE_STRING); attrSourceCode.setDefaultValue("Hello World"); attrSourceCode.setIsOptional(true); AtlasAttributeDef attrCost = new AtlasAttributeDef("Cost", AtlasBaseTypeDef.ATLAS_TYPE_INT); attrCost.setIsOptional(true); attrCost.setDefaultValue("30"); AtlasAttributeDef attrDiskUsage = new AtlasAttributeDef("diskUsage", AtlasBaseTypeDef.ATLAS_TYPE_FLOAT); attrDiskUsage.setIsOptional(true); attrDiskUsage.setDefaultValue("70.50"); AtlasAttributeDef attrisStoreUse = new AtlasAttributeDef("isstoreUse", AtlasBaseTypeDef.ATLAS_TYPE_BOOLEAN); attrisStoreUse.setIsOptional(true); attrisStoreUse.setDefaultValue("true"); newtestEntityDef.addAttribute(attrName); newtestEntityDef.addAttribute(attrDescription); newtestEntityDef.addAttribute(attrcheck); newtestEntityDef.addAttribute(attrSourceCode); newtestEntityDef.addAttribute(attrCost); newtestEntityDef.addAttribute(attrDiskUsage); newtestEntityDef.addAttribute(attrisStoreUse); populateSystemAttributes(newtestEntityDef); return newtestEntityDef; }
AttributeDefinition ret = new AttributeDefinition(); ret.setName(attributeDef.getName()); ret.setDataTypeName(attributeDef.getTypeName()); ret.setIsUnique(attributeDef.getIsUnique()); ret.setIsIndexable(attributeDef.getIsIndexable()); ret.setIsComposite(attribute.isOwnedRef()); ret.setReverseAttributeName(attribute.getInverseRefAttributeName()); ret.setDefaultValue(attributeDef.getDefaultValue()); ret.setDescription(attributeDef.getDescription()); if (attributeDef.getCardinality() == AtlasAttributeDef.Cardinality.SINGLE) { lower = attributeDef.getIsOptional() ? 0 : 1; upper = 1; } else { if(attributeDef.getIsOptional()) { lower = 0; } else { lower = attributeDef.getValuesMinCount() < 1 ? 1 : attributeDef.getValuesMinCount(); upper = attributeDef.getValuesMaxCount() < 2 ? Integer.MAX_VALUE : attributeDef.getValuesMaxCount(); multiplicity.setLower(lower); multiplicity.setUpper(upper); multiplicity.setIsUnique(AtlasAttributeDef.Cardinality.SET.equals(attributeDef.getCardinality()));
@Override public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { Map<String, AtlasAttribute> a = new HashMap<>(); for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { AtlasType attrType = typeRegistry.getType(attributeDef.getTypeName()); AtlasAttribute attribute = new AtlasAttribute(this, attributeDef, attrType); Cardinality cardinality = attributeDef.getCardinality(); if (cardinality == Cardinality.LIST || cardinality == Cardinality.SET) { if (!(attrType instanceof AtlasArrayType)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_ATTRIBUTE_TYPE_FOR_CARDINALITY, getTypeName(), attributeDef.getName()); } AtlasArrayType arrayType = (AtlasArrayType)attrType; arrayType.setMinCount(attributeDef.getValuesMinCount()); arrayType.setMaxCount(attributeDef.getValuesMaxCount()); } a.put(attributeDef.getName(), attribute); } resolveConstraints(typeRegistry); this.allAttributes = Collections.unmodifiableMap(a); this.uniqAttributes = getUniqueAttributes(this.allAttributes); }
private void rectifyAttributesIfNeeded(final Set<String> entityNames, AtlasStructDef structDef) { List<AtlasAttributeDef> attributeDefs = structDef.getAttributeDefs(); if (CollectionUtils.isNotEmpty(attributeDefs)) { for (AtlasAttributeDef attributeDef : attributeDefs) { if (!hasOwnedReferenceConstraint(attributeDef.getConstraints())) { continue; } Set<String> referencedTypeNames = AtlasTypeUtil.getReferencedTypeNames(attributeDef.getTypeName()); boolean valid = false; for (String referencedTypeName : referencedTypeNames) { if (entityNames.contains(referencedTypeName)) { valid = true; break; } } if (!valid) { rectifyOwnedReferenceError(structDef, attributeDef); } } } }
private void rectifyAttributesIfNeeded(final Set<String> entityNames, AtlasStructDef structDef) { List<AtlasAttributeDef> attributeDefs = structDef.getAttributeDefs(); if (CollectionUtils.isNotEmpty(attributeDefs)) { for (AtlasAttributeDef attributeDef : attributeDefs) { if (!hasOwnedReferenceConstraint(attributeDef.getConstraints())) { continue; } Set<String> referencedTypeNames = AtlasTypeUtil.getReferencedTypeNames(attributeDef.getTypeName()); boolean valid = false; for (String referencedTypeName : referencedTypeNames) { if (entityNames.contains(referencedTypeName)) { valid = true; break; } } if (!valid) { rectifyOwnedReferenceError(structDef, attributeDef); } } } }
private void cleanupIndexForAttribute(AtlasGraphManagement management, String typeName, AtlasAttributeDef attributeDef) { final String propertyName = AtlasGraphUtilsV2.encodePropertyKey(typeName + "." + attributeDef.getName()); String attribTypeName = attributeDef.getTypeName(); boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName); boolean isArrayType = isArrayType(attribTypeName); boolean isMapType = isMapType(attribTypeName); try { AtlasType atlasType = typeRegistry.getType(attribTypeName); if (isClassificationType(atlasType) || isEntityType(atlasType)) { LOG.warn("Ignoring non-indexable attribute {}", attribTypeName); } else if (isBuiltInType || isEnumType(atlasType) || isArrayType || isMapType) { cleanupIndex(management, propertyName); } else if (isStructType(atlasType)) { AtlasStructDef structDef = typeRegistry.getStructDefByName(attribTypeName); cleanupIndices(management, structDef); } } catch (AtlasBaseException e) { LOG.error("No type exists for {}", attribTypeName, e); } }
public static void updateVertexPreCreate(AtlasStructDef structDef, AtlasStructType structType, AtlasVertex vertex, AtlasTypeDefGraphStoreV2 typeDefStore) throws AtlasBaseException { List<String> attrNames = new ArrayList<>(structDef.getAttributeDefs().size()); for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { // Validate the mandatory features of an attribute (compatibility with legacy type system) if (StringUtils.isEmpty(attributeDef.getName())) { throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE, structDef.getName(), "name"); } if (StringUtils.isEmpty(attributeDef.getTypeName())) { throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE, structDef.getName(), "typeName"); } String propertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef, attributeDef.getName()); String encodedPropertyKey = AtlasGraphUtilsV2.encodePropertyKey(propertyKey); vertex.setProperty(encodedPropertyKey, toJsonFromAttribute(structType.getAttribute(attributeDef.getName()))); attrNames.add(attributeDef.getName()); } String typeNamePropertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef); String encodedtypeNamePropertyKey = AtlasGraphUtilsV2.encodePropertyKey(typeNamePropertyKey); vertex.setProperty(encodedtypeNamePropertyKey, attrNames); }
public static void updateVertexPreCreate(AtlasStructDef structDef, AtlasStructType structType, AtlasVertex vertex, AtlasTypeDefGraphStoreV2 typeDefStore) throws AtlasBaseException { List<String> attrNames = new ArrayList<>(structDef.getAttributeDefs().size()); for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) { // Validate the mandatory features of an attribute (compatibility with legacy type system) if (StringUtils.isEmpty(attributeDef.getName())) { throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE, structDef.getName(), "name"); } if (StringUtils.isEmpty(attributeDef.getTypeName())) { throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ATTRIBUTE, structDef.getName(), "typeName"); } String propertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef, attributeDef.getName()); String encodedPropertyKey = AtlasGraphUtilsV2.encodePropertyKey(propertyKey); vertex.setProperty(encodedPropertyKey, toJsonFromAttribute(structType.getAttribute(attributeDef.getName()))); attrNames.add(attributeDef.getName()); } String typeNamePropertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef); String encodedtypeNamePropertyKey = AtlasGraphUtilsV2.encodePropertyKey(typeNamePropertyKey); vertex.setProperty(encodedtypeNamePropertyKey, attrNames); }
private Object mapPrimitiveValue(AttributeMutationContext ctx) { boolean isIndexableStrAttr = ctx.getAttributeDef().getIsIndexable() && ctx.getAttrType() instanceof AtlasBuiltInTypes.AtlasStringType; Object ret = ctx.getValue(); // Janus bug, when an indexed string attribute has a value longer than a certain length then the reverse indexed key generated by JanusGraph // exceeds the HBase row length's hard limit (Short.MAX). This trimming and hashing procedure is to circumvent that limitation if (ret != null && isIndexableStrAttr) { String value = (String) ctx.getValue(); if (value.length() > INDEXED_STR_MAX_ALLOWED_LEN) { LOG.warn("Indexed-String-Attribute: {} exceeds {} characters, trimming and appending checksum", ctx.getAttribute().getQualifiedName(), INDEXED_STR_MAX_ALLOWED_LEN); String sha256Hex = DigestUtils.shaHex(value); // Storing SHA checksum in case verification is needed after retrieval ret = value.substring(0, (INDEXED_STR_MAX_ALLOWED_LEN - 1) - sha256Hex.length()) + ":" + sha256Hex; } } AtlasGraphUtilsV2.setEncodedProperty(ctx.getReferringVertex(), ctx.getVertexProperty(), ret); return ret; }