/** * Get the column families required by this schema. * * @return The set of column families. */ public Set<String> getRequiredColumnFamilies() { Set<String> set = new HashSet<String>(); for (FieldMapping mapping : fieldMappings) { if (FieldMapping.MappingType.KEY != mapping.getMappingType()) set.add(mapping.getFamilyAsString()); } return set; }
/** * Get the columns required by this schema. * * @return The set of columns */ public Set<String> getRequiredColumns() { Set<String> set = new HashSet<String>(); for (FieldMapping fieldMapping : fieldMappings) { if (FieldMapping.MappingType.KEY == fieldMapping.getMappingType()) { continue; } else if (FieldMapping.MappingType.KEY_AS_COLUMN == fieldMapping.getMappingType()) { set.add(fieldMapping.getFamilyAsString() + ":"); } else { set.add(fieldMapping.getFamilyAsString() + ":" + fieldMapping.getQualifierAsString()); } } return set; }
/** * Validate that a {@link FieldMapping} is compatible with this builder's * current set of mappings and add it to the set of mappings. * * A mapping is not compatible if it results in: * <pre> * 1. Multiple occVersion mappings in the mapping set * 2. Both a counter and an occVersion mapping in the mapping set * </pre> * * @param fm a {@code FieldMapping} to add to this builder */ private void addField(FieldMapping fm) { // validate! if (fm.getMappingType() == FieldMapping.MappingType.OCC_VERSION) { ValidationException.check(!hasOCCVersion, "Cannot use multiple occVersion fields"); ValidationException.check(!hasCounter, "Cannot use both counter and occVersion fields"); hasOCCVersion = true; } else if (fm.getMappingType() == FieldMapping.MappingType.COUNTER) { ValidationException.check(!hasOCCVersion, "Cannot use both counter and occVersion fields"); hasCounter = true; } fieldMappings.add(fm); } }
public static Map<Integer, FieldMapping> parseKeyMappingsFromSchemaFields( Schema schema) { Map<Integer, FieldMapping> keyMappings = Maps.newHashMap(); if (Schema.Type.RECORD == schema.getType()) { for (Schema.Field field : schema.getFields()) { if (field.getJsonProp(MAPPING) != null) { // parse the String because Avro uses com.codehaus.jackson JsonNode mappingNode = JsonUtil.parse( field.getJsonProp(MAPPING).toString()); FieldMapping fm = parseFieldMapping(field.name(), mappingNode); if (FieldMapping.MappingType.KEY == fm.getMappingType() && mappingNode.has(VALUE)) { Integer index = mappingNode.get(VALUE).asInt(); keyMappings.put(index, fm); } } } return keyMappings; } throw new IllegalArgumentException( "Cannot parse field-level mappings from non-Record"); }
public RegexEntityFilter(EntitySchema entitySchema, EntitySerDe<?> entitySerDe, String fieldName, String regex, boolean isEqual) { FieldMapping fieldMapping = entitySchema.getColumnMappingDescriptor() .getFieldMapping(fieldName); if (fieldMapping.getMappingType() != MappingType.COLUMN) { throw new DatasetException( "SingleColumnValueFilter only compatible with COLUMN mapping types."); } this.filter = constructFilter(regex, isEqual, fieldMapping); }
public RegexEntityFilter(EntitySchema entitySchema, EntitySerDe<?> entitySerDe, String fieldName, String regex, boolean isEqual) { FieldMapping fieldMapping = entitySchema.getColumnMappingDescriptor() .getFieldMapping(fieldName); if (fieldMapping.getMappingType() != MappingType.COLUMN) { throw new DatasetException( "SingleColumnValueFilter only compatible with COLUMN mapping types."); } this.filter = constructFilter(regex, isEqual, fieldMapping); }
/** * Initialize the AvroRecordBuilderFactories for all keyAsColumn mapped fields * that are record types. We need to be able to get record builders for these * since the records are broken across many columns, and need to be * constructed by the composer. */ private void initRecordBuilderFactories() { for (FieldMapping fieldMapping : avroSchema.getColumnMappingDescriptor().getFieldMappings()) { if (fieldMapping.getMappingType() == MappingType.KEY_AS_COLUMN) { String fieldName = fieldMapping.getFieldName(); Schema fieldSchema = avroSchema.getAvroSchema().getField(fieldName) .schema(); Schema.Type fieldSchemaType = fieldSchema.getType(); if (fieldSchemaType == Schema.Type.RECORD) { AvroRecordBuilderFactory<E> factory = buildAvroRecordBuilderFactory(fieldSchema); kacRecordBuilderFactories.put(fieldName, factory); } } } }
/** * Initialize the AvroRecordBuilderFactories for all keyAsColumn mapped fields * that are record types. We need to be able to get record builders for these * since the records are broken across many columns, and need to be * constructed by the composer. */ private void initRecordBuilderFactories() { for (FieldMapping fieldMapping : avroSchema.getColumnMappingDescriptor().getFieldMappings()) { if (fieldMapping.getMappingType() == MappingType.KEY_AS_COLUMN) { String fieldName = fieldMapping.getFieldName(); Schema fieldSchema = avroSchema.getAvroSchema().getField(fieldName) .schema(); Schema.Type fieldSchemaType = fieldSchema.getType(); if (fieldSchemaType == Schema.Type.RECORD) { AvroRecordBuilderFactory<E> factory = buildAvroRecordBuilderFactory(fieldSchema); kacRecordBuilderFactories.put(fieldName, factory); } } } }
private static String getColumnValue(FieldMapping fm) { switch (fm.getMappingType()) { case COLUMN: case COUNTER: return fm.getFamilyAsString() + ":" + fm.getQualifierAsString(); case KEY_AS_COLUMN: return fm.getFamilyAsString() + ":" + (fm.getPrefix() == null ? "" : fm.getPrefix()); default: return null; } }
private static String getColumnValue(FieldMapping fm) { switch (fm.getMappingType()) { case COLUMN: case COUNTER: return fm.getFamilyAsString() + ":" + fm.getQualifierAsString(); case KEY_AS_COLUMN: return fm.getFamilyAsString() + ":" + (fm.getPrefix() == null ? "" : fm.getPrefix()); default: return null; } }
@Override public long mapFromIncrementResult(Result result, String fieldName) { FieldMapping fieldMapping = entitySchema.getColumnMappingDescriptor() .getFieldMapping(fieldName); if (fieldMapping == null) { throw new DatasetException("Unknown field in the schema: " + fieldName); } if (fieldMapping.getMappingType() != MappingType.COUNTER) { throw new DatasetException("Field is not a counter type: " + fieldName); } return (Long) entitySerDe.deserialize(fieldMapping, result); }
@Override public long mapFromIncrementResult(Result result, String fieldName) { FieldMapping fieldMapping = entitySchema.getColumnMappingDescriptor() .getFieldMapping(fieldName); if (fieldMapping == null) { throw new DatasetException("Unknown field in the schema: " + fieldName); } if (fieldMapping.getMappingType() != MappingType.COUNTER) { throw new DatasetException("Field is not a counter type: " + fieldName); } return (Long) entitySerDe.deserialize(fieldMapping, result); }
@Override public PutAction mapFromEntity(E entity) { List<PutAction> putActionList = new ArrayList<PutAction>(); byte[] keyBytes; if (keySchema == null || keySerDe == null) { keyBytes = new byte[] { (byte) 0 }; } else { keyBytes = keySerDe.serialize(mapToKey(entity)); } for (FieldMapping fieldMapping : entitySchema.getColumnMappingDescriptor() .getFieldMappings()) { if (fieldMapping.getMappingType() == MappingType.KEY) { continue; } Object fieldValue = getEntityComposer().extractField(entity, fieldMapping.getFieldName()); if (fieldValue != null) { PutAction put = entitySerDe.serialize(keyBytes, fieldMapping, fieldValue); putActionList.add(put); } } return HBaseUtils.mergePutActions(keyBytes, putActionList); }
public SingleFieldEntityFilter(EntitySchema entitySchema, EntitySerDe<?> entitySerDe, String fieldName, Object filterValue, CompareFilter.CompareOp equalityOperator) { FieldMapping fieldMapping = entitySchema.getColumnMappingDescriptor() .getFieldMapping(fieldName); if (fieldMapping.getMappingType() != MappingType.COLUMN) { throw new DatasetException( "SingleColumnValueFilter only compatible with COLUMN mapping types."); } byte[] family = fieldMapping.getFamily(); byte[] qualifier = fieldMapping.getQualifier(); byte[] comparisonBytes = entitySerDe.serializeColumnValueToBytes(fieldName, filterValue); this.filter = new SingleColumnValueFilter(family, qualifier, equalityOperator, comparisonBytes); }
public SingleFieldEntityFilter(EntitySchema entitySchema, EntitySerDe<?> entitySerDe, String fieldName, Object filterValue, CompareFilter.CompareOp equalityOperator) { FieldMapping fieldMapping = entitySchema.getColumnMappingDescriptor() .getFieldMapping(fieldName); if (fieldMapping.getMappingType() != MappingType.COLUMN) { throw new DatasetException( "SingleColumnValueFilter only compatible with COLUMN mapping types."); } byte[] family = fieldMapping.getFamily(); byte[] qualifier = fieldMapping.getQualifier(); byte[] comparisonBytes = entitySerDe.serializeColumnValueToBytes(fieldName, filterValue); this.filter = new SingleColumnValueFilter(family, qualifier, equalityOperator, comparisonBytes); }
@Test public void testGoodSchema() { AvroEntitySchema avroEntitySchema = parser.parseEntitySchema(entitySchema); ColumnMapping descriptor = avroEntitySchema .getColumnMappingDescriptor(); assertEquals(9, descriptor.getFieldMappings().size()); assertEquals(FieldMapping.MappingType.COLUMN, descriptor.getFieldMapping("field1") .getMappingType()); assertEquals(FieldMapping.MappingType.KEY_AS_COLUMN, descriptor .getFieldMapping("field4").getMappingType()); assertEquals(FieldMapping.MappingType.OCC_VERSION, descriptor.getFieldMapping("version") .getMappingType()); AvroKeySchema avroKeySchema = parser.parseKeySchema(entitySchema); assertEquals(Type.STRING, avroKeySchema.getAvroSchema() .getField("keyPart1_copy").schema().getType()); assertEquals(Type.STRING, avroKeySchema.getAvroSchema() .getField("keyPart2_copy").schema().getType()); assertEquals(2, Accessor.getDefault().getFieldPartitioners(avroKeySchema .getPartitionStrategy()) .size()); }
@Override public Increment mapToIncrement(PartitionKey key, String fieldName, long amount) { FieldMapping fieldMapping = entitySchema.getColumnMappingDescriptor() .getFieldMapping(fieldName); if (fieldMapping == null) { throw new DatasetException("Unknown field in the schema: " + fieldName); } if (fieldMapping.getMappingType() != MappingType.COUNTER) { throw new DatasetException("Field is not a counter type: " + fieldName); } byte[] keyBytes; if (keySerDe == null) { keyBytes = new byte[] { (byte) 0 }; } else { keyBytes = keySerDe.serialize(key); } Increment increment = new Increment(keyBytes); increment.addColumn(fieldMapping.getFamily(), fieldMapping.getQualifier(), amount); return increment; }
@Override public Increment mapToIncrement(PartitionKey key, String fieldName, long amount) { FieldMapping fieldMapping = entitySchema.getColumnMappingDescriptor() .getFieldMapping(fieldName); if (fieldMapping == null) { throw new DatasetException("Unknown field in the schema: " + fieldName); } if (fieldMapping.getMappingType() != MappingType.COUNTER) { throw new DatasetException("Field is not a counter type: " + fieldName); } byte[] keyBytes; if (keySerDe == null) { keyBytes = new byte[] { (byte) 0 }; } else { keyBytes = keySerDe.serialize(key); } Increment increment = new Increment(keyBytes); increment.addColumn(fieldMapping.getFamily(), fieldMapping.getQualifier(), amount); return increment; }
@Test public void testOverrideColumnMapping() { ColumnMapping desc = new ColumnMapping.Builder() .column("field1", "override", "field1") .counter("version", "override", "version").build(); AvroEntitySchema avroEntitySchema = parser.parseEntitySchema(entitySchema, desc); desc = avroEntitySchema.getColumnMappingDescriptor(); assertEquals(2, desc.getFieldMappings().size()); assertEquals(FieldMapping.MappingType.COLUMN, desc.getFieldMapping("field1") .getMappingType()); assertEquals(FieldMapping.MappingType.COUNTER, desc.getFieldMapping("version") .getMappingType()); }
/** * Deserialize an entity field from the HBase Result. * * @param fieldMapping * The FieldMapping that specifies this field's mapping type and * field name. * @param result * The HBase Result that represents a row in HBase. * @return The field Object we deserialized from the Result. */ public Object deserialize(FieldMapping fieldMapping, Result result) { String fieldName = fieldMapping.getFieldName(); MappingType mappingType = fieldMapping.getMappingType(); if (mappingType == MappingType.COLUMN || mappingType == MappingType.COUNTER) { return deserializeColumn(fieldMapping.getFieldName(), fieldMapping.getFamily(), fieldMapping.getQualifier(), result); } else if (mappingType == MappingType.KEY_AS_COLUMN) { return deserializeKeyAsColumn(fieldMapping.getFieldName(), fieldMapping.getFamily(), fieldMapping.getPrefix(), result); } else if (mappingType == MappingType.OCC_VERSION) { return deserializeOCCColumn(result); } else { throw new ValidationException( "Invalid field mapping for field with name: " + fieldName); } }