private static JsonNode toJson(ColumnMapping mapping) { ArrayNode mappingJson = JsonNodeFactory.instance.arrayNode(); for (FieldMapping fm : mapping.getFieldMappings()) { mappingJson.add(toJson(fm)); } return mappingJson; }
/** * Initialize the AvroRecordBuilderFactories for all keyAsColumn mapped fields * that are record types. We need to be able to get record builders for these * since the records are broken across many columns, and need to be * constructed by the composer. */ private void initRecordBuilderFactories() { for (FieldMapping fieldMapping : avroSchema.getColumnMappingDescriptor().getFieldMappings()) { if (fieldMapping.getMappingType() == MappingType.KEY_AS_COLUMN) { String fieldName = fieldMapping.getFieldName(); Schema fieldSchema = avroSchema.getAvroSchema().getField(fieldName) .schema(); Schema.Type fieldSchemaType = fieldSchema.getType(); if (fieldSchemaType == Schema.Type.RECORD) { AvroRecordBuilderFactory<E> factory = buildAvroRecordBuilderFactory(fieldSchema); kacRecordBuilderFactories.put(fieldName, factory); } } } }
/** * Initialize the AvroRecordBuilderFactories for all keyAsColumn mapped fields * that are record types. We need to be able to get record builders for these * since the records are broken across many columns, and need to be * constructed by the composer. */ private void initRecordBuilderFactories() { for (FieldMapping fieldMapping : avroSchema.getColumnMappingDescriptor().getFieldMappings()) { if (fieldMapping.getMappingType() == MappingType.KEY_AS_COLUMN) { String fieldName = fieldMapping.getFieldName(); Schema fieldSchema = avroSchema.getAvroSchema().getField(fieldName) .schema(); Schema.Type fieldSchemaType = fieldSchema.getType(); if (fieldSchemaType == Schema.Type.RECORD) { AvroRecordBuilderFactory<E> factory = buildAvroRecordBuilderFactory(fieldSchema); kacRecordBuilderFactories.put(fieldName, factory); } } } }
boolean foundOccMapping = false; for (FieldMapping fieldMapping : entitySchema1.getColumnMappingDescriptor() .getFieldMappings()) { if (fieldMapping.getMappingType() == MappingType.OCC_VERSION) { foundOccMapping = true; .getColumnMappingDescriptor().getFieldMappings()) { if (fieldMapping.getMappingType() == MappingType.OCC_VERSION) { LOG.warn("Field: " + fieldMapping.getFieldName() + " in schema "
boolean foundOccMapping = false; for (FieldMapping fieldMapping : entitySchema1.getColumnMappingDescriptor() .getFieldMappings()) { if (fieldMapping.getMappingType() == MappingType.OCC_VERSION) { foundOccMapping = true; .getColumnMappingDescriptor().getFieldMappings()) { if (fieldMapping.getMappingType() == MappingType.OCC_VERSION) { LOG.warn("Field: " + fieldMapping.getFieldName() + " in schema "
"Cannot map non-records: %s", schema); Set<String> keyMappedFields = Sets.newHashSet(); for (FieldMapping fm : mappings.getFieldMappings()) { Schema fieldSchema = SchemaUtil.fieldSchema(schema, fm.getFieldName()); ValidationException.check(
/** * Ensure that the column mappings for the shared fields between the old and * new schema haven't changed. * * @param oldSchema * @param newSchema * @return true if the mappings are compatible, false if not. */ private static boolean mappingCompatible(EntitySchema oldSchema, EntitySchema newSchema) { for (FieldMapping oldFieldMapping : oldSchema.getColumnMappingDescriptor() .getFieldMappings()) { FieldMapping newFieldMapping = newSchema.getColumnMappingDescriptor() .getFieldMapping(oldFieldMapping.getFieldName()); if (newFieldMapping != null) { if (!oldFieldMapping.equals(newFieldMapping)) { return false; } } } return true; }
/** * Ensure that the column mappings for the shared fields between the old and * new schema haven't changed. * * @param oldSchema * @param newSchema * @return true if the mappings are compatible, false if not. */ private static boolean mappingCompatible(EntitySchema oldSchema, EntitySchema newSchema) { for (FieldMapping oldFieldMapping : oldSchema.getColumnMappingDescriptor() .getFieldMappings()) { FieldMapping newFieldMapping = newSchema.getColumnMappingDescriptor() .getFieldMapping(oldFieldMapping.getFieldName()); if (newFieldMapping != null) { if (!oldFieldMapping.equals(newFieldMapping)) { return false; } } } return true; }
@Override public PutAction mapFromEntity(E entity) { List<PutAction> putActionList = new ArrayList<PutAction>(); byte[] keyBytes; if (keySchema == null || keySerDe == null) { keyBytes = new byte[] { (byte) 0 }; } else { keyBytes = keySerDe.serialize(mapToKey(entity)); } for (FieldMapping fieldMapping : entitySchema.getColumnMappingDescriptor() .getFieldMappings()) { if (fieldMapping.getMappingType() == MappingType.KEY) { continue; } Object fieldValue = getEntityComposer().extractField(entity, fieldMapping.getFieldName()); if (fieldValue != null) { PutAction put = entitySerDe.serialize(keyBytes, fieldMapping, fieldValue); putActionList.add(put); } } return HBaseUtils.mergePutActions(keyBytes, putActionList); }
List<String> entitySchema1KeyAsColumns = new ArrayList<String>(); for (FieldMapping fieldMapping1 : entitySchema1 .getColumnMappingDescriptor().getFieldMappings()) { if (fieldMapping1.getMappingType() == MappingType.COLUMN || fieldMapping1.getMappingType() == MappingType.COUNTER) { .getColumnMappingDescriptor().getFieldMappings()) { if (fieldMapping2.getMappingType() == MappingType.COLUMN || fieldMapping2.getMappingType() == MappingType.COUNTER) {
@Override public PutAction mapFromEntity(E entity) { List<PutAction> putActionList = new ArrayList<PutAction>(); byte[] keyBytes; if (keySchema == null || keySerDe == null) { keyBytes = new byte[] { (byte) 0 }; } else { keyBytes = keySerDe.serialize(mapToKey(entity)); } for (FieldMapping fieldMapping : entitySchema.getColumnMappingDescriptor() .getFieldMappings()) { if (fieldMapping.getMappingType() == MappingType.KEY) { continue; } Object fieldValue = getEntityComposer().extractField(entity, fieldMapping.getFieldName()); if (fieldValue != null) { PutAction put = entitySerDe.serialize(keyBytes, fieldMapping, fieldValue); putActionList.add(put); } } return HBaseUtils.mergePutActions(keyBytes, putActionList); }
List<String> entitySchema1KeyAsColumns = new ArrayList<String>(); for (FieldMapping fieldMapping1 : entitySchema1 .getColumnMappingDescriptor().getFieldMappings()) { if (fieldMapping1.getMappingType() == MappingType.COLUMN || fieldMapping1.getMappingType() == MappingType.COUNTER) { .getColumnMappingDescriptor().getFieldMappings()) { if (fieldMapping2.getMappingType() == MappingType.COLUMN || fieldMapping2.getMappingType() == MappingType.COUNTER) {
for (FieldMapping fieldMapping : avroSchema.getColumnMappingDescriptor().getFieldMappings()) { String fieldName = fieldMapping.getFieldName(); Schema fieldSchema = avroSchema.getAvroSchema().getField(fieldName)
for (FieldMapping fieldMapping : avroSchema.getColumnMappingDescriptor().getFieldMappings()) { String fieldName = fieldMapping.getFieldName(); Schema fieldSchema = avroSchema.getAvroSchema().getField(fieldName)
@Test public void testBasic() { DatasetDescriptor desc = provider.create("default", tableName + ".TestEntity", new DatasetDescriptor.Builder().schemaLiteral(testEntity).build()); ColumnMapping columnMapping = desc.getColumnMapping(); PartitionStrategy partStrat = desc.getPartitionStrategy(); assertEquals(9, columnMapping.getFieldMappings().size()); assertEquals(2, Accessor.getDefault().getFieldPartitioners(partStrat).size()); }
.getFieldMappings()) { Object fieldValue; if (fieldMapping.getMappingType() == MappingType.KEY) {
.getFieldMappings()) { Object fieldValue; if (fieldMapping.getMappingType() == MappingType.KEY) {
@Test public void testGoodSchema() { AvroEntitySchema avroEntitySchema = parser.parseEntitySchema(entitySchema); ColumnMapping descriptor = avroEntitySchema .getColumnMappingDescriptor(); assertEquals(9, descriptor.getFieldMappings().size()); assertEquals(FieldMapping.MappingType.COLUMN, descriptor.getFieldMapping("field1") .getMappingType()); assertEquals(FieldMapping.MappingType.KEY_AS_COLUMN, descriptor .getFieldMapping("field4").getMappingType()); assertEquals(FieldMapping.MappingType.OCC_VERSION, descriptor.getFieldMapping("version") .getMappingType()); AvroKeySchema avroKeySchema = parser.parseKeySchema(entitySchema); assertEquals(Type.STRING, avroKeySchema.getAvroSchema() .getField("keyPart1_copy").schema().getType()); assertEquals(Type.STRING, avroKeySchema.getAvroSchema() .getField("keyPart2_copy").schema().getType()); assertEquals(2, Accessor.getDefault().getFieldPartitioners(avroKeySchema .getPartitionStrategy()) .size()); }
@Test public void testOverrideColumnMapping() { ColumnMapping desc = new ColumnMapping.Builder() .column("field1", "override", "field1") .counter("version", "override", "version").build(); AvroEntitySchema avroEntitySchema = parser.parseEntitySchema(entitySchema, desc); desc = avroEntitySchema.getColumnMappingDescriptor(); assertEquals(2, desc.getFieldMappings().size()); assertEquals(FieldMapping.MappingType.COLUMN, desc.getFieldMapping("field1") .getMappingType()); assertEquals(FieldMapping.MappingType.COUNTER, desc.getFieldMapping("version") .getMappingType()); }