Refine search
public static void fillComplexFullResyncDelta(GenericRecord delta) { GenericRecord testField2 = new GenericData.Record(getSchemaByFullName( delta.getSchema().getField("testField2").schema().getTypes(), "org.kaa.config.testRecordT")); testField2.put("testField3", 456); byte[] rawUuid = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; GenericFixed uuid = new GenericData.Fixed(delta.getSchema() .getField("__uuid").schema(), rawUuid); delta.put("testField1", "abc"); delta.put("testField2", testField2); delta.put("__uuid", uuid); }
private Object deserializeList(Object datum, Schema fileSchema, Schema recordSchema, ListTypeInfo columnType) throws AvroSerdeException { if(recordSchema.getType().equals(Schema.Type.FIXED)) { List<Byte> asList = new ArrayList<Byte>(fixed.bytes().length); for(int j = 0; j < fixed.bytes().length; j++) { asList.add(fixed.bytes()[j]); } else if(recordSchema.getType().equals(Schema.Type.BYTES)) { } else { // An actual list, deser its values List listData = (List) datum; Schema listSchema = recordSchema.getElementType(); List<Object> listContents = new ArrayList<Object>(listData.size()); for(Object obj : listData) {
@SuppressWarnings(value="unchecked") private Object generate(Schema schema, Random random, int d) { switch (schema.getType()) { case RECORD: GenericRecord record = new GenericData.Record(schema); for (Schema.Field field : schema.getFields()) { Object value = (field.getObjectProp(USE_DEFAULT) == null) ? generate(field.schema(), random, d+1) : GenericData.get().getDefaultValue(field); record.put(field.name(), value); List<String> symbols = schema.getEnumSymbols(); return new GenericData.EnumSymbol (schema, symbols.get(random.nextInt(symbols.size()))); byte[] bytes = new byte[schema.getFixedSize()]; random.nextBytes(bytes); return new GenericData.Fixed(schema, bytes); case STRING: return randomString(random, 40); case BYTES: return randomBytes(random, 40);
switch (schema.getType()) { case RECORD: if (item instanceof Map) { Map<String,Object> map = (Map) item; IndexedRecord record = new GenericData.Record(schema); for (Field field : schema.getFields()) { Object value = map.get(field.name()); Object result = toAvro(value, field); if (result == ERROR) { return ERROR; record.put(field.pos(), result); if (schema.hasEnumSymbol(item.toString())) { return item.toString(); case FIXED: if (item instanceof byte[]) { return new GenericData.Fixed(schema, (byte[])item);
for (Field f : PROTOCOL.getType("TestRecord").getFields()) fields.add(new Field(f.name(), f.schema(), null, null)); fields.add(new Field("extra", Schema.create(Schema.Type.BOOLEAN), null, true)); Schema record = Schema.createRecord("TestRecord", null, "org.apache.avro.test", false); record.setFields(fields); GenericRecord args = new GenericData.Record(message.getRequest()); GenericRecord rec = new GenericData.Record(record); rec.put("name", new Utf8("foo")); rec.put("kind", new GenericData.EnumSymbol (PROTOCOL.getType("Kind"), "BAR")); rec.put("hash", new GenericData.Fixed (PROTOCOL.getType("MD5"), new byte[]{0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5}));
return null; switch(schema.getType()) { case RECORD: return convertToRelRecord((IndexedRecord) avroObj); .map(v -> convertToJavaObject(v, getNonNullUnionSchema(schema).getElementType())) .collect(Collectors.toList())); return retVal; .collect(Collectors.toMap( Map.Entry::getKey, e -> convertToJavaObject(e.getValue(), getNonNullUnionSchema(schema).getValueType())))); return retVal; case FIXED: GenericData.Fixed fixed = (GenericData.Fixed) avroObj; return new ByteString(fixed.bytes()); case BYTES: return new ByteString(((ByteBuffer) avroObj).array());
return null; switch (schema.getType()) { case RECORD: if (object instanceof Row) { return new GenericData.EnumSymbol(schema, object.toString()); case ARRAY: final Schema elementSchema = schema.getElementType(); final Object[] array = (Object[]) object; final GenericData.Array<Object> convertedArray = new GenericData.Array<>(array.length, schema); convertedMap.put( new Utf8(entry.getKey().toString()), convertFlinkType(schema.getValueType(), entry.getValue())); return new GenericData.Fixed( schema, convertFromDecimal(schema, (BigDecimal) object)); return new GenericData.Fixed(schema, (byte[]) object); case STRING: return new Utf8(object.toString());
Queue<FieldAttribute> fieldQueue) throws DeltaCalculatorException { List<Schema.Field> fields = delta.getSchema().getFields(); if (resetFields.contains(field.name())) { putReset(delta, field.name()); } else if (uuidFields.containsKey(field.name())) { Schema arraySchema = getArraySchema(delta, field.name()); Schema uuidSchema = getSchemaByFullName(arraySchema, KAA_NAMESPACE + "." + UUID_TYPE); GenericArray arrayField = new GenericData.Array(uuids.size(), arraySchema); for (byte[] uuid : uuids) { GenericFixed uuidFixed = new GenericData.Fixed(uuidSchema, uuid); arrayField.add(uuidFixed); delta.put(field.name(), arrayField); } else if (!field.name().equals(UUID_FIELD)) { putUnchanged(delta, field.name()); GenericRecord subDelta = new GenericData.Record(nextField.getFieldSchema()); fillDeltaArrayFields(subDelta, resetFields, uuidFields, fieldQueue); delta.put(field.name(), subDelta); } else if (!field.name().equals(UUID_FIELD)) { putUnchanged(delta, field.name());
private static Object getAvroValue(Object input, Schema schema) { if (input instanceof byte[] && Schema.Type.FIXED.equals(schema.getType())) { return new GenericData.Fixed(schema, (byte[]) input); } else if (input instanceof Map && !((Map) input).isEmpty()) { GenericRecord result; result = new GenericData.Record(schema); for (Map.Entry<String, Object> entry: ((Map<String, Object>) input).entrySet()) { result.put(entry.getKey(), getAvroValue(entry.getValue(), schema.getField(entry.getKey()).schema())); } return result; } else if (input instanceof Collection && !((Collection) input).isEmpty()) { // for array even though we(Schema in streamline registry) support different types of elements in an array, avro expects an array // schema to have elements of same type. Hence, for now we will restrict array to have elements of same type. Other option is convert // a streamline Schema Array field to Record in avro. However, with that the issue is that avro Field constructor does not allow a // null name. We could potentiall hack it by plugging in a dummy name like arrayfield, but seems hacky so not taking that path List<Object> values = new ArrayList<>(((Collection) input).size()); for (Object value: (Collection) input) { values.add(getAvroValue(value, schema.getElementType())); } return new GenericData.Array<Object>(schema, values); } else { return input; } } }
public static void fillArrayItemUpdateDelta(GenericRecord item) { byte[] rawItemUuid2 = new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}; GenericFixed itemUuid2 = new GenericData.Fixed(item .getSchema().getField("__uuid").schema(), rawItemUuid2); item.put("__uuid", itemUuid2); item.put("testField2", 22); }
"\"name\":\"Fixed2\",\"size\":2,\"logicalType\":\"decimal\",\"precision\":4,\"scale\":2}}]}"; final Schema schema = new Schema.Parser().parse(schemaString); GenericRecord addr = new GenericData.Record(schema.getField("type_nested").schema().getTypes().get(1)); addr.put("num", 42); addr.put("street", "Main Street 42"); addr.put("city", "Test City"); addr.put("state", "Test State"); addr.put("zip", "12345"); user.put("type_array_boolean", Arrays.asList(true, true, false)); user.put("type_nullable_array", null); user.put("type_enum", new GenericData.EnumSymbol(schema.getField("type_enum").schema(), "RED")); user.put("type_map", Collections.singletonMap("test", 12L)); user.put("type_fixed", new Fixed16(new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16})); ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray())); user.put("type_decimal_fixed", new GenericData.Fixed( schema.getField("type_decimal_fixed").schema(), BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
public static void fillArrayItemRemoveDelta(GenericRecord delta) { GenericArray testField1 = new GenericData.Array(1, getArraySchema(delta, "testField1")); byte[] rawUuidToRemove = new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}; GenericFixed uuidToRemove = new GenericData.Fixed(getSchemaByFullName( testField1.getSchema().getElementType().getTypes(), "org.kaaproject.configuration.uuidT"), rawUuidToRemove); testField1.add(uuidToRemove); delta.put("testField1", testField1); }
private Object serializePrimitive(TypeInfo typeInfo, PrimitiveObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException { switch(fieldOI.getPrimitiveCategory()) { case BINARY: if (schema.getType() == Type.BYTES){ return AvroSerdeUtils.getBufferFromBytes((byte[])fieldOI.getPrimitiveJavaObject(structFieldData)); } else if (schema.getType() == Type.FIXED){ Fixed fixed = new GenericData.Fixed(schema, (byte[])fieldOI.getPrimitiveJavaObject(structFieldData)); return fixed; } else { throw new AvroSerdeException("Unexpected Avro schema for Binary TypeInfo: " + schema.getType());
if (recordSchema.getType() == Type.FIXED){ Fixed fixed = (Fixed) datum; return fixed.bytes(); } else if (recordSchema.getType() == Type.BYTES){ return AvroSerdeUtils.getBytesFromByteBuffer((ByteBuffer) datum); } else { throw new AvroSerdeException("Unexpected Avro schema for Binary TypeInfo: " + recordSchema.getType());
@SuppressWarnings("unchecked") @Override public Object get(final int pos) throws ExecException { Schema s = avroObject.getSchema().getFields().get(pos).schema(); Object o = avroObject.get(pos); switch(s.getType()) { case STRING: // unwrap avro UTF8 encoding return o.toString(); case MAP: return new AvroMapWrapper((Map<CharSequence, Object>) o); case RECORD: return new AvroTupleWrapper<T>((T) o); case ENUM: return o.toString(); case ARRAY: return new AvroBagWrapper<GenericData.Record>( (GenericArray<GenericData.Record>) o); case FIXED: return new DataByteArray(((GenericData.Fixed) o).bytes()); case BYTES: return new DataByteArray(((ByteBuffer) o).array()); case UNION: return getPigObject(o); default: return o; } }
break; GenericData.Fixed fixed = new GenericData.Fixed(avroSchema); fixed.bytes(bytes); result = fixed; break; DataMap map = (DataMap) value; DataSchema valueDataSchema = ((MapDataSchema) dereferencedDataSchema).getValues(); Schema valueAvroSchema = avroSchema.getValueType(); Map<String, Object> avroMap = new HashMap<String, Object>(map.size()); for (Map.Entry<String, Object> entry : map.entrySet()) DataList list = (DataList) value; DataSchema elementDataSchema = ((ArrayDataSchema) dereferencedDataSchema).getItems(); Schema elementAvroSchema = avroSchema.getElementType(); GenericData.Array<Object> avroList = new GenericData.Array<Object>(list.size(), avroSchema); for (int i = 0; i < list.size(); i++) Schema.Field avroField = avroSchema.getField(fieldName); if (avroField == null) Schema fieldAvroSchema = avroField.schema(); Object fieldValue = map.get(fieldName); boolean isOptional = field.getOptional();
@Override public void writeValue(Object value) throws JsonMappingException { _verifyValueWrite(); if (_nextField != null) { Schema schema = _nextField.schema(); if (schema.getType() == Schema.Type.FIXED) { // 13-Nov-2014 josh: AvroGenerator wraps all binary values in ByteBuffers, // but avro wants FIXED, so rewrap the array, copying if necessary ByteBuffer bb = (ByteBuffer) value; byte[] bytes = bb.array(); if (bb.arrayOffset() != 0 || bb.remaining() != bytes.length) { bytes = Arrays.copyOfRange(bytes, bb.arrayOffset(), bb.remaining()); } value = new GenericData.Fixed(schema, bytes); } _record.put(_nextField.pos(), value); } }
Schema fixed = Schema.createFixed("myfixed", null, null, 2); Schema enumeration = Schema.createEnum("myenum", null, null, Lists.newArrayList("a", "b")); colGenerator(true, Schema.create(Schema.Type.BOOLEAN), true, "BIT"), colGenerator(100, Schema.create(Schema.Type.INT), 100, "INTEGER"), colGenerator(200L, Schema.create(Schema.Type.LONG), 200L, "BIGINT"), colGenerator(ByteBuffer.wrap(b), Schema.create(Schema.Type.BYTES), b, "VARBINARY(8)"), colGenerator(new GenericData.Fixed(fixed, b), fixed, b, "BINARY(2)"), colGenerator(new GenericData.EnumSymbol(enumeration, "a"), enumeration,
assertNull(actual.get("nullField")); assertEquals(Arrays.asList(10.0, 20.0), actual.get("arrayField")); GenericData.Record expected = new GenericData.Record(schema.getField("mapField").schema().getValueType()); expected.put("label", "car"); assertEquals(ImmutableMap.of("myMap", expected), actual.get("mapField")); assertEquals("B", actual.get("enumField")); assertEquals( new GenericData.Fixed(schema.getField("fixedField").schema(), fixed), actual.get("fixedField")); expected = new GenericData.Record(schema.getField("recordField").schema()); expected.put("label", "house"); expected.put("children", new ArrayList(Arrays.asList(ByteBuffer.wrap(bytes))));
private Object nextValue(Schema s, int column) throws IOException { Object v = values[column].nextValue(); switch (s.getType()) { case ENUM: return new GenericData.EnumSymbol(s, s.getEnumSymbols().get((Integer)v)); case FIXED: return new GenericData.Fixed(s, ((ByteBuffer)v).array()); } return v; }