protected int compare(Object o1, Object o2, Schema s, boolean equals) { if (o1 == o2) return 0; switch (s.getType()) { case RECORD: for (Field f : s.getFields()) { if (f.order() == Field.Order.IGNORE) continue; // ignore this field int pos = f.pos(); String name = f.name(); int compare = compare(getField(o1, name, pos), getField(o2, name, pos), return s.getEnumOrdinal(o1.toString()) - s.getEnumOrdinal(o2.toString()); case ARRAY: Collection a1 = (Collection)o1; return 0; case STRING: Utf8 u1 = o1 instanceof Utf8 ? (Utf8)o1 : new Utf8(o1.toString()); Utf8 u2 = o2 instanceof Utf8 ? (Utf8)o2 : new Utf8(o2.toString()); return u1.compareTo(u2); default: return ((Comparable)o1).compareTo(o2);
@Override public Object getField(String fieldName) { Object value = record.get(fieldName); if (value instanceof Utf8) { return ((Utf8) value).toString(); } else if (value instanceof org.apache.avro.generic.GenericRecord) { org.apache.avro.generic.GenericRecord avroRecord = (org.apache.avro.generic.GenericRecord) value; org.apache.avro.Schema recordSchema = avroRecord.getSchema(); List<Field> fields = recordSchema.getFields() .stream() .map(f -> new Field(f.name(), f.pos())) .collect(Collectors.toList()); return new GenericAvroRecord(schema, fields, avroRecord); } else { return value; } }
@Override public Utf8 readString(Utf8 old) throws IOException { int length = readInt(); Utf8 result = (old != null ? old : new Utf8()); result.setByteLength(length); if (length > 0) { in.readFully(result.getBytes(), 0, length); } return result; }
@Override public void writeString(Utf8 utf8) throws IOException { writeBytes(utf8.getBytes(), 0, utf8.getByteLength()); }
@Test public void testArrayReusedWhenLargerThanRequestedSize() { byte[] bs = "55555".getBytes(StandardCharsets.UTF_8); Utf8 u = new Utf8(bs); assertEquals(5, u.getByteLength()); byte[] content = u.getBytes(); u.setByteLength(3); assertEquals(3, u.getByteLength()); assertSame(content, u.getBytes()); u.setByteLength(4); assertEquals(4, u.getByteLength()); assertSame(content, u.getBytes()); } }
switch (schema.getType()) { case ARRAY: List<Object> arrayValue = (List) value; arrayValue.size(), schema); for (Object obj : arrayValue) { arrayCopy.add(deepCopy(schema.getElementType(), obj)); for (Map.Entry<CharSequence, Object> entry : mapValue.entrySet()) { mapCopy.put((CharSequence)(deepCopy(STRINGS, entry.getKey())), deepCopy(schema.getValueType(), entry.getValue())); Object newState = getRecordState(newRecord, schema); for (Field f : schema.getFields()) { int pos = f.pos(); String name = f.name(); Object newValue = deepCopy(f.schema(), getField(value, name, pos, oldState)); setField(newRecord, name, pos, newValue, newState); return new Utf8((Utf8)value); return new Utf8(value.toString()); case UNION: return deepCopy(
for (Field f : PROTOCOL.getType("TestRecord").getFields()) fields.add(new Field(f.name(), f.schema(), null, null)); fields.add(new Field("extra", Schema.create(Schema.Type.BOOLEAN), null, true)); Schema record = Schema.createRecord("TestRecord", null, "org.apache.avro.test", false); record.setFields(fields); Protocol protocol = new Protocol("Simple", "org.apache.avro.test"); List<Field> params = new ArrayList<>(); params.add(new Field("record", record, null, null)); protocol.createMessage("echo", null, Schema.createRecord(params), record, Schema.createUnion(new ArrayList<>())); try { GenericRequestor r = new GenericRequestor(protocol, t); GenericRecord args = new GenericData.Record(message.getRequest()); GenericRecord rec = new GenericData.Record(record); rec.put("name", new Utf8("foo")); rec.put("kind", new GenericData.EnumSymbol (PROTOCOL.getType("Kind"), "BAR")); rec.put("hash", new GenericData.Fixed (PROTOCOL.getType("MD5"), new byte[]{0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5}));
for (final Field f : schema.getFields()) { final String fieldName = f.name(); if (dimensions.contains(fieldName) || metrics.contains(fieldName) || time.equals(fieldName)) { isSingleValueMap.put(fieldName, isSingleValueField(f)); Object value = record.get(column); switch (schema.getField(column).schema().getType()) { case INT: value = 0; value = ((Utf8) value).toString(); final String groupbyKeyBase = column + ":" + record.get(column) + ":" + metricName; int dimCounter = 1; for (final String dim : cardinalityCountsMap.keySet()) { final String groupbyKey = groupbyKeyBase + ":" + dim; if (sumGroupBy.containsKey(groupbyKey)) { if (sumGroupBy.get(groupbyKey).containsKey(record.get(dim))) { sumGroupBy.get(groupbyKey).put(record.get(dim), getAppropriateNumberType(metricName, record.get(metricName),
@Override public String getProperty(String key) throws Exception { if(key == null) { throw new NullPointerException("key"); } Schema schema = _protocol.getMessages().get(ProtocolConstants.MSG_META).getRequest(); GenericRecord req = new GenericData.Record(schema); req.put("src", _sourceUtf8); req.put("opt", ProtocolConstants.OPT_GET_PROPERTY_UTF8); req.put("key", new Utf8(key)); Utf8 res = (Utf8)send(ProtocolConstants.MSG_META, req); return res == null ? null : res.toString(); }
model.addLogicalTypeConversion(manager.getHandler()); Schema parentSchema = Schema.createRecord("Parent", null, null, false); parentFields.add(new Schema.Field("id", Schema.create(Schema.Type.LONG), null, null)); parentFields.add(new Schema.Field("p", Schema.create(Schema.Type.STRING), null, null)); parentFields.add(new Schema.Field("child", childSchema, null, null)); System.out.println("Schema: " + schema.toString(true)); Record child = new Record(childSchema); 1L, actual.get("id")); Assert.assertEquals("Should correctly read back the parent data", new Utf8("parent data!"), actual.get("p")); Record actualChild = (Record) actual.get("child"); Assert.assertEquals("Should correctly read back the child data", new Utf8("child data!"), actualChild.get("c")); Object childParent = actualChild.get("parent"); Assert.assertTrue("Should have a parent Record object", 1L, childParentRecord.get("id")); Assert.assertEquals("Should have the right parent data", new Utf8("parent data!"), childParentRecord.get("p"));
@Test /** Construct and use a different protocol whose "hello" method has an extra argument to check that schema is sent to parse request. */ public void testHandshake() throws IOException { Protocol protocol = new Protocol("Simple", "org.apache.avro.test"); List<Field> fields = new ArrayList<>(); fields.add(new Schema.Field("extra", Schema.create(Schema.Type.BOOLEAN), null, null)); fields.add(new Schema.Field("greeting", Schema.create(Schema.Type.STRING), null, null)); Protocol.Message message = protocol.createMessage("hello", null /* doc */, Schema.createRecord(fields), Schema.create(Schema.Type.STRING), Schema.createUnion(new ArrayList<>())); protocol.getMessages().put("hello", message); Transceiver t = new SocketTransceiver(new InetSocketAddress(server.getPort())); try { GenericRequestor r = new GenericRequestor(protocol, t); GenericRecord params = new GenericData.Record(message.getRequest()); params.put("extra", Boolean.TRUE); params.put("greeting", new Utf8("bob")); Utf8 response = (Utf8)r.request("hello", params); assertEquals(new Utf8("goodbye"), response); } finally { t.close(); } }
public GenericRecord getValueRecord(Schema valueSchema, GenericDBWritable val) { GenericRecord valueRecord = new GenericData.Record(valueSchema); List<Schema.Field> fieldSchemas = valueSchema.getFields(); for (int i = 0; i < val.getValues().size(); ++i) { Schema.Type type = fieldSchemas.get(i).schema().getType(); if (type.equals(Schema.Type.STRING)) { Utf8 utf8 = new Utf8((String) val.getValues().get(i).toString()); valueRecord.put(i, utf8); } else { valueRecord.put(i, val.getValues().get(i)); } } return valueRecord; }
final String[] columns = new String[avroSchema.getFields().size()]; int i = 0; for (final Field f : avroSchema.getFields()) { columns[i] = f.name(); i++; final GenericRecord rec = avroReader.next(); for (final String column : columns) { Object val = rec.get(column); if (val instanceof Utf8) { val = ((Utf8) val).toString();
@Test public void testEquals() { Schema s = recordSchema(); GenericRecord r0 = new GenericData.Record(s); GenericRecord r1 = new GenericData.Record(s); GenericRecord r2 = new GenericData.Record(s); Collection<CharSequence> l0 = new ArrayDeque<>(); List<CharSequence> l1 = new ArrayList<>(); GenericArray<CharSequence> l2 = new GenericData.Array<>(1, s.getFields().get(0).schema()); String foo = "foo"; l0.add(new StringBuffer(foo)); l1.add(foo); l2.add(new Utf8(foo)); r0.put(0, l0); r1.put(0, l1); r2.put(0, l2); assertEquals(r0, r1); assertEquals(r0, r2); assertEquals(r1, r2); }
@Test public void testGetStringArrayUtf8() throws IOException { // Expectation: Even though we read an Avro object with UTF8 underneath, the accessor converts it into a // Java String List<String> expectedQuotes = ImmutableList.of("abc", "defg"); GenericData.Array<Utf8> strings = new GenericData.Array<Utf8>(2, Schema.createArray(Schema.create(Schema.Type.STRING))); expectedQuotes.forEach(s -> strings.add(new Utf8(s))); record.put("favorite_quotes", strings); Assert.assertEquals(accessor.getGeneric("favorite_quotes"), expectedQuotes); }
public Object respond(Message message, Object request) throws AvroRemoteException { GenericRecord params = (GenericRecord)request; if ("hello".equals(message.getName())) { LOG.info("hello: "+params.get("greeting")); return new Utf8("goodbye"); } if ("echo".equals(message.getName())) { Object record = params.get("record"); LOG.info("echo: "+record); return record; } if ("echoBytes".equals(message.getName())) { Object data = params.get("data"); LOG.info("echoBytes: "+data); return data; } if ("error".equals(message.getName())) { if (throwUndeclaredError) throw new RuntimeException("foo"); GenericRecord error = new GenericData.Record(PROTOCOL.getType("TestError")); error.put("message", new Utf8("an error")); throw new AvroRemoteException(error); } throw new AvroRuntimeException("unexpected message: "+message.getName()); }
@Test public void testHello() throws IOException { GenericRecord params = new GenericData.Record(PROTOCOL.getMessages().get("hello").getRequest()); params.put("greeting", new Utf8("bob")); Utf8 response = (Utf8)requestor.request("hello", params); assertEquals(new Utf8("goodbye"), response); }
@Test public void testSerialization() throws Exception { Schema testSchema = getTestSchema(); GenericRecord message = new Record(testSchema); message.put("name", "testValue"); byte[] data = getSerializedMessage(message, testSchema); GenericDatumReader<IndexedRecord> reader = new GenericDatumReader<>(testSchema); SeekableInput in = new SeekableByteArrayInput(data); FileReader<IndexedRecord> dfr = null; IndexedRecord result = null; try { dfr = DataFileReader.openReader(in, reader); result = dfr.next(); } finally { if (dfr != null) { dfr.close(); } } Assert.assertNotNull(result); Assert.assertTrue(result instanceof GenericRecord); Assert.assertEquals(new Utf8("testValue"), ((GenericRecord) result).get("name")); } }
private byte[] createDataFile(byte[] sync) throws IOException { Schema schema = Schema.create(Type.STRING); DataFileWriter<Utf8> w = new DataFileWriter<>(new GenericDatumWriter<>(schema)); ByteArrayOutputStream baos = new ByteArrayOutputStream(); w.create(schema, baos, sync); w.append(new Utf8("apple")); w.append(new Utf8("banana")); w.sync(); w.append(new Utf8("celery")); w.append(new Utf8("date")); w.sync(); w.append(new Utf8("endive")); w.append(new Utf8("fig")); w.close(); return baos.toByteArray(); }
assertEquals("The schemas should be equal", userSchema, u.getSchema()); String name = u.get("name").toString(); assertNotNull("empty record", name); assertEquals("name not equal", TEST_NAME, name); List<CharSequence> sl = (List<CharSequence>) u.get("type_array_string"); assertEquals("element 0 not equal", TEST_ARRAY_STRING_1, sl.get(0).toString()); assertEquals("element 1 not equal", TEST_ARRAY_STRING_2, sl.get(1).toString()); Map<CharSequence, Long> lm = (Map<CharSequence, Long>) u.get("type_map"); assertEquals("map value of key 1 not equal", TEST_MAP_VALUE1, lm.get(new Utf8(TEST_MAP_KEY1)).longValue()); assertEquals("map value of key 2 not equal", TEST_MAP_VALUE2, lm.get(new Utf8(TEST_MAP_KEY2)).longValue());