@Override void readInternal(Decoder d) throws IOException { for (int i = 0; i < count; i++) { reader.read(reuse, d); } } @Override
/** * Decode json data. * * @param data the data * @param reuse the reuse * @return the decoded object * @throws IOException Signals that an I/O exception has occurred. */ public T decodeJson(String data, T reuse) throws IOException { jsonDecoder = DecoderFactory.get().jsonDecoder(this.schema, data, true); return avroReader.read(null, jsonDecoder); }
/** * Creates object from byte array * * @param data the data * @param reuse object to reuse * @return the result of conversion * @throws IOException Signals that an I/O exception has occurred. */ public T fromByteArray(byte[] data, T reuse) throws IOException { decoder = DecoderFactory.get().binaryDecoder(data, decoder); return avroReader.read(reuse, decoder); } }
@Override public AvroJobSpec deserialize(String topic, byte[] data) { try (InputStream is = new ByteArrayInputStream(data)) { _versionWriter.readSchemaVersioningInformation(new DataInputStream(is)); Decoder decoder = DecoderFactory.get().binaryDecoder(is, _decoder); return _reader.read(null, decoder); } catch (IOException e) { throw new RuntimeException("Could not decode message"); } }
private AvroJobSpec decodeRecord(ByteArrayBasedKafkaRecord kafkaConsumerRecord) throws IOException { InputStream is = new ByteArrayInputStream(kafkaConsumerRecord.getMessageBytes()); _versionWriter.readSchemaVersioningInformation(new DataInputStream(is)); Decoder decoder = DecoderFactory.get().binaryDecoder(is, _decoder); return _reader.read(null, decoder); }
@Override public Collection<Either<JobSpec, URI>> parseJobSpec(byte[] message) throws IOException { InputStream is = new ByteArrayInputStream(message); this.versionWriter.readSchemaVersioningInformation(new DataInputStream(is)); Decoder decoder = DecoderFactory.get().binaryDecoder(is, this.decoder.get()); try { T decodedMessage = this.reader.get().read(null, decoder); return parseJobSpec(decodedMessage); } catch (AvroRuntimeException | IOException exc) { this.messageParseFailures.mark(); if (this.messageParseFailures.getFiveMinuteRate() < 1) { log.warn("Unable to decode input message.", exc); } else { log.warn("Unable to decode input message."); } return Lists.newArrayList(); } }
private void parseTopics() { if (state.getProperty(TOPIC_LIST) != null) { byte[] data = base64.decodeBase64(state.getProperty(TOPIC_LIST)); BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); SpecificDatumReader<Topic> avroReader = new SpecificDatumReader<>(Topic.class); try { // NOSONAR Topic decodedTopic; while (!decoder.isEnd()) { decodedTopic = avroReader.read(null, decoder); LOG.debug("Loaded {}", decodedTopic); topicMap.put(decodedTopic.getId(), decodedTopic); } } catch (Exception ex) { LOG.error("Unexpected exception occurred while reading information from decoder", ex); } } else { LOG.info("No topic list found in state"); } }
/** * Decode records from <code>ByteBuffer</code> and return <code>Iterable</code>. * * @param bb the <code>ByteBuffer</code> which decode * @return decoded <code>Iterable</code> */ public Iterable<T> decodeRecords(ByteBuffer bb) throws IOException { byte[] recordDataBody = toByteArray(bb); BinaryDecoder recordDataDecoder = DecoderFactory.get().binaryDecoder(recordDataBody, null); RecordData recordData = recordDataReader.read(null, recordDataDecoder); LOG.debug("Avro event header: {}", recordData.getRecordHeader()); LOG.debug("Avro event data.size: {}", recordData.getEventRecords().size()); List<T> results = new ArrayList<T>(); BinaryDecoder recordDecoder = null; for (ByteBuffer eventBuf : recordData.getEventRecords()) { byte[] recordBody = toByteArray(eventBuf); recordDecoder = DecoderFactory.get().binaryDecoder(recordBody, recordDecoder); T record = recordReader.read(null, recordDecoder); results.add(record); LOG.trace("Parsed record: {}", record); } return results; }
private Event deserializeValue(byte[] value, boolean parseAsFlumeEvent) throws IOException { Event e; if (parseAsFlumeEvent) { ByteArrayInputStream in = new ByteArrayInputStream(value); decoder = DecoderFactory.get().directBinaryDecoder(in, decoder); if (!reader.isPresent()) { reader = Optional.of( new SpecificDatumReader<AvroFlumeEvent>(AvroFlumeEvent.class)); } AvroFlumeEvent event = reader.get().read(null, decoder); e = EventBuilder.withBody(event.getBody().array(), toStringMap(event.getHeaders())); } else { e = EventBuilder.withBody(value, Collections.EMPTY_MAP); } return e; } }
@Override public void readExternal(ObjectInput in) throws IOException { new SpecificDatumReader(getSchema()) .read(this, SpecificData.getDecoder(in)); }
private <T extends SpecificRecordBase> T deserialize(byte[] bytes, SpecificDatumReader<T> datumReader) { try { final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); return datumReader.read(null, DecoderFactory.get().directBinaryDecoder(byteArrayInputStream, null)); } catch (IOException e) { throw new RuntimeException(e); } } }
@Override public void readExternal(ObjectInput in) throws IOException { new SpecificDatumReader(getSchema()) .read(this, SpecificData.getDecoder(in)); } }
@Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { int size = in.readInt(); byte[] data = new byte[size]; in.read(data); @SuppressWarnings("unchecked") AvroReader<T> reader = (AvroReader<T>) recordReaderMap.get().get(className); if (reader == null) { reader = new AvroReader<T>( new SpecificDatumReader<T>(clazz), DecoderFactory.get().binaryDecoder(data, null) ); recordReaderMap.get().put(className, reader); } BinaryDecoder recordDataDecoder = DecoderFactory.get().binaryDecoder(data, reader.getDecoder()); avroObject = reader.getReader().read(null, recordDataDecoder); }
public T toObject(byte[] bytes) { Decoder decoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null); SpecificDatumReader<T> reader = null; try { reader = new SpecificDatumReader<T>(clazz); return reader.read(null, decoder); } catch(IOException e) { throw new SerializationException(e); } } }
@Override public Object[] parse(InputStream stream) { try { BinaryDecoder in = DECODER_FACTORY.binaryDecoder(stream, null); Schema reqSchema = message.getRequest(); GenericRecord request = (GenericRecord) new SpecificDatumReader<>(reqSchema).read(null, in); Object[] args = new Object[reqSchema.getFields().size()]; int i = 0; for (Schema.Field field : reqSchema.getFields()) { args[i++] = request.get(field.name()); } return args; } catch (IOException e) { throw Status.INTERNAL.withCause(e). withDescription("Error deserializing avro request arguments").asRuntimeException(); } finally { AvroGrpcUtils.skipAndCloseQuietly(stream); } }
/** * Process incoming flume event. * * @param event the event for processing */ public Map<KaaSinkKey, List<KaaRecordEvent>> processIncomingFlumeEvent(Event event) throws IOException { final Map<KaaSinkKey, List<KaaRecordEvent>> eventsMap = new LinkedHashMap<>(); byte[] body = event.getBody(); decoder = DecoderFactory.get().binaryDecoder(body, decoder); RecordData data = avroReader.read(null, decoder); KaaSinkKey sinkKey = new KaaSinkKey(data.getApplicationToken(), data.getSchemaVersion()); Map<String, String> headers = headersMap.get(sinkKey); if (headers == null) { headers = new HashMap<>(); sinkKey.updateHeaders(headers); headersMap.put(sinkKey, headers); } List<KaaRecordEvent> events = Lists.newArrayList(); for (ByteBuffer eventData : data.getEventRecords()) { KaaRecordEvent kaaRecordEvent = new KaaRecordEvent(data.getRecordHeader(), headers, eventData.array()); events.add(kaaRecordEvent); } eventsMap.put(sinkKey, events); return eventsMap; }
@Override public Object parse(InputStream stream) { try { if (message.isOneWay()) return null; BinaryDecoder in = DECODER_FACTORY.binaryDecoder(stream, null); if (!in.readBoolean()) { Object response = new SpecificDatumReader(message.getResponse()).read(null, in); return response; } else { Object value = new SpecificDatumReader(message.getErrors()).read(null, in); if (value instanceof Exception) { return value; } return new AvroRuntimeException(value.toString()); } } catch (IOException e) { throw Status.INTERNAL.withCause(e). withDescription("Error deserializing avro response").asRuntimeException(); } finally { AvroGrpcUtils.skipAndCloseQuietly(stream); } }
void input(ByteBuffer data, long count) { try { decoder = decoderFactory.binaryDecoder(data.array(), decoder); for (long i = 0; i < count; i++) { switch (taskType) { case MAP: inRecord = inReader.read(inRecord, decoder); map(inRecord, midCollector); break; case REDUCE: MID prev = midRecord; midRecord = midReader.read(midRecordSpare, decoder); if (prev != null && !midRecord.equals(prev)) reduceFlush(prev, outCollector); reduce(midRecord, outCollector); midRecordSpare = prev; break; } } } catch (Throwable e) { LOG.warn("failing: "+e, e); fail(e.toString()); } }
@Test public void testStringables() throws IOException { StringablesRecord.Builder newBuilder = StringablesRecord.newBuilder(); newBuilder.setValue(new BigDecimal("42.11")); HashMap<String, BigDecimal> mapWithBigDecimalElements = new HashMap<>(); mapWithBigDecimalElements.put("test", new BigDecimal("11.11")); newBuilder.setMapWithBigDecimalElements(mapWithBigDecimalElements); HashMap<BigInteger, String> mapWithBigIntKeys = new HashMap<>(); mapWithBigIntKeys.put(BigInteger.ONE, "test"); newBuilder.setMapWithBigIntKeys(mapWithBigIntKeys); StringablesRecord stringablesRecord = newBuilder.build(); byte[] recordBytes = serializeRecord(stringablesRecord); Decoder decoder = DecoderFactory.get().binaryDecoder(recordBytes, null); SpecificDatumReader<StringablesRecord> specificDatumReader = new SpecificDatumReader<>(StringablesRecord.SCHEMA$); StringablesRecord deserialized = new StringablesRecord(); specificDatumReader.read(deserialized, decoder); assertEquals(stringablesRecord, deserialized); }
@Test public void testRead() throws IOException { Builder newBuilder = FooBarSpecificRecord.newBuilder(); newBuilder.setId(42); newBuilder.setName("foo"); newBuilder.setNicknames(Arrays.asList("bar")); newBuilder.setRelatedids(Arrays.asList(1,2,3)); FooBarSpecificRecord specificRecord = newBuilder.build(); byte[] recordBytes = serializeRecord(specificRecord); Decoder decoder = DecoderFactory.get().binaryDecoder(recordBytes, null); SpecificDatumReader<FooBarSpecificRecord> specificDatumReader = new SpecificDatumReader<>(FooBarSpecificRecord.SCHEMA$); FooBarSpecificRecord deserialized = new FooBarSpecificRecord(); specificDatumReader.read(deserialized, decoder); assertEquals(specificRecord, deserialized); }