@Override public Encoder apply(OutputStream input) { return new BinaryEncoder(input); } };
@Override public Encoder writeBytes(byte[] bytes) throws IOException { return writeBytes(bytes, 0, bytes.length); }
@Override public Encoder writeBytes(ByteBuffer buffer) throws IOException { writeInt(buffer.remaining()); if (buffer.hasArray()) { output.write(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); } else { byte[] bytes = new byte[buffer.remaining()]; int pos = buffer.position(); buffer.get(bytes); output.write(bytes); buffer.position(pos); } return this; } }
@Override public Encoder writeBytes(byte[] bytes, int off, int len) throws IOException { writeLong(len); output.write(bytes, off, len); return this; }
@Override public Encoder writeBytes(byte[] bytes, int off, int len) throws IOException { writeLong(len); output.write(bytes, off, len); return this; }
private static byte[] serializeEmptyHashKeys() { try { // we don't synchronize here: the worst thing that go wrong here is repeated assignment to the same value ByteArrayOutputStream bos = new ByteArrayOutputStream(); Encoder encoder = new BinaryEncoder(bos); encoder.writeInt(0); return bos.toByteArray(); } catch (IOException e) { throw new RuntimeException("encoding empty hash keys went wrong - bailing out: " + e.getMessage(), e); } } private static final byte[] SERIALIZED_EMPTY_HASH_KEYS = serializeEmptyHashKeys();
@Override public Encoder writeBytes(byte[] bytes) throws IOException { return writeBytes(bytes, 0, bytes.length); }
@Override public Encoder writeBytes(ByteBuffer buffer) throws IOException { writeInt(buffer.remaining()); if (buffer.hasArray()) { output.write(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); } else { byte[] bytes = new byte[buffer.remaining()]; int pos = buffer.position(); buffer.get(bytes); output.write(bytes); buffer.position(pos); } return this; } }
private byte[] encode(T object) { // encode T using schema ByteArrayOutputStream bos = new ByteArrayOutputStream(); BinaryEncoder encoder = new BinaryEncoder(bos); try { this.datumWriter.encode(object, encoder); } catch (IOException e) { // SHOULD NEVER happen throw new DataSetException("Failed to encode object to be written: " + e.getMessage(), e); } return bos.toByteArray(); }
@Override public Encoder writeString(String s) throws IOException { return writeBytes(Charsets.UTF_8.encode(s)); }
private byte[] encode(T object) { // encode T using schema ByteArrayOutputStream bos = new ByteArrayOutputStream(); BinaryEncoder encoder = new BinaryEncoder(bos); try { this.datumWriter.encode(object, encoder); } catch (IOException e) { // SHOULD NEVER happen throw new DataSetException("Failed to encode object to be written: " + e.getMessage(), e); } return bos.toByteArray(); }
@Override public Encoder writeString(String s) throws IOException { return writeBytes(Charsets.UTF_8.encode(s)); }
@Override public void emit(T data, Map<String, Object> partitions) { try { ByteArrayOutputStream output = new ByteArrayOutputStream(); output.write(schemaHash); writer.encode(data, new BinaryEncoder(output)); producerSupplier.get().enqueue(new QueueEntry(Maps.transformValues(partitions, PARTITION_MAP_TRANSFORMER), output.toByteArray())); } catch (IOException e) { throw Throwables.propagate(e); } }
public static byte[] serializeHashKeys(Map<String, Integer> hashKeys) throws IOException { // many entries will have no hash keys. Reuse a static value for that if (hashKeys == null || hashKeys.isEmpty()) { return SERIALIZED_EMPTY_HASH_KEYS; } ByteArrayOutputStream bos = new ByteArrayOutputStream(); Encoder encoder = new BinaryEncoder(bos); encoder.writeInt(hashKeys.size()); for (Map.Entry<String, Integer> entry : hashKeys.entrySet()) { encoder.writeString(entry.getKey()).writeInt(entry.getValue()); } encoder.writeInt(0); // per Avro spec, end with a (block of length) zero return bos.toByteArray(); }
@Inject MessagingMetricsCollectionService(CConfiguration cConf, MessagingService messagingService, DatumWriter<MetricValues> recordWriter) { String topicPrefix = cConf.get(Constants.Metrics.TOPIC_PREFIX); int totalTopicNum = cConf.getInt(Constants.Metrics.MESSAGING_TOPIC_NUM); Preconditions.checkArgument(totalTopicNum > 0, "Constants.Metrics.MESSAGING_TOPIC_NUM must be a positive integer"); this.messagingService = messagingService; this.recordWriter = recordWriter; // Parent guarantees the publish method would not get called concurrently, hence safe to reuse the same instances. this.encoderOutputStream = new ByteArrayOutputStream(1024); this.encoder = new BinaryEncoder(encoderOutputStream); RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, "system.metrics."); this.topicPayloads = new LinkedHashMap<>(totalTopicNum); for (int i = 0; i < totalTopicNum; i++) { topicPayloads.put(i, new TopicPayload(NamespaceId.SYSTEM.topic(topicPrefix + i), retryStrategy)); } }
@Inject MessagingMetricsCollectionService(CConfiguration cConf, MessagingService messagingService, DatumWriter<MetricValues> recordWriter) { super(TimeUnit.SECONDS.toMillis(cConf.getInt(Constants.Metrics.METRICS_MINIMUM_RESOLUTION_SECONDS))); String topicPrefix = cConf.get(Constants.Metrics.TOPIC_PREFIX); int totalTopicNum = cConf.getInt(Constants.Metrics.MESSAGING_TOPIC_NUM); Preconditions.checkArgument(totalTopicNum > 0, "Constants.Metrics.MESSAGING_TOPIC_NUM must be a positive integer"); this.messagingService = messagingService; this.recordWriter = recordWriter; // Parent guarantees the publish method would not get called concurrently, hence safe to reuse the same instances. this.encoderOutputStream = new ByteArrayOutputStream(1024); this.encoder = new BinaryEncoder(encoderOutputStream); RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, "system.metrics."); this.topicPayloads = new LinkedHashMap<>(totalTopicNum); for (int i = 0; i < totalTopicNum; i++) { topicPayloads.put(i, new TopicPayload(NamespaceId.SYSTEM.topic(topicPrefix + i), retryStrategy)); } }
@Override public byte[] encode(KeyIdentifier keyIdentifier) throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); Encoder encoder = new BinaryEncoder(bos); encoder.writeInt(KeyIdentifier.Schemas.getVersion()); DatumWriter<KeyIdentifier> writer = writerFactory.create(KEY_IDENTIFIER_TYPE, KeyIdentifier.Schemas.getCurrentSchema()); writer.encode(keyIdentifier, encoder); return bos.toByteArray(); }
@Override public byte[] encode(AccessTokenIdentifier identifier) throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); Encoder encoder = new BinaryEncoder(bos); encoder.writeInt(AccessTokenIdentifier.Schemas.getVersion()); DatumWriter<AccessTokenIdentifier> writer = writerFactory.create(ACCESS_TOKEN_IDENTIFIER_TYPE, AccessTokenIdentifier.Schemas.getCurrentSchema()); writer.encode(identifier, encoder); return bos.toByteArray(); }
@Override public byte[] encode(AccessToken token) throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); Encoder encoder = new BinaryEncoder(bos); encoder.writeInt(AccessToken.Schemas.getVersion()); DatumWriter<AccessToken> writer = writerFactory.create(ACCESS_TOKEN_TYPE, AccessToken.Schemas.getCurrentSchema()); writer.encode(token, encoder); return bos.toByteArray(); }
/** * Encodes the given {@link StreamEvent} into {@code byte[]} that could become * payload of a QueueEntry. * * @param event The {@link StreamEvent} to encode. * @return Encoded {@code byte[]}. */ public byte[] encodePayload(StreamEvent event) { // TODO: This is a bit hacky to do it directly for now, for performance reason. ByteBuffer body = event.getBody(); Map<String, String> headers = event.getHeaders(); long timestamp = event.getTimestamp(); // Some assumption on the header size to minimize array copying // 16 bytes Schema hash + body size + (header size) * (50 bytes key/value pair) + 9 bytes timestamp (vlong encoding) ByteArrayOutputStream os = new ByteArrayOutputStream(16 + body.remaining() + headers.size() * 50 + 9); Encoder encoder = new BinaryEncoder(os); try { // Write the schema hash os.write(STREAM_EVENT_SCHEMA.getSchemaHash().toByteArray()); StreamEventDataCodec.encode(event, encoder); encoder.writeLong(timestamp); return os.toByteArray(); } catch (IOException e) { // It should never happens, otherwise something very wrong. throw Throwables.propagate(e); } }