@Override public String apply(PubsubMessage input) { return new String(input.getPayload(), StandardCharsets.UTF_8); } }
@Override public void encode(PubsubMessage value, OutputStream outStream, Context context) throws IOException { PAYLOAD_CODER.encode(value.getPayload(), outStream, context); }
@Override public T apply(PubsubMessage input) { try { return CoderUtils.decodeFromByteArray(coder, input.getPayload()); } catch (CoderException e) { throw new RuntimeException("Could not decode Pubsub message", e); } } }
private PubsubClient.OutgoingMessage toOutgoingMessage(PubsubMessage message) { return new PubsubClient.OutgoingMessage( message.getPayload(), message.getAttributeMap(), DateTime.now().getMillis(), null); } }
@DoFn.ProcessElement public void processElement(ProcessContext c) throws IOException { if (schema == null) { schema = new Schema.Parser().parse(schemaStr); datumReader = new GenericDatumReader<GenericRecord>(schema); } decoder = DecoderFactory.get().binaryDecoder(c.element().getPayload(), decoder); GenericRecord record = datumReader.read(null, decoder); c.output(record); } }
@DoFn.ProcessElement public void processElement(ProcessContext c) { String record = new String(c.element().getPayload(), Charset.forName("UTF-8")); c.output(record.split(fieldDelimiter)); }
private static String data(PubsubMessage message) { return new String(message.getPayload(), StandardCharsets.UTF_8); }
@ProcessElement public void processElement(ProcessContext context) { PubsubMessage message = context.element(); context.output( FailsafeElement.of(message, new String(message.getPayload(), StandardCharsets.UTF_8))); } }
public void encode(PubsubMessage value, OutputStream outStream, Context context) throws IOException { PAYLOAD_CODER.encode(value.getPayload(), outStream); ATTRIBUTES_CODER.encode(value.getAttributeMap(), outStream, context); }
private Row parsePayloadJsonRow(PubsubMessage pubsubMessage) { String payloadJson = new String(pubsubMessage.getPayload(), StandardCharsets.UTF_8); if (objectMapper == null) { objectMapper = newObjectMapperWith(RowJsonDeserializer.forSchema(payloadSchema())); } return JsonToRowUtils.jsonToRow(objectMapper, payloadJson); }
private static boolean messagesEqual(PubsubMessage message1, PubsubMessage message2) { return message1.getAttributeMap().equals(message2.getAttributeMap()) && Arrays.equals(message1.getPayload(), message2.getPayload()); }
@ProcessElement public void processElement(ProcessContext c) throws Exception { elementCounter.inc(); PubsubMessage message = c.element(); byte[] elementBytes = message.getPayload(); Map<String, String> attributes = message.getAttributeMap(); long timestampMsSinceEpoch = c.timestamp().getMillis(); @Nullable String recordId = null; switch (recordIdMethod) { case NONE: break; case DETERMINISTIC: recordId = Hashing.murmur3_128().hashBytes(elementBytes).toString(); break; case RANDOM: // Since these elements go through a GroupByKey, any failures while sending to // Pubsub will be retried without falling back and generating a new record id. // Thus even though we may send the same message to Pubsub twice, it is guaranteed // to have the same record id. recordId = UUID.randomUUID().toString(); break; } c.output( KV.of( ThreadLocalRandom.current().nextInt(numShards), new OutgoingMessage(elementBytes, attributes, timestampMsSinceEpoch, recordId))); }
@ProcessElement public void processElement(ProcessContext context) { FailsafeElement<PubsubMessage, String> failsafeElement = context.element(); final PubsubMessage message = failsafeElement.getOriginalPayload(); // Format the timestamp for insertion String timestamp = TIMESTAMP_FORMATTER.print(context.timestamp().toDateTime(DateTimeZone.UTC)); // Build the table row final TableRow failedRow = new TableRow() .set("timestamp", timestamp) .set("attributes", attributeMapToTableRows(message.getAttributeMap())) .set("errorMessage", failsafeElement.getErrorMessage()) .set("stacktrace", failsafeElement.getStacktrace()); // Only set the payload if it's populated on the message. if (message.getPayload() != null) { failedRow .set("payloadString", new String(message.getPayload())) .set("payloadBytes", message.getPayload()); } context.output(failedRow); } }
@ProcessElement public void processElement(ProcessContext context) { PubsubMessage message = context.element(); context.output( new AvroPubsubMessageRecord( message.getPayload(), message.getAttributeMap(), context.timestamp().getMillis())); } }
convertToSet(messages, m -> new String(m.getPayload(), UTF_8))); return null; });