private PubsubClient.OutgoingMessage toOutgoingMessage(PubsubMessage message) { return new PubsubClient.OutgoingMessage( message.getPayload(), message.getAttributeMap(), DateTime.now().getMillis(), null); } }
@Override public PubsubMessage getCurrent() throws NoSuchElementException { if (current == null) { throw new NoSuchElementException(); } return new PubsubMessage(current.elementBytes, current.attributes); }
@Override public String apply(PubsubMessage input) { return new String(input.getPayload(), StandardCharsets.UTF_8); } }
private Object getValueForField( Schema.Field field, Instant timestamp, PubsubMessage pubsubMessage) { switch (field.getName()) { case TIMESTAMP_FIELD: return timestamp; case ATTRIBUTES_FIELD: return pubsubMessage.getAttributeMap(); case PAYLOAD_FIELD: return parsePayloadJsonRow(pubsubMessage); default: throw new IllegalArgumentException( "Unexpected field '" + field.getName() + "' in top level schema" + " for Pubsub message. Top level schema should only contain " + "'timestamp', 'attributes', and 'payload' fields"); } }
@ProcessElement public void processElement(ProcessContext context) { if (!this.doFilter) { // Filter is not enabled context.output(context.element()); } else { PubsubMessage message = context.element(); String extractedValue = message.getAttribute(this.inputFilterKey); if (Objects.equals(extractedValue, this.inputFilterValue)) { context.output(message); } } } }
@Override public void encode(PubsubMessage value, OutputStream outStream, Context context) throws IOException { PAYLOAD_CODER.encode(value.getPayload(), outStream, context); }
public void encode(PubsubMessage value, OutputStream outStream, Context context) throws IOException { PAYLOAD_CODER.encode(value.getPayload(), outStream); ATTRIBUTES_CODER.encode(value.getAttributeMap(), outStream, context); }
@Override public T apply(PubsubMessage input) { try { return CoderUtils.decodeFromByteArray(coder, input.getPayload()); } catch (CoderException e) { throw new RuntimeException("Could not decode Pubsub message", e); } } }
@Override public PubsubMessage apply(String input) { return new PubsubMessage(input.getBytes(StandardCharsets.UTF_8), ImmutableMap.of()); } }
private static boolean messagesEqual(PubsubMessage message1, PubsubMessage message2) { return message1.getAttributeMap().equals(message2.getAttributeMap()) && Arrays.equals(message1.getPayload(), message2.getPayload()); }
@DoFn.ProcessElement public void processElement(ProcessContext c) throws IOException { if (schema == null) { schema = new Schema.Parser().parse(schemaStr); datumReader = new GenericDatumReader<GenericRecord>(schema); } decoder = DecoderFactory.get().binaryDecoder(c.element().getPayload(), decoder); GenericRecord record = datumReader.read(null, decoder); c.output(record); } }
@Override public PubsubMessage apply(T input) { try { return new PubsubMessage(CoderUtils.encodeToByteArray(coder, input), ImmutableMap.of()); } catch (CoderException e) { throw new RuntimeException("Could not decode Pubsub message", e); } } }
@ProcessElement public void processElement(ProcessContext context) { PubsubMessage message = context.element(); context.output( new AvroPubsubMessageRecord( message.getPayload(), message.getAttributeMap(), context.timestamp().getMillis())); } }
@DoFn.ProcessElement public void processElement(ProcessContext c) { String record = new String(c.element().getPayload(), Charset.forName("UTF-8")); c.output(record.split(fieldDelimiter)); }
@Override public PubsubMessage decode(InputStream inStream, Context context) throws IOException { return new PubsubMessage(PAYLOAD_CODER.decode(inStream, context), ImmutableMap.of()); } }
@ProcessElement public void processElement(ProcessContext c) throws Exception { elementCounter.inc(); PubsubMessage message = c.element(); byte[] elementBytes = message.getPayload(); Map<String, String> attributes = message.getAttributeMap(); long timestampMsSinceEpoch = c.timestamp().getMillis(); @Nullable String recordId = null; switch (recordIdMethod) { case NONE: break; case DETERMINISTIC: recordId = Hashing.murmur3_128().hashBytes(elementBytes).toString(); break; case RANDOM: // Since these elements go through a GroupByKey, any failures while sending to // Pubsub will be retried without falling back and generating a new record id. // Thus even though we may send the same message to Pubsub twice, it is guaranteed // to have the same record id. recordId = UUID.randomUUID().toString(); break; } c.output( KV.of( ThreadLocalRandom.current().nextInt(numShards), new OutgoingMessage(elementBytes, attributes, timestampMsSinceEpoch, recordId))); }
private static String data(PubsubMessage message) { return new String(message.getPayload(), StandardCharsets.UTF_8); }
@Override public PubsubMessage apply(IndexedRecord input) { int size = input.getSchema().getFields().size(); for (int i = 0; i < size; i++) { if (sb.length() != 0) sb.append(fieldDelimiter); sb.append(input.get(i)); } byte[] bytes = sb.toString().getBytes(Charset.forName("UTF-8")); sb.setLength(0); return new PubsubMessage(bytes, ImmutableMap.<String, String> of()); } }
@ProcessElement public void processElement(ProcessContext context) { FailsafeElement<PubsubMessage, String> failsafeElement = context.element(); final PubsubMessage message = failsafeElement.getOriginalPayload(); // Format the timestamp for insertion String timestamp = TIMESTAMP_FORMATTER.print(context.timestamp().toDateTime(DateTimeZone.UTC)); // Build the table row final TableRow failedRow = new TableRow() .set("timestamp", timestamp) .set("attributes", attributeMapToTableRows(message.getAttributeMap())) .set("errorMessage", failsafeElement.getErrorMessage()) .set("stacktrace", failsafeElement.getStacktrace()); // Only set the payload if it's populated on the message. if (message.getPayload() != null) { failedRow .set("payloadString", new String(message.getPayload())) .set("payloadBytes", message.getPayload()); } context.output(failedRow); } }
@ProcessElement public void processElement(ProcessContext context) { PubsubMessage message = context.element(); context.output( FailsafeElement.of(message, new String(message.getPayload(), StandardCharsets.UTF_8))); } }