"timing", new WriteWindowedToBigQuery.FieldInfo<>( "STRING", (c, w) -> c.pane().getTiming().toString())); return tableConfigure;
private PendingJobData startWriteRename( TableDestination finalTableDestination, Iterable<String> tempTableNames, ProcessContext c) throws Exception { WriteDisposition writeDisposition = (c.pane().getIndex() == 0) ? firstPaneWriteDisposition : WriteDisposition.WRITE_APPEND; CreateDisposition createDisposition = (c.pane().getIndex() == 0) ? firstPaneCreateDisposition : CreateDisposition.CREATE_NEVER; List<TableReference> tempTables = StreamSupport.stream(tempTableNames.spliterator(), false) .map(table -> BigQueryHelpers.fromJsonString(table, TableReference.class)) .collect(Collectors.toList()); ; // Make sure each destination table gets a unique job id. String jobIdPrefix = BigQueryHelpers.createJobId( c.sideInput(jobIdToken), finalTableDestination, -1, c.pane().getIndex()); BigQueryHelpers.PendingJob retryJob = startCopy( bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)), bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)), jobIdPrefix, finalTableDestination.getTableReference(), tempTables, writeDisposition, createDisposition); return new PendingJobData(retryJob, finalTableDestination, tempTables); }
/** Accumulates the input into JsonTableRows and uniqueIdsForTableRows. */ @ProcessElement public void processElement(ProcessContext context, BoundedWindow window) { String tableSpec = context.element().getKey().getKey(); List<ValueInSingleWindow<TableRow>> rows = BigQueryHelpers.getOrCreateMapListValue(tableRows, tableSpec); List<String> uniqueIds = BigQueryHelpers.getOrCreateMapListValue(uniqueIdsForTableRows, tableSpec); rows.add( ValueInSingleWindow.of( context.element().getValue().tableRow, context.timestamp(), window, context.pane())); uniqueIds.add(context.element().getValue().uniqueId); }
@ProcessElement public void processElement(ProcessContext context, BoundedWindow window) throws IOException { dynamicDestinations.setSideInputAccessorFromProcessContext(context); ValueInSingleWindow<T> element = ValueInSingleWindow.of( context.element(), context.timestamp(), window, context.pane()); DestinationT tableDestination = dynamicDestinations.getDestination(element); checkArgument( tableDestination != null, "DynamicDestinations.getDestination() may not return null, " + "but %s returned null on element %s", dynamicDestinations, element); TableRow tableRow = formatFunction.apply(context.element()); checkArgument( tableRow != null, "formatFunction may not return null, but %s returned null on element %s", formatFunction, context.element()); context.output(KV.of(tableDestination, tableRow)); } })
@ProcessElement public void processElement(ProcessContext c, BoundedWindow window) throws Exception { String[] values = c.element().getValue().split(",", -1); TableRow row = new TableRow() .set("trigger_type", triggerType) .set("freeway", c.element().getKey()) .set("total_flow", Integer.parseInt(values[0])) .set("number_of_records", Long.parseLong(values[1])) .set("window", window.toString()) .set("isFirst", c.pane().isFirst()) .set("isLast", c.pane().isLast()) .set("timing", c.pane().getTiming().toString()) .set("event_time", c.timestamp().toString()) .set("processing_time", Instant.now().toString()); c.output(row); } }
@ProcessElement public void processElement(ProcessContext c, BoundedWindow untypedWindow) throws Exception { @SuppressWarnings("unchecked") W window = (W) untypedWindow; c.output( KV.of( coder.hash(ImmutableList.of(c.element().getKey())), KV.of( KV.of(c.element().getKey(), window), WindowedValue.of( c.element().getValue(), c.timestamp(), untypedWindow, c.pane())))); } }
@Override public PaneInfo pane() { return outerContext.pane(); }
@Override public PaneInfo paneInfo(DoFn<InputT, OutputT> doFn) { return outerContext.pane(); }
@ProcessElement public void processElement(ProcessContext c, BoundedWindow untypedWindow) throws Exception { @SuppressWarnings("unchecked") W window = (W) untypedWindow; c.output( KV.of( ismCoderForHash.hash(ImmutableList.of(window)), KV.of(window, WindowedValue.of(c.element(), c.timestamp(), window, c.pane())))); } }
@ProcessElement public void processElement(final ProcessContext c, final BoundedWindow window) { c.output( KV.of( c.element().getKey(), WindowedValue.of(c.element(), c.timestamp(), window, c.pane()))); } }
@ProcessElement public void processElement(ProcessContext c) { c.output(c.element() + ": " + c.pane().isLast()); } }
@ProcessElement public void processElement(ProcessContext c) throws Exception { c.output( IsmRecord.of( ImmutableList.of(GlobalWindow.INSTANCE, indexInBundle), WindowedValue.of(c.element(), c.timestamp(), GlobalWindow.INSTANCE, c.pane()))); indexInBundle += 1; } }
@ProcessElement public void processElement(final ProcessContext c, final BoundedWindow window) { c.output( KV.of( c.element().getKey(), KV.of( c.timestamp(), WindowedValue.of(c.element(), c.timestamp(), window, c.pane())))); } }
@ProcessElement public void processElement(ProcessContext c) { if (c.pane().isLast()) { c.output(c.element()); } } }