/** Writes the accumulated rows into BigQuery with streaming API. */ @FinishBundle public void finishBundle(FinishBundleContext context) throws Exception { List<ValueInSingleWindow<ErrorT>> failedInserts = Lists.newArrayList(); BigQueryOptions options = context.getPipelineOptions().as(BigQueryOptions.class); for (Map.Entry<String, List<ValueInSingleWindow<TableRow>>> entry : tableRows.entrySet()) { TableReference tableReference = BigQueryHelpers.parseTableSpec(entry.getKey()); flushRows( tableReference, entry.getValue(), uniqueIdsForTableRows.get(entry.getKey()), options, failedInserts); } tableRows.clear(); uniqueIdsForTableRows.clear(); for (ValueInSingleWindow<ErrorT> row : failedInserts) { context.output(failedOutputTag, row.getValue(), row.getTimestamp(), row.getWindow()); } }
/** @deprecated Use {@link TestPipeline} with the {@code DirectRunner}. */ @Deprecated public List<TimestampedValue<OutputT>> peekOutputElementsInWindow( TupleTag<OutputT> tag, BoundedWindow window) { ImmutableList.Builder<TimestampedValue<OutputT>> valuesBuilder = ImmutableList.builder(); for (ValueInSingleWindow<OutputT> value : getImmutableOutput(tag)) { if (value.getWindow().equals(window)) { valuesBuilder.add(TimestampedValue.of(value.getValue(), value.getTimestamp())); } } return valuesBuilder.build(); }
@Override public Integer getDestination(ValueInSingleWindow<String> element) { assertThat( element.getWindow(), Matchers.instanceOf(PartitionedGlobalWindow.class)); Matcher matcher = userPattern.matcher(element.getValue()); if (matcher.matches()) { // Since we name tables by userid, we can simply store an Integer to represent // a table. return Integer.valueOf(matcher.group(2)); } throw new RuntimeException("Unmatching destination " + element.getValue()); }
@Override public void encode(ValueInSingleWindow<T> windowedElem, OutputStream outStream, Context context) throws IOException { InstantCoder.of().encode(windowedElem.getTimestamp(), outStream); windowCoder.encode(windowedElem.getWindow(), outStream); PaneInfo.PaneInfoCoder.INSTANCE.encode(windowedElem.getPane(), outStream); valueCoder.encode(windowedElem.getValue(), outStream, context); }
PartitionedGlobalWindow window = (PartitionedGlobalWindow) input.getWindow();