private <ElemT, ViewT> void translateTyped( CreateDataflowView<ElemT, ViewT> transform, TranslationContext context) { StepTranslationContext stepContext = context.addStep(transform, "CollectionToSingleton"); PCollection<ElemT> input = context.getInput(transform); stepContext.addInput(PropertyNames.PARALLEL_INPUT, input); stepContext.addCollectionToSingletonOutput( input, PropertyNames.OUTPUT, transform.getView()); } });
/** * Returns a {@link PTransformOverrideFactory} that replaces a single-output {@link ParDo} with a * composite transform specialized for the {@link DataflowRunner}. */ public static <K, InputT, OutputT> PTransformOverrideFactory< PCollection<KV<K, InputT>>, PCollection<OutputT>, ParDo.SingleOutput<KV<K, InputT>, OutputT>> singleOutputOverrideFactory(DataflowPipelineOptions options) { return new SingleOutputOverrideFactory<>(isFnApi(options)); }
/** * Returns a {@link PTransformOverrideFactory} that replaces a multi-output {@link ParDo} with a * composite transform specialized for the {@link DataflowRunner}. */ public static <K, InputT, OutputT> PTransformOverrideFactory< PCollection<KV<K, InputT>>, PCollectionTuple, ParDo.MultiOutput<KV<K, InputT>, OutputT>> multiOutputOverrideFactory(DataflowPipelineOptions options) { return new MultiOutputOverrideFactory<>(isFnApi(options)); }
private <K1, K2, V> void groupByKeyAndSortValuesHelper( GroupByKeyAndSortValuesOnly<K1, K2, V> transform, TranslationContext context) { StepTranslationContext stepContext = context.addStep(transform, "GroupByKey"); PCollection<KV<K1, KV<K2, V>>> input = context.getInput(transform); stepContext.addInput(PropertyNames.PARALLEL_INPUT, input); stepContext.addOutput(PropertyNames.OUTPUT, context.getOutput(transform)); stepContext.addInput(PropertyNames.SORT_VALUES, true); // TODO: Add support for combiner lifting once the need arises. stepContext.addInput(PropertyNames.DISALLOW_COMBINER_LIFTING, true); } });
/** Constructs a runner from the provided options. */ public static TestDataflowRunner fromOptions(PipelineOptions options) { TestDataflowPipelineOptions dataflowOptions = options.as(TestDataflowPipelineOptions.class); String tempLocation = Joiner.on("/") .join(dataflowOptions.getTempRoot(), dataflowOptions.getJobName(), "output", "results"); dataflowOptions.setTempLocation(tempLocation); return new TestDataflowRunner( dataflowOptions, DataflowClient.create(options.as(DataflowPipelineOptions.class))); }
private <W extends BoundedWindow> PCollection<?> applyInternal(PCollection<KV<K, V>> input) { try { return applyForMapLike(runner, input, view, false /* unique keys not expected */); } catch (NonDeterministicException e) { runner.recordViewUsesNonDeterministicKeyCoder(this); // Since the key coder is not deterministic, we convert the map into a singleton // and return a singleton view equivalent. return applyForSingletonFallback(input); } }
private static void translateInputs( StepTranslationContext stepContext, PCollection<?> input, List<PCollectionView<?>> sideInputs, TranslationContext context) { stepContext.addInput(PropertyNames.PARALLEL_INPUT, input); translateSideInputs(stepContext, sideInputs, context); }
@Override public void translate(ReadWithIds<?> transform, TranslationContext context) { ReadTranslator.translateReadHelper(transform.getSource(), transform, context); } }
private <W extends BoundedWindow> PCollection<?> applyInternal(PCollection<KV<K, V>> input) { try { return BatchViewAsMultimap.applyForMapLike(runner, input, view, true /* unique keys */); } catch (NonDeterministicException e) { runner.recordViewUsesNonDeterministicKeyCoder(this); // Since the key coder is not deterministic, we convert the map into a singleton // and return a singleton view equivalent. return applyForSingletonFallback(input); } }
private JobMetrics generateMockMetricResponse(boolean success, boolean tentative) throws Exception { List<MetricUpdate> metrics = generateMockMetrics(success, tentative); return buildJobMetrics(metrics); }
public static DataflowMetricQueryResultsFactory create( DataflowPipelineJob dataflowPipelineJob, Iterable<MetricUpdate> metricUpdates, MetricsFilter filter) { return new DataflowMetricQueryResultsFactory(dataflowPipelineJob, metricUpdates, filter); }
@Override public void translate( SplittableParDo.ProcessKeyedElements transform, TranslationContext context) { translateTyped(transform, context); }
public static MetricQueryResults create( Iterable<MetricResult<Long>> counters, Iterable<MetricResult<DistributionResult>> distributions, Iterable<MetricResult<GaugeResult>> gauges) { return new AutoValue_DataflowMetrics_DataflowMetricQueryResults( counters, distributions, gauges); } }
/** * Returns a {@link PTransformOverrideFactory} that inserts a {@link Reshuffle.ViaRandomKey} * before a {@link ParDo.SingleOutput} that uses the {@link RequiresStableInput} annotation. */ static <InputT, OutputT> PTransformOverrideFactory< PCollection<InputT>, PCollection<OutputT>, ParDo.SingleOutput<InputT, OutputT>> singleOutputOverrideFactory() { return new SingleOutputOverrideFactory<>(); }
@Override public String toString() { return "TestDataflowRunner#" + options.getAppName(); }
@Override public void translate(View.CreatePCollectionView transform, TranslationContext context) { translateTyped(transform, context); }
@Override public void translate(GroupByKey transform, TranslationContext context) { groupByKeyHelper(transform, context); }
@Override public void translate(ParDo.MultiOutput transform, TranslationContext context) { translateMultiHelper(transform, context); }
public static <T> MetricResult<T> create( MetricName name, String scope, T committed, T attempted) { return new AutoValue_DataflowMetrics_DataflowMetricResult<>( name, scope, committed, attempted); } }
@Override public Integer extractOutput(Integer accumulator, Context c) { return delegate.extractOutput(accumulator); } }