@Override public void translate(ReadWithIds<?> transform, TranslationContext context) { ReadTranslator.translateReadHelper(transform.getSource(), transform, context); } }
public static <T> void translateReadHelper( Source<T> source, PTransform<?, ? extends PCollection<?>> transform, TranslationContext context) { try { StepTranslationContext stepContext = context.addStep(transform, "ParallelRead"); stepContext.addInput(PropertyNames.FORMAT, PropertyNames.CUSTOM_SOURCE_FORMAT); stepContext.addInput( PropertyNames.SOURCE_STEP_INPUT, cloudSourceToDictionary( CustomSources.serializeToCloudSource(source, context.getPipelineOptions()))); stepContext.addOutput(PropertyNames.OUTPUT, context.getOutput(transform)); } catch (Exception e) { throw new RuntimeException(e); } }
private static Map<String, Object> cloudSourceToDictionary( com.google.api.services.dataflow.model.Source source) { // Do not translate encoding - the source's encoding is translated elsewhere // to the step's output info. Map<String, Object> res = new HashMap<>(); addDictionary(res, PropertyNames.SOURCE_SPEC, source.getSpec()); if (source.getMetadata() != null) { addDictionary( res, PropertyNames.SOURCE_METADATA, cloudSourceMetadataToDictionary(source.getMetadata())); } if (source.getDoesNotNeedSplitting() != null) { addBoolean( res, PropertyNames.SOURCE_DOES_NOT_NEED_SPLITTING, source.getDoesNotNeedSplitting()); } return res; }
@Override public void translate(Read.Bounded<?> transform, TranslationContext context) { translateReadHelper(transform.getSource(), transform, context); }