@Override public PDone expand(PCollection<T> input) { return PDone.in(input.getPipeline()); } }
/** Creates a {@link PDone} in the given {@link Pipeline}. */ public static PDone in(Pipeline pipeline) { return new PDone(pipeline); }
private static AppliedPTransform<?, ?, ?> rawPTransformWithNullSpec(Pipeline pipeline) { PTransformTranslation.RawPTransform<PBegin, PDone> rawPTransform = new PTransformTranslation.RawPTransform<PBegin, PDone>() { @Override public String getUrn() { return "fake/urn"; } @Nullable @Override public RunnerApi.FunctionSpec getSpec() { return null; } }; return AppliedPTransform.<PBegin, PDone, PTransform<PBegin, PDone>>of( "RawPTransformWithNoSpec", pipeline.begin().expand(), PDone.in(pipeline).expand(), rawPTransform, pipeline); }
@Override public PDone expand(PCollection<PubsubMessage> input) { return PDone.in(input.getPipeline()); }
@Override public PDone expand(PCollection<T> input) { input.apply(inner); return PDone.in(input.getPipeline()); }
@Override public PDone expand(PCollection<String> input) { inner.expand(input); return PDone.in(input.getPipeline()); } }
@Override public POutput expand(PInput input) { return PDone.in(input.getPipeline()); }
@Override public PDone expand(PCollection<RedisMutation> input) { input.apply(ParDo.of(dofn)); return PDone.in(input.getPipeline()); }
@Override public PDone expand(PInput input) { return PDone.in(input.getPipeline()); } });
@Override public POutput expand(PInput input) { return PDone.in(input.getPipeline()); } };
@Override public PDone expand(PBegin begin) { return PDone.in(begin.getPipeline()); } }
@Override public PDone expand(PCollection<T> input) { input.apply(ParDo.of(function)); return PDone.in(input.getPipeline()); }
@Override public PDone expand(PCollection<T> input) { input.apply(ParDo.of(function)); return PDone.in(input.getPipeline()); }
@Override public PDone expand(PCollection<FeatureRowExtended> input) { List<FeatureSpec> storeFeatureSpecs = specs.getFeatureSpecByServingStoreId(storeId); Set<String> tableNames = new HashSet<>(); Map<String, Set<FeatureSpec>> featuresByTable = new HashMap<>(); // Group feature specs by table name for (FeatureSpec featureSpec : storeFeatureSpecs) { String tableName = getTableName(featureSpec); tableNames.add(tableName); Set<FeatureSpec> featureSpecs = featuresByTable.getOrDefault(tableName, new HashSet<>()); featureSpecs.add(featureSpec); featuresByTable.put(tableName, featureSpecs); } MultiOutputSplit<String> splitter = new MultiOutputSplit<>(Write::getTableName, tableNames, specs); PCollectionTuple splitByTable = input.apply(splitter); for (String tableName : tableNames) { log.info("Initialising write transform for table " + tableName); TupleTag<FeatureRowExtended> tag = splitter.getStrategy().getTag(tableName); writeTable(splitByTable.get(tag), Lists.newArrayList(featuresByTable.get(tableName))); } return PDone.in(input.getPipeline()); }
@Override public POutput buildIOWriter(PCollection<Row> input) { input.apply( ParDo.of( new DoFn<Row, Void>() { @ProcessElement public void processElement(ProcessContext c) { CONTENT.add(c.element()); } @Teardown public void close() { CONTENT.clear(); } })); return PDone.in(input.getPipeline()); } }
@Override public PDone expand(PCollection<T> input) { input.apply(ParDo.of(new GridFsWriteFn<>(this))); return PDone.in(input.getPipeline()); } }
@Override public PDone expand(PCollection<T> input) { checkArgument(projectId != null, "withProjectId() is required"); if (projectId.isAccessible()) { checkArgument(projectId.get() != null, "projectId can not be null"); } checkArgument(mutationFn != null, "mutationFn can not be null"); input .apply("Convert to Mutation", MapElements.via(mutationFn)) .apply( "Write Mutation to Datastore", ParDo.of(new DatastoreWriterFn(projectId, localhost))); return PDone.in(input.getPipeline()); }
@Override public POutput buildIOWriter(PCollection<Row> input) { input.apply(ParDo.of(new CollectorFn(tableWithRows))); return PDone.in(input.getPipeline()); }
@Override public PDone expand(PCollection<T> input) { input.apply(ParDo.of(new WriteFn<>(this))); return PDone.in(input.getPipeline()); }
@Override public PDone expand(PCollection<Message> input) { input.apply(ParDo.of(new WriteFn(this))); return PDone.in(input.getPipeline()); }