@Override public void populateDAG(DAG dag, Configuration conf) { dataGenerator = dag.addOperator("DataGenerator", DataGenerator.class); JsonParser parser = dag.addOperator("Parser", JsonParser.class); /** * FSLoader is used to configure Enricher backend. Property of FSLoader file which is fileName is set in * properties.xml file. * The format that is used to read the file is present as an example in resources/circleMapping.txt file. */ JsonFSLoader fsLoader = new JsonFSLoader(); POJOEnricher enrich = dag.addOperator("Enrich", POJOEnricher.class); enrich.setStore(fsLoader); ArrayList<String> includeFields = new ArrayList<>(); includeFields.add("circleName"); ArrayList<String> lookupFields = new ArrayList<>(); lookupFields.add("circleId"); enrich.setIncludeFields(includeFields); enrich.setLookupFields(lookupFields); MapFunctionOperator<Object, ?> out = dag.addOperator("out", new MapFunctionOperator<>(outputFn)); dag.addStream("Parse", dataGenerator.output, parser.in); dag.addStream("Enrich", parser.out, enrich.input); dag.addStream("Console", enrich.output, out.input); }
@Override public void populateDAG(DAG dag, Configuration conf) { pojoDataGenerator = dag.addOperator("Input", new POJOGenerator()); TransformOperator transform = dag.addOperator("Process", new TransformOperator()); // Set expression map Map<String, String> expMap = new HashMap<>(); expMap.put("name", "{$.firstName}.concat(\" \").concat({$.lastName})"); expMap.put("age", "(new java.util.Date()).getYear() - {$.dateOfBirth}.getYear()"); expMap.put("address", "{$.address}.toLowerCase()"); transform.setExpressionMap(expMap); FunctionOperator.MapFunctionOperator<Object, ?> output = dag.addOperator("out", new FunctionOperator.MapFunctionOperator<>(outputFn)); dag.addStream("InputToTransform", pojoDataGenerator.output, transform.input); dag.addStream("TransformToOutput", transform.output, output.input); dag.setInputPortAttribute(transform.input, Context.PortContext.TUPLE_CLASS, CustomerEvent.class); dag.setOutputPortAttribute(transform.output, Context.PortContext.TUPLE_CLASS, CustomerInfo.class); setPartitioner(dag,conf,transform); }
@Override public void process(IN t) { Function.MapFunction<IN, OUT> f = getFunction(); output.emit(f.f(t)); } };
@Override public <O, STREAM extends ApexStream<O>> STREAM map(Function.MapFunction<T, O> mf, Option... opts) { FunctionOperator.MapFunctionOperator<T, O> opt = new FunctionOperator.MapFunctionOperator<>(mf); return addOperator(opt, opt.input, opt.output, opts); }
@Override public void process(Tuple<IN> t) { Function.MapFunction<IN, OUT> f = getFunction(); if (t instanceof Tuple.PlainTuple) { TupleUtil.buildOf((Tuple.PlainTuple<IN>)t, f.f(t.getValue())); } else { output.emit(f.f(t.getValue())); } } };