@Override public void setup(Context.OperatorContext context) { readFunction(); }
@Override public void process(IN t) { Function.MapFunction<IN, OUT> f = getFunction(); output.emit(f.f(t)); } };
@Override public void process(IN t) { Function.FlatMapFunction<IN, OUT> f = getFunction(); for (OUT out : f.f(t)) { output.emit(out); } } };
@Override public <O, STREAM extends ApexStream<O>> STREAM map(Function.MapFunction<T, O> mf, Option... opts) { FunctionOperator.MapFunctionOperator<T, O> opt = new FunctionOperator.MapFunctionOperator<>(mf); return addOperator(opt, opt.input, opt.output, opts); }
@Override public void process(IN t) { Function.FilterFunction<IN> f = getFunction(); // fold the value if (f.f(t)) { output.emit(t); } } };
public FunctionOperator(FUNCTION f) { isAnnonymous = f.getClass().isAnonymousClass(); if (isAnnonymous) { // legacy behavior for functions that don't support Java serialization annonymousFunctionClass = functionClassData(f); try { readFunction(); } catch (Exception e) { // cannot be handled by custom serialization, but may be Java serializable annonymousFunctionClass = null; statefulF.setValue(f); stateful = true; } } else if (f instanceof Function.Stateful) { statelessF = f; } else { statefulF.setValue(f); stateful = true; } }
@Override @SuppressWarnings("unchecked") public <STREAM extends ApexStream<T>> STREAM filter(final Function.FilterFunction<T> filter, Option... opts) { FunctionOperator.FilterFunctionOperator<T> filterFunctionOperator = new FunctionOperator.FilterFunctionOperator<>(filter); return addOperator(filterFunctionOperator, filterFunctionOperator.input, filterFunctionOperator.output, opts); }
@Override public <O, STREAM extends ApexStream<O>> STREAM flatMap(FlatMapFunction<T, O> flatten, Option... opts) { FunctionOperator.FlatMapFunctionOperator<T, O> opt = new FunctionOperator.FlatMapFunctionOperator<>(flatten); return addOperator(opt, opt.input, opt.output, opts); }
AnnonymousClassModifier annonymousClassModifier = new AnnonymousClassModifier(Opcodes.ASM4, cw); cr.accept(annonymousClassModifier, 0); classBytes = cw.toByteArray();
@Override public void populateDAG(DAG dag, Configuration conf) { dataGenerator = dag.addOperator("DataGenerator", DataGenerator.class); JsonParser parser = dag.addOperator("Parser", JsonParser.class); /** * FSLoader is used to configure Enricher backend. Property of FSLoader file which is fileName is set in * properties.xml file. * The format that is used to read the file is present as an example in resources/circleMapping.txt file. */ JsonFSLoader fsLoader = new JsonFSLoader(); POJOEnricher enrich = dag.addOperator("Enrich", POJOEnricher.class); enrich.setStore(fsLoader); ArrayList<String> includeFields = new ArrayList<>(); includeFields.add("circleName"); ArrayList<String> lookupFields = new ArrayList<>(); lookupFields.add("circleId"); enrich.setIncludeFields(includeFields); enrich.setLookupFields(lookupFields); MapFunctionOperator<Object, ?> out = dag.addOperator("out", new MapFunctionOperator<>(outputFn)); dag.addStream("Parse", dataGenerator.output, parser.in); dag.addStream("Enrich", parser.out, enrich.input); dag.addStream("Console", enrich.output, out.input); }
@Override public void process(Tuple<IN> t) { Function.MapFunction<IN, OUT> f = getFunction(); if (t instanceof Tuple.PlainTuple) { TupleUtil.buildOf((Tuple.PlainTuple<IN>)t, f.f(t.getValue())); } else { output.emit(f.f(t.getValue())); } } };
@Override public void process(Tuple<IN> t) { Function.FlatMapFunction<IN, OUT> f = getFunction(); if (t instanceof Tuple.PlainTuple) { for (OUT out : f.f(t.getValue())) { tupleOutput.emit(TupleUtil.buildOf((Tuple.PlainTuple<IN>)t, out)); } } else { for (OUT out : f.f(t.getValue())) { output.emit(out); } } } };
@Override public void process(Tuple<IN> t) { Function.FilterFunction<IN> f = getFunction(); if (f.f(t.getValue())) { tupleOutput.emit(t); } } };
public FUNCTION getFunction() { readFunction(); if (stateful) { return statefulF.getValue(); } else { return statelessF; } }
@Override public void populateDAG(DAG dag, Configuration conf) { pojoDataGenerator = dag.addOperator("Input", new POJOGenerator()); TransformOperator transform = dag.addOperator("Process", new TransformOperator()); // Set expression map Map<String, String> expMap = new HashMap<>(); expMap.put("name", "{$.firstName}.concat(\" \").concat({$.lastName})"); expMap.put("age", "(new java.util.Date()).getYear() - {$.dateOfBirth}.getYear()"); expMap.put("address", "{$.address}.toLowerCase()"); transform.setExpressionMap(expMap); FunctionOperator.MapFunctionOperator<Object, ?> output = dag.addOperator("out", new FunctionOperator.MapFunctionOperator<>(outputFn)); dag.addStream("InputToTransform", pojoDataGenerator.output, transform.input); dag.addStream("TransformToOutput", transform.output, output.input); dag.setInputPortAttribute(transform.input, Context.PortContext.TUPLE_CLASS, CustomerEvent.class); dag.setOutputPortAttribute(transform.output, Context.PortContext.TUPLE_CLASS, CustomerInfo.class); setPartitioner(dag,conf,transform); }