@Override public void populateDAG(DAG dag, Configuration configuration) { RandomPOJOGenerator input = dag.addOperator("Input", RandomPOJOGenerator.class); POJOEnricher enrich = dag.addOperator("Enrich", POJOEnricher.class); EnrichVerifier verify = dag.addOperator("Verify", EnrichVerifier.class); verify.address = testMeta.address; verify.age = testMeta.age; verify.names = testMeta.name; verify.salary = testMeta.salary; enrich.setStore(loader); ArrayList<String> lookupFields = new ArrayList<>(); lookupFields.add("ID"); ArrayList<String> includeFields = new ArrayList<>(); includeFields.add("NAME"); includeFields.add("AGE"); includeFields.add("ADDRESS"); includeFields.add("SALARY"); enrich.setLookupFields(lookupFields); enrich.setIncludeFields(includeFields); dag.getMeta(enrich).getMeta(enrich.input).getAttributes().put(Context.PortContext.TUPLE_CLASS, Order.class); dag.getMeta(enrich).getMeta(enrich.output).getAttributes() .put(Context.PortContext.TUPLE_CLASS, EmployeeOrder.class); dag.addStream("S1", input.output, enrich.input); dag.addStream("S2", enrich.output, verify.input); }
@Override public void populateDAG(DAG dag, Configuration conf) { ParquetFilePOJOReader parquetReader = dag.addOperator("parquetReader", getParquetFilePOJOReader()); ConsoleOutputOperator pojoOp = dag.addOperator("pojoOp", new ConsoleOutputOperator()); dag.getMeta(parquetReader).getMeta(parquetReader.output).getAttributes().put(Context.PortContext.TUPLE_CLASS, EventRecord.class); dag.addStream("pojo", parquetReader.output, pojoOp.input); }
@Override public void populateDAG(DAG dag, Configuration conf) { AvroFileInputOperator avroInputOperator = dag.addOperator("avroInputOperator", getAvroFileInput()); AvroToPojo avroToPojo = dag.addOperator("AvroToPojo", getAvroToPojo()); ConsoleOutputOperator consoleOutput = dag.addOperator("GenericRecordOp", new ConsoleOutputOperator()); dag.getMeta(avroToPojo).getMeta(avroToPojo.output).getAttributes().put(Context.PortContext.TUPLE_CLASS, SimpleOrder.class); dag.addStream("GenericRecords", avroInputOperator.output, avroToPojo.data).setLocality(Locality.THREAD_LOCAL); dag.addStream("POJO", avroToPojo.output, consoleOutput.input).setLocality(Locality.CONTAINER_LOCAL); }
@Override public void populateDAG(DAG dag, Configuration conf) { FileInputOperator fileInput = dag.addOperator("fileInput", getFileInput()); StreamingJsonParser parser = dag.addOperator("parser", getParser()); dag.getMeta(parser).getMeta(parser.output).getAttributes().put(Context.PortContext.TUPLE_CLASS, Person.class); ConsoleOutputOperator consoleOutput = dag.addOperator("output", new ConsoleOutputOperator()); dag.addStream("Input", fileInput.output, parser.in).setLocality(Locality.CONTAINER_LOCAL); dag.addStream("pojo", parser.output, consoleOutput.input).setLocality(Locality.CONTAINER_LOCAL); }
@Override public void populateDAG(DAG dag, Configuration conf) { PojoEmitter input = dag.addOperator("data", new PojoEmitter()); CsvFormatter formatter = dag.addOperator("formatter", new CsvFormatter()); dag.getMeta(formatter).getMeta(formatter.in).getAttributes().put(Context.PortContext.TUPLE_CLASS, Ad.class); formatter.setSchema(SchemaUtils.jarResourceFileToString("schema.json")); ConsoleOutputOperator output = dag.addOperator("output", new ConsoleOutputOperator()); ConsoleOutputOperator error = dag.addOperator("error", new ConsoleOutputOperator()); output.setDebug(true); dag.addStream("input", input.output, formatter.in); dag.addStream("output", formatter.out, output.input); dag.addStream("err", formatter.err, error.input); } }
@Override public void populateDAG(DAG dag, Configuration conf) { JsonDataEmitterOperator input = dag.addOperator("data", new JsonDataEmitterOperator()); JsonParser parser = dag.addOperator("jsonparser", new JsonParser()); parser.setClazz(Product.class); dag.getMeta(parser).getMeta(parser.out).getAttributes().put(Context.PortContext.TUPLE_CLASS, Product.class); parser.setJsonSchema(SchemaUtils.jarResourceFileToString("json-parser-schema.json")); ConsoleOutputOperator jsonObjectOp = dag.addOperator("jsonObjectOp", new ConsoleOutputOperator()); ConsoleOutputOperator pojoOp = dag.addOperator("pojoOp", new ConsoleOutputOperator()); jsonObjectOp.setDebug(true); dag.addStream("input", input.output, parser.in); dag.addStream("output", parser.parsedOutput, jsonObjectOp.input); dag.addStream("pojo", parser.out, pojoOp.input); } }
@Override public void populateDAG(DAG dag, Configuration conf) { locality = Locality.CONTAINER_LOCAL; dag.getAttributes().put(DAG.STREAMING_WINDOW_SIZE_MILLIS, 1000); SimpleSinglePortZeroMQPullStringInputOperator input = dag.addOperator("input", new SimpleSinglePortZeroMQPullStringInputOperator(addr)); ApacheLogParseOperator parse = dag.addOperator("parse", new ApacheLogParseOperator()); UniqueCounter<String> ipAddrCount = dag.addOperator("ipAddrCount", new UniqueCounter<String>()); UniqueCounter<String> urlCount = dag.addOperator("urlCount", new UniqueCounter<String>()); UniqueCounter<String> httpStatusCount = dag.addOperator("httpStatusCount", new UniqueCounter<String>()); Sum<Long> numOfBytesSum = dag.addOperator("numOfBytesSum", new Sum<Long>()); //ArrayListAggregator<Long> agg = dag.addOperator("agg", new ArrayListAggregator<Long>()); //dag.getOperatorWrapper(agg).getAttributes().put(OperatorContext.APPLICATION_WINDOW_COUNT, 3); dag.getMeta(numOfBytesSum).getAttributes().put(OperatorContext.APPLICATION_WINDOW_COUNT, 3); dag.addStream("input-parse", input.outputPort, parse.data).setLocality(locality); dag.addStream("parse-ipAddrCount", parse.outputIPAddress, ipAddrCount.data).setLocality(locality); dag.addStream("parse-urlCount", parse.outputUrl, urlCount.data).setLocality(locality); dag.addStream("parse-httpStatusCount", parse.outputStatusCode, httpStatusCount.data).setLocality(locality); dag.addStream("parse-numOfBytesSum", parse.outputBytes, numOfBytesSum.data).setLocality(locality); //dag.addStream("numOfBytesSum-agg", numOfBytesSum.sumLong, agg.input); ConsoleOutputOperator consoleOperator1 = dag.addOperator("console1", new ConsoleOutputOperator()); ConsoleOutputOperator consoleOperator2 = dag.addOperator("console2", new ConsoleOutputOperator()); ConsoleOutputOperator consoleOperator3 = dag.addOperator("console3", new ConsoleOutputOperator()); ConsoleOutputOperator consoleOperator4 = dag.addOperator("console4", new ConsoleOutputOperator()); dag.addStream("ipAddrCount-console", ipAddrCount.count, consoleOperator1.input); dag.addStream("urlCount-console", urlCount.count, consoleOperator2.input); dag.addStream("httpStatusCount-console", httpStatusCount.count, consoleOperator3.input); //dag.addStream("agg-console", agg.output, consoleOperator4.input); dag.addStream("numOfBytesSum-console", numOfBytesSum.sumLong, consoleOperator4.input); }
@Override public void populateDAG(DAG dag, Configuration conf) { locality = Locality.CONTAINER_LOCAL; dag.getAttributes().put(DAG.STREAMING_WINDOW_SIZE_MILLIS, 1000); SimpleSinglePortZeroMQPullStringInputOperator input = dag.addOperator("input", new SimpleSinglePortZeroMQPullStringInputOperator(addr)); ApacheLogParseOperator parse = dag.addOperator("parse", new ApacheLogParseOperator()); UniqueCounter<String> ipAddrCount = dag.addOperator("ipAddrCount", new UniqueCounter<String>()); UniqueCounter<String> urlCount = dag.addOperator("urlCount", new UniqueCounter<String>()); UniqueCounter<String> httpStatusCount = dag.addOperator("httpStatusCount", new UniqueCounter<String>()); Sum<Long> numOfBytesSum = dag.addOperator("numOfBytesSum", new Sum<Long>()); //ArrayListAggregator<Long> agg = dag.addOperator("agg", new ArrayListAggregator<Long>()); //dag.getOperatorWrapper(agg).getAttributes().put(OperatorContext.APPLICATION_WINDOW_COUNT, 3); dag.getMeta(numOfBytesSum).getAttributes().put(OperatorContext.APPLICATION_WINDOW_COUNT, 3); dag.addStream("input-parse", input.outputPort, parse.data).setLocality(locality); dag.addStream("parse-ipAddrCount", parse.outputIPAddress, ipAddrCount.data).setLocality(locality); dag.addStream("parse-urlCount", parse.outputUrl, urlCount.data).setLocality(locality); dag.addStream("parse-httpStatusCount", parse.outputStatusCode, httpStatusCount.data).setLocality(locality); dag.addStream("parse-numOfBytesSum", parse.outputBytes, numOfBytesSum.data).setLocality(locality); //dag.addStream("numOfBytesSum-agg", numOfBytesSum.sumLong, agg.input); ConsoleOutputOperator consoleOperator1 = dag.addOperator("console1", new ConsoleOutputOperator()); ConsoleOutputOperator consoleOperator2 = dag.addOperator("console2", new ConsoleOutputOperator()); ConsoleOutputOperator consoleOperator3 = dag.addOperator("console3", new ConsoleOutputOperator()); ConsoleOutputOperator consoleOperator4 = dag.addOperator("console4", new ConsoleOutputOperator()); dag.addStream("ipAddrCount-console", ipAddrCount.count, consoleOperator1.input); dag.addStream("urlCount-console", urlCount.count, consoleOperator2.input); dag.addStream("httpStatusCount-console", httpStatusCount.count, consoleOperator3.input); //dag.addStream("agg-console", agg.output, consoleOperator4.input); dag.addStream("numOfBytesSum-console", numOfBytesSum.sumLong, consoleOperator4.input); }