/** * Create the DAG */ @Override public void populateDAG(DAG dag, Configuration conf) { InputGenerator randomInputGenerator = dag.addOperator("rand", new InputGenerator()); FaithfulRScript rScriptOp = dag.addOperator("rScriptOp", new FaithfulRScript("com/datatorrent/examples/r/oldfaithful/eruptionModel.R", "eruptionModel", "retVal")); ConsoleOutputOperator consoles = dag.addOperator("consoles", new ConsoleOutputOperator()); Map<String, FaithfulRScript.REXP_TYPE> argTypeMap = new HashMap<String, FaithfulRScript.REXP_TYPE>(); argTypeMap.put("ELAPSEDTIME", FaithfulRScript.REXP_TYPE.REXP_INT); argTypeMap.put("ERUPTIONS", FaithfulRScript.REXP_TYPE.REXP_ARRAY_DOUBLE); argTypeMap.put("WAITING", FaithfulRScript.REXP_TYPE.REXP_ARRAY_INT); rScriptOp.setArgTypeMap(argTypeMap); dag.addStream("ingen_faithfulRscript", randomInputGenerator.outputPort, rScriptOp.faithfulInput).setLocality(locality); dag.addStream("ingen_faithfulRscript_eT", randomInputGenerator.elapsedTime, rScriptOp.inputElapsedTime).setLocality(locality); dag.addStream("faithfulRscript_console_s", rScriptOp.strOutput, consoles.input).setLocality(locality); } }
@Override public void populateDAG(DAG dag, Configuration config) { KafkaSinglePortInputOperator in = dag.addOperator("kafkaIn", new KafkaSinglePortInputOperator()); TwitterStreamOperator tweeterStream = dag.addOperator("tweeterStream", TwitterStreamOperator.class); HdfsSinkOperator hdfsSinkOperator = dag.addOperator("hdfsSinkOperator", HdfsSinkOperator.class); hdfsSinkOperator.setFileName(config.get(DiPConfiguration.HDFS_FILE_NAME)); HBaseSinkOperator hBaseSinkOperator = dag.addOperator("hBaseSinkOperator",new HBaseSinkOperator(config)); hBaseSinkOperator.getStore().setTableName(config.get(DiPConfiguration.HBASE_TABLE_NAME)); hBaseSinkOperator.getStore().setZookeeperQuorum(config.get(DiPConfiguration.ZK_HOST)); hBaseSinkOperator.getStore().setZookeeperClientPort(config.getInt(DiPConfiguration.ZK_PORT, 2181)); dag.addStream("kafkaStream", in.outputPort, tweeterStream.inputPort).setLocality(Locality.CONTAINER_LOCAL); dag.addStream("hdfsStream", tweeterStream.hdfsOutputPort, hdfsSinkOperator.input); dag.addStream("hbaseStream", tweeterStream.hBaseOutputPort, hBaseSinkOperator.input); }
@Override public void populateDAG(DAG dag, Configuration conf) { // Setup the operator to get the data from twitter sample stream injected into the system. TwitterSampleInput twitterFeed = new TwitterSampleInput(); twitterFeed = dag.addOperator("TweetSampler", twitterFeed); // Setup a node to count the unique Hashtags within a window. UniqueCounter<String> uniqueCounter = dag.addOperator("UniqueHashtagCounter", new UniqueCounter<String>()); // Get the aggregated Hashtag counts and count them over last 5 mins. WindowedTopCounter<String> topCounts = dag.addOperator("TopCounter", new WindowedTopCounter<String>()); topCounts.setTopCount(10); topCounts.setSlidingWindowWidth(600); topCounts.setDagWindowWidth(1); dag.addStream("TwittedHashtags", twitterFeed.hashtag, uniqueCounter.data).setLocality(locality); // Count unique Hashtags dag.addStream("UniqueHashtagCounts", uniqueCounter.count, topCounts.input); TwitterTopCounterApplication.consoleOutput(dag, "topHashtags", topCounts.output, SNAPSHOT_SCHEMA, "hashtag"); } }
@Override public void populateDAG(DAG dag, Configuration conf) { TailFsInputOperator log = dag.addOperator("log", new TailFsInputOperator()); log.setDelimiter('\n'); log.setFilePath("/var/log/apache2/access.log"); ApacheLogParseMapOutputOperator parse = dag.addOperator("parse", new ApacheLogParseMapOutputOperator()); GeoIPExtractor geoIPExtractor = new GeoIPExtractor(); // Can't put this file in resources until licensing issue is straightened out geoIPExtractor.setDatabasePath("/home/david/GeoLiteCity.dat"); parse.registerInformationExtractor("ip", geoIPExtractor); parse.registerInformationExtractor("agent", new UserAgentExtractor()); TimestampExtractor timestampExtractor = new TimestampExtractor(); timestampExtractor.setDateFormatString("dd/MMM/yyyy:HH:mm:ss Z"); parse.registerInformationExtractor("time", timestampExtractor); ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator()); dag.addStream("log-parse", log.output, parse.data); dag.addStream("parse-console", parse.output, console.input).setLocality(Locality.CONTAINER_LOCAL); }
@Override public void populateDAG(DAG dag, Configuration conf) { final SiteToSiteClientConfig clientConfig = new SiteToSiteClient.Builder() .url("http://localhost:8080/nifi") .portName("Apex") .buildConfig(); final int batchSize = 1; final SiteToSiteClient.Builder builder = new SiteToSiteClient.Builder().fromConfig(clientConfig); final NiFiDataPacketBuilder<String> dataPacketBuilder = new StringNiFiDataPacketBuilder(); final WindowDataManager windowDataManager = new WindowDataManager.NoopWindowDataManager(); RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator()); NiFiSinglePortOutputOperator nifi = dag.addOperator("nifi", new NiFiSinglePortOutputOperator(builder, dataPacketBuilder, windowDataManager, batchSize)); dag.addStream("rand_nifi", rand.string_data, nifi.inputPort).setLocality(null); }
@Override public void populateDAG(DAG dag, Configuration conf) { JdbcPOJOInputOperator jdbcInputOperator = dag.addOperator("JdbcInput", new JdbcPOJOInputOperator()); /** * The class given below can be updated to the user defined class based on * input table schema The addField infos method needs to be updated * accordingly This line can be commented and class can be set from the * properties file */ // dag.setOutputPortAttribute(jdbcInputOperator.outputPort, Context.PortContext.TUPLE_CLASS, PojoEvent.class); jdbcInputOperator.setFieldInfos(addFieldInfos()); JdbcStore store = new JdbcStore(); jdbcInputOperator.setStore(store); FileLineOutputOperator fileOutput = dag.addOperator("FileOutputOperator", new FileLineOutputOperator()); dag.addStream("POJO's", jdbcInputOperator.outputPort, fileOutput.input).setLocality(Locality.CONTAINER_LOCAL); }
@Override public void populateDAG(DAG dag, Configuration conf) { // Sample DAG with 2 operators // Replace this code with the DAG you want to build SeedEventGenerator seedGen = dag.addOperator("seedGen", SeedEventGenerator.class); seedGen.setSeedStart(1); seedGen.setSeedEnd(10); seedGen.addKeyData("x", 0, 10); seedGen.addKeyData("y", 0, 100); ConsoleOutputOperator cons = dag.addOperator("console", new ConsoleOutputOperator()); cons.setStringFormat("hello: %s"); dag.addStream("seeddata", seedGen.val_list, cons.input).setLocality(Locality.CONTAINER_LOCAL); } }
@Override public void populateDAG(DAG dag, Configuration conf) { TwitterSampleInput twitterFeed = new TwitterSampleInput(); twitterFeed = dag.addOperator("TweetSampler", twitterFeed); TwitterStatusWordExtractor wordExtractor = dag.addOperator("WordExtractor", TwitterStatusWordExtractor.class); UniqueCounter<String> uniqueCounter = dag.addOperator("UniqueWordCounter", new UniqueCounter<String>()); WindowedTopCounter<String> topCounts = dag.addOperator("TopCounter", new WindowedTopCounter<String>()); topCounts.setSlidingWindowWidth(120); topCounts.setDagWindowWidth(1); dag.addStream("TweetStream", twitterFeed.text, wordExtractor.input); dag.addStream("TwittedWords", wordExtractor.output, uniqueCounter.data); dag.addStream("UniqueWordCounts", uniqueCounter.count, topCounts.input).setLocality(Locality.CONTAINER_LOCAL); TwitterTopCounterApplication.consoleOutput(dag, "topWords", topCounts.output, SNAPSHOT_SCHEMA, "word"); } }
@Override public void populateDAG(DAG dag, Configuration conf) { final SiteToSiteClientConfig clientConfig = new SiteToSiteClient.Builder() .url("http://localhost:8080/nifi") .portName("Apex") .requestBatchCount(5) .buildConfig(); final SiteToSiteClient.Builder builder = new SiteToSiteClient.Builder().fromConfig(clientConfig); final WindowDataManager windowDataManager = new WindowDataManager.NoopWindowDataManager(); NiFiSinglePortInputOperator nifi = dag.addOperator("nifi", new NiFiSinglePortInputOperator(builder, windowDataManager)); ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator()); dag.addStream("nifi_console", nifi.outputPort, console.input).setLocality(null); }
@Override public void populateDAG(DAG dag, Configuration conf) { //dag.setAttribute(DAGContext.APPLICATION_NAME, "TweetsDump"); TwitterSampleInput twitterStream = dag.addOperator("TweetSampler", new TwitterSampleInput()); //ConsoleOutputOperator dbWriter = dag.addOperator("DatabaseWriter", new ConsoleOutputOperator()); Status2Database dbWriter = dag.addOperator("DatabaseWriter", new Status2Database()); dbWriter.getStore().setDatabaseDriver("com.mysql.jdbc.Driver"); dbWriter.getStore().setDatabaseUrl("jdbc:mysql://node6.morado.com:3306/twitter"); dbWriter.getStore().setConnectionProperties("user:twitter"); dag.addStream("Statuses", twitterStream.status, dbWriter.input).setLocality(Locality.CONTAINER_LOCAL); }
@Override public void populateDAG(DAG dag, Configuration conf) { FileInputOperator fileInput = dag.addOperator("fileInput", getFileInput()); StreamingJsonParser parser = dag.addOperator("parser", getParser()); dag.getMeta(parser).getMeta(parser.output).getAttributes().put(Context.PortContext.TUPLE_CLASS, Person.class); ConsoleOutputOperator consoleOutput = dag.addOperator("output", new ConsoleOutputOperator()); dag.addStream("Input", fileInput.output, parser.in).setLocality(Locality.CONTAINER_LOCAL); dag.addStream("pojo", parser.output, consoleOutput.input).setLocality(Locality.CONTAINER_LOCAL); }
@Override public void populateDAG(DAG dag, Configuration configuration) { AvroFileInputOperator avroFileInputOperator = dag.addOperator("AvroFileInputOperator", this.avroFileInputOperator); AvroToPojo avroToPojo = dag.addOperator("AvroGenericObjectToPojo", new AvroToPojo()); dag.setOutputPortAttribute(avroToPojo.output, Context.PortContext.TUPLE_CLASS, pojoClass); dag.addStream("avroFileContainerToPojo", avroFileInputOperator.output, avroToPojo.data) .setLocality(DAG.Locality.CONTAINER_LOCAL); output.set(avroToPojo.output); errorPort.set(avroToPojo.errorPort); completedAvroFilesPort.set(avroFileInputOperator.completedFilesPort); avroErrorRecordsPort.set(avroFileInputOperator.errorRecordsPort); }
@Override public void populateDAG(DAG dag, Configuration conf) { RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator()); PiCalculateOperator calc = dag.addOperator("picalc", new PiCalculateOperator()); ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator()); dag.addStream("rand_calc", rand.integer_data, calc.input).setLocality(locality); dag.addStream("rand_console",calc.output, console.input).setLocality(locality); }
@Override public void populateDAG(DAG dag, Configuration conf) { //dag.setAttribute(DAGContext.APPLICATION_NAME, "TweetsDump"); TwitterSampleInput twitterStream = dag.addOperator("TweetSampler", new TwitterSampleInput()); Status2Hbase hBaseWriter = dag.addOperator("DatabaseWriter", new Status2Hbase()); dag.addStream("Statuses", twitterStream.status, hBaseWriter.input).setLocality(Locality.CONTAINER_LOCAL); }
@Override public void populateDAG(DAG dag, Configuration conf) { S3InputModule inputModule = dag.addModule("S3InputModule", new S3InputModule()); HDFSFileCopyModule outputModule = dag.addModule("HDFSFileCopyModule", new HDFSFileCopyModule()); dag.addStream("FileMetaData", inputModule.filesMetadataOutput, outputModule.filesMetadataInput); dag.addStream("BlocksMetaData", inputModule.blocksMetadataOutput, outputModule.blocksMetadataInput) .setLocality(Locality.THREAD_LOCAL); dag.addStream("BlocksData", inputModule.messages, outputModule.blockData).setLocality(Locality.THREAD_LOCAL); }
@Override public void populateDAG(DAG dag, Configuration conf) { AvroFileInputOperator avroInputOperator = dag.addOperator("avroInputOperator", getAvroFileInput()); AvroToPojo avroToPojo = dag.addOperator("AvroToPojo", getAvroToPojo()); ConsoleOutputOperator consoleOutput = dag.addOperator("GenericRecordOp", new ConsoleOutputOperator()); dag.getMeta(avroToPojo).getMeta(avroToPojo.output).getAttributes().put(Context.PortContext.TUPLE_CLASS, SimpleOrder.class); dag.addStream("GenericRecords", avroInputOperator.output, avroToPojo.data).setLocality(Locality.THREAD_LOCAL); dag.addStream("POJO", avroToPojo.output, consoleOutput.input).setLocality(Locality.CONTAINER_LOCAL); }
@Override public void populateDAG(DAG dag, Configuration conf) { FSInputModule inputModule = dag.addModule("HDFSInputModule", new FSInputModule()); S3OutputModule outputModule = dag.addModule("S3OutputModule", new S3OutputModule()); dag.addStream("FileMetaData", inputModule.filesMetadataOutput, outputModule.filesMetadataInput); dag.addStream("BlocksMetaData", inputModule.blocksMetadataOutput, outputModule.blocksMetadataInput) .setLocality(DAG.Locality.CONTAINER_LOCAL); dag.addStream("BlocksData", inputModule.messages, outputModule.blockData).setLocality(DAG.Locality.CONTAINER_LOCAL); } }
@Override public void populateDAG(DAG dag, Configuration conf) { AvroFileInputOperator avroInputOperator = dag.addOperator("avroInputOperator", getAvroFileInput()); ConsoleOutputOperator consoleOutput = dag.addOperator("GenericRecordOp", new ConsoleOutputOperator()); dag.addStream("pojo", avroInputOperator.output, consoleOutput.input).setLocality(Locality.CONTAINER_LOCAL); }
@Override public void populateDAG(DAG dag, Configuration conf) { MessageReceiver receiver = dag.addOperator("Message Receiver", MessageReceiver.class); MessageResponder responder = dag.addOperator("Message Responder", MessageResponder.class); // Locality has to be container so that the operators use the same socket dag.addStream("messages", receiver.messageOutput, responder.messageInput).setLocality(DAG.Locality.CONTAINER_LOCAL); } }
@Override public void populateDAG(DAG dag, Configuration conf) { LineByLineFileInputOperator in = dag.addOperator("lines", LineByLineFileInputOperator.class); KafkaSinglePortOutputOperator<String,String> out = dag.addOperator("kafkaOutput", new KafkaSinglePortOutputOperator<String,String>()); dag.addStream("data", in.output, out.inputPort).setLocality(Locality.CONTAINER_LOCAL); } }