HdfsSpout spout = new HdfsSpout().withOutputFields(TextFileReader.defaultFields) .setReaderType(fileFormat) .setHdfsUri(hdfsUri)
private AutoCloseableHdfsSpout makeSpout(String readerType, String[] outputFields) { HdfsSpout spout = new HdfsSpout().withOutputFields(outputFields) .setReaderType(readerType) .setHdfsUri(DFS_CLUSTER_RULE.getDfscluster().getURI().toString()) .setSourceDir(source.toString()) .setArchiveDir(archive.toString()) .setBadFilesDir(badfiles.toString()); return new AutoCloseableHdfsSpout(spout); }
static StormTopology getTopology(Map<String, Object> config) { final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); final int boltNum = Helper.getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM); final String fileFormat = Helper.getStr(config, "text"); final String hdfsUri = Helper.getStr(config, HDFS_URI); final String sourceDir = Helper.getStr(config, SOURCE_DIR); final String archiveDir = Helper.getStr(config, ARCHIVE_DIR); final String badDir = Helper.getStr(config, BAD_DIR); // 1 - Setup Hdfs Spout -------- HdfsSpout spout = new HdfsSpout() .setReaderType(fileFormat) .setHdfsUri(hdfsUri) .setSourceDir(sourceDir) .setArchiveDir(archiveDir) .setBadFilesDir(badDir) .withOutputFields(TextFileReader.defaultFields); // 2 - DevNull Bolt -------- DevNullBolt bolt = new DevNullBolt(); // 3 - Setup Topology -------- TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, spout, spoutNum); builder.setBolt(BOLT_ID, bolt, boltNum) .localOrShuffleGrouping(SPOUT_ID); return builder.createTopology(); }