for (Pair<Attribute, Object> attr : nm.inputPortAttributes.get(input)) { logger.debug("Building DAG: set port attribute {} to {} for port {}", attr.getLeft(), attr.getValue(), input); dag.setInputPortAttribute(input, attr.getLeft(), attr.getValue());
@Override public void populateDAG(DAG dag, Configuration conf) { RandomNumberGenerator random = dag.addOperator("randomInt", RandomNumberGenerator.class); TestPartition testPartition = dag.addOperator("testPartition", TestPartition.class); Codec3 codec = new Codec3(); dag.setInputPortAttribute(testPartition.in, PortContext.STREAM_CODEC, codec); //Add locality if needed, e.g.: .setLocality(Locality.CONTAINER_LOCAL); dag.addStream("randomData", random.out, testPartition.in); } }
dag.setInputPortAttribute(port, PortContext.STREAM_CODEC, streamCodec);
@Override public void populateDAG(DAG dag, Configuration conf) { pojoDataGenerator = dag.addOperator("Input", new POJOGenerator()); TransformOperator transform = dag.addOperator("Process", new TransformOperator()); // Set expression map Map<String, String> expMap = new HashMap<>(); expMap.put("name", "{$.firstName}.concat(\" \").concat({$.lastName})"); expMap.put("age", "(new java.util.Date()).getYear() - {$.dateOfBirth}.getYear()"); expMap.put("address", "{$.address}.toLowerCase()"); transform.setExpressionMap(expMap); FunctionOperator.MapFunctionOperator<Object, ?> output = dag.addOperator("out", new FunctionOperator.MapFunctionOperator<>(outputFn)); dag.addStream("InputToTransform", pojoDataGenerator.output, transform.input); dag.addStream("TransformToOutput", transform.output, output.input); dag.setInputPortAttribute(transform.input, Context.PortContext.TUPLE_CLASS, CustomerEvent.class); dag.setOutputPortAttribute(transform.output, Context.PortContext.TUPLE_CLASS, CustomerInfo.class); setPartitioner(dag,conf,transform); }
@Override public void populateDAG(DAG dag, Configuration conf) { BlockWriter blockWriter = dag.addOperator("BlockWriter", new BlockWriter()); Synchronizer synchronizer = dag.addOperator("BlockSynchronizer", new Synchronizer()); dag.setInputPortAttribute(blockWriter.input, PortContext.PARTITION_PARALLEL, true); dag.setInputPortAttribute(blockWriter.blockMetadataInput, PortContext.PARTITION_PARALLEL, true); dag.addStream("CompletedBlockmetadata", blockWriter.blockMetadataOutput, synchronizer.blocksMetadataInput); HDFSFileMerger merger = new HDFSFileMerger(); merger = dag.addOperator("FileMerger", merger); dag.addStream("MergeTrigger", synchronizer.trigger, merger.input); merger.setFilePath(outputDirectoryPath); merger.setOverwriteOnConflict(overwriteOnConflict); blockWriter.setBlocksDirectory(blocksDirectory); merger.setBlocksDirectory(blocksDirectory); filesMetadataInput.set(synchronizer.filesMetadataInput); blocksMetadataInput.set(blockWriter.blockMetadataInput); blockData.set(blockWriter.input); }
@Override public void populateDAG(DAG dag, Configuration conf) { JdbcPOJOPollInputOperator poller = dag.addOperator("JdbcPoller", new JdbcPOJOPollInputOperator()); JdbcStore store = new JdbcStore(); poller.setStore(store); poller.setFieldInfos(addFieldInfos()); FileLineOutputOperator writer = dag.addOperator("Writer", new FileLineOutputOperator()); dag.setInputPortAttribute(writer.input, PortContext.PARTITION_PARALLEL, true); writer.setRotationWindows(60); dag.addStream("dbrecords", poller.outputPort, writer.input); }
@Override public void populateDAG(DAG dag, Configuration conf) { JsonGenerator generator = dag.addOperator("JsonGenerator", JsonGenerator.class); JsonParser jsonParser = dag.addOperator("jsonParser", JsonParser.class); CsvFormatter formatter = dag.addOperator("formatter", CsvFormatter.class); formatter.setSchema(SchemaUtils.jarResourceFileToString(filename)); dag.setInputPortAttribute(formatter.in, PortContext.TUPLE_CLASS, PojoEvent.class); HDFSOutputOperator<String> hdfsOutput = dag.addOperator("HDFSOutputOperator", HDFSOutputOperator.class); hdfsOutput.setLineDelimiter(""); dag.addStream("parserStream", generator.out, jsonParser.in); dag.addStream("formatterStream", jsonParser.out, formatter.in); dag.addStream("outputStream", formatter.out, hdfsOutput.input); } }
/** * Populate Yahoo Finance Example Application DAG. */ @SuppressWarnings("unchecked") @Override public void populateDAG(DAG dag, Configuration conf) { dag.getAttributes().put(DAG.STREAMING_WINDOW_SIZE_MILLIS, streamingWindowSizeMilliSeconds); StockTickInput tick = getStockTickInputOperator("StockTickInput", dag); SumKeyVal<String, Long> dailyVolume = getDailyVolumeOperator("DailyVolume", dag); ConsolidatorKeyVal<String,Double,Long,String,?,?> quoteOperator = getQuoteOperator("Quote", dag); RangeKeyVal<String, Double> highlow = getHighLowOperator("HighLow", dag, appWindowCountMinute); SumKeyVal<String, Long> minuteVolume = getMinuteVolumeOperator("MinuteVolume", dag, appWindowCountMinute); ConsolidatorKeyVal<String,HighLow<Double>,Long,?,?,?> chartOperator = getChartOperator("Chart", dag); SimpleMovingAverage<String, Double> priceSMA = getPriceSimpleMovingAverageOperator("PriceSMA", dag, appWindowCountSMA); DefaultPartitionCodec<String, Double> codec = new DefaultPartitionCodec<String, Double>(); dag.setInputPortAttribute(highlow.data, PortContext.STREAM_CODEC, codec); dag.setInputPortAttribute(priceSMA.data, PortContext.STREAM_CODEC, codec); dag.addStream("price", tick.price, quoteOperator.in1, highlow.data, priceSMA.data); dag.addStream("vol", tick.volume, dailyVolume.data, minuteVolume.data); dag.addStream("time", tick.time, quoteOperator.in3); dag.addStream("daily_vol", dailyVolume.sum, quoteOperator.in2); dag.addStream("quote_data", quoteOperator.out, getConsole("quoteConsole", dag, "QUOTE")); dag.addStream("high_low", highlow.range, chartOperator.in1); dag.addStream("vol_1min", minuteVolume.sum, chartOperator.in2); dag.addStream("chart_data", chartOperator.out, getConsole("chartConsole", dag, "CHART")); dag.addStream("sma_price", priceSMA.doubleSMA, getConsole("priceSMAConsole", dag, "Price SMA")); }
@Override public void populateDAG(DAG dag, Configuration conf) { // Test Data Generator Operator RandomDataGeneratorOperator gen = dag.addOperator("RandomGenerator", new RandomDataGeneratorOperator()); // Dedup Operator. Configuration through resources/META-INF/properties.xml TimeBasedDedupOperator dedup = dag.addOperator("Deduper", new TimeBasedDedupOperator()); // Console output operator for unique tuples ConsoleOutputOperator consoleUnique = dag.addOperator("ConsoleUnique", new ConsoleOutputOperator()); // Console output operator for duplicate tuples ConsoleOutputOperator consoleDuplicate = dag.addOperator("ConsoleDuplicate", new ConsoleOutputOperator()); // Console output operator for duplicate tuples ConsoleOutputOperator consoleExpired = dag.addOperator("ConsoleExpired", new ConsoleOutputOperator()); // Streams dag.addStream("Generator to Dedup", gen.output, dedup.input); // Connect Dedup unique to Console dag.addStream("Dedup Unique to Console", dedup.unique, consoleUnique.input); // Connect Dedup duplicate to Console dag.addStream("Dedup Duplicate to Console", dedup.duplicate, consoleDuplicate.input); // Connect Dedup expired to Console dag.addStream("Dedup Expired to Console", dedup.expired, consoleExpired.input); // Set Attribute TUPLE_CLASS for supplying schema information to the port dag.setInputPortAttribute(dedup.input, Context.PortContext.TUPLE_CLASS, TestEvent.class); // Uncomment the following line to create multiple partitions for Dedup operator. In this case: 2 // dag.setAttribute(dedup, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TimeBasedDedupOperator>(2)); }
dag.setInputPortAttribute(blockUploader.blockInput, Context.PortContext.PARTITION_PARALLEL, true); dag.setInputPortAttribute(blockUploader.blockMetadataInput, Context.PortContext.PARTITION_PARALLEL, true);
dag.setInputPortAttribute(rangeOperator.data, Context.PortContext.STREAM_CODEC, txCodec); dag.setInputPortAttribute(smaOperator.data, Context.PortContext.STREAM_CODEC, txCodec); dag.setInputPortAttribute(avgAlertingOperator.txInputPort, Context.PortContext.STREAM_CODEC, txCodec);
dag.setInputPortAttribute(recordReader.blocksMetadataInput, Context.PortContext.STREAM_CODEC, new SequentialFileBlockMetadataCodec());
dag.setInputPortAttribute(blockReader.blocksMetadataInput, Context.PortContext.STREAM_CODEC, new SequentialFileBlockMetadataCodec());