@Override public void execute(List<TridentTuple> tuples) throws IOException { for (TridentTuple tuple : tuples) { byte[] bytes = this.format.format(tuple); out.write(bytes); this.offset += bytes.length; } } }
@Override Path createOutputFile() throws IOException { Path p = new Path(this.fsUrl + this.fileNameFormat.getPath(), this.fileNameFormat.getName(this.rotation, System.currentTimeMillis())); this.writer = SequenceFile.createWriter( this.hdfsConfig, SequenceFile.Writer.file(p), SequenceFile.Writer.keyClass(this.format.keyClass()), SequenceFile.Writer.valueClass(this.format.valueClass()), SequenceFile.Writer.compression(this.compressionType, this.codecFactory.getCodecByName(this.compressionCodec)) ); return p; }
public static StormTopology buildTopology(String hdfsUrl) { FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l), new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l), new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l)); spout.setCycle(true); TridentTopology topology = new TridentTopology(); Stream stream = topology.newStream("spout1", spout); Fields hdfsFields = new Fields("sentence", "key"); FileNameFormat fileNameFormat = new DefaultFileNameFormat() .withPath("/tmp/trident") .withPrefix("trident") .withExtension(".txt"); RecordFormat recordFormat = new DelimitedRecordFormat() .withFields(hdfsFields); FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB); HdfsState.Options options = new HdfsState.HdfsFileOptions() .withFileNameFormat(fileNameFormat) .withRecordFormat(recordFormat) .withRotationPolicy(rotationPolicy) .withFsUrl(hdfsUrl) .withConfigKey("hdfs.config"); StateFactory factory = new HdfsStateFactory().withOptions(options); TridentState state = stream .partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields()); return topology.build(); }
@Test public void testParameters() { SimpleFileNameFormat format = new SimpleFileNameFormat() .withName("$TIME.$HOST.$PARTITION.$NUM.txt") .withPath("/mypath") .withTimeFormat("yyyy-MM-dd HH:mm:ss"); format.prepare(null, 3, 5); long now = System.currentTimeMillis(); String path = format.getPath(); String name = format.getName(1, now); Assert.assertEquals("/mypath", path); String time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(now); String host = null; try { host = Utils.localHostname(); } catch (UnknownHostException e) { e.printStackTrace(); } Assert.assertEquals(time + "." + host + ".3.1.txt", name); }
@Override void doRecover(Path srcPath, long nBytes) throws Exception { SequenceFile.Reader reader = new SequenceFile.Reader(this.hdfsConfig, SequenceFile.Reader.file(srcPath), SequenceFile.Reader.length(nBytes)); Writable key = (Writable) this.format.keyClass().newInstance(); Writable value = (Writable) this.format.valueClass().newInstance(); while (reader.next(key, value)) { this.writer.append(key, value); } }
public static StormTopology buildTopology(String hdfsUrl) { FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l), new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l), new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l)); spout.setCycle(true); TridentTopology topology = new TridentTopology(); Stream stream = topology.newStream("spout1", spout); Fields hdfsFields = new Fields("sentence", "key"); FileNameFormat fileNameFormat = new DefaultFileNameFormat() .withPath("/tmp/trident") .withPrefix("trident") .withExtension(".seq"); FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB); HdfsState.Options seqOpts = new HdfsState.SequenceFileOptions() .withFileNameFormat(fileNameFormat) .withSequenceFormat(new DefaultSequenceFormat("key", "sentence")) .withRotationPolicy(rotationPolicy) .withFsUrl(hdfsUrl) .withConfigKey("hdfs.config") .addRotationAction(new MoveFileAction().toDestination("/tmp/dest2/")); StateFactory factory = new HdfsStateFactory().withOptions(seqOpts); TridentState state = stream .partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields()); return topology.build(); }
@Test public void testDefaults() { SimpleFileNameFormat format = new SimpleFileNameFormat(); format.prepare(null, 3, 5); long now = System.currentTimeMillis(); String path = format.getPath(); String name = format.getName(1, now); Assert.assertEquals("/storm", path); String time = new SimpleDateFormat("yyyyMMddHHmmss").format(now); Assert.assertEquals(time + ".1.txt", name); }
private HdfsState createHdfsState() { Fields hdfsFields = new Fields("f1"); RecordFormat recordFormat = new DelimitedRecordFormat().withFields(hdfsFields); FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB); HdfsState.Options options = new HdfsState.HdfsFileOptions() .withFileNameFormat(fileNameFormat) .withRecordFormat(recordFormat) .withRotationPolicy(rotationPolicy) .withFsUrl("file://" + TEST_OUT_DIR); Map<String, Object> conf = new HashMap<>(); conf.put(Config.TOPOLOGY_NAME, TEST_TOPOLOGY_NAME); HdfsState state = new HdfsState(options); state.prepare(conf, null, 0, 1); return state; }
@Test(expected = IllegalArgumentException.class) public void testTimeFormat() { SimpleFileNameFormat format = new SimpleFileNameFormat() .withTimeFormat("xyz"); format.prepare(null, 3, 5); } }
private void initLastTxn(Map<String, Object> conf, int partition) { // include partition id in the file name so that index for different partitions are independent. String indexFileName = String.format(".index.%s.%d", conf.get(Config.TOPOLOGY_NAME), partition); this.indexFilePath = new Path(options.fileNameFormat.getPath(), indexFileName); try { this.lastSeenTxn = getTxnRecord(indexFilePath); LOG.debug("initLastTxn updated lastSeenTxn to [{}]", this.lastSeenTxn); } catch (IOException e) { LOG.warn("initLastTxn failed due to IOException.", e); throw new RuntimeException(e); } }
throw new IllegalStateException("File system URL must be specified."); this.fileNameFormat.prepare(conf, partitionIndex, numPartitions); this.hdfsConfig = new Configuration(); Map<String, Object> map = (Map<String, Object>) conf.get(this.configKey);
@Override Path createOutputFile() throws IOException { Path p = new Path(this.fsUrl + this.fileNameFormat.getPath(), this.fileNameFormat.getName(this.rotation, System.currentTimeMillis())); this.writer = SequenceFile.createWriter( this.hdfsConfig, SequenceFile.Writer.file(p), SequenceFile.Writer.keyClass(this.format.keyClass()), SequenceFile.Writer.valueClass(this.format.valueClass()), SequenceFile.Writer.compression(this.compressionType, this.codecFactory.getCodecByName(this.compressionCodec)) ); return p; }
@Override void doRecover(Path srcPath, long nBytes) throws Exception { SequenceFile.Reader reader = new SequenceFile.Reader(this.hdfsConfig, SequenceFile.Reader.file(srcPath), SequenceFile.Reader.length(nBytes)); Writable key = (Writable) this.format.keyClass().newInstance(); Writable value = (Writable) this.format.valueClass().newInstance(); while(reader.next(key, value)) { this.writer.append(key, value); } }
@Override public void execute(List<TridentTuple> tuples) throws IOException { for (TridentTuple tuple : tuples) { byte[] bytes = this.format.format(tuple); out.write(bytes); this.offset += bytes.length; } } }
@Override Path createOutputFile() throws IOException { Path p = new Path(this.fsUrl + this.fileNameFormat.getPath(), this.fileNameFormat.getName(this.rotation, System.currentTimeMillis())); this.writer = SequenceFile.createWriter( this.hdfsConfig, SequenceFile.Writer.file(p), SequenceFile.Writer.keyClass(this.format.keyClass()), SequenceFile.Writer.valueClass(this.format.valueClass()), SequenceFile.Writer.compression(this.compressionType, this.codecFactory.getCodecByName(this.compressionCodec)) ); return p; }