@Override public State makeState(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { LOG.info("makeState(partitonIndex={}, numpartitions={}", partitionIndex, numPartitions); HdfsState state = new HdfsState(this.options); state.prepare(conf, metrics, partitionIndex, numPartitions); return state; } }
@Override public void updateState(HdfsState state, List<TridentTuple> tuples, TridentCollector collector) { state.updateState(tuples, collector); } }
private void initLastTxn(Map<String, Object> conf, int partition) { // include partition id in the file name so that index for different partitions are independent. String indexFileName = String.format(".index.%s.%d", conf.get(Config.TOPOLOGY_NAME), partition); this.indexFilePath = new Path(options.fileNameFormat.getPath(), indexFileName); try { this.lastSeenTxn = getTxnRecord(indexFilePath); LOG.debug("initLastTxn updated lastSeenTxn to [{}]", this.lastSeenTxn); } catch (IOException e) { LOG.warn("initLastTxn failed due to IOException.", e); throw new RuntimeException(e); } }
/** * Reads the last txn record from index file if it exists, if not from .tmp file if exists. * * @param indexFilePath the index file path * @return the txn record from the index file or a default initial record. * * @throws IOException */ private TxnRecord getTxnRecord(Path indexFilePath) throws IOException { Path tmpPath = tmpFilePath(indexFilePath.toString()); if (this.options.fs.exists(indexFilePath)) { return readTxnRecord(indexFilePath); } else if (this.options.fs.exists(tmpPath)) { return readTxnRecord(tmpPath); } return new TxnRecord(0, options.currentFile.toString(), 0); }
state.beginCommit(1L); state.updateState(createMockTridentTuples(batch1Count), null); state.commit(1L); state.beginCommit(2L); state.updateState(createMockTridentTuples(batch2Count), null); state.commit(2L); state.beginCommit(3L); state.updateState(createMockTridentTuples(batch3Count), null); state.commit(3L); state.beginCommit(3L); state.updateState(createMockTridentTuples(batch3ReplayCount), null); state.commit(3L); state.close();
void prepare(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { this.options.prepare(conf, partitionIndex, numPartitions); initLastTxn(conf, partitionIndex); }
@Test public void testIndexFileCreation() throws Exception { HdfsState state = createHdfsState(); state.beginCommit(1L); Collection<File> files = FileUtils.listFiles(new File(TEST_OUT_DIR), null, false); File hdfsIndexFile = Paths.get(TEST_OUT_DIR, INDEX_FILE_PREFIX + TEST_TOPOLOGY_NAME + ".0").toFile(); Assert.assertTrue(files.contains(hdfsIndexFile)); }
@Test public void testUpdateState() throws Exception { HdfsState state = createHdfsState(); state.beginCommit(1L); int tupleCount = 100; state.updateState(createMockTridentTuples(tupleCount), null); state.commit(1L); state.close(); List<String> lines = getLinesFromCurrentDataFile(); List<String> expected = new ArrayList<>(); for (int i = 0; i < tupleCount; i++) { expected.add("data"); } Assert.assertEquals(tupleCount, lines.size()); Assert.assertEquals(expected, lines); }
/** * Reads the last txn record from index file if it exists, if not * from .tmp file if exists. * * @param indexFilePath the index file path * @return the txn record from the index file or a default initial record. * @throws IOException */ private TxnRecord getTxnRecord(Path indexFilePath) throws IOException { Path tmpPath = tmpFilePath(indexFilePath.toString()); if (this.options.fs.exists(indexFilePath)) { return readTxnRecord(indexFilePath); } else if (this.options.fs.exists(tmpPath)) { return readTxnRecord(tmpPath); } return new TxnRecord(0, options.currentFile.toString(), 0); }
void prepare(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { this.options.prepare(conf, partitionIndex, numPartitions); initLastTxn(conf, partitionIndex); }
private HdfsState createHdfsState() { Fields hdfsFields = new Fields("f1"); RecordFormat recordFormat = new DelimitedRecordFormat().withFields(hdfsFields); FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB); HdfsState.Options options = new HdfsState.HdfsFileOptions() .withFileNameFormat(fileNameFormat) .withRecordFormat(recordFormat) .withRotationPolicy(rotationPolicy) .withFsUrl("file://" + TEST_OUT_DIR); Map<String, Object> conf = new HashMap<>(); conf.put(Config.TOPOLOGY_NAME, TEST_TOPOLOGY_NAME); HdfsState state = new HdfsState(options); state.prepare(conf, null, 0, 1); return state; }
@Test public void testRecoverOneBatch() throws Exception { HdfsState state = createHdfsState(); // batch 1 is played with 25 tuples initially. state.beginCommit(1L); state.updateState(createMockTridentTuples(25), null); // batch 1 is replayed with 50 tuples. int replayBatchSize = 50; state.beginCommit(1L); state.updateState(createMockTridentTuples(replayBatchSize), null); state.commit(1L); // close the state to force flush state.close(); // Ensure that the original batch1 is discarded and new one is persisted. List<String> lines = getLinesFromCurrentDataFile(); Assert.assertEquals(replayBatchSize, lines.size()); List<String> expected = new ArrayList<>(); for (int i = 0; i < replayBatchSize; i++) { expected.add("data"); } Assert.assertEquals(expected, lines); }
@Override public void updateState(HdfsState state, List<TridentTuple> tuples, TridentCollector collector) { state.updateState(tuples, collector); } }
private void initLastTxn(Map conf, int partition) { // include partition id in the file name so that index for different partitions are independent. String indexFileName = String.format(".index.%s.%d", conf.get(Config.TOPOLOGY_NAME), partition); this.indexFilePath = new Path(options.fileNameFormat.getPath(), indexFileName); try { this.lastSeenTxn = getTxnRecord(indexFilePath); LOG.debug("initLastTxn updated lastSeenTxn to [{}]", this.lastSeenTxn); } catch (IOException e) { LOG.warn("initLastTxn failed due to IOException.", e); throw new RuntimeException(e); } }
@Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { LOG.info("makeState(partitonIndex={}, numpartitions={}", partitionIndex, numPartitions); HdfsState state = new HdfsState(this.options); state.prepare(conf, metrics, partitionIndex, numPartitions); return state; } }
@Override public void updateState(HdfsState state, List<TridentTuple> tuples, TridentCollector collector) { state.updateState(tuples, collector); } }
@Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { LOG.info("makeState(partitonIndex={}, numpartitions={}", partitionIndex, numPartitions); HdfsState state = new HdfsState(this.options); state.prepare(conf, metrics, partitionIndex, numPartitions); return state; } }