public HDFSTransactionLogReaderV2(SequenceFile.Reader reader) { this.reader = reader; this.transactionEdits = new ArrayDeque<>(); this.key = new LongWritable(); this.commitMarkerCodec = new CommitMarkerCodec(); }
public HDFSTransactionLogReaderV2(SequenceFile.Reader reader) { this.reader = reader; this.transactionEdits = new ArrayDeque<>(); this.key = new LongWritable(); this.commitMarkerCodec = new CommitMarkerCodec(); }
@Test public void testRandomCommitMarkers() throws Exception { List<Integer> randomInts = new ArrayList<>(); Path newLog = new Path(TMP_FOLDER.newFolder().getAbsolutePath(), LOG_FILE); // Write a bunch of random commit markers try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, newLog, LongWritable.class, LongWritable.class, SequenceFile.CompressionType.NONE)) { for (int i = 0; i < 1000; i++) { int randomNum = RANDOM.nextInt(Integer.MAX_VALUE); CommitMarkerCodec.writeMarker(writer, randomNum); randomInts.add(randomNum); } writer.hflush(); writer.hsync(); } // Read the commit markers back to verify the marker try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, newLog, conf); CommitMarkerCodec markerCodec = new CommitMarkerCodec()) { for (int num : randomInts) { Assert.assertEquals(num, markerCodec.readMarker(reader)); } } }
@Test public void testIncompleteCommitMarker() throws Exception { Path newLog = new Path(TMP_FOLDER.newFolder().getAbsolutePath(), LOG_FILE); try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, newLog, LongWritable.class, LongWritable.class, SequenceFile.CompressionType.NONE)) { String key = TxConstants.TransactionLog.NUM_ENTRIES_APPENDED; SequenceFile.ValueBytes valueBytes = new IncompleteValueBytes(); writer.appendRaw(key.getBytes(), 0, key.length(), valueBytes); writer.hflush(); writer.hsync(); } // Read the incomplete commit marker try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, newLog, conf); CommitMarkerCodec markerCodec = new CommitMarkerCodec()) { try { markerCodec.readMarker(reader); Assert.fail("Expected EOF Exception to be thrown"); } catch (EOFException e) { // expected since we didn't write the value bytes } } }
@Test public void testIncorrectCommitMarker() throws Exception { Path newLog = new Path(TMP_FOLDER.newFolder().getAbsolutePath(), LOG_FILE); // Write an incorrect marker try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, newLog, LongWritable.class, LongWritable.class, SequenceFile.CompressionType.NONE)) { String invalidKey = "IncorrectKey"; SequenceFile.ValueBytes valueBytes = new CommitMarkerCodec.CommitEntriesCount(100); writer.appendRaw(invalidKey.getBytes(), 0, invalidKey.length(), valueBytes); writer.hflush(); writer.hsync(); } // Read the commit markers back to verify the marker try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, newLog, conf); CommitMarkerCodec markerCodec = new CommitMarkerCodec()) { try { markerCodec.readMarker(reader); Assert.fail("Expected an IOException to be thrown"); } catch (IOException e) { // expected } } } }