public TFileReadable(FileSystem fs, Path path, int osBufferSize, Configuration conf) throws IOException { this.fsdis = fs.open(path, osBufferSize); this.reader = new TFile.Reader(fsdis, fs.getFileStatus(path).getLen(), conf); this.scanner = reader.createScanner(); keyBuffer = new byte[32]; valueBuffer = new byte[32]; }
private void checkBlockIndex(int count, int recordIndex, int blockIndexExpected) throws IOException { Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Scanner scanner = reader.createScanner(); scanner.seekTo(composeSortedKey(KEY, count, recordIndex).getBytes()); Assert.assertEquals(blockIndexExpected, scanner.currentLocation .getBlockIndex()); scanner.close(); reader.close(); }
public void testNoDataEntry() throws IOException { if (skip) return; closeOutput(); Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Assert.assertTrue(reader.isSorted()); Scanner scanner = reader.createScanner(); Assert.assertTrue(scanner.atEnd()); scanner.close(); reader.close(); }
void unsortedWithSomeCodec(String codec) throws IOException { Path uTfile = new Path(ROOT, "unsorted.tfile"); FSDataOutputStream fout = createFSOutput(uTfile); Writer writer = new Writer(fout, minBlockSize, codec, null, conf); writeRecords(writer); writer.close(); fout.close(); FSDataInputStream fin = fs.open(uTfile); Reader reader = new Reader(fs.open(uTfile), fs.getFileStatus(uTfile).getLen(), conf); Scanner scanner = reader.createScanner(); readAllRecords(scanner); scanner.close(); reader.close(); fin.close(); fs.delete(uTfile, true); }
@Test public void testLocate() throws IOException { if (skip) return; writeRecords(3 * records1stBlock); Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Scanner scanner = reader.createScanner(); locate(scanner, composeSortedKey(KEY, 2).getBytes()); locate(scanner, composeSortedKey(KEY, records1stBlock - 1).getBytes()); locate(scanner, composeSortedKey(KEY, records1stBlock).getBytes()); Location locX = locate(scanner, "keyX".getBytes()); Assert.assertEquals(scanner.endLocation, locX); scanner.close(); reader.close(); }
@Test public void testLocate() throws IOException { if (skip) return; writeRecords(3 * records1stBlock); Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Scanner scanner = reader.createScanner(); locate(scanner, composeSortedKey(KEY, 2).getBytes()); locate(scanner, composeSortedKey(KEY, records1stBlock - 1).getBytes()); locate(scanner, composeSortedKey(KEY, records1stBlock).getBytes()); Location locX = locate(scanner, "keyX".getBytes()); Assert.assertEquals(scanner.endLocation, locX); scanner.close(); reader.close(); }
public void reset() throws IOException { IOUtils.cleanup(LOG, scanner); scanner = reader.createScanner(); }
public void reset() throws IOException { IOUtils.cleanup(LOG, scanner); scanner = reader.createScanner(); }
public LogReader(Configuration conf, Path remoteAppLogFile) throws IOException { FileContext fileContext = FileContext.getFileContext(remoteAppLogFile.toUri(), conf); this.fsDataIStream = fileContext.open(remoteAppLogFile); reader = new TFile.Reader(this.fsDataIStream, fileContext.getFileStatus( remoteAppLogFile).getLen(), conf); this.scanner = reader.createScanner(); }
private void checkBlockIndex(int recordIndex, int blockIndexExpected) throws IOException { Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Scanner scanner = reader.createScanner(); scanner.seekTo(composeSortedKey(KEY, recordIndex).getBytes()); Assert.assertEquals(blockIndexExpected, scanner.currentLocation .getBlockIndex()); scanner.close(); reader.close(); }
public void reset() throws IOException { IOUtils.cleanupWithLogger(LOG, scanner); scanner = reader.createScanner(); }
public TFileReader(FSDataInputStream fsdis, long fileLength, Configuration conf) throws IOException { this.fsdis = fsdis; reader = new Reader(fsdis, fileLength, conf); scanner = reader.createScanner(); }
@Test public void testNoDataEntry() throws IOException { if (skip) return; closeOutput(); Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Assert.assertTrue(reader.isSorted()); Scanner scanner = reader.createScanner(); Assert.assertTrue(scanner.atEnd()); scanner.close(); reader.close(); }
public LogReader(Configuration conf, DistributedFileSystemOps dfs, Path remoteAppLogFile) throws IOException { this.fsDataIStream = dfs.open(remoteAppLogFile); reader = new TFile.Reader(this.fsDataIStream, dfs.getFileStatus( remoteAppLogFile).getLen(), conf); this.scanner = reader.createScanner(); } private boolean atBeginning = true;
public LogReader(Configuration conf, Path remoteAppLogFile) throws IOException { FileContext fileContext = FileContext.getFileContext(remoteAppLogFile.toUri(), conf); this.fsDataIStream = fileContext.open(remoteAppLogFile); reader = new TFile.Reader(this.fsDataIStream, fileContext.getFileStatus( remoteAppLogFile).getLen(), conf); this.scanner = reader.createScanner(); }
public LogReader(Configuration conf, Path remoteAppLogFile) throws IOException { FileContext fileContext = FileContext.getFileContext(conf); this.fsDataIStream = fileContext.open(remoteAppLogFile); reader = new TFile.Reader(this.fsDataIStream, fileContext.getFileStatus( remoteAppLogFile).getLen(), conf); this.scanner = reader.createScanner(); }
private void checkBlockIndex(int recordIndex, int blockIndexExpected) throws IOException { Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Scanner scanner = reader.createScanner(); scanner.seekTo(composeSortedKey(KEY, recordIndex).getBytes()); Assert.assertEquals(blockIndexExpected, scanner.currentLocation .getBlockIndex()); scanner.close(); reader.close(); }
public TFileReadable(FileSystem fs, Path path, int osBufferSize, Configuration conf) throws IOException { this.fsdis = fs.open(path, osBufferSize); this.reader = new TFile.Reader(fsdis, fs.getFileStatus(path).getLen(), conf); this.scanner = reader.createScanner(); keyBuffer = new byte[32]; valueBuffer = new byte[32]; }
@Test public void testNoDataEntry() throws IOException { if (skip) return; closeOutput(); Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Assert.assertTrue(reader.isSorted()); Scanner scanner = reader.createScanner(); Assert.assertTrue(scanner.atEnd()); scanner.close(); reader.close(); }
public TFileReadable(FileSystem fs, Path path, int osBufferSize, Configuration conf) throws IOException { this.fsdis = fs.open(path, osBufferSize); this.reader = new TFile.Reader(fsdis, fs.getFileStatus(path).getLen(), conf); this.scanner = reader.createScanner(); keyBuffer = new byte[32]; valueBuffer = new byte[32]; }