long length = fs.getFileStatus(path).getLen(); FSDataInputStream fsdis = fs.open(path); TFile.Reader reader = new TFile.Reader(fsdis, length, conf); try { LinkedHashMap<String, String> properties =
public TFileReadable(FileSystem fs, Path path, int osBufferSize, Configuration conf) throws IOException { this.fsdis = fs.open(path, osBufferSize); this.reader = new TFile.Reader(fsdis, fs.getFileStatus(path).getLen(), conf); this.scanner = reader.createScanner(); keyBuffer = new byte[32]; valueBuffer = new byte[32]; }
public HistoryFileReader(Path historyFile) throws IOException { fsdis = fs.open(historyFile); reader = new TFile.Reader(fsdis, fs.getFileStatus(historyFile).getLen(), getConfig()); reset(); }
public void testMetaBlocks() throws IOException { Path mFile = new Path(ROOT, "meta.tfile"); FSDataOutputStream fout = createFSOutput(mFile); Writer writer = new Writer(fout, minBlockSize, "none", null, conf); someTestingWithMetaBlock(writer, "none"); writer.close(); fout.close(); FSDataInputStream fin = fs.open(mFile); Reader reader = new Reader(fin, fs.getFileStatus(mFile).getLen(), conf); someReadingWithMetaBlock(reader); fs.delete(mFile, true); reader.close(); fin.close(); } }
public HistoryFileReader(Path historyFile) throws IOException { fsdis = fs.open(historyFile); reader = new TFile.Reader(fsdis, fs.getFileStatus(historyFile).getLen(), getConfig()); reset(); }
public LogReader(Configuration conf, Path remoteAppLogFile) throws IOException { FileContext fileContext = FileContext.getFileContext(remoteAppLogFile.toUri(), conf); this.fsDataIStream = fileContext.open(remoteAppLogFile); reader = new TFile.Reader(this.fsDataIStream, fileContext.getFileStatus( remoteAppLogFile).getLen(), conf); this.scanner = reader.createScanner(); }
private void checkBlockIndex(int recordIndex, int blockIndexExpected) throws IOException { Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Scanner scanner = reader.createScanner(); scanner.seekTo(composeSortedKey(KEY, recordIndex).getBytes()); Assert.assertEquals(blockIndexExpected, scanner.currentLocation .getBlockIndex()); scanner.close(); reader.close(); }
public HistoryFileReader(Path historyFile) throws IOException { fsdis = fs.open(historyFile); reader = new TFile.Reader(fsdis, fs.getFileStatus(historyFile).getLen(), getConfig()); reset(); }
public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException { FileSplit split = (FileSplit) genericSplit; Configuration job = context.getConfiguration(); start = split.getStart(); end = start + split.getLength(); final Path file = split.getPath(); // open the file and seek to the start of the split FileSystem fs = file.getFileSystem(job); fileIn = fs.open(split.getPath()); reader = new Reader(fileIn, fs.getFileStatus(file).getLen(), job); scanner = reader.createScannerByByteRange(start, split.getLength()); }
public TFileReader(FSDataInputStream fsdis, long fileLength, Configuration conf) throws IOException { this.fsdis = fsdis; reader = new Reader(fsdis, fileLength, conf); scanner = reader.createScanner(); }
@Test public void testNoDataEntry() throws IOException { if (skip) return; closeOutput(); Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Assert.assertTrue(reader.isSorted()); Scanner scanner = reader.createScanner(); Assert.assertTrue(scanner.atEnd()); scanner.close(); reader.close(); }
public LogReader(Configuration conf, DistributedFileSystemOps dfs, Path remoteAppLogFile) throws IOException { this.fsDataIStream = dfs.open(remoteAppLogFile); reader = new TFile.Reader(this.fsDataIStream, dfs.getFileStatus( remoteAppLogFile).getLen(), conf); this.scanner = reader.createScanner(); } private boolean atBeginning = true;
private void checkBlockIndex(int recordIndex, int blockIndexExpected) throws IOException { Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Scanner scanner = reader.createScanner(); scanner.seekTo(composeSortedKey(KEY, recordIndex).getBytes()); Assert.assertEquals(blockIndexExpected, scanner.currentLocation .getBlockIndex()); scanner.close(); reader.close(); }
@Test public void testNoDataEntry() throws IOException { if (skip) return; closeOutput(); Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Assert.assertTrue(reader.isSorted()); Scanner scanner = reader.createScanner(); Assert.assertTrue(scanner.atEnd()); scanner.close(); reader.close(); }
public TFileReadable(FileSystem fs, Path path, int osBufferSize, Configuration conf) throws IOException { this.fsdis = fs.open(path, osBufferSize); this.reader = new TFile.Reader(fsdis, fs.getFileStatus(path).getLen(), conf); this.scanner = reader.createScanner(); keyBuffer = new byte[32]; valueBuffer = new byte[32]; }
public LogReader(Configuration conf, Path remoteAppLogFile) throws IOException { FileContext fileContext = FileContext.getFileContext(remoteAppLogFile.toUri(), conf); this.fsDataIStream = fileContext.open(remoteAppLogFile); reader = new TFile.Reader(this.fsDataIStream, fileContext.getFileStatus( remoteAppLogFile).getLen(), conf); this.scanner = reader.createScanner(); }
private void checkBlockIndex(int count, int recordIndex, int blockIndexExpected) throws IOException { Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Scanner scanner = reader.createScanner(); scanner.seekTo(composeSortedKey(KEY, count, recordIndex).getBytes()); Assert.assertEquals(blockIndexExpected, scanner.currentLocation .getBlockIndex()); scanner.close(); reader.close(); }
public void testNoDataEntry() throws IOException { if (skip) return; closeOutput(); Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Assert.assertTrue(reader.isSorted()); Scanner scanner = reader.createScanner(); Assert.assertTrue(scanner.atEnd()); scanner.close(); reader.close(); }
public LogReader(Configuration conf, Path remoteAppLogFile) throws IOException { FileContext fileContext = FileContext.getFileContext(conf); this.fsDataIStream = fileContext.open(remoteAppLogFile); reader = new TFile.Reader(this.fsDataIStream, fileContext.getFileStatus( remoteAppLogFile).getLen(), conf); this.scanner = reader.createScanner(); }
public TFileReadable(FileSystem fs, Path path, int osBufferSize, Configuration conf) throws IOException { this.fsdis = fs.open(path, osBufferSize); this.reader = new TFile.Reader(fsdis, fs.getFileStatus(path).getLen(), conf); this.scanner = reader.createScanner(); keyBuffer = new byte[32]; valueBuffer = new byte[32]; }