/** * Creates a reader that reads from beginning of file * * @param file file to read * @return * @throws IOException */ private FileReader createFileReader(Path file) throws IOException { if (readerType.equalsIgnoreCase(Configs.SEQ)) { return new SequenceFileReader(this.hdfs, file, conf); } if (readerType.equalsIgnoreCase(Configs.TEXT)) { return new TextFileReader(this.hdfs, file, conf); } try { Class<?> clsType = Class.forName(readerType); Constructor<?> constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class); return (FileReader) constructor.newInstance(this.hdfs, file, conf); } catch (Exception e) { LOG.error(e.getMessage(), e); throw new RuntimeException("Unable to instantiate " + readerType + " reader", e); } }
@Override public void close() { try { reader.close(); } catch (IOException e) { LOG.warn("Ignoring error when closing file " + getFilePath(), e); } }
public SequenceFileReader(FileSystem fs, Path file, Map<String, Object> conf, String offset) throws IOException { super(fs, file); int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf.get(BUFFER_SIZE).toString()); this.offset = new SequenceFileReader.Offset(offset); this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), SequenceFile.Reader.bufferSize(bufferSize)); this.key = (Key) ReflectionUtils.newInstance(reader.getKeyClass(), fs.getConf()); this.value = (Value) ReflectionUtils.newInstance(reader.getValueClass(), fs.getConf()); skipToOffset(this.reader, this.offset, this.key); }
/** * Creates a reader that starts reading from 'offset' * * @param file the file to read * @param offset the offset string should be understandable by the reader type being used * @return * @throws IOException */ private FileReader createFileReader(Path file, String offset) throws IOException { if (readerType.equalsIgnoreCase(Configs.SEQ)) { return new SequenceFileReader(this.hdfs, file, conf, offset); } if (readerType.equalsIgnoreCase(Configs.TEXT)) { return new TextFileReader(this.hdfs, file, conf, offset); } try { Class<?> clsType = Class.forName(readerType); Constructor<?> constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class, String.class); return (FileReader) constructor.newInstance(this.hdfs, file, conf, offset); } catch (Exception e) { LOG.error(e.getMessage(), e); throw new RuntimeException("Unable to instantiate " + readerType, e); } }
public SequenceFileReader(FileSystem fs, Path file, Map conf, String offset) throws IOException { super(fs, file); int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt( conf.get(BUFFER_SIZE).toString() ); this.offset = new SequenceFileReader.Offset(offset); this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), SequenceFile.Reader.bufferSize(bufferSize) ); this.key = (Key) ReflectionUtils.newInstance(reader.getKeyClass(), fs.getConf() ); this.value = (Value) ReflectionUtils.newInstance(reader.getValueClass(), fs.getConf() ); skipToOffset(this.reader, this.offset, this.key); }
@Override public void close() { try { reader.close(); } catch (IOException e) { LOG.warn("Ignoring error when closing file " + getFilePath(), e); } }
/** * Creates a reader that reads from beginning of file * * @param file file to read * @return * @throws IOException */ private FileReader createFileReader(Path file) throws IOException { if (readerType.equalsIgnoreCase(Configs.SEQ)) { return new SequenceFileReader(this.hdfs, file, conf); } if (readerType.equalsIgnoreCase(Configs.TEXT)) { return new TextFileReader(this.hdfs, file, conf); } try { Class<?> clsType = Class.forName(readerType); Constructor<?> constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class); return (FileReader) constructor.newInstance(this.hdfs, file, conf); } catch (Exception e) { LOG.error(e.getMessage(), e); throw new RuntimeException("Unable to instantiate " + readerType + " reader", e); } }
/** * Creates a reader that starts reading from 'offset' * * @param file the file to read * @param offset the offset string should be understandable by the reader type being used * @return * @throws IOException */ private FileReader createFileReader(Path file, String offset) throws IOException { if (readerType.equalsIgnoreCase(Configs.SEQ)) { return new SequenceFileReader(this.hdfs, file, conf, offset); } if (readerType.equalsIgnoreCase(Configs.TEXT)) { return new TextFileReader(this.hdfs, file, conf, offset); } try { Class<?> clsType = Class.forName(readerType); Constructor<?> constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class, String.class); return (FileReader) constructor.newInstance(this.hdfs, file, conf, offset); } catch (Exception e) { LOG.error(e.getMessage(), e); throw new RuntimeException("Unable to instantiate " + readerType, e); } }