/** * Creates a reader that starts reading from 'offset' * * @param file the file to read * @param offset the offset string should be understandable by the reader type being used * @return * @throws IOException */ private FileReader createFileReader(Path file, String offset) throws IOException { if (readerType.equalsIgnoreCase(Configs.SEQ)) { return new SequenceFileReader(this.hdfs, file, conf, offset); } if (readerType.equalsIgnoreCase(Configs.TEXT)) { return new TextFileReader(this.hdfs, file, conf, offset); } try { Class<?> clsType = Class.forName(readerType); Constructor<?> constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class, String.class); return (FileReader) constructor.newInstance(this.hdfs, file, conf, offset); } catch (Exception e) { LOG.error(e.getMessage(), e); throw new RuntimeException("Unable to instantiate " + readerType, e); } }
@Override public List<Object> next() throws IOException, ParseException { readAttempts++; if (readAttempts == 3 || readAttempts == 4) { throw new IOException("mock test exception"); } else if (readAttempts > 5) { throw new ParseException("mock test exception", null); } return super.next(); } }
@Override public void close() { try { reader.close(); } catch (IOException e) { LOG.warn("Ignoring error when closing file " + getFilePath(), e); } }
TextFileReader reader = new TextFileReader(fs, file, null); FileOffset pos0 = tracker.getCommitPosition(); Assert.assertNull(pos0); TextFileReader.Offset currOffset = reader.getFileOffset(); Assert.assertNotNull(currOffset); Assert.assertEquals(0, currOffset.charOffset); Assert.assertNotNull(reader.next()); TextFileReader.Offset pos1 = reader.getFileOffset(); tracker.recordAckedOffset(pos1); Assert.assertNotNull(reader.next()); TextFileReader.Offset pos2 = reader.getFileOffset(); tracker.recordAckedOffset(pos2); Assert.assertNotNull(reader.next());//3 TextFileReader.Offset pos3 = reader.getFileOffset(); Assert.assertNotNull(reader.next());//4 TextFileReader.Offset pos4 = reader.getFileOffset(); Assert.assertNotNull(reader.next());//5 TextFileReader.Offset pos5 = reader.getFileOffset(); Assert.assertNotNull(reader.next());//6 TextFileReader.Offset pos6 = reader.getFileOffset(); Assert.assertNotNull(reader.next());//7 TextFileReader.Offset pos7 = reader.getFileOffset();
public class ViewController implements Initializable { @FXML private TextField urlTextField; @FXML private TextArea linesTextArea; private static final Logger LOG = Logger.getLogger(ViewController.class.getName()); private Future<List<String>> future; private ExecutorService executorService = Executors.newSingleThreadExecutor(); private TextFileReader reader = new TextFileReader(); public void initialize(URL url, ResourceBundle rb) { } @FXML @SuppressWarnings("NestedAssignment") public void showFileLines() throws InterruptedException, ExecutionException { future = executorService.submit(new Callable<List<String>>() { public List<String> call() throws Exception { return reader.read(new File(ViewController.this.urlTextField.getText())); } }); List<String> lines = future.get(); executorService.shutdownNow(); linesTextArea.clear(); for (String line : lines ) { linesTextArea.appendText(line + "\n"); } } }
public List<Object> next() throws IOException, ParseException { String line = readLineAndTrackOffset(reader); if (line != null) { return Collections.singletonList((Object) line); } return null; }
public List<Object> next() throws IOException, ParseException { String line = readLineAndTrackOffset(reader); if(line!=null) { return Collections.singletonList((Object) line); } return null; }
/** * Creates a reader that reads from beginning of file * * @param file file to read * @return * @throws IOException */ private FileReader createFileReader(Path file) throws IOException { if (readerType.equalsIgnoreCase(Configs.SEQ)) { return new SequenceFileReader(this.hdfs, file, conf); } if (readerType.equalsIgnoreCase(Configs.TEXT)) { return new TextFileReader(this.hdfs, file, conf); } try { Class<?> clsType = Class.forName(readerType); Constructor<?> constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class); return (FileReader) constructor.newInstance(this.hdfs, file, conf); } catch (Exception e) { LOG.error(e.getMessage(), e); throw new RuntimeException("Unable to instantiate " + readerType + " reader", e); } }
public List<Object> next() throws IOException, ParseException { List<Object> lineTuple = super.next(); if (lineTuple==null) return null; String jsonLine = (String) lineTuple.get(0); if ( jsonLine.trim().isEmpty() ) return next(); try { //1- convert Json to Map<> HashMap<String, Object> jsonMap = new ObjectMapper().readValue(jsonLine, HashMap.class); //2- make StreamlineEvent from map StreamlineEventImpl slEvent = StreamlineEventImpl.builder().putAll(jsonMap).dataSourceId("HdfsSpout").build(); //3- create tuple from StreamlineEvent return Collections.singletonList(slEvent); } catch (JsonProcessingException e) { throw new ParseException("Json parsing error at location : " + getFileOffset().toString(), e); } } }
@Override public void close() { try { reader.close(); } catch (IOException e) { LOG.warn("Ignoring error when closing file " + getFilePath(), e); } }
/** * Creates a reader that reads from beginning of file * * @param file file to read * @return * @throws IOException */ private FileReader createFileReader(Path file) throws IOException { if (readerType.equalsIgnoreCase(Configs.SEQ)) { return new SequenceFileReader(this.hdfs, file, conf); } if (readerType.equalsIgnoreCase(Configs.TEXT)) { return new TextFileReader(this.hdfs, file, conf); } try { Class<?> clsType = Class.forName(readerType); Constructor<?> constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class); return (FileReader) constructor.newInstance(this.hdfs, file, conf); } catch (Exception e) { LOG.error(e.getMessage(), e); throw new RuntimeException("Unable to instantiate " + readerType + " reader", e); } }
/** * Creates a reader that starts reading from 'offset' * * @param file the file to read * @param offset the offset string should be understandable by the reader type being used * @return * @throws IOException */ private FileReader createFileReader(Path file, String offset) throws IOException { if (readerType.equalsIgnoreCase(Configs.SEQ)) { return new SequenceFileReader(this.hdfs, file, conf, offset); } if (readerType.equalsIgnoreCase(Configs.TEXT)) { return new TextFileReader(this.hdfs, file, conf, offset); } try { Class<?> clsType = Class.forName(readerType); Constructor<?> constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class, String.class); return (FileReader) constructor.newInstance(this.hdfs, file, conf, offset); } catch (Exception e) { LOG.error(e.getMessage(), e); throw new RuntimeException("Unable to instantiate " + readerType, e); } }