private void incrementBytesReadCounter() { try { this.counters.inc(CounterNames.FileBytesRead, this.fsHelper.getFileSize(this.currentFile)); } catch (FileBasedHelperException e) { LOG.info("Unable to get file size. Will skip increment to bytes counter " + e.getMessage()); LOG.debug(e.getMessage(), e); } catch (UnsupportedOperationException e) { LOG.info("Unable to get file size. Will skip increment to bytes counter " + e.getMessage()); LOG.debug(e.getMessage(), e); } } }
@Override public List<String> getcurrentFsSnapshot(State state) { List<String> results = Lists.newArrayList(); String path = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_DATA_DIRECTORY); try { LOGGER.info("Running ls command with input " + path); results = this.fsHelper.ls(path); } catch (FileBasedHelperException e) { LOGGER.error("Not able to run ls command due to " + e.getMessage() + " will not pull any files", e); } return results; } }
@SuppressWarnings("unchecked") public Iterator<D> downloadFile(String file) throws IOException { log.info("Beginning to download gzip compressed file: " + file); try { InputStream inputStream = this.fileBasedExtractor.getCloser().register(this.fileBasedExtractor.getFsHelper().getFileStream(file)); Iterator<D> fileItr = (Iterator<D>) IOUtils.lineIterator(new GZIPInputStream(inputStream), ConfigurationKeys.DEFAULT_CHARSET_ENCODING); if (this.fileBasedExtractor.isShouldSkipFirstRecord() && fileItr.hasNext()) { fileItr.next(); } return fileItr; } catch (FileBasedHelperException e) { throw new IOException("Exception while downloading file " + file + " with message " + e.getMessage(), e); } } }
@SuppressWarnings("unchecked") public Iterator<D> downloadFile(String file) throws IOException { log.info("Beginning to download file: " + file); try { InputStream inputStream = this.fileBasedExtractor.getCloser().register(this.fileBasedExtractor.getFsHelper().getFileStream(file)); Iterator<D> fileItr = (Iterator<D>) IOUtils.lineIterator(inputStream, ConfigurationKeys.DEFAULT_CHARSET_ENCODING); if (this.fileBasedExtractor.isShouldSkipFirstRecord() && fileItr.hasNext()) { fileItr.next(); } return fileItr; } catch (FileBasedHelperException e) { throw new IOException("Exception while downloading file " + file + " with message " + e.getMessage(), e); } } }
private void incrementBytesReadCounter() { try { this.counters.inc(CounterNames.FileBytesRead, this.fsHelper.getFileSize(this.currentFile)); } catch (FileBasedHelperException e) { LOG.info("Unable to get file size. Will skip increment to bytes counter " + e.getMessage()); LOG.debug(e.getMessage(), e); } catch (UnsupportedOperationException e) { LOG.info("Unable to get file size. Will skip increment to bytes counter " + e.getMessage()); LOG.debug(e.getMessage(), e); } } }
@Override public List<String> getcurrentFsSnapshot(State state) { List<String> results = Lists.newArrayList(); String path = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_DATA_DIRECTORY); try { LOGGER.info("Running ls command with input " + path); results = this.fsHelper.ls(path); } catch (FileBasedHelperException e) { LOGGER.error("Not able to run ls command due to " + e.getMessage() + " will not pull any files", e); } return results; } }
@SuppressWarnings("unchecked") public Iterator<D> downloadFile(String file) throws IOException { log.info("Beginning to download gzip compressed file: " + file); try { InputStream inputStream = this.fileBasedExtractor.getCloser().register(this.fileBasedExtractor.getFsHelper().getFileStream(file)); Iterator<D> fileItr = (Iterator<D>) IOUtils.lineIterator(new GZIPInputStream(inputStream), ConfigurationKeys.DEFAULT_CHARSET_ENCODING); if (this.fileBasedExtractor.isShouldSkipFirstRecord() && fileItr.hasNext()) { fileItr.next(); } return fileItr; } catch (FileBasedHelperException e) { throw new IOException("Exception while downloading file " + file + " with message " + e.getMessage(), e); } } }
@SuppressWarnings("unchecked") public Iterator<D> downloadFile(String file) throws IOException { log.info("Beginning to download file: " + file); try { InputStream inputStream = this.fileBasedExtractor.getCloser().register(this.fileBasedExtractor.getFsHelper().getFileStream(file)); Iterator<D> fileItr = (Iterator<D>) IOUtils.lineIterator(inputStream, ConfigurationKeys.DEFAULT_CHARSET_ENCODING); if (this.fileBasedExtractor.isShouldSkipFirstRecord() && fileItr.hasNext()) { fileItr.next(); } return fileItr; } catch (FileBasedHelperException e) { throw new IOException("Exception while downloading file " + file + " with message " + e.getMessage(), e); } } }