private static String getFileProgress(FileReader reader) { return reader.getFilePath() + " " + reader.getFileOffset(); }
private void markFileAsDone(Path filePath) { try { Path newFile = renameCompletedFile(reader.getFilePath()); LOG.info("Completed processing {}. Spout Id = {}", newFile, spoutId); } catch (IOException e) { LOG.error("Unable to archive completed file" + filePath + " Spout ID " + spoutId, e); } closeReaderAndResetTrackers(); }
@Override public void ack(Object msgId) { LOG.trace("Ack received for msg {} on spout {}", msgId, spoutId); if (!ackEnabled) { return; } MessageId id = (MessageId) msgId; inflight.remove(id); ++acksSinceLastCommit; tracker.recordAckedOffset(id.offset); commitProgress(tracker.getCommitPosition()); if (fileReadCompletely && inflight.isEmpty()) { markFileAsDone(reader.getFilePath()); reader = null; } super.ack(msgId); }
fileReadCompletely = false; ++tupleCounter; MessageId msgId = new MessageId(tupleCounter, reader.getFilePath(), reader.getFileOffset()); emitData(tuple, msgId); markFileAsDone(reader.getFilePath()); LOG.error("Parsing error when processing at file location " + getFileProgress(reader) + ". Skipping remainder of file.", e); markFileAsBad(reader.getFilePath());
private static String getFileProgress(FileReader reader) { return reader.getFilePath() + " " + reader.getFileOffset(); }
private void markFileAsDone(Path filePath) { try { Path newFile = renameCompletedFile(reader.getFilePath()); LOG.info("Completed processing {}. Spout Id = {}", newFile, spoutId); } catch (IOException e) { LOG.error("Unable to archive completed file" + filePath + " Spout ID " + spoutId, e); } closeReaderAndResetTrackers(); }
@Override public void ack(Object msgId) { LOG.trace("Ack received for msg {} on spout {}", msgId, spoutId); if (!ackEnabled) { return; } MessageId id = (MessageId) msgId; inflight.remove(id); ++acksSinceLastCommit; tracker.recordAckedOffset(id.offset); commitProgress(tracker.getCommitPosition()); if (fileReadCompletely && inflight.isEmpty()) { markFileAsDone(reader.getFilePath()); reader = null; } super.ack(msgId); }
fileReadCompletely = false; ++tupleCounter; MessageId msgId = new MessageId(tupleCounter, reader.getFilePath(), reader.getFileOffset()); emitData(tuple, msgId); fileReadCompletely = true; if (!ackEnabled) { markFileAsDone(reader.getFilePath()); LOG.error("Parsing error when processing at file location " + getFileProgress(reader) + ". Skipping remainder of file.", e); markFileAsBad(reader.getFilePath());