/** * Recovers nBytes from srcFile to the new file created by calling rotateOutputFile and then deletes the srcFile. */ private void recover(String srcFile, long nBytes) { try { Path srcPath = new Path(srcFile); rotateOutputFile(false); this.rotationPolicy.reset(); if (nBytes > 0) { doRecover(srcPath, nBytes); LOG.info("Recovered {} bytes from {} to {}", nBytes, srcFile, currentFile); } else { LOG.info("Nothing to recover from {}", srcFile); } fs.delete(srcPath, false); LOG.info("Deleted file {} that had partial commits.", srcFile); } catch (Exception e) { LOG.warn("Recovery failed.", e); throw new RuntimeException(e); } }
@Override public void doCommit(Long txId) throws IOException { if (this.rotationPolicy.mark(this.offset)) { rotateOutputFile(); this.offset = 0; this.rotationPolicy.reset(); } else { if (this.out instanceof HdfsDataOutputStream) { ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); } else { this.out.hsync(); } } }
@Override public void doCommit(Long txId) throws IOException { if (this.rotationPolicy.mark(this.writer.getLength())) { rotateOutputFile(); this.rotationPolicy.reset(); } else { this.writer.hsync(); } }
/** * Recovers nBytes from srcFile to the new file created * by calling rotateOutputFile and then deletes the srcFile. */ private void recover(String srcFile, long nBytes) { try { Path srcPath = new Path(srcFile); rotateOutputFile(false); this.rotationPolicy.reset(); if (nBytes > 0) { doRecover(srcPath, nBytes); LOG.info("Recovered {} bytes from {} to {}", nBytes, srcFile, currentFile); } else { LOG.info("Nothing to recover from {}", srcFile); } fs.delete(srcPath, false); LOG.info("Deleted file {} that had partial commits.", srcFile); } catch (Exception e) { LOG.warn("Recovery failed.", e); throw new RuntimeException(e); } }
@Override public void doCommit(Long txId) throws IOException { if (this.rotationPolicy.mark(this.offset)) { rotateOutputFile(); this.offset = 0; this.rotationPolicy.reset(); } else { if (this.out instanceof HdfsDataOutputStream) { ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); } else { this.out.hsync(); } } }
@Override public void doCommit(Long txId) throws IOException { if (this.rotationPolicy.mark(this.writer.getLength())) { rotateOutputFile(); this.rotationPolicy.reset(); } else { this.writer.hsync(); } }
@Override public void execute(List<TridentTuple> tuples) throws IOException { boolean rotated = false; synchronized (this.writeLock) { for (TridentTuple tuple : tuples) { byte[] bytes = this.format.format(tuple); out.write(bytes); this.offset += bytes.length; if (this.rotationPolicy.mark(tuple, this.offset)) { rotateOutputFile(); this.offset = 0; this.rotationPolicy.reset(); rotated = true; } } if (!rotated) { if (this.out instanceof HdfsDataOutputStream) { ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); } else { this.out.hsync(); } } } } }
@Override public void execute(List<TridentTuple> tuples) throws IOException { long offset; for(TridentTuple tuple : tuples) { synchronized (this.writeLock) { this.writer.append(this.format.key(tuple), this.format.value(tuple)); offset = this.writer.getLength(); } if (this.rotationPolicy.mark(tuple, offset)) { rotateOutputFile(); this.rotationPolicy.reset(); } } }