protected void rotateOutputFile(boolean doRotateAction) throws IOException { LOG.info("Rotating output file..."); long start = System.currentTimeMillis(); closeOutputFile(); this.rotation++; Path newFile = createOutputFile(); if (doRotateAction) { LOG.info("Performing {} file rotation actions.", this.rotationActions.size()); for (RotationAction action : this.rotationActions) { action.execute(this.fs, this.currentFile); } } this.currentFile = newFile; long time = System.currentTimeMillis() - start; LOG.info("File rotation took {} ms.", time); }
doPrepare(conf, partitionIndex, numPartitions); this.currentFile = createOutputFile();
public void updateState(List<TridentTuple> tuples, TridentCollector tridentCollector) { try { this.options.execute(tuples); } catch (IOException e) { LOG.warn("Failing batch due to IOException.", e); throw new FailedException(e); } }
doPrepare(conf, partitionIndex, numPartitions); this.currentFile = createOutputFile();
doPrepare(conf, partitionIndex, numPartitions); this.currentFile = createOutputFile();
private void updateIndex(long txId) { LOG.debug("Starting index update."); final Path tmpPath = tmpFilePath(indexFilePath.toString()); try (FSDataOutputStream out = this.options.fs.create(tmpPath, true); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out))) { TxnRecord txnRecord = new TxnRecord(txId, options.currentFile.toString(), this.options.getCurrentOffset()); bw.write(txnRecord.toString()); bw.newLine(); bw.flush(); out.close(); /* In non error scenarios, for the Azure Data Lake Store File System (adl://), the output stream must be closed before the file associated with it is deleted. For ADLFS deleting the file also removes any handles to the file, hence out.close() will fail. */ /* * Delete the current index file and rename the tmp file to atomically * replace the index file. Orphan .tmp files are handled in getTxnRecord. */ options.fs.delete(this.indexFilePath, false); options.fs.rename(tmpPath, this.indexFilePath); lastSeenTxn = txnRecord; LOG.debug("updateIndex updated lastSeenTxn to [{}]", this.lastSeenTxn); } catch (IOException e) { LOG.warn("Begin commit failed due to IOException. Failing batch", e); throw new FailedException(e); } }
/** * Recovers nBytes from srcFile to the new file created * by calling rotateOutputFile and then deletes the srcFile. */ private void recover(String srcFile, long nBytes) { try { Path srcPath = new Path(srcFile); rotateOutputFile(false); this.rotationPolicy.reset(); if (nBytes > 0) { doRecover(srcPath, nBytes); LOG.info("Recovered {} bytes from {} to {}", nBytes, srcFile, currentFile); } else { LOG.info("Nothing to recover from {}", srcFile); } fs.delete(srcPath, false); LOG.info("Deleted file {} that had partial commits.", srcFile); } catch (Exception e) { LOG.warn("Recovery failed.", e); throw new RuntimeException(e); } }
protected void rotateOutputFile() throws IOException { LOG.info("Rotating output file..."); long start = System.currentTimeMillis(); synchronized (this.writeLock) { closeOutputFile(); this.rotation++; Path newFile = createOutputFile(); LOG.info("Performing {} file rotation actions.", this.rotationActions.size()); for (RotationAction action : this.rotationActions) { action.execute(this.fs, this.currentFile); } this.currentFile = newFile; } long time = System.currentTimeMillis() - start; LOG.info("File rotation took {} ms.", time); }
protected void rotateOutputFile(boolean doRotateAction) throws IOException { LOG.info("Rotating output file..."); long start = System.currentTimeMillis(); closeOutputFile(); this.rotation++; Path newFile = createOutputFile(); if (doRotateAction) { LOG.info("Performing {} file rotation actions.", this.rotationActions.size()); for (RotationAction action : this.rotationActions) { action.execute(this.fs, this.currentFile); } } this.currentFile = newFile; long time = System.currentTimeMillis() - start; LOG.info("File rotation took {} ms.", time); }
void prepare(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions){ this.options.prepare(conf, partitionIndex, numPartitions); }
protected void rotateOutputFile() throws IOException { rotateOutputFile(true); }
public void updateState(List<TridentTuple> tuples, TridentCollector tridentCollector){ try{ this.options.execute(tuples); } catch (IOException e){ LOG.warn("Failing batch due to IOException.", e); throw new FailedException(e); } } }
void prepare(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { this.options.prepare(conf, partitionIndex, numPartitions); initLastTxn(conf, partitionIndex); }
@Override public void run() { try { rotateOutputFile(); } catch(IOException e){ LOG.warn("IOException during scheduled file rotation.", e); } } };
public void updateState(List<TridentTuple> tuples, TridentCollector tridentCollector) { try { this.options.execute(tuples); } catch (IOException e) { LOG.warn("Failing batch due to IOException.", e); throw new FailedException(e); } }
/** * for unit tests */ void close() throws IOException { this.options.closeOutputFile(); } }
@Override public void commit(Long txId) { try { options.doCommit(txId); } catch (IOException e) { LOG.warn("Commit failed due to IOException. Failing the batch.", e); throw new FailedException(e); } }
private void updateIndex(long txId) { LOG.debug("Starting index update."); final Path tmpPath = tmpFilePath(indexFilePath.toString()); try (FSDataOutputStream out = this.options.fs.create(tmpPath, true); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out))) { TxnRecord txnRecord = new TxnRecord(txId, options.currentFile.toString(), this.options.getCurrentOffset()); bw.write(txnRecord.toString()); bw.newLine(); bw.flush(); out.close(); /* In non error scenarios, for the Azure Data Lake Store File System (adl://), the output stream must be closed before the file associated with it is deleted. For ADLFS deleting the file also removes any handles to the file, hence out.close() will fail. */ /* * Delete the current index file and rename the tmp file to atomically * replace the index file. Orphan .tmp files are handled in getTxnRecord. */ options.fs.delete(this.indexFilePath, false); options.fs.rename(tmpPath, this.indexFilePath); lastSeenTxn = txnRecord; LOG.debug("updateIndex updated lastSeenTxn to [{}]", this.lastSeenTxn); } catch (IOException e) { LOG.warn("Begin commit failed due to IOException. Failing batch", e); throw new FailedException(e); } }
/** * Recovers nBytes from srcFile to the new file created by calling rotateOutputFile and then deletes the srcFile. */ private void recover(String srcFile, long nBytes) { try { Path srcPath = new Path(srcFile); rotateOutputFile(false); this.rotationPolicy.reset(); if (nBytes > 0) { doRecover(srcPath, nBytes); LOG.info("Recovered {} bytes from {} to {}", nBytes, srcFile, currentFile); } else { LOG.info("Nothing to recover from {}", srcFile); } fs.delete(srcPath, false); LOG.info("Deleted file {} that had partial commits.", srcFile); } catch (Exception e) { LOG.warn("Recovery failed.", e); throw new RuntimeException(e); } }
void prepare(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { this.options.prepare(conf, partitionIndex, numPartitions); initLastTxn(conf, partitionIndex); }