private TxnRecord readTxnRecord(Path path) throws IOException { FSDataInputStream inputStream = null; try { inputStream = this.options.fs.open(path); BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); String line; if ((line = reader.readLine()) != null) { String[] fields = line.split(","); return new TxnRecord(Long.valueOf(fields[0]), fields[1], Long.valueOf(fields[2])); } } finally { if (inputStream != null) { inputStream.close(); } } return new TxnRecord(0, options.currentFile.toString(), 0); }
private void updateIndex(long txId) { LOG.debug("Starting index update."); final Path tmpPath = tmpFilePath(indexFilePath.toString()); try (FSDataOutputStream out = this.options.fs.create(tmpPath, true); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out))) { TxnRecord txnRecord = new TxnRecord(txId, options.currentFile.toString(), this.options.getCurrentOffset()); bw.write(txnRecord.toString()); bw.newLine(); bw.flush(); out.close(); /* In non error scenarios, for the Azure Data Lake Store File System (adl://), the output stream must be closed before the file associated with it is deleted. For ADLFS deleting the file also removes any handles to the file, hence out.close() will fail. */ /* * Delete the current index file and rename the tmp file to atomically * replace the index file. Orphan .tmp files are handled in getTxnRecord. */ options.fs.delete(this.indexFilePath, false); options.fs.rename(tmpPath, this.indexFilePath); lastSeenTxn = txnRecord; LOG.debug("updateIndex updated lastSeenTxn to [{}]", this.lastSeenTxn); } catch (IOException e) { LOG.warn("Begin commit failed due to IOException. Failing batch", e); throw new FailedException(e); } }
/** * Reads the last txn record from index file if it exists, if not from .tmp file if exists. * * @param indexFilePath the index file path * @return the txn record from the index file or a default initial record. * * @throws IOException */ private TxnRecord getTxnRecord(Path indexFilePath) throws IOException { Path tmpPath = tmpFilePath(indexFilePath.toString()); if (this.options.fs.exists(indexFilePath)) { return readTxnRecord(indexFilePath); } else if (this.options.fs.exists(tmpPath)) { return readTxnRecord(tmpPath); } return new TxnRecord(0, options.currentFile.toString(), 0); }
private void updateIndex(long txId) { LOG.debug("Starting index update."); final Path tmpPath = tmpFilePath(indexFilePath.toString()); try (FSDataOutputStream out = this.options.fs.create(tmpPath, true); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out))) { TxnRecord txnRecord = new TxnRecord(txId, options.currentFile.toString(), this.options.getCurrentOffset()); bw.write(txnRecord.toString()); bw.newLine(); bw.flush(); out.close(); /* In non error scenarios, for the Azure Data Lake Store File System (adl://), the output stream must be closed before the file associated with it is deleted. For ADLFS deleting the file also removes any handles to the file, hence out.close() will fail. */ /* * Delete the current index file and rename the tmp file to atomically * replace the index file. Orphan .tmp files are handled in getTxnRecord. */ options.fs.delete(this.indexFilePath, false); options.fs.rename(tmpPath, this.indexFilePath); lastSeenTxn = txnRecord; LOG.debug("updateIndex updated lastSeenTxn to [{}]", this.lastSeenTxn); } catch (IOException e) { LOG.warn("Begin commit failed due to IOException. Failing batch", e); throw new FailedException(e); } }
private TxnRecord readTxnRecord(Path path) throws IOException { FSDataInputStream inputStream = null; try { inputStream = this.options.fs.open(path); BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); String line; if ((line = reader.readLine()) != null) { String[] fields = line.split(","); return new TxnRecord(Long.valueOf(fields[0]), fields[1], Long.valueOf(fields[2])); } } finally { if (inputStream != null) { inputStream.close(); } } return new TxnRecord(0, options.currentFile.toString(), 0); }
/** * Reads the last txn record from index file if it exists, if not * from .tmp file if exists. * * @param indexFilePath the index file path * @return the txn record from the index file or a default initial record. * @throws IOException */ private TxnRecord getTxnRecord(Path indexFilePath) throws IOException { Path tmpPath = tmpFilePath(indexFilePath.toString()); if (this.options.fs.exists(indexFilePath)) { return readTxnRecord(indexFilePath); } else if (this.options.fs.exists(tmpPath)) { return readTxnRecord(tmpPath); } return new TxnRecord(0, options.currentFile.toString(), 0); }