public TableBuilder setMaterializedViewValidTxnList(ValidTxnList validTxnList) { mvValidTxnList = validTxnList.writeToString(); return this; }
private void findRecordsWithInvalidTransactionIds(VectorizedRowBatch batch, BitSet selectedBitSet) { if (batch.cols[OrcRecordUpdater.CURRENT_TRANSACTION].isRepeating) { // When we have repeating values, we can unset the whole bitset at once // if the repeating value is not a valid transaction. long currentTransactionIdForBatch = ((LongColumnVector) batch.cols[OrcRecordUpdater.CURRENT_TRANSACTION]).vector[0]; if (!validTxnList.isTxnValid(currentTransactionIdForBatch)) { selectedBitSet.clear(0, batch.size); } return; } long[] currentTransactionVector = ((LongColumnVector) batch.cols[OrcRecordUpdater.CURRENT_TRANSACTION]).vector; // Loop through the bits that are set to true and mark those rows as false, if their // current transactions are not valid. for (int setBitIndex = selectedBitSet.nextSetBit(0); setBitIndex >= 0; setBitIndex = selectedBitSet.nextSetBit(setBitIndex+1)) { if (!validTxnList.isTxnValid(currentTransactionVector[setBitIndex])) { selectedBitSet.clear(setBitIndex); } } }
private static long getCompactorTxnId(Configuration jobConf) { String snapshot = jobConf.get(ValidTxnList.VALID_TXNS_KEY); if(Strings.isNullOrEmpty(snapshot)) { throw new IllegalStateException(ValidTxnList.VALID_TXNS_KEY + " not found for writing to " + jobConf.get(FINAL_LOCATION)); } ValidTxnList validTxnList = new ValidReadTxnList(); validTxnList.readFromString(snapshot); //this is id of the current (compactor) txn return validTxnList.getHighWatermark(); } private void getWriter(Reporter reporter, ObjectInspector inspector,
private List<Long> getOpenTxns(ValidTxnList validTxnList) { long[] invalidTxns = validTxnList.getInvalidTransactions(); List<Long> openTxns = new ArrayList<>(); for (long invalidTxn : invalidTxns) { if (!validTxnList.isTxnAborted(invalidTxn)) { openTxns.add(invalidTxn); } } return openTxns; }
/** * checks {@code visibilityTxnId} to see if {@code child} is committed in current snapshot */ private static boolean isDirUsable(FileStatus child, long visibilityTxnId, List<FileStatus> aborted, ValidTxnList validTxnList) { if(validTxnList == null) { throw new IllegalArgumentException("No ValidTxnList for " + child.getPath()); } if(!validTxnList.isTxnValid(visibilityTxnId)) { boolean isAborted = validTxnList.isTxnAborted(visibilityTxnId); if(isAborted) { aborted.add(child);//so we can clean it up } LOG.debug("getChildState() ignoring(" + aborted + ") " + child); return false; } return true; } public static HdfsFileStatusWithId createOriginalObj(
long minOpenWriteId = Long.MAX_VALUE; BitSet abortedBits = new BitSet(); long txnHwm = validTxnList.getHighWatermark(); long txnId = rs.getLong(1); long writeId = rs.getLong(2); if (validTxnList.isTxnValid(txnId)) { if (validTxnList.isTxnAborted(txnId)) { abortedBits.set(invalidWriteIdList.size() - 1); } else {
final ValidTxnList txns = TxnUtils.createValidCompactTxnList(txnHandler.getOpenTxnsInfo()); LOG.debug("ValidCompactTxnList: " + txns.writeToString()); txnHandler.setCompactionHighestTxnId(ci, txns.getHighWatermark()); final StringBuilder jobName = new StringBuilder(name); jobName.append("-compactor-");
if (next.maxTransaction > current) { if (txnList.isTxnRangeValid(current+1, next.maxTransaction) != ValidTxnList.RangeResponse.NONE) { deltas.add(next); long[] exceptions = txnList.getInvalidTransactions(); String minOpenTxn = exceptions != null && exceptions.length > 0 ? Long.toString(exceptions[0]) : "x"; throw new IOException(ErrorMsg.ACID_NOT_ENOUGH_HISTORY.format( Long.toString(txnList.getHighWatermark()), minOpenTxn, bestBase.oldestBase.toString()));
Assert.assertEquals("1:" + Long.MAX_VALUE + "::", asString); validTxns = new ValidReadTxnList(asString); Assert.assertEquals(1, validTxns.getHighWatermark()); Assert.assertNotNull(validTxns.getInvalidTransactions()); Assert.assertEquals(0, validTxns.getInvalidTransactions().length); asString = validTxns.toString(); Assert.assertEquals("1:" + Long.MAX_VALUE + "::", asString); validTxns = new ValidReadTxnList(asString); Assert.assertEquals(1, validTxns.getHighWatermark()); Assert.assertNotNull(validTxns.getInvalidTransactions()); Assert.assertEquals(0, validTxns.getInvalidTransactions().length); Assert.assertEquals(10, validTxns.getHighWatermark()); Assert.assertNotNull(validTxns.getInvalidTransactions()); Assert.assertEquals(2, validTxns.getInvalidTransactions().length); boolean sawThree = false, sawFive = false; for (long tid : validTxns.getInvalidTransactions()) { if (tid == 3) sawThree = true; else if (tid == 5) sawFive = true;
(fn.startsWith(DELTA_PREFIX)) ? DELTA_PREFIX : DELETE_DELTA_PREFIX; ParsedDelta delta = parseDelta(child, deltaPrefix); if (txnList.isTxnRangeValid(delta.minTransaction, delta.maxTransaction) != ValidTxnList.RangeResponse.NONE) {
if (validTxnList.isTxnValid(txnId) || validTxnList.isTxnAborted(txnId)) {
if (txnList.isTxnRangeValid(delta.minTransaction, delta.maxTransaction) != ValidTxnList.RangeResponse.NONE) { if (next.maxTransaction > current) { if (txnList.isTxnRangeValid(current+1, next.maxTransaction) != ValidTxnList.RangeResponse.NONE) { deltas.add(next);
private ValidWriteIdList getTransactionContext(Configuration conf) throws Exception { ValidTxnList validTxnList = msClient.getValidTxns(); conf.set(ValidTxnList.VALID_TXNS_KEY, validTxnList.writeToString()); List<TableValidWriteIds> v = msClient.getValidWriteIds(Collections .singletonList(TableName.getDbTable(dbName, tblName)), validTxnList.writeToString()); return TxnCommonUtils.createValidReaderWriteIdList(v.get(0)); } private void checkNothingWritten(Path partitionPath) throws Exception {
((LongColumnVector) batch.cols[OrcRecordUpdater.CURRENT_TRANSACTION]).vector[currentTransactionIndex]; ++indexPtrInBatch; if (validTxnList.isTxnValid(currentTransaction)) { isValidNext = true; deleteRecordKey.set(originalTransaction, rowId);
private ValidWriteIdList getTransactionContext(Configuration conf) throws Exception { ValidTxnList validTxnList = msClient.getValidTxns(); conf.set(ValidTxnList.VALID_TXNS_KEY, validTxnList.writeToString()); List<TableValidWriteIds> v = msClient.getValidWriteIds(Collections .singletonList(TableName.getDbTable(dbName, tblName)), validTxnList.writeToString()); return TxnCommonUtils.createValidReaderWriteIdList(v.get(0)); } private void checkNothingWritten(Path partitionPath) throws Exception {
@Test public void testTxns() throws Exception { List<Long> tids = client.openTxns("me", 3).getTxn_ids(); Assert.assertEquals(1L, (long) tids.get(0)); Assert.assertEquals(2L, (long) tids.get(1)); Assert.assertEquals(3L, (long) tids.get(2)); client.rollbackTxn(1); client.commitTxn(2); ValidTxnList validTxns = client.getValidTxns(); Assert.assertFalse(validTxns.isTxnValid(1)); Assert.assertTrue(validTxns.isTxnValid(2)); Assert.assertFalse(validTxns.isTxnValid(3)); Assert.assertFalse(validTxns.isTxnValid(4)); }
StreamingAssert(IMetaStoreClient metaStoreClient, HiveConf conf, Table table, List<String> partition) throws Exception { this.metaStoreClient = metaStoreClient; this.table = table; this.partition = partition; validTxnList = metaStoreClient.getValidTxns(); conf.set(ValidTxnList.VALID_TXNS_KEY, validTxnList.writeToString()); List<TableValidWriteIds> v = metaStoreClient.getValidWriteIds(Collections .singletonList(TableName.getDbTable(table.getDbName(), table.getTableName())), validTxnList.writeToString()); writeIds = TxnCommonUtils.createValidReaderWriteIdList(v.get(0)); partitionLocation = getPartitionLocation(); dir = AcidUtils.getAcidState(partitionLocation, conf, writeIds); assertEquals(0, dir.getObsolete().size()); assertEquals(0, dir.getOriginalFiles().size()); currentDeltas = dir.getCurrentDirectories(); min = Long.MAX_VALUE; max = Long.MIN_VALUE; System.out.println("Files found: "); for (AcidUtils.ParsedDelta parsedDelta : currentDeltas) { System.out.println(parsedDelta.getPath().toString()); max = Math.max(parsedDelta.getMaxWriteId(), max); min = Math.min(parsedDelta.getMinWriteId(), min); } }
@Test public void testOpenTxnNotExcluded() throws Exception { List<Long> tids = client.openTxns("me", 3).getTxn_ids(); Assert.assertEquals(1L, (long) tids.get(0)); Assert.assertEquals(2L, (long) tids.get(1)); Assert.assertEquals(3L, (long) tids.get(2)); client.rollbackTxn(1); client.commitTxn(2); ValidTxnList validTxns = client.getValidTxns(3); Assert.assertFalse(validTxns.isTxnValid(1)); Assert.assertTrue(validTxns.isTxnValid(2)); Assert.assertTrue(validTxns.isTxnValid(3)); Assert.assertFalse(validTxns.isTxnValid(4)); }
conf.set(ValidTxnList.VALID_TXNS_KEY, validTxnList.writeToString()); rqst.setValidTxnList(validTxnList.writeToString()); final ValidWriteIdList tblValidWriteIds = TxnUtils.createValidCompactWriteIdList( txnHandler.getValidWriteIds(rqst).getTblValidWriteIds().get(0));
if (!validTxnList.isTxnValid( ((ReaderKey) recordIdentifier).getCurrentTransactionId())) { continue;