final CompactionInfo ci = txnHandler.findNextToCompact(name); LOG.info("Unable to find table " + ci.getFullTableName() + ", assuming it was dropped and moving on."); txnHandler.markCleaned(ci); continue; txnHandler.markCleaned(ci); continue; LOG.info("Unable to find partition " + ci.getFullPartitionName() + ", assuming it was dropped and moving on."); txnHandler.markCleaned(ci); continue; txnHandler.markCleaned(ci); continue; txnHandler.markCleaned(ci); continue; TxnUtils.createValidCompactTxnList(txnHandler.getOpenTxnsInfo()); LOG.debug("ValidCompactTxnList: " + txns.writeToString()); txnHandler.setCompactionHighestTxnId(ci, txns.getHighWatermark()); final StringBuilder jobName = new StringBuilder(name); jobName.append("-compactor-"); if (ci.runAs == null) { runAs = findUserToRunAs(sd.getLocation(), t);
@Test public void testMarkCleaned() throws Exception { CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); assertEquals(0, txnHandler.findReadyToClean().size()); CompactionInfo ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); assertEquals(0, txnHandler.findReadyToClean().size()); txnHandler.markCompacted(ci); assertNull(txnHandler.findNextToCompact("fred")); List<CompactionInfo> toClean = txnHandler.findReadyToClean(); assertEquals(1, toClean.size()); assertNull(txnHandler.findNextToCompact("fred")); txnHandler.markCleaned(ci); assertNull(txnHandler.findNextToCompact("fred")); assertEquals(0, txnHandler.findReadyToClean().size()); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); assertEquals(1, rsp.getCompactsSize()); assertTrue(TxnHandler.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState())); }
handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.Cleaner.name()); startedAt = System.currentTimeMillis(); List<CompactionInfo> toClean = txnHandler.findReadyToClean(); ShowLocksResponse locksResponse = txnHandler.showLocks(new ShowLocksRequest());
@Override public void run() { TxnStore.MutexAPI.LockHandle handle = null; try { handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.WriteSetCleaner.name()); long startTime = System.currentTimeMillis(); txnHandler.performWriteSetGC(); LOG.debug("cleaner ran for " + (System.currentTimeMillis() - startTime)/1000 + "seconds."); } catch(Throwable t) { LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); } finally { if(handle != null) { handle.releaseLocks(); } } } }
@Override public void run() { TxnStore.MutexAPI.LockHandle handle = null; try { handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.HouseKeeper.name()); long startTime = System.currentTimeMillis(); txnHandler.performTimeOuts(); LOG.debug("timeout reaper ran for " + (System.currentTimeMillis() - startTime)/1000 + "seconds."); } catch(Throwable t) { LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); } finally { if(handle != null) { handle.releaseLocks(); } } } }
@Override public void run() { TxnStore.MutexAPI.LockHandle handle = null; try { handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.CompactionHistory.name()); long startTime = System.currentTimeMillis(); txnHandler.purgeCompactionHistory(); LOG.debug("History reaper reaper ran for " + (System.currentTimeMillis() - startTime)/1000 + "seconds."); } catch(Throwable t) { LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); } finally { if(handle != null) { handle.releaseLocks(); } } } }
@Test public void testCompactMajorWithPartition() throws Exception { CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MAJOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); assertEquals(1, compacts.size()); ShowCompactResponseElement c = compacts.get(0); assertEquals("foo", c.getDbname()); assertEquals("bar", c.getTablename()); assertEquals("ds=today", c.getPartitionname()); assertEquals(CompactionType.MAJOR, c.getType()); assertEquals("initiated", c.getState()); assertEquals(0L, c.getStart()); }
handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.Initiator.name()); startedAt = System.currentTimeMillis(); ShowCompactResponse currentCompactions = txnHandler.showCompact(new ShowCompactRequest()); Set<CompactionInfo> potentials = txnHandler.findPotentialCompactions(abortedThreshold); LOG.debug("Found " + potentials.size() + " potential compactions, " + "checking to see if we should compact any of them"); if(txnHandler.checkFailedCompactions(ci)) { LOG.warn("Will not initiate compaction for " + ci.getFullPartitionName() + " since last " + HiveConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD + " attempts to compact it failed."); txnHandler.markFailed(ci); continue; .createValidReadTxnList(txnHandler.getOpenTxns(), 0); conf.set(ValidTxnList.VALID_TXNS_KEY, validTxnList.writeToString()); rqst.setValidTxnList(validTxnList.writeToString()); final ValidWriteIdList tblValidWriteIds = TxnUtils.createValidCompactWriteIdList( txnHandler.getValidWriteIds(rqst).getTblValidWriteIds().get(0)); ci + ". Marking failed to avoid repeated failures, " + "" + StringUtils.stringifyException(t)); txnHandler.markFailed(ci); txnHandler.cleanEmptyAbortedTxns(); txnHandler.cleanTxnToWriteIdTable(); } catch (Throwable t) {
LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); assertTrue(res.getState() == LockState.ACQUIRED); txnHandler.abortTxn(new AbortTxnRequest(txnid)); req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); res = txnHandler.lock(req); assertTrue(res.getState() == LockState.ACQUIRED); txnHandler.abortTxn(new AbortTxnRequest(txnid)); req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); res = txnHandler.lock(req); assertTrue(res.getState() == LockState.ACQUIRED); req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); res = txnHandler.lock(req); assertTrue(res.getState() == LockState.ACQUIRED); txnHandler.abortTxn(new AbortTxnRequest(txnid)); txnHandler.compact(rqst); assertEquals(0, txnHandler.findReadyToClean().size()); ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); txnHandler.markCompacted(ci);
handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.Initiator.name()); startedAt = System.currentTimeMillis(); ShowCompactResponse currentCompactions = txnHandler.showCompact(new ShowCompactRequest()); ValidTxnList txns = TxnUtils.createValidCompactTxnList(txnHandler.getOpenTxnsInfo()); Set<CompactionInfo> potentials = txnHandler.findPotentialCompactions(abortedThreshold); LOG.debug("Found " + potentials.size() + " potential compactions, " + "checking to see if we should compact any of them"); continue; if(txnHandler.checkFailedCompactions(ci)) { LOG.warn("Will not initiate compaction for " + ci.getFullPartitionName() + " since last " + HiveConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD + " attempts to compact it failed."); txnHandler.markFailed(ci); continue; ci + ". Marking failed to avoid repeated failures, " + "" + StringUtils.stringifyException(t)); txnHandler.markFailed(ci); txnHandler.cleanEmptyAbortedTxns(); } catch (Throwable t) { LOG.error("Initiator loop caught unexpected exception this time through the loop: " +
@Test public void noCompactTableDynamicPartitioning() throws Exception { Table t = newTable("default", "nctdp", true); Partition p = newPartition(t, "today"); addBaseFile(t, p, 20L, 20); addDeltaFile(t, p, 21L, 22L, 2); addDeltaFile(t, p, 23L, 24L, 2); burnThroughTransactions("default", "nctdp", 23); long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("nctdp"); comp.setOperationType(DataOperationType.UPDATE); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); long writeid = allocateWriteId("default", "nctdp", txnid); Assert.assertEquals(24, writeid); txnHandler.commitTxn(new CommitTxnRequest(txnid)); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(0, compacts.size()); }
@Test public void testAbortTxn() throws Exception { OpenTxnsResponse openedTxns = txnHandler.openTxns(new OpenTxnRequest(3, "me", "localhost")); List<Long> txnList = openedTxns.getTxn_ids(); long first = txnList.get(0); long second = txnList.get(1); assertEquals(2L, second); txnHandler.abortTxn(new AbortTxnRequest(1)); List<String> parts = new ArrayList<String>(); parts.add("p=1"); AllocateTableWriteIdsResponse writeIds = txnHandler.allocateTableWriteIds(rqst); long writeId = writeIds.getTxnToWriteIds().get(0).getWriteId(); assertEquals(3, writeIds.getTxnToWriteIds().get(0).getTxnId()); txnHandler.addDynamicPartitions(adp); GetOpenTxnsInfoResponse txnsInfo = txnHandler.getOpenTxnsInfo(); assertEquals(3, txnsInfo.getTxn_high_water_mark()); assertEquals(3, txnsInfo.getOpen_txns().size()); GetOpenTxnsResponse txns = txnHandler.getOpenTxns(); assertEquals(3, txns.getTxn_high_water_mark()); assertEquals(3, txns.getOpen_txns().size()); txnHandler.commitTxn(new CommitTxnRequest(2)); txnHandler.abortTxn(new AbortTxnRequest(1)); boolean gotException = false; try {
CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); assertEquals(0, txnHandler.findReadyToClean().size()); CompactionInfo ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); assertEquals(0, txnHandler.findReadyToClean().size()); txnHandler.markFailed(ci); assertNull(txnHandler.findNextToCompact("fred")); boolean failedCheck = txnHandler.checkFailedCompactions(ci); assertFalse(failedCheck); try { txnHandler.markFailed(ci); fail("The first call to markFailed() must have failed as this call did " + "not throw the expected exception"); assertFalse(txnHandler.checkFailedCompactions(ci)); assertTrue(txnHandler.checkFailedCompactions(ci)); addFailedCompaction("foo", "bar", CompactionType.MINOR, "ds=today"); txnHandler.purgeCompactionHistory();
@Test public void testValidTxnsNoneOpen() throws Exception { txnHandler.openTxns(new OpenTxnRequest(2, "me", "localhost")); txnHandler.commitTxn(new CommitTxnRequest(1)); txnHandler.commitTxn(new CommitTxnRequest(2)); GetOpenTxnsInfoResponse txnsInfo = txnHandler.getOpenTxnsInfo(); assertEquals(2L, txnsInfo.getTxn_high_water_mark()); assertEquals(0, txnsInfo.getOpen_txns().size()); GetOpenTxnsResponse txns = txnHandler.getOpenTxns(); assertEquals(2L, txns.getTxn_high_water_mark()); assertEquals(0, txns.getOpen_txns().size()); }
@Test public void noCompactWhenNoCompactSet() throws Exception { Map<String, String> parameters = new HashMap<String, String>(1); parameters.put("NO_AUTO_COMPACTION", "true"); Table t = newTable("default", "ncwncs", false, parameters); HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); for (int i = 0; i < 11; i++) { long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("ncwncs"); comp.setOperationType(DataOperationType.UPDATE); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.abortTxn(new AbortTxnRequest(txnid)); } startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); }
CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); txnHandler.compact(rqst); CompactionInfo ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); assertEquals("foo", ci.dbname); assertEquals(CompactionType.MINOR, ci.type); ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); assertEquals("foo", ci.dbname); assertEquals(CompactionType.MINOR, ci.type); assertNull(txnHandler.findNextToCompact("fred")); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); assertEquals(2, compacts.size());
@Test public void testRevokeTimedOutWorkers() throws Exception { CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR); txnHandler.compact(rqst); rqst = new CompactionRequest("foo", "baz", CompactionType.MINOR); txnHandler.compact(rqst); assertNotNull(txnHandler.findNextToCompact("fred-193892")); Thread.sleep(200); assertNotNull(txnHandler.findNextToCompact("fred-193892")); txnHandler.revokeTimedoutWorkers(100); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); assertEquals(2, compacts.size()); boolean sawWorking = false, sawInitiated = false; for (ShowCompactResponseElement c : compacts) { if (c.getState().equals("working")) sawWorking = true; else if (c.getState().equals("initiated")) sawInitiated = true; else fail("Unexpected state"); } assertTrue(sawWorking); assertTrue(sawInitiated); }
@Test public void testRevokeFromLocalWorkers() throws Exception { CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR); txnHandler.compact(rqst); rqst = new CompactionRequest("foo", "baz", CompactionType.MINOR); txnHandler.compact(rqst); rqst = new CompactionRequest("foo", "bazzoo", CompactionType.MINOR); txnHandler.compact(rqst); assertNotNull(txnHandler.findNextToCompact("fred-193892")); assertNotNull(txnHandler.findNextToCompact("bob-193892")); assertNotNull(txnHandler.findNextToCompact("fred-193893")); txnHandler.revokeFromLocalWorkers("fred"); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); assertEquals(3, compacts.size()); boolean sawWorkingBob = false; int initiatedCount = 0; for (ShowCompactResponseElement c : compacts) { if (c.getState().equals("working")) { assertEquals("bob-193892", c.getWorkerid()); sawWorkingBob = true; } else if (c.getState().equals("initiated")) { initiatedCount++; } else { fail("Unexpected state"); } } assertTrue(sawWorkingBob); assertEquals(2, initiatedCount); }
@Test public void testRecoverManyTimeouts() throws Exception { long timeout = txnHandler.setTimeout(1); try { txnHandler.openTxns(new OpenTxnRequest(503, "me", "localhost")); Thread.sleep(10); txnHandler.performTimeOuts(); GetOpenTxnsInfoResponse rsp = txnHandler.getOpenTxnsInfo(); int numAborted = 0; for (TxnInfo txnInfo : rsp.getOpen_txns()) { assertEquals(TxnState.ABORTED, txnInfo.getState()); numAborted++; } assertEquals(503, numAborted); } finally { txnHandler.setTimeout(timeout); } }
LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.abortTxn(new AbortTxnRequest(txnid)); txnHandler.compact(rqst); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(1, compacts.size()); rsp = txnHandler.showCompact(new ShowCompactRequest()); compacts = rsp.getCompacts(); Assert.assertEquals(1, compacts.size());