@Override public void mark_compacted(CompactionInfoStruct cr) throws MetaException { getTxnHandler().markCompacted(CompactionInfo.compactionStructToInfo(cr)); }
@Test public void testMarkCleaned() throws Exception { CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); assertEquals(0, txnHandler.findReadyToClean().size()); CompactionInfo ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); assertEquals(0, txnHandler.findReadyToClean().size()); txnHandler.markCompacted(ci); assertNull(txnHandler.findNextToCompact("fred")); List<CompactionInfo> toClean = txnHandler.findReadyToClean(); assertEquals(1, toClean.size()); assertNull(txnHandler.findNextToCompact("fred")); txnHandler.markCleaned(ci); assertNull(txnHandler.findNextToCompact("fred")); assertEquals(0, txnHandler.findReadyToClean().size()); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); assertEquals(1, rsp.getCompactsSize()); assertTrue(TxnHandler.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState())); }
@Test public void testMarkCompacted() throws Exception { CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); CompactionInfo ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); txnHandler.markCompacted(ci); assertNull(txnHandler.findNextToCompact("fred")); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); assertEquals(1, compacts.size()); ShowCompactResponseElement c = compacts.get(0); assertEquals("foo", c.getDbname()); assertEquals("bar", c.getTablename()); assertEquals("ds=today", c.getPartitionname()); assertEquals(CompactionType.MINOR, c.getType()); assertEquals("ready for cleaning", c.getState()); assertNull(c.getWorkerid()); }
@Test public void testFindNextToClean() throws Exception { CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); assertEquals(0, txnHandler.findReadyToClean().size()); CompactionInfo ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); assertEquals(0, txnHandler.findReadyToClean().size()); txnHandler.markCompacted(ci); assertNull(txnHandler.findNextToCompact("fred")); List<CompactionInfo> toClean = txnHandler.findReadyToClean(); assertEquals(1, toClean.size()); assertNull(txnHandler.findNextToCompact("fred")); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); assertEquals(1, compacts.size()); ShowCompactResponseElement c = compacts.get(0); assertEquals("foo", c.getDbname()); assertEquals("bar", c.getTablename()); assertEquals("ds=today", c.getPartitionname()); assertEquals(CompactionType.MINOR, c.getType()); assertEquals("ready for cleaning", c.getState()); assertNull(c.getWorkerid()); }
txnHandler.markCompacted(ci); if (conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST)) { mrJob = mr.getMrJob();
ci.runAs = System.getProperty("user.name"); txnHandler.updateCompactorState(ci, openTxn()); txnHandler.markCompacted(ci);
@Test public void cleanupAfterMajorTableCompaction() throws Exception { Table t = newTable("default", "camtc", false); addBaseFile(t, null, 20L, 20); addDeltaFile(t, null, 21L, 22L, 2); addDeltaFile(t, null, 23L, 24L, 2); addBaseFile(t, null, 25L, 25); burnThroughTransactions("default", "camtc", 25); CompactionRequest rqst = new CompactionRequest("default", "camtc", CompactionType.MAJOR); txnHandler.compact(rqst); CompactionInfo ci = txnHandler.findNextToCompact("fred"); ci.runAs = System.getProperty("user.name"); txnHandler.updateCompactorState(ci, openTxn()); txnHandler.markCompacted(ci); startCleaner(); // Check there are no compactions requests left. ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(1, rsp.getCompactsSize()); Assert.assertTrue(TxnStore.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState())); // Check that the files are removed List<Path> paths = getDirectories(conf, t, null); Assert.assertEquals(1, paths.size()); Assert.assertEquals("base_25", paths.get(0).getName()); }
ci.runAs = System.getProperty("user.name"); txnHandler.updateCompactorState(ci, openTxn()); txnHandler.markCompacted(ci);
@Test public void cleanupAfterMajorPartitionCompactionNoBase() throws Exception { Table t = newTable("default", "campcnb", true); Partition p = newPartition(t, "today"); addDeltaFile(t, p, 1L, 22L, 22); addDeltaFile(t, p, 23L, 24L, 2); addBaseFile(t, p, 25L, 25); burnThroughTransactions("default", "campcnb", 25); CompactionRequest rqst = new CompactionRequest("default", "campcnb", CompactionType.MAJOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); CompactionInfo ci = txnHandler.findNextToCompact("fred"); txnHandler.markCompacted(ci); ci.runAs = System.getProperty("user.name"); txnHandler.updateCompactorState(ci, openTxn()); startCleaner(); // Check there are no compactions requests left. ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(1, rsp.getCompactsSize()); Assert.assertTrue(TxnStore.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState())); // Check that the files are removed List<Path> paths = getDirectories(conf, t, p); Assert.assertEquals(1, paths.size()); Assert.assertEquals("base_25", paths.get(0).getName()); }
@Test public void cleanupAfterMajorPartitionCompaction() throws Exception { Table t = newTable("default", "campc", true); Partition p = newPartition(t, "today"); addBaseFile(t, p, 20L, 20); addDeltaFile(t, p, 21L, 22L, 2); addDeltaFile(t, p, 23L, 24L, 2); addBaseFile(t, p, 25L, 25); burnThroughTransactions("default", "campc", 25); CompactionRequest rqst = new CompactionRequest("default", "campc", CompactionType.MAJOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); CompactionInfo ci = txnHandler.findNextToCompact("fred"); ci.runAs = System.getProperty("user.name"); txnHandler.updateCompactorState(ci, openTxn()); txnHandler.markCompacted(ci); startCleaner(); // Check there are no compactions requests left. ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(1, rsp.getCompactsSize()); Assert.assertTrue(TxnStore.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState())); // Check that the files are removed List<Path> paths = getDirectories(conf, t, p); Assert.assertEquals(1, paths.size()); Assert.assertEquals("base_25", paths.get(0).getName()); }
ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); txnHandler.markCompacted(ci); ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); txnHandler.markCompacted(ci);
@Test public void droppedPartition() throws Exception { Table t = newTable("default", "dp", true); Partition p = newPartition(t, "today"); addDeltaFile(t, p, 1L, 22L, 22); addDeltaFile(t, p, 23L, 24L, 2); addBaseFile(t, p, 25L, 25); burnThroughTransactions("default", "dp", 25); CompactionRequest rqst = new CompactionRequest("default", "dp", CompactionType.MAJOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); CompactionInfo ci = txnHandler.findNextToCompact("fred"); ci.runAs = System.getProperty("user.name"); txnHandler.updateCompactorState(ci, openTxn()); txnHandler.markCompacted(ci); // Drop partition will clean the partition entry from the compaction queue and hence cleaner have no effect ms.dropPartition("default", "dp", Collections.singletonList("today"), true); startCleaner(); // Check there are no compactions requests left. ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); } @Override
@Test public void droppedTable() throws Exception { Table t = newTable("default", "dt", false); addDeltaFile(t, null, 1L, 22L, 22); addDeltaFile(t, null, 23L, 24L, 2); addBaseFile(t, null, 25L, 25); burnThroughTransactions("default", "dt", 25); CompactionRequest rqst = new CompactionRequest("default", "dt", CompactionType.MINOR); txnHandler.compact(rqst); CompactionInfo ci = txnHandler.findNextToCompact("fred"); ci.runAs = System.getProperty("user.name"); txnHandler.updateCompactorState(ci, openTxn()); txnHandler.markCompacted(ci); // Drop table will clean the table entry from the compaction queue and hence cleaner have no effect ms.dropTable("default", "dt"); startCleaner(); // Check there are no compactions requests left. ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); }