@Override public ShowCompactResponse show_compact(ShowCompactRequest rqst) throws TException { return getTxnHandler().showCompact(rqst); }
@Test public void testCompactMajorWithPartition() throws Exception { CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MAJOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); assertEquals(1, compacts.size()); ShowCompactResponseElement c = compacts.get(0); assertEquals("foo", c.getDbname()); assertEquals("bar", c.getTablename()); assertEquals("ds=today", c.getPartitionname()); assertEquals(CompactionType.MAJOR, c.getType()); assertEquals("initiated", c.getState()); assertEquals(0L, c.getStart()); }
@Test public void testCompactMinorNoPartition() throws Exception { CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR); rqst.setRunas("fred"); txnHandler.compact(rqst); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); assertEquals(1, compacts.size()); ShowCompactResponseElement c = compacts.get(0); assertEquals("foo", c.getDbname()); assertEquals("bar", c.getTablename()); assertNull(c.getPartitionname()); assertEquals(CompactionType.MINOR, c.getType()); assertEquals("initiated", c.getState()); assertEquals(0L, c.getStart()); assertEquals("fred", c.getRunAs()); }
@Test public void testMarkCleaned() throws Exception { CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); assertEquals(0, txnHandler.findReadyToClean().size()); CompactionInfo ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); assertEquals(0, txnHandler.findReadyToClean().size()); txnHandler.markCompacted(ci); assertNull(txnHandler.findNextToCompact("fred")); List<CompactionInfo> toClean = txnHandler.findReadyToClean(); assertEquals(1, toClean.size()); assertNull(txnHandler.findNextToCompact("fred")); txnHandler.markCleaned(ci); assertNull(txnHandler.findNextToCompact("fred")); assertEquals(0, txnHandler.findReadyToClean().size()); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); assertEquals(1, rsp.getCompactsSize()); assertTrue(TxnHandler.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState())); }
@Test public void testRevokeTimedOutWorkers() throws Exception { CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR); txnHandler.compact(rqst); rqst = new CompactionRequest("foo", "baz", CompactionType.MINOR); txnHandler.compact(rqst); assertNotNull(txnHandler.findNextToCompact("fred-193892")); Thread.sleep(200); assertNotNull(txnHandler.findNextToCompact("fred-193892")); txnHandler.revokeTimedoutWorkers(100); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); assertEquals(2, compacts.size()); boolean sawWorking = false, sawInitiated = false; for (ShowCompactResponseElement c : compacts) { if (c.getState().equals("working")) sawWorking = true; else if (c.getState().equals("initiated")) sawInitiated = true; else fail("Unexpected state"); } assertTrue(sawWorking); assertTrue(sawInitiated); }
@Test public void testFindNextToClean() throws Exception { CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); assertEquals(0, txnHandler.findReadyToClean().size()); CompactionInfo ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); assertEquals(0, txnHandler.findReadyToClean().size()); txnHandler.markCompacted(ci); assertNull(txnHandler.findNextToCompact("fred")); List<CompactionInfo> toClean = txnHandler.findReadyToClean(); assertEquals(1, toClean.size()); assertNull(txnHandler.findNextToCompact("fred")); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); assertEquals(1, compacts.size()); ShowCompactResponseElement c = compacts.get(0); assertEquals("foo", c.getDbname()); assertEquals("bar", c.getTablename()); assertEquals("ds=today", c.getPartitionname()); assertEquals(CompactionType.MINOR, c.getType()); assertEquals("ready for cleaning", c.getState()); assertNull(c.getWorkerid()); }
@Test public void testMarkCompacted() throws Exception { CompactionRequest rqst = new CompactionRequest("foo", "bar", CompactionType.MINOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); CompactionInfo ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); txnHandler.markCompacted(ci); assertNull(txnHandler.findNextToCompact("fred")); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); assertEquals(1, compacts.size()); ShowCompactResponseElement c = compacts.get(0); assertEquals("foo", c.getDbname()); assertEquals("bar", c.getTablename()); assertEquals("ds=today", c.getPartitionname()); assertEquals(CompactionType.MINOR, c.getType()); assertEquals("ready for cleaning", c.getState()); assertNull(c.getWorkerid()); }
@Test public void recoverFailedRemoteWorkers() throws Exception { Table t = newTable("default", "rfrw1", false); CompactionRequest rqst = new CompactionRequest("default", "rfrw1", CompactionType.MINOR); txnHandler.compact(rqst); txnHandler.findNextToCompact("nosuchhost-193892"); conf.setTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_WORKER_TIMEOUT, 1L, TimeUnit.MILLISECONDS); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(1, compacts.size()); Assert.assertEquals("initiated", compacts.get(0).getState()); }
@Test public void noCompactWhenNoCompactSet() throws Exception { Map<String, String> parameters = new HashMap<String, String>(1); parameters.put("NO_AUTO_COMPACTION", "true"); Table t = newTable("default", "ncwncs", false, parameters); HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); for (int i = 0; i < 11; i++) { long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("ncwncs"); comp.setOperationType(DataOperationType.UPDATE); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.abortTxn(new AbortTxnRequest(txnid)); } startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); }
@Test public void noCompactWhenNoCompactSetLowerCase() throws Exception { Map<String, String> parameters = new HashMap<String, String>(1); parameters.put("no_auto_compaction", "true"); Table t = newTable("default", "ncwncs", false, parameters); HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); for (int i = 0; i < 11; i++) { long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setOperationType(DataOperationType.DELETE); comp.setTablename("ncwncs"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.abortTxn(new AbortTxnRequest(txnid)); } startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); }
/** * https://issues.apache.org/jira/browse/HIVE-17391 */ @Test public void testEmptyInTblproperties() throws Exception { runStatementOnDriver("create table t1 " + "(a int, b int) stored as orc TBLPROPERTIES ('serialization.null.format'='', 'transactional'='true')"); runStatementOnDriver("insert into t1 " + "(a,b) values(1,7),(3,7)"); runStatementOnDriver("update t1" + " set b = -2 where b = 2"); runStatementOnDriver("alter table t1 " + " compact 'MAJOR'"); runWorker(hiveConf); TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals("Unexpected number of compactions in history", 1, resp.getCompactsSize()); Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(0).getState()); Assert.assertTrue(resp.getCompacts().get(0).getHadoopJobId().startsWith("job_local")); }
@Test public void cleanupAfterMajorTableCompaction() throws Exception { Table t = newTable("default", "camtc", false); addBaseFile(t, null, 20L, 20); addDeltaFile(t, null, 21L, 22L, 2); addDeltaFile(t, null, 23L, 24L, 2); addBaseFile(t, null, 25L, 25); burnThroughTransactions("default", "camtc", 25); CompactionRequest rqst = new CompactionRequest("default", "camtc", CompactionType.MAJOR); txnHandler.compact(rqst); CompactionInfo ci = txnHandler.findNextToCompact("fred"); ci.runAs = System.getProperty("user.name"); txnHandler.updateCompactorState(ci, openTxn()); txnHandler.markCompacted(ci); startCleaner(); // Check there are no compactions requests left. ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(1, rsp.getCompactsSize()); Assert.assertTrue(TxnStore.SUCCEEDED_RESPONSE.equals(rsp.getCompacts().get(0).getState())); // Check that the files are removed List<Path> paths = getDirectories(conf, t, null); Assert.assertEquals(1, paths.size()); Assert.assertEquals("base_25", paths.get(0).getName()); }
@Test public void droppedTable() throws Exception { Table t = newTable("default", "dt", false); addDeltaFile(t, null, 1L, 2L, 2); addDeltaFile(t, null, 3L, 4L, 2); burnThroughTransactions("default", "dt", 4); CompactionRequest rqst = new CompactionRequest("default", "dt", CompactionType.MAJOR); txnHandler.compact(rqst); // Drop table will clean the table entry from the compaction queue and hence worker have no effect ms.dropTable("default", "dt"); startWorker(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(0, compacts.size()); }
@Test public void droppedPartition() throws Exception { Table t = newTable("default", "dp", true); Partition p = newPartition(t, "today"); addBaseFile(t, p, 20L, 20); addDeltaFile(t, p, 21L, 22L, 2); addDeltaFile(t, p, 23L, 24L, 2); burnThroughTransactions("default", "dp", 25); CompactionRequest rqst = new CompactionRequest("default", "dp", CompactionType.MINOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); // Drop partition will clean the partition entry from the compaction queue and hence worker have no effect ms.dropPartition("default", "dp", Collections.singletonList("today"), true); startWorker(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(0, compacts.size()); }
@Test public void testCompactWithDelete() throws Exception { int[][] tableData = {{1,2},{3,4}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'"); runWorker(hiveConf); runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 4"); runStatementOnDriver("update " + Table.ACIDTBL + " set b = -2 where b = 2"); runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MINOR'"); runWorker(hiveConf); TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals("Unexpected number of compactions in history", 2, resp.getCompactsSize()); Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(0).getState()); Assert.assertEquals("Unexpected 1 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(1).getState()); }
@Test public void noCompactOnManyDifferentPartitionAborts() throws Exception { Table t = newTable("default", "ncomdpa", true); for (int i = 0; i < 11; i++) { Partition p = newPartition(t, "day-" + i); } HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); for (int i = 0; i < 11; i++) { long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("ncomdpa"); comp.setPartitionname("ds=day-" + i); comp.setOperationType(DataOperationType.UPDATE); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.abortTxn(new AbortTxnRequest(txnid)); } startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); }
@Test public void droppedPartition() throws Exception { Table t = newTable("default", "dp", true); Partition p = newPartition(t, "today"); addDeltaFile(t, p, 1L, 22L, 22); addDeltaFile(t, p, 23L, 24L, 2); addBaseFile(t, p, 25L, 25); burnThroughTransactions("default", "dp", 25); CompactionRequest rqst = new CompactionRequest("default", "dp", CompactionType.MAJOR); rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); CompactionInfo ci = txnHandler.findNextToCompact("fred"); ci.runAs = System.getProperty("user.name"); txnHandler.updateCompactorState(ci, openTxn()); txnHandler.markCompacted(ci); // Drop partition will clean the partition entry from the compaction queue and hence cleaner have no effect ms.dropPartition("default", "dp", Collections.singletonList("today"), true); startCleaner(); // Check there are no compactions requests left. ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); } @Override
@Test public void noCompactTableNotEnoughDeltas() throws Exception { Table t = newTable("default", "nctned", false); addBaseFile(t, null, 200L, 200); addDeltaFile(t, null, 201L, 205L, 5); addDeltaFile(t, null, 206L, 211L, 6); burnThroughTransactions("default", "nctned", 210); long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("nctned"); comp.setOperationType(DataOperationType.UPDATE); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); long writeid = allocateWriteId("default", "nctned", txnid); Assert.assertEquals(211, writeid); txnHandler.commitTxn(new CommitTxnRequest(txnid)); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); }
@Test public void droppedTable() throws Exception { Table t = newTable("default", "dt", false); addDeltaFile(t, null, 1L, 22L, 22); addDeltaFile(t, null, 23L, 24L, 2); addBaseFile(t, null, 25L, 25); burnThroughTransactions("default", "dt", 25); CompactionRequest rqst = new CompactionRequest("default", "dt", CompactionType.MINOR); txnHandler.compact(rqst); CompactionInfo ci = txnHandler.findNextToCompact("fred"); ci.runAs = System.getProperty("user.name"); txnHandler.updateCompactorState(ci, openTxn()); txnHandler.markCompacted(ci); // Drop table will clean the table entry from the compaction queue and hence cleaner have no effect ms.dropTable("default", "dt"); startCleaner(); // Check there are no compactions requests left. ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); }
@Test public void noCompactTableDeltaPctNotHighEnough() throws Exception { Table t = newTable("default", "nctdpnhe", false); addBaseFile(t, null, 50L, 50); addDeltaFile(t, null, 21L, 22L, 2); addDeltaFile(t, null, 23L, 24L, 2); burnThroughTransactions("default", "nctdpnhe", 53); long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("nctdpnhe"); comp.setOperationType(DataOperationType.UPDATE); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); long writeid = allocateWriteId("default", "nctdpnhe", txnid); Assert.assertEquals(54, writeid); txnHandler.commitTxn(new CommitTxnRequest(txnid)); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); }