@Override public CommandProcessorResponse compileAndRespond(String statement) { currentQuery = statement; return coreDriver.compileAndRespond(statement); }
@Override public CommandProcessorResponse compileAndRespond(String command) { return compileAndRespond(command, false); }
@Test public void createTable() throws Exception { dropTable(new String[] {"T"}); CommandProcessorResponse cpr = driver.compileAndRespond("create table if not exists T (a int, b int)", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", null, null, locks); txnMgr.commitTxn(); Assert.assertEquals("Lock remained", 0, getLocks().size()); } @Test
/** * txns update same resource but do not overlap in time - no conflict */ @Test public void testWriteSetTracking1() throws Exception { dropTable(new String[] {"TAB_PART"}); CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " + "partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); checkCmdOnDriver(driver.compileAndRespond("select * from TAB_PART", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "Nicholas"); txnMgr.commitTxn(); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); checkCmdOnDriver(driver.compileAndRespond("update TAB_PART set b = 7 where p = 'blah'", true)); checkCmdOnDriver(driver.compileAndRespond("update TAB_PART set b = 7 where p = 'blah'", true)); txnMgr2.acquireLocks(driver.getPlan(), ctx, "Alexandra"); txnMgr2.commitTxn(); } private void dropTable(String[] tabs) throws Exception {
@Test public void testShowLocksAgentInfo() throws Exception { CommandProcessorResponse cpr = driver.run("create table if not exists XYZ (a int, b int)"); checkCmdOnDriver(cpr); checkCmdOnDriver(driver.compileAndRespond("select a from XYZ where b = 8", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "XYZ"); List<ShowLocksResponseElement> locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "XYZ", null, locks); Assert.assertEquals("Wrong AgentInfo", driver.getPlan().getQueryId(), locks.get(0).getAgentInfo()); } @Test
@Test public void testDummyTxnManagerOnAcidTable() throws Exception { dropTable(new String[] {"T10", "T11"}); // Create an ACID table with DbTxnManager CommandProcessorResponse cpr = driver.run("create table T10 (a int, b int) clustered by(b) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); cpr = driver.run("create table T11 (a int, b int) clustered by(b) into 2 buckets stored as orc"); checkCmdOnDriver(cpr); // All DML should fail with DummyTxnManager on ACID table useDummyTxnManagerTemporarily(conf); cpr = driver.compileAndRespond("select * from T10", true); Assert.assertEquals(ErrorMsg.TXNMGR_NOT_ACID.getErrorCode(), cpr.getResponseCode()); Assert.assertTrue(cpr.getErrorMessage().contains("This command is not allowed on an ACID table")); useDummyTxnManagerTemporarily(conf); cpr = driver.compileAndRespond("insert into table T10 values (1, 2)", true); Assert.assertEquals(ErrorMsg.TXNMGR_NOT_ACID.getErrorCode(), cpr.getResponseCode()); Assert.assertTrue(cpr.getErrorMessage().contains("This command is not allowed on an ACID table")); useDummyTxnManagerTemporarily(conf); cpr = driver.compileAndRespond("update T10 set a=0 where b=1", true); Assert.assertEquals(ErrorMsg.ACID_OP_ON_NONACID_TXNMGR.getErrorCode(), cpr.getResponseCode()); Assert.assertTrue(cpr.getErrorMessage().contains("Attempt to do update or delete using transaction manager that does not support these operations.")); useDummyTxnManagerTemporarily(conf); cpr = driver.compileAndRespond("delete from T10", true); Assert.assertEquals(ErrorMsg.ACID_OP_ON_NONACID_TXNMGR.getErrorCode(), cpr.getResponseCode()); Assert.assertTrue(cpr.getErrorMessage().contains("Attempt to do update or delete using transaction manager that does not support these operations.")); conf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"); }
/** * check that locks in Waiting state show what they are waiting on * This test is somewhat abusive in that it make DbLockManager retain locks for 2 * different queries (which are not part of the same transaction) which can never * happen in real use cases... but it makes testing convenient. * @throws Exception */ @Test public void testLockBlockedBy() throws Exception { dropTable(new String[] {"TAB_BLOCKED"}); CommandProcessorResponse cpr = driver.run("create table TAB_BLOCKED (a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("select * from TAB_BLOCKED", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "I AM SAM"); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_BLOCKED", null, locks); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); cpr = driver.compileAndRespond("drop table TAB_BLOCKED", true); checkCmdOnDriver(cpr); ((DbTxnManager)txnMgr2).acquireLocks(driver.getPlan(), ctx, "SAM I AM", false);//make non-blocking locks = getLocks(); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_BLOCKED", null, locks); checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "TAB_BLOCKED", null, locks); Assert.assertEquals("BlockedByExtId doesn't match", locks.get(0).getLockid(), locks.get(1).getBlockedByExtId()); Assert.assertEquals("BlockedByIntId doesn't match", locks.get(0).getLockIdInternal(), locks.get(1).getBlockedByIntId()); }
@Test public void testLockingOnInsertIntoNonNativeTables() throws Exception { dropTable(new String[] {"tab_not_acid"}); checkCmdOnDriver(driver.run("create table if not exists tab_not_acid (a int, b int) " + " STORED BY 'org.apache.hadoop.hive.ql.metadata.StorageHandlerMock'")); txnMgr.openTxn(ctx, "T1"); checkCmdOnDriver(driver.compileAndRespond("insert into tab_not_acid values(1,2)", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "T1"); List<ShowLocksResponseElement> locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); }
@Test public void testLockingOnInsertOverwriteNonNativeTables() throws Exception { dropTable(new String[] {"tab_not_acid"}); checkCmdOnDriver(driver.run("create table if not exists tab_not_acid (a int, b int) " + " STORED BY 'org.apache.hadoop.hive.ql.metadata.StorageHandlerMock'")); txnMgr.openTxn(ctx, "T1"); checkCmdOnDriver(driver.compileAndRespond("insert overwrite table tab_not_acid values(1,2)", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "T1"); List<ShowLocksResponseElement> locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", null, locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); }
@Test public void testMmConversionLocks() throws Exception { dropTable(new String[] {"T"}); CommandProcessorResponse cpr = driver.run("create table T (a int, b int) tblproperties('transactional'='false')"); checkCmdOnDriver(cpr); cpr = driver.run("insert into T values(0,2),(1,4)"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("ALTER TABLE T set tblproperties" + "('transactional'='true', 'transactional_properties'='insert_only')", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");//gets X lock on T List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T", null, locks); } @Test
@Test public void testLoadData() throws Exception { dropTable(new String[] {"T2"}); CommandProcessorResponse cpr = driver.run("create table T2(a int) " + "stored as ORC TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); checkCmdOnDriver(driver.run("insert into T2 values(1)")); String exportLoc = exportFolder.newFolder("1").toString(); checkCmdOnDriver(driver.run("export table T2 to '" + exportLoc + "/2'")); cpr = driver.compileAndRespond( "load data inpath '" + exportLoc + "/2/data' overwrite into table T2"); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T2", null, locks); txnMgr.commitTxn(); } @Test
CommandProcessorResponse cpr = driver.run("create table if not exists T6(a int)"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("select a from T6", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");//gets S lock on T6 HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); cpr = driver.compileAndRespond("drop table if exists T6", true); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("select a from T6", true); checkCmdOnDriver(cpr); ((DbTxnManager)txnMgr3).acquireLocks(driver.getPlan(), ctx, "Fifer", false);//gets S lock on T6
@Test public void testTruncate() throws Exception { dropTable(new String[] {"T"}); CommandProcessorResponse cpr = driver.run("create table T (a int, b int) stored as" + " orc tblproperties('transactional'='true')"); checkCmdOnDriver(cpr); checkCmdOnDriver(driver.run("insert into T values(0,2),(1,4)")); checkCmdOnDriver(driver.run("truncate table T")); cpr = driver.compileAndRespond("truncate table T"); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");//gets X lock on T List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T", null, locks); } }
@Test public void testLocksInSubquery() throws Exception { dropTable(new String[] {"T","S", "R"}); checkCmdOnDriver(driver.run("create table if not exists T (a int, b int)")); checkCmdOnDriver(driver.run("create table if not exists S (a int, b int) clustered by(b) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')")); checkCmdOnDriver(driver.run("create table if not exists R (a int, b int) clustered by(b) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')")); checkCmdOnDriver(driver.compileAndRespond("delete from S where a in (select a from T where b = 1)", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "one"); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T", null, locks); checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "S", null, locks); txnMgr.rollbackTxn(); checkCmdOnDriver(driver.compileAndRespond("update S set a = 7 where a in (select a from T where b = 1)", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "one"); locks = getLocks(); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T", null, locks); checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "S", null, locks); txnMgr.rollbackTxn(); checkCmdOnDriver(driver.compileAndRespond("insert into R select * from S where a in (select a from T where b = 1)", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "three"); locks = getLocks(); Assert.assertEquals("Unexpected lock count", 3, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T", null, locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "S", null, locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "R", null, locks); txnMgr.rollbackTxn(); } @Test
@Test public void lockConflictDbTable() throws Exception { dropTable(new String[] {"temp.T7"}); CommandProcessorResponse cpr = driver.run("create database if not exists temp"); checkCmdOnDriver(cpr); cpr = driver.run("create table if not exists temp.T7(a int, b int) clustered by(b) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("update temp.T7 set a = 5 where b = 6", true);//gets SS lock on T7 checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); checkCmdOnDriver(driver.compileAndRespond("drop database if exists temp", true)); ((DbTxnManager)txnMgr2).acquireLocks(driver.getPlan(), ctx, "Fiddler", false); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "temp", "T7", null, locks); checkLock(LockType.EXCLUSIVE, LockState.WAITING, "temp", null, null, locks); txnMgr.commitTxn(); ((DbLockManager)txnMgr2.getLockManager()).checkLock(locks.get(1).getLockid()); locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "temp", null, null, locks); txnMgr2.commitTxn(); } @Test
/** * txns overlap in time but do not update same resource - no conflict */ @Test public void testWriteSetTracking2() throws Exception { dropTable(new String[] {"TAB_PART", "TAB2"}); CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " + "partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); cpr = driver.run("create table if not exists TAB2 (a int, b int) partitioned by (p string) " + "clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); txnMgr.openTxn(ctx, "Peter"); checkCmdOnDriver(driver.compileAndRespond("update TAB_PART set b = 7 where p = 'blah'", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "Peter"); txnMgr2.openTxn(ctx, "Catherine"); List<ShowLocksResponseElement> locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 1, locks.size()); //note that "update" uses dynamic partitioning thus lock is on the table not partition checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB_PART", null, locks); txnMgr.commitTxn(); checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 9 where p = 'doh'", true)); txnMgr2.acquireLocks(driver.getPlan(), ctx, "Catherine"); txnMgr2.commitTxn(); } /**
@Test public void testLockRetryLimit() throws Exception { dropTable(new String[] {"T9"}); conf.setIntVar(HiveConf.ConfVars.HIVE_LOCK_NUMRETRIES, 2); conf.setBoolVar(HiveConf.ConfVars.TXN_MGR_DUMP_LOCK_STATE_ON_ACQUIRE_TIMEOUT, true); CommandProcessorResponse cpr = driver.run("create table T9(a int)"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("select * from T9", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Vincent Vega"); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T9", null, locks); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); cpr = driver.compileAndRespond("drop table T9", true); checkCmdOnDriver(cpr); try { txnMgr2.acquireLocks(driver.getPlan(), ctx, "Winston Winnfield"); } catch(LockException ex) { Assert.assertEquals("Got wrong lock exception", ErrorMsg.LOCK_ACQUIRE_TIMEDOUT, ex.getCanonicalErrorMsg()); } locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T9", null, locks); txnMgr2.closeTxnManager(); }
/** * see also {@link org.apache.hadoop.hive.ql.TestTxnAddPartition} */ @Test public void testAddPartitionLocks() throws Exception { dropTable(new String[] {"T", "Tstage"}); CommandProcessorResponse cpr = driver.run("create table T (a int, b int) partitioned by (p int) " + "stored as orc tblproperties('transactional'='true')"); checkCmdOnDriver(cpr); //bucketed just so that we get 2 files cpr = driver.run("create table Tstage (a int, b int) clustered by (a) into 2 " + "buckets stored as orc tblproperties('transactional'='false')"); checkCmdOnDriver(cpr); cpr = driver.run("insert into Tstage values(0,2),(1,4)"); checkCmdOnDriver(cpr); String exportLoc = exportFolder.newFolder("1").toString(); cpr = driver.run("export table Tstage to '" + exportLoc + "'"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("ALTER TABLE T ADD if not exists PARTITION (p=0)" + " location '" + exportLoc + "/data'", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");//gets X lock on T List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T", null, locks); } @Test
private void insertOverwriteCreate(boolean isTransactional) throws Exception { if(isTransactional) { MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID, true); } dropTable(new String[] {"T2", "T3"}); CommandProcessorResponse cpr = driver.run("create table if not exists T2(a int)"); checkCmdOnDriver(cpr); cpr = driver.run("create table T3(a int) stored as ORC"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("insert overwrite table T3 select a from T2", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T2", null, locks); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T3", null, locks); txnMgr.commitTxn(); Assert.assertEquals("Lock remained", 0, getLocks().size()); cpr = driver.run("drop table if exists T1"); checkCmdOnDriver(cpr); cpr = driver.run("drop table if exists T2"); checkCmdOnDriver(cpr); } @Test
private void insertOverwritePartitionedCreate(boolean isTransactional) throws Exception { dropTable(new String[] {"T4", "T5"}); if(isTransactional) { MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID, true); } CommandProcessorResponse cpr = driver.run("create table T4" + "(name string, gpa double) partitioned by (age int) stored as ORC"); checkCmdOnDriver(cpr); cpr = driver.run("create table T5(name string, age int, gpa double)"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("INSERT OVERWRITE TABLE T4 PARTITION (age) SELECT " + "name, age, gpa FROM T5", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T5", null, locks); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T4", null, locks); txnMgr.commitTxn(); Assert.assertEquals("Lock remained", 0, getLocks().size()); cpr = driver.run("drop table if exists T5"); checkCmdOnDriver(cpr); cpr = driver.run("drop table if exists T4"); checkCmdOnDriver(cpr); } @Test