@Override public QueryPlan getPlan() { return coreDriver.getPlan(); }
private void doIndexUpdate(List<Index> tblIndexes) throws HiveException { Driver driver = new Driver(this.conf); for (Index idx : tblIndexes) { StringBuilder sb = new StringBuilder(); sb.append("ALTER INDEX "); sb.append(idx.getIndexName()); sb.append(" ON "); sb.append(idx.getDbName()).append('.'); sb.append(idx.getOrigTableName()); sb.append(" REBUILD"); driver.compile(sb.toString(), false); tasks.addAll(driver.getPlan().getRootTasks()); inputs.addAll(driver.getPlan().getInputs()); } }
private String getColumnType(String query) { Driver driver = createDriver(); int rc = driver.compile(query); if (rc != 0) { return null; } QueryPlan plan = driver.getPlan(); DDLTask task = (DDLTask) plan.getRootTasks().get(0); DDLWork work = task.getWork(); CreateTableDesc spec = work.getCreateTblDesc(); FieldSchema fs = spec.getCols().get(0); return fs.getType(); }
@Test public void testShowLocksAgentInfo() throws Exception { CommandProcessorResponse cpr = driver.run("create table if not exists XYZ (a int, b int)"); checkCmdOnDriver(cpr); checkCmdOnDriver(driver.compileAndRespond("select a from XYZ where b = 8", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "XYZ"); List<ShowLocksResponseElement> locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "XYZ", null, locks); Assert.assertEquals("Wrong AgentInfo", driver.getPlan().getQueryId(), locks.get(0).getAgentInfo()); } @Test
@Test public void testQueryTable1() throws ParseException { String query = "select * from t1"; Driver driver = createDriver(); int rc = driver.compile(query); Assert.assertEquals("Checking command success", 0, rc); QueryPlan plan = driver.getPlan(); // check access columns from ColumnAccessInfo ColumnAccessInfo columnAccessInfo = plan.getColumnAccessInfo(); List<String> cols = columnAccessInfo.getTableToColumnAccessMap().get("default@t1"); Assert.assertNotNull(cols); Assert.assertEquals(2, cols.size()); Assert.assertNotNull(cols.contains("id1")); Assert.assertNotNull(cols.contains("name1")); // check access columns from readEntity Map<String, List<String>> tableColsMap = getColsFromReadEntity(plan.getInputs()); cols = tableColsMap.get("default@t1"); Assert.assertNotNull(cols); Assert.assertEquals(2, cols.size()); Assert.assertNotNull(cols.contains("id1")); Assert.assertNotNull(cols.contains("name1")); }
@Test public void createTable() throws Exception { dropTable(new String[] {"T"}); CommandProcessorResponse cpr = driver.compileAndRespond("create table if not exists T (a int, b int)", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", null, null, locks); txnMgr.commitTxn(); Assert.assertEquals("Lock remained", 0, getLocks().size()); } @Test
/** * check that locks in Waiting state show what they are waiting on * This test is somewhat abusive in that it make DbLockManager retain locks for 2 * different queries (which are not part of the same transaction) which can never * happen in real use cases... but it makes testing convenient. * @throws Exception */ @Test public void testLockBlockedBy() throws Exception { dropTable(new String[] {"TAB_BLOCKED"}); CommandProcessorResponse cpr = driver.run("create table TAB_BLOCKED (a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("select * from TAB_BLOCKED", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "I AM SAM"); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_BLOCKED", null, locks); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); cpr = driver.compileAndRespond("drop table TAB_BLOCKED", true); checkCmdOnDriver(cpr); ((DbTxnManager)txnMgr2).acquireLocks(driver.getPlan(), ctx, "SAM I AM", false);//make non-blocking locks = getLocks(); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_BLOCKED", null, locks); checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "TAB_BLOCKED", null, locks); Assert.assertEquals("BlockedByExtId doesn't match", locks.get(0).getLockid(), locks.get(1).getBlockedByExtId()); Assert.assertEquals("BlockedByIntId doesn't match", locks.get(0).getLockIdInternal(), locks.get(1).getBlockedByIntId()); }
@Test public void testLockingOnInsertIntoNonNativeTables() throws Exception { dropTable(new String[] {"tab_not_acid"}); checkCmdOnDriver(driver.run("create table if not exists tab_not_acid (a int, b int) " + " STORED BY 'org.apache.hadoop.hive.ql.metadata.StorageHandlerMock'")); txnMgr.openTxn(ctx, "T1"); checkCmdOnDriver(driver.compileAndRespond("insert into tab_not_acid values(1,2)", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "T1"); List<ShowLocksResponseElement> locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); }
@Test public void testLockingOnInsertOverwriteNonNativeTables() throws Exception { dropTable(new String[] {"tab_not_acid"}); checkCmdOnDriver(driver.run("create table if not exists tab_not_acid (a int, b int) " + " STORED BY 'org.apache.hadoop.hive.ql.metadata.StorageHandlerMock'")); txnMgr.openTxn(ctx, "T1"); checkCmdOnDriver(driver.compileAndRespond("insert overwrite table tab_not_acid values(1,2)", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "T1"); List<ShowLocksResponseElement> locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", null, locks); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "_dummy_database", "_dummy_table", null, locks); }
@Test public void testMmConversionLocks() throws Exception { dropTable(new String[] {"T"}); CommandProcessorResponse cpr = driver.run("create table T (a int, b int) tblproperties('transactional'='false')"); checkCmdOnDriver(cpr); cpr = driver.run("insert into T values(0,2),(1,4)"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("ALTER TABLE T set tblproperties" + "('transactional'='true', 'transactional_properties'='insert_only')", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");//gets X lock on T List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T", null, locks); } @Test
/** * txns update same resource but do not overlap in time - no conflict */ @Test public void testWriteSetTracking1() throws Exception { dropTable(new String[] {"TAB_PART"}); CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " + "partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); checkCmdOnDriver(driver.compileAndRespond("select * from TAB_PART", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "Nicholas"); txnMgr.commitTxn(); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); checkCmdOnDriver(driver.compileAndRespond("update TAB_PART set b = 7 where p = 'blah'", true)); checkCmdOnDriver(driver.compileAndRespond("update TAB_PART set b = 7 where p = 'blah'", true)); txnMgr2.acquireLocks(driver.getPlan(), ctx, "Alexandra"); txnMgr2.commitTxn(); } private void dropTable(String[] tabs) throws Exception {
@Test public void testLoadData() throws Exception { dropTable(new String[] {"T2"}); CommandProcessorResponse cpr = driver.run("create table T2(a int) " + "stored as ORC TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); checkCmdOnDriver(driver.run("insert into T2 values(1)")); String exportLoc = exportFolder.newFolder("1").toString(); checkCmdOnDriver(driver.run("export table T2 to '" + exportLoc + "/2'")); cpr = driver.compileAndRespond( "load data inpath '" + exportLoc + "/2/data' overwrite into table T2"); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T2", null, locks); txnMgr.commitTxn(); } @Test
cpr = driver.compileAndRespond("select a from T6", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");//gets S lock on T6 HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); checkCmdOnDriver(cpr); LockState lockState = ((DbTxnManager) txnMgr2).acquireLocks(driver.getPlan(), ctx, "Fiddler", false); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 2, locks.size()); ((DbTxnManager)txnMgr3).acquireLocks(driver.getPlan(), ctx, "Fifer", false);//gets S lock on T6 locks = getLocks(); Assert.assertEquals("Unexpected lock count", 3, locks.size());
@Test public void testTruncate() throws Exception { dropTable(new String[] {"T"}); CommandProcessorResponse cpr = driver.run("create table T (a int, b int) stored as" + " orc tblproperties('transactional'='true')"); checkCmdOnDriver(cpr); checkCmdOnDriver(driver.run("insert into T values(0,2),(1,4)")); checkCmdOnDriver(driver.run("truncate table T")); cpr = driver.compileAndRespond("truncate table T"); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");//gets X lock on T List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T", null, locks); } }
@Test public void lockConflictDbTable() throws Exception { dropTable(new String[] {"temp.T7"}); CommandProcessorResponse cpr = driver.run("create database if not exists temp"); checkCmdOnDriver(cpr); cpr = driver.run("create table if not exists temp.T7(a int, b int) clustered by(b) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("update temp.T7 set a = 5 where b = 6", true);//gets SS lock on T7 checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); checkCmdOnDriver(driver.compileAndRespond("drop database if exists temp", true)); ((DbTxnManager)txnMgr2).acquireLocks(driver.getPlan(), ctx, "Fiddler", false); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "temp", "T7", null, locks); checkLock(LockType.EXCLUSIVE, LockState.WAITING, "temp", null, null, locks); txnMgr.commitTxn(); ((DbLockManager)txnMgr2.getLockManager()).checkLock(locks.get(1).getLockid()); locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "temp", null, null, locks); txnMgr2.commitTxn(); } @Test
/** * txns overlap in time but do not update same resource - no conflict */ @Test public void testWriteSetTracking2() throws Exception { dropTable(new String[] {"TAB_PART", "TAB2"}); CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " + "partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); cpr = driver.run("create table if not exists TAB2 (a int, b int) partitioned by (p string) " + "clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); checkCmdOnDriver(cpr); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); txnMgr.openTxn(ctx, "Peter"); checkCmdOnDriver(driver.compileAndRespond("update TAB_PART set b = 7 where p = 'blah'", true)); txnMgr.acquireLocks(driver.getPlan(), ctx, "Peter"); txnMgr2.openTxn(ctx, "Catherine"); List<ShowLocksResponseElement> locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 1, locks.size()); //note that "update" uses dynamic partitioning thus lock is on the table not partition checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB_PART", null, locks); txnMgr.commitTxn(); checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 9 where p = 'doh'", true)); txnMgr2.acquireLocks(driver.getPlan(), ctx, "Catherine"); txnMgr2.commitTxn(); } /**
@Test public void testLockRetryLimit() throws Exception { dropTable(new String[] {"T9"}); conf.setIntVar(HiveConf.ConfVars.HIVE_LOCK_NUMRETRIES, 2); conf.setBoolVar(HiveConf.ConfVars.TXN_MGR_DUMP_LOCK_STATE_ON_ACQUIRE_TIMEOUT, true); CommandProcessorResponse cpr = driver.run("create table T9(a int)"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("select * from T9", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Vincent Vega"); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T9", null, locks); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); swapTxnManager(txnMgr2); cpr = driver.compileAndRespond("drop table T9", true); checkCmdOnDriver(cpr); try { txnMgr2.acquireLocks(driver.getPlan(), ctx, "Winston Winnfield"); } catch(LockException ex) { Assert.assertEquals("Got wrong lock exception", ErrorMsg.LOCK_ACQUIRE_TIMEDOUT, ex.getCanonicalErrorMsg()); } locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T9", null, locks); txnMgr2.closeTxnManager(); }
/** * see also {@link org.apache.hadoop.hive.ql.TestTxnAddPartition} */ @Test public void testAddPartitionLocks() throws Exception { dropTable(new String[] {"T", "Tstage"}); CommandProcessorResponse cpr = driver.run("create table T (a int, b int) partitioned by (p int) " + "stored as orc tblproperties('transactional'='true')"); checkCmdOnDriver(cpr); //bucketed just so that we get 2 files cpr = driver.run("create table Tstage (a int, b int) clustered by (a) into 2 " + "buckets stored as orc tblproperties('transactional'='false')"); checkCmdOnDriver(cpr); cpr = driver.run("insert into Tstage values(0,2),(1,4)"); checkCmdOnDriver(cpr); String exportLoc = exportFolder.newFolder("1").toString(); cpr = driver.run("export table Tstage to '" + exportLoc + "'"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("ALTER TABLE T ADD if not exists PARTITION (p=0)" + " location '" + exportLoc + "/data'", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");//gets X lock on T List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T", null, locks); } @Test
private void insertOverwriteCreate(boolean isTransactional) throws Exception { if(isTransactional) { MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID, true); } dropTable(new String[] {"T2", "T3"}); CommandProcessorResponse cpr = driver.run("create table if not exists T2(a int)"); checkCmdOnDriver(cpr); cpr = driver.run("create table T3(a int) stored as ORC"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("insert overwrite table T3 select a from T2", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T2", null, locks); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T3", null, locks); txnMgr.commitTxn(); Assert.assertEquals("Lock remained", 0, getLocks().size()); cpr = driver.run("drop table if exists T1"); checkCmdOnDriver(cpr); cpr = driver.run("drop table if exists T2"); checkCmdOnDriver(cpr); } @Test
private void insertOverwritePartitionedCreate(boolean isTransactional) throws Exception { dropTable(new String[] {"T4", "T5"}); if(isTransactional) { MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID, true); } CommandProcessorResponse cpr = driver.run("create table T4" + "(name string, gpa double) partitioned by (age int) stored as ORC"); checkCmdOnDriver(cpr); cpr = driver.run("create table T5(name string, age int, gpa double)"); checkCmdOnDriver(cpr); cpr = driver.compileAndRespond("INSERT OVERWRITE TABLE T4 PARTITION (age) SELECT " + "name, age, gpa FROM T5", true); checkCmdOnDriver(cpr); txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer"); List<ShowLocksResponseElement> locks = getLocks(); Assert.assertEquals("Unexpected lock count", 2, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T5", null, locks); checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "T4", null, locks); txnMgr.commitTxn(); Assert.assertEquals("Lock remained", 0, getLocks().size()); cpr = driver.run("drop table if exists T5"); checkCmdOnDriver(cpr); cpr = driver.run("drop table if exists T4"); checkCmdOnDriver(cpr); } @Test