@Test public void exchangePartition() throws Exception { runStatementOnDriver("create database ex1"); runStatementOnDriver("create database ex2"); runStatementOnDriver("CREATE TABLE ex1.exchange_part_test1 (f1 string) PARTITIONED BY (ds STRING)"); runStatementOnDriver("CREATE TABLE ex2.exchange_part_test2 (f1 string) PARTITIONED BY (ds STRING)"); runStatementOnDriver("ALTER TABLE ex2.exchange_part_test2 ADD PARTITION (ds='2013-04-05')"); runStatementOnDriver("ALTER TABLE ex1.exchange_part_test1 EXCHANGE PARTITION (ds='2013-04-05') WITH TABLE ex2.exchange_part_test2"); } @Test
@Test public void testImplicitRollback() throws Exception { runStatementOnDriver("START TRANSACTION"); runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)"); List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Can't see my own write", 1, rs0.size()); //next command should produce an error CommandProcessorResponse cpr = runStatementOnDriverNegative("select * from no_such_table"); Assert.assertEquals("Txn didn't fail?", "FAILED: SemanticException [Error 10001]: Line 1:14 Table not found 'no_such_table'", cpr.getErrorMessage()); runStatementOnDriver("start transaction"); List<String> rs1 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); runStatementOnDriver("commit"); Assert.assertEquals("Didn't rollback as expected", 0, rs1.size()); } @Test
@Test public void testMergeCase() throws Exception { runStatementOnDriver("create table merge_test (c1 integer, c2 integer, c3 integer) CLUSTERED BY (c1) into 2 buckets stored as orc tblproperties(\"transactional\"=\"true\")"); runStatementOnDriver("create table if not exists e011_02 (c1 float, c2 double, c3 float)"); runStatementOnDriver("merge into merge_test using e011_02 on (merge_test.c1 = e011_02.c1) when not matched then insert values (case when e011_02.c1 > 0 then e011_02.c1 + 1 else e011_02.c1 end, e011_02.c2 + e011_02.c3, coalesce(e011_02.c3, 1))"); } /**
@Test public void testSimpleAcidInsert() throws Exception { int[][] rows1 = {{1,2},{3,4}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1)); //List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); //Assert.assertEquals("Data didn't match in autocommit=true (rs)", stringifyValues(rows1), rs); runStatementOnDriver("START TRANSACTION"); int[][] rows2 = {{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows2)); List<String> allData = stringifyValues(rows1); allData.addAll(stringifyValues(rows2)); List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Data didn't match inside tx (rs0)", allData, rs0); runStatementOnDriver("COMMIT WORK"); dumpTableData(Table.ACIDTBL, 1, 0); dumpTableData(Table.ACIDTBL, 2, 0); runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); CommandProcessorResponse cpr = runStatementOnDriverNegative("COMMIT");//txn started implicitly by previous statement Assert.assertEquals("Error didn't match: " + cpr, ErrorMsg.OP_NOT_ALLOWED_WITHOUT_TXN.getErrorCode(), cpr.getErrorCode()); List<String> rs1 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Data didn't match inside tx (rs0)", allData, rs1); }
@Test public void testQuotedIdentifier2() throws Exception { String target = "`aci/d_u/ami`"; String src = "`src/name`"; runStatementOnDriver("drop table if exists " + target); runStatementOnDriver("drop table if exists " + src); runStatementOnDriver("create table " + target + "(i int," + "`d?*de e` decimal(5,2)," + "vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); runStatementOnDriver("create table " + src + "(`g/h` int, j decimal(5,2), k varchar(128))"); runStatementOnDriver("merge into " + target + " as `d/8` using " + src + " as `a/b` on i=`g/h`" + "\nwhen matched and `g/h` > 5 then delete " + "\nwhen matched and `g/h` < 0 then update set vc='∆∋', `d?*de e` = `d?*de e` * j + 1" + "\nwhen not matched and `d?*de e` <> 0 then insert values(`a/b`.`g/h`,`a/b`.j,`a/b`.k)"); runStatementOnDriver("merge into " + target + " as `d/8` using " + src + " as `a/b` on i=`g/h`" + "\nwhen matched and `g/h` > 5 then delete" + "\n when matched and `g/h` < 0 then update set vc='∆∋' , `d?*de e` = `d?*de e` * j + 1 " + "\n when not matched and `d?*de e` <> 0 then insert values(`a/b`.`g/h`,`a/b`.j,`a/b`.k)"); } /**
@Ignore("not needed but useful for testing") @Test public void testNonAcidInsert() throws Exception { runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(1,2)"); List<String> rs = runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL); runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(2,3)"); List<String> rs1 = runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL); }
@Test public void testExplicitRollback() throws Exception { runStatementOnDriver("START TRANSACTION"); runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)"); runStatementOnDriver("ROLLBACK"); List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Rollback didn't rollback", 0, rs.size()); }
@Test public void testReadMyOwnInsert() throws Exception { runStatementOnDriver("START TRANSACTION"); List<String> rs = runStatementOnDriver("select * from " + Table.ACIDTBL); Assert.assertEquals("Expected empty " + Table.ACIDTBL, 0, rs.size()); runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)"); List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Can't see my own write", 1, rs0.size()); runStatementOnDriver("commit"); runStatementOnDriver("START TRANSACTION"); List<String> rs1 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); runStatementOnDriver("rollback work"); Assert.assertEquals("Can't see write after commit", 1, rs1.size()); } @Test
@Test public void testDeleteIn() throws Exception { runStatementOnDriver("delete from " + Table.ACIDTBL + " where a IN (SELECT A.a from " + Table.ACIDTBL + " A)"); int[][] tableData = {{1,2},{3,2},{5,2},{1,3},{3,3},{5,3}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); runStatementOnDriver("insert into " + Table.ACIDTBL2 + "(a,b,c) values(1,7,17),(3,7,17)"); // runStatementOnDriver("select b from " + Table.ACIDTBL + " where a in (select b from " + Table.NONACIDORCTBL + ")"); runStatementOnDriver("delete from " + Table.ACIDTBL + " where a in(select a from " + Table.ACIDTBL2 + ")"); // runStatementOnDriver("delete from " + Table.ACIDTBL + " where a in(select a from " + Table.NONACIDORCTBL + ")"); runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) select a,b from " + Table.ACIDTBL2); List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] updatedData = {{1,7},{3,7},{5,2},{5,3}}; Assert.assertEquals("Bulk update failed", stringifyValues(updatedData), rs); } @Test
@Test public void testDelete() throws Exception { int[][] rows1 = {{1,2},{3,4}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1)); List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Content didn't match rs0", stringifyValues(rows1), rs0); runStatementOnDriver("START TRANSACTION"); runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 4"); int[][] updatedData2 = {{1,2}}; List<String> rs3 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after delete", stringifyValues(updatedData2), rs3); runStatementOnDriver("commit"); List<String> rs4 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after commit", stringifyValues(updatedData2), rs4); }
private IMetaStoreClient prepareParallelTest(String tableName, int val) throws Exception, MetaException, TException, NoSuchObjectException { hiveConf.setBoolean("hive.stats.autogather", true); hiveConf.setBoolean("hive.stats.column.autogather", true); runStatementOnDriver("drop table if exists " + tableName); runStatementOnDriver(String.format("create table %s (a int) stored as orc " + "TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')", tableName)); runStatementOnDriver(String.format("insert into %s (a) values (" + val + ")", tableName)); runStatementOnDriver(String.format("insert into %s (a) values (" + val + ")", tableName)); IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf); // Stats should be valid after serial inserts. List<ColumnStatisticsObj> stats = getTxnTableStats(msClient, tableName); Assert.assertEquals(1, stats.size()); return msClient; }
@Test public void testMergeUpdateDelete() throws Exception { int[][] baseValsOdd = {{2,2},{4,44},{5,5},{11,11}}; runStatementOnDriver("insert into " + Table.NONACIDORCTBL + " " + makeValuesClause(baseValsOdd)); int[][] vals = {{2,1},{4,3},{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(vals)); String query = "merge into " + Table.ACIDTBL + " as t using " + Table.NONACIDORCTBL + " s ON t.a = s.a " + "WHEN MATCHED AND s.a < 3 THEN update set b = 0 " + //updates (2,1) -> (2,0) "WHEN MATCHED and t.a > 3 and t.a < 5 THEN DELETE " +//deletes (4,3) "WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b) ";//inserts (11,11) runStatementOnDriver(query); List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] rExpected = {{2,0},{5,6},{7,8},{11,11}}; Assert.assertEquals(stringifyValues(rExpected), r); } @Test
@Test public void testMergeDeleteUpdate() throws Exception { int[][] sourceVals = {{2,2},{4,44},{5,5},{11,11}}; runStatementOnDriver("insert into " + Table.NONACIDORCTBL + " " + makeValuesClause(sourceVals)); int[][] targetVals = {{2,1},{4,3},{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(targetVals)); String query = "merge into " + Table.ACIDTBL + " as t using " + Table.NONACIDORCTBL + " s ON t.a = s.a " + "WHEN MATCHED and s.a < 5 THEN DELETE " + "WHEN MATCHED AND s.a < 3 THEN update set b = 0 " + "WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b) "; runStatementOnDriver(query); List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] rExpected = {{5,6},{7,8},{11,11}}; Assert.assertEquals(stringifyValues(rExpected), r); }
/** * tests that a failing Insert Overwrite (which creates a new base_x) is properly marked as * aborted. */ @Test public void testInsertOverwrite() throws Exception { runStatementOnDriver("insert overwrite table " + Table.NONACIDORCTBL + " select a,b from " + Table.NONACIDORCTBL2); runStatementOnDriver("create table " + Table.NONACIDORCTBL2 + "3(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='false')"); runStatementOnDriver("insert into " + Table.ACIDTBL + " values(1,2)"); List<String> rs = runStatementOnDriver("select a from " + Table.ACIDTBL + " where b = 2"); Assert.assertEquals(1, rs.size()); Assert.assertEquals("1", rs.get(0)); hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, true); runStatementOnDriver("insert overwrite table " + Table.ACIDTBL + " values(3,2)"); hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false); runStatementOnDriver("insert into " + Table.ACIDTBL + " values(5,6)"); rs = runStatementOnDriver("select a from " + Table.ACIDTBL + " order by a"); Assert.assertEquals(2, rs.size()); Assert.assertEquals("1", rs.get(0)); Assert.assertEquals("5", rs.get(1)); }
@Test public void testMultipleInserts() throws Exception { runStatementOnDriver("START TRANSACTION"); int[][] rows1 = {{1,2},{3,4}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1)); int[][] rows2 = {{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows2)); List<String> allData = stringifyValues(rows1); allData.addAll(stringifyValues(rows2)); List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Content didn't match before commit rs", allData, rs); runStatementOnDriver("commit"); dumpTableData(Table.ACIDTBL, 1, 0); dumpTableData(Table.ACIDTBL, 1, 1); List<String> rs1 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Content didn't match after commit rs1", allData, rs1); } @Test
/** * Writing UTs that need multiple threads is challenging since Derby chokes on concurrent access. * This tests that "AND WAIT" actually blocks and responds to interrupt * @throws Exception */ @Test public void testCompactionBlocking() throws Exception { Timer cancelCompact = new Timer("CancelCompactionTimer", false); final Thread threadToInterrupt= Thread.currentThread(); cancelCompact.schedule(new TimerTask() { @Override public void run() { threadToInterrupt.interrupt(); } }, 5000); long start = System.currentTimeMillis(); runStatementOnDriver("alter table "+ TestTxnCommands2.Table.ACIDTBL +" compact 'major' AND WAIT"); //no Worker so it stays in initiated state //w/o AND WAIT the above alter table retunrs almost immediately, so the test here to check that //> 2 seconds pass, i.e. that the command in Driver actually blocks before cancel is fired Assert.assertTrue(System.currentTimeMillis() > start + 2); }
@Test public void testUpdateOfInserts() throws Exception { int[][] rows1 = {{1,2},{3,4}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1)); List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Content didn't match rs0", stringifyValues(rows1), rs0); runStatementOnDriver("START TRANSACTION"); int[][] rows2 = {{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows2)); List<String> rs1 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); List<String> allData = stringifyValues(rows1); allData.addAll(stringifyValues(rows2)); Assert.assertEquals("Content didn't match rs1", allData, rs1); runStatementOnDriver("update " + Table.ACIDTBL + " set b = 1 where b != 1"); int[][] updatedData = {{1,1},{3,1},{5,1},{7,1}}; List<String> rs2 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after update", stringifyValues(updatedData), rs2); runStatementOnDriver("commit"); List<String> rs4 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after commit", stringifyValues(updatedData), rs4); } @Test
@Test public void testMoreBucketsThanReducers() throws Exception { //see bucket_num_reducers.q bucket_num_reducers2.q // todo: try using set VerifyNumReducersHook.num.reducers=10; d.destroy(); HiveConf hc = new HiveConf(hiveConf); hc.setIntVar(HiveConf.ConfVars.MAXREDUCERS, 1); //this is used in multiple places, SemanticAnalyzer.getBucketingSortingDest() among others hc.setIntVar(HiveConf.ConfVars.HADOOPNUMREDUCERS, 1); hc.setBoolVar(HiveConf.ConfVars.HIVE_EXPLAIN_USER, false); d = new Driver(hc); d.setMaxRows(10000); runStatementOnDriver("insert into " + Table.ACIDTBL + " values(1,1)");//txn X write to bucket1 runStatementOnDriver("insert into " + Table.ACIDTBL + " values(0,0),(3,3)");// txn X + 1 write to bucket0 + bucket1 runStatementOnDriver("update " + Table.ACIDTBL + " set b = -1"); List<String> r = runStatementOnDriver("select * from " + Table.ACIDTBL + " order by a, b"); int[][] expected = {{0, -1}, {1, -1}, {3, -1}}; Assert.assertEquals(stringifyValues(expected), r); } @Ignore("Moved to Tez")
@Test public void testMergeUpdateDeleteNoCardCheck() throws Exception { d.destroy(); HiveConf hc = new HiveConf(hiveConf); hc.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, false); d = new Driver(hc); d.setMaxRows(10000); int[][] baseValsOdd = {{2,2},{4,44},{5,5},{11,11}}; runStatementOnDriver("insert into " + Table.NONACIDORCTBL + " " + makeValuesClause(baseValsOdd)); int[][] vals = {{2,1},{4,3},{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(vals)); String query = "merge into " + Table.ACIDTBL + " as t using " + Table.NONACIDORCTBL + " s ON t.a = s.a " + "WHEN MATCHED AND s.a < 3 THEN update set b = 0 " + "WHEN MATCHED and t.a > 3 and t.a < 5 THEN DELETE "; runStatementOnDriver(query); List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] rExpected = {{2,0},{5,6},{7,8}}; Assert.assertEquals(stringifyValues(rExpected), r); } @Test
@Test public void testParallelTruncateAnalyzeStats() throws Exception { String tableName = "mm_table"; List<ColumnStatisticsObj> stats; IMetaStoreClient msClient = prepareParallelTest(tableName, 0); String[] queries = { String.format("truncate table %s", tableName), String.format("analyze table %s compute statistics for columns", tableName) }; runParallelQueries(queries); // Verify stats are either invalid, or valid and correct. stats = getTxnTableStats(msClient, tableName); boolean hasStats = 0 != stats.size(); if (hasStats) { verifyLongStats(0, 0, 0, stats); } // Stats should be valid after analyze. runStatementOnDriver(String.format("analyze table %s compute statistics for columns", tableName)); verifyLongStats(0, 0, 0, getTxnTableStats(msClient, tableName)); }