@Test public void testNonAcidInsert() throws Exception { runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(1,2)"); List<String> rs = runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL); runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(2,3)"); List<String> rs1 = runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL); }
@Test public void testAlterTable() throws Exception { int[][] tableData = {{1,2}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'"); runWorker(hiveConf); int[][] tableData2 = {{5,6}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData2)); List<String> rs1 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " where b > 0 order by a,b"); runStatementOnDriver("alter table " + Table.ACIDTBL + " add columns(c int)"); int[][] moreTableData = {{7,8,9}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b,c) " + makeValuesClause(moreTableData)); List<String> rs0 = runStatementOnDriver("select a,b,c from " + Table.ACIDTBL + " where a > 0 order by a,b,c"); } // @Ignore("not needed but useful for testing")
protected void dropTables() throws Exception { for(Table t : Table.values()) { runStatementOnDriver("drop table if exists " + t); } } @After
@Test public void testAcidWithSchemaEvolution() throws Exception { hiveConf.setVar(HiveConf.ConfVars.HIVE_ORC_SPLIT_STRATEGY, "ETL"); String tblName = "acidTblWithSchemaEvol"; runStatementOnDriver("drop table if exists " + tblName); runStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) " + " CLUSTERED BY(a) INTO 2 BUCKETS" + //currently ACID requires table to be bucketed " STORED AS ORC TBLPROPERTIES ('transactional'='true')"); runStatementOnDriver("INSERT INTO " + tblName + " VALUES (1, 'foo'), (2, 'bar')"); // Major compact to create a base that has ACID schema. runStatementOnDriver("ALTER TABLE " + tblName + " COMPACT 'MAJOR'"); runWorker(hiveConf); // Alter table for perform schema evolution. runStatementOnDriver("ALTER TABLE " + tblName + " ADD COLUMNS(c int)"); // Validate there is an added NULL for column c. List<String> rs = runStatementOnDriver("SELECT * FROM " + tblName + " ORDER BY a"); String[] expectedResult = { "1\tfoo\tNULL", "2\tbar\tNULL" }; Assert.assertEquals(Arrays.asList(expectedResult), rs); } /**
@Test public void testDeleteIn() throws Exception { int[][] tableData = {{1,2},{3,2},{5,2},{1,3},{3,3},{5,3}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(1,7),(3,7)"); //todo: once multistatement txns are supported, add a test to run next 2 statements in a single txn runStatementOnDriver("delete from " + Table.ACIDTBL + " where a in(select a from " + Table.NONACIDORCTBL + ")"); runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) select a,b from " + Table.NONACIDORCTBL); List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] updatedData = {{1,7},{3,7},{5,2},{5,3}}; Assert.assertEquals("Bulk update failed", stringifyValues(updatedData), rs); runStatementOnDriver("update " + Table.ACIDTBL + " set b=19 where b in(select b from " + Table.NONACIDORCTBL + " where a = 3)"); List<String> rs2 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] updatedData2 = {{1,19},{3,19},{5,2},{5,3}}; Assert.assertEquals("Bulk update2 failed", stringifyValues(updatedData2), rs2); }
@Test public void testUpdateMixedCase() throws Exception { int[][] tableData = {{1,2},{3,3},{5,3}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); runStatementOnDriver("update " + Table.ACIDTBL + " set B = 7 where A=1"); List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] updatedData = {{1,7},{3,3},{5,3}}; Assert.assertEquals("Update failed", stringifyValues(updatedData), rs); runStatementOnDriver("update " + Table.ACIDTBL + " set B = B + 1 where A=1"); List<String> rs2 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] updatedData2 = {{1,8},{3,3},{5,3}}; Assert.assertEquals("Update failed", stringifyValues(updatedData2), rs2); } @Test
final void assertUniqueID(Table table) throws Exception { String partCols = table.getPartitionColumns(); //check to make sure there are no duplicate ROW__IDs - HIVE-16832 StringBuilder sb = new StringBuilder("select "); if(partCols != null && partCols.length() > 0) { sb.append(partCols).append(","); } sb.append(" ROW__ID, count(*) from ").append(table).append(" group by "); if(partCols != null && partCols.length() > 0) { sb.append(partCols).append(","); } sb.append("ROW__ID having count(*) > 1"); List<String> r = runStatementOnDriver(sb.toString()); Assert.assertTrue("Duplicate ROW__ID: " + r.toString(),r.size() == 0); }
/** * https://issues.apache.org/jira/browse/HIVE-10151 */ @Test public void testBucketizedInputFormat() throws Exception { int[][] tableData = {{1,2}}; runStatementOnDriver("insert into " + Table.ACIDTBLPART + " partition(p=1) (a,b) " + makeValuesClause(tableData)); runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) select a,b from " + Table.ACIDTBLPART + " where p = 1"); List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL);//no order by as it's just 1 row Assert.assertEquals("Insert into " + Table.ACIDTBL + " didn't match:", stringifyValues(tableData), rs); runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) select a,b from " + Table.ACIDTBLPART + " where p = 1"); List<String> rs2 = runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL);//no order by as it's just 1 row Assert.assertEquals("Insert into " + Table.NONACIDORCTBL + " didn't match:", stringifyValues(tableData), rs2); } @Test
/** * In current implementation of ACID, altering the value of transactional_properties or trying to * set a value for previously unset value for an acid table will throw an exception. * @throws Exception */ @Test public void testFailureOnAlteringTransactionalProperties() throws Exception { expectedException.expect(RuntimeException.class); expectedException.expectMessage("TBLPROPERTIES with 'transactional_properties' cannot be altered after the table is created"); runStatementOnDriver("create table acidTblLegacy (a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='true')"); runStatementOnDriver("alter table acidTblLegacy SET TBLPROPERTIES ('transactional_properties' = 'insert_only')"); } /**
@Ignore("Covered elsewhere") @Test public void testMergeAliasedTarget() throws Exception { int[][] baseValsOdd = {{2,2},{4,44},{5,5},{11,11}}; runStatementOnDriver("insert into " + Table.NONACIDORCTBL + " " + makeValuesClause(baseValsOdd)); int[][] vals = {{2,1},{4,3},{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(vals)); String query = "merge into " + Table.ACIDTBL + " as target using " + Table.NONACIDORCTBL + " source ON target.a = source.a " + "WHEN MATCHED THEN update set b = 0 " + "WHEN NOT MATCHED THEN INSERT VALUES(source.a, source.b) "; runStatementOnDriver(query); List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] rExpected = {{2,0},{4,0},{5,0},{7,8},{11,11}}; Assert.assertEquals(stringifyValues(rExpected), r); }
@Test @Ignore("Values clause with table constructor not yet supported") public void testValuesSource() throws Exception { int[][] targetVals = {{2,1},{4,3},{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(targetVals)); String query = "merge into " + Table.ACIDTBL + " as t using (select * from (values (2,2),(4,44),(5,5),(11,11)) as F(a,b)) s ON t.a = s.a " + "WHEN MATCHED and s.a < 5 THEN DELETE " + "WHEN MATCHED AND s.a < 3 THEN update set b = 0 " + "WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b) "; runStatementOnDriver(query); List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] rExpected = {{5,6},{7,8},{11,11}}; Assert.assertEquals(stringifyValues(rExpected), r); }
@Test public void testETLSplitStrategyForACID() throws Exception { hiveConf.setVar(HiveConf.ConfVars.HIVE_ORC_SPLIT_STRATEGY, "ETL"); hiveConf.setBoolVar(HiveConf.ConfVars.HIVEOPTINDEXFILTER, true); runStatementOnDriver("insert into " + Table.ACIDTBL + " values(1,2)"); runStatementOnDriver("alter table " + Table.ACIDTBL + " compact 'MAJOR'"); runWorker(hiveConf); List<String> rs = runStatementOnDriver("select * from " + Table.ACIDTBL + " where a = 1"); int[][] resultData = new int[][] {{1,2}}; Assert.assertEquals(stringifyValues(resultData), rs); }
/** * make sure Aborted txns don't red-flag a base_xxxx (HIVE-14350) */ @Test public void testNoHistory() throws Exception { int[][] tableData = {{1,2},{3,4}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, true); runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false); runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'"); runWorker(hiveConf); runCleaner(hiveConf); runStatementOnDriver("select count(*) from " + Table.ACIDTBL); }
@Test public void testMergeWithPredicate() throws Exception { int[][] baseValsOdd = {{2,2},{5,5},{8,8},{11,11}}; runStatementOnDriver("insert into " + Table.NONACIDPART2 + " PARTITION(p2='odd') " + makeValuesClause(baseValsOdd)); int[][] vals = {{2,1},{4,3},{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(vals)); List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals(stringifyValues(vals), r); String query = "merge into " + Table.ACIDTBL + " t using " + Table.NONACIDPART2 + " s ON t.a = s.a2 " + "WHEN MATCHED AND t.b between 1 and 3 THEN UPDATE set b = s.b2 " + "WHEN NOT MATCHED and s.b2 >= 8 THEN INSERT VALUES(s.a2, s.b2)"; runStatementOnDriver(query); r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] rExpected = {{2,2},{4,3},{5,6},{7,8},{8,8},{11,11}}; Assert.assertEquals(stringifyValues(rExpected), r); assertUniqueID(Table.ACIDTBL); }
private void verifyDirAndResult(int expectedDeltas) throws Exception { FileSystem fs = FileSystem.get(hiveConf); // Verify the content of subdirs FileStatus[] status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" + (Table.MMTBL).toString().toLowerCase()), FileUtils.HIDDEN_FILES_PATH_FILTER); int sawDeltaTimes = 0; for (int i = 0; i < status.length; i++) { Assert.assertTrue(status[i].getPath().getName().matches("delta_.*")); sawDeltaTimes++; FileStatus[] files = fs.listStatus(status[i].getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER); Assert.assertEquals(1, files.length); Assert.assertTrue(files[0].getPath().getName().equals("000000_0")); } Assert.assertEquals(expectedDeltas, sawDeltaTimes); // Verify query result int [][] resultData = new int[][] {{1,2}, {3,4}}; List<String> rs = runStatementOnDriver("select a,b from " + Table.MMTBL); Assert.assertEquals(stringifyValues(resultData), rs); }
@Test public void testSimpleRead() throws Exception { hiveConf.setVar(HiveConf.ConfVars.HIVEFETCHTASKCONVERSION, "more"); int[][] tableData = {{1,2},{3,3}}; runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(tableData)); int[][] tableData2 = {{5,3}}; //this will cause next txn to be marked aborted but the data is still written to disk hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, true); runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(tableData2)); assert hiveConf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY) == null : "previous txn should've cleaned it"; //so now if HIVEFETCHTASKCONVERSION were to use a stale value, it would use a //ValidWriteIdList with HWM=MAX_LONG, i.e. include the data for aborted txn List<String> rs = runStatementOnDriver("select * from " + Table.ACIDTBL); Assert.assertEquals("Extra data", 2, rs.size()); } @Test
@Test public void testValidTxnsBookkeeping() throws Exception { // 1. Run a query against a non-ACID table, and we shouldn't have txn logged in conf runStatementOnDriver("select * from " + Table.NONACIDORCTBL); String value = hiveConf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY); Assert.assertNull("The entry should be null for query that doesn't involve ACID tables", value); }
/** * https://issues.apache.org/jira/browse/HIVE-17391 */ @Test public void testEmptyInTblproperties() throws Exception { runStatementOnDriver("create table t1 " + "(a int, b int) stored as orc TBLPROPERTIES ('serialization.null.format'='', 'transactional'='true')"); runStatementOnDriver("insert into t1 " + "(a,b) values(1,7),(3,7)"); runStatementOnDriver("update t1" + " set b = -2 where b = 2"); runStatementOnDriver("alter table t1 " + " compact 'MAJOR'"); runWorker(hiveConf); TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals("Unexpected number of compactions in history", 1, resp.getCompactsSize()); Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(0).getState()); Assert.assertTrue(resp.getCompacts().get(0).getHadoopJobId().startsWith("job_local")); }
@Test public void testCompactWithDelete() throws Exception { int[][] tableData = {{1,2},{3,4}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'"); runWorker(hiveConf); runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 4"); runStatementOnDriver("update " + Table.ACIDTBL + " set b = -2 where b = 2"); runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MINOR'"); runWorker(hiveConf); TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals("Unexpected number of compactions in history", 2, resp.getCompactsSize()); Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(0).getState()); Assert.assertEquals("Unexpected 1 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(1).getState()); }
/** * Simulate the scenario when a heartbeat failed due to client errors such as no locks or no txns being found. * When a heartbeat fails, the query should be failed too. * @throws Exception */ @Test public void testFailHeartbeater() throws Exception { // Fail heartbeater, so that we can get a RuntimeException from the query. // More specifically, it's the original IOException thrown by either MR's or Tez's progress monitoring loop. hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILHEARTBEATER, true); Exception exception = null; try { runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(new int[][]{{1, 2}, {3, 4}})); } catch (RuntimeException e) { exception = e; } Assert.assertNotNull(exception); Assert.assertTrue(exception.getMessage().contains("HIVETESTMODEFAILHEARTBEATER=true")); }