runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(new int[][]{{1, 2}, {3, 4}})); List<String> explain; String query = "update " + Table.ACIDTBL + " set b = 5 where a = 3"; if (enablePPD) { explain = runStatementOnDriver("explain " + query); assertExplainHasString("filterExpr: (a = 3)", explain, "PPD wasn't pushed"); runStatementOnDriver(query); query = "select a,b from " + Table.ACIDTBL + " where b = 4 order by a,b"; if (enablePPD) { explain = runStatementOnDriver("explain " + query); assertExplainHasString("filterExpr: (b = 4)", explain, "PPD wasn't pushed"); List<String> rs0 = runStatementOnDriver(query); Assert.assertEquals("Read failed", 0, rs0.size()); runStatementOnDriver("alter table " + Table.ACIDTBL + " compact 'MAJOR'"); runWorker(hiveConf); runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData2)); explain = runStatementOnDriver("explain delete from " + Table.ACIDTBL + " where a=7 and b=8"); assertExplainHasString("filterExpr: ((a = 7) and (b = 8))", explain, "PPD wasn't pushed"); runStatementOnDriver("delete from " + Table.ACIDTBL + " where a=7 and b=8"); explain = runStatementOnDriver("explain " + query); assertExplainHasString("filterExpr: (a > 1)", explain, "PPD wasn't pushed");
@Test public void testMergeWithPredicate() throws Exception { int[][] baseValsOdd = {{2,2},{5,5},{8,8},{11,11}}; runStatementOnDriver("insert into " + Table.NONACIDPART2 + " PARTITION(p2='odd') " + makeValuesClause(baseValsOdd)); int[][] vals = {{2,1},{4,3},{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(vals)); List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals(stringifyValues(vals), r); String query = "merge into " + Table.ACIDTBL + " t using " + Table.NONACIDPART2 + " s ON t.a = s.a2 " + "WHEN MATCHED AND t.b between 1 and 3 THEN UPDATE set b = s.b2 " + "WHEN NOT MATCHED and s.b2 >= 8 THEN INSERT VALUES(s.a2, s.b2)"; runStatementOnDriver(query); r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] rExpected = {{2,2},{4,3},{5,6},{7,8},{8,8},{11,11}}; Assert.assertEquals(stringifyValues(rExpected), r); assertUniqueID(Table.ACIDTBL); }
/** * Make sure there's no FileSystem$Cache$Key leak due to UGI use * @throws Exception */ @Test public void testFileSystemUnCaching() throws Exception { int cacheSizeBefore; int cacheSizeAfter; // get the size of cache BEFORE cacheSizeBefore = getFileSystemCacheSize(); // Insert a row to ACID table runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)"); // Perform a major compaction runStatementOnDriver("alter table " + Table.ACIDTBL + " compact 'major'"); runWorker(hiveConf); runCleaner(hiveConf); // get the size of cache AFTER cacheSizeAfter = getFileSystemCacheSize(); Assert.assertEquals(cacheSizeBefore, cacheSizeAfter); }
@Test public void testAlterTable() throws Exception { int[][] tableData = {{1,2}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'"); runWorker(hiveConf); int[][] tableData2 = {{5,6}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData2)); List<String> rs1 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " where b > 0 order by a,b"); runStatementOnDriver("alter table " + Table.ACIDTBL + " add columns(c int)"); int[][] moreTableData = {{7,8,9}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b,c) " + makeValuesClause(moreTableData)); List<String> rs0 = runStatementOnDriver("select a,b,c from " + Table.ACIDTBL + " where a > 0 order by a,b,c"); } // @Ignore("not needed but useful for testing")
@Test public void testUpdateMixedCase() throws Exception { int[][] tableData = {{1,2},{3,3},{5,3}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); runStatementOnDriver("update " + Table.ACIDTBL + " set B = 7 where A=1"); List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] updatedData = {{1,7},{3,3},{5,3}}; Assert.assertEquals("Update failed", stringifyValues(updatedData), rs); runStatementOnDriver("update " + Table.ACIDTBL + " set B = B + 1 where A=1"); List<String> rs2 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] updatedData2 = {{1,8},{3,3},{5,3}}; Assert.assertEquals("Update failed", stringifyValues(updatedData2), rs2); } @Test
/** * make sure Aborted txns don't red-flag a base_xxxx (HIVE-14350) */ @Test public void testNoHistory() throws Exception { int[][] tableData = {{1,2},{3,4}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, true); runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false); runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'"); runWorker(hiveConf); runCleaner(hiveConf); runStatementOnDriver("select count(*) from " + Table.ACIDTBL); }
void testInitiatorWithMultipleFailedCompactionsForVariousTblProperties(String tblProperties) throws Exception { String tblName = "hive12353"; runStatementOnDriver("drop table if exists " + tblName); runStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) " + " CLUSTERED BY(a) INTO 1 BUCKETS" + //currently ACID requires table to be bucketed " STORED AS ORC TBLPROPERTIES ( " + tblProperties + " )"); runStatementOnDriver("insert into " + tblName + " values(" + (i + 1) + ", 'foo'),(" + (i + 2) + ", 'bar'),(" + (i + 3) + ", 'baz')"); runWorker(hiveConf); init.run(); int numAttemptedCompactions = 1; checkCompactionState(new CompactionsByState(numAttemptedCompactions,numFailedCompactions,0,0,0,0,numFailedCompactions + numAttemptedCompactions), countCompacts(txnHandler)); compactionHistoryService.setConf(hiveConf); compactionHistoryService.run(); checkCompactionState(new CompactionsByState(numAttemptedCompactions,numFailedCompactions,0,0,0,0,numFailedCompactions + numAttemptedCompactions), countCompacts(txnHandler)); runWorker(hiveConf);//will fail txnHandler.compact(new CompactionRequest("default", tblName, CompactionType.MINOR)); runWorker(hiveConf);//will fail init.run(); numAttemptedCompactions++; init.run(); numAttemptedCompactions++; checkCompactionState(new CompactionsByState(numAttemptedCompactions,numFailedCompactions + 2,0,0,0,0,numFailedCompactions + 2 + numAttemptedCompactions), countCompacts(txnHandler)); checkCompactionState(new CompactionsByState( hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED),
public void testMmTableCompaction() throws Exception { runStatementOnDriver("insert into " + Table.MMTBL + "(a,b) values(1,2)"); runStatementOnDriver("insert into " + Table.MMTBL + "(a,b) values(3,4)"); verifyDirAndResult(2); runStatementOnDriver("alter table "+ Table.MMTBL + " compact 'MINOR'"); runWorker(hiveConf); verifyDirAndResult(2); runStatementOnDriver("insert into " + Table.MMTBL + "(a,b) values(5,6)"); hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false); verifyDirAndResult(3); runStatementOnDriver("alter table "+ Table.MMTBL + " compact 'MINOR'"); runWorker(hiveConf); verifyDirAndResult(2); runCleaner(hiveConf); verifyDirAndResult(2);
private void writeBetweenWorkerAndCleanerForVariousTblProperties(String tblProperties) throws Exception { String tblName = "hive12352"; runStatementOnDriver("drop table if exists " + tblName); runStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) " + " CLUSTERED BY(a) INTO 1 BUCKETS" + //currently ACID requires table to be bucketed " STORED AS ORC TBLPROPERTIES ( " + tblProperties + " )"); runStatementOnDriver("insert into " + tblName + " values(1, 'foo'),(2, 'bar'),(3, 'baz')"); runStatementOnDriver("update " + tblName + " set b = 'blah' where a = 3"); runWorker(hiveConf); runStatementOnDriver("delete from " + tblName + " where a = 1"); hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false); expected.add("3\tblah"); Assert.assertEquals("", expected, runStatementOnDriver("select a,b from " + tblName + " order by a")); runCleaner(hiveConf); runInitiator(hiveConf); runStatementOnDriver("select a,b from " + tblName + " order by a"));
int[][] data = {{1, 2, 10}, {3, 4, 11}, {5, 6, 12}}; runStatementOnDriver("insert into Tstage" + TestTxnCommands2.makeValuesClause(data)); TestTxnCommands2.stringifyValues(data), rs);
@Test public void testDynamicPartitionsMerge() throws Exception { d.destroy(); hiveConf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); runStatementOnDriver("insert into " + Table.ACIDTBLPART + " partition(p) values(1,1,'p1'),(2,2,'p1'),(3,3,'p1'),(4,4,'p2')"); List<String> r1 = runStatementOnDriver("select count(*) from " + Table.ACIDTBLPART); Assert.assertEquals("4", r1.get(0)); int[][] sourceVals = {{2,15},{4,44},{5,5},{11,11}}; runStatementOnDriver("insert into " + Table.NONACIDORCTBL + " " + makeValuesClause(sourceVals)); runStatementOnDriver("merge into " + Table.ACIDTBLPART + " using " + Table.NONACIDORCTBL + " as s ON " + Table.ACIDTBLPART + ".a = s.a " + "when matched then update set b = s.b " + "when not matched then insert values(s.a, s.b, 'new part')"); r1 = runStatementOnDriver("select p,a,b from " + Table.ACIDTBLPART + " order by p, a, b"); String result= r1.toString(); Assert.assertEquals("[new part\t5\t5, new part\t11\t11, p1\t1\t1, p1\t2\t15, p1\t3\t3, p2\t4\t44]", result); //note: inserts go into 'new part'... so this won't fail assertUniqueID(Table.ACIDTBLPART); } /**
protected void testACIDwithSchemaEvolutionForVariousTblProperties(String tblProperties) throws Exception { String tblName = "acidWithSchemaEvol"; int numBuckets = 1; runStatementOnDriver("drop table if exists " + tblName); runStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) " + " CLUSTERED BY(a) INTO " + numBuckets +" BUCKETS" + //currently ACID requires table to be bucketed " STORED AS ORC TBLPROPERTIES ( " + tblProperties + " )"); runStatementOnDriver("insert into " + tblName + " values(1, 'foo'),(2, 'bar'),(3, 'baz')"); runStatementOnDriver("update " + tblName + " set b = 'blah' where a = 3"); runStatementOnDriver("alter table " + tblName + " add columns(c int, d string)"); runStatementOnDriver("insert into " + tblName + " values(4, 'acid', 100, 'orc')," + "(5, 'llap', 200, 'tez')"); runStatementOnDriver("update " + tblName + " set d = 'hive' where a <= 3"); runStatementOnDriver("update " + tblName + " set c = 999 where a <= 3"); List<String> rs = runStatementOnDriver("select * from " + tblName + " order by a"); String[] expectedResult = { "1\tfoo\t999\thive", "2\tbar\t999\thive", "3\tblah\t999\thive", "4\tacid\t100\torc", "5\tllap\t200\ttez" }; Assert.assertEquals(Arrays.asList(expectedResult), rs); runStatementOnDriver("alter table "+ tblName + " compact 'MAJOR'"); runWorker(hiveConf); runCleaner(hiveConf); // Cleaner would remove the obsolete files. rs = runStatementOnDriver("select * from " + tblName + " order by a"); Assert.assertEquals(Arrays.asList(expectedResult), rs);
TestTxnCommands2.runWorker(hiveConf); rs = runStatementOnDriver("select ROW__ID, c1, c2, c3, INPUT__FILE__NAME from nobuckets order by INPUT__FILE__NAME, ROW__ID"); LOG.warn("after major compact"); assertExpectedFileSet(expectedFiles, getWarehouseDir() + "/nobuckets"); TestTxnCommands2.runCleaner(hiveConf); rs = runStatementOnDriver("select c1, c2, c3 from nobuckets order by c1, c2, c3"); int[][] result = {{0,0,17},{1,1,17},{2,2,2},{3,3,3}};
@Test public void testSimpleRead() throws Exception { hiveConf.setVar(HiveConf.ConfVars.HIVEFETCHTASKCONVERSION, "more"); int[][] tableData = {{1,2},{3,3}}; runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(tableData)); int[][] tableData2 = {{5,3}}; //this will cause next txn to be marked aborted but the data is still written to disk hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, true); runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(tableData2)); assert hiveConf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY) == null : "previous txn should've cleaned it"; //so now if HIVEFETCHTASKCONVERSION were to use a stale value, it would use a //ValidWriteIdList with HWM=MAX_LONG, i.e. include the data for aborted txn List<String> rs = runStatementOnDriver("select * from " + Table.ACIDTBL); Assert.assertEquals("Extra data", 2, rs.size()); } @Test
@Test public void testMultiInsert() throws Exception { runStatementOnDriver("create temporary table if not exists data1 (x int)"); runStatementOnDriver("insert into data1 values (1),(2),(1)"); d.destroy(); hiveConf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); d = new Driver(hiveConf); runStatementOnDriver(" from data1 " + "insert into " + Table.ACIDTBLPART + " partition(p) select 0, 0, 'p' || x " + "insert into " + Table.ACIDTBLPART + " partition(p='p1') select 0, 1"); /** * Using {@link BucketCodec.V0} the output * is missing 1 of the (p1,0,1) rows because they have the same ROW__ID and only differ by * StatementId so {@link org.apache.hadoop.hive.ql.io.orc.OrcRawRecordMerger} skips one. * With split update (and V0), the data is read correctly (insert deltas are now the base) but we still * should get duplicate ROW__IDs. */ List<String> r = runStatementOnDriver("select p,a,b from " + Table.ACIDTBLPART + " order by p, a, b"); Assert.assertEquals("[p1\t0\t0, p1\t0\t0, p1\t0\t1, p1\t0\t1, p1\t0\t1, p2\t0\t0]", r.toString()); assertUniqueID(Table.ACIDTBLPART); /** * this delete + select covers VectorizedOrcAcidRowBatchReader */ runStatementOnDriver("delete from " + Table.ACIDTBLPART); r = runStatementOnDriver("select p,a,b from " + Table.ACIDTBLPART + " order by p, a, b"); Assert.assertEquals("[]", r.toString()); } /**
d = new Driver(new QueryState.Builder().withHiveConf(hiveConf).nonIsolated().build(), null); d.setMaxRows(10000); dropTables(); runStatementOnDriver("create table " + Table.ACIDTBL + "(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES (" + tableProperties + ")"); runStatementOnDriver("create table " + Table.ACIDTBLPART + "(a int, b int) partitioned by (p string) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES (" + tableProperties + ")"); runStatementOnDriver("create table " + Table.NONACIDORCTBL + "(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='false')"); runStatementOnDriver("create table " + Table.NONACIDPART + "(a int, b int) partitioned by (p string) stored as orc TBLPROPERTIES ('transactional'='false')"); runStatementOnDriver("create table " + Table.NONACIDPART2 + "(a2 int, b2 int) partitioned by (p2 string) stored as orc TBLPROPERTIES ('transactional'='false')"); runStatementOnDriver("create table " + Table.ACIDNESTEDPART + "(a int, b int) partitioned by (p int, q int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES (" + tableProperties + ")"); runStatementOnDriver("create table " + Table.MMTBL + "(a int, b int) TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')");
protected String makeValuesClause(int[][] rows) { return TestTxnCommands2.makeValuesClause(rows); } public static void runWorker(HiveConf hiveConf) throws Exception {
@After public void tearDown() throws Exception { try { if (d != null) { dropTables(); d.close(); d.destroy(); d = null; } TxnDbUtil.cleanDb(hiveConf); } finally { FileUtils.deleteDirectory(new File(TEST_DATA_DIR)); } } @Test
TestTxnCommands2.runInitiator(hiveConf); TestTxnCommands2.runInitiator(hiveConf); Assert.assertEquals(TxnDbUtil.queryToString(hiveConf, "select * from TXNS"), 0, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from TXNS"));
@Test public void testDeleteIn() throws Exception { int[][] tableData = {{1,2},{3,2},{5,2},{1,3},{3,3},{5,3}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(1,7),(3,7)"); //todo: once multistatement txns are supported, add a test to run next 2 statements in a single txn runStatementOnDriver("delete from " + Table.ACIDTBL + " where a in(select a from " + Table.NONACIDORCTBL + ")"); runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) select a,b from " + Table.NONACIDORCTBL); List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] updatedData = {{1,7},{3,7},{5,2},{5,3}}; Assert.assertEquals("Bulk update failed", stringifyValues(updatedData), rs); runStatementOnDriver("update " + Table.ACIDTBL + " set b=19 where b in(select b from " + Table.NONACIDORCTBL + " where a = 3)"); List<String> rs2 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] updatedData2 = {{1,19},{3,19},{5,2},{5,3}}; Assert.assertEquals("Bulk update2 failed", stringifyValues(updatedData2), rs2); }