public static IDriver newDriver(QueryState queryState, String userName, QueryInfo queryInfo) { boolean enabled = queryState.getConf().getBoolVar(ConfVars.HIVE_QUERY_REEXECUTION_ENABLED); if (!enabled) { return new Driver(queryState, userName, queryInfo); } String strategies = queryState.getConf().getVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES); strategies = Strings.nullToEmpty(strategies).trim().toLowerCase(); ArrayList<IReExecutionPlugin> plugins = new ArrayList<>(); for (String string : strategies.split(",")) { if (string.trim().isEmpty()) { continue; } plugins.add(buildReExecPlugin(string)); } return new ReExecDriver(queryState, userName, queryInfo, plugins); }
private void doIndexUpdate(List<Index> tblIndexes) throws HiveException { Driver driver = new Driver(this.conf); for (Index idx : tblIndexes) { StringBuilder sb = new StringBuilder(); sb.append("ALTER INDEX "); sb.append(idx.getIndexName()); sb.append(" ON "); sb.append(idx.getDbName()).append('.'); sb.append(idx.getOrigTableName()); sb.append(" REBUILD"); driver.compile(sb.toString(), false); tasks.addAll(driver.getPlan().getRootTasks()); inputs.addAll(driver.getPlan().getInputs()); } }
private Driver createDriver() throws IllegalAccessException, ClassNotFoundException, InstantiationException { SessionState.start(conf); Driver driver = new Driver(conf); return driver; }
public ReExecDriver(QueryState queryState, String userName, QueryInfo queryInfo, ArrayList<IReExecutionPlugin> plugins) { this.queryState = queryState; coreDriver = new Driver(queryState, userName, queryInfo, null); coreDriver.getHookRunner().addSemanticAnalyzerHook(new HandleReOptimizationExplain()); this.plugins = plugins; for (IReExecutionPlugin p : plugins) { p.initialize(coreDriver); } }
private void initDriver(HiveConf conf, int threadCount) throws Exception { driver = Mockito.spy(new Driver(conf)); resetParallelCompilationLimit(conf); AtomicInteger count = new AtomicInteger(threadCount); Mockito.doAnswer(invocation -> { Thread.sleep(500); verifyThatWaitingCompileOpsCountIsEqualTo(count.decrementAndGet()); return null; }).when(driver).compile(eq(SHORT_QUERY), eq(true), eq(false)); Mockito.doAnswer(invocation -> { Thread.sleep(5000); verifyThatWaitingCompileOpsCountIsEqualTo(count.decrementAndGet()); return null; }).when(driver).compile(eq(LONG_QUERY), eq(true), eq(false)); }
private static Driver createDriver(HiveConf conf) { HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); SessionState.start(conf); Driver driver = new Driver(conf); return driver; }
public void runWithinSession(Consumer<Driver> driverConsumer) { final HiveConf hiveConf = new HiveConf(SessionState.class); driverConf.forEach(hiveConf::set); SessionState ss = new SessionState(hiveConf); try (Closeable ssClose = ss::close) { SessionState.start(ss); driverConsumer.accept(new Driver(hiveConf)); } catch (IOException e) { throw new RuntimeException("Exception was thrown while closing SessionState", e); } }
private Driver createDriver() { HiveConf conf = new HiveConf(Driver.class); conf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); SessionState.start(conf); Driver driver = new Driver(conf); return driver; }
public static Task<?> createRootTask( HiveConf builderConf, Set<ReadEntity> inputs, Set<WriteEntity> outputs, StringBuilder command, LinkedHashMap<String, String> partSpec, String indexTableName, String dbName){ // Don't try to index optimize the query to build the index HiveConf.setBoolVar(builderConf, HiveConf.ConfVars.HIVEOPTINDEXFILTER, false); Driver driver = new Driver(builderConf, SessionState.get().getUserName()); driver.compile(command.toString(), false); Task<?> rootTask = driver.getPlan().getRootTasks().get(0); inputs.addAll(driver.getPlan().getInputs()); outputs.addAll(driver.getPlan().getOutputs()); IndexMetadataChangeWork indexMetaChange = new IndexMetadataChangeWork(partSpec, indexTableName, dbName); IndexMetadataChangeTask indexMetaChangeTsk = (IndexMetadataChangeTask) TaskFactory.get(indexMetaChange, builderConf); indexMetaChangeTsk.setWork(indexMetaChange); rootTask.addDependentTask(indexMetaChangeTsk); driver.destroy(); return rootTask; }
/** * Create driver with the test hook set in config */ private static Driver createDriver() { HiveConf conf = new HiveConf(Driver.class); conf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); conf.setVar(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK, CheckInputReadEntityDirect.class.getName()); HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); SessionState.start(conf); Driver driver = new Driver(conf); return driver; }
@BeforeClass public static void onetimeSetup() throws Exception { HiveConf conf = new HiveConf(Driver.class); conf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); conf.setVar(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK, CheckInputReadEntity.class.getName()); HiveConf .setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); SessionState.start(conf); driver = new Driver(conf); }
private static Driver createDriver() { HiveConf conf = new HiveConf(Driver.class); conf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS, true); SessionState.start(conf); Driver driver = new Driver(conf); return driver; }
@Override public void run() { SessionState ss = SessionState.start(hiveConf); try { ss.applyAuthorizationPolicy(); } catch (HiveException e) { throw new RuntimeException(e); } QueryState qs = new QueryState.Builder().withHiveConf(hiveConf).nonIsolated().build(); Driver d = new Driver(qs, null); try { LOG.info("Ready to run the query: " + query); syncThreadStart(cdlIn, cdlOut); try { CommandProcessorResponse cpr = d.run(query); if(cpr.getResponseCode() != 0) { throw new RuntimeException(query + " failed: " + cpr); } d.getResults(new ArrayList<String>()); } catch (Exception e) { throw new RuntimeException(e); } } finally { d.close(); } } }
@Test public void testInsertOverwriteToAcidWithUnionRemove() throws Exception { hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_UNION_REMOVE, true); hiveConf.setVar(HiveConf.ConfVars.HIVEFETCHTASKCONVERSION, "none"); d.close(); d = new Driver(hiveConf); int[][] values = {{1, 2}, {3, 4}, {5, 6}, {7, 8}, {9, 10}}; runStatementOnDriver("drop table if exists T"); runStatementOnDriver("create table T (a int, b int) stored as ORC TBLPROPERTIES ('transactional'='true')"); CommandProcessorResponse cpr = runStatementOnDriverNegative( "insert overwrite table T select a, b from " + TxnCommandsBaseForTests.Table.ACIDTBL + " where a between 1 and 3 group by a, b union all select a, b from " + TxnCommandsBaseForTests.Table.ACIDTBL + " where a between 5 and 7 union all select a, b from " + TxnCommandsBaseForTests.Table.ACIDTBL + " where a >= 9"); Assert.assertTrue("", cpr.getErrorMessage().contains("not supported due to OVERWRITE and UNION ALL")); } /**
@Test public void testMultiInsert() throws Exception { runStatementOnDriver("create temporary table if not exists data1 (x int)"); runStatementOnDriver("insert into data1 values (1),(2),(1)"); d.destroy(); hiveConf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); d = new Driver(hiveConf); runStatementOnDriver(" from data1 " + "insert into " + Table.ACIDTBLPART + " partition(p) select 0, 0, 'p' || x " + "insert into " + Table.ACIDTBLPART + " partition(p='p1') select 0, 1"); /** * Using {@link BucketCodec.V0} the output * is missing 1 of the (p1,0,1) rows because they have the same ROW__ID and only differ by * StatementId so {@link org.apache.hadoop.hive.ql.io.orc.OrcRawRecordMerger} skips one. * With split update (and V0), the data is read correctly (insert deltas are now the base) but we still * should get duplicate ROW__IDs. */ List<String> r = runStatementOnDriver("select p,a,b from " + Table.ACIDTBLPART + " order by p, a, b"); Assert.assertEquals("[p1\t0\t0, p1\t0\t0, p1\t0\t1, p1\t0\t1, p1\t0\t1, p2\t0\t0]", r.toString()); assertUniqueID(Table.ACIDTBLPART); /** * this delete + select covers VectorizedOrcAcidRowBatchReader */ runStatementOnDriver("delete from " + Table.ACIDTBLPART); r = runStatementOnDriver("select p,a,b from " + Table.ACIDTBLPART + " order by p, a, b"); Assert.assertEquals("[]", r.toString()); } /**
@Before public void setUp() throws Exception { SessionState.start(conf); ctx = new Context(conf); driver = new Driver(new QueryState.Builder().withHiveConf(conf).nonIsolated().build(), null); driver2 = new Driver(new QueryState.Builder().withHiveConf(conf).build(), null); TxnDbUtil.cleanDb(conf); TxnDbUtil.prepDb(conf); SessionState ss = SessionState.get(); ss.initTxnMgr(conf); txnMgr = ss.getTxnMgr(); Assert.assertTrue(txnMgr instanceof DbTxnManager); txnHandler = TxnUtils.getTxnStore(conf); } @After
private void runSparkTestSession(HiveConf conf, int threadId) throws Exception { conf.setVar(HiveConf.ConfVars.SPARK_SESSION_TIMEOUT, "10s"); conf.setVar(HiveConf.ConfVars.SPARK_SESSION_TIMEOUT_PERIOD, "1s"); Driver driver = null; try { driver = new Driver(new QueryState.Builder() .withGenerateNewQueryId(true) .withHiveConf(conf).build(), null, null); SparkSession sparkSession = SparkUtilities.getSparkSession(conf, SparkSessionManagerImpl.getInstance()); Assert.assertEquals(0, driver.run("show tables").getResponseCode()); barrier.await(); SparkContext sparkContext = getSparkContext(sparkSession); Assert.assertFalse(sparkContext.isStopped()); if(threadId == 1) { barrier.await(); closeSparkSession(sparkSession); Assert.assertTrue(sparkContext.isStopped()); } else { closeSparkSession(sparkSession); Assert.assertFalse(sparkContext.isStopped()); barrier.await(); } } finally { if (driver != null) { driver.destroy(); } } }
@Test public void testMoreBucketsThanReducers() throws Exception { //see bucket_num_reducers.q bucket_num_reducers2.q // todo: try using set VerifyNumReducersHook.num.reducers=10; d.destroy(); HiveConf hc = new HiveConf(hiveConf); hc.setIntVar(HiveConf.ConfVars.MAXREDUCERS, 1); //this is used in multiple places, SemanticAnalyzer.getBucketingSortingDest() among others hc.setIntVar(HiveConf.ConfVars.HADOOPNUMREDUCERS, 1); hc.setBoolVar(HiveConf.ConfVars.HIVE_EXPLAIN_USER, false); d = new Driver(hc); d.setMaxRows(10000); runStatementOnDriver("insert into " + Table.ACIDTBL + " values(1,1)");//txn X write to bucket1 runStatementOnDriver("insert into " + Table.ACIDTBL + " values(0,0),(3,3)");// txn X + 1 write to bucket0 + bucket1 runStatementOnDriver("update " + Table.ACIDTBL + " set b = -1"); List<String> r = runStatementOnDriver("select * from " + Table.ACIDTBL + " order by a, b"); int[][] expected = {{0, -1}, {1, -1}, {3, -1}}; Assert.assertEquals(stringifyValues(expected), r); } @Ignore("Moved to Tez")
@Test public void testMergeUpdateDeleteNoCardCheck() throws Exception { d.destroy(); HiveConf hc = new HiveConf(hiveConf); hc.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, false); d = new Driver(hc); d.setMaxRows(10000); int[][] baseValsOdd = {{2,2},{4,44},{5,5},{11,11}}; runStatementOnDriver("insert into " + Table.NONACIDORCTBL + " " + makeValuesClause(baseValsOdd)); int[][] vals = {{2,1},{4,3},{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(vals)); String query = "merge into " + Table.ACIDTBL + " as t using " + Table.NONACIDORCTBL + " s ON t.a = s.a " + "WHEN MATCHED AND s.a < 3 THEN update set b = 0 " + "WHEN MATCHED and t.a > 3 and t.a < 5 THEN DELETE "; runStatementOnDriver(query); List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); int[][] rExpected = {{2,0},{5,6},{7,8}}; Assert.assertEquals(stringifyValues(rExpected), r); } @Test
@Ignore("Moved to Tez") @Test public void testMoreBucketsThanReducers2() throws Exception { //todo: try using set VerifyNumReducersHook.num.reducers=10; //see bucket_num_reducers.q bucket_num_reducers2.q d.destroy(); HiveConf hc = new HiveConf(hiveConf); hc.setIntVar(HiveConf.ConfVars.MAXREDUCERS, 2); //this is used in multiple places, SemanticAnalyzer.getBucketingSortingDest() among others hc.setIntVar(HiveConf.ConfVars.HADOOPNUMREDUCERS, 2); d = new Driver(hc); d.setMaxRows(10000); runStatementOnDriver("create table fourbuckets (a int, b int) clustered by (a) into 4 buckets stored as orc TBLPROPERTIES ('transactional'='true')"); //below value for a is bucket id, for b - txn id (logically) runStatementOnDriver("insert into fourbuckets values(0,1),(1,1)");//txn X write to b0 + b1 runStatementOnDriver("insert into fourbuckets values(2,2),(3,2)");// txn X + 1 write to b2 + b3 runStatementOnDriver("insert into fourbuckets values(0,3),(1,3)");//txn X + 2 write to b0 + b1 runStatementOnDriver("insert into fourbuckets values(2,4),(3,4)");//txn X + 3 write to b2 + b3 //so with 2 FileSinks and 4 buckets, FS1 should see (0,1),(2,2),(0,3)(2,4) since data is sorted by ROW__ID where tnxid is the first component //FS2 should see (1,1),(3,2),(1,3),(3,4) runStatementOnDriver("update fourbuckets set b = -1"); List<String> r = runStatementOnDriver("select * from fourbuckets order by a, b"); int[][] expected = {{0, -1},{0, -1}, {1, -1}, {1, -1}, {2, -1}, {2, -1}, {3, -1}, {3, -1}}; Assert.assertEquals(stringifyValues(expected), r); } @Test