private void checkQFileTestHack() { boolean hackOn = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST) || MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEZ_TEST); if (hackOn) { LOG.info("Hacking in canned values for transaction manager"); // Set up the transaction/locking db in the derby metastore TxnDbUtil.setConfValues(conf); try { TxnDbUtil.prepDb(conf); } catch (Exception e) { // We may have already created the tables and thus don't need to redo it. if (e.getMessage() != null && !e.getMessage().contains("already exists")) { throw new RuntimeException("Unable to set up transaction database for" + " testing: " + e.getMessage(), e); } } } }
public void prepareTransactionDatabase(HiveConf conf) throws Exception { TxnDbUtil.setConfValues(conf); TxnDbUtil.cleanDb(conf); TxnDbUtil.prepDb(conf); }
Statement stmt = null; try { conn = getConnection(conf); stmt = conn.createStatement(); stmt.execute("CREATE TABLE TXNS (" + prepDb(conf); } else { throw e; closeResources(conn, stmt, null);
/** * Utility method used to run COUNT queries like "select count(*) from ..." against metastore tables * @param countQuery countQuery text * @return count countQuery result * @throws Exception */ public static int countQueryAgent(Configuration conf, String countQuery) throws Exception { Connection conn = null; Statement stmt = null; ResultSet rs = null; try { conn = getConnection(conf); stmt = conn.createStatement(); rs = stmt.executeQuery(countQuery); if (!rs.next()) { return 0; } return rs.getInt(1); } finally { closeResources(conn, stmt, rs); } } @VisibleForTesting
Statement stmt = null; try { conn = getConnection(conf); stmt = conn.createStatement(); success &= dropTable(stmt, "TXN_COMPONENTS", retryCount); success &= dropTable(stmt, "COMPLETED_TXN_COMPONENTS", retryCount); success &= dropTable(stmt, "TXNS", retryCount); success &= dropTable(stmt, "NEXT_TXN_ID", retryCount); success &= dropTable(stmt, "TXN_TO_WRITE_ID", retryCount); success &= dropTable(stmt, "NEXT_WRITE_ID", retryCount); success &= dropTable(stmt, "MIN_HISTORY_LEVEL", retryCount); success &= dropTable(stmt, "HIVE_LOCKS", retryCount); success &= dropTable(stmt, "NEXT_LOCK_ID", retryCount); success &= dropTable(stmt, "COMPACTION_QUEUE", retryCount); success &= dropTable(stmt, "NEXT_COMPACTION_QUEUE_ID", retryCount); success &= dropTable(stmt, "COMPLETED_COMPACTIONS", retryCount); success &= dropTable(stmt, "AUX_TABLE", retryCount); success &= dropTable(stmt, "WRITE_SET", retryCount); success &= dropTable(stmt, "REPL_TXN_MAP", retryCount); success &= dropTable(stmt, "MATERIALIZATION_REBUILD_LOCKS", retryCount); closeResources(conn, stmt, null);
public TestHiveWriter() throws Exception { port = 9083; metaStoreURI = null; int callTimeoutPoolSize = 1; callTimeoutPool = Executors.newFixedThreadPool(callTimeoutPoolSize, new ThreadFactoryBuilder().setNameFormat("hiveWriterTest").build()); // 1) Start metastore conf = HiveSetupUtil.getHiveConf(); TxnDbUtil.setConfValues(conf); if (metaStoreURI != null) { conf.setVar(HiveConf.ConfVars.METASTOREURIS, metaStoreURI); } }
@Before public void setup() throws Exception { conf = new HiveConf(); TxnDbUtil.setConfValues(conf); TxnDbUtil.cleanDb(conf); ms = new HiveMetaStoreClient(conf); txnHandler = TxnUtils.getTxnStore(conf); tmpdir = new File(Files.createTempDirectory("compactor_test_table_").toString()); }
@Test public void testOperationsOnCompletedTxnComponentsForMmTable() throws Exception { // Insert two rows into the table. runStatementOnDriver("insert into " + TableExtended.MMTBL + "(a,b) values(1,2)"); runStatementOnDriver("insert into " + TableExtended.MMTBL + "(a,b) values(3,4)"); // There should be 2 delta directories verifyDirAndResult(2); Assert.assertEquals(TxnDbUtil.queryToString(hiveConf, "select * from COMPLETED_TXN_COMPONENTS"), 2, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from COMPLETED_TXN_COMPONENTS")); Assert.assertEquals(TxnDbUtil.queryToString(hiveConf, "select * from TXNS"), 0, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from TXNS")); // Initiate a minor compaction request on the table. runStatementOnDriver("alter table " + TableExtended.MMTBL + " compact 'MAJOR'"); // Run worker. runWorker(hiveConf); // Run Cleaner. runCleaner(hiveConf); Assert.assertEquals(TxnDbUtil.queryToString(hiveConf, "select * from COMPLETED_TXN_COMPONENTS"), 0, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from COMPLETED_TXN_COMPONENTS")); Assert.assertEquals(TxnDbUtil.queryToString(hiveConf, "select * from TXNS"), 0, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from TXNS")); }
@Before public void setUp() throws Exception { TxnDbUtil.prepDb(conf); txnHandler = TxnUtils.getTxnStore(conf); }
@Before public void setUp() throws Exception { SessionState.start(conf); ctx = new Context(conf); driver = new Driver(new QueryState.Builder().withHiveConf(conf).nonIsolated().build(), null); driver2 = new Driver(new QueryState.Builder().withHiveConf(conf).build(), null); TxnDbUtil.cleanDb(conf); TxnDbUtil.prepDb(conf); SessionState ss = SessionState.get(); ss.initTxnMgr(conf); txnMgr = ss.getTxnMgr(); Assert.assertTrue(txnMgr instanceof DbTxnManager); txnHandler = TxnUtils.getTxnStore(conf); } @After
@After public void tearDown() throws Exception { TxnDbUtil.cleanDb(conf); } }
@VisibleForTesting public static String queryToString(Configuration conf, String query) throws Exception { return queryToString(conf, query, true); } public static String queryToString(Configuration conf, String query, boolean includeHeader)
/** * A tool to count the number of partitions, tables, * and databases locked by a particular lockId. * * @param lockId lock id to look for lock components * * @return number of components, or 0 if there is no lock */ public static int countLockComponents(Configuration conf, long lockId) throws Exception { Connection conn = null; PreparedStatement stmt = null; ResultSet rs = null; try { conn = getConnection(conf); stmt = conn.prepareStatement("SELECT count(*) FROM hive_locks WHERE hl_lock_ext_id = ?"); stmt.setLong(1, lockId); rs = stmt.executeQuery(); if (!rs.next()) { return 0; } return rs.getInt(1); } finally { closeResources(conn, stmt, rs); } }
public static void cleanDb() throws Exception { Connection conn = null; Statement stmt = null; try { conn = getConnection(); stmt = conn.createStatement(); // We want to try these, whether they succeed or fail. try { stmt.execute("DROP INDEX HL_TXNID_INDEX"); } catch (Exception e) { System.err.println("Unable to drop index HL_TXNID_INDEX " + e.getMessage()); } dropTable(stmt, "TXN_COMPONENTS"); dropTable(stmt, "COMPLETED_TXN_COMPONENTS"); dropTable(stmt, "TXNS"); dropTable(stmt, "NEXT_TXN_ID"); dropTable(stmt, "HIVE_LOCKS"); dropTable(stmt, "NEXT_LOCK_ID"); dropTable(stmt, "COMPACTION_QUEUE"); dropTable(stmt, "NEXT_COMPACTION_QUEUE_ID"); conn.commit(); } finally { closeResources(conn, stmt, null); } }
public TestHiveBolt() throws Exception { //metaStoreURI = "jdbc:derby:;databaseName="+System.getProperty("java.io.tmpdir") +"metastore_db;create=true"; metaStoreURI = null; conf = HiveSetupUtil.getHiveConf(); TxnDbUtil.setConfValues(conf); if (metaStoreURI != null) { conf.setVar(HiveConf.ConfVars.METASTOREURIS, metaStoreURI); } }
Assert.assertEquals(0, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_TABLE='t'")); Assert.assertEquals(0, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from COMPACTION_QUEUE where CQ_TABLE='t'")); Assert.assertEquals(0, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from WRITE_SET where WS_TABLE='t'")); Assert.assertEquals(0, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from TXN_TO_WRITE_ID where T2W_TABLE='t'")); Assert.assertEquals(0, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from NEXT_WRITE_ID where NWI_TABLE='t'")); TxnDbUtil.queryToString(hiveConf, "select * from COMPLETED_TXN_COMPONENTS"), 2, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_TABLE='s'")); Assert.assertEquals(1, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from COMPACTION_QUEUE where CQ_TABLE='s'")); Assert.assertEquals(1, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from WRITE_SET where WS_TABLE='s'")); Assert.assertEquals(3, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from TXN_TO_WRITE_ID where T2W_TABLE='s'")); Assert.assertEquals(1, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from NEXT_WRITE_ID where NWI_TABLE='s'")); TxnDbUtil.queryToString(hiveConf, "select * from COMPLETED_TXN_COMPONENTS"), 2, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_TABLE='bar'")); Assert.assertEquals(1, TxnDbUtil.countQueryAgent(hiveConf, "select count(*) from COMPACTION_QUEUE where CQ_TABLE='bar'")); Assert.assertEquals(1, TxnDbUtil.countQueryAgent(hiveConf,
@Before public void setUp() throws Exception { TxnDbUtil.prepDb(conf); txnHandler = TxnUtils.getTxnStore(conf); }
@After public void tearDown() throws Exception { conn.close(); TxnDbUtil.cleanDb(conf); } }
private void checkReplTxnForTest(Long startTxnId, Long endTxnId, String replPolicy, List<Long> targetTxnId) throws Exception { String[] output = TxnDbUtil.queryToString(conf, "select RTM_TARGET_TXN_ID from REPL_TXN_MAP where " + " RTM_SRC_TXN_ID >= " + startTxnId + "and RTM_SRC_TXN_ID <= " + endTxnId + " and RTM_REPL_POLICY = \'" + replPolicy + "\'").split("\n"); assertEquals(output.length - 1, targetTxnId.size()); for (int idx = 1; idx < output.length; idx++) { long txnId = Long.parseLong(output[idx].trim()); assertEquals(txnId, targetTxnId.get(idx-1).longValue()); } }
@Before public void setUp() throws Exception { conf = MetastoreConf.newMetastoreConf(); TxnDbUtil.setConfValues(conf); TxnDbUtil.prepDb(conf); }