private void setHiveQueueName() throws SQLException { this.conn.executeStatements("set " + MAPREDUCE_JOB_QUEUENAME + "=" + CompactionRunner.jobProperties.getProperty(HIVE_QUEUE_NAME, "default")); }
HiveJdbcConnector hiveJdbcConnector = closer.register(HiveJdbcConnector.newConnectorWithProps(props)); for (String query : queries) { String hiveOutput = "hiveConversionValidationOutput_" + UUID.randomUUID().toString(); try { if (this.hiveSettings.size() > 0) { hiveJdbcConnector.executeStatements(this.hiveSettings.toArray(new String[this.hiveSettings.size()])); hiveJdbcConnector.executeStatements("SET hive.exec.compress.output=false","SET hive.auto.convert.join=false", query); FileStatus[] fileStatusList = this.fs.listStatus(hiveTempDir); List<FileStatus> files = new ArrayList<>();
@Override public void close() throws IOException { this.hiveJdbcConnector.close(); }
/** * Create a new {@link HiveJdbcConnector} using the specified Hive server version. * @param hiveServerVersion is the Hive server version to use * @return a HiveJdbcConnector with the specified hiveServerVersion * @throws SQLException */ @SuppressWarnings("resource") public static HiveJdbcConnector newEmbeddedConnector(int hiveServerVersion) throws SQLException { return new HiveJdbcConnector().withHiveServerVersion(hiveServerVersion).withHiveEmbeddedConnection(); }
/** * Create a new {@link HiveJdbcConnector} based on the configs in a {@link Properties} object * @param compactRunProps contains the configuration keys to construct the {@link HiveJdbcConnector} * @throws SQLException if there is a problem setting up the JDBC connection * @return */ public static HiveJdbcConnector newConnectorWithProps(Properties compactRunProps) throws SQLException { boolean isSimulate = Boolean.valueOf(compactRunProps.getProperty(HIVE_EXECUTION_SIMULATE)); HiveJdbcConnector hiveJdbcConnector = new HiveJdbcConnector(isSimulate); // Set the Hive Server type hiveJdbcConnector.withHiveServerVersion( Integer.parseInt(compactRunProps.getProperty(HIVESERVER_VERSION, DEFAULT_HIVESERVER_VERSION))); // Add the Hive Site Dir to the classpath if (compactRunProps.containsKey(HIVESITE_DIR)) { HiveJdbcConnector.addHiveSiteDirToClasspath(compactRunProps.getProperty(HIVESITE_DIR)); } // Set and create the Hive JDBC connection if (compactRunProps.containsKey(HIVESERVER_CONNECTION_STRING)) { hiveJdbcConnector.withHiveConnectionFromUrl(compactRunProps.getProperty(HIVESERVER_CONNECTION_STRING)); } else if (compactRunProps.containsKey(HIVESERVER_URL)) { hiveJdbcConnector.withHiveConnectionFromUrlUserPassword(compactRunProps.getProperty(HIVESERVER_URL), compactRunProps.getProperty(HIVESERVER_USER), PasswordManager.getInstance(compactRunProps).readPassword(compactRunProps.getProperty(HIVESERVER_PASSWORD))); } else { hiveJdbcConnector.withHiveEmbeddedConnection(); } // Set Hive properties hiveJdbcConnector.setHiveProperties(compactRunProps); return hiveJdbcConnector; }
HiveJdbcConnector hiveJdbcConnector = HiveJdbcConnector.newConnectorWithProps(props); statement = hiveJdbcConnector.getConnection().createStatement();
@Override public DataWriter<QueryBasedHiveConversionEntity> build() throws IOException { try { return new HiveQueryExecutionWriter(HiveJdbcConnector.newConnectorWithProps(this.destination.getProperties().getProperties()), this.destination.getProperties()); } catch (SQLException e) { throw new RuntimeException(e); } } }
@BeforeClass public void setup() throws Exception { this.jdbcConnector = HiveJdbcConnector.newEmbeddedConnector(2); this.dataFile = new File(getClass().getClassLoader().getResource("hiveMaterializerTest/source/").toURI()); this.localHiveMetastore.dropDatabaseIfExists(this.dbName); this.localHiveMetastore.createTestDb(this.dbName); this.jdbcConnector.executeStatements( String.format("CREATE EXTERNAL TABLE %s.%s (id STRING, name String) PARTITIONED BY (%s String) " + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE", this.dbName, this.sourceTableName, this.partitionColumn), String.format("ALTER TABLE %s.%s ADD PARTITION (part = 'part1') LOCATION '%s'", this.dbName, this.sourceTableName, this.dataFile.getAbsolutePath() + "/part1"), String.format("ALTER TABLE %s.%s ADD PARTITION (part = 'part2') LOCATION '%s'", this.dbName, this.sourceTableName, this.dataFile.getAbsolutePath() + "/part2")); List<List<String>> allTable = executeStatementAndGetResults(this.jdbcConnector, String.format("SELECT * FROM %s.%s", this.dbName, this.sourceTableName), 3); Assert.assertEquals(allTable.size(), 8); List<List<String>> part1 = executeStatementAndGetResults(this.jdbcConnector, String.format("SELECT * FROM %s.%s WHERE %s='part1'", this.dbName, this.sourceTableName, this.partitionColumn), 3); Assert.assertEquals(part1.size(), 4); this.pool = HiveMetastoreClientPool.get(new Properties(), Optional.absent()); Table table; try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { table = new Table(client.get().getTable(this.dbName, this.sourceTableName)); } this.dataset = new HiveDataset(FileSystem.getLocal(new Configuration()), pool, table, new Properties()); }
private List<List<String>> executeStatementAndGetResults(HiveJdbcConnector connector, String query, int columns) throws SQLException { Connection conn = connector.getConnection(); List<List<String>> result = new ArrayList<>(); try (Statement stmt = conn.createStatement()) { stmt.execute(query); ResultSet rs = stmt.getResultSet(); while (rs.next()) { List<String> thisResult = new ArrayList<>(); for (int i = 0; i < columns; i++) { thisResult.add(rs.getString(i + 1)); } result.add(thisResult); } } return result; }
/*** * Executes the given SQL statements. * * @param statements SQL statements to be executed. * @throws SQLException if any issue in executing any statement. */ public void executeStatements(String... statements) throws SQLException { Preconditions.checkNotNull(this.conn, "The Hive connection must be set before any queries can be run"); for (String statement : statements) { if (isSimulate) { LOG.info("[SIMULATE MODE] STATEMENT NOT RUN: " + choppedStatementNoLineChange(statement)); } else { LOG.info("RUNNING STATEMENT: " + choppedStatementNoLineChange(statement)); try (Statement stmt = this.conn.createStatement()) { try { stmt.execute(statement); } catch (SQLException sqe) { LOG.error("Failed statement: " + choppedStatementNoLineChange(statement)); throw sqe; } } } } }
/** * Create a new {@link HiveJdbcConnector} based on the configs in a {@link Properties} object * @param compactRunProps contains the configuration keys to construct the {@link HiveJdbcConnector} * @throws SQLException if there is a problem setting up the JDBC connection * @return */ public static HiveJdbcConnector newConnectorWithProps(Properties compactRunProps) throws SQLException { boolean isSimulate = Boolean.valueOf(compactRunProps.getProperty(HIVE_EXECUTION_SIMULATE)); HiveJdbcConnector hiveJdbcConnector = new HiveJdbcConnector(isSimulate); // Set the Hive Server type hiveJdbcConnector.withHiveServerVersion( Integer.parseInt(compactRunProps.getProperty(HIVESERVER_VERSION, DEFAULT_HIVESERVER_VERSION))); // Add the Hive Site Dir to the classpath if (compactRunProps.containsKey(HIVESITE_DIR)) { HiveJdbcConnector.addHiveSiteDirToClasspath(compactRunProps.getProperty(HIVESITE_DIR)); } // Set and create the Hive JDBC connection if (compactRunProps.containsKey(HIVESERVER_CONNECTION_STRING)) { hiveJdbcConnector.withHiveConnectionFromUrl(compactRunProps.getProperty(HIVESERVER_CONNECTION_STRING)); } else if (compactRunProps.containsKey(HIVESERVER_URL)) { hiveJdbcConnector.withHiveConnectionFromUrlUserPassword(compactRunProps.getProperty(HIVESERVER_URL), compactRunProps.getProperty(HIVESERVER_USER), PasswordManager.getInstance(compactRunProps).readPassword(compactRunProps.getProperty(HIVESERVER_PASSWORD))); } else { hiveJdbcConnector.withHiveEmbeddedConnection(); } // Set Hive properties hiveJdbcConnector.setHiveProperties(compactRunProps); return hiveJdbcConnector; }
@Override public void compact() throws IOException { checkSchemaCompatibility(); Closer closer = Closer.create(); try { this.conn = closer.register(HiveJdbcConnector.newConnectorWithProps(CompactionRunner.properties)); setHiveParameters(); createTables(); HiveTable mergedDelta = mergeDeltas(); HiveManagedTable notUpdated = getNotUpdatedRecords(this.snapshot, mergedDelta); unionNotUpdatedRecordsAndDeltas(notUpdated, mergedDelta); } catch (SQLException e) { LOG.error("SQLException during compaction: " + e.getMessage()); throw new RuntimeException(e); } catch (IOException e) { LOG.error("IOException during compaction: " + e.getMessage()); throw new RuntimeException(e); } catch (RuntimeException e) { LOG.error("Runtime Exception during compaction: " + e.getMessage()); throw e; } finally { try { deleteTmpFiles(); } finally { closer.close(); } } }
HiveJdbcConnector hiveJdbcConnector = HiveJdbcConnector.newConnectorWithProps(props); statement = hiveJdbcConnector.getConnection().createStatement();
/** * Create a new {@link HiveJdbcConnector} using the specified Hive server version. * @param hiveServerVersion is the Hive server version to use * @return a HiveJdbcConnector with the specified hiveServerVersion * @throws SQLException */ @SuppressWarnings("resource") public static HiveJdbcConnector newEmbeddedConnector(int hiveServerVersion) throws SQLException { return new HiveJdbcConnector().withHiveServerVersion(hiveServerVersion).withHiveEmbeddedConnection(); }
@Test public void testChoppedStatementNoLineChange() { String example1 = "This is\na test"; String example2 = "This is\r\na test\nstring"; String expected1 = "This is a test"; String expected2 = "This is a test string"; Assert.assertEquals(HiveJdbcConnector.choppedStatementNoLineChange(example1), expected1); Assert.assertEquals(HiveJdbcConnector.choppedStatementNoLineChange(example2), expected2); // Generate a random string longer than 1000 charaters int iter = 501; StringBuilder exampleExpected = new StringBuilder(); StringBuilder exampleResult = new StringBuilder(); while (iter > 0) { exampleExpected.append("a "); exampleResult.append("a\n"); iter -- ; } String expected = exampleExpected.toString().substring(0, 1000) + "...... (2 characters omitted)"; Assert.assertEquals(HiveJdbcConnector.choppedStatementNoLineChange(exampleResult.toString()), expected); } }
private void setHiveDbName() throws SQLException { this.conn.executeStatements("use " + CompactionRunner.jobProperties.getProperty(HIVE_DB_NAME, "default")); }
HiveJdbcConnector hiveJdbcConnector = closer.register(HiveJdbcConnector.newConnectorWithProps(props)); for (String query : queries) { String hiveOutput = "hiveConversionValidationOutput_" + UUID.randomUUID().toString(); try { if (this.hiveSettings.size() > 0) { hiveJdbcConnector.executeStatements(this.hiveSettings.toArray(new String[this.hiveSettings.size()])); hiveJdbcConnector.executeStatements("SET hive.exec.compress.output=false","SET hive.auto.convert.join=false", query); FileStatus[] fileStatusList = this.fs.listStatus(hiveTempDir); List<FileStatus> files = new ArrayList<>();
public HiveTask(TaskContext taskContext) { super(taskContext); this.taskContext = taskContext; this.workUnitState = taskContext.getTaskState(); this.workUnit = new HiveWorkUnit(this.workUnitState.getWorkunit()); this.eventSubmitter = new EventSubmitter.Builder(this.metricContext, "gobblin.HiveTask") .build(); this.hiveExecutionQueries = Lists.newArrayList(); this.publishEntity = new QueryBasedHivePublishEntity(); try { this.hiveJdbcConnector = HiveJdbcConnector.newConnectorWithProps(this.workUnitState.getProperties()); } catch (SQLException se) { throw new RuntimeException("Error in creating JDBC Connector", se); } this.addFiles = this.workUnitState.getPropAsList(ADD_FILES, ""); this.addJars = this.workUnitState.getPropAsList(ADD_JARS, ""); this.setupQueries = Splitter.on(";").trimResults().omitEmptyStrings().splitToList(this.workUnitState.getProp(SETUP_QUERIES, "")); }
@Override public void close() throws IOException { this.avroSchemaManager.cleanupTempSchemas(); this.hiveJdbcConnector.close(); }
private void setNumberOfReducers() throws SQLException { boolean numOfReducersSpecified = CompactionRunner.jobProperties.containsKey(MAPREDUCE_JOB_NUM_REDUCERS); if (numOfReducersSpecified) { this.conn.executeStatements("set " + MAPREDUCE_JOB_REDUCES + "=" + CompactionRunner.jobProperties.getProperty(MAPREDUCE_JOB_NUM_REDUCERS)); } }