@SuppressWarnings("nls") private void initialize() { LOG.debug("ObjectStore, initialize called"); // if this method fails, PersistenceManagerProvider will retry for the configured number of times // before giving up pm = PersistenceManagerProvider.getPersistenceManager(); LOG.info("RawStore: {}, with PersistenceManager: {}" + " created in the thread with id: {}", this, pm, Thread.currentThread().getId()); try { String productName = MetaStoreDirectSql.getProductName(pm); sqlGenerator = new SQLGenerator(DatabaseProduct.determineDatabaseProduct(productName), conf); } catch (SQLException e) { LOG.error("error trying to figure out the database product", e); throw new RuntimeException(e); } isInitialized = pm != null; if (isInitialized) { dbType = determineDatabaseProduct(); expressionProxy = createExpressionProxy(conf); if (MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL)) { String schema = PersistenceManagerProvider.getProperty("javax.jdo.mapping.Schema"); schema = org.apache.commons.lang.StringUtils.defaultIfBlank(schema, null); directSql = new MetaStoreDirectSql(pm, conf, schema); } } }
@Override protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx) throws MetaException { return directSql.getPartitionsViaSqlFilter(catName, dbName, tblName, partNames); } @Override
@Override protected AggrStats getSqlResult(GetHelper<AggrStats> ctx) throws MetaException { return directSql.aggrColStatsForPartitions(catName, dbName, tblName, partNames, colNames, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector); } @Override
@Override protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx) throws MetaException { // If we have some sort of expression tree, try SQL filter pushdown. List<Partition> result = null; if (exprTree != null) { SqlFilterForPushdown filter = new SqlFilterForPushdown(); if (directSql.generateSqlFilterForPushdown(ctx.getTable(), exprTree, defaultPartitionName, filter)) { return directSql.getPartitionsViaSqlFilter(filter, null); } } // We couldn't do SQL filter pushdown. Get names via normal means. List<String> partNames = new LinkedList<>(); hasUnknownPartitions.set(getPartitionNamesPrunedByExprNoTxn( ctx.getTable(), expr, defaultPartitionName, maxParts, partNames)); return directSql.getPartitionsViaSqlFilter(catName, dbName, tblName, partNames); }
DatabaseProduct dbType = null; try { dbType = DatabaseProduct.determineDatabaseProduct(getProductName(pm)); } catch (SQLException e) { LOG.warn("Cannot determine database product; assuming OTHER", e); if (f.getAnnotation(TableName.class) == null) continue; try { String value = getFullyQualifiedName(schema, f.getName()); f.set(this, value); fieldNameToTableNameBuilder.put(f.getName(), value); } else { boolean isInTest = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST); isCompatibleDatastore = (!isInTest || ensureDbInit()) && runTestQuery(); if (isCompatibleDatastore) { LOG.debug("Using direct SQL, underlying DB is " + dbType);
partSb.append(extractSqlLong(partitionId)).append(","); String partIds = trimCommaList(partSb); Query query = pm.newQuery("javax.jdo.query.SQL", queryText); @SuppressWarnings("unchecked") List<Object[]> sqlResult = executeWithArray(query, null, queryText); long queryTime = doTrace ? System.nanoTime() : 0; Deadline.checkTimeout(); long partitionId = extractSqlLong(fields[0]); Long sdId = extractSqlLong(fields[1]); Long colId = extractSqlLong(fields[2]); Long serdeId = extractSqlLong(fields[3]); isView = isViewTable(dbName, tblName); part.setDbName(dbName); part.setTableName(tblName); if (fields[4] != null) part.setCreateTime(extractSqlInt(fields[4])); if (fields[5] != null) part.setLastAccessTime(extractSqlInt(fields[5])); partitions.put(partitionId, part); new ArrayList<List<String>>(), new HashMap<List<String>, String>())); sd.setInputFormat((String)fields[6]); Boolean tmpBoolean = extractSqlBoolean(fields[7]); if (tmpBoolean != null) sd.setCompressed(tmpBoolean); tmpBoolean = extractSqlBoolean(fields[8]); if (tmpBoolean != null) sd.setStoredAsSubDirectories(tmpBoolean);
String partIds = getIdListForIn(partIdList); String queryText = "select " + PARTITIONS + ".\"PART_ID\", " + SDS + ".\"SD_ID\", " + SDS + ".\"CD_ID\"," + " " List<Object[]> sqlResult = executeWithArray(query, null, queryText); long queryTime = doTrace ? System.nanoTime() : 0; Deadline.checkTimeout(); isView = isViewTable(catName, dbName, tblName); String sdIds = trimCommaList(sdSb); String serdeIds = trimCommaList(serdeSb); String colIds = trimCommaList(colsSb);
if (isConfigEnabled && directSql == null) { dbType = determineDatabaseProduct(); directSql = new MetaStoreDirectSql(pm, getConf(), ""); if (!allowJdo && isConfigEnabled && !directSql.isCompatibleDatastore()) { throw new MetaException("SQL is not operational"); // test path; SQL is enabled and broken. this.doUseDirectSql = allowSql && isConfigEnabled && directSql.isCompatibleDatastore();
String partIds = trimCommaList(partSb); if (doTrace) { LOG.debug("Direct SQL query in " + (queryTime - start) / 1000000.0 + "ms + " + new ArrayList<List<String>>(), new HashMap<SkewedValueList, String>())); sd.setInputFormat((String)fields[6]); Boolean tmpBoolean = extractSqlBoolean(fields[7]); if (tmpBoolean != null) sd.setCompressed(tmpBoolean); tmpBoolean = extractSqlBoolean(fields[8]); if (tmpBoolean != null) sd.setStoredAsSubDirectories(tmpBoolean); sd.setLocation((String)fields[9]); String sdIds = trimCommaList(sdSb), serdeIds = trimCommaList(serdeSb), colIds = trimCommaList(colsSb); loopJoinOrderedResult(partitions, queryText, 0, new ApplyFunc<Partition>() { public void apply(Partition t, Object[] fields) { t.putToParameters((String)fields[1], (String)fields[2]); loopJoinOrderedResult(partitions, queryText, 0, new ApplyFunc<Partition>() { public void apply(Partition t, Object[] fields) { t.addToValues((String)fields[1]); loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { public void apply(StorageDescriptor t, Object[] fields) { t.putToParameters((String)fields[1], (String)fields[2]); loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { public void apply(StorageDescriptor t, Object[] fields) {
@Override protected List<ColumnStatistics> getSqlResult( GetHelper<List<ColumnStatistics>> ctx) throws MetaException { return directSql.getPartitionStats(catName, dbName, tblName, partNames, colNames, enableBitVector); } @Override
@SuppressWarnings("nls") private void initialize(Properties dsProps) { LOG.info("ObjectStore, initialize called"); prop = dsProps; pm = getPersistenceManager(); isInitialized = pm != null; if (isInitialized) { directSql = new MetaStoreDirectSql(pm); } return; }
private DB determineDbType() { DB dbType = DB.OTHER; if (runDbCheck("SET @@session.sql_mode=ANSI_QUOTES", "MySql")) { dbType = DB.MYSQL; } else if (runDbCheck("SELECT version FROM v$instance", "Oracle")) { dbType = DB.ORACLE; } else if (runDbCheck("SELECT @@version", "MSSQL")) { dbType = DB.MSSQL; } else { // TODO: maybe we should use getProductName to identify all the DBs String productName = getProductName(); if (productName != null && productName.toLowerCase().contains("derby")) { dbType = DB.DERBY; } } return dbType; }
@Override protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx) throws MetaException { Integer max = (maxParts < 0) ? null : maxParts; return directSql.getPartitions(catName, dbName, tblName, max); } @Override
@Override protected ColumnStatistics getSqlResult(GetHelper<ColumnStatistics> ctx) throws MetaException { return directSql.getTableStats(catName, dbName, tblName, colNames, enableBitVector); } @Override
@Override protected Database getSqlResult(GetHelper<Database> ctx) throws MetaException { return directSql.getDatabase(catalogName, dbName); }
public GetHelper(String dbName, String tblName, boolean allowSql, boolean allowJdo) throws MetaException { assert allowSql || allowJdo; this.allowJdo = allowJdo; this.dbName = HiveStringUtils.normalizeIdentifier(dbName); if (tblName != null){ this.tblName = HiveStringUtils.normalizeIdentifier(tblName); } else { // tblName can be null in cases of Helper being used at a higher // abstraction level, such as with datbases this.tblName = null; this.table = null; } this.doTrace = LOG.isDebugEnabled(); this.isInTxn = isActiveTransaction(); // SQL usage inside a larger transaction (e.g. droptable) may not be desirable because // some databases (e.g. Postgres) abort the entire transaction when any query fails, so // the fallback from failed SQL to JDO is not possible. boolean isConfigEnabled = HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL) && (HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL_DDL) || !isInTxn); if (!allowJdo && isConfigEnabled && !directSql.isCompatibleDatastore()) { throw new MetaException("SQL is not operational"); // test path; SQL is enabled and broken. } this.doUseDirectSql = allowSql && isConfigEnabled && directSql.isCompatibleDatastore(); }
partSb.append(extractSqlLong(partitionId)).append(","); String partIds = trimCommaList(partSb); Query query = pm.newQuery("javax.jdo.query.SQL", queryText); @SuppressWarnings("unchecked") List<Object[]> sqlResult = executeWithArray(query, null, queryText); long queryTime = doTrace ? System.nanoTime() : 0; Deadline.checkTimeout(); long partitionId = extractSqlLong(fields[0]); Long sdId = extractSqlLong(fields[1]); Long colId = extractSqlLong(fields[2]); Long serdeId = extractSqlLong(fields[3]); isView = isViewTable(dbName, tblName); part.setDbName(dbName); part.setTableName(tblName); if (fields[4] != null) part.setCreateTime(extractSqlInt(fields[4])); if (fields[5] != null) part.setLastAccessTime(extractSqlInt(fields[5])); partitions.put(partitionId, part); new ArrayList<List<String>>(), new HashMap<List<String>, String>())); sd.setInputFormat((String)fields[6]); Boolean tmpBoolean = extractSqlBoolean(fields[7]); if (tmpBoolean != null) sd.setCompressed(tmpBoolean); tmpBoolean = extractSqlBoolean(fields[8]); if (tmpBoolean != null) sd.setStoredAsSubDirectories(tmpBoolean);
public GetHelper(String catalogName, String dbName, String tblName, boolean allowSql, boolean allowJdo) throws MetaException { assert allowSql || allowJdo; this.allowJdo = allowJdo; this.catName = (catalogName != null) ? normalizeIdentifier(catalogName) : null; this.dbName = (dbName != null) ? normalizeIdentifier(dbName) : null; if (tblName != null) { this.tblName = normalizeIdentifier(tblName); } else { // tblName can be null in cases of Helper being used at a higher // abstraction level, such as with datbases this.tblName = null; this.table = null; } this.doTrace = LOG.isDebugEnabled(); this.isInTxn = isActiveTransaction(); // SQL usage inside a larger transaction (e.g. droptable) may not be desirable because // some databases (e.g. Postgres) abort the entire transaction when any query fails, so // the fallback from failed SQL to JDO is not possible. boolean isConfigEnabled = MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL) && (MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL_DDL) || !isInTxn); if (isConfigEnabled && directSql == null) { dbType = determineDatabaseProduct(); directSql = new MetaStoreDirectSql(pm, getConf(), ""); } if (!allowJdo && isConfigEnabled && !directSql.isCompatibleDatastore()) { throw new MetaException("SQL is not operational"); // test path; SQL is enabled and broken. } this.doUseDirectSql = allowSql && isConfigEnabled && directSql.isCompatibleDatastore(); }
@Override protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx) throws MetaException { // If we have some sort of expression tree, try SQL filter pushdown. List<Partition> result = null; if (exprTree != null) { SqlFilterForPushdown filter = new SqlFilterForPushdown(); if (directSql.generateSqlFilterForPushdown(ctx.getTable(), exprTree, filter)) { return directSql.getPartitionsViaSqlFilter(filter, null); } } // We couldn't do SQL filter pushdown. Get names via normal means. List<String> partNames = new LinkedList<>(); hasUnknownPartitions.set(getPartitionNamesPrunedByExprNoTxn( ctx.getTable(), expr, defaultPartitionName, maxParts, partNames)); return directSql.getPartitionsViaSqlFilter(catName, dbName, tblName, partNames); }
private List<ColumnStatisticsObj> aggrStatsUseJava(String catName, String dbName, String tableName, List<String> partNames, List<String> colNames, boolean areAllPartsFound, boolean useDensityFunctionForNDVEstimation, double ndvTuner) throws MetaException { // 1. get all the stats for colNames in partNames; List<ColumnStatistics> partStats = getPartitionStats(catName, dbName, tableName, partNames, colNames, true); // 2. use util function to aggr stats return MetaStoreServerUtils.aggrPartitionStats(partStats, catName, dbName, tableName, partNames, colNames, areAllPartsFound, useDensityFunctionForNDVEstimation, ndvTuner); }