/** * Is this an external table? * * @param table * Check if this table is external. * @return True if the table is external, otherwise false. */ private boolean isExternal(Table table) { return MetaStoreUtils.isExternalTable(table); }
@Override public void rollbackCreateTable(Table table) throws MetaException { String tableName = getHBaseTableName(table); boolean isPurge = !MetaStoreUtils.isExternalTable(table) || MetaStoreUtils.isExternalTablePurge(table); try { if (isPurge && getHBaseAdmin().tableExists(TableName.valueOf(tableName))) { // we have created an HBase table, so we delete it to roll back; if (getHBaseAdmin().isTableEnabled(TableName.valueOf(tableName))) { getHBaseAdmin().disableTable(TableName.valueOf(tableName)); } getHBaseAdmin().deleteTable(TableName.valueOf(tableName)); } } catch (IOException ie) { throw new MetaException(StringUtils.stringifyException(ie)); } }
/** * Are the basic stats for the table up-to-date for query planning. * Can run additional checks compared to the version in StatsSetupConst. */ public static boolean areBasicStatsUptoDateForQueryAnswering(Table table, Map<String, String> params) { // HIVE-19332: external tables should not be considered to have up-to-date stats. if (MetaStoreUtils.isExternalTable(table.getTTable())) { return false; } return StatsSetupConst.areBasicStatsUptoDate(params); }
@Override public void commitDropTable(Table tbl, boolean deleteData) throws MetaException { try { String tableName = getHBaseTableName(tbl); boolean isPurge = !MetaStoreUtils.isExternalTable(tbl) || MetaStoreUtils.isExternalTablePurge(tbl); if (deleteData && isPurge) { LOG.info("Dropping with purge all the data for data source {}", tableName); if (getHBaseAdmin().tableExists(TableName.valueOf(tableName))) { if (getHBaseAdmin().isTableEnabled(TableName.valueOf(tableName))) { getHBaseAdmin().disableTable(TableName.valueOf(tableName)); } getHBaseAdmin().deleteTable(TableName.valueOf(tableName)); } } } catch (IOException ie) { throw new MetaException(StringUtils.stringifyException(ie)); } }
/** * Are the column stats for the table up-to-date for query planning. * Can run additional checks compared to the version in StatsSetupConst. */ public static boolean areColumnStatsUptoDateForQueryAnswering(Table table, Map<String, String> params, String colName) { // HIVE-19332: external tables should not be considered to have up-to-date stats. if (MetaStoreUtils.isExternalTable(table.getTTable())) { return false; } return StatsSetupConst.areColumnStatsUptoDate(params, colName); }
public Path getDefaultTablePath(Database db, Table table) throws MetaException { return getDefaultTablePath(db, table.getTableName(), MetaStoreUtils.isExternalTable(table)); }
private static boolean hasExternalTableAncestor(Operator op, StringBuilder sb) { boolean result = false; Operator ancestor = OperatorUtils.findSingleOperatorUpstream(op, TableScanOperator.class); if (ancestor != null) { TableScanOperator ts = (TableScanOperator) ancestor; if (MetaStoreUtils.isExternalTable(ts.getConf().getTableMetadata().getTTable())) { sb.append(ts.getConf().getTableMetadata().getFullyQualifiedName()); return true; } } return result; } }
if (MetaStoreUtils.isExternalTable(newTable)) { throw new MetaException(Warehouse.getQualifiedName(newTable) + " cannot be declared transactional because it's an external table");
private Status checkThresholdWithMetastoreStats(final Table table, final PrunedPartitionList partsList, final long threshold) { Status status = Status.UNAVAILABLE; if (table != null && !table.isPartitioned()) { long dataSize = StatsUtils.getTotalSize(table); if (dataSize <= 0) { LOG.warn("Cannot determine basic stats for table: {} from metastore. Falling back.", table.getCompleteName()); return Status.UNAVAILABLE; } status = (threshold - dataSize) >= 0 ? Status.PASS : Status.FAIL; } else if (table != null && table.isPartitioned() && partsList != null) { List<Long> dataSizes = StatsUtils.getBasicStatForPartitions(table, partsList.getNotDeniedPartns(), StatsSetupConst.TOTAL_SIZE); long totalDataSize = StatsUtils.getSumIgnoreNegatives(dataSizes); if (totalDataSize <= 0) { LOG.warn("Cannot determine basic stats for partitioned table: {} from metastore. Falling back.", table.getCompleteName()); return Status.UNAVAILABLE; } status = (threshold - totalDataSize) >= 0 ? Status.PASS : Status.FAIL; } if (status == Status.PASS && MetaStoreUtils.isExternalTable(table.getTTable())) { // External table should also check the underlying file size. LOG.warn("Table {} is external table, falling back to filesystem scan.", table.getCompleteName()); status = Status.UNAVAILABLE; } return status; }
if (deleteData && !MetaStoreUtils.isExternalTable(table)) { try { boolean ifPurge = false;
if (conf.getBoolVar(HiveConf.ConfVars.HIVE_DISABLE_UNSAFE_EXTERNALTABLE_OPERATIONS)) { if (MetaStoreUtils.isExternalTable(ts.getConf().getTableMetadata().getTTable())) { LOG.debug("Disabling semijoin optimzation on {} since it is an external table.", ts.getConf().getTableMetadata().getFullyQualifiedName()); if (MetaStoreUtils.isExternalTable(joinKeyTs.getConf().getTableMetadata().getTTable())) { LOG.debug("Join key {} is from {} which is an external table. Disabling semijoin optimization.", columnOrigin.col,
/** * validates if a table can be exported, similar to EximUtil.shouldExport with few replication * specific checks. */ public static Boolean shouldReplicate(ReplicationSpec replicationSpec, Table tableHandle, HiveConf hiveConf) { if (replicationSpec == null) { replicationSpec = new ReplicationSpec(); } if (replicationSpec.isNoop() || tableHandle == null) { return false; } // if its metadata only, then dump metadata of non native tables also. if (tableHandle.isNonNative() && !replicationSpec.isMetadataOnly()) { return false; } if (replicationSpec.isInReplicationScope()) { if (tableHandle.isTemporary()) { return false; } if (MetaStoreUtils.isExternalTable(tableHandle.getTTable())) { return hiveConf.getBoolVar(HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES) || replicationSpec.isMetadataOnly(); } } return true; }
if (MetaStoreUtils.isExternalTable(tbl.getTTable())) { Logger.info("Table " + tbl.getTableName() + " is external. Skip StatsOptimizer."); return null;
&& (oldt.getSd().getLocation().compareTo(newt.getSd().getLocation()) == 0 || StringUtils.isEmpty(newt.getSd().getLocation())) && !MetaStoreUtils.isExternalTable(oldt)) { Database olddb = msdb.getDatabase(catName, dbname);
public TableWrapper(org.apache.hadoop.hive.metastore.api.Table apiTable) { org.apache.hadoop.hive.metastore.api.Table wrapperApiTable = apiTable.deepCopy(); if (wrapperApiTable.getTableType() == null){ // TableType specified was null, we need to figure out what type it was. if (MetaStoreUtils.isExternalTable(wrapperApiTable)){ wrapperApiTable.setTableType(TableType.EXTERNAL_TABLE.toString()); } else if (MetaStoreUtils.isMaterializedViewTable(wrapperApiTable)) { wrapperApiTable.setTableType(TableType.MATERIALIZED_VIEW.toString()); } else if ((wrapperApiTable.getSd() == null) || (wrapperApiTable.getSd().getLocation() == null)) { wrapperApiTable.setTableType(TableType.VIRTUAL_VIEW.toString()); } else { wrapperApiTable.setTableType(TableType.MANAGED_TABLE.toString()); } } initialize(wrapperApiTable); } }
/** * @param tableEvent table event. * @throws MetaException */ @Override public void onCreateTable(CreateTableEvent tableEvent) throws MetaException { Table t = tableEvent.getTable(); FileIterator fileIter = MetaStoreUtils.isExternalTable(t) ? null : new FileIterator(t.getSd().getLocation()); CreateTableMessage msg = MessageBuilder.getInstance().buildCreateTableMessage(t, fileIter); NotificationEvent event = new NotificationEvent(0, now(), EventType.CREATE_TABLE.toString(), msgEncoder.getSerializer().serialize(msg)); event.setCatName(t.isSetCatName() ? t.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); process(event, tableEvent); }
/** * @param partitionEvent partition event * @throws MetaException */ @Override public void onAddPartition(AddPartitionEvent partitionEvent) throws MetaException { Table t = partitionEvent.getTable(); PartitionFilesIterator fileIter = MetaStoreUtils.isExternalTable(t) ? null : new PartitionFilesIterator(partitionEvent.getPartitionIterator(), t); EventMessage msg = MessageBuilder.getInstance() .buildAddPartitionMessage(t, partitionEvent.getPartitionIterator(), fileIter); MessageSerializer serializer = msgEncoder.getSerializer(); NotificationEvent event = new NotificationEvent(0, now(), EventType.ADD_PARTITION.toString(), serializer.serialize(msg)); event.setCatName(t.isSetCatName() ? t.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); process(event, partitionEvent); }
/** * Is this an external table? * * @param table * Check if this table is external. * @return True if the table is external, otherwise false. */ private boolean isExternal(Table table) { return MetaStoreUtils.isExternalTable(table); }
protected boolean isExternalTable(Table table) { return MetaStoreUtils.isExternalTable(table); }
public Path getDefaultTablePath(Database db, Table table) throws MetaException { return getDefaultTablePath(db, table.getTableName(), MetaStoreUtils.isExternalTable(table)); }