public SourceLocationManager getLocationManager(Table table, String eventId) throws IOException { if (MetaStoreUtils.isView(table)) { return new ViewLocationManager(); } return new HdfsSnapshotLocationManager(getHiveConf(), eventId, table, snapshotsDisabled, sourceTableLocation, sourceCatalogListener); }
public SourceLocationManager getLocationManager(Table table, String eventId) throws IOException { if (MetaStoreUtils.isView(table)) { return new ViewLocationManager(); } return new HdfsSnapshotLocationManager(getHiveConf(), eventId, table, snapshotsDisabled, sourceTableLocation, sourceCatalogListener); }
private void initializeAddedPartition( final Table tbl, final PartitionSpecProxy.PartitionIterator part, boolean madeDir) throws MetaException { if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVESTATSAUTOGATHER) && !MetaStoreUtils.isView(tbl)) { MetaStoreUtils.updatePartitionStatsFast(part, wh, madeDir, false); } // set create time long time = System.currentTimeMillis() / 1000; part.setCreateTime((int) time); if (part.getParameters() == null || part.getParameters().get(hive_metastoreConstants.DDL_TIME) == null) { part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(time)); } // Inherit table properties into partition properties. Map<String, String> tblParams = tbl.getParameters(); String inheritProps = hiveConf.getVar(ConfVars.METASTORE_PART_INHERIT_TBL_PROPS).trim(); // Default value is empty string in which case no properties will be inherited. // * implies all properties needs to be inherited Set<String> inheritKeys = new HashSet<String>(Arrays.asList(inheritProps.split(","))); if (inheritKeys.contains("*")) { inheritKeys = tblParams.keySet(); } for (String key : inheritKeys) { String paramVal = tblParams.get(key); if (null != paramVal) { // add the property only if it exists in table properties part.putToParameters(key, paramVal); } } }
private void initializeAddedPartition( final Table tbl, final PartitionSpecProxy.PartitionIterator part, boolean madeDir) throws MetaException { if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVESTATSAUTOGATHER) && !MetaStoreUtils.isView(tbl)) { MetaStoreUtils.updatePartitionStatsFast(part, wh, madeDir, false); } // set create time long time = System.currentTimeMillis() / 1000; part.setCreateTime((int) time); if (part.getParameters() == null || part.getParameters().get(hive_metastoreConstants.DDL_TIME) == null) { part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(time)); } // Inherit table properties into partition properties. Map<String, String> tblParams = tbl.getParameters(); String inheritProps = hiveConf.getVar(ConfVars.METASTORE_PART_INHERIT_TBL_PROPS).trim(); // Default value is empty string in which case no properties will be inherited. // * implies all properties needs to be inherited Set<String> inheritKeys = new HashSet<String>(Arrays.asList(inheritProps.split(","))); if (inheritKeys.contains("*")) { inheritKeys = tblParams.keySet(); } for (String key : inheritKeys) { String paramVal = tblParams.get(key); if (null != paramVal) { // add the property only if it exists in table properties part.putToParameters(key, paramVal); } } }
public static boolean requireCalStats(Configuration hiveConf, Partition oldPart, Partition newPart, Table tbl) { if (MetaStoreUtils.isView(tbl)) { return false; } if (oldPart == null && newPart == null) { return true; } // requires to calculate stats if new partition doesn't have it if ((newPart == null) || (newPart.getParameters() == null) || !containsAllFastStats(newPart.getParameters())) { return true; } if(newPart.getParameters().containsKey(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK)) { return true; } // requires to calculate stats if new and old have different fast stats if ((oldPart != null) && (oldPart.getParameters() != null)) { for (String stat : StatsSetupConst.fastStats) { if (oldPart.getParameters().containsKey(stat)) { Long oldStat = Long.parseLong(oldPart.getParameters().get(stat)); Long newStat = Long.parseLong(newPart.getParameters().get(stat)); if (!oldStat.equals(newStat)) { return true; } } } } return false; }
public static boolean requireCalStats(Configuration hiveConf, Partition oldPart, Partition newPart, Table tbl) { if (MetaStoreUtils.isView(tbl)) { return false; } if (oldPart == null && newPart == null) { return true; } // requires to calculate stats if new partition doesn't have it if ((newPart == null) || (newPart.getParameters() == null) || !containsAllFastStats(newPart.getParameters())) { return true; } if(newPart.getParameters().containsKey(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK)) { return true; } // requires to calculate stats if new and old have different fast stats if ((oldPart != null) && (oldPart.getParameters() != null)) { for (String stat : StatsSetupConst.fastStats) { if (oldPart.getParameters().containsKey(stat)) { Long oldStat = Long.parseLong(oldPart.getParameters().get(stat)); Long newStat = Long.parseLong(newPart.getParameters().get(stat)); if (!oldStat.equals(newStat)) { return true; } } } } return false; }
public SourceLocationManager getLocationManager( Table table, List<Partition> partitions, String eventId, Map<String, Object> copierOptions) throws IOException { if (MetaStoreUtils.isView(table)) { return new ViewLocationManager(); } HdfsSnapshotLocationManager hdfsSnapshotLocationManager = new HdfsSnapshotLocationManager(getHiveConf(), eventId, table, partitions, snapshotsDisabled, sourceTableLocation, sourceCatalogListener); boolean ignoreMissingFolder = MapUtils.getBooleanValue(copierOptions, CopierOptions.IGNORE_MISSING_PARTITION_FOLDER_ERRORS, false); if (ignoreMissingFolder) { return new FilterMissingPartitionsLocationManager(hdfsSnapshotLocationManager, getHiveConf()); } return hdfsSnapshotLocationManager; }
public SourceLocationManager getLocationManager( Table table, List<Partition> partitions, String eventId, Map<String, Object> copierOptions) throws IOException { if (MetaStoreUtils.isView(table)) { return new ViewLocationManager(); } HdfsSnapshotLocationManager hdfsSnapshotLocationManager = new HdfsSnapshotLocationManager(getHiveConf(), eventId, table, partitions, snapshotsDisabled, sourceTableLocation, sourceCatalogListener); boolean ignoreMissingFolder = MapUtils.getBooleanValue(copierOptions, CopierOptions.IGNORE_MISSING_PARTITION_FOLDER_ERRORS, false); if (ignoreMissingFolder) { return new FilterMissingPartitionsLocationManager(hdfsSnapshotLocationManager, getHiveConf()); } return hdfsSnapshotLocationManager; }
private void validate(TableReplication tableReplication, Source source, Replica replica) { source.getDatabase(tableReplication.getSourceTable().getDatabaseName()); replica.getDatabase(tableReplication.getReplicaDatabaseName()); TableAndStatistics sourceTableAndStatistics = source.getTableAndStatistics(tableReplication); if (tableReplication.getReplicationMode() != ReplicationMode.METADATA_MIRROR && MetaStoreUtils.isView(sourceTableAndStatistics.getTable())) { throw new CircusTrainException(String .format("Cannot replicate view %s. Only %s is supported for views", tableReplication.getSourceTable().getQualifiedName(), ReplicationMode.METADATA_MIRROR.name())); } }
private void validate(TableReplication tableReplication, Source source, Replica replica) { source.getDatabase(tableReplication.getSourceTable().getDatabaseName()); replica.getDatabase(tableReplication.getReplicaDatabaseName()); TableAndStatistics sourceTableAndStatistics = source.getTableAndStatistics(tableReplication); if (tableReplication.getReplicationMode() != ReplicationMode.METADATA_MIRROR && MetaStoreUtils.isView(sourceTableAndStatistics.getTable())) { throw new CircusTrainException(String .format("Cannot replicate view %s. Only %s is supported for views", tableReplication.getSourceTable().getQualifiedName(), ReplicationMode.METADATA_MIRROR.name())); } }
@Override public Table transform(Table table) { if (!MetaStoreUtils.isView(table)) { return table; } LOG.info("Translating HQL of view {}.{}", table.getDbName(), table.getTableName()); String tableQualifiedName = Warehouse.getQualifiedName(table); String hql = hqlTranslator.translate(tableQualifiedName, table.getViewOriginalText()); String expandedHql = hqlTranslator.translate(tableQualifiedName, table.getViewExpandedText()); Table transformedView = new Table(table); transformedView.setViewOriginalText(hql); transformedView.setViewExpandedText(expandedHql); if (!replicaHiveConf.getBoolean(SKIP_TABLE_EXIST_CHECKS, false)) { LOG .info("Validating that tables used by the view {}.{} exist in the replica catalog", table.getDbName(), table.getTableName()); validateReferencedTables(transformedView); } return transformedView; }
@Override public Table transform(Table table) { if (!MetaStoreUtils.isView(table)) { return table; } LOG.info("Translating HQL of view {}.{}", table.getDbName(), table.getTableName()); String tableQualifiedName = Warehouse.getQualifiedName(table); String hql = hqlTranslator.translate(tableQualifiedName, table.getViewOriginalText()); String expandedHql = hqlTranslator.translate(tableQualifiedName, table.getViewExpandedText()); Table transformedView = new Table(table); transformedView.setViewOriginalText(hql); transformedView.setViewExpandedText(expandedHql); if (!replicaHiveConf.getBoolean(SKIP_TABLE_EXIST_CHECKS, false)) { LOG .info("Validating that tables used by the view {}.{} exist in the replica catalog", table.getDbName(), table.getTableName()); validateReferencedTables(transformedView); } return transformedView; }
!MetaStoreUtils.isView(tbl)) { MetaStoreUtils.updatePartitionStatsFast(part, wh, madeDir);
!MetaStoreUtils.isView(tbl)) { MetaStoreUtils.updatePartitionStatsFast(part, wh, madeDir);
!MetaStoreUtils.isView(tbl)) { if (tbl.getPartitionKeysSize() == 0) { // Unpartitioned table MetaStoreUtils.updateUnpartitionedTableStatsFast(db, tbl, wh, madeDir);
!MetaStoreUtils.isView(tbl)) { if (tbl.getPartitionKeysSize() == 0) { // Unpartitioned table MetaStoreUtils.updateUnpartitionedTableStatsFast(db, tbl, wh, madeDir);