private void dropExtraColumnStatisticsAfterAlterPartition( String databaseName, String tableName, PartitionWithStatistics partitionWithStatistics) { List<String> dataColumns = partitionWithStatistics.getPartition().getColumns().stream() .map(Column::getName) .collect(toImmutableList()); Set<String> columnsWithMissingStatistics = new HashSet<>(dataColumns); columnsWithMissingStatistics.removeAll(partitionWithStatistics.getStatistics().getColumnStatistics().keySet()); // In case new partition had the statistics computed for all the columns, the storePartitionColumnStatistics // call in the alterPartition will just overwrite the old statistics. There is no need to explicitly remove anything. if (columnsWithMissingStatistics.isEmpty()) { return; } // check if statistics for the columnsWithMissingStatistics are actually stored in the metastore // when trying to remove any missing statistics the metastore throws NoSuchObjectException String partitionName = partitionWithStatistics.getPartitionName(); List<ColumnStatisticsObj> statisticsToBeRemoved = getMetastorePartitionColumnStatistics( databaseName, tableName, ImmutableSet.of(partitionName), ImmutableList.copyOf(columnsWithMissingStatistics)) .getOrDefault(partitionName, ImmutableList.of()); for (ColumnStatisticsObj statistics : statisticsToBeRemoved) { deletePartitionColumnStatistics(databaseName, tableName, partitionName, statistics.getColName()); } }
private void storePartitionColumnStatistics(String databaseName, String tableName, String partitionName, PartitionWithStatistics partitionWithStatistics) { PartitionStatistics statistics = partitionWithStatistics.getStatistics(); Map<String, HiveColumnStatistics> columnStatistics = statistics.getColumnStatistics(); if (columnStatistics.isEmpty()) { return; } Map<String, HiveType> columnTypes = partitionWithStatistics.getPartition().getColumns().stream() .collect(toImmutableMap(Column::getName, Column::getType)); setPartitionColumnStatistics(databaseName, tableName, partitionName, columnTypes, columnStatistics, statistics.getBasicStatistics().getRowCount()); }
private Builder(Partition partition) { this.storageBuilder = Storage.builder(partition.getStorage()); this.databaseName = partition.getDatabaseName(); this.tableName = partition.getTableName(); this.values = partition.getValues(); this.columns = partition.getColumns(); this.parameters = partition.getParameters(); }
List<Column> partitionColumns = partition.getColumns(); if ((tableColumns == null) || (partitionColumns == null)) { throw new PrestoException(HIVE_INVALID_METADATA, format("Table '%s' or partition '%s' has null columns", tableName, partName));
public PartitionMetadata(Table table, PartitionWithStatistics partitionWithStatistics) { Partition partition = partitionWithStatistics.getPartition(); PartitionStatistics statistics = partitionWithStatistics.getStatistics(); this.columns = partition.getColumns(); this.parameters = updateStatisticsParameters(partition.getParameters(), statistics.getBasicStatistics()); StorageFormat tableFormat = partition.getStorage().getStorageFormat(); storageFormat = Arrays.stream(HiveStorageFormat.values()) .filter(format -> tableFormat.equals(StorageFormat.fromHiveStorageFormat(format))) .findFirst(); if (table.getTableType().equals(TableType.EXTERNAL_TABLE.name())) { externalLocation = Optional.of(partition.getStorage().getLocation()); } else { externalLocation = Optional.empty(); } bucketProperty = partition.getStorage().getBucketProperty(); serdeParameters = partition.getStorage().getSerdeParameters(); columnStatistics = ImmutableMap.copyOf(statistics.getColumnStatistics()); }
public static Properties getHiveSchema(Partition partition, Table table) { // Mimics function in Hive: MetaStoreUtils.getSchema(Partition, Table) return getHiveSchema( partition.getStorage(), partition.getColumns(), table.getDataColumns(), table.getParameters(), table.getDatabaseName(), table.getTableName(), table.getPartitionColumns()); }
List<Column> existingPartitionColumns = partition.get().getColumns(); for (int i = 0; i < min(existingPartitionColumns.size(), tableColumns.size()); i++) { HiveType tableType = tableColumns.get(i).getType();
public static PartitionInput convertPartition(Partition partition) { PartitionInput input = new PartitionInput(); input.setValues(partition.getValues()); input.setStorageDescriptor(convertStorage(partition.getStorage(), partition.getColumns())); input.setParameters(partition.getParameters()); return input; }
@Test public void testConvertPartition() { com.facebook.presto.hive.metastore.Partition prestoPartition = GlueToPrestoConverter.convertPartition(testPartition); assertEquals(prestoPartition.getDatabaseName(), testPartition.getDatabaseName()); assertEquals(prestoPartition.getTableName(), testPartition.getTableName()); assertColumnList(prestoPartition.getColumns(), testPartition.getStorageDescriptor().getColumns()); assertEquals(prestoPartition.getValues(), testPartition.getValues()); assertStorage(prestoPartition.getStorage(), testPartition.getStorageDescriptor()); assertEquals(prestoPartition.getParameters(), testPartition.getParameters()); }
public static org.apache.hadoop.hive.metastore.api.Partition toMetastoreApiPartition(Partition partition) { org.apache.hadoop.hive.metastore.api.Partition result = new org.apache.hadoop.hive.metastore.api.Partition(); result.setDbName(partition.getDatabaseName()); result.setTableName(partition.getTableName()); result.setValues(partition.getValues()); result.setSd(makeStorageDescriptor(partition.getTableName(), partition.getColumns(), partition.getStorage())); result.setParameters(partition.getParameters()); return result; }
@Test public void testConvertPartition() { com.facebook.presto.hive.metastore.Partition prestoPartition = GlueToPrestoConverter.convertPartition(testPartition); assertEquals(prestoPartition.getDatabaseName(), testPartition.getDatabaseName()); assertEquals(prestoPartition.getTableName(), testPartition.getTableName()); assertColumnList(prestoPartition.getColumns(), testPartition.getStorageDescriptor().getColumns()); assertEquals(prestoPartition.getValues(), testPartition.getValues()); assertStorage(prestoPartition.getStorage(), testPartition.getStorageDescriptor()); assertEquals(prestoPartition.getParameters(), testPartition.getParameters()); }