public void addPartition(PartitionWithStatistics partition) { checkArgument(getPrestoQueryId(partition.getPartition()).isPresent()); partitions.add(partition); }
private void dropExtraColumnStatisticsAfterAlterPartition( String databaseName, String tableName, PartitionWithStatistics partitionWithStatistics) { List<String> dataColumns = partitionWithStatistics.getPartition().getColumns().stream() .map(Column::getName) .collect(toImmutableList()); Set<String> columnsWithMissingStatistics = new HashSet<>(dataColumns); columnsWithMissingStatistics.removeAll(partitionWithStatistics.getStatistics().getColumnStatistics().keySet()); // In case new partition had the statistics computed for all the columns, the storePartitionColumnStatistics // call in the alterPartition will just overwrite the old statistics. There is no need to explicitly remove anything. if (columnsWithMissingStatistics.isEmpty()) { return; } // check if statistics for the columnsWithMissingStatistics are actually stored in the metastore // when trying to remove any missing statistics the metastore throws NoSuchObjectException String partitionName = partitionWithStatistics.getPartitionName(); List<ColumnStatisticsObj> statisticsToBeRemoved = getMetastorePartitionColumnStatistics( databaseName, tableName, ImmutableSet.of(partitionName), ImmutableList.copyOf(columnsWithMissingStatistics)) .getOrDefault(partitionName, ImmutableList.of()); for (ColumnStatisticsObj statistics : statisticsToBeRemoved) { deletePartitionColumnStatistics(databaseName, tableName, partitionName, statistics.getColName()); } }
@Override public synchronized void alterPartition(String databaseName, String tableName, PartitionWithStatistics partitionWithStatistics) { Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition()); if (partition.getParameters() == null) { partition.setParameters(ImmutableMap.of()); } PartitionName partitionKey = PartitionName.partition(databaseName, tableName, partitionWithStatistics.getPartitionName()); partitions.put(partitionKey, partition); partitionColumnStatistics.put(partitionKey, partitionWithStatistics.getStatistics()); }
public String getDescription() { return format( "alter partition %s.%s %s", newPartition.getPartition().getDatabaseName(), newPartition.getPartition().getTableName(), newPartition.getPartition().getValues()); }
@Override public synchronized void addPartitions(String databaseName, String tableName, List<PartitionWithStatistics> partitionsWithStatistics) { for (PartitionWithStatistics partitionWithStatistics : partitionsWithStatistics) { Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition()); if (partition.getParameters() == null) { partition.setParameters(ImmutableMap.of()); } PartitionName partitionKey = PartitionName.partition(databaseName, tableName, partitionWithStatistics.getPartitionName()); partitions.put(partitionKey, partition); partitionColumnStatistics.put(partitionKey, partitionWithStatistics.getStatistics()); } }
public AlterPartitionOperation(PartitionWithStatistics newPartition, PartitionWithStatistics oldPartition) { this.newPartition = requireNonNull(newPartition, "newPartition is null"); this.oldPartition = requireNonNull(oldPartition, "oldPartition is null"); checkArgument(newPartition.getPartition().getDatabaseName().equals(oldPartition.getPartition().getDatabaseName())); checkArgument(newPartition.getPartition().getTableName().equals(oldPartition.getPartition().getTableName())); checkArgument(newPartition.getPartition().getValues().equals(oldPartition.getPartition().getValues())); }
@Override public synchronized void alterPartition(String databaseName, String tableName, PartitionWithStatistics partitionWithStatistics) { Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition()); if (partition.getParameters() == null) { partition.setParameters(ImmutableMap.of()); } PartitionName partitionKey = PartitionName.partition(databaseName, tableName, partitionWithStatistics.getPartitionName()); partitions.put(partitionKey, partition); partitionColumnStatistics.put(partitionKey, partitionWithStatistics.getStatistics()); }
public void run(ExtendedHiveMetastore metastore) { undo = true; metastore.alterPartition(newPartition.getPartition().getDatabaseName(), newPartition.getPartition().getTableName(), newPartition); }
@Override public synchronized void addPartitions(String databaseName, String tableName, List<PartitionWithStatistics> partitionsWithStatistics) { for (PartitionWithStatistics partitionWithStatistics : partitionsWithStatistics) { Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition()); if (partition.getParameters() == null) { partition.setParameters(ImmutableMap.of()); } PartitionName partitionKey = PartitionName.partition(databaseName, tableName, partitionWithStatistics.getPartitionName()); partitions.put(partitionKey, partition); partitionColumnStatistics.put(partitionKey, partitionWithStatistics.getStatistics()); } }
public void undo(ExtendedHiveMetastore metastore) { if (!undo) { return; } metastore.alterPartition(oldPartition.getPartition().getDatabaseName(), oldPartition.getPartition().getTableName(), oldPartition); } }
metastore.addPartitions(schemaName, tableName, batch); for (PartitionWithStatistics partition : batch) { createdPartitionValues.add(partition.getPartition().getValues()); for (PartitionWithStatistics partition : batch) { try { Optional<Partition> remotePartition = metastore.getPartition(schemaName, tableName, partition.getPartition().getValues()); if (remotePartition.isPresent() && getPrestoQueryId(remotePartition.get()).equals(getPrestoQueryId(partition.getPartition()))) { createdPartitionValues.add(partition.getPartition().getValues());
private void storePartitionColumnStatistics(String databaseName, String tableName, String partitionName, PartitionWithStatistics partitionWithStatistics) { PartitionStatistics statistics = partitionWithStatistics.getStatistics(); Map<String, HiveColumnStatistics> columnStatistics = statistics.getColumnStatistics(); if (columnStatistics.isEmpty()) { return; } Map<String, HiveType> columnTypes = partitionWithStatistics.getPartition().getColumns().stream() .collect(toImmutableMap(Column::getName, Column::getType)); setPartitionColumnStatistics(databaseName, tableName, partitionName, columnTypes, columnStatistics, statistics.getBasicStatistics().getRowCount()); }
Map<Path, byte[]> schemaFiles = new LinkedHashMap<>(); for (PartitionWithStatistics partitionWithStatistics : partitions) { Partition partition = partitionWithStatistics.getPartition(); verifiedPartition(table, partition); Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues());
public static PartitionInput convertPartition(PartitionWithStatistics partitionWithStatistics) { PartitionInput input = convertPartition(partitionWithStatistics.getPartition()); PartitionStatistics statistics = partitionWithStatistics.getStatistics(); if (!statistics.getColumnStatistics().isEmpty()) { throw new PrestoException(NOT_SUPPORTED, "Glue metastore does not support column level statistics"); } input.setParameters(updateStatisticsParameters(input.getParameters(), statistics.getBasicStatistics())); return input; }
public static org.apache.hadoop.hive.metastore.api.Partition toMetastoreApiPartition(PartitionWithStatistics partitionWithStatistics) { org.apache.hadoop.hive.metastore.api.Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition()); partition.setParameters(updateStatisticsParameters(partition.getParameters(), partitionWithStatistics.getStatistics().getBasicStatistics())); return partition; }
@Override public synchronized void alterPartition(String databaseName, String tableName, PartitionWithStatistics partitionWithStatistics) { Table table = getRequiredTable(databaseName, tableName); Partition partition = partitionWithStatistics.getPartition(); verifiedPartition(table, partition); Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues()); writeSchemaFile("partition", partitionMetadataDirectory, partitionCodec, new PartitionMetadata(table, partitionWithStatistics), true); }
public PartitionMetadata(Table table, PartitionWithStatistics partitionWithStatistics) { Partition partition = partitionWithStatistics.getPartition(); PartitionStatistics statistics = partitionWithStatistics.getStatistics(); this.columns = partition.getColumns(); this.parameters = updateStatisticsParameters(partition.getParameters(), statistics.getBasicStatistics()); StorageFormat tableFormat = partition.getStorage().getStorageFormat(); storageFormat = Arrays.stream(HiveStorageFormat.values()) .filter(format -> tableFormat.equals(StorageFormat.fromHiveStorageFormat(format))) .findFirst(); if (table.getTableType().equals(TableType.EXTERNAL_TABLE.name())) { externalLocation = Optional.of(partition.getStorage().getLocation()); } else { externalLocation = Optional.empty(); } bucketProperty = partition.getStorage().getBucketProperty(); serdeParameters = partition.getStorage().getSerdeParameters(); columnStatistics = ImmutableMap.copyOf(statistics.getColumnStatistics()); }
@Override public void alterPartition(String databaseName, String tableName, PartitionWithStatistics partition) { try { PartitionInput newPartition = GlueInputConverter.convertPartition(partition); glueClient.updatePartition(new UpdatePartitionRequest() .withDatabaseName(databaseName) .withTableName(tableName) .withPartitionInput(newPartition) .withPartitionValueList(partition.getPartition().getValues())); } catch (EntityNotFoundException e) { throw new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), partition.getPartition().getValues()); } catch (AmazonServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } }