public Partition getAugmentedPartitionForInTransactionRead() { // This method augments the location field of the partition to the staging location. // This way, if the partition is accessed in an ongoing transaction, staged data // can be found and accessed. Partition partition = this.partition; String currentLocation = this.currentLocation.toString(); if (!currentLocation.equals(partition.getStorage().getLocation())) { partition = Partition.builder(partition) .withStorage(storage -> storage.setLocation(currentLocation)) .build(); } return partition; }
private Builder(Partition partition) { this.storageBuilder = Storage.builder(partition.getStorage()); this.databaseName = partition.getDatabaseName(); this.tableName = partition.getTableName(); this.values = partition.getValues(); this.columns = partition.getColumns(); this.parameters = partition.getParameters(); }
public Partition build() { return new Partition(databaseName, tableName, values, storageBuilder.build(), columns, parameters); } }
public static void checkPartitionIsWritable(String partitionName, Partition partition) { checkWritable( new SchemaTableName(partition.getDatabaseName(), partition.getTableName()), Optional.of(partitionName), getProtectMode(partition), partition.getParameters(), partition.getStorage()); }
public String getDescription() { return format( "alter partition %s.%s %s", newPartition.getPartition().getDatabaseName(), newPartition.getPartition().getTableName(), newPartition.getPartition().getValues()); }
private void prepareInsertExistingPartition(HdfsContext context, PartitionAndMore partitionAndMore) { deleteOnly = false; Partition partition = partitionAndMore.getPartition(); Path targetPath = new Path(partition.getStorage().getLocation()); Path currentPath = partitionAndMore.getCurrentLocation(); cleanUpTasksForAbort.add(new DirectoryCleanUpTask(context, targetPath, false)); if (!targetPath.equals(currentPath)) { asyncRename(hdfsEnvironment, renameExecutor, fileRenameCancelled, fileRenameFutures, context, currentPath, targetPath, partitionAndMore.getFileNames()); } updateStatisticsOperations.add(new UpdateStatisticsOperation( new SchemaTableName(partition.getDatabaseName(), partition.getTableName()), Optional.of(getPartitionName(partition.getDatabaseName(), partition.getTableName(), partition.getValues())), partitionAndMore.getStatisticsUpdate(), true)); }
public static PartitionInput convertPartition(Partition partition) { PartitionInput input = new PartitionInput(); input.setValues(partition.getValues()); input.setStorageDescriptor(convertStorage(partition.getStorage(), partition.getColumns())); input.setParameters(partition.getParameters()); return input; }
@Test public void testConvertPartition() { PartitionInput partitionInput = GlueInputConverter.convertPartition(testPartition); assertEquals(partitionInput.getParameters(), testPartition.getParameters()); assertStorage(partitionInput.getStorageDescriptor(), testPartition.getStorage()); assertEquals(partitionInput.getValues(), testPartition.getValues()); }
private void verifiedPartition(Table table, Partition partition) { Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues()); if (table.getTableType().equals(MANAGED_TABLE.name())) { if (!partitionMetadataDirectory.equals(new Path(partition.getStorage().getLocation()))) { throw new PrestoException(HIVE_METASTORE_ERROR, "Partition directory must be " + partitionMetadataDirectory); } } else if (table.getTableType().equals(EXTERNAL_TABLE.name())) { try { Path externalLocation = new Path(partition.getStorage().getLocation()); FileSystem externalFileSystem = hdfsEnvironment.getFileSystem(hdfsContext, externalLocation); if (!externalFileSystem.isDirectory(externalLocation)) { throw new PrestoException(HIVE_METASTORE_ERROR, "External partition location does not exist"); } if (isChildDirectory(catalogDirectory, externalLocation)) { throw new PrestoException(HIVE_METASTORE_ERROR, "External partition location can not be inside the system metadata directory"); } } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not validate external partition location", e); } } else { throw new PrestoException(NOT_SUPPORTED, "Partitions can not be added to " + table.getTableType()); } }
public PartitionMetadata(Table table, PartitionWithStatistics partitionWithStatistics) { Partition partition = partitionWithStatistics.getPartition(); PartitionStatistics statistics = partitionWithStatistics.getStatistics(); this.columns = partition.getColumns(); this.parameters = updateStatisticsParameters(partition.getParameters(), statistics.getBasicStatistics()); StorageFormat tableFormat = partition.getStorage().getStorageFormat(); storageFormat = Arrays.stream(HiveStorageFormat.values()) .filter(format -> tableFormat.equals(StorageFormat.fromHiveStorageFormat(format))) .findFirst(); if (table.getTableType().equals(TableType.EXTERNAL_TABLE.name())) { externalLocation = Optional.of(partition.getStorage().getLocation()); } else { externalLocation = Optional.empty(); } bucketProperty = partition.getStorage().getBucketProperty(); serdeParameters = partition.getStorage().getSerdeParameters(); columnStatistics = ImmutableMap.copyOf(statistics.getColumnStatistics()); }
public PartitionWithStatistics(Partition partition, String partitionName, PartitionStatistics statistics) { this.partition = requireNonNull(partition, "partition is null"); this.partitionName = requireNonNull(partitionName, "partitionName is null"); checkArgument(toPartitionValues(partitionName).equals(partition.getValues()), "unexpected partition name: %s != %s", partitionName, partition.getValues()); this.statistics = requireNonNull(statistics, "statistics is null"); }
private static String getPartitionLocation(Table table, Optional<Partition> partition) { if (!partition.isPresent()) { return table.getStorage().getLocation(); } return partition.get().getStorage().getLocation(); }
public static Partition convertPartition(com.amazonaws.services.glue.model.Partition gluePartition) { requireNonNull(gluePartition.getStorageDescriptor(), "Partition StorageDescriptor is null"); StorageDescriptor sd = gluePartition.getStorageDescriptor(); Partition.Builder partitionBuilder = Partition.builder() .setDatabaseName(gluePartition.getDatabaseName()) .setTableName(gluePartition.getTableName()) .setValues(gluePartition.getValues()) .setColumns(sd.getColumns().stream() .map(GlueToPrestoConverter::convertColumn) .collect(toList())) .setParameters(firstNonNull(gluePartition.getParameters(), ImmutableMap.of())); setStorageBuilder(sd, partitionBuilder.getStorageBuilder()); return partitionBuilder.build(); }
public void undo(ExtendedHiveMetastore metastore) { if (!undo) { return; } metastore.alterPartition(oldPartition.getPartition().getDatabaseName(), oldPartition.getPartition().getTableName(), oldPartition); } }
public static Properties getHiveSchema(Partition partition, Table table) { // Mimics function in Hive: MetaStoreUtils.getSchema(Partition, Table) return getHiveSchema( partition.getStorage(), partition.getColumns(), table.getDataColumns(), table.getParameters(), table.getDatabaseName(), table.getTableName(), table.getPartitionColumns()); }
private static Optional<String> getPrestoQueryId(Partition partition) { return Optional.ofNullable(partition.getParameters().get(PRESTO_QUERY_ID_NAME)); }
@Override public void updatePartitionStatistics(String databaseName, String tableName, String partitionName, Function<PartitionStatistics, PartitionStatistics> update) { PartitionStatistics currentStatistics = getPartitionStatistics(databaseName, tableName, ImmutableSet.of(partitionName)).get(partitionName); if (currentStatistics == null) { throw new PrestoException(HIVE_PARTITION_DROPPED_DURING_QUERY, "Statistics result does not contain entry for partition: " + partitionName); } PartitionStatistics updatedStatistics = update.apply(currentStatistics); if (!updatedStatistics.getColumnStatistics().isEmpty()) { throw new PrestoException(NOT_SUPPORTED, "Glue metastore does not support column level statistics"); } List<String> partitionValues = toPartitionValues(partitionName); Partition partition = getPartition(databaseName, tableName, partitionValues) .orElseThrow(() -> new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), partitionValues)); try { PartitionInput partitionInput = GlueInputConverter.convertPartition(partition); partitionInput.setParameters(updateStatisticsParameters(partition.getParameters(), updatedStatistics.getBasicStatistics())); glueClient.updatePartition(new UpdatePartitionRequest() .withDatabaseName(databaseName) .withTableName(tableName) .withPartitionValueList(partition.getValues()) .withPartitionInput(partitionInput)); } catch (EntityNotFoundException e) { throw new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), partitionValues); } catch (AmazonServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } }
private void dropExtraColumnStatisticsAfterAlterPartition( String databaseName, String tableName, PartitionWithStatistics partitionWithStatistics) { List<String> dataColumns = partitionWithStatistics.getPartition().getColumns().stream() .map(Column::getName) .collect(toImmutableList()); Set<String> columnsWithMissingStatistics = new HashSet<>(dataColumns); columnsWithMissingStatistics.removeAll(partitionWithStatistics.getStatistics().getColumnStatistics().keySet()); // In case new partition had the statistics computed for all the columns, the storePartitionColumnStatistics // call in the alterPartition will just overwrite the old statistics. There is no need to explicitly remove anything. if (columnsWithMissingStatistics.isEmpty()) { return; } // check if statistics for the columnsWithMissingStatistics are actually stored in the metastore // when trying to remove any missing statistics the metastore throws NoSuchObjectException String partitionName = partitionWithStatistics.getPartitionName(); List<ColumnStatisticsObj> statisticsToBeRemoved = getMetastorePartitionColumnStatistics( databaseName, tableName, ImmutableSet.of(partitionName), ImmutableList.copyOf(columnsWithMissingStatistics)) .getOrDefault(partitionName, ImmutableList.of()); for (ColumnStatisticsObj statistics : statisticsToBeRemoved) { deletePartitionColumnStatistics(databaseName, tableName, partitionName, statistics.getColName()); } }
private void prepareAddPartition(HdfsContext context, PartitionAndMore partitionAndMore) { deleteOnly = false; Partition partition = partitionAndMore.getPartition(); String targetLocation = partition.getStorage().getLocation(); Path currentPath = partitionAndMore.getCurrentLocation(); Path targetPath = new Path(targetLocation); SchemaTableName schemaTableName = new SchemaTableName(partition.getDatabaseName(), partition.getTableName()); PartitionAdder partitionAdder = partitionAdders.computeIfAbsent( schemaTableName, ignored -> new PartitionAdder(partition.getDatabaseName(), partition.getTableName(), delegate, PARTITION_COMMIT_BATCH_SIZE)); if (pathExists(context, hdfsEnvironment, currentPath)) { if (!targetPath.equals(currentPath)) { renameDirectory( context, hdfsEnvironment, currentPath, targetPath, () -> cleanUpTasksForAbort.add(new DirectoryCleanUpTask(context, targetPath, true))); } } else { cleanUpTasksForAbort.add(new DirectoryCleanUpTask(context, targetPath, true)); createDirectory(context, hdfsEnvironment, targetPath); } String partitionName = getPartitionName(partition.getDatabaseName(), partition.getTableName(), partition.getValues()); partitionAdder.addPartition(new PartitionWithStatistics(partition, partitionName, partitionAndMore.getStatisticsUpdate())); }
throw new PrestoException(GENERIC_INTERNAL_ERROR, "Partition not loaded: " + hivePartition); String partName = makePartName(table.getPartitionColumns(), partition.getValues()); verifyOnline(tableName, Optional.of(partName), getProtectMode(partition), partition.getParameters()); String partitionNotReadable = partition.getParameters().get(OBJECT_NOT_READABLE); if (!isNullOrEmpty(partitionNotReadable)) { throw new HiveNotReadableException(tableName, Optional.of(partName), partitionNotReadable); List<Column> partitionColumns = partition.getColumns(); if ((tableColumns == null) || (partitionColumns == null)) { throw new PrestoException(HIVE_INVALID_METADATA, format("Table '%s' or partition '%s' has null columns", tableName, partName)); Optional<HiveBucketProperty> partitionBucketProperty = partition.getStorage().getBucketProperty(); if (!partitionBucketProperty.isPresent()) { throw new PrestoException(HIVE_PARTITION_SCHEMA_MISMATCH, format(