public AlterPartitionOperation(PartitionWithStatistics newPartition, PartitionWithStatistics oldPartition) { this.newPartition = requireNonNull(newPartition, "newPartition is null"); this.oldPartition = requireNonNull(oldPartition, "oldPartition is null"); checkArgument(newPartition.getPartition().getDatabaseName().equals(oldPartition.getPartition().getDatabaseName())); checkArgument(newPartition.getPartition().getTableName().equals(oldPartition.getPartition().getTableName())); checkArgument(newPartition.getPartition().getValues().equals(oldPartition.getPartition().getValues())); }
public String getDescription() { return format( "alter partition %s.%s %s", newPartition.getPartition().getDatabaseName(), newPartition.getPartition().getTableName(), newPartition.getPartition().getValues()); }
public void undo(ExtendedHiveMetastore metastore) { if (!undo) { return; } metastore.alterPartition(oldPartition.getPartition().getDatabaseName(), oldPartition.getPartition().getTableName(), oldPartition); } }
public void run(ExtendedHiveMetastore metastore) { undo = true; metastore.alterPartition(newPartition.getPartition().getDatabaseName(), newPartition.getPartition().getTableName(), newPartition); }
private PartitionStatistics getExistingPartitionStatistics(Partition partition, String partitionName) { try { PartitionStatistics statistics = delegate.getPartitionStatistics(partition.getDatabaseName(), partition.getTableName(), ImmutableSet.of(partitionName)) .get(partitionName); if (statistics == null) { throw new PrestoException( TRANSACTION_CONFLICT, format("The partition that this transaction modified was deleted in another transaction. %s %s", partition.getTableName(), partition.getValues())); } return statistics; } catch (PrestoException e) { if (e.getErrorCode().equals(HIVE_CORRUPTED_COLUMN_STATISTICS.toErrorCode())) { log.warn( e, "Corrupted statistics found when altering partition. Table: %s.%s. Partition: %s", partition.getDatabaseName(), partition.getTableName(), partition.getValues()); return PartitionStatistics.empty(); } throw e; } }
public static void checkPartitionIsWritable(String partitionName, Partition partition) { checkWritable( new SchemaTableName(partition.getDatabaseName(), partition.getTableName()), Optional.of(partitionName), getProtectMode(partition), partition.getParameters(), partition.getStorage()); }
private void prepareInsertExistingPartition(HdfsContext context, PartitionAndMore partitionAndMore) { deleteOnly = false; Partition partition = partitionAndMore.getPartition(); Path targetPath = new Path(partition.getStorage().getLocation()); Path currentPath = partitionAndMore.getCurrentLocation(); cleanUpTasksForAbort.add(new DirectoryCleanUpTask(context, targetPath, false)); if (!targetPath.equals(currentPath)) { asyncRename(hdfsEnvironment, renameExecutor, fileRenameCancelled, fileRenameFutures, context, currentPath, targetPath, partitionAndMore.getFileNames()); } updateStatisticsOperations.add(new UpdateStatisticsOperation( new SchemaTableName(partition.getDatabaseName(), partition.getTableName()), Optional.of(getPartitionName(partition.getDatabaseName(), partition.getTableName(), partition.getValues())), partitionAndMore.getStatisticsUpdate(), true)); }
public static org.apache.hadoop.hive.metastore.api.Partition toMetastoreApiPartition(Partition partition) { org.apache.hadoop.hive.metastore.api.Partition result = new org.apache.hadoop.hive.metastore.api.Partition(); result.setDbName(partition.getDatabaseName()); result.setTableName(partition.getTableName()); result.setValues(partition.getValues()); result.setSd(makeStorageDescriptor(partition.getTableName(), partition.getColumns(), partition.getStorage())); result.setParameters(partition.getParameters()); return result; }
private Builder(Partition partition) { this.storageBuilder = Storage.builder(partition.getStorage()); this.databaseName = partition.getDatabaseName(); this.tableName = partition.getTableName(); this.values = partition.getValues(); this.columns = partition.getColumns(); this.parameters = partition.getParameters(); }
private void prepareAddPartition(HdfsContext context, PartitionAndMore partitionAndMore) { deleteOnly = false; Partition partition = partitionAndMore.getPartition(); String targetLocation = partition.getStorage().getLocation(); Path currentPath = partitionAndMore.getCurrentLocation(); Path targetPath = new Path(targetLocation); SchemaTableName schemaTableName = new SchemaTableName(partition.getDatabaseName(), partition.getTableName()); PartitionAdder partitionAdder = partitionAdders.computeIfAbsent( schemaTableName, ignored -> new PartitionAdder(partition.getDatabaseName(), partition.getTableName(), delegate, PARTITION_COMMIT_BATCH_SIZE)); if (pathExists(context, hdfsEnvironment, currentPath)) { if (!targetPath.equals(currentPath)) { renameDirectory( context, hdfsEnvironment, currentPath, targetPath, () -> cleanUpTasksForAbort.add(new DirectoryCleanUpTask(context, targetPath, true))); } } else { cleanUpTasksForAbort.add(new DirectoryCleanUpTask(context, targetPath, true)); createDirectory(context, hdfsEnvironment, targetPath); } String partitionName = getPartitionName(partition.getDatabaseName(), partition.getTableName(), partition.getValues()); partitionAdder.addPartition(new PartitionWithStatistics(partition, partitionName, partitionAndMore.getStatisticsUpdate())); }
Optional<Partition> oldPartition = delegate.getPartition(partition.getDatabaseName(), partition.getTableName(), partition.getValues()); if (!oldPartition.isPresent()) { throw new PrestoException( TRANSACTION_CONFLICT, format("The partition that this transaction modified was deleted in another transaction. %s %s", partition.getTableName(), partition.getValues())); String partitionName = getPartitionName(partition.getDatabaseName(), partition.getTableName(), partition.getValues()); PartitionStatistics oldPartitionStatistics = getExistingPartitionStatistics(partition, partitionName); String oldPartitionLocation = oldPartition.get().getStorage().getLocation();
@Test public void testConvertPartition() { com.facebook.presto.hive.metastore.Partition prestoPartition = GlueToPrestoConverter.convertPartition(testPartition); assertEquals(prestoPartition.getDatabaseName(), testPartition.getDatabaseName()); assertEquals(prestoPartition.getTableName(), testPartition.getTableName()); assertColumnList(prestoPartition.getColumns(), testPartition.getStorageDescriptor().getColumns()); assertEquals(prestoPartition.getValues(), testPartition.getValues()); assertStorage(prestoPartition.getStorage(), testPartition.getStorageDescriptor()); assertEquals(prestoPartition.getParameters(), testPartition.getParameters()); }
@Test public void testConvertPartition() { com.facebook.presto.hive.metastore.Partition prestoPartition = GlueToPrestoConverter.convertPartition(testPartition); assertEquals(prestoPartition.getDatabaseName(), testPartition.getDatabaseName()); assertEquals(prestoPartition.getTableName(), testPartition.getTableName()); assertColumnList(prestoPartition.getColumns(), testPartition.getStorageDescriptor().getColumns()); assertEquals(prestoPartition.getValues(), testPartition.getValues()); assertStorage(prestoPartition.getStorage(), testPartition.getStorageDescriptor()); assertEquals(prestoPartition.getParameters(), testPartition.getParameters()); }