@Deprecated public Partition appendPartition(String dbName, String tableName, List<String> partVals, EnvironmentContext ec) throws TException { return client.append_partition_with_environment_context(prependCatalogToDbName(dbName, conf), tableName, partVals, ec).deepCopy(); }
@Override public synchronized Optional<Partition> getPartition(String databaseName, String tableName, List<String> partitionValues) { PartitionName name = PartitionName.partition(databaseName, tableName, partitionValues); Partition partition = partitions.get(name); if (partition == null) { return Optional.empty(); } return Optional.of(partition.deepCopy()); }
@Override public synchronized List<Partition> getPartitionsByNames(String databaseName, String tableName, List<String> partitionNames) { ImmutableList.Builder<Partition> builder = ImmutableList.builder(); for (String name : partitionNames) { PartitionName partitionName = PartitionName.partition(databaseName, tableName, name); Partition partition = partitions.get(partitionName); if (partition == null) { return ImmutableList.of(); } builder.add(partition.deepCopy()); } return builder.build(); }
/** * Sets create time if not already set. */ private Partition getPartitionWithCreateTime(Partition partition, int createTime) { if (partition.isSetCreateTime() && partition.getCreateTime() > 0) { return partition; } Partition actualPartition = partition.deepCopy(); actualPartition.setCreateTime(createTime); return actualPartition; }
@Override public synchronized void updatePartitionStatistics(String databaseName, String tableName, String partitionName, Function<PartitionStatistics, PartitionStatistics> update) { PartitionStatistics currentStatistics = requireNonNull( getPartitionStatistics(databaseName, tableName, ImmutableSet.of(partitionName)).get(partitionName), "getPartitionStatistics() returned null"); PartitionStatistics updatedStatistics = update.apply(currentStatistics); List<Partition> partitions = getPartitionsByNames(databaseName, tableName, ImmutableList.of(partitionName)); if (partitions.size() != 1) { throw new PrestoException(HIVE_METASTORE_ERROR, "Metastore returned multiple partitions for name: " + partitionName); } Partition originalPartition = getOnlyElement(partitions); Partition modifiedPartition = originalPartition.deepCopy(); HiveBasicStatistics basicStatistics = updatedStatistics.getBasicStatistics(); modifiedPartition.setParameters(updateStatisticsParameters(modifiedPartition.getParameters(), basicStatistics)); alterPartitionWithoutStatistics(databaseName, tableName, modifiedPartition); Map<String, HiveType> columns = modifiedPartition.getSd().getCols().stream() .collect(toImmutableMap(FieldSchema::getName, schema -> HiveType.valueOf(schema.getType()))); setPartitionColumnStatistics(databaseName, tableName, partitionName, columns, updatedStatistics.getColumnStatistics(), basicStatistics.getRowCount()); Set<String> removedStatistics = difference(currentStatistics.getColumnStatistics().keySet(), updatedStatistics.getColumnStatistics().keySet()); removedStatistics.forEach(column -> deletePartitionColumnStatistics(databaseName, tableName, partitionName, column)); }
static Partition assemble(PartitionWrapper wrapper, SharedCache sharedCache) { Partition p = wrapper.getPartition().deepCopy(); if (wrapper.getSdHash() != null) { StorageDescriptor sdCopy = sharedCache.getSdFromCache(wrapper.getSdHash()).deepCopy(); if (sdCopy.getBucketCols() == null) { sdCopy.setBucketCols(Collections.emptyList()); } if (sdCopy.getSortCols() == null) { sdCopy.setSortCols(Collections.emptyList()); } if (sdCopy.getSkewedInfo() == null) { sdCopy.setSkewedInfo(new SkewedInfo(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap())); } sdCopy.setLocation(wrapper.getLocation()); sdCopy.setParameters(wrapper.getParameters()); p.setSd(sdCopy); } return p; }
private Partition getTargetPartition(Partition originPartition, Path targetLocation) throws IOException { try { Partition targetPartition = new Partition(this.hiveCopyEntityHelper.getTargetTable(), originPartition.getTPartition().deepCopy()); targetPartition.getTable().setDbName(this.hiveCopyEntityHelper.getTargetDatabase()); targetPartition.getTPartition().setDbName(this.hiveCopyEntityHelper.getTargetDatabase()); targetPartition.getTPartition().putToParameters(HiveDataset.REGISTERER, HiveCopyEntityHelper.GOBBLIN_DISTCP); targetPartition.getTPartition().putToParameters(HiveDataset.REGISTRATION_GENERATION_TIME_MILLIS, Long.toString(this.hiveCopyEntityHelper.getStartTime())); targetPartition.setLocation(targetLocation.toString()); targetPartition.getTPartition().unsetCreateTime(); return targetPartition; } catch (HiveException he) { throw new IOException(he); } }
private PartitionWrapper makePartitionWrapper(Partition part, SharedCache sharedCache) { Partition partCopy = part.deepCopy(); PartitionWrapper wrapper; if (part.getSd() != null) { byte[] sdHash = MetaStoreServerUtils.hashStorageDescriptor(part.getSd(), md); StorageDescriptor sd = part.getSd(); sharedCache.increSd(sd, sdHash); partCopy.setSd(null); wrapper = new PartitionWrapper(partCopy, sdHash, sd.getLocation(), sd.getParameters()); } else { wrapper = new PartitionWrapper(partCopy, null, null, null); } return wrapper; } }
public PartitionWrapper(org.apache.hadoop.hive.metastore.api.Partition mapiPart, PreEventContext context) throws HiveException, NoSuchObjectException, MetaException { org.apache.hadoop.hive.metastore.api.Partition wrapperApiPart = mapiPart.deepCopy(); org.apache.hadoop.hive.metastore.api.Table t = context.getHandler().get_table_core( mapiPart.getDbName(), mapiPart.getTableName()); if (wrapperApiPart.getSd() == null){ // In the cases of create partition, by the time this event fires, the partition // object has not yet come into existence, and thus will not yet have a // location or an SD, but these are needed to create a ql.metadata.Partition, // so we use the table's SD. The only place this is used is by the // authorization hooks, so we will not affect code flow in the metastore itself. wrapperApiPart.setSd(t.getSd().deepCopy()); } initialize(new TableWrapper(t),wrapperApiPart); } }
public PartitionWrapper(org.apache.hadoop.hive.metastore.api.Partition mapiPart, PreEventContext context) throws HiveException, NoSuchObjectException, MetaException { org.apache.hadoop.hive.metastore.api.Partition wrapperApiPart = mapiPart.deepCopy(); String catName = mapiPart.isSetCatName() ? mapiPart.getCatName() : MetaStoreUtils.getDefaultCatalog(context.getHandler().getConf()); org.apache.hadoop.hive.metastore.api.Table t = context.getHandler().get_table_core( catName, mapiPart.getDbName(), mapiPart.getTableName()); if (wrapperApiPart.getSd() == null){ // In the cases of create partition, by the time this event fires, the partition // object has not yet come into existence, and thus will not yet have a // location or an SD, but these are needed to create a ql.metadata.Partition, // so we use the table's SD. The only place this is used is by the // authorization hooks, so we will not affect code flow in the metastore itself. wrapperApiPart.setSd(t.getSd().deepCopy()); } initialize(new TableWrapper(t),wrapperApiPart); } }
/** * Sets create time if not already set. */ private Partition getPartitionWithCreateTime(Partition partition, int createTime) { if (partition.isSetCreateTime() && partition.getCreateTime() > 0) { return partition; } Partition actualPartition = partition.deepCopy(); actualPartition.setCreateTime(createTime); return actualPartition; }
@Override public Optional<Partition> getPartition(String databaseName, String tableName, String partitionName) { PartitionName name = new PartitionName(databaseName, tableName, partitionName); Partition partition = partitions.get(name); if (partition == null) { return Optional.empty(); } return Optional.of(partition.deepCopy()); }
/** * Sets create time if not already set. */ private Partition getPartitionWithCreateTime(Partition partition, int createTime) { if (partition.isSetCreateTime() && partition.getCreateTime() > 0) { return partition; } Partition actualPartition = partition.deepCopy(); actualPartition.setCreateTime(createTime); return actualPartition; }
@Override public synchronized Optional<Partition> getPartition(String databaseName, String tableName, List<String> partitionValues) { PartitionName name = PartitionName.partition(databaseName, tableName, partitionValues); Partition partition = partitions.get(name); if (partition == null) { return Optional.empty(); } return Optional.of(partition.deepCopy()); }
@Override public synchronized Optional<Partition> getPartition(String databaseName, String tableName, List<String> partitionValues) { PartitionName name = PartitionName.partition(databaseName, tableName, partitionValues); Partition partition = partitions.get(name); if (partition == null) { return Optional.empty(); } return Optional.of(partition.deepCopy()); }
@Override public synchronized Optional<Partition> getPartition(String databaseName, String tableName, List<String> partitionValues) { PartitionName name = PartitionName.partition(databaseName, tableName, partitionValues); Partition partition = partitions.get(name); if (partition == null) { return Optional.empty(); } return Optional.of(partition.deepCopy()); }
@Override public synchronized List<Partition> getPartitionsByNames(String databaseName, String tableName, List<String> partitionNames) { ImmutableList.Builder<Partition> builder = ImmutableList.builder(); for (String name : partitionNames) { PartitionName partitionName = PartitionName.partition(databaseName, tableName, name); Partition partition = partitions.get(partitionName); if (partition == null) { return ImmutableList.of(); } builder.add(partition.deepCopy()); } return builder.build(); }
@Override public synchronized List<Partition> getPartitionsByNames(String databaseName, String tableName, List<String> partitionNames) { ImmutableList.Builder<Partition> builder = ImmutableList.builder(); for (String name : partitionNames) { PartitionName partitionName = PartitionName.partition(databaseName, tableName, name); Partition partition = partitions.get(partitionName); if (partition == null) { return ImmutableList.of(); } builder.add(partition.deepCopy()); } return builder.build(); }
@Override public Optional<Map<String, Partition>> getPartitionsByNames(String databaseName, String tableName, List<String> partitionNames) { ImmutableMap.Builder<String, Partition> builder = ImmutableMap.builder(); for (String name : partitionNames) { PartitionName partitionName = new PartitionName(databaseName, tableName, name); Partition partition = partitions.get(partitionName); if (partition == null) { return Optional.empty(); } builder.put(name, partition.deepCopy()); } return Optional.of(builder.build()); }
@Override public PartitionAndMetadata apply(PartitionAndMetadata partitionAndMetadata) { if (partitionAndMetadata == null) { return null; } Partition partitionCopy = partitionAndMetadata.getPartition().deepCopy(); for (CircusTrainTableParameter p : CircusTrainTableParameter.values()) { partitionCopy.getParameters().remove(p.parameterName()); } for (ExcludedHiveParameter p : ExcludedHiveParameter.values()) { partitionCopy.getParameters().remove(p.parameterName()); } return new PartitionAndMetadata(partitionAndMetadata.getSourceTable(), partitionAndMetadata.getSourceLocation(), partitionCopy); }