/** * Constructor. * * @param name name * @param requestContext context * @param source The source object which threw this event * @param partitions partition dtos */ public MetacatDeleteTablePartitionPostEvent( @Nonnull @NonNull final QualifiedName name, @Nonnull @NonNull final MetacatRequestContext requestContext, @Nonnull @NonNull final Object source, @Nonnull @NonNull final List<PartitionDto> partitions ) { super(name, requestContext, source); this.partitions = Collections.unmodifiableList(partitions); this.partitionIds = partitions.stream() .map(dto -> dto.getName().getPartitionName()).collect(Collectors.toList()); } }
/** * Constructor. * * @param name name * @param requestContext context * @param source The source object which threw this event * @param partitions partition dtos */ public MetacatDeleteTablePartitionPostEvent( @Nonnull @NonNull final QualifiedName name, @Nonnull @NonNull final MetacatRequestContext requestContext, @Nonnull @NonNull final Object source, @Nonnull @NonNull final List<PartitionDto> partitions ) { super(name, requestContext, source); this.partitions = Collections.unmodifiableList(partitions); this.partitionIds = partitions.stream() .map(dto -> dto.getName().getPartitionName()).collect(Collectors.toList()); } }
/** * get descending order deletion column value. * * @param partitionDtos partition DTOs * @param deleteColumn delete column name * @return descending order deletion column */ @VisibleForTesting private static List<String> getSortedDeletionPartitionKeys(final List<PartitionDto> partitionDtos, final String deleteColumn) { return partitionDtos.stream() .map(x -> PartitionUtil.getPartitionKeyValues(x.getName().toString()).get(deleteColumn)) .filter(Objects::nonNull) .sorted(Comparator.reverseOrder()) .collect(Collectors.toList()); }
/** * Creates the qualified name representing a partition. * * @param tableName table name * @param partitionDto partition * @return qualified name */ public static QualifiedName ofPartition( @NonNull final QualifiedName tableName, @NonNull final PartitionDto partitionDto ) { return ofPartition( tableName.catalogName, tableName.databaseName, tableName.tableName, partitionDto.getName().getPartitionName() ); }
/** * Creates the qualified name representing a partition. * * @param tableName table name * @param partitionDto partition * @return qualified name */ public static QualifiedName ofPartition( @NonNull final QualifiedName tableName, @NonNull final PartitionDto partitionDto ) { return ofPartition( tableName.catalogName, tableName.databaseName, tableName.tableName, partitionDto.getName().getPartitionName() ); }
/** * Save all tables to index it in elastic search. * * @param tableName database name * @param dtos partition dtos * @return future */ private ListenableFuture<Void> indexPartitionDtos(final QualifiedName tableName, final List<PartitionDto> dtos) { return esService.submit(() -> { final List<ElasticSearchDoc> docs = dtos.stream().filter(dto -> dto != null).map( dto -> { final String userName = dto.getAudit() != null ? dto.getAudit().getCreatedBy() : "admin"; return new ElasticSearchDoc(dto.getName().toString(), dto, userName, false, refreshMarkerText); }).collect(Collectors.toList()); log.info("Saving partitions for tableName: {}", tableName); elasticSearchUtil.save(ElasticSearchDoc.Type.partition.name(), docs); return null; }); }
/** * Subscriber. * * @param event event */ @EventListener public void metacatSaveTablePartitionPostEventHandler(final MetacatSaveTablePartitionPostEvent event) { log.debug("Received SaveTablePartitionEvent {}", event); this.partitionSaveEventsDelayTimer .record(System.currentTimeMillis() - event.getRequestContext().getTimestamp(), TimeUnit.MILLISECONDS); if (config.isElasticSearchPublishPartitionEnabled()) { this.partitionSaveTimer.record(() -> { final List<PartitionDto> partitionDtos = event.getPartitions(); final MetacatRequestContext context = event.getRequestContext(); final List<ElasticSearchDoc> docs = partitionDtos.stream() .map(dto -> new ElasticSearchDoc(dto.getName().toString(), dto, context.getUserName(), false)) .collect(Collectors.toList()); es.save(ElasticSearchDoc.Type.partition.name(), docs); }); } } }
names.add(partitionDto.getName()); if (partitionDto.isDataExternal()) { uris.add(partitionDto.getDataUri()); final Map<String, ObjectNode> dataMetadataMap = metadataResults.get(1); result.forEach(partitionDto -> userMetadataService.populateMetadata(partitionDto, definitionMetadataMap.get(partitionDto.getName().toString()), prePopulatedMap.containsKey(partitionDto.getName().toString()) ? prePopulatedMap.get(partitionDto.getName().toString()) //using the prepopulated datametric : dataMetadataMap.get(partitionDto.getDataUri()))); } catch (Exception e) {
partitionDto.setName(QualifiedName .ofPartition(viewQName.getCatalogName(), viewQName.getDatabaseName(), viewQName.getTableName(), partitionDto.getName().getPartitionName()))); final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext(); eventBus.post(new MetacatSaveMViewPartitionPreEvent(name, metacatRequestContext, this, dto)); partitionDto -> partitionDto.getName().getPartitionName()).collect(Collectors.toList()); final List<PartitionDto> existingPartitions = partitionService.list(viewQName, null, null, false, false, final Map<String, PartitionDto> existingPartitionsMap = existingPartitions.stream() .collect(Collectors .toMap(partitionDto -> partitionDto.getName().getPartitionName(), Function.identity())); final List<PartitionDto> mergedPartitions = partitionDtos.stream() .map(partitionDto -> { final String partitionName = partitionDto.getName().getPartitionName(); final PartitionDto existingPartition = existingPartitionsMap.get(partitionName); return mergePartition(partitionDto, existingPartition);
@Override public ConnectorPartition fromPartitionDto(final PartitionDto partitionDto) { return new ConnectorPartitionDetailImpl(partitionDto.getName().getPartitionName(), TupleDomain.none(), fromStorageDto(partitionDto.getSerde()), partitionDto.getMetadata(), fromAuditDto(partitionDto.getAudit())); } }
final Partition result = new Partition(); final QualifiedName name = partitionDto.getName(); final List<String> values = Lists.newArrayListWithCapacity(16); String databaseName = ""; if (name != null) { if (name.getPartitionName() != null) { for (String partialPartName : SLASH_SPLITTER.split(partitionDto.getName().getPartitionName())) { final List<String> nameValues = ImmutableList.copyOf(EQUAL_SPLITTER.split(partialPartName)); if (nameValues.size() != 2) { throw new IllegalStateException("Unrecognized partition name: " + partitionDto.getName());
/** * {@inheritDoc} */ @Override public boolean drop_partition_with_environment_context( final String dbName, final String tblName, final List<String> partVals, final boolean deleteData, @Nullable final EnvironmentContext environmentContext ) throws TException { return requestWrapper("drop_partition_with_environment_context", new Object[]{dbName, tblName, partVals, deleteData, environmentContext}, () -> { final TableDto tableDto = getTableDto(dbName, tblName); final String partName = hiveConverters.getNameFromPartVals(tableDto, partVals); final QualifiedName partitionName = getPartitionDtoByName(tableDto, partName).getName(); if (deleteData) { log.warn("Ignoring command to delete data for {}/{}/{}/{}", catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(), partitionName.getPartitionName()); } partV1.deletePartitions( catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(), ImmutableList.of(partitionName.getPartitionName())); return true; }); }
/** * {@inheritDoc} */ @Override public boolean drop_partition_with_environment_context( final String dbName, final String tblName, final List<String> partVals, final boolean deleteData, @Nullable final EnvironmentContext environmentContext ) throws TException { return requestWrapper("drop_partition_with_environment_context", new Object[]{dbName, tblName, partVals, deleteData, environmentContext}, () -> { final TableDto tableDto = getTableDto(dbName, tblName); final String partName = hiveConverters.getNameFromPartVals(tableDto, partVals); final QualifiedName partitionName = getPartitionDtoByName(tableDto, partName).getName(); if (deleteData) { log.warn("Ignoring command to delete data for {}/{}/{}/{}", catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(), partitionName.getPartitionName()); } partV1.deletePartitions( catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(), ImmutableList.of(partitionName.getPartitionName())); return true; }); }
final Partition result = new Partition(); final QualifiedName name = partitionDto.getName(); List<String> values = Lists.newArrayListWithCapacity(16); String databaseName = null;
final Partition result = new Partition(); final QualifiedName name = partitionDto.getName(); List<String> values = Lists.newArrayListWithCapacity(16); String databaseName = null;