@Override public ConnectorPartition fromPartitionDto(final PartitionDto partitionDto) { return new ConnectorPartitionDetailImpl(partitionDto.getName().getPartitionName(), TupleDomain.none(), fromStorageDto(partitionDto.getSerde()), partitionDto.getMetadata(), fromAuditDto(partitionDto.getAudit())); } }
@Override public PartitionDto toPartitionDto(QualifiedName name, ConnectorPartition partition) { if ( name == null && partition == null ) { return null; } PartitionDto partitionDto = new PartitionDto(); if ( name != null ) { partitionDto.setName( name ); } if ( partition != null ) { partitionDto.setMetadata( toMetadata( partition ) ); partitionDto.setSerde( toStorageDto( partition ) ); partitionDto.setAudit( toAuditDto( partition ) ); } return partitionDto; }
private void appendPartitionsCore(final String dbName, final String tblName, final String partName) throws TException { final PartitionsSaveRequestDto partitionsSaveRequestDto = new PartitionsSaveRequestDto(); final PartitionDto partitionDto = new PartitionDto(); partitionDto.setName(QualifiedName.ofPartition(catalogName, dbName, tblName, partName)); partitionDto.setSerde(new StorageDto()); partitionsSaveRequestDto.setPartitions(Lists.newArrayList(partitionDto)); partV1.savePartitions(catalogName, dbName, tblName, partitionsSaveRequestDto); }
/** * Constructor. * * @param name name * @param requestContext context * @param source The source object which threw this event * @param partitions partition dtos */ public MetacatDeleteTablePartitionPostEvent( @Nonnull @NonNull final QualifiedName name, @Nonnull @NonNull final MetacatRequestContext requestContext, @Nonnull @NonNull final Object source, @Nonnull @NonNull final List<PartitionDto> partitions ) { super(name, requestContext, source); this.partitions = Collections.unmodifiableList(partitions); this.partitionIds = partitions.stream() .map(dto -> dto.getName().getPartitionName()).collect(Collectors.toList()); } }
/** * Save all tables to index it in elastic search. * * @param tableName database name * @param dtos partition dtos * @return future */ private ListenableFuture<Void> indexPartitionDtos(final QualifiedName tableName, final List<PartitionDto> dtos) { return esService.submit(() -> { final List<ElasticSearchDoc> docs = dtos.stream().filter(dto -> dto != null).map( dto -> { final String userName = dto.getAudit() != null ? dto.getAudit().getCreatedBy() : "admin"; return new ElasticSearchDoc(dto.getName().toString(), dto, userName, false, refreshMarkerText); }).collect(Collectors.toList()); log.info("Saving partitions for tableName: {}", tableName); elasticSearchUtil.save(ElasticSearchDoc.Type.partition.name(), docs); return null; }); }
QualifiedName.ofTable(name.getCatalogName(), VIEW_DB_NAME, createViewName(name)); partitionDtos.forEach(partitionDto -> partitionDto.setName(QualifiedName .ofPartition(viewQName.getCatalogName(), viewQName.getDatabaseName(), viewQName.getTableName(), partitionDto.getName().getPartitionName()))); final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext(); eventBus.post(new MetacatSaveMViewPartitionPreEvent(name, metacatRequestContext, this, dto)); partitionDto -> partitionDto.getName().getPartitionName()).collect(Collectors.toList()); final List<PartitionDto> existingPartitions = partitionService.list(viewQName, null, null, false, false, final Map<String, PartitionDto> existingPartitionsMap = existingPartitions.stream() .collect(Collectors .toMap(partitionDto -> partitionDto.getName().getPartitionName(), Function.identity())); final List<PartitionDto> mergedPartitions = partitionDtos.stream() .map(partitionDto -> { final String partitionName = partitionDto.getName().getPartitionName(); final PartitionDto existingPartition = existingPartitionsMap.get(partitionName); return mergePartition(partitionDto, existingPartition);
names.add(partitionDto.getName()); if (partitionDto.isDataExternal()) { uris.add(partitionDto.getDataUri()); final Map<String, ObjectNode> dataMetadataMap = metadataResults.get(1); result.forEach(partitionDto -> userMetadataService.populateMetadata(partitionDto, definitionMetadataMap.get(partitionDto.getName().toString()), prePopulatedMap.containsKey(partitionDto.getName().toString()) ? prePopulatedMap.get(partitionDto.getName().toString()) //using the prepopulated datametric : dataMetadataMap.get(partitionDto.getDataUri()))); } catch (Exception e) { Throwables.propagate(e);
private PartitionDto mergePartition(final PartitionDto partitionDto, @Nullable final PartitionDto existingPartition) { if (existingPartition != null) { final StorageDto existingSerde = existingPartition.getSerde(); if (existingSerde != null) { StorageDto serde = partitionDto.getSerde(); if (serde == null) { serde = new StorageDto(); partitionDto.setSerde(serde); } if (serde.getUri() == null || serde.getUri().equals(existingSerde.getUri())) { serde.setUri(existingSerde.getUri()); if (serde.getInputFormat() == null) { serde.setInputFormat(existingSerde.getInputFormat()); } if (serde.getOutputFormat() == null) { serde.setOutputFormat(existingSerde.getOutputFormat()); } if (serde.getSerializationLib() == null) { serde.setSerializationLib(existingSerde.getSerializationLib()); } } } } return partitionDto; }
/** * Constructor. * * @param name name * @param requestContext context * @param source The source object which threw this event * @param partitions partition dtos */ public MetacatDeleteTablePartitionPostEvent( @Nonnull @NonNull final QualifiedName name, @Nonnull @NonNull final MetacatRequestContext requestContext, @Nonnull @NonNull final Object source, @Nonnull @NonNull final List<PartitionDto> partitions ) { super(name, requestContext, source); this.partitions = Collections.unmodifiableList(partitions); this.partitionIds = partitions.stream() .map(dto -> dto.getName().getPartitionName()).collect(Collectors.toList()); } }
private void appendPartitionsCore(final String dbName, final String tblName, final String partName) throws TException { final PartitionsSaveRequestDto partitionsSaveRequestDto = new PartitionsSaveRequestDto(); final PartitionDto partitionDto = new PartitionDto(); partitionDto.setName(QualifiedName.ofPartition(catalogName, dbName, tblName, partName)); partitionDto.setSerde(new StorageDto()); partitionsSaveRequestDto.setPartitions(Lists.newArrayList(partitionDto)); partV1.savePartitions(catalogName, dbName, tblName, partitionsSaveRequestDto); }
final Partition result = new Partition(); final QualifiedName name = partitionDto.getName(); final List<String> values = Lists.newArrayListWithCapacity(16); String databaseName = ""; if (name != null) { if (name.getPartitionName() != null) { for (String partialPartName : SLASH_SPLITTER.split(partitionDto.getName().getPartitionName())) { final List<String> nameValues = ImmutableList.copyOf(EQUAL_SPLITTER.split(partialPartName)); if (nameValues.size() != 2) { throw new IllegalStateException("Unrecognized partition name: " + partitionDto.getName()); result.setTableName(tableName); Map<String, String> metadata = partitionDto.getMetadata(); if (metadata == null) { metadata = Collections.EMPTY_MAP; result.setSd(fromStorageDto(partitionDto.getSerde())); final StorageDescriptor sd = result.getSd(); if (tableDto != null) { final AuditDto auditDto = partitionDto.getAudit(); if (auditDto != null) { if (auditDto.getCreatedDate() != null) {
@Override public PartitionDto hiveToMetacatPartition(final TableDto tableDto, final Partition partition) { final QualifiedName tableName = tableDto.getName(); final QualifiedName partitionName = QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), getNameFromPartVals(tableDto, partition.getValues())); final PartitionDto result = new PartitionDto(); String owner = ""; if (tableDto.getSerde() != null) { owner = tableDto.getSerde().getOwner(); } result.setSerde(toStorageDto(partition.getSd(), owner)); result.setMetadata(partition.getParameters()); final AuditDto auditDto = new AuditDto(); auditDto.setCreatedDate(epochSecondsToDate(partition.getCreateTime())); auditDto.setLastModifiedDate(epochSecondsToDate(partition.getLastAccessTime())); result.setAudit(auditDto); result.setName(partitionName); return result; }
/** * Creates the qualified name representing a partition. * * @param tableName table name * @param partitionDto partition * @return qualified name */ public static QualifiedName ofPartition( @NonNull final QualifiedName tableName, @NonNull final PartitionDto partitionDto ) { return ofPartition( tableName.catalogName, tableName.databaseName, tableName.tableName, partitionDto.getName().getPartitionName() ); }
final Partition result = new Partition(); final QualifiedName name = partitionDto.getName(); List<String> values = Lists.newArrayListWithCapacity(16); String databaseName = null; result.setTableName(tableName); Map<String, String> metadata = partitionDto.getMetadata(); if (metadata == null) { metadata = Maps.newHashMap(); result.setSd(fromStorageDto(partitionDto.getSerde(), tableName)); final StorageDescriptor sd = result.getSd(); if (tableDto != null) { final AuditDto auditDto = partitionDto.getAudit(); if (auditDto != null) { if (auditDto.getCreatedDate() != null) {
/** * {@inheritDoc} */ @Override public PartitionDto hiveToMetacatPartition(final TableDto tableDto, final Partition partition) { final QualifiedName tableName = tableDto.getName(); final QualifiedName partitionName = QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), getNameFromPartVals(tableDto, partition.getValues())); final PartitionDto result = new PartitionDto(); String owner = ""; if (tableDto.getSerde() != null) { owner = tableDto.getSerde().getOwner(); } //not setting Serde to view if (null == tableDto.getView() || Strings.isNullOrEmpty(tableDto.getView().getViewOriginalText())) { result.setSerde(toStorageDto(partition.getSd(), owner)); } result.setMetadata(partition.getParameters()); final AuditDto auditDto = new AuditDto(); auditDto.setCreatedDate(epochSecondsToDate(partition.getCreateTime())); auditDto.setLastModifiedDate(epochSecondsToDate(partition.getLastAccessTime())); result.setAudit(auditDto); result.setName(partitionName); return result; }
/** * get descending order deletion column value. * * @param partitionDtos partition DTOs * @param deleteColumn delete column name * @return descending order deletion column */ @VisibleForTesting private static List<String> getSortedDeletionPartitionKeys(final List<PartitionDto> partitionDtos, final String deleteColumn) { return partitionDtos.stream() .map(x -> PartitionUtil.getPartitionKeyValues(x.getName().toString()).get(deleteColumn)) .filter(Objects::nonNull) .sorted(Comparator.reverseOrder()) .collect(Collectors.toList()); }
final Partition result = new Partition(); final QualifiedName name = partitionDto.getName(); List<String> values = Lists.newArrayListWithCapacity(16); String databaseName = null; result.setTableName(tableName); Map<String, String> metadata = partitionDto.getMetadata(); if (metadata == null) { metadata = Maps.newHashMap(); result.setSd(fromStorageDto(partitionDto.getSerde(), tableName)); final StorageDescriptor sd = result.getSd(); if (tableDto != null) { final AuditDto auditDto = partitionDto.getAudit(); if (auditDto != null) { if (auditDto.getCreatedDate() != null) {
/** * {@inheritDoc} */ @Override public PartitionDto hiveToMetacatPartition(final TableDto tableDto, final Partition partition) { final QualifiedName tableName = tableDto.getName(); final QualifiedName partitionName = QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), getNameFromPartVals(tableDto, partition.getValues())); final PartitionDto result = new PartitionDto(); String owner = ""; if (tableDto.getSerde() != null) { owner = tableDto.getSerde().getOwner(); } //not setting Serde to view if (null == tableDto.getView() || Strings.isNullOrEmpty(tableDto.getView().getViewOriginalText())) { result.setSerde(toStorageDto(partition.getSd(), owner)); } result.setMetadata(partition.getParameters()); final AuditDto auditDto = new AuditDto(); auditDto.setCreatedDate(epochSecondsToDate(partition.getCreateTime())); auditDto.setLastModifiedDate(epochSecondsToDate(partition.getLastAccessTime())); result.setAudit(auditDto); result.setName(partitionName); return result; }
/** * Creates the qualified name representing a partition. * * @param tableName table name * @param partitionDto partition * @return qualified name */ public static QualifiedName ofPartition( @NonNull final QualifiedName tableName, @NonNull final PartitionDto partitionDto ) { return ofPartition( tableName.catalogName, tableName.databaseName, tableName.tableName, partitionDto.getName().getPartitionName() ); }
/** * Subscriber. * * @param event event */ @EventListener public void metacatSaveTablePartitionPostEventHandler(final MetacatSaveTablePartitionPostEvent event) { log.debug("Received SaveTablePartitionEvent {}", event); this.partitionSaveEventsDelayTimer .record(System.currentTimeMillis() - event.getRequestContext().getTimestamp(), TimeUnit.MILLISECONDS); if (config.isElasticSearchPublishPartitionEnabled()) { this.partitionSaveTimer.record(() -> { final List<PartitionDto> partitionDtos = event.getPartitions(); final MetacatRequestContext context = event.getRequestContext(); final List<ElasticSearchDoc> docs = partitionDtos.stream() .map(dto -> new ElasticSearchDoc(dto.getName().toString(), dto, context.getUserName(), false)) .collect(Collectors.toList()); es.save(ElasticSearchDoc.Type.partition.name(), docs); }); } } }