final List<String> partitionNames = partitionInfos.stream() .map(part -> { final String partitionName = part.getName().getPartitionName(); PartitionUtil.validatePartitionName(partitionName, getPartitionKeys(table.getPartitionKeys())); return partitionName; final String partitionName = partitionInfo.getName().getPartitionName(); final PartitionHolder existingPartitionHolder = existingPartitionMap.get(partitionName); if (existingPartitionHolder == null) { } else { final String partitionUri = partitionInfo.getSerde() != null ? partitionInfo.getSerde().getUri() : null; final String existingPartitionUri = getPartitionUri(existingPartitionHolder); if (partitionUri == null || !partitionUri.equals(existingPartitionUri)) { existingPartitionNames.add(partitionName); if (partitionInfo.getSerde() == null) { partitionInfo.setSerde(new StorageInfo()); if (partitionInfo.getAudit() == null) { partitionInfo.setAudit(new AuditInfo()); partitionInfo.getSerde().setUri(existingPartitionUri); if (existingPartitionHolder.getPartition() != null) { final Partition existingPartition = existingPartitionHolder.getPartition(); partitionInfo.getSerde().setParameters(existingPartition.getParameters()); partitionInfo.getAudit().setCreatedDate(
/** * Converts from partition info to s3 partition object. * @param partitionInfo partition info * @return s3 partition */ Partition fromPartitionInfo(final PartitionInfo partitionInfo) { final Partition result = new Partition(); result.setName(partitionInfo.getName().getPartitionName()); result.setUri(partitionInfo.getSerde().getUri()); final AuditInfo auditInfo = partitionInfo.getAudit(); if (auditInfo != null) { result.setCreatedDate(auditInfo.getCreatedDate()); result.setLastUpdatedDate(auditInfo.getLastModifiedDate()); } return result; }
for (PartitionHolder partitionHolder : partitionHolders) { final PartitionInfo partition = partitionHolder.getPartitionInfo(); final StorageInfo storageInfo = partition.getSerde(); final long partId = partitionHolder.getId(); final long sdsId = partitionHolder.getSdId(); final long serdeId = partitionHolder.getSerdeId(); final Map<String, String> parameters = partition.getMetadata(); if (parameters != null) { parameters storageInfo.getInputFormat(), sdsId, }); partitionNames.add(partition.getName().toString());
final PartitionInfo partition ) { final QualifiedName name = partition.getName(); final List<String> values = Lists.newArrayListWithCapacity(16); Map<String, String> metadata = partition.getMetadata(); if (metadata == null) { metadata = new HashMap<>(); .collect(Collectors.toList()); final StorageDescriptor sd = fromStorageInfo(partition.getSerde(), fieldSchemas); sd.getSerdeInfo().setSerializationLib(tableInfo.getSerde().getSerializationLib()); final AuditInfo auditInfo = partition.getAudit(); final int createTime = (notNull(auditInfo) && notNull(auditInfo.getCreatedDate())) ? dateToEpochSeconds(auditInfo.getCreatedDate()) : 0; for (String partialPartName : SLASH_SPLITTER.split(partition.getName().getPartitionName())) { final List<String> nameValues = ImmutableList.copyOf(EQUAL_SPLITTER.split(partialPartName)); Preconditions.checkState(nameValues.size() == 2, "Unrecognized partition name: " + partition.getName()); values.add(nameValues.get(1));
PartitionInfo.builder().name(QualifiedName.ofPartition(catalogName, databaseName, tableName, name)).auditInfo(auditInfo).serde(storageInfo).build())); partitionHolder.getPartitionInfo().setMetadata(partitionParams.get(partitionHolder.getId())); partitionHolder.getPartitionInfo().getSerde() .setParameters(sdParams.get(partitionHolder.getSdId())); partitionHolder.getPartitionInfo().getSerde() .setSerdeInfoParameters(serdeParams.get(partitionHolder.getSerdeId()));
@Override public List<PartitionInfo> list(@Nonnull final ConnectorRequestContext context, @Nonnull final QualifiedName name, @Nullable final QualifiedName prefix, @Nullable final Sort sort, @Nullable final Pageable pageable) { log.debug("Get partitions for table {} with name prefix {}", name, prefix); return _getPartitions(name, null, null, sort, pageable, true).stream() .filter(p -> p.getName().getPartitionName().startsWith(prefix.getPartitionName())) .collect(Collectors.toList()); }
.map(partitionName -> PartitionInfo.builder().name( QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), final Comparator<PartitionInfo> nameComparator = Comparator.comparing(p -> p.getName().toString()); ConnectorUtils.sort(filteredPartitionList, sort, nameComparator);
final Table table, final boolean doFileSystemCalls) { String location = partitionInfo.getSerde().getUri(); Path path = null; if (StringUtils.isBlank(location)) { throw new InvalidMetaException(tableQName, null); final String partitionName = partitionInfo.getName().getPartitionName(); final List<String> partValues = PartitionUtil .getPartValuesFromPartName(tableQName, table, partitionName); partitionInfo.getSerde().setUri(location); if (doFileSystemCalls) { registry.counter(registry.createId(HiveMetrics.CounterHivePartitionFileSystemCall.getMetricName())
/** * Convert from data source to partitionInfo. * * @param segment segment object * @return partition info object */ public PartitionInfo getPartitionInfoFromSegment(final Segment segment) { final Map<String, String> metadata = new HashMap<>(); metadata.put(DruidConfigConstants.LOADSPEC_KEY, segment.getLoadSpec().getKeys().toString()); metadata.put(DruidConfigConstants.LOADSPEC_BUCKET, segment.getLoadSpec().getBucket()); metadata.put(DruidConfigConstants.LOADSPEC_TYPE, segment.getLoadSpec().getType()); metadata.put(DruidConfigConstants.DIMENSIONS, segment.getDimensions()); metadata.put(DruidConfigConstants.METRICS, segment.getMetric()); final StorageInfo storageInfo = StorageInfo.builder().uri(segment.getLoadSpec().getUri()).build(); return PartitionInfo.builder().metadata(metadata).serde(storageInfo).build(); }
private void copyTableSdToPartitionInfoSd(final PartitionInfo partitionInfo, final Table table) { StorageInfo sd = partitionInfo.getSerde(); // // Partitions can be provided in the request without the storage information. // if (sd == null) { sd = new StorageInfo(); partitionInfo.setSerde(sd); } final StorageDescriptor tableSd = table.getSd(); if (StringUtils.isBlank(sd.getInputFormat())) { sd.setInputFormat(tableSd.getInputFormat()); } if (StringUtils.isBlank(sd.getOutputFormat())) { sd.setOutputFormat(tableSd.getOutputFormat()); } if (sd.getParameters() == null || sd.getParameters().isEmpty()) { sd.setParameters(tableSd.getParameters()); } final SerDeInfo tableSerde = tableSd.getSerdeInfo(); if (tableSerde != null) { if (StringUtils.isBlank(sd.getSerializationLib())) { sd.setSerializationLib(tableSerde.getSerializationLib()); } if (sd.getSerdeInfoParameters() == null || sd.getSerdeInfoParameters().isEmpty()) { sd.setSerdeInfoParameters(tableSerde.getParameters()); } } }
final List<String> uris = Lists.newArrayList(); final Map<String, ObjectNode> prePopulatedMap = new HashMap<>(); resultInfo.stream().filter(partitionInfo -> partitionInfo.getDataMetrics() != null) .forEach(partitionInfo -> prePopulatedMap.put(partitionInfo.getName().toString(), partitionInfo.getDataMetrics()));
for (PartitionHolder partitionHolder : partitionHolders) { final PartitionInfo partition = partitionHolder.getPartitionInfo(); final StorageInfo storageInfo = partition.getSerde(); final long partId = partitionHolder.getId(); final long sdsId = partitionHolder.getSdId(); final long serdeId = partitionHolder.getSerdeId(); final Map<String, String> parameters = partition.getMetadata(); if (parameters != null) { parameters storageInfo.getInputFormat(), sdsId, }); partitionNames.add(partition.getName().toString());
final PartitionInfo partition ) { final QualifiedName name = partition.getName(); final List<String> values = Lists.newArrayListWithCapacity(16); Map<String, String> metadata = partition.getMetadata(); if (metadata == null) { metadata = new HashMap<>(); .collect(Collectors.toList()); final StorageDescriptor sd = fromStorageInfo(partition.getSerde(), fieldSchemas); sd.getSerdeInfo().setSerializationLib(tableInfo.getSerde().getSerializationLib()); final AuditInfo auditInfo = partition.getAudit(); final int createTime = (notNull(auditInfo) && notNull(auditInfo.getCreatedDate())) ? dateToEpochSeconds(auditInfo.getCreatedDate()) : 0; for (String partialPartName : SLASH_SPLITTER.split(partition.getName().getPartitionName())) { final List<String> nameValues = ImmutableList.copyOf(EQUAL_SPLITTER.split(partialPartName)); Preconditions.checkState(nameValues.size() == 2, "Unrecognized partition name: " + partition.getName()); values.add(nameValues.get(1));
PartitionInfo.builder().name(QualifiedName.ofPartition(catalogName, databaseName, tableName, name)).auditInfo(auditInfo).serde(storageInfo).build())); partitionHolder.getPartitionInfo().setMetadata(partitionParams.get(partitionHolder.getId())); partitionHolder.getPartitionInfo().getSerde() .setParameters(sdParams.get(partitionHolder.getSdId())); partitionHolder.getPartitionInfo().getSerde() .setSerdeInfoParameters(serdeParams.get(partitionHolder.getSerdeId()));
@Override public List<String> getPartitionKeys(@Nonnull final ConnectorRequestContext context, @Nonnull final QualifiedName tableName, @Nonnull final PartitionListRequest partitionsRequest, final TableInfo tableInfo) { log.debug("Get partition keys for table {}", tableName); return _getPartitions(tableName, partitionsRequest.getFilter(), partitionsRequest.getPartitionNames(), partitionsRequest.getSort(), partitionsRequest.getPageable(), true).stream() .map(p -> p.getName().getPartitionName()).collect(Collectors.toList()); }
.map(partitionName -> PartitionInfo.builder().name( QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), final Comparator<PartitionInfo> nameComparator = Comparator.comparing(p -> p.getName().toString()); ConnectorUtils.sort(filteredPartitionList, sort, nameComparator);
final Table table, final boolean doFileSystemCalls) { String location = partitionInfo.getSerde().getUri(); Path path = null; if (StringUtils.isBlank(location)) { throw new InvalidMetaException(tableQName, null); final String partitionName = partitionInfo.getName().getPartitionName(); final List<String> partValues = PartitionUtil .getPartValuesFromPartName(tableQName, table, partitionName); partitionInfo.getSerde().setUri(location); if (doFileSystemCalls) { registry.counter(registry.createId(HiveMetrics.CounterHivePartitionFileSystemCall.getMetricName())
@Override public PartitionInfo toPartitionInfo(final TableInfo tableInfo, final Partition partition) { final QualifiedName tableName = tableInfo.getName(); final StorageInfo storageInfo = tableInfo.getSerde(); storageInfo.setUri(partition.getUri()); final AuditInfo auditInfo = AuditInfo.builder().createdDate(partition.getCreatedDate()) .lastModifiedDate(partition.getLastUpdatedDate()) .build(); final AuditInfo tableAuditInfo = tableInfo.getAudit(); if (tableAuditInfo != null) { auditInfo.setCreatedBy(tableAuditInfo.getCreatedBy()); auditInfo.setLastModifiedBy(tableAuditInfo.getLastModifiedBy()); } return PartitionInfo.builder() .name(QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), partition.getName())) .serde(storageInfo) .auditInfo(auditInfo) .build(); }
private void copyTableSdToPartitionInfoSd(final PartitionInfo partitionInfo, final Table table) { StorageInfo sd = partitionInfo.getSerde(); // // Partitions can be provided in the request without the storage information. // if (sd == null) { sd = new StorageInfo(); partitionInfo.setSerde(sd); } final StorageDescriptor tableSd = table.getSd(); if (StringUtils.isBlank(sd.getInputFormat())) { sd.setInputFormat(tableSd.getInputFormat()); } if (StringUtils.isBlank(sd.getOutputFormat())) { sd.setOutputFormat(tableSd.getOutputFormat()); } if (sd.getParameters() == null || sd.getParameters().isEmpty()) { sd.setParameters(tableSd.getParameters()); } final SerDeInfo tableSerde = tableSd.getSerdeInfo(); if (tableSerde != null) { if (StringUtils.isBlank(sd.getSerializationLib())) { sd.setSerializationLib(tableSerde.getSerializationLib()); } if (sd.getSerdeInfoParameters() == null || sd.getSerdeInfoParameters().isEmpty()) { sd.setSerdeInfoParameters(tableSerde.getParameters()); } } }