/** * {@inheritDoc} */ @Override public void snapshotPartitions(final QualifiedName name, @Nullable final String filter) { final List<PartitionDto> partitionDtos = partitionService.list(name, null, null, false, false, new GetPartitionsRequestDto(filter, null, true, true)); if (partitionDtos != null && !partitionDtos.isEmpty()) { log.info("Snapshot partitions({}) for view {}.", partitionDtos.size(), name); final PartitionsSaveRequestDto dto = new PartitionsSaveRequestDto(); dto.setPartitions(partitionDtos); savePartitions(name, dto, false); } }
/** * Creates the partition list connector request. * * @param partitionsRequestDto request containing the filter and other properties used for listing * @param pageable pageable info * @param sort sort info * @return connector request */ public PartitionListRequest toPartitionListRequest(final GetPartitionsRequestDto partitionsRequestDto, final Pageable pageable, final Sort sort) { if (partitionsRequestDto != null) { if (partitionsRequestDto.getIncludePartitionDetails() == null) { partitionsRequestDto.setIncludePartitionDetails(false); } if (partitionsRequestDto.getIncludeAuditOnly() == null) { partitionsRequestDto.setIncludeAuditOnly(false); } final PartitionListRequest result = mapper.map(partitionsRequestDto, PartitionListRequest.class); result.setPageable(pageable); result.setSort(sort); return result; } else { return new PartitionListRequest(null, null, false, pageable, sort, false); } }
private List<Partition> getPartitionsByFilter(final String dbName, final String tblName, @Nullable final String filter, final short maxParts) { final String databaseName = normalizeIdentifier(dbName); final String tableName = normalizeIdentifier(tblName); final TableDto tableDto = v1.getTable(catalogName, databaseName, tableName, true, false, false); final Integer maxValues = maxParts > 0 ? Short.toUnsignedInt(maxParts) : null; final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(filter, null, true, false); final List<PartitionDto> metacatPartitions = partV1.getPartitionsForRequest(catalogName, dbName, tblName, null, null, null, maxValues, false, dto); final List<Partition> result = Lists.newArrayListWithCapacity(metacatPartitions.size()); for (PartitionDto partition : metacatPartitions) { result.add(hiveConverters.metacatToHivePartition(partition, tableDto)); } return result; }
/** * Creates the partition list connector request. * * @param partitionsRequestDto request containing the filter and other properties used for listing * @param pageable pageable info * @param sort sort info * @return connector request */ public PartitionListRequest toPartitionListRequest(final GetPartitionsRequestDto partitionsRequestDto, final Pageable pageable, final Sort sort) { if (partitionsRequestDto != null) { if (partitionsRequestDto.getIncludePartitionDetails() == null) { partitionsRequestDto.setIncludePartitionDetails(false); } if (partitionsRequestDto.getIncludeAuditOnly() == null) { partitionsRequestDto.setIncludeAuditOnly(false); } final PartitionListRequest result = mapper.map(partitionsRequestDto, PartitionListRequest.class); result.setPageable(pageable); result.setSort(sort); return result; } else { return new PartitionListRequest(null, null, false, pageable, sort, false); } }
final Set<String> existingUnmarkedPartitionNames = Sets.newHashSet( partitionService.getPartitionKeys(tableName, null, null, new GetPartitionsRequestDto(null, unmarkedPartitionNames, false, true))); final List<String> partitionIds = unmarkedPartitionDtos.stream() .filter(p -> !existingUnmarkedPartitionNames.contains(
private List<Partition> getPartitionsByFilter(final String dbName, final String tblName, @Nullable final String filter, final short maxParts) { final String databaseName = normalizeIdentifier(dbName); final String tableName = normalizeIdentifier(tblName); final TableDto tableDto = v1.getTable(catalogName, databaseName, tableName, true, false, false); final Integer maxValues = maxParts > 0 ? Short.toUnsignedInt(maxParts) : null; final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(filter, null, true, false); final List<PartitionDto> metacatPartitions = partV1.getPartitionsForRequest(catalogName, dbName, tblName, null, null, null, maxValues, false, dto); final List<Partition> result = Lists.newArrayListWithCapacity(metacatPartitions.size()); for (PartitionDto partition : metacatPartitions) { result.add(hiveConverters.metacatToHivePartition(partition, tableDto)); } return result; }
/** * {@inheritDoc} */ @Override public PartitionDto get(final QualifiedName name) { PartitionDto result = null; final QualifiedName tableName = QualifiedName .ofTable(name.getCatalogName(), name.getDatabaseName(), name.getTableName()); final List<PartitionDto> dtos = list(tableName, null, null, true, true, new GetPartitionsRequestDto(null, Lists.newArrayList(name.getPartitionName()), true, true)); if (!dtos.isEmpty()) { result = dtos.get(0); } return result; }
private PartitionDto getPartitionDtoByName(final TableDto tableDto, final String partName) throws TException { final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(null, ImmutableList.of(partName), true, false); final List<PartitionDto> partitionDtos = partV1.getPartitionsForRequest( catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(), null, null, null, null, false, dto); if (partitionDtos == null || partitionDtos.isEmpty()) { throw new NoSuchObjectException("Partition (" + partName + ") not found on " + tableDto.getName()); } else if (partitionDtos.size() != 1) { // I don't think this is even possible throw new NoSuchObjectException("Partition (" + partName + ") matched extra on " + tableDto.getName()); } return partitionDtos.get(0); }
private PartitionDto getPartitionDtoByName(final TableDto tableDto, final String partName) throws TException { final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(null, ImmutableList.of(partName), true, false); final List<PartitionDto> partitionDtos = partV1.getPartitionsForRequest( catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(), null, null, null, null, false, dto); if (partitionDtos == null || partitionDtos.isEmpty()) { throw new NoSuchObjectException("Partition (" + partName + ") not found on " + tableDto.getName()); } else if (partitionDtos.size() != 1) { // I don't think this is even possible throw new NoSuchObjectException("Partition (" + partName + ") matched extra on " + tableDto.getName()); } return partitionDtos.get(0); }
/** * {@inheritDoc} */ @Override public List<Partition> get_partitions_by_names(final String dbName, final String tblName, final List<String> names) throws TException { return requestWrapper("get_partitions_by_names", new Object[]{dbName, tblName, names}, () -> { final String databaseName = normalizeIdentifier(dbName); final String tableName = normalizeIdentifier(tblName); final TableDto tableDto = v1.getTable(catalogName, databaseName, tableName, true, false, false); final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(null, names, true, false); final List<PartitionDto> metacatPartitions = partV1.getPartitionsForRequest(catalogName, databaseName, tableName, null, null, null, null, false, dto); final List<Partition> result = Lists.newArrayListWithCapacity(metacatPartitions.size()); for (PartitionDto partition : metacatPartitions) { result.add(hiveConverters.metacatToHivePartition(partition, tableDto)); } return result; }); }
/** * {@inheritDoc} */ @Override public List<Partition> get_partitions_by_names(final String dbName, final String tblName, final List<String> names) throws TException { return requestWrapper("get_partitions_by_names", new Object[]{dbName, tblName, names}, () -> { final String databaseName = normalizeIdentifier(dbName); final String tableName = normalizeIdentifier(tblName); final TableDto tableDto = v1.getTable(catalogName, databaseName, tableName, true, false, false); final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(null, names, true, false); final List<PartitionDto> metacatPartitions = partV1.getPartitionsForRequest(catalogName, databaseName, tableName, null, null, null, null, false, dto); final List<Partition> result = Lists.newArrayListWithCapacity(metacatPartitions.size()); for (PartitionDto partition : metacatPartitions) { result.add(hiveConverters.metacatToHivePartition(partition, tableDto)); } return result; }); }
final List<PartitionDto> partitionDtos = partitionService.list(tableName, sort, pageable, true, true, new GetPartitionsRequestDto(null, null, true, true)); count = partitionDtos.size(); if (!partitionDtos.isEmpty()) {
/** * {@inheritDoc} */ @Override public List<Partition> get_partitions_ps(final String dbName, final String tblName, final List<String> partVals, final short maxParts) throws TException { return requestWrapper("get_partitions_ps", new Object[]{dbName, tblName, partVals, maxParts}, () -> { final String databaseName = normalizeIdentifier(dbName); final String tableName = normalizeIdentifier(tblName); final TableDto tableDto = v1.getTable(catalogName, databaseName, tableName, true, false, false); final String partFilter = partition_values_to_partition_filter(tableDto, partVals); final Integer maxValues = maxParts > 0 ? Short.toUnsignedInt(maxParts) : null; final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(partFilter, null, true, false); final List<PartitionDto> metacatPartitions = partV1.getPartitionsForRequest(catalogName, dbName, tblName, null, null, null, maxValues, false, dto); final List<Partition> result = Lists.newArrayListWithCapacity(metacatPartitions.size()); for (PartitionDto partition : metacatPartitions) { result.add(hiveConverters.metacatToHivePartition(partition, tableDto)); } return result; }); }
/** * {@inheritDoc} */ @Override public List<Partition> get_partitions_ps(final String dbName, final String tblName, final List<String> partVals, final short maxParts) throws TException { return requestWrapper("get_partitions_ps", new Object[]{dbName, tblName, partVals, maxParts}, () -> { final String databaseName = normalizeIdentifier(dbName); final String tableName = normalizeIdentifier(tblName); final TableDto tableDto = v1.getTable(catalogName, databaseName, tableName, true, false, false); final String partFilter = partition_values_to_partition_filter(tableDto, partVals); final Integer maxValues = maxParts > 0 ? Short.toUnsignedInt(maxParts) : null; final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(partFilter, null, true, false); final List<PartitionDto> metacatPartitions = partV1.getPartitionsForRequest(catalogName, dbName, tblName, null, null, null, maxValues, false, dto); final List<Partition> result = Lists.newArrayListWithCapacity(metacatPartitions.size()); for (PartitionDto partition : metacatPartitions) { result.add(hiveConverters.metacatToHivePartition(partition, tableDto)); } return result; }); }
final ConnectorPartitionService service = connectorManager.getPartitionService(name); final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto(null, partitionIds, false, true); final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
/** * {@inheritDoc} */ @Override public List<PartitionSpec> get_part_specs_by_filter(final String dbName, final String tblName, final String filter, final int maxParts) throws TException { //TODO: Handle the use case of grouping return requestWrapper("get_partitions_pspec", new Object[]{dbName, tblName, filter, maxParts}, () -> { final String databaseName = normalizeIdentifier(dbName); final String tableName = normalizeIdentifier(tblName); final TableDto tableDto = v1.getTable(catalogName, databaseName, tableName, true, false, false); final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(filter, null, true, false); final List<PartitionDto> metacatPartitions = partV1.getPartitionsForRequest(catalogName, dbName, tblName, null, null, null, maxParts, false, dto); final List<Partition> partitions = Lists.newArrayListWithCapacity(metacatPartitions.size()); for (PartitionDto partition : metacatPartitions) { partitions.add(hiveConverters.metacatToHivePartition(partition, tableDto)); } final PartitionSpec pSpec = new PartitionSpec(); pSpec.setPartitionList(new PartitionListComposingSpec(partitions)); pSpec.setDbName(dbName); pSpec.setTableName(tblName); if (tableDto != null && tableDto.getSerde() != null) { pSpec.setRootPath(tableDto.getSerde().getUri()); } return Arrays.asList(pSpec); }); }
final List<PartitionDto> existingPartitions = partitionService.list(viewQName, null, null, false, false, new GetPartitionsRequestDto(null, partitionNames, false, true)); final Map<String, PartitionDto> existingPartitionsMap = existingPartitions.stream() .collect(Collectors
/** * {@inheritDoc} */ @Override public List<PartitionSpec> get_part_specs_by_filter(final String dbName, final String tblName, final String filter, final int maxParts) throws TException { //TODO: Handle the use case of grouping return requestWrapper("get_partitions_pspec", new Object[]{dbName, tblName, filter, maxParts}, () -> { final String databaseName = normalizeIdentifier(dbName); final String tableName = normalizeIdentifier(tblName); final TableDto tableDto = v1.getTable(catalogName, databaseName, tableName, true, false, false); final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(filter, null, true, false); final List<PartitionDto> metacatPartitions = partV1.getPartitionsForRequest(catalogName, dbName, tblName, null, null, null, maxParts, false, dto); final List<Partition> partitions = Lists.newArrayListWithCapacity(metacatPartitions.size()); for (PartitionDto partition : metacatPartitions) { partitions.add(hiveConverters.metacatToHivePartition(partition, tableDto)); } final PartitionSpec pSpec = new PartitionSpec(); pSpec.setPartitionList(new PartitionListComposingSpec(partitions)); pSpec.setDbName(dbName); pSpec.setTableName(tblName); if (tableDto != null && tableDto.getSerde() != null) { pSpec.setRootPath(tableDto.getSerde().getUri()); } return Arrays.asList(pSpec); }); }
this.partitionDeleteDistSummary.withTags(name.parts())).record(partitionIdsForDeletes.size()); final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto(null, partitionIdsForDeletes, false, true); final List<PartitionInfo> deletePartitionInfos = service.getPartitions(connectorRequestContext, name, converterUtil.toPartitionListRequest(requestDto, null, null), this.getTableInfo(name));
includeUserMetadata, includeUserMetadata, new GetPartitionsRequestDto(filter, null, false, false) new Pageable(limit, offset), includeUserMetadata, new GetPartitionsRequestDto(filter, null, false, true) offset, limit, new GetPartitionsRequestDto(filter, null, false, false) ); offset, limit, new GetPartitionsRequestDto(filter, null, false, true) ); offset, limit, new GetPartitionsRequestDto(filter, null, false, false) ); offset, limit, new GetPartitionsRequestDto(filter, null, false, true) );