private void copyTableSdToPartitionInfoSd(final PartitionInfo partitionInfo, final Table table) { StorageInfo sd = partitionInfo.getSerde(); // // Partitions can be provided in the request without the storage information. // if (sd == null) { sd = new StorageInfo(); partitionInfo.setSerde(sd); } final StorageDescriptor tableSd = table.getSd(); if (StringUtils.isBlank(sd.getInputFormat())) { sd.setInputFormat(tableSd.getInputFormat()); } if (StringUtils.isBlank(sd.getOutputFormat())) { sd.setOutputFormat(tableSd.getOutputFormat()); } if (sd.getParameters() == null || sd.getParameters().isEmpty()) { sd.setParameters(tableSd.getParameters()); } final SerDeInfo tableSerde = tableSd.getSerdeInfo(); if (tableSerde != null) { if (StringUtils.isBlank(sd.getSerializationLib())) { sd.setSerializationLib(tableSerde.getSerializationLib()); } if (sd.getSerdeInfoParameters() == null || sd.getSerdeInfoParameters().isEmpty()) { sd.setSerdeInfoParameters(tableSerde.getParameters()); } } }
/** * Converts from storage info to s3 location. * @param storageInfo storage info * @return location */ Location fromStorageInfo(final StorageInfo storageInfo) { final Location result = new Location(); if (storageInfo != null) { result.setUri(storageInfo.getUri()); final Info info = new Info(); info.setLocation(result); info.setOwner(storageInfo.getOwner()); info.setInputFormat(storageInfo.getInputFormat()); info.setOutputFormat(storageInfo.getOutputFormat()); info.setSerializationLib(storageInfo.getSerializationLib()); final Map<String, String> parameters = Maps.newHashMap(); if (storageInfo.getParameters() != null) { parameters.putAll(storageInfo.getParameters()); } if (storageInfo.getSerdeInfoParameters() != null) { parameters.putAll(storageInfo.getSerdeInfoParameters()); } info.setParameters(parameters); result.setInfo(info); } return result; }
final String inputFormat = storageInfo.getInputFormat(); final String location = notNull(storageInfo.getUri()) ? storageInfo.getUri() : ""; final String outputFormat = storageInfo.getOutputFormat(); final Map<String, String> sdParams = notNull(storageInfo.getParameters()) ? storageInfo.getParameters() : new HashMap<>(); final Map<String, String> serdeParams = notNull(storageInfo.getSerdeInfoParameters()) ? storageInfo.getSerdeInfoParameters() : new HashMap<>(); final String serializationLib = storageInfo.getSerializationLib(); return new StorageDescriptor( cols,
/** * Converts from s3 table info to storage info. * @param table table info * @return table info */ StorageInfo toStorageInfo(final Table table) { StorageInfo result = null; final Location location = table.getLocation(); if (location != null) { final Map<String, String> infoParameters = Maps.newHashMap(); result = new StorageInfo(); result.setUri(location.getUri()); final Info info = location.getInfo(); if (info != null) { result.setOwner(info.getOwner()); result.setInputFormat(info.getInputFormat()); result.setOutputFormat(info.getOutputFormat()); result.setSerializationLib(info.getSerializationLib()); if (info.getParameters() != null) { infoParameters.putAll(info.getParameters()); } } result.setSerdeInfoParameters(infoParameters); result.setParameters(Maps.newHashMap()); } return result; }
} else { final String partitionUri = partitionInfo.getSerde() != null ? partitionInfo.getSerde().getUri() : null; final String existingPartitionUri = getPartitionUri(existingPartitionHolder); if (partitionUri == null || !partitionUri.equals(existingPartitionUri)) { partitionInfo.setSerde(new StorageInfo()); partitionInfo.getSerde().setUri(existingPartitionUri); if (existingPartitionHolder.getPartition() != null) { final Partition existingPartition = existingPartitionHolder.getPartition(); partitionInfo.getSerde().setParameters(existingPartition.getParameters()); partitionInfo.getAudit().setCreatedDate( HiveConnectorInfoConverter.epochSecondsToDate(existingPartition.getCreateTime())); if (existingPartitionInfo.getSerde() != null) { partitionInfo.getSerde() .setParameters(existingPartitionInfo.getSerde().getParameters());
private StorageInfo toStorageInfo(final StorageDescriptor sd, final String owner) { if (sd == null) { return new StorageInfo(); } if (sd.getSerdeInfo() != null) { return StorageInfo.builder().owner(owner) .uri(sd.getLocation()) .inputFormat(sd.getInputFormat()) .outputFormat(sd.getOutputFormat()) .parameters(sd.getParameters()) .serializationLib(sd.getSerdeInfo().getSerializationLib()) .serdeInfoParameters(sd.getSerdeInfo().getParameters()) .build(); } return StorageInfo.builder().owner(owner).uri(sd.getLocation()).inputFormat(sd.getInputFormat()) .outputFormat(sd.getOutputFormat()).parameters(sd.getParameters()).build(); }
final Table table, final boolean doFileSystemCalls) { String location = partitionInfo.getSerde().getUri(); Path path = null; if (StringUtils.isBlank(location)) { partitionInfo.getSerde().setUri(location); if (doFileSystemCalls) { registry.counter(registry.createId(HiveMetrics.CounterHivePartitionFileSystemCall.getMetricName())
@Override public List<String> getPartitionUris(@Nonnull final ConnectorRequestContext context, @Nonnull final QualifiedName tableName, @Nonnull final PartitionListRequest partitionsRequest, final TableInfo tableInfo) { log.debug("Get partition uris for table {}", tableName); return _getPartitions(tableName, partitionsRequest.getFilter(), partitionsRequest.getPartitionNames(), partitionsRequest.getSort(), partitionsRequest.getPageable(), true).stream() .filter(p -> p.getSerde() != null && p.getSerde().getUri() != null) .map(p -> p.getSerde().getUri()).collect(Collectors.toList()); }
/** * Convert from data source to partitionInfo. * * @param segment segment object * @return partition info object */ public PartitionInfo getPartitionInfoFromSegment(final Segment segment) { final Map<String, String> metadata = new HashMap<>(); metadata.put(DruidConfigConstants.LOADSPEC_KEY, segment.getLoadSpec().getKeys().toString()); metadata.put(DruidConfigConstants.LOADSPEC_BUCKET, segment.getLoadSpec().getBucket()); metadata.put(DruidConfigConstants.LOADSPEC_TYPE, segment.getLoadSpec().getType()); metadata.put(DruidConfigConstants.DIMENSIONS, segment.getDimensions()); metadata.put(DruidConfigConstants.METRICS, segment.getMetric()); final StorageInfo storageInfo = StorageInfo.builder().uri(segment.getLoadSpec().getUri()).build(); return PartitionInfo.builder().metadata(metadata).serde(storageInfo).build(); }
@Override public void create(@Nonnull final ConnectorRequestContext context, @Nonnull final TableInfo tableInfo) { log.debug("Start: Create table {}", tableInfo.getName()); Preconditions.checkArgument(tableInfo.getSerde() == null || !Strings.isNullOrEmpty(tableInfo.getSerde().getOwner()), "Table owner is null or empty"); final QualifiedName tableName = tableInfo.getName(); if (tableDao.getBySourceDatabaseTableName(catalogName, tableName.getDatabaseName(), tableName.getTableName()) != null) { throw new TableAlreadyExistsException(tableName); } final Database database = databaseDao .getBySourceDatabaseName(catalogName, tableName.getDatabaseName()); if (database == null) { throw new DatabaseNotFoundException(QualifiedName.ofDatabase(catalogName, tableName.getDatabaseName())); } tableDao.save(infoConverter.fromTableInfo(database, tableInfo)); log.debug("End: Create table {}", tableInfo.getName()); }
&& Strings.isNullOrEmpty(sd.getSerdeInfo().getSerializationLib()) ) { sd.getSerdeInfo().setSerializationLib(tableInfo.getSerde().getSerializationLib());
final String outputFormat = rs.getString("output_format"); final String serializationLib = rs.getString("slib"); final StorageInfo storageInfo = new StorageInfo(); storageInfo.setUri(uri); storageInfo.setInputFormat(inputFormat); storageInfo.setOutputFormat(outputFormat); storageInfo.setSerializationLib(serializationLib); final AuditInfo auditInfo = new AuditInfo(); auditInfo.setCreatedDate(Date.from(Instant.ofEpochSecond(createdDate))); partitionHolder.getPartitionInfo().setMetadata(partitionParams.get(partitionHolder.getId())); partitionHolder.getPartitionInfo().getSerde() .setParameters(sdParams.get(partitionHolder.getSdId())); partitionHolder.getPartitionInfo().getSerde() .setSerdeInfoParameters(serdeParams.get(partitionHolder.getSerdeId()));
} else { final String partitionUri = partitionInfo.getSerde() != null ? partitionInfo.getSerde().getUri() : null; final String existingPartitionUri = getPartitionUri(existingPartitionHolder); if (partitionUri == null || !partitionUri.equals(existingPartitionUri)) { partitionInfo.setSerde(new StorageInfo()); partitionInfo.getSerde().setUri(existingPartitionUri); if (existingPartitionHolder.getPartition() != null) { final Partition existingPartition = existingPartitionHolder.getPartition(); partitionInfo.getSerde().setParameters(existingPartition.getParameters()); partitionInfo.getAudit().setCreatedDate( HiveConnectorInfoConverter.epochSecondsToDate(existingPartition.getCreateTime())); if (existingPartitionInfo.getSerde() != null) { partitionInfo.getSerde() .setParameters(existingPartitionInfo.getSerde().getParameters());
private StorageInfo toStorageInfo(final StorageDescriptor sd, final String owner) { if (sd == null) { return new StorageInfo(); } if (sd.getSerdeInfo() != null) { return StorageInfo.builder().owner(owner) .uri(sd.getLocation()) .inputFormat(sd.getInputFormat()) .outputFormat(sd.getOutputFormat()) .parameters(sd.getParameters()) .serializationLib(sd.getSerdeInfo().getSerializationLib()) .serdeInfoParameters(sd.getSerdeInfo().getParameters()) .build(); } return StorageInfo.builder().owner(owner).uri(sd.getLocation()).inputFormat(sd.getInputFormat()) .outputFormat(sd.getOutputFormat()).parameters(sd.getParameters()).build(); }
final Table table, final boolean doFileSystemCalls) { String location = partitionInfo.getSerde().getUri(); Path path = null; if (StringUtils.isBlank(location)) { partitionInfo.getSerde().setUri(location); if (doFileSystemCalls) { registry.counter(registry.createId(HiveMetrics.CounterHivePartitionFileSystemCall.getMetricName())
/** * Gets the partition uri. * @param partitionInfo partition * @return uri */ public String getUri(final PartitionInfo partitionInfo) { return partitionInfo.getSerde() == null ? null : partitionInfo.getSerde().getUri(); }
/** * Converts IcebergTable to TableDto. * * @param name qualified name * @param table iceberg table object * @param tableLoc iceberg table metadata location * @param auditInfo audit information * @return Metacat table Info */ public TableInfo fromIcebergTableToTableInfo(final QualifiedName name, final com.netflix.iceberg.Table table, final String tableLoc, final AuditInfo auditInfo) { final List<FieldInfo> allFields = this.hiveTypeConverter.icebergeSchemaTofieldDtos(table.schema(), table.spec().fields()); final Map<String, String> tableParameters = new HashMap<>(); tableParameters.put(DirectSqlTable.PARAM_TABLE_TYPE, DirectSqlTable.ICEBERG_TABLE_TYPE); tableParameters.put(DirectSqlTable.PARAM_METADATA_LOCATION, tableLoc); //adding iceberg table properties tableParameters.putAll(table.properties()); return TableInfo.builder().fields(allFields) .metadata(tableParameters) .serde(StorageInfo.builder().uri(table.location()).build()) .name(name).auditInfo(auditInfo) .build(); }
&& Strings.isNullOrEmpty(sd.getSerdeInfo().getSerializationLib()) ) { sd.getSerdeInfo().setSerializationLib(tableInfo.getSerde().getSerializationLib());
private void copyTableSdToPartitionInfoSd(final PartitionInfo partitionInfo, final Table table) { StorageInfo sd = partitionInfo.getSerde(); // // Partitions can be provided in the request without the storage information. // if (sd == null) { sd = new StorageInfo(); partitionInfo.setSerde(sd); } final StorageDescriptor tableSd = table.getSd(); if (StringUtils.isBlank(sd.getInputFormat())) { sd.setInputFormat(tableSd.getInputFormat()); } if (StringUtils.isBlank(sd.getOutputFormat())) { sd.setOutputFormat(tableSd.getOutputFormat()); } if (sd.getParameters() == null || sd.getParameters().isEmpty()) { sd.setParameters(tableSd.getParameters()); } final SerDeInfo tableSerde = tableSd.getSerdeInfo(); if (tableSerde != null) { if (StringUtils.isBlank(sd.getSerializationLib())) { sd.setSerializationLib(tableSerde.getSerializationLib()); } if (sd.getSerdeInfoParameters() == null || sd.getSerdeInfoParameters().isEmpty()) { sd.setSerdeInfoParameters(tableSerde.getParameters()); } } }