/** * get iceberg table metadata location. * * @param tableInfo table info * @return true for iceberg table */ public static String getIcebergTableMetadataLocation(final TableInfo tableInfo) { return tableInfo.getMetadata().get(DirectSqlTable.PARAM_METADATA_LOCATION); } }
@Override public PartitionInfo toPartitionInfo(final TableInfo tableInfo, final Partition partition) { final QualifiedName tableName = tableInfo.getName(); final StorageInfo storageInfo = tableInfo.getSerde(); storageInfo.setUri(partition.getUri()); final AuditInfo auditInfo = AuditInfo.builder().createdDate(partition.getCreatedDate()) .lastModifiedDate(partition.getLastUpdatedDate()) .build(); final AuditInfo tableAuditInfo = tableInfo.getAudit(); if (tableAuditInfo != null) { auditInfo.setCreatedBy(tableAuditInfo.getCreatedBy()); auditInfo.setLastModifiedBy(tableAuditInfo.getLastModifiedBy()); } return PartitionInfo.builder() .name(QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), partition.getName())) .serde(storageInfo) .auditInfo(auditInfo) .build(); }
/** * Converts from column metadata to field. * @param field column * @return field info */ public FieldInfo toFieldInfo(final Field field) { return FieldInfo.builder().name(field.getName()).partitionKey(field.isPartitionKey()) .comment(field.getComment()).sourceType(field.getSourceType()).type(toType(field.getType())).build(); }
@Override public DatabaseInfo toDatabaseInfo(final QualifiedName catalogName, final Database database) { final AuditInfo auditInfo = AuditInfo.builder().createdDate(database.getCreatedDate()) .lastModifiedDate(database.getLastUpdatedDate()).build(); return DatabaseInfo.builder().name(QualifiedName.ofDatabase(catalogName.getCatalogName(), database.getName())) .auditInfo(auditInfo).build(); }
/** * Converts to DatabaseDto. * * @param database connector database * @return Metacat database Info */ @Override public DatabaseInfo toDatabaseInfo( final QualifiedName qualifiedName, final Database database ) { return DatabaseInfo.builder() .name(qualifiedName) .uri(database.getLocationUri()) .metadata(database.getParameters()) .build(); }
private StorageInfo toStorageInfo(final StorageDescriptor sd, final String owner) { if (sd == null) { return new StorageInfo(); } if (sd.getSerdeInfo() != null) { return StorageInfo.builder().owner(owner) .uri(sd.getLocation()) .inputFormat(sd.getInputFormat()) .outputFormat(sd.getOutputFormat()) .parameters(sd.getParameters()) .serializationLib(sd.getSerdeInfo().getSerializationLib()) .serdeInfoParameters(sd.getSerdeInfo().getParameters()) .build(); } return StorageInfo.builder().owner(owner).uri(sd.getLocation()).inputFormat(sd.getInputFormat()) .outputFormat(sd.getOutputFormat()).parameters(sd.getParameters()).build(); }
/** * Convert from data source to partitionInfo. * * @param segment segment object * @return partition info object */ public PartitionInfo getPartitionInfoFromSegment(final Segment segment) { final Map<String, String> metadata = new HashMap<>(); metadata.put(DruidConfigConstants.LOADSPEC_KEY, segment.getLoadSpec().getKeys().toString()); metadata.put(DruidConfigConstants.LOADSPEC_BUCKET, segment.getLoadSpec().getBucket()); metadata.put(DruidConfigConstants.LOADSPEC_TYPE, segment.getLoadSpec().getType()); metadata.put(DruidConfigConstants.DIMENSIONS, segment.getDimensions()); metadata.put(DruidConfigConstants.METRICS, segment.getMetric()); final StorageInfo storageInfo = StorageInfo.builder().uri(segment.getLoadSpec().getUri()).build(); return PartitionInfo.builder().metadata(metadata).serde(storageInfo).build(); }
@Override public List<String> getPartitionUris(@Nonnull final ConnectorRequestContext context, @Nonnull final QualifiedName tableName, @Nonnull final PartitionListRequest partitionsRequest, final TableInfo tableInfo) { log.debug("Get partition uris for table {}", tableName); return _getPartitions(tableName, partitionsRequest.getFilter(), partitionsRequest.getPartitionNames(), partitionsRequest.getSort(), partitionsRequest.getPageable(), true).stream() .filter(p -> p.getSerde() != null && p.getSerde().getUri() != null) .map(p -> p.getSerde().getUri()).collect(Collectors.toList()); }
/** * Converts from partition info to s3 partition object. * @param partitionInfo partition info * @return s3 partition */ Partition fromPartitionInfo(final PartitionInfo partitionInfo) { final Partition result = new Partition(); result.setName(partitionInfo.getName().getPartitionName()); result.setUri(partitionInfo.getSerde().getUri()); final AuditInfo auditInfo = partitionInfo.getAudit(); if (auditInfo != null) { result.setCreatedDate(auditInfo.getCreatedDate()); result.setLastUpdatedDate(auditInfo.getLastModifiedDate()); } return result; }
/** * hiveToMetacatField. * * @param field field * @param isPartitionKey boolean * @return field info obj */ private FieldInfo hiveToMetacatField(final FieldSchema field, final boolean isPartitionKey) { return FieldInfo.builder().name(field.getName()) .type(hiveTypeConverter.toMetacatType(field.getType())) .sourceType(field.getType()) .comment(field.getComment()) .partitionKey(isPartitionKey) .build(); }
/** * {@inheritDoc} */ @Override public DatabaseInfo get(@Nonnull final ConnectorRequestContext context, @Nonnull final QualifiedName name) { final String databaseName = name.getDatabaseName(); log.debug("Beginning to get database metadata for {} for request {}", databaseName, context); return DatabaseInfo.builder().name(name).build(); }
/** * Converts to DatabaseDto. * * @param database connector database * @return Metacat database Info */ @Override public DatabaseInfo toDatabaseInfo( final QualifiedName qualifiedName, final Database database ) { return DatabaseInfo.builder() .name(qualifiedName) .uri(database.getLocationUri()) .metadata(database.getParameters()) .build(); }
private String getPartitionUri(final PartitionHolder partition) { String result = null; if (partition.getPartition() != null) { final Partition hivePartition = partition.getPartition(); result = hivePartition.getSd() != null ? hivePartition.getSd().getLocation() : null; } else if (partition.getPartitionInfo() != null) { final PartitionInfo partitionInfo = partition.getPartitionInfo(); result = partitionInfo.getSerde() != null ? partitionInfo.getSerde().getUri() : null; } return result; }
/** * get iceberg table metadata location. * * @param tableInfo table info * @return true for iceberg table */ public static String getIcebergTableMetadataLocation(final TableInfo tableInfo) { return tableInfo.getMetadata().get(DirectSqlTable.PARAM_METADATA_LOCATION); } }
/** * hiveToMetacatField. * * @param field field * @param isPartitionKey boolean * @return field info obj */ private FieldInfo hiveToMetacatField(final FieldSchema field, final boolean isPartitionKey) { return FieldInfo.builder().name(field.getName()) .type(hiveTypeConverter.toMetacatType(field.getType())) .sourceType(field.getType()) .comment(field.getComment()) .partitionKey(isPartitionKey) .build(); }
/** * {@inheritDoc} */ @Override public DatabaseInfo get(@Nonnull final ConnectorRequestContext context, @Nonnull final QualifiedName name) { final String databaseName = name.getDatabaseName(); log.debug("Beginning to get database metadata for {} for request {}", databaseName, context); return DatabaseInfo.builder().name(name).build(); }
private String getPartitionUri(final PartitionHolder partition) { String result = null; if (partition.getPartition() != null) { final Partition hivePartition = partition.getPartition(); result = hivePartition.getSd() != null ? hivePartition.getSd().getLocation() : null; } else if (partition.getPartitionInfo() != null) { final PartitionInfo partitionInfo = partition.getPartitionInfo(); result = partitionInfo.getSerde() != null ? partitionInfo.getSerde().getUri() : null; } return result; }