@Override public Optional<ConnectorNewTableLayout> getNewTableLayout(ConnectorSession connectorSession, ConnectorTableMetadata tableMetadata) { List<String> distributeColumns = (List<String>) tableMetadata.getProperties().get(DISTRIBUTED_ON); if (distributeColumns.isEmpty()) { return Optional.empty(); } Set<String> undefinedColumns = Sets.difference( ImmutableSet.copyOf(distributeColumns), tableMetadata.getColumns().stream() .map(ColumnMetadata::getName) .collect(toSet())); if (!undefinedColumns.isEmpty()) { throw new PrestoException(INVALID_TABLE_PROPERTY, "Distribute columns not defined on table: " + undefinedColumns); } return Optional.of(new ConnectorNewTableLayout(BlackHolePartitioningHandle.INSTANCE, distributeColumns)); }
@Override public Optional<ConnectorNewTableLayout> getNewTableLayout(ConnectorSession connectorSession, ConnectorTableMetadata tableMetadata) { List<String> distributeColumns = (List<String>) tableMetadata.getProperties().get(DISTRIBUTED_ON); if (distributeColumns.isEmpty()) { return Optional.empty(); } Set<String> undefinedColumns = Sets.difference( ImmutableSet.copyOf(distributeColumns), tableMetadata.getColumns().stream() .map(ColumnMetadata::getName) .collect(toSet())); if (!undefinedColumns.isEmpty()) { throw new PrestoException(INVALID_TABLE_PROPERTY, "Distribute columns not defined on table: " + undefinedColumns); } return Optional.of(new ConnectorNewTableLayout(BlackHolePartitioningHandle.INSTANCE, distributeColumns)); }
private static void validatePartitionColumns(ConnectorTableMetadata tableMetadata) { List<String> partitionedBy = getPartitionedBy(tableMetadata.getProperties()); List<String> allColumns = tableMetadata.getColumns().stream() .map(ColumnMetadata::getName) .collect(toList()); if (!allColumns.containsAll(partitionedBy)) { throw new PrestoException(INVALID_TABLE_PROPERTY, format("Partition columns %s not present in schema", Sets.difference(ImmutableSet.copyOf(partitionedBy), ImmutableSet.copyOf(allColumns)))); } if (allColumns.size() == partitionedBy.size()) { throw new PrestoException(INVALID_TABLE_PROPERTY, "Table contains only partition columns"); } if (!allColumns.subList(allColumns.size() - partitionedBy.size(), allColumns.size()).equals(partitionedBy)) { throw new PrestoException(HIVE_COLUMN_ORDER_MISMATCH, "Partition keys must be the last columns in the table and in the same order as the table properties: " + partitionedBy); } }
@Override public TableStatisticsMetadata getStatisticsCollectionMetadata(ConnectorSession session, ConnectorTableMetadata tableMetadata) { List<String> partitionedBy = firstNonNull(getPartitionedBy(tableMetadata.getProperties()), ImmutableList.of()); return getStatisticsCollectionMetadata(tableMetadata.getColumns(), partitionedBy, true); }
@Override public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata column) { ConnectorTableMetadata tableMetadata = getTableMetadata(session, tableHandle); SchemaTableName tableName = getTableName(tableHandle); ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder(); columns.addAll(tableMetadata.getColumns()); columns.add(column); tables.put(tableName, new ConnectorTableMetadata(tableName, columns.build(), tableMetadata.getProperties(), tableMetadata.getComment())); }
@Override public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata column) { ConnectorTableMetadata tableMetadata = getTableMetadata(session, tableHandle); SchemaTableName tableName = getTableName(tableHandle); ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder(); columns.addAll(tableMetadata.getColumns()); columns.add(column); tables.put(tableName, new ConnectorTableMetadata(tableName, columns.build(), tableMetadata.getProperties(), tableMetadata.getComment())); }
private static void validateBucketColumns(ConnectorTableMetadata tableMetadata) { Optional<HiveBucketProperty> bucketProperty = getBucketProperty(tableMetadata.getProperties()); if (!bucketProperty.isPresent()) { return; } Set<String> allColumns = tableMetadata.getColumns().stream() .map(ColumnMetadata::getName) .collect(toSet()); List<String> bucketedBy = bucketProperty.get().getBucketedBy(); if (!allColumns.containsAll(bucketedBy)) { throw new PrestoException(INVALID_TABLE_PROPERTY, format("Bucketing columns %s not present in schema", Sets.difference(ImmutableSet.copyOf(bucketedBy), ImmutableSet.copyOf(allColumns)))); } List<String> sortedBy = bucketProperty.get().getSortedBy().stream() .map(SortingColumn::getColumnName) .collect(toImmutableList()); if (!allColumns.containsAll(sortedBy)) { throw new PrestoException(INVALID_TABLE_PROPERTY, format("Sorting columns %s not present in schema", Sets.difference(ImmutableSet.copyOf(sortedBy), ImmutableSet.copyOf(allColumns)))); } }
@Override public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(ConnectorSession session, ConnectorTableMetadata tableMetadata) { if (!isCollectColumnStatisticsOnWrite(session)) { return TableStatisticsMetadata.empty(); } List<String> partitionedBy = firstNonNull(getPartitionedBy(tableMetadata.getProperties()), ImmutableList.of()); return getStatisticsCollectionMetadata(tableMetadata.getColumns(), partitionedBy, false); }
@Override public Optional<ConnectorNewTableLayout> getNewTableLayout(ConnectorSession session, ConnectorTableMetadata metadata) { ImmutableMap.Builder<String, RaptorColumnHandle> map = ImmutableMap.builder(); long columnId = 1; for (ColumnMetadata column : metadata.getColumns()) { map.put(column.getName(), new RaptorColumnHandle(connectorId, column.getName(), columnId, column.getType())); columnId++; } Optional<DistributionInfo> distribution = getOrCreateDistribution(map.build(), metadata.getProperties()); if (!distribution.isPresent()) { return Optional.empty(); } List<String> partitionColumns = distribution.get().getBucketColumns().stream() .map(RaptorColumnHandle::getColumnName) .collect(toList()); ConnectorPartitioningHandle partitioning = getPartitioningHandle(distribution.get().getDistributionId()); return Optional.of(new ConnectorNewTableLayout(partitioning, partitionColumns)); }
@Override public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle source, String target) { ConnectorTableMetadata tableMetadata = getTableMetadata(session, tableHandle); SchemaTableName tableName = getTableName(tableHandle); ColumnMetadata columnMetadata = getColumnMetadata(session, tableHandle, source); List<ColumnMetadata> columns = new ArrayList<>(tableMetadata.getColumns()); columns.set(columns.indexOf(columnMetadata), new ColumnMetadata(target, columnMetadata.getType(), columnMetadata.getComment(), columnMetadata.isHidden())); tables.put(tableName, new ConnectorTableMetadata(tableName, ImmutableList.copyOf(columns), tableMetadata.getProperties(), tableMetadata.getComment())); }
@Override public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle source, String target) { ConnectorTableMetadata tableMetadata = getTableMetadata(session, tableHandle); SchemaTableName tableName = getTableName(tableHandle); ColumnMetadata columnMetadata = getColumnMetadata(session, tableHandle, source); List<ColumnMetadata> columns = new ArrayList<>(tableMetadata.getColumns()); columns.set(columns.indexOf(columnMetadata), new ColumnMetadata(target, columnMetadata.getType(), columnMetadata.getComment(), columnMetadata.isHidden())); tables.put(tableName, new ConnectorTableMetadata(tableName, ImmutableList.copyOf(columns), tableMetadata.getProperties(), tableMetadata.getComment())); }
@Override public ConnectorTableHandle getTableHandleForStatisticsCollection(ConnectorSession session, SchemaTableName tableName, Map<String, Object> analyzeProperties) { HiveTableHandle handle = getTableHandle(session, tableName); if (handle == null) { return null; } Optional<List<List<String>>> partitionValuesList = getPartitionList(analyzeProperties); ConnectorTableMetadata tableMetadata = getTableMetadata(handle.getSchemaTableName()); handle = handle.withAnalyzePartitionValues(partitionValuesList); List<String> partitionedBy = getPartitionedBy(tableMetadata.getProperties()); if (partitionValuesList.isPresent() && partitionedBy.isEmpty()) { throw new PrestoException(INVALID_ANALYZE_PROPERTY, "Only partitioned table can be analyzed with a partition list"); } return handle; }
@Test public void testBucketedCatalog() { String bucketedCatalog = bucketedSession.getCatalog().get(); String bucketedSchema = bucketedSession.getSchema().get(); TableMetadata ordersTableMetadata = getTableMetadata(bucketedCatalog, bucketedSchema, "orders"); assertEquals(ordersTableMetadata.getMetadata().getProperties().get(BUCKETED_BY_PROPERTY), ImmutableList.of("custkey")); assertEquals(ordersTableMetadata.getMetadata().getProperties().get(BUCKET_COUNT_PROPERTY), 11); TableMetadata customerTableMetadata = getTableMetadata(bucketedCatalog, bucketedSchema, "customer"); assertEquals(customerTableMetadata.getMetadata().getProperties().get(BUCKETED_BY_PROPERTY), ImmutableList.of("custkey")); assertEquals(customerTableMetadata.getMetadata().getProperties().get(BUCKET_COUNT_PROPERTY), 11); }
@Test public void testBucketedCatalog() { String bucketedCatalog = bucketedSession.getCatalog().get(); String bucketedSchema = bucketedSession.getSchema().get(); TableMetadata ordersTableMetadata = getTableMetadata(bucketedCatalog, bucketedSchema, "orders"); assertEquals(ordersTableMetadata.getMetadata().getProperties().get(BUCKETED_BY_PROPERTY), ImmutableList.of("custkey")); assertEquals(ordersTableMetadata.getMetadata().getProperties().get(BUCKET_COUNT_PROPERTY), 11); TableMetadata customerTableMetadata = getTableMetadata(bucketedCatalog, bucketedSchema, "customer"); assertEquals(customerTableMetadata.getMetadata().getProperties().get(BUCKETED_BY_PROPERTY), ImmutableList.of("custkey")); assertEquals(customerTableMetadata.getMetadata().getProperties().get(BUCKET_COUNT_PROPERTY), 11); }
private static void assertTableEqual(ConnectorTableMetadata actual, ConnectorTableMetadata expected) { assertEquals(actual.getTable(), expected.getTable()); List<ColumnMetadata> actualColumns = actual.getColumns().stream() .filter(columnMetadata -> !columnMetadata.isHidden()) .collect(Collectors.toList()); List<ColumnMetadata> expectedColumns = expected.getColumns(); assertEquals(actualColumns.size(), expectedColumns.size()); for (int i = 0; i < actualColumns.size(); i++) { ColumnMetadata actualColumn = actualColumns.get(i); ColumnMetadata expectedColumn = expectedColumns.get(i); assertEquals(actualColumn.getName(), expectedColumn.getName()); assertEquals(actualColumn.getType(), expectedColumn.getType()); } assertEquals(actual.getProperties(), expected.getProperties()); }
private void verifyPartitionedBucketedTable(HiveStorageFormat storageFormat, String tableName) { TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, tableName); assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat); assertEquals(tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY), ImmutableList.of("orderstatus")); assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKETED_BY_PROPERTY), ImmutableList.of("custkey", "custkey2")); assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKET_COUNT_PROPERTY), 11); List<?> partitions = getPartitions(tableName); assertEquals(partitions.size(), 3); assertQuery("SELECT * from " + tableName, "SELECT custkey, custkey, comment, orderstatus FROM orders"); for (int i = 1; i <= 30; i++) { assertQuery( format("SELECT * from " + tableName + " where custkey = %d and custkey2 = %d", i, i), format("SELECT custkey, custkey, comment, orderstatus FROM orders where custkey = %d", i)); } assertThatThrownBy(() -> assertUpdate("INSERT INTO " + tableName + " VALUES (1, 1, 'comment', 'O')", 1)) .hasMessage(getExpectedErrorMessageForInsertExistingBucketedTable( getInsertExistingPartitionsBehavior(getConnectorSession(getSession())), "orderstatus=O")); }
private void verifyPartition(boolean hasPartition, TableMetadata tableMetadata, List<String> partitionKeys) { Object partitionByProperty = tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY); if (hasPartition) { assertEquals(partitionByProperty, partitionKeys); for (ColumnMetadata columnMetadata : tableMetadata.getColumns()) { boolean partitionKey = partitionKeys.contains(columnMetadata.getName()); assertEquals(columnMetadata.getExtraInfo(), columnExtraInfo(partitionKey)); } } else { assertNull(partitionByProperty); } }
private void verifyPartition(boolean hasPartition, TableMetadata tableMetadata, List<String> partitionKeys) { Object partitionByProperty = tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY); if (hasPartition) { assertEquals(partitionByProperty, partitionKeys); for (ColumnMetadata columnMetadata : tableMetadata.getColumns()) { boolean partitionKey = partitionKeys.contains(columnMetadata.getName()); assertEquals(columnMetadata.getExtraInfo(), columnExtraInfo(partitionKey)); } } else { assertNull(partitionByProperty); } }
private void verifyPartitionedBucketedTableAsFewRows(HiveStorageFormat storageFormat, String tableName) { TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, tableName); assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat); assertEquals(tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY), ImmutableList.of("partition_key")); assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKETED_BY_PROPERTY), ImmutableList.of("bucket_key")); assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKET_COUNT_PROPERTY), 11); List<?> partitions = getPartitions(tableName); assertEquals(partitions.size(), 3); MaterializedResult actual = computeActual("SELECT * from " + tableName); MaterializedResult expected = resultBuilder(getSession(), canonicalizeType(createUnboundedVarcharType()), canonicalizeType(createUnboundedVarcharType()), canonicalizeType(createUnboundedVarcharType())) .row("a", "b", "c") .row("aa", "bb", "cc") .row("aaa", "bbb", "ccc") .build(); assertEqualsIgnoreOrder(actual.getMaterializedRows(), expected.getMaterializedRows()); }
private void verifyPartitionedBucketedTableAsFewRows(HiveStorageFormat storageFormat, String tableName) { TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, tableName); assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat); assertEquals(tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY), ImmutableList.of("partition_key")); assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKETED_BY_PROPERTY), ImmutableList.of("bucket_key")); assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKET_COUNT_PROPERTY), 11); List<?> partitions = getPartitions(tableName); assertEquals(partitions.size(), 3); MaterializedResult actual = computeActual("SELECT * from " + tableName); MaterializedResult expected = resultBuilder(getSession(), canonicalizeType(createUnboundedVarcharType()), canonicalizeType(createUnboundedVarcharType()), canonicalizeType(createUnboundedVarcharType())) .row("a", "b", "c") .row("aa", "bb", "cc") .row("aaa", "bbb", "ccc") .build(); assertEqualsIgnoreOrder(actual.getMaterializedRows(), expected.getMaterializedRows()); }