@Override public Optional<ConnectorNewTableLayout> getNewTableLayout(ConnectorSession connectorSession, ConnectorTableMetadata tableMetadata) { List<String> distributeColumns = (List<String>) tableMetadata.getProperties().get(DISTRIBUTED_ON); if (distributeColumns.isEmpty()) { return Optional.empty(); } Set<String> undefinedColumns = Sets.difference( ImmutableSet.copyOf(distributeColumns), tableMetadata.getColumns().stream() .map(ColumnMetadata::getName) .collect(toSet())); if (!undefinedColumns.isEmpty()) { throw new PrestoException(INVALID_TABLE_PROPERTY, "Distribute columns not defined on table: " + undefinedColumns); } return Optional.of(new ConnectorNewTableLayout(BlackHolePartitioningHandle.INSTANCE, distributeColumns)); }
private static void validatePartitionColumns(ConnectorTableMetadata tableMetadata) { List<String> partitionedBy = getPartitionedBy(tableMetadata.getProperties()); List<String> allColumns = tableMetadata.getColumns().stream() .map(ColumnMetadata::getName) .collect(toList()); if (!allColumns.containsAll(partitionedBy)) { throw new PrestoException(INVALID_TABLE_PROPERTY, format("Partition columns %s not present in schema", Sets.difference(ImmutableSet.copyOf(partitionedBy), ImmutableSet.copyOf(allColumns)))); } if (allColumns.size() == partitionedBy.size()) { throw new PrestoException(INVALID_TABLE_PROPERTY, "Table contains only partition columns"); } if (!allColumns.subList(allColumns.size() - partitionedBy.size(), allColumns.size()).equals(partitionedBy)) { throw new PrestoException(HIVE_COLUMN_ORDER_MISMATCH, "Partition keys must be the last columns in the table and in the same order as the table properties: " + partitionedBy); } }
/** * Gets the row ID based on a table properties or the first column name. * * @param meta ConnectorTableMetadata * @return Lowercase Presto column name mapped to the Accumulo row ID */ private static String getRowIdColumn(ConnectorTableMetadata meta) { Optional<String> rowIdColumn = AccumuloTableProperties.getRowId(meta.getProperties()); return rowIdColumn.orElse(meta.getColumns().get(0).getName()).toLowerCase(Locale.ENGLISH); }
private Map<String, String> getEmptyTableProperties(ConnectorTableMetadata tableMetadata, boolean partitioned, HdfsContext hdfsContext) { Builder<String, String> tableProperties = ImmutableMap.builder(); // Hook point for extended versions of the Hive Plugin tableProperties.putAll(tableParameterCodec.encode(tableMetadata.getProperties())); // ORC format specific properties List<String> columns = getOrcBloomFilterColumns(tableMetadata.getProperties()); if (columns != null && !columns.isEmpty()) { tableProperties.put(ORC_BLOOM_FILTER_COLUMNS_KEY, Joiner.on(",").join(columns)); tableProperties.put(ORC_BLOOM_FILTER_FPP_KEY, String.valueOf(getOrcBloomFilterFpp(tableMetadata.getProperties()))); } // Avro specific properties String avroSchemaUrl = getAvroSchemaUrl(tableMetadata.getProperties()); if (avroSchemaUrl != null) { HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(tableMetadata.getProperties()); if (hiveStorageFormat != HiveStorageFormat.AVRO) { throw new PrestoException(INVALID_TABLE_PROPERTY, format("Cannot specify %s table property for storage format: %s", AVRO_SCHEMA_URL, hiveStorageFormat)); } tableProperties.put(AVRO_SCHEMA_URL_KEY, validateAndNormalizeAvroSchemaUrl(avroSchemaUrl, hdfsContext)); } // Table comment property tableMetadata.getComment().ifPresent(value -> tableProperties.put(TABLE_COMMENT, value)); return tableProperties.build(); }
@Override public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata column) { ConnectorTableMetadata tableMetadata = getTableMetadata(session, tableHandle); SchemaTableName tableName = getTableName(tableHandle); ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder(); columns.addAll(tableMetadata.getColumns()); columns.add(column); tables.put(tableName, new ConnectorTableMetadata(tableName, columns.build(), tableMetadata.getProperties(), tableMetadata.getComment())); }
private static void validateBucketColumns(ConnectorTableMetadata tableMetadata) { Optional<HiveBucketProperty> bucketProperty = getBucketProperty(tableMetadata.getProperties()); if (!bucketProperty.isPresent()) { return; } Set<String> allColumns = tableMetadata.getColumns().stream() .map(ColumnMetadata::getName) .collect(toSet()); List<String> bucketedBy = bucketProperty.get().getBucketedBy(); if (!allColumns.containsAll(bucketedBy)) { throw new PrestoException(INVALID_TABLE_PROPERTY, format("Bucketing columns %s not present in schema", Sets.difference(ImmutableSet.copyOf(bucketedBy), ImmutableSet.copyOf(allColumns)))); } List<String> sortedBy = bucketProperty.get().getSortedBy().stream() .map(SortingColumn::getColumnName) .collect(toImmutableList()); if (!allColumns.containsAll(sortedBy)) { throw new PrestoException(INVALID_TABLE_PROPERTY, format("Sorting columns %s not present in schema", Sets.difference(ImmutableSet.copyOf(sortedBy), ImmutableSet.copyOf(allColumns)))); } }
int splitCount = (Integer) tableMetadata.getProperties().get(SPLIT_COUNT_PROPERTY); int pagesPerSplit = (Integer) tableMetadata.getProperties().get(PAGES_PER_SPLIT_PROPERTY); int rowsPerPage = (Integer) tableMetadata.getProperties().get(ROWS_PER_PAGE_PROPERTY); int fieldsLength = (Integer) tableMetadata.getProperties().get(FIELD_LENGTH_PROPERTY); Duration pageProcessingDelay = (Duration) tableMetadata.getProperties().get(PAGE_PROCESSING_DELAY);
@Override public TableStatisticsMetadata getStatisticsCollectionMetadata(ConnectorSession session, ConnectorTableMetadata tableMetadata) { if (!isCollectColumnStatisticsOnWrite(session)) { return TableStatisticsMetadata.empty(); } List<String> partitionedBy = firstNonNull(getPartitionedBy(tableMetadata.getProperties()), ImmutableList.of()); return getStatisticsCollectionMetadata(tableMetadata.getColumns(), partitionedBy); }
@Override public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle source, String target) { ConnectorTableMetadata tableMetadata = getTableMetadata(session, tableHandle); SchemaTableName tableName = getTableName(tableHandle); ColumnMetadata columnMetadata = getColumnMetadata(session, tableHandle, source); List<ColumnMetadata> columns = new ArrayList<>(tableMetadata.getColumns()); columns.set(columns.indexOf(columnMetadata), new ColumnMetadata(target, columnMetadata.getType(), columnMetadata.getComment(), columnMetadata.isHidden())); tables.put(tableName, new ConnectorTableMetadata(tableName, ImmutableList.copyOf(columns), tableMetadata.getProperties(), tableMetadata.getComment())); }
@Override public Optional<ConnectorNewTableLayout> getNewTableLayout(ConnectorSession session, ConnectorTableMetadata tableMetadata) { validatePartitionColumns(tableMetadata); validateBucketColumns(tableMetadata); Optional<HiveBucketProperty> bucketProperty = getBucketProperty(tableMetadata.getProperties()); if (!bucketProperty.isPresent()) { return Optional.empty(); } if (!bucketProperty.get().getSortedBy().isEmpty() && !isSortedWritingEnabled(session)) { throw new PrestoException(NOT_SUPPORTED, "Writing to bucketed sorted Hive tables is disabled"); } List<String> bucketedBy = bucketProperty.get().getBucketedBy(); Map<String, HiveType> hiveTypeMap = tableMetadata.getColumns().stream() .collect(toMap(ColumnMetadata::getName, column -> toHiveType(typeTranslator, column.getType()))); return Optional.of(new ConnectorNewTableLayout( new HivePartitioningHandle( bucketProperty.get().getBucketCount(), bucketedBy.stream() .map(hiveTypeMap::get) .collect(toList()), OptionalInt.of(bucketProperty.get().getBucketCount())), bucketedBy)); }
/** * Validates the given metadata for a series of conditions to ensure the table is well-formed. * * @param meta Table metadata */ private void validateCreateTable(ConnectorTableMetadata meta) { validateColumns(meta); validateLocalityGroups(meta); if (!AccumuloTableProperties.isExternal(meta.getProperties())) { validateInternalTable(meta); } }
@Test public void testBucketedCatalog() { String bucketedCatalog = bucketedSession.getCatalog().get(); String bucketedSchema = bucketedSession.getSchema().get(); TableMetadata ordersTableMetadata = getTableMetadata(bucketedCatalog, bucketedSchema, "orders"); assertEquals(ordersTableMetadata.getMetadata().getProperties().get(BUCKETED_BY_PROPERTY), ImmutableList.of("custkey")); assertEquals(ordersTableMetadata.getMetadata().getProperties().get(BUCKET_COUNT_PROPERTY), 11); TableMetadata customerTableMetadata = getTableMetadata(bucketedCatalog, bucketedSchema, "customer"); assertEquals(customerTableMetadata.getMetadata().getProperties().get(BUCKETED_BY_PROPERTY), ImmutableList.of("custkey")); assertEquals(customerTableMetadata.getMetadata().getProperties().get(BUCKET_COUNT_PROPERTY), 11); }
private void verifyPartitionedBucketedTable(HiveStorageFormat storageFormat, String tableName) { TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, tableName); assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat); assertEquals(tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY), ImmutableList.of("orderstatus")); assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKETED_BY_PROPERTY), ImmutableList.of("custkey", "custkey2")); assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKET_COUNT_PROPERTY), 11); List<?> partitions = getPartitions(tableName); assertEquals(partitions.size(), 3); assertQuery("SELECT * from " + tableName, "SELECT custkey, custkey, comment, orderstatus FROM orders"); for (int i = 1; i <= 30; i++) { assertQuery( format("SELECT * from " + tableName + " where custkey = %d and custkey2 = %d", i, i), format("SELECT custkey, custkey, comment, orderstatus FROM orders where custkey = %d", i)); } assertThatThrownBy(() -> assertUpdate("INSERT INTO " + tableName + " VALUES (1, 1, 'comment', 'O')", 1)) .hasMessage(getExpectedErrorMessageForInsertExistingBucketedTable( getInsertExistingPartitionsBehavior(getConnectorSession(getSession())), "orderstatus=O")); }
private void verifyPartition(boolean hasPartition, TableMetadata tableMetadata, List<String> partitionKeys) { Object partitionByProperty = tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY); if (hasPartition) { assertEquals(partitionByProperty, partitionKeys); for (ColumnMetadata columnMetadata : tableMetadata.getColumns()) { boolean partitionKey = partitionKeys.contains(columnMetadata.getName()); assertEquals(columnMetadata.getExtraInfo(), columnExtraInfo(partitionKey)); } } else { assertNull(partitionByProperty); } }
private void validateInternalTable(ConnectorTableMetadata meta) { String table = AccumuloTable.getFullTableName(meta.getTable()); String indexTable = Indexer.getIndexTableName(meta.getTable()); String metricsTable = Indexer.getMetricsTableName(meta.getTable()); if (tableManager.exists(table)) { throw new PrestoException(ACCUMULO_TABLE_EXISTS, "Cannot create internal table when an Accumulo table already exists"); } if (AccumuloTableProperties.getIndexColumns(meta.getProperties()).isPresent()) { if (tableManager.exists(indexTable) || tableManager.exists(metricsTable)) { throw new PrestoException(ACCUMULO_TABLE_EXISTS, "Internal table is indexed, but the index table and/or index metrics table(s) already exist"); } } }
private void testCreatePartitionedTableAs(Session session, HiveStorageFormat storageFormat) { @Language("SQL") String createTable = "" + "CREATE TABLE test_create_partitioned_table_as " + "WITH (" + "format = '" + storageFormat + "', " + "partitioned_by = ARRAY[ 'SHIP_PRIORITY', 'ORDER_STATUS' ]" + ") " + "AS " + "SELECT orderkey AS order_key, shippriority AS ship_priority, orderstatus AS order_status " + "FROM tpch.tiny.orders"; assertUpdate(session, createTable, "SELECT count(*) from orders"); TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, "test_create_partitioned_table_as"); assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat); assertEquals(tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY), ImmutableList.of("ship_priority", "order_status")); List<?> partitions = getPartitions("test_create_partitioned_table_as"); assertEquals(partitions.size(), 3); assertQuery(session, "SELECT * from test_create_partitioned_table_as", "SELECT orderkey, shippriority, orderstatus FROM orders"); assertUpdate(session, "DROP TABLE test_create_partitioned_table_as"); assertFalse(getQueryRunner().tableExists(session, "test_create_partitioned_table_as")); }
private void verifyPartitionedBucketedTableAsFewRows(HiveStorageFormat storageFormat, String tableName) { TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, tableName); assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat); assertEquals(tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY), ImmutableList.of("partition_key")); assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKETED_BY_PROPERTY), ImmutableList.of("bucket_key")); assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKET_COUNT_PROPERTY), 11); List<?> partitions = getPartitions(tableName); assertEquals(partitions.size(), 3); MaterializedResult actual = computeActual("SELECT * from " + tableName); MaterializedResult expected = resultBuilder(getSession(), canonicalizeType(createUnboundedVarcharType()), canonicalizeType(createUnboundedVarcharType()), canonicalizeType(createUnboundedVarcharType())) .row("a", "b", "c") .row("aa", "bb", "cc") .row("aaa", "bbb", "ccc") .build(); assertEqualsIgnoreOrder(actual.getMaterializedRows(), expected.getMaterializedRows()); }
assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat);
public KuduTable createTable(ConnectorTableMetadata tableMetadata, boolean ignoreExisting) { try { String rawName = schemaEmulation.toRawName(tableMetadata.getTable()); if (ignoreExisting) { if (client.tableExists(rawName)) { return null; } } if (!schemaEmulation.existsSchema(client, tableMetadata.getTable().getSchemaName())) { throw new SchemaNotFoundException(tableMetadata.getTable().getSchemaName()); } List<ColumnMetadata> columns = tableMetadata.getColumns(); Map<String, Object> properties = tableMetadata.getProperties(); Schema schema = buildSchema(columns, properties); CreateTableOptions options = buildCreateTableOptions(schema, properties); return client.createTable(rawName, schema, options); } catch (KuduException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, e); } }
Map<String, Object> tableProperties = meta.getProperties(); String rowIdColumn = getRowIdColumn(meta);