@Test public void testEmptyTableCreation() throws Exception { for (HiveStorageFormat storageFormat : createTableFormats) { SchemaTableName temporaryCreateEmptyTable = temporaryTable("create_empty"); try { doCreateEmptyTable(temporaryCreateEmptyTable, storageFormat, CREATE_TABLE_COLUMNS); } finally { dropTable(temporaryCreateEmptyTable); } } }
@Test public void testUpdateBasicTableStatistics() throws Exception { SchemaTableName tableName = temporaryTable("update_basic_table_statistics"); try { doCreateEmptyTable(tableName, ORC, STATISTICS_TABLE_COLUMNS); testUpdateTableStatistics(tableName, EMPTY_TABLE_STATISTICS, BASIC_STATISTICS_1, BASIC_STATISTICS_2); } finally { dropTable(tableName); } }
@Test public void testUpdateTableColumnStatisticsEmptyOptionalFields() throws Exception { SchemaTableName tableName = temporaryTable("update_table_column_statistics_empty_optional_fields"); try { doCreateEmptyTable(tableName, ORC, STATISTICS_TABLE_COLUMNS); testUpdateTableStatistics(tableName, EMPTY_TABLE_STATISTICS, STATISTICS_EMPTY_OPTIONAL_FIELDS); } finally { dropTable(tableName); } }
@Test public void testUpdateTableColumnStatistics() throws Exception { SchemaTableName tableName = temporaryTable("update_table_column_statistics"); try { doCreateEmptyTable(tableName, ORC, STATISTICS_TABLE_COLUMNS); testUpdateTableStatistics(tableName, EMPTY_TABLE_STATISTICS, STATISTICS_1_1, STATISTICS_1_2, STATISTICS_2); } finally { dropTable(tableName); } }
protected void createDummyPartitionedTable(SchemaTableName tableName, List<ColumnMetadata> columns) throws Exception { doCreateEmptyTable(tableName, ORC, columns); ExtendedHiveMetastore metastoreClient = getMetastoreClient(); Table table = metastoreClient.getTable(tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new TableNotFoundException(tableName)); List<String> firstPartitionValues = ImmutableList.of("2016-01-01"); List<String> secondPartitionValues = ImmutableList.of("2016-01-02"); String firstPartitionName = makePartName(ImmutableList.of("ds"), firstPartitionValues); String secondPartitionName = makePartName(ImmutableList.of("ds"), secondPartitionValues); List<PartitionWithStatistics> partitions = ImmutableList.of(firstPartitionName, secondPartitionName) .stream() .map(partitionName -> new PartitionWithStatistics(createDummyPartition(table, partitionName), partitionName, PartitionStatistics.empty())) .collect(toImmutableList()); metastoreClient.addPartitions(tableName.getSchemaName(), tableName.getTableName(), partitions); metastoreClient.updatePartitionStatistics(tableName.getSchemaName(), tableName.getTableName(), firstPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS); metastoreClient.updatePartitionStatistics(tableName.getSchemaName(), tableName.getTableName(), secondPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS); }
private void doInsertIntoExistingPartitionEmptyStatistics(HiveStorageFormat storageFormat, SchemaTableName tableName) throws Exception { doCreateEmptyTable(tableName, storageFormat, CREATE_TABLE_COLUMNS_PARTITIONED); insertData(tableName, CREATE_TABLE_PARTITIONED_DATA); eraseStatistics(tableName); insertData(tableName, CREATE_TABLE_PARTITIONED_DATA); try (Transaction transaction = newTransaction()) { List<String> partitionNames = transaction.getMetastore(tableName.getSchemaName()).getPartitionNames(tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new AssertionError("Table does not exist: " + tableName)); for (String partitionName : partitionNames) { HiveBasicStatistics statistics = getBasicStatisticsForPartition(transaction, tableName, partitionName); assertThat(statistics.getRowCount()).isNotPresent(); assertThat(statistics.getInMemoryDataSizeInBytes()).isNotPresent(); // fileCount and rawSize statistics are computed on the fly by the metastore, thus cannot be erased } } }
doCreateEmptyTable(tableName, storageFormat, CREATE_TABLE_COLUMNS_PARTITIONED);
doCreateEmptyTable(tableName, storageFormat, CREATE_TABLE_COLUMNS_PARTITIONED);
doCreateEmptyTable(tableName, storageFormat, CREATE_TABLE_COLUMNS);
doCreateEmptyTable(tableName, storageFormat, CREATE_TABLE_COLUMNS_PARTITIONED);
String tableName = schemaTableName.getTableName(); doCreateEmptyTable(schemaTableName, storageFormat, tableBefore);
doCreateEmptyTable(tableName, ORC, columns);
@Test public void testEmptyTableCreation() throws Exception { for (HiveStorageFormat storageFormat : createTableFormats) { try { doCreateEmptyTable(temporaryCreateEmptyTable, storageFormat, CREATE_TABLE_COLUMNS); } finally { dropTable(temporaryCreateEmptyTable); } } }
@Test public void testEmptyTableCreation() throws Exception { for (HiveStorageFormat storageFormat : createTableFormats) { SchemaTableName temporaryCreateEmptyTable = temporaryTable("create_empty"); try { doCreateEmptyTable(temporaryCreateEmptyTable, storageFormat, CREATE_TABLE_COLUMNS); } finally { dropTable(temporaryCreateEmptyTable); } } }
@Test public void testUpdateTableColumnStatistics() throws Exception { SchemaTableName tableName = temporaryTable("update_table_column_statistics"); try { doCreateEmptyTable(tableName, ORC, STATISTICS_TABLE_COLUMNS); testUpdateTableStatistics(tableName, EMPTY_TABLE_STATISTICS, STATISTICS_1_1, STATISTICS_1_2, STATISTICS_2); } finally { dropTable(tableName); } }
@Test public void testUpdateTableColumnStatisticsEmptyOptionalFields() throws Exception { SchemaTableName tableName = temporaryTable("update_table_column_statistics_empty_optional_fields"); try { doCreateEmptyTable(tableName, ORC, STATISTICS_TABLE_COLUMNS); testUpdateTableStatistics(tableName, EMPTY_TABLE_STATISTICS, STATISTICS_EMPTY_OPTIONAL_FIELDS); } finally { dropTable(tableName); } }
@Test public void testUpdateBasicTableStatistics() throws Exception { SchemaTableName tableName = temporaryTable("update_basic_table_statistics"); try { doCreateEmptyTable(tableName, ORC, STATISTICS_TABLE_COLUMNS); testUpdateTableStatistics(tableName, EMPTY_TABLE_STATISTICS, BASIC_STATISTICS_1, BASIC_STATISTICS_2); } finally { dropTable(tableName); } }
protected void createDummyPartitionedTable(SchemaTableName tableName, List<ColumnMetadata> columns) throws Exception { doCreateEmptyTable(tableName, ORC, columns); ExtendedHiveMetastore metastoreClient = getMetastoreClient(); Table table = metastoreClient.getTable(tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new TableNotFoundException(tableName)); List<String> firstPartitionValues = ImmutableList.of("2016-01-01"); List<String> secondPartitionValues = ImmutableList.of("2016-01-02"); String firstPartitionName = makePartName(ImmutableList.of("ds"), firstPartitionValues); String secondPartitionName = makePartName(ImmutableList.of("ds"), secondPartitionValues); List<PartitionWithStatistics> partitions = ImmutableList.of(firstPartitionName, secondPartitionName) .stream() .map(partitionName -> new PartitionWithStatistics(createDummyPartition(table, partitionName), partitionName, PartitionStatistics.empty())) .collect(toImmutableList()); metastoreClient.addPartitions(tableName.getSchemaName(), tableName.getTableName(), partitions); metastoreClient.updatePartitionStatistics(tableName.getSchemaName(), tableName.getTableName(), firstPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS); metastoreClient.updatePartitionStatistics(tableName.getSchemaName(), tableName.getTableName(), secondPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS); }
private void doInsertIntoExistingPartitionEmptyStatistics(HiveStorageFormat storageFormat, SchemaTableName tableName) throws Exception { doCreateEmptyTable(tableName, storageFormat, CREATE_TABLE_COLUMNS_PARTITIONED); insertData(tableName, CREATE_TABLE_PARTITIONED_DATA); eraseStatistics(tableName); insertData(tableName, CREATE_TABLE_PARTITIONED_DATA); try (Transaction transaction = newTransaction()) { List<String> partitionNames = transaction.getMetastore(tableName.getSchemaName()).getPartitionNames(tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new AssertionError("Table does not exist: " + tableName)); for (String partitionName : partitionNames) { HiveBasicStatistics statistics = getBasicStatisticsForPartition(transaction, tableName, partitionName); assertThat(statistics.getRowCount()).isNotPresent(); assertThat(statistics.getInMemoryDataSizeInBytes()).isNotPresent(); // fileCount and rawSize statistics are computed on the fly by the metastore, thus cannot be erased } } }
doCreateEmptyTable(tableName, storageFormat, CREATE_TABLE_COLUMNS_PARTITIONED);