private static String buildColumnTypes(List<HiveColumnHandle> columns) { if (columns == null || columns.isEmpty()) { return ""; } return columns.stream() .map(column -> column.getHiveType().getTypeInfo().getTypeName()) .collect(Collectors.joining(",")); }
public HiveCoercionRecordCursor( List<ColumnMapping> columnMappings, TypeManager typeManager, RecordCursor delegate) { requireNonNull(columnMappings, "columns is null"); requireNonNull(typeManager, "typeManager is null"); this.bridgingRecordCursor = new BridgingRecordCursor(); this.delegate = requireNonNull(delegate, "delegate is null"); this.columnMappings = ImmutableList.copyOf(columnMappings); int size = columnMappings.size(); this.coercers = new Coercer[size]; for (int columnIndex = 0; columnIndex < size; columnIndex++) { ColumnMapping columnMapping = columnMappings.get(columnIndex); if (columnMapping.getCoercionFrom().isPresent()) { coercers[columnIndex] = createCoercer(typeManager, columnMapping.getCoercionFrom().get(), columnMapping.getHiveColumnHandle().getHiveType(), bridgingRecordCursor); } } }
private List<String> toConjuncts(List<HiveColumnHandle> columns, TupleDomain<HiveColumnHandle> tupleDomain) { Builder<String> builder = builder(); for (HiveColumnHandle column : columns) { Type type = column.getHiveType().getType(typeManager); if (tupleDomain.getDomains().isPresent() && isSupported(type)) { Domain domain = tupleDomain.getDomains().get().get(column); if (domain != null) { builder.add(toPredicate(domain, type, column.getHiveColumnIndex())); } } } return builder.build(); }
public static TupleDomain<ColumnDescriptor> getParquetTupleDomain(Map<List<String>, RichColumnDescriptor> descriptorsByPath, TupleDomain<HiveColumnHandle> effectivePredicate) { if (effectivePredicate.isNone()) { return TupleDomain.none(); } ImmutableMap.Builder<ColumnDescriptor, Domain> predicate = ImmutableMap.builder(); for (Entry<HiveColumnHandle, Domain> entry : effectivePredicate.getDomains().get().entrySet()) { HiveColumnHandle columnHandle = entry.getKey(); // skip looking up predicates for complex types as Parquet only stores stats for primitives if (!columnHandle.getHiveType().getCategory().equals(PRIMITIVE)) { continue; } RichColumnDescriptor descriptor = descriptorsByPath.get(ImmutableList.of(columnHandle.getName())); if (descriptor != null) { predicate.put(descriptor, entry.getValue()); } } return TupleDomain.withColumnDomains(predicate.build()); }
hiveTypesBuilder.add(column.getHiveType());
dataColumnsInputIndex.add(inputIndex); dataColumnNameToIdMap.put(column.getName(), inputIndex); dataColumnNameToTypeMap.put(column.getName(), column.getHiveType());
public void validateColumns(List<HiveColumnHandle> handles) { if (this == AVRO) { for (HiveColumnHandle handle : handles) { if (!handle.isPartitionKey()) { validateAvroType(handle.getHiveType().getTypeInfo(), handle.getName()); } } } }
private static List<HiveColumnHandle> getPhysicalHiveColumnHandles(List<HiveColumnHandle> columns, boolean useOrcColumnNames, OrcReader reader, Path path) { if (!useOrcColumnNames) { return columns; } verifyFileHasColumnNames(reader.getColumnNames(), path); Map<String, Integer> physicalNameOrdinalMap = buildPhysicalNameOrdinalMap(reader); int nextMissingColumnIndex = physicalNameOrdinalMap.size(); ImmutableList.Builder<HiveColumnHandle> physicalColumns = ImmutableList.builder(); for (HiveColumnHandle column : columns) { Integer physicalOrdinal = physicalNameOrdinalMap.get(column.getName()); if (physicalOrdinal == null) { // if the column is missing from the file, assign it a column number larger // than the number of columns in the file so the reader will fill it with nulls physicalOrdinal = nextMissingColumnIndex; nextMissingColumnIndex++; } physicalColumns.add(new HiveColumnHandle(column.getName(), column.getHiveType(), column.getTypeSignature(), physicalOrdinal, column.getColumnType(), column.getComment())); } return physicalColumns.build(); }
@Test public void testColumnHandleDeserialize() throws Exception { String json = objectMapper.writeValueAsString(COLUMN_HANDLE_AS_MAP); HiveColumnHandle columnHandle = objectMapper.readValue(json, HiveColumnHandle.class); assertEquals(columnHandle.getName(), "column"); assertEquals(columnHandle.getTypeSignature(), DOUBLE.getTypeSignature()); assertEquals(columnHandle.getHiveType(), HiveType.HIVE_FLOAT); assertEquals(columnHandle.getHiveColumnIndex(), -1); assertEquals(columnHandle.isPartitionKey(), true); }
hiveTypes[i] = column.getHiveType();
ImmutableList.Builder<DataColumn> dataColumns = ImmutableList.builder(); for (HiveColumnHandle column : inputColumns) { HiveType hiveType = column.getHiveType(); if (column.isPartitionKey()) { partitionColumnNames.add(column.getName());
private void testRoundTrip(HiveColumnHandle expected) { String json = codec.toJson(expected); HiveColumnHandle actual = codec.fromJson(json); assertEquals(actual.getName(), expected.getName()); assertEquals(actual.getHiveType(), expected.getHiveType()); assertEquals(actual.getHiveColumnIndex(), expected.getHiveColumnIndex()); assertEquals(actual.isPartitionKey(), expected.isPartitionKey()); } }
List<Column> partitionColumns = partitionedBy.stream() .map(columnHandlesByName::get) .map(column -> new Column(column.getName(), column.getHiveType(), column.getComment())) .collect(toList()); for (HiveColumnHandle columnHandle : columnHandles) { String name = columnHandle.getName(); HiveType type = columnHandle.getHiveType(); if (!partitionColumnNames.contains(name)) { verify(!columnHandle.isPartitionKey(), "Column handles are not consistent with partitioned by property");
.toArray(); List<HiveType> bucketColumnHiveTypes = conversion.getBucketColumnHandles().stream() .map(columnHandle -> hiveIndexToBlockIndex.get(columnHandle.getHiveColumnIndex()).getHiveColumnHandle().getHiveType()) .collect(toImmutableList()); return new BucketAdaptation(bucketColumnIndices, bucketColumnHiveTypes, conversion.getTableBucketCount(), conversion.getPartitionBucketCount(), bucketNumber.getAsInt());
ImmutableMap.Builder<Integer, Type> readColumns = ImmutableMap.builder(); for (HiveColumnHandle column : columns) { readColumns.put(column.getHiveColumnIndex(), column.getHiveType().getType(typeManager));
coercers[columnIndex] = createCoercer(typeManager, columnMapping.getCoercionFrom().get(), columnMapping.getHiveColumnHandle().getHiveType());
.collect(toImmutableMap(HiveColumnHandle::getName, column -> column.getHiveType().getType(typeManager))); Map<List<String>, ComputedStatistics> partitionComputedStatistics = createComputedStatisticsToPartitionMap(computedStatistics, handle.getPartitionedBy(), columnTypes);
.collect(toImmutableList()); Map<String, Type> columnTypes = handle.getInputColumns().stream() .collect(toImmutableMap(HiveColumnHandle::getName, column -> column.getHiveType().getType(typeManager))); Map<List<String>, ComputedStatistics> partitionComputedStatistics = createComputedStatisticsToPartitionMap(computedStatistics, partitionedBy, columnTypes);
List<Column> partitionColumns = partitionedBy.stream() .map(columnHandlesByName::get) .map(column -> new Column(column.getName(), column.getHiveType(), column.getComment())) .collect(toList()); checkPartitionTypesSupported(partitionColumns);
List<Column> partitionColumns = partitionedBy.stream() .map(columnHandlesByName::get) .map(column -> new Column(column.getName(), column.getHiveType(), column.getComment())) .collect(toList()); checkPartitionTypesSupported(partitionColumns);