private PrimitiveType primitive(String name, PrimitiveTypeName primitive, OriginalType originalType) { return new PrimitiveType(Type.Repetition.OPTIONAL, primitive, name, originalType); } }
public NodeWriteSupport(boolean excludeMetadata) { super(excludeMetadata); latType = new PrimitiveType(REQUIRED, DOUBLE, "latitude"); longType = new PrimitiveType(REQUIRED, DOUBLE, "longitude"); }
private PrimitiveType primitive(String name, PrimitiveTypeName primitive, OriginalType originalType) { return new PrimitiveType(Type.Repetition.OPTIONAL, primitive, name, originalType); } }
private PrimitiveType primitive(String name, PrimitiveTypeName primitive) { return new PrimitiveType(Type.Repetition.OPTIONAL, primitive, name, null); }
private PrimitiveType primitive(String name, PrimitiveTypeName primitive) { return new PrimitiveType(Type.Repetition.OPTIONAL, primitive, name, null); }
public OsmEntityWriteSupport(boolean excludeMetadata) { idType = new PrimitiveType(REQUIRED, INT64, "id"); tagKeyType = new PrimitiveType(REQUIRED, BINARY, "key"); tagValueType = new PrimitiveType(OPTIONAL, BINARY, "value"); tags = new GroupType(REPEATED, "tags", tagKeyType, tagValueType); versionType = new PrimitiveType(OPTIONAL, INT32, "version"); timestampType = new PrimitiveType(OPTIONAL, INT64, "timestamp"); changesetType = new PrimitiveType(OPTIONAL, INT64, "changeset"); uidType = new PrimitiveType(OPTIONAL, INT32, "uid"); userSidType = new PrimitiveType(OPTIONAL, BINARY, "user_sid"); this.excludeMetadata = excludeMetadata; }
public RelationWriteSupport(boolean excludeMetadata) { super(excludeMetadata); memberIdType = new PrimitiveType(REQUIRED, INT64, "id"); memberRoleType = new PrimitiveType(REQUIRED, BINARY, "role"); memberTypeType = new PrimitiveType(REQUIRED, BINARY, "type"); membersType = new GroupType(REPEATED, "members", memberIdType, memberRoleType, memberTypeType); }
public WayWriteSupport(boolean excludeMetadata) { super(excludeMetadata); nodeIndexType = new PrimitiveType(REQUIRED, INT32, "index"); nodeIdType = new PrimitiveType(REQUIRED, INT64, "nodeId"); nodes = new GroupType(REPEATED, "nodes", nodeIndexType, nodeIdType); }
/** * @param path the path of this column in the write schema * @param type the primitive type of this column * @param codec the compression codec used for this column * @param encodings a set of encodings used by this column * @return column chunk properties * @deprecated will be removed in 2.0.0. Use {@link #get(ColumnPath, PrimitiveType, CompressionCodecName, Set)} * instead. */ @Deprecated public static ColumnChunkProperties get(ColumnPath path, PrimitiveTypeName type, CompressionCodecName codec, Set<Encoding> encodings) { return get(path, new PrimitiveType(Type.Repetition.OPTIONAL, type, ""), codec, encodings); }
/** * @param path the path of this column in the write schema * @param type the primitive type of this column * @param codec the compression codec used for this column * @param encodings a set of encodings used by this column * @return column chunk properties * @deprecated will be removed in 2.0.0. Use {@link #get(ColumnPath, PrimitiveType, CompressionCodecName, Set)} * instead. */ @Deprecated public static ColumnChunkProperties get(ColumnPath path, PrimitiveTypeName type, CompressionCodecName codec, Set<Encoding> encodings) { return get(path, new PrimitiveType(Type.Repetition.OPTIONAL, type, ""), codec, encodings); }
/** * @param createdBy the created-by string from the file * @param statistics parquet format statistics * @param type a primitive type name * @return the statistics * @deprecated will be removed in 2.0.0. */ @Deprecated public static org.apache.parquet.column.statistics.Statistics fromParquetStatistics (String createdBy, Statistics statistics, PrimitiveTypeName type) { return fromParquetStatisticsInternal(createdBy, statistics, new PrimitiveType(Repetition.OPTIONAL, type, "fake_type"), defaultSortOrder(type)); }
/** * @param id the field id * @return a new PrimitiveType with the same fields and a new id */ @Override public PrimitiveType withId(int id) { return new PrimitiveType(getRepetition(), primitive, length, getName(), getOriginalType(), decimalMeta, new ID(id), columnOrder); }
/** * @param id the field id * @return a new PrimitiveType with the same fields and a new id */ @Override public PrimitiveType withId(int id) { return new PrimitiveType(getRepetition(), primitive, length, getName(), getLogicalTypeAnnotation(), new ID(id), columnOrder); }
@Test public void testMapInt32ToPrestoDate() { // int32 primitive with original type of date should map to a Presto date PrimitiveType dateType = new PrimitiveType(OPTIONAL, PrimitiveTypeName.INT32, "date_col", OriginalType.DATE); ColumnDescriptor columnDescriptor = new ColumnDescriptor(new String[] {"date_col"}, PrimitiveTypeName.INT32, 0, 1); RichColumnDescriptor dateColumn = new RichColumnDescriptor(columnDescriptor, dateType); assertEquals(getPrestoType(TupleDomain.all(), dateColumn), DATE); } }
@Test public void testMatchesWithDescriptors() throws ParquetCorruptionException { ColumnDescriptor columnDescriptor = new ColumnDescriptor(new String[] {"path"}, BINARY, 0, 0); RichColumnDescriptor column = new RichColumnDescriptor(columnDescriptor, new PrimitiveType(OPTIONAL, BINARY, "Test column")); TupleDomain<ColumnDescriptor> effectivePredicate = getEffectivePredicate(column, createVarcharType(255), EMPTY_SLICE); TupleDomainParquetPredicate parquetPredicate = new TupleDomainParquetPredicate(effectivePredicate, singletonList(column)); DictionaryPage page = new DictionaryPage(Slices.wrappedBuffer(new byte[] {0, 0, 0, 0}), 1, PLAIN_DICTIONARY); assertTrue(parquetPredicate.matches(singletonMap(column, new DictionaryDescriptor(column, Optional.of(page))))); }
@Test public void testMapInt32ToPrestoInteger() { PrimitiveType intType = new PrimitiveType(OPTIONAL, PrimitiveTypeName.INT32, "int_col", OriginalType.INT_32); ColumnDescriptor columnDescriptor = new ColumnDescriptor(new String[] {"int_col"}, PrimitiveTypeName.INT32, 0, 1); RichColumnDescriptor intColumn = new RichColumnDescriptor(columnDescriptor, intType); assertEquals(getPrestoType(TupleDomain.all(), intColumn), INTEGER); }
@Test public void testMapInt32WithoutOriginalTypeToPrestoInteger() { // int32 primitive should default to Presto integer if original type metadata isn't available PrimitiveType intType = new PrimitiveType(OPTIONAL, PrimitiveTypeName.INT32, "int_col"); ColumnDescriptor columnDescriptor = new ColumnDescriptor(new String[] {"int_col"}, PrimitiveTypeName.INT32, 0, 1); RichColumnDescriptor intColumn = new RichColumnDescriptor(columnDescriptor, intType); assertEquals(getPrestoType(TupleDomain.all(), intColumn), INTEGER); }
@Test public void testParquetTupleDomainPrimitiveArray() { HiveColumnHandle columnHandle = new HiveColumnHandle("my_array", HiveType.valueOf("array<int>"), parseTypeSignature(StandardTypes.ARRAY), 0, REGULAR, Optional.empty()); TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(new ArrayType(INTEGER)))); MessageType fileSchema = new MessageType("hive_schema", new GroupType(OPTIONAL, "my_array", new GroupType(REPEATED, "bag", new PrimitiveType(OPTIONAL, INT32, "array_element")))); Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema); TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain); assertTrue(tupleDomain.getDomains().get().isEmpty()); }
@Test public void testParquetTupleDomainPrimitiveArray() { HiveColumnHandle columnHandle = new HiveColumnHandle("my_array", HiveType.valueOf("array<int>"), parseTypeSignature(StandardTypes.ARRAY), 0, REGULAR, Optional.empty()); TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(new ArrayType(INTEGER)))); MessageType fileSchema = new MessageType("hive_schema", new GroupType(OPTIONAL, "my_array", new GroupType(REPEATED, "bag", new PrimitiveType(OPTIONAL, INT32, "array_element")))); Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema); TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain); assertTrue(tupleDomain.getDomains().get().isEmpty()); }
private PrimitiveType getPrimitiveType(Field field) { MajorType majorType = getMajorTypeForField(field); MinorType minorType = majorType.getMinorType(); String name = field.getName(); PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType); if (primitiveTypeName == null) { return null; } OriginalType originalType = ParquetTypeHelper.getOriginalTypeForMinorType(minorType); int length = ParquetTypeHelper.getLengthForMinorType(minorType); DecimalMetadata decimalMetadata = ParquetTypeHelper.getDecimalMetadataForField(majorType); return new PrimitiveType(OPTIONAL, primitiveTypeName, length, name, originalType, decimalMetadata, null); }