static Schema getSchema(Types.StructType partitionType) { return AvroSchemaUtil.convert(partitionType, PartitionData.class.getName()); }
public static Schema convert(Types.StructType type, String name) { return convert(type, ImmutableMap.of(type, name)); }
public static Schema convert(Type type) { return convert(type, ImmutableMap.of()); }
/** * Adds an {@link com.netflix.iceberg.Schema Iceberg schema} that can be used to decode buffers. * * @param writeSchema a schema to use when decoding buffers */ public void addSchema(com.netflix.iceberg.Schema writeSchema) { addSchema(AvroSchemaUtil.convert(writeSchema, "table")); }
private static org.apache.avro.Schema getAvroSchema(Types.StructType partitionType) { Types.StructType type = DataFile.getType(partitionType); return AvroSchemaUtil.convert(type, ImmutableMap.of( type, GenericDataFile.class.getName(), partitionType, PartitionData.class.getName())); }
public static Schema convert(com.netflix.iceberg.Schema schema, String tableName) { return convert(schema, ImmutableMap.of(schema.asStruct(), tableName)); }
ManifestEntry(StructType partitionType) { this.schema = AvroSchemaUtil.convert(getSchema(partitionType), "manifest_entry"); }
/** * Used by Avro reflection to instantiate this class when reading manifest files. */ public PartitionData(Schema schema) { this.partitionType = AvroSchemaUtil.convert(schema).asNestedType().asStructType(); this.size = partitionType.fields().size(); this.data = new Object[size]; this.stringSchema = schema.toString(); this.schema = schema; }
@SuppressWarnings("unchecked") private <T> WriteSupport<T> getWriteSupport(MessageType type) { if (writeSupport != null) { return (WriteSupport<T>) writeSupport; } else { return new AvroWriteSupport<>( type, ParquetAvro.parquetAvroSchema(AvroSchemaUtil.convert(schema, name)), ParquetAvro.DEFAULT_MODEL); } }
@Override public ValueReader<?> record(Schema record, List<String> names, List<ValueReader<?>> fields) { return GenericReaders.struct(AvroSchemaUtil.convert(record).asStructType(), fields); }
public <D> FileAppender<D> build() throws IOException { Preconditions.checkNotNull(schema, "Schema is required"); Preconditions.checkNotNull(name, "Table name is required and cannot be null"); // add the Iceberg schema to keyValueMetadata meta("iceberg.schema", SchemaParser.toJson(schema)); return new AvroFileAppender<>( AvroSchemaUtil.convert(schema, name), file, createWriterFunc, codec(), metadata); } }
private static Record projectFlat(Schema projection, Record record) { org.apache.avro.Schema avroSchema = AvroSchemaUtil.convert(projection, "test"); Record result = new Record(avroSchema); List<Types.NestedField> fields = projection.asStruct().fields(); for (int i = 0; i < fields.size(); i += 1) { Types.NestedField field = fields.get(i); result.put(i, record.get(field.name())); } return result; }
@Test public void testMapOfLongToBytes() { Type map = Types.MapType.ofRequired(33, 34, Types.LongType.get(), Types.BinaryType.get()); Schema schema = AvroSchemaUtil.createMap( 33, Schema.create(Schema.Type.LONG), 34, Schema.create(Schema.Type.BYTES)); Assert.assertEquals("Avro schema to map", map, AvroSchemaUtil.convert(schema)); Assert.assertEquals("Map to Avro schema", schema, AvroSchemaUtil.convert(map)); }
@Test public void testList() { Type list = Types.ListType.ofRequired(34, Types.UUIDType.get()); Schema schema = addElementId(34, SchemaBuilder.array().items( LogicalTypes.uuid().addToSchema(Schema.createFixed("uuid_fixed", null, null, 16)))); Assert.assertEquals("Avro schema to list", list, AvroSchemaUtil.convert(schema)); Assert.assertEquals("List to Avro schema", schema, AvroSchemaUtil.convert(list)); }
@Test public void testMapOfStringToBytes() { Type map = Types.MapType.ofRequired(33, 34, Types.StringType.get(), Types.BinaryType.get()); Schema schema = addKeyId(33, addValueId(34, SchemaBuilder.map().values( Schema.create(Schema.Type.BYTES)))); Assert.assertEquals("Avro schema to map", map, AvroSchemaUtil.convert(schema)); Assert.assertEquals("Map to Avro schema", schema, AvroSchemaUtil.convert(map)); }
@Test public void testCompatibleReadWithSchemaFromLookup() throws Exception { MessageEncoder<Record> v1Encoder = new IcebergEncoder<>(SCHEMA_V1); SchemaStore.Cache schemaCache = new SchemaStore.Cache(); schemaCache.addSchema(AvroSchemaUtil.convert(SCHEMA_V1, "table")); IcebergDecoder<Record> v2Decoder = new IcebergDecoder<>(SCHEMA_V2, schemaCache); ByteBuffer v1Buffer = v1Encoder.encode(V1_RECORDS.get(2)); Record record = v2Decoder.decode(v1Buffer); Assert.assertEquals(v2Record(4L, "m-4", null), record); }
@Test public void testFullProjection() throws Exception { Schema schema = new Schema( Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()) ); Record record = new Record(AvroSchemaUtil.convert(schema, "table")); record.put("id", 34L); record.put("data", "test"); Record projected = writeAndRead("full_projection", schema, schema, record); Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id")); int cmp = Comparators.charSequences() .compare("test", (CharSequence) projected.get("data")); Assert.assertTrue("Should contain the correct data value", cmp == 0); }
@Test public void testFullProjection() throws Exception { Schema schema = new Schema( Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()) ); Record record = new Record(AvroSchemaUtil.convert(schema, "table")); record.put("id", 34L); record.put("data", "test"); Record projected = writeAndRead("full_projection", schema, schema, record); Assert.assertEquals("Should contain the correct id value", 34L, (long) projected.get("id")); int cmp = Comparators.charSequences() .compare("test", (CharSequence) projected.get("data")); Assert.assertEquals("Should contain the correct data value", 0, cmp); }
@Test public void testEmptyProjection() throws Exception { Schema schema = new Schema( Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()) ); Record record = new Record(AvroSchemaUtil.convert(schema, "table")); record.put("id", 34L); record.put("data", "test"); Record projected = writeAndRead("empty_projection", schema, schema.select(), record); Assert.assertNotNull("Should read a non-null record", projected); try { projected.get(0); Assert.fail("Should not retrieve value with ordinal 0"); } catch (ArrayIndexOutOfBoundsException e) { // this is expected because there are no values } }
@Test public void testMapOfStringToStructs() { Type map = Types.MapType.ofRequired(33, 34, Types.StringType.get(), Types.StructType.of( required(35, "a", Types.IntegerType.get()), optional(36, "b", Types.IntegerType.get()) )); Schema schema = addKeyId(33, addValueId(34, SchemaBuilder.map().values( record("r34", requiredField(35, "a", Schema.create(Schema.Type.INT)), optionalField(36, "b", Schema.create(Schema.Type.INT)))))); Assert.assertEquals("Avro schema to map", map, AvroSchemaUtil.convert(schema)); Assert.assertEquals("Map to Avro schema", schema, AvroSchemaUtil.convert(map)); }