static void toJson(Types.MapType map, JsonGenerator generator) throws IOException { generator.writeStartObject(); generator.writeStringField(TYPE, MAP); generator.writeNumberField(KEY_ID, map.keyId()); generator.writeFieldName(KEY); toJson(map.keyType(), generator); generator.writeNumberField(VALUE_ID, map.valueId()); generator.writeFieldName(VALUE); toJson(map.valueType(), generator); generator.writeBooleanField(VALUE_REQUIRED, !map.isValueOptional()); generator.writeEndObject(); }
@Override public List<String> map(Types.MapType readMap, Supplier<List<String>> keyErrors, Supplier<List<String>> valueErrors) { if (!currentType.isMapType()) { return ImmutableList.of(String.format(": %s cannot be read as a map", currentType)); } Types.MapType map = currentType.asNestedType().asMapType(); List<String> errors = Lists.newArrayList(); try { if (readMap.isValueRequired() && map.isValueOptional()) { errors.add(": values should be required, but are optional"); } this.currentType = map.keyType(); errors.addAll(keyErrors.get()); this.currentType = map.valueType(); errors.addAll(valueErrors.get()); return errors; } finally { this.currentType = map; } }
@Override public Type map(Types.MapType map, Type kResult, Type vResult) { // if any updates are intended for the key, throw an exception int keyId = map.fields().get(0).fieldId(); if (deletes.contains(keyId)) { throw new IllegalArgumentException("Cannot delete map keys: " + map); } else if (updates.containsKey(keyId)) { throw new IllegalArgumentException("Cannot update map keys: " + map); } else if (adds.containsKey(keyId)) { throw new IllegalArgumentException("Cannot add fields to map keys: " + map); } else if (!map.keyType().equals(kResult)) { throw new IllegalArgumentException("Cannot alter map keys: " + map); } // use field to apply updates to the value Type valueResult = field(map.fields().get(1), vResult); if (valueResult == null) { throw new IllegalArgumentException("Cannot delete value type from map: " + map); } if (map.valueType() == valueResult) { return map; } if (map.isValueOptional()) { return Types.MapType.ofOptional(map.keyId(), map.valueId(), map.keyType(), valueResult); } else { return Types.MapType.ofRequired(map.keyId(), map.valueId(), map.keyType(), valueResult); } }
case MAP: final Types.MapType mapType = type.asMapType(); return format("map<%s,%s>", convert(mapType.keyType()), convert(mapType.valueType())); default: throw new UnsupportedOperationException(type +" is not supported");
keyResult = visit(map.keyType(), visitor); } finally { visitor.fieldIds.pop();
case MAP: final Types.MapType mapType = type.asMapType(); return String.format("map<%s,%s>", fromIcebergToHiveType(mapType.keyType()), fromIcebergToHiveType(mapType.valueType())); default:
null == TypeUtil.find( schema, type -> type.isMapType() && type.asMapType().keyType() != Types.StringType.get()));
Types.MapType map = type.asNestedType().asMapType(); return visitor.map(map, new VisitFuture<>(map.keyType(), visitor), new VisitFuture<>(map.valueType(), visitor));
@Override public Object map(Types.MapType map, Supplier<Object> keyResult, Supplier<Object> valueResult) { int numEntries = random.nextInt(20); Map<Object, Object> result = Maps.newLinkedHashMap(); Supplier<Object> keyFunc; if (map.keyType() == Types.StringType.get()) { keyFunc = () -> keyResult.get().toString(); } else { keyFunc = keyResult; } Set<Object> keySet = Sets.newHashSet(); for (int i = 0; i < numEntries; i += 1) { Object key = keyFunc.get(); // ensure no collisions while (keySet.contains(key)) { key = keyFunc.get(); } keySet.add(key); // return null 5% of the time when the value is optional if (map.isValueOptional() && random.nextInt(20) == 1) { result.put(key, null); } else { result.put(key, valueResult.get()); } } return result; }
@Override public Object map(Types.MapType map, Supplier<Object> keyResult, Supplier<Object> valueResult) { int numEntries = random.nextInt(20); Map<Object, Object> result = Maps.newLinkedHashMap(); Supplier<Object> keyFunc; if (map.keyType() == Types.StringType.get()) { keyFunc = () -> keyResult.get().toString(); } else { keyFunc = keyResult; } Set<Object> keySet = Sets.newHashSet(); for (int i = 0; i < numEntries; i += 1) { Object key = keyFunc.get(); // ensure no collisions while (keySet.contains(key)) { key = keyFunc.get(); } keySet.add(key); // return null 5% of the time when the value is optional if (map.isValueOptional() && random.nextInt(20) == 1) { result.put(key, null); } else { result.put(key, valueResult.get()); } } return result; }
protected void writeAndValidate(Schema schema) throws IOException { Assume.assumeTrue("Parquet Avro cannot write non-string map keys", null == TypeUtil.find(schema, type -> type.isMapType() && type.asMapType().keyType() != Types.StringType.get())); List<GenericData.Record> expected = RandomData.generateList(schema, 100, 0L); File testFile = temp.newFile(); Assert.assertTrue("Delete should succeed", testFile.delete()); try (FileAppender<GenericData.Record> writer = Parquet.write(Files.localOutput(testFile)) .schema(schema) .named("test") .build()) { writer.addAll(expected); } try (CloseableIterable<InternalRow> reader = Parquet.read(Files.localInput(testFile)) .project(schema) .createReaderFunc(type -> SparkParquetReaders.buildReader(schema, type)) .build()) { Iterator<InternalRow> rows = reader.iterator(); for (int i = 0; i < expected.size(); i += 1) { Assert.assertTrue("Should have expected number of rows", rows.hasNext()); assertEqualsUnsafe(schema.asStruct(), expected.get(i), rows.next()); } Assert.assertFalse("Should not have extra rows", rows.hasNext()); } } }
@Override public Type map(Types.MapType map, Supplier<Type> keyResult, Supplier<Type> valueResult) { Preconditions.checkArgument(current instanceof MapType, "Not a map: %s", current); MapType m = (MapType) current; Preconditions.checkArgument(m.valueContainsNull() || !map.isValueOptional(), "Cannot project a map of optional values as required values: %s", map); Preconditions.checkArgument(StringType.class.isInstance(m.keyType()), "Invalid map key type (not string): %s", m.keyType()); this.current = m.valueType(); try { Type valueType = valueResult.get(); if (map.valueType() == valueType) { return map; } if (map.isValueOptional()) { return Types.MapType.ofOptional(map.keyId(), map.valueId(), map.keyType(), valueType); } else { return Types.MapType.ofRequired(map.keyId(), map.valueId(), map.keyType(), valueType); } } finally { this.current = m; } }
@Override public Type map(Types.MapType map, Supplier<Type> keyTypeFuture, Supplier<Type> valueTypeFuture) { Preconditions.checkArgument(sourceType.isMapType(), "Not a map: " + sourceType); Types.MapType sourceMap = sourceType.asMapType(); try { this.sourceType = sourceMap.keyType(); Type keyType = keyTypeFuture.get(); this.sourceType = sourceMap.valueType(); Type valueType = valueTypeFuture.get(); if (map.keyType() == keyType && map.valueType() == valueType) { return map; } if (map.isValueOptional()) { return Types.MapType.ofOptional(map.keyId(), map.valueId(), keyType, valueType); } else { return Types.MapType.ofRequired(map.keyId(), map.valueId(), keyType, valueType); } } finally { this.sourceType = sourceMap; } }
@Override public Type map(Types.MapType map, Supplier<Type> keyTypeFuture, Supplier<Type> valueTypeFuture) { Preconditions.checkArgument(sourceType.isMapType(), "Not a map: " + sourceType); Types.MapType sourceMap = sourceType.asMapType(); int sourceKeyId = sourceMap.keyId(); int sourceValueId = sourceMap.valueId(); try { this.sourceType = sourceMap.keyType(); Type keyType = keyTypeFuture.get(); this.sourceType = sourceMap.valueType(); Type valueType = valueTypeFuture.get(); if (map.isValueOptional()) { return Types.MapType.ofOptional(sourceKeyId, sourceValueId, keyType, valueType); } else { return Types.MapType.ofRequired(sourceKeyId, sourceValueId, keyType, valueType); } } finally { this.sourceType = sourceMap; } }
@Override public Schema map(Schema map, Supplier<Schema> value) { Preconditions.checkArgument(current.isNestedType() && current.asNestedType().isMapType(), "Incompatible projected type: %s", current); Types.MapType m = current.asNestedType().asMapType(); Preconditions.checkArgument(m.keyType() == Types.StringType.get(), "Incompatible projected type: key type %s is not string", m.keyType()); this.current = m.valueType(); try { Schema valueSchema = value.get(); // element was changed, create a new map if (valueSchema != map.getValueType()) { return Schema.createMap(valueSchema); } return map; } finally { this.current = m; } }
@Override public Type map(Types.MapType map, Supplier<Type> keyResult, Supplier<Type> valueResult) { Preconditions.checkArgument(current instanceof MapType, "Not a map: %s", current); MapType m = (MapType) current; Preconditions.checkArgument(m.valueContainsNull() || !map.isValueOptional(), "Cannot project a map of optional values as required values: %s", map); this.current = m.valueType(); try { Type valueType = valueResult.get(); if (map.valueType() == valueType) { return map; } if (map.isValueOptional()) { return Types.MapType.ofOptional(map.keyId(), map.valueId(), map.keyType(), valueType); } else { return Types.MapType.ofRequired(map.keyId(), map.valueId(), map.keyType(), valueType); } } finally { this.current = m; } }
private static void assertEqualsSafe(Types.MapType map, Map<?, ?> expected, Map<?, ?> actual) { Type keyType = map.keyType(); Type valueType = map.valueType(); for (Object expectedKey : expected.keySet()) { Object matchingKey = null; for (Object actualKey : actual.keySet()) { try { assertEqualsSafe(keyType, expectedKey, actualKey); matchingKey = actualKey; } catch (AssertionError e) { // failed } } Assert.assertNotNull("Should have a matching key", matchingKey); assertEqualsSafe(valueType, expected.get(expectedKey), actual.get(matchingKey)); } }
@Override public Type map(Types.MapType map, Type ignored, Type valueResult) { if (selected.contains(map.valueId())) { return map; } else if (valueResult != null) { if (map.valueType() == valueResult) { return map; } else if (map.isValueOptional()) { return Types.MapType.ofOptional(map.keyId(), map.valueId(), map.keyType(), valueResult); } else { return Types.MapType.ofRequired(map.keyId(), map.valueId(), map.keyType(), valueResult); } } else if (selected.contains(map.keyId())) { // right now, maps can't be selected without values return map; } return null; }
private static void assertEqualsUnsafe(Types.MapType map, Map<?, ?> expected, MapData actual) { Type keyType = map.keyType(); Type valueType = map.valueType(); List<Map.Entry<?, ?>> expectedElements = Lists.newArrayList(expected.entrySet()); ArrayData actualKeys = actual.keyArray(); ArrayData actualValues = actual.valueArray(); for (int i = 0; i < expectedElements.size(); i += 1) { Map.Entry<?, ?> expectedPair = expectedElements.get(i); Object actualKey = actualKeys.get(i, convert(keyType)); Object actualValue = actualValues.get(i, convert(keyType)); assertEqualsUnsafe(keyType, expectedPair.getKey(), actualKey); assertEqualsUnsafe(valueType, expectedPair.getValue(), actualValue); } }