/** * Validates and returns the underlying {@link LogicalType} of the given {@link Schema.Field}. * * @param field field with logical type * @param allowedTypes acceptable logical types * @return the underlying {@link LogicalType} for the field * @throws UnexpectedFormatException if the field provided does not have a {@link LogicalType} that is * one of the acceptable types */ @Nullable private static LogicalType validateAndGetLogicalType(Schema.Field field, Set<LogicalType> allowedTypes) { if (field == null) { return null; } String fieldName = field.getName(); LogicalType logicalType = getLogicalType(field.getSchema(), allowedTypes); if (logicalType == null) { throw new UnexpectedFormatException(String.format("Field %s does not have a logical type.", fieldName)); } if (!allowedTypes.contains(logicalType)) { throw new UnexpectedFormatException(String.format("Field %s must be of logical type %s, instead it is of type %s", fieldName, allowedTypes, logicalType)); } return logicalType; }
/** * Sets the date value for {@link LogicalType#DATE} field * * @param fieldName name of the field to set * @param localDate value for the field * @return this builder * @throws UnexpectedFormatException if the field is not in the schema, or the field is not nullable but a null * value is given or if the provided date is an invalid date */ public Builder setDate(String fieldName, @Nullable LocalDate localDate) { validateAndGetLogicalType(validateAndGetField(fieldName, localDate), EnumSet.of(LogicalType.DATE)); if (localDate == null) { fields.put(fieldName, null); return this; } try { fields.put(fieldName, Math.toIntExact(localDate.toEpochDay())); } catch (ArithmeticException e) { // Highest integer is 2,147,483,647 which is Jan 1 2038. throw new UnexpectedFormatException(String.format("Field %s was set to a date that is too large." + "Valid date should be below Jan 1 2038", fieldName)); } return this; }
fields.put(fieldName, millis); } catch (ArithmeticException e) { throw new UnexpectedFormatException(String.format("Field %s was set to a time that is too large.", fieldName));
private int addNextLogEntry(String data, int start, List<String> parts) { int end = -1; if (data.charAt(start) == '"') { // Find the closing '"' and extract values within start = start + 1; end = findNext(data, start, '"'); } else if (data.charAt(start) == '[') { // find the closing ']' and extract values start = start + 1; end = findNext(data, start, ']'); } else { // find the next ' ' and extract values end = findNext(data, start + 1, ' '); } if (end == -1) { throw new UnexpectedFormatException(String.format("Could not parse data in CLF format. Entry %s", data)); } parts.add(data.substring(start, end)); return end + 1; }
/** * Sets the date value for {@link LogicalType#DATE} field * * @param fieldName name of the field to set * @param localDate value for the field * @return this builder * @throws UnexpectedFormatException if the field is not in the schema, or the field is not nullable but a null * value is given or if the provided date is an invalid date */ public Builder setDate(String fieldName, @Nullable LocalDate localDate) { validateAndGetLogicalType(validateAndGetField(fieldName, localDate), EnumSet.of(LogicalType.DATE)); if (localDate == null) { fields.put(fieldName, null); return this; } try { fields.put(fieldName, Math.toIntExact(localDate.toEpochDay())); } catch (ArithmeticException e) { // Highest integer is 2,147,483,647 which is Jan 1 2038. throw new UnexpectedFormatException(String.format("Field %s was set to a date that is too large." + "Valid date should be below Jan 1 2038", fieldName)); } return this; }
/** * Validates and returns the underlying {@link LogicalType} of the given {@link Schema.Field}. * * @param field field with logical type * @param allowedTypes acceptable logical types * @return the underlying {@link LogicalType} for the field * @throws UnexpectedFormatException if the field provided does not have a {@link LogicalType} that is * one of the acceptable types */ @Nullable private static LogicalType validateAndGetLogicalType(Schema.Field field, Set<LogicalType> allowedTypes) { if (field == null) { return null; } String fieldName = field.getName(); LogicalType logicalType = getLogicalType(field.getSchema(), allowedTypes); if (logicalType == null) { throw new UnexpectedFormatException(String.format("Field %s does not have a logical type.", fieldName)); } if (!allowedTypes.contains(logicalType)) { throw new UnexpectedFormatException(String.format("Field %s must be of logical type %s, instead it is of type %s", fieldName, allowedTypes, logicalType)); } return logicalType; }
fields.put(fieldName, millis); } catch (ArithmeticException e) { throw new UnexpectedFormatException(String.format("Field %s was set to a time that is too large.", fieldName));
private int addNextLogEntry(String data, int start, List<String> parts) { int end = -1; if (data.charAt(start) == '"') { // Find the closing '"' and extract values within start = start + 1; end = findNext(data, start, '"'); } else if (data.charAt(start) == '[') { // find the closing ']' and extract values start = start + 1; end = findNext(data, start, ']'); } else { // find the next ' ' and extract values end = findNext(data, start + 1, ' '); } if (end == -1) { throw new UnexpectedFormatException(String.format("Could not parse data in CLF format. Entry %s", data)); } parts.add(data.substring(start, end)); return end + 1; }
/** * Get a builder for creating a record with the given schema. * * @param schema schema for the record to build. * @return builder for creating a record with the given schema. * @throws UnexpectedFormatException if the given schema is not a record with at least one field. */ public static Builder builder(Schema schema) throws UnexpectedFormatException { if (schema == null || schema.getType() != Schema.Type.RECORD || schema.getFields().size() < 1) { throw new UnexpectedFormatException("Schema must be a record with at least one field."); } return new Builder(schema); }
/** * Get a builder for creating a record with the given schema. * * @param schema schema for the record to build. * @return builder for creating a record with the given schema. * @throws UnexpectedFormatException if the given schema is not a record with at least one field. */ public static Builder builder(Schema schema) throws UnexpectedFormatException { if (schema == null || schema.getType() != Schema.Type.RECORD || schema.getFields().size() < 1) { throw new UnexpectedFormatException("Schema must be a record with at least one field."); } return new Builder(schema); }
return this; } catch (ArithmeticException e) { throw new UnexpectedFormatException(String.format("Field %s was set to a timestamp that is too large.", fieldName));
return this; } catch (ArithmeticException e) { throw new UnexpectedFormatException(String.format("Field %s was set to a timestamp that is too large.", fieldName));
private static Object convertUnion(Object value, List<Schema> schemas) { boolean isNullable = false; for (Schema possibleSchema : schemas) { if (possibleSchema.getType() == Schema.Type.NULL) { isNullable = true; if (value == null) { return value; } } else { try { return convertField(value, possibleSchema); } catch (Exception e) { // if we couldn't convert, move to the next possibility } } } if (isNullable) { return null; } throw new UnexpectedFormatException("Unable to determine the union type."); }
private Schema.Field validateAndGetField(String fieldName, Object val) { Schema.Field field = schema.getField(fieldName); if (field == null) { throw new UnexpectedFormatException("field " + fieldName + " is not in the schema."); } Schema fieldSchema = field.getSchema(); if (val == null) { if (fieldSchema.getType() == Schema.Type.NULL) { return field; } if (fieldSchema.getType() != Schema.Type.UNION) { throw new UnexpectedFormatException("field " + fieldName + " cannot be set to a null value."); } for (Schema unionSchema : fieldSchema.getUnionSchemas()) { if (unionSchema.getType() == Schema.Type.NULL) { return field; } } throw new UnexpectedFormatException("field " + fieldName + " cannot be set to a null value."); } return field; } }
@Override public StructuredRecord read(ByteBuffer event) { try { binaryDecoder = decoderFactory.binaryDecoder(byteBufferInput.reset(event), binaryDecoder); return datumReader.read(null, binaryDecoder); } catch (IOException e) { throw new UnexpectedFormatException("Unable to decode the stream body as avro.", e); } }
private Schema.Field validateAndGetField(String fieldName, Object val) { Schema.Field field = schema.getField(fieldName); if (field == null) { throw new UnexpectedFormatException("field " + fieldName + " is not in the schema."); } Schema fieldSchema = field.getSchema(); if (val == null) { if (fieldSchema.getType() == Schema.Type.NULL) { return field; } if (fieldSchema.getType() != Schema.Type.UNION) { throw new UnexpectedFormatException("field " + fieldName + " cannot be set to a null value."); } for (Schema unionSchema : fieldSchema.getUnionSchemas()) { if (unionSchema.getType() == Schema.Type.NULL) { return field; } } throw new UnexpectedFormatException("field " + fieldName + " cannot be set to a null value."); } return field; } }
@Override public StructuredRecord make(Schema schema, Iterator<String> bodyFields) { StructuredRecord.Builder builder = StructuredRecord.builder(schema); Iterator<Schema.Field> fieldsIterator = schema.getFields().iterator(); while (fieldsIterator.hasNext()) { Schema.Field field = fieldsIterator.next(); Schema fieldSchema = field.getSchema(); String fieldName = field.getName(); if (isStringArray(fieldSchema)) { if (!fieldsIterator.hasNext()) { // only do varargs-style string array parsing on bodyField if it's the last field List<String> fields = Lists.newArrayList(bodyFields); builder.set(fieldName, fields.toArray(new String[fields.size()])); } else { throw new UnexpectedFormatException( String.format("string array type field '%s' must be the last schema field", fieldName)); } } else { // simple type (not string array) String bodyField = bodyFields.hasNext() ? bodyFields.next() : null; String val = parseBodyValue(bodyField, fieldSchema); builder.convertAndSet(fieldName, val); } } return builder.build(); } }
simpleType = schema.getType(); if (strVal == null && simpleType != Schema.Type.NULL) { throw new UnexpectedFormatException("Cannot set non-nullable field to a null value."); throw new UnexpectedFormatException("Cannot convert a string to schema " + schema); default: throw new UnexpectedFormatException("Cannot convert a string to schema " + schema);
@Override public StructuredRecord make(Schema schema, Iterator<String> bodyFields) { StructuredRecord.Builder builder = StructuredRecord.builder(schema); Iterator<Schema.Field> fieldsIterator = schema.getFields().iterator(); while (fieldsIterator.hasNext()) { Schema.Field field = fieldsIterator.next(); Schema fieldSchema = field.getSchema(); String fieldName = field.getName(); if (isStringArray(fieldSchema)) { if (!fieldsIterator.hasNext()) { // only do varargs-style string array parsing on bodyField if it's the last field List<String> fields = Lists.newArrayList(bodyFields); builder.set(fieldName, fields.toArray(new String[fields.size()])); } else { throw new UnexpectedFormatException( String.format("string array type field '%s' must be the last schema field", fieldName)); } } else { // simple type (not string array) String bodyField = bodyFields.hasNext() ? bodyFields.next() : null; String val = parseBodyValue(bodyField, fieldSchema); builder.convertAndSet(fieldName, val); } } return builder.build(); } }
simpleType = schema.getType(); if (strVal == null && simpleType != Schema.Type.NULL) { throw new UnexpectedFormatException("Cannot set non-nullable field to a null value."); throw new UnexpectedFormatException("Cannot convert a string to schema " + schema); default: throw new UnexpectedFormatException("Cannot convert a string to schema " + schema);