/** * Set the partition key of the output partition when using PartitionedFileSet as an OutputFormatProvider. * This key is used as the partition key for the new file, and also to generate an output file path - if that path * is not explicitly given as an argument itself. * * @param key the partition key * @param arguments the runtime arguments for a partitioned dataset */ public static void setOutputPartitionKey(Map<String, String> arguments, PartitionKey key) { for (Map.Entry<String, ? extends Comparable> entry : key.getFields().entrySet()) { arguments.put(OUTPUT_PARTITION_KEY_PREFIX + entry.getKey(), entry.getValue().toString()); } }
@Override public JsonElement serialize(PartitionKey partitionKey, Type type, JsonSerializationContext jsonSerializationContext) { JsonObject jsonObj = new JsonObject(); for (Map.Entry<String, Comparable> entry : partitionKey.getFields().entrySet()) { jsonObj.add(entry.getKey(), serializeComparable(entry.getValue(), jsonSerializationContext)); } return jsonObj; } }
private String generateHivePartitionKey(PartitionKey key) { StringBuilder builder = new StringBuilder("("); String sep = ""; for (Map.Entry<String, ? extends Comparable> entry : key.getFields().entrySet()) { String fieldName = entry.getKey(); Comparable fieldValue = entry.getValue(); String quote = fieldValue instanceof String ? "'" : ""; builder.append(sep); if (shouldEscapeColumns) { // a literal backtick(`) is just a double backtick(``) builder.append('`').append(fieldName.replace("`", "``")).append('`'); } else { builder.append(fieldName); } builder.append("=").append(quote).append(fieldValue.toString()).append(quote); sep = ", "; } builder.append(")"); return builder.toString(); } }
private String generateHivePartitionKey(PartitionKey key) { StringBuilder builder = new StringBuilder("("); String sep = ""; for (Map.Entry<String, ? extends Comparable> entry : key.getFields().entrySet()) { String fieldName = entry.getKey(); Comparable fieldValue = entry.getValue(); String quote = fieldValue instanceof String ? "'" : ""; builder.append(sep); if (shouldEscapeColumns) { // a literal backtick(`) is just a double backtick(``) builder.append('`').append(fieldName.replace("`", "``")).append('`'); } else { builder.append(fieldName); } builder.append("=").append(quote).append(fieldValue.toString()).append(quote); sep = ", "; } builder.append(")"); return builder.toString(); } }
/** * Validates the partition key against the partitioning. */ private static void validatePartitionKey(PartitionKey key, Partitioning partitioning) { if (!partitioning.getFields().keySet().equals(key.getFields().keySet())) { throw new IllegalArgumentException(String.format( "Partition key is invalid: It contains fields %s, but the partitioning requires %s", key.getFields().keySet(), partitioning.getFields().keySet())); } for (Map.Entry<String, FieldType> entry : partitioning.getFields().entrySet()) { String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); Comparable fieldValue = key.getField(fieldName); if (fieldValue == null) { throw new IllegalArgumentException( String.format("Incomplete partition key: value for field '%s' is missing", fieldName)); } try { fieldType.validate(fieldValue); } catch (IllegalArgumentException e) { throw new IllegalArgumentException(String.format( "Invalid partition key: Value for field '%s' is incompatible with the partitioning: %s", fieldName, e.getMessage())); } } }
/** * Validates the partition key against the partitioning. */ private static void validatePartitionKey(PartitionKey key, Partitioning partitioning) { if (!partitioning.getFields().keySet().equals(key.getFields().keySet())) { throw new IllegalArgumentException(String.format( "Partition key is invalid: It contains fields %s, but the partitioning requires %s", key.getFields().keySet(), partitioning.getFields().keySet())); } for (Map.Entry<String, FieldType> entry : partitioning.getFields().entrySet()) { String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); Comparable fieldValue = key.getField(fieldName); if (fieldValue == null) { throw new IllegalArgumentException( String.format("Incomplete partition key: value for field '%s' is missing", fieldName)); } try { fieldType.validate(fieldValue); } catch (IllegalArgumentException e) { throw new IllegalArgumentException(String.format( "Invalid partition key: Value for field '%s' is incompatible with the partitioning: %s", fieldName, e.getMessage())); } } }