public static <T> T valueForDirname(FieldPartitioner<?, T> field, Schema schema, String name) { return Conversions.convert(dirnameToValueString(name), SchemaUtil.getPartitionType(field, schema)); }
@SuppressWarnings("unchecked") private <S, T> T valueFor(FieldPartitioner<S, T> fp) { if (values.containsKey(fp.getName())) { return Conversions.convert(values.get(fp.getName()), SchemaUtil.getPartitionType(fp, schema)); } else if (values.containsKey(fp.getSourceName())) { return fp.apply(Conversions.convert(values.get(fp.getSourceName()), SchemaUtil.getSourceType(fp, schema))); } else { throw new IllegalStateException( "Cannot create Key, missing data for field:" + fp.getName()); } } }
@SuppressWarnings("deprecation") static List<FieldSchema> partitionColumns(PartitionStrategy strategy, Schema schema) { List<FieldSchema> columns = Lists.newArrayList(); for (FieldPartitioner<?, ?> fp : Accessor.getDefault().getFieldPartitioners(strategy)) { columns.add(new FieldSchema(fp.getName(), getHiveType(SchemaUtil.getPartitionType(fp, schema)), "Partition column derived from '" + fp.getSourceName() + "' column, " + "generated by Kite.")); } return columns; }
@SuppressWarnings("deprecation") static List<FieldSchema> partitionColumns(PartitionStrategy strategy, Schema schema) { List<FieldSchema> columns = Lists.newArrayList(); for (FieldPartitioner<?, ?> fp : Accessor.getDefault().getFieldPartitioners(strategy)) { columns.add(new FieldSchema(fp.getName(), getHiveType(SchemaUtil.getPartitionType(fp, schema)), "Partition column derived from '" + fp.getSourceName() + "' column, " + "generated by Kite.")); } return columns; }
public static void checkStrategyUpdate(PartitionStrategy existing, PartitionStrategy other, Schema schema) { List<FieldPartitioner> existingFields = Accessor.getDefault() .getFieldPartitioners(existing); List<FieldPartitioner> otherFields = Accessor.getDefault() .getFieldPartitioners(other); ValidationException.check(existingFields.size() == otherFields.size(), "Not compatible: cannot replace %s partitioners with %s partitioners", existingFields.size(), otherFields.size()); for (int i = 0; i < existingFields.size(); i += 1) { FieldPartitioner fp = existingFields.get(i); FieldPartitioner replacement = otherFields.get(i); if (fp.equals(replacement)) { continue; } ValidationException.check(fp instanceof ProvidedFieldPartitioner, "Cannot replace partition %s: not a provided partitioner", fp.getName()); ValidationException.check(fp.getName().equals(replacement.getName()), "Cannot change the name of partition %s (to %s)", fp.getName(), replacement.getName()); Class<?> outputType = SchemaUtil.getPartitionType(replacement, schema); ValidationException.check( isCompatibleWithProvidedType(fp.getType(), outputType), "Cannot change the data type of partition %s", fp.getName()); } }
/** * Builds a Schema for the FieldPartitioner using the given Schema to * determine types not fixed by the FieldPartitioner. * * @param fp a FieldPartitioner * @param schema an entity Schema that will be partitioned * @return a Schema for the field partitioner */ public static Schema partitionFieldSchema(FieldPartitioner<?, ?> fp, Schema schema) { if (fp instanceof IdentityFieldPartitioner) { // copy the schema directly from the entity to preserve annotations return fieldSchema(schema, fp.getSourceName()); } else { Class<?> fieldType = getPartitionType(fp, schema); if (fieldType == Integer.class) { return Schema.create(Schema.Type.INT); } else if (fieldType == Long.class) { return Schema.create(Schema.Type.LONG); } else if (fieldType == String.class) { return Schema.create(Schema.Type.STRING); } else { throw new ValidationException( "Cannot encode partition " + fp.getName() + " with type " + fp.getSourceType() ); } } }