@Override public PartitionKey deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext) throws JsonParseException { JsonObject jsonObject = jsonElement.getAsJsonObject(); PartitionKey.Builder builder = PartitionKey.builder(); for (Map.Entry<String, JsonElement> entry : jsonObject.entrySet()) { JsonArray jsonArray = entry.getValue().getAsJsonArray(); builder.addField(entry.getKey(), deserializeComparable(jsonArray, jsonDeserializationContext)); } return builder.build(); }
private PartitionKey getPartitionKey(Partitioning partitioning, String relativePath) { List<String> pathParts = Arrays.asList(relativePath.split(Path.SEPARATOR)); if (pathParts.size() != partitioning.getFields().size()) { throw new IllegalArgumentException( String.format("relativePath '%s' does not have same number of components as partitioning '%s", relativePath, partitioning)); } PartitionKey.Builder builder = PartitionKey.builder(); int i = 0; for (Map.Entry<String, Partitioning.FieldType> entry : partitioning.getFields().entrySet()) { String keyName = entry.getKey(); Comparable keyValue = entry.getValue().parse(pathParts.get(i)); builder.addField(keyName, keyValue); i++; } return builder.build(); }
private PartitionKey getPartitionKey(Partitioning partitioning, String relativePath) { List<String> pathParts = Arrays.asList(relativePath.split(Path.SEPARATOR)); if (pathParts.size() != partitioning.getFields().size()) { throw new IllegalArgumentException( String.format("relativePath '%s' does not have same number of components as partitioning '%s", relativePath, partitioning)); } PartitionKey.Builder builder = PartitionKey.builder(); int i = 0; for (Map.Entry<String, Partitioning.FieldType> entry : partitioning.getFields().entrySet()) { String keyName = entry.getKey(); Comparable keyValue = entry.getValue().parse(pathParts.get(i)); builder.addField(keyName, keyValue); i++; } return builder.build(); }
@Override protected void initialize() throws Exception { // add a partition to the pfs PartitionedFileSet pfs = getContext().getDataset("pfs"); PartitionKey key = PartitionKey.builder().addField("x", 1).build(); PartitionOutput partitionOutput = pfs.getPartitionOutput(key); partitionOutput.addPartition(); // configure the same partition as output for the MR Map<String, String> args = new HashMap<>(); PartitionedFileSetArguments.setOutputPartitionKey(args, key); getContext().addOutput(Output.ofDataset("pfs", args)); // configure an input KeyValueTable kv = getContext().getDataset("recorder"); kv.write("hello", "world"); getContext().addInput(Input.ofDataset("recorder")); // configure mapper and no reducers Job job = getContext().getHadoopJob(); job.setMapperClass(IdentityMapper.class); job.setNumReduceTasks(0); }
@Test public void testBuilderGetter() { PartitionKey key = PartitionKey.builder() .addField("a", "value") .addField("b", 1L) .addField("c", -17) .addField("d", true) .addIntField("e", 42) .addLongField("f", 15) .addStringField("g", "ghijk") .build(); Assert.assertEquals("value", key.getField("a")); Assert.assertEquals(1L, key.getField("b")); Assert.assertEquals(-17, key.getField("c")); Assert.assertEquals(true, key.getField("d")); Assert.assertEquals(42, key.getField("e")); Assert.assertEquals(15L, key.getField("f")); Assert.assertEquals("ghijk", key.getField("g")); }
@Test public void testEqualityHashCode() { PartitionKey key1 = PartitionKey.builder() .addField("a", "value") .addField("b", 1L) .addField("c", -17) .build(); PartitionKey key2 = PartitionKey.builder() .addField("b", 1L) .addField("c", -17) .addField("a", "value") .build(); Assert.assertEquals(key1, key2); Assert.assertEquals(key1.hashCode(), key2.hashCode()); } }
@Test(expected = IllegalArgumentException.class) public void testIncompatibleMatch() { PartitionFilter filter = PartitionFilter.builder() .addValueCondition("year", 2012) .addRangeCondition("month", 4, 7) .addValueCondition("market", "asia") .build(); // field of incompatible type filter.match(PartitionKey .builder() .addField("month", "january") .addField("market", "latin") .addField("year", 2012) .build()); }
public void put(String key, String value) throws Exception { table.put(Bytes.toBytes(key), COL, Bytes.toBytes(value)); files.getPartition(PartitionKey.builder().addField(key, value).build()); }
/** * Add field of type LONG. * * @param name the field name * @param value the value of the field * * @throws java.lang.IllegalArgumentException if the field name is null, empty, or already exists, * if the value is null, or if the partitioning is known and does not contain the field name, * or the field is defined with a different type. */ public Builder addLongField(String name, long value) { return addField(name, value); }
/** * Add field of type INT. * * @param name the field name * @param value the value of the field * * @throws java.lang.IllegalArgumentException if the field name is null, empty, or already exists, * if the value is null, or if the partitioning is known and does not contain the field name, * or the field is defined with a different type. */ public Builder addIntField(String name, int value) { return addField(name, value); }
/** * Add field of type STRING. * * @param name the field name * @param value the value of the field * * @throws java.lang.IllegalArgumentException if the field name is null, empty, or already exists, * if the value is null, or if the partitioning is known and does not contain the field name, * or the field is defined with a different type. */ public Builder addStringField(String name, String value) { return addField(name, value); }
@Nullable @Override public PartitionKey.Builder apply(@Nullable PartitionKey.Builder builder) { return builder.addField("z", 1L); } });
@Nullable @Override public PartitionKey.Builder apply(@Nullable PartitionKey.Builder builder) { return builder.addField("z", 1); } });
@Nullable @Override public PartitionKey.Builder apply(@Nullable PartitionKey.Builder builder) { return builder.addField("x", 1L); } });
@Nullable @Override public PartitionKey.Builder apply(@Nullable PartitionKey.Builder builder) { return builder.addField("y", "a"); } });
@Nullable @Override public PartitionKey.Builder apply(@Nullable PartitionKey.Builder builder) { return builder.addField("y", 1); } });
@Nullable @Override public PartitionKey.Builder apply(@Nullable PartitionKey.Builder builder) { return builder.addField("x", "a"); } });