PartitionStrategy.Builder builder = new PartitionStrategy.Builder(); for (Iterator<JsonNode> it = node.elements(); it.hasNext();) { JsonNode fieldPartitioner = it.next(); builder.identity(source, name); } else if (type.equals("hash")) { ValidationException.check(fieldPartitioner.has(BUCKETS), name == null ? source : name, fieldPartitioner.get(BUCKETS).asText()); builder.hash(source, name, buckets); } else if (type.equals("range")) { ValidationException.check(fieldPartitioner.has(SIZE), name == null ? source : name, fieldPartitioner.get(SIZE).asText()); builder.fixedSizeRange(source, name, size); } else if (type.equals("year")) { builder.year(source, name); } else if (type.equals("month")) { builder.month(source, name); } else if (type.equals("day")) { builder.day(source, name); } else if (type.equals("hour")) { builder.hour(source, name); } else if (type.equals("minute")) { builder.minute(source, name); } else if (type.equals("dateFormat")) { ValidationException.check(name != null,
@Test @SuppressWarnings("deprecation") public void testPartitionedWriterDouble() throws IOException { PartitionStrategy partitionStrategy = new PartitionStrategy.Builder() .hash("username", 2).hash("email", 3).build();
@Test public void testDescriptorValidationFails() { final PartitionStrategy missingStartField = new PartitionStrategy.Builder() .identity("p.latitude") .identity("p.longitude") .build(); @Override public void run() { new DatasetDescriptor.Builder() .schema(schema) .partitionStrategy(missingStartField) new PartitionStrategy.Builder() .identity("position.latitude") .identity("position.long") .build(); @Override public void run() { new DatasetDescriptor.Builder() .schema(schema) .partitionStrategy(missingLongField) new PartitionStrategy.Builder() .identity("position.latitude") .year("position.longitude") .build();
@Test @SuppressWarnings("deprecation") public void testPartitionedWriterSingle() throws IOException { PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash( "username", 2).build();
@Test public void testProvided() { checkParser(new PartitionStrategy.Builder().provided("version").build(), "[ {\"type\": \"provided\", \"name\": \"version\"} ]"); checkParser(new PartitionStrategy.Builder().provided("version").build(), "[ {\"type\": \"provided\", \"name\": \"version\", \"values\": \"string\"} ]"); checkParser(new PartitionStrategy.Builder().provided("version", "string").build(), "[ {\"type\": \"provided\", \"name\": \"version\", \"values\": \"string\"} ]"); checkParser(new PartitionStrategy.Builder().provided("version", "int").build(), "[ {\"type\": \"provided\", \"name\": \"version\", \"values\": \"int\"} ]"); checkParser(new PartitionStrategy.Builder().provided("version", "long").build(), "[ {\"type\": \"provided\", \"name\": \"version\", \"values\": \"long\"} ]");
.requiredString("email") .endRecord(); PartitionStrategy strategy = new PartitionStrategy.Builder() .provided("version", "int") .build(); DatasetDescriptor descriptor = new DatasetDescriptor.Builder() .schema(user)
@Test @SuppressWarnings("deprecation") public void testPartitionedWriterSingleNullableField() throws IOException { PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash( "username", 2).build();
@Test public void testAlignedWithPartitionBoundaries() { PartitionStrategy hashStrategy = new PartitionStrategy.Builder() .hash("id", "bucket", 32) .build(); PartitionStrategy withColor = new PartitionStrategy.Builder() .range("color", "blue", "green", "red") .year("timestamp") .month("timestamp") .day("timestamp") .identity("id") .build();
@Override public int run() throws IOException { PartitionStrategy.Builder strategyBuilder = new PartitionStrategy.Builder(); for (String partition : partitions) { Matcher m = PARTITION_FIELD.matcher(partition); ValidationException.check(width != null, "Missing number of hash partitions: %s:hash[?]", fieldName); strategyBuilder.hash(fieldName, Integer.parseInt(width)); } else if ("copy".equals(partitionerType)) { strategyBuilder.identity(fieldName); } else if ("year".equals(partitionerType)) { strategyBuilder.year(fieldName); } else if ("month".equals(partitionerType)) { strategyBuilder.month(fieldName); } else if ("day".equals(partitionerType)) { strategyBuilder.day(fieldName); } else if ("hour".equals(partitionerType)) { strategyBuilder.hour(fieldName); } else if ("minute".equals(partitionerType)) { strategyBuilder.minute(fieldName); } else if ("provided".equals(partitionerType)) { strategyBuilder.provided(fieldName); } else { throw new ValidationException( .partitionStrategy(strategyBuilder.build()) .schema(open(avroSchemaFile)) .build();
@Test public void testMerge() throws IOException { PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash( "username", 2).build();
@Test public void testProvidedPartitionSizeChange() { final PartitionStrategy provided = new PartitionStrategy.Builder() .provided("part", "string") .provided("part2", "string") .build(); Compatibility.checkStrategyUpdate( provided, new PartitionStrategy.Builder() .identity("s", "part") .build(), new PartitionStrategy.Builder() .identity("s", "part") .identity("s", "part2") .build(), PROVIDED_TEST_SCHEMA);
@Test @SuppressWarnings("deprecation") public void testDropPartition() throws IOException { PartitionStrategy partitionStrategy = new PartitionStrategy.Builder() .hash("username", 2).build();
@Override public int run(String[] args) throws Exception { PartitionStrategy partitionStrategy = new PartitionStrategy.Builder() .identity("favoriteColor", "favorite_color") .build();
.partitionStrategy(new PartitionStrategy.Builder() .identity("data", "d_copy") .build()) .schema(TestRecord.class) .build();
@Test public void testProvidedPartitionStringUpdate() { final PartitionStrategy provided = new PartitionStrategy.Builder() .provided("part", "string") .build(); new PartitionStrategy.Builder() .identity("s", "part") .build(), PROVIDED_TEST_SCHEMA);
.add("day", 1) .build(); PartitionStrategy strategy = new PartitionStrategy.Builder() .year("timestamp") .month("timestamp") .day("timestamp") .build(); comparator = new MarkerComparator(strategy);
descriptor.isPartitioned()); PartitionStrategy expected = new PartitionStrategy.Builder() .hash("username", 16) .identity("username", "u") .build(); Assert.assertEquals(expected, descriptor.getPartitionStrategy()); expected = new PartitionStrategy.Builder() .identity("real_name", "n") .build(); DatasetDescriptor override = new DatasetDescriptor.Builder() .schema(schema)
.add("day", 1) .build(); PartitionStrategy strategy = new PartitionStrategy.Builder() .year("timestamp") .month("timestamp") .day("timestamp") .build(); comparator = new MarkerComparator(strategy);
@Test public void testHash() { checkParser(new PartitionStrategy.Builder().hash("id", 64).build(), "[ {\"type\": \"hash\", \"source\": \"id\", \"buckets\": 64} ]"); checkParser(new PartitionStrategy.Builder().hash("id", "h", 64).build(), "[ {\"type\": \"hash\", " + "\"source\": \"id\", " +