private static PartitionStrategy buildPartitionStrategyForKeyMappings( Map<Integer, FieldMapping> keyMappings) { PartitionStrategy.Builder builder = new PartitionStrategy.Builder(); for (Integer index : new TreeSet<Integer>(keyMappings.keySet())) { builder.identity(keyMappings.get(index).getFieldName()); } return builder.build(); }
public void testToKey() { PartitionStrategy strategy = new PartitionStrategy.Builder() .identity("id") .year("timestamp") .month("timestamp")
@Test @SuppressWarnings("unchecked") public void toDirNameIdentityWithSlashes() { PartitionStrategy strategy = new PartitionStrategy.Builder() .identity("name") .identity("address") .build(); StorageKey key = new StorageKey(strategy); key.replaceValues((List) Lists.newArrayList("John Doe", "NY/USA")); Assert.assertEquals( new Path("name_copy=John+Doe/address_copy=NY%2FUSA"), convert.fromKey(key)); }
@Test public void testAddEmbeddedPartitionStrategy() { PartitionStrategy strategy = new PartitionStrategy.Builder() .hash("username", 16) .identity("username", "u") .build(); Schema original = new Schema.Parser().parse("{" + " \"type\": \"record\"," + " \"name\": \"User\"," + " \"fields\": [" + " {\"name\": \"id\", \"type\": \"long\"}," + " {\"name\": \"username\", \"type\": \"string\"}," + " {\"name\": \"real_name\", \"type\": \"string\"}" + " ]" + "}"); Schema embedded = PartitionStrategyParser.embedPartitionStrategy(original, strategy); Assert.assertTrue(PartitionStrategyParser.hasEmbeddedStrategy(embedded)); Assert.assertEquals(strategy, PartitionStrategyParser.parseFromSchema(embedded)); }
@Test public void testUpdateNonProvided() { final PartitionStrategy provided = new PartitionStrategy.Builder() .identity("s", "part") .build(); TestHelpers.assertThrows("Should not allow replacing if not provided", ValidationException.class, new Runnable() { @Override public void run() { Compatibility.checkStrategyUpdate( provided, new PartitionStrategy.Builder() .dateFormat("l", "part", "yyyy-MM-dd") .build(), PROVIDED_TEST_SCHEMA); } }); }
@Test public void testIdentity() { // right now, the field type is taken from the Schema checkParser(new PartitionStrategy.Builder() .identity("username", "id") .build(), "[ {\"type\": \"identity\", " + "\"source\": \"username\", " + "\"name\": \"id\"} ]" ); checkParser(new PartitionStrategy.Builder() .identity("username", "username_copy") .build(), "[ {\"type\": \"identity\", \"source\": \"username\"} ]" ); }
@Test public void test() throws Exception { final PartitionStrategy p = new PartitionStrategy.Builder() .identity("month", "month_ordinal", 12) .hash("userId", 7) .build(); List<FieldPartitioner> fieldPartitioners = p.getFieldPartitioners(); Assert.assertEquals(2, fieldPartitioners.size()); FieldPartitioner fp0 = fieldPartitioners.get(0); assertEquals("month_ordinal", fp0.getName()); assertEquals(12, fp0.getCardinality()); FieldPartitioner fp1 = fieldPartitioners.get(1); assertEquals("userId_hash", fp1.getName()); assertEquals(7, fp1.getCardinality()); assertEquals(12 * 7, p.getCardinality()); // useful for writers }
@Test @SuppressWarnings("unchecked") public void toDirNameIdentityWithNonString() { PartitionStrategy strategy = new PartitionStrategy.Builder() .identity("id") .build(); StorageKey expected = new StorageKey(strategy); expected.replace(0, 0L); Assert.assertEquals("Should convert to schema type", expected, convert.toKey(new Path("id=0"), new StorageKey(strategy))); } }
@Override public void run() { Compatibility.checkStrategyUpdate( provided, new PartitionStrategy.Builder() .identity("i", "part") .build(), PROVIDED_TEST_SCHEMA); } });
@Override public void run() { Compatibility.checkDescriptor( new DatasetDescriptor.Builder() .schema(schema) .partitionStrategy(new PartitionStrategy.Builder() .day("timestamp") .identity("number", "timestamp") .build()) .build()); } });
@Override public void run() { Compatibility.checkDescriptor( new DatasetDescriptor.Builder() .schema(schema) .partitionStrategy(new PartitionStrategy.Builder() .identity("number", "day.of.month") .build()) .build()); } });
@Override public void run() { Compatibility.checkDescriptor( new DatasetDescriptor.Builder() .schema(schema) .partitionStrategy(new PartitionStrategy.Builder() .identity("day_of_month", "day-of-month") .build()) .build()); } });
@Test @SuppressWarnings("deprecation") public void testIntIdentity() { PartitionStrategy intIdStrategy = new PartitionStrategy.Builder() .identity("intField") .build(); PartitionStrategy copy = new PartitionExpression( PartitionExpression.toExpression(intIdStrategy), true).evaluate(); Assert.assertEquals(intIdStrategy, copy); }
@Override public void run() { Compatibility.checkStrategyUpdate( provided, new PartitionStrategy.Builder() .identity("s", "part") .build(), PROVIDED_TEST_SCHEMA); } });
@Override public void run() { Compatibility.checkStrategyUpdate( provided, new PartitionStrategy.Builder() .identity("i", "part") .build(), PROVIDED_TEST_SCHEMA); } });
@Override public void run() { Compatibility.checkDescriptor( new DatasetDescriptor.Builder() .schema(schema) .partitionStrategy(new PartitionStrategy.Builder() .day("timestamp") .identity("number", "day") .build()) .build()); } });
@Override public void run() { Compatibility.checkStrategyUpdate( provided, new PartitionStrategy.Builder() .identity("s", "other") .build(), PROVIDED_TEST_SCHEMA); } });
@Override public void run() { Compatibility.checkStrategyUpdate( provided, new PartitionStrategy.Builder() .identity("i", "part") .build(), PROVIDED_TEST_SCHEMA); } });