@Test public void testDescriptorValidationPasses() { DatasetDescriptor descriptor = new DatasetDescriptor.Builder() .schema(schema) .partitionStrategy(strategy) .build(); Assert.assertEquals("Descriptor should have correct schema", schema, descriptor.getSchema()); Assert.assertEquals("Descriptor should have correct strategy", strategy, descriptor.getPartitionStrategy()); }
@Override public void run() { new DatasetDescriptor.Builder() .schema(SchemaBuilder.record("Record").fields() .requiredInt("timestamp") .endRecord()) .partitionStrategy(new PartitionStrategy.Builder() .year("timestamp") .build()) .build(); } });
@Override public void run() { new DatasetDescriptor.Builder() .schema(SchemaBuilder.record("Record").fields() .requiredString("field") .endRecord()) .partitionStrategy(new PartitionStrategy.Builder() .hash("array", 48) .build()) .build(); } });
@Override public void run() { new DatasetDescriptor.Builder() .schema(Schema.createArray(Schema.create(Schema.Type.FLOAT))) .partitionStrategy(new PartitionStrategy.Builder() .hash("array", 48) .build()) .build(); } });
@BeforeClass public static void createTestDataset() { Datasets.delete("dataset:file:/tmp/test_name"); test = Datasets.create("dataset:file:/tmp/test_name", new DatasetDescriptor.Builder() .schema(SCHEMA) .partitionStrategy(STRATEGY) .build()); }
@Override public void run() { new DatasetDescriptor.Builder() .schema(schema) .partitionStrategy(missingStartField) .build(); } });
@Override public void run() { new DatasetDescriptor.Builder() .partitionStrategy(new PartitionStrategy.Builder() .hash("array", 48) .build()) .build(); } });
@Override public void run() { Compatibility.checkDescriptor( new DatasetDescriptor.Builder() .schema(schema) .partitionStrategy(new PartitionStrategy.Builder() .day("timestamp") .identity("number", "timestamp") .build()) .build()); } });
@Override public void run() { Compatibility.checkDescriptor( new DatasetDescriptor.Builder() .schema(schema) .partitionStrategy(new PartitionStrategy.Builder() .identity("number", "day.of.month") .build()) .build()); } });
@Override public void run() { Compatibility.checkDescriptor( new DatasetDescriptor.Builder() .schema(schema) .partitionStrategy(new PartitionStrategy.Builder() .identity("day_of_month", "day-of-month") .build()) .build()); } });
@Override public AvroKeySchema parseKeySchema(String rawSchema, PartitionStrategy partitionStrategy) { // use DatasetDescriptor.Builder because it checks consistency DatasetDescriptor descriptor = new DatasetDescriptor.Builder() .schemaLiteral(rawSchema) .partitionStrategy(partitionStrategy) .build(); return new AvroKeySchema( descriptor.getSchema(), descriptor.getPartitionStrategy()); }
@Override public void run() { new DatasetDescriptor.Builder() .schema(SchemaBuilder.record("Record").fields() .requiredInt("number") .endRecord()) .partitionStrategy(new PartitionStrategy.Builder() .range("number", "m", "z") .build()) .build(); } });
@Override public void run() { new DatasetDescriptor.Builder() .schema(SchemaBuilder.record("Record").fields() .requiredString("username") .endRecord()) .partitionStrategy(new PartitionStrategy.Builder() .range("number", 5, 10, 15) .build()) .build(); } });
@Override public void run() { new DatasetDescriptor.Builder() .schema((Schema) null) .partitionStrategy(new PartitionStrategy.Builder() .hash("array", 48) .build()) .build(); } });
@Override public void run() { Compatibility.checkDescriptor( new DatasetDescriptor.Builder() .schema(schema) .partitionStrategy(new PartitionStrategy.Builder() .day("timestamp") .identity("number", "day") .build()) .build()); } });
@Override public void run() { new DatasetDescriptor.Builder() .schema(schema) .partitionStrategy(badExpectedType) .build(); } });
@Override public void run() { new DatasetDescriptor.Builder() .schema(SchemaBuilder.record("Record").fields() .requiredInt("timestamp") .endRecord()) .partitionStrategy(new PartitionStrategy.Builder() .year("timestamp").month("timestamp").day("timestamp") .build()) .build(); } });
@Override public AvroKeySchema parseKeySchema(String rawSchema, PartitionStrategy partitionStrategy) { // use DatasetDescriptor.Builder because it checks consistency DatasetDescriptor descriptor = new DatasetDescriptor.Builder() .schemaLiteral(rawSchema) .partitionStrategy(partitionStrategy) .build(); return new AvroKeySchema( descriptor.getSchema(), descriptor.getPartitionStrategy()); }
@Before public void createFileSystemDataset() { String uri = "dataset:file:/tmp/datasets/ns/test"; DatasetDescriptor descriptor = new DatasetDescriptor.Builder() .schema(schema) .partitionStrategy(ymd) .build(); Datasets.delete(uri); this.dataset = Datasets.create(uri, descriptor); }
@Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .partitionStrategy(new PartitionStrategy.Builder() .year("created_at") .build()) .build(); } }