/** * Convert a URI for a partition directory in a filesystem dataset to a {@link View} * object representing that partition. * @param dataset the (partitioned) filesystem dataset * @param uri the path to the partition directory * @return a view of the partition */ public static <E> View<E> viewForUri(Dataset<E> dataset, String uri) { return viewForUri(dataset, URI.create(uri)); }
eventsToProcess = Datasets.load(args[0], StandardEvent.class); } else { eventsToProcess = FileSystemDatasets.viewForPath(eventsDataset, new Path(args[0]));
@Override public void run() { FileSystemDatasets.viewForUri(ds, "y=2014/m=03/d=14"); } });
@Override public void run() { FileSystemDatasets.viewForUri( dataset, "file:/tmp/datasets/ns/test2/y=2014/m=03/d=14"); } });
@Override public void run() { FileSystemDatasets.viewForUri( dataset, "hdfs:/tmp/datasets/ns/test/y=2014/m=03/d=14"); } });
@Test public void testNoConstraints() { View<GenericRecord> view = FileSystemDatasets.viewForUri( dataset, "file:/tmp/datasets/ns/test/"); Assert.assertEquals("Should create correct view", view, dataset); }
@Test public void testViewForUri() { Path path = new Path("/tmp/datasets/ns/test/y=2014/m=03/d=14"); View<GenericRecord> view = FileSystemDatasets.viewForUri( dataset, "file:" + path); Assert.assertEquals("Should create correct view", view, dataset.getPartitionView(path)); view = FileSystemDatasets.viewForUri( dataset, path.toString()); Assert.assertEquals("Should create correct view", view, dataset.getPartitionView(path)); Constraints expected = ((AbstractRefinableView<GenericRecord>) dataset.with("y", 2014).with("m", 3).with("d", 14)).getConstraints(); Constraints actual = ((AbstractRefinableView<GenericRecord>) view) .getConstraints(); Assert.assertEquals("Constraints should match expected", expected, actual); }
@Test public void testViewForIncompleteUri() { Path path = new Path("/tmp/datasets/ns/test/y=2014/m=03"); View<GenericRecord> view = FileSystemDatasets.viewForUri( dataset, path.toString()); Assert.assertEquals("Should create correct view", view, dataset.getPartitionView(path)); Constraints expected = ((AbstractRefinableView<GenericRecord>) dataset.with("y", 2014).with("m", 3)).getConstraints(); Constraints actual = ((AbstractRefinableView<GenericRecord>) view) .getConstraints(); Assert.assertEquals("Constraints should match expected", expected, actual); }
@Test public void testViewForDifferentPartitionNames() { Path path = new Path("year=2014/month=3/day=14"); // like PathConversion, this uses names from the partition strategy // and will accept partitions that have a different "name=" component View<GenericRecord> view = FileSystemDatasets.viewForUri( dataset, path.toString()); Assert.assertEquals("Should create correct view", view, dataset.getPartitionView(path)); Constraints expected = ((AbstractRefinableView<GenericRecord>) dataset.with("y", 2014).with("m", 3).with("d", 14)).getConstraints(); Constraints actual = ((AbstractRefinableView<GenericRecord>) view) .getConstraints(); Assert.assertEquals("Constraints should match expected", expected, actual); }
@Test public void testViewForMissingPartitionNames() { Path path = new Path("2014/3/14"); // like PathConversion, this uses names from the partition strategy // and will accept partitions that don't have a "name=" component View<GenericRecord> view = FileSystemDatasets.viewForUri( dataset, path.toString()); Assert.assertEquals("Should create correct view", view, dataset.getPartitionView(path)); Constraints expected = ((AbstractRefinableView<GenericRecord>) dataset.with("y", 2014).with("m", 3).with("d", 14)).getConstraints(); Constraints actual = ((AbstractRefinableView<GenericRecord>) view) .getConstraints(); Assert.assertEquals("Constraints should match expected", expected, actual); }
@Test public void testViewForRelativeUri() { Path path = new Path("/tmp/datasets/ns/test/y=2014/m=03/d=14"); View<GenericRecord> view = FileSystemDatasets.viewForUri( dataset, "y=2014/m=03/d=14"); Assert.assertEquals("Should create correct view", view, dataset.getPartitionView(path)); Constraints expected = ((AbstractRefinableView<GenericRecord>) dataset.with("y", 2014).with("m", 3).with("d", 14)).getConstraints(); Constraints actual = ((AbstractRefinableView<GenericRecord>) view) .getConstraints(); Assert.assertEquals("Constraints should match expected", expected, actual); }
@Test public void testIgnoresAuthority() { Path path = new Path("/tmp/datasets/ns/test/y=2014/m=03/d=14"); View<GenericRecord> view = FileSystemDatasets.viewForUri( dataset, "file://127.0.0.1/tmp/datasets/ns/test/y=2014/m=03/d=14"); Assert.assertEquals("Should create correct view", view, dataset.getPartitionView(path)); Constraints expected = ((AbstractRefinableView<GenericRecord>) dataset.with("y", 2014).with("m", 3).with("d", 14)).getConstraints(); Constraints actual = ((AbstractRefinableView<GenericRecord>) view) .getConstraints(); Assert.assertEquals("Constraints should match expected", expected, actual); }
@Test public void testDatasetNotPartitioned() { Datasets.delete("dataset:file:/tmp/datasets/ns/test"); final Dataset<GenericRecord> ds = Datasets.create( "dataset:file:/tmp/datasets/ns/test", new DatasetDescriptor.Builder() .schema(schema) .build()); Assert.assertEquals("Should work for empty relative directory", ds, FileSystemDatasets.viewForUri(ds, "file:/tmp/datasets/ns/test")); TestHelpers.assertThrows("Should reject paths in a non-partitioned dataset", IllegalArgumentException.class, new Runnable() { @Override public void run() { FileSystemDatasets.viewForUri(ds, "y=2014/m=03/d=14"); } }); }