View<E> viewForUri(URI location) { Preconditions.checkNotNull(location, "Partition location cannot be null"); PartitionView<E> view = getPartitionView(location); if (view == unbounded) { return this; } return view; }
@Override public void run() { partitioned.getPartitionView(URI.create( "file:/tmp/datasets/partitioned/id_hash=trees")); } });
@Override public void run() { partitioned.getPartitionView(URI.create( "hdfs:/tmp/datasets/partitioned")); } });
@Override public void run() { partitioned.getPartitionView(URI.create( "hdfs:/tmp/datasets/partitioned/id_hash=0/data_hash=2")); } });
@Override public void run() { partitioned.getPartitionView(URI.create( "file:/tmp/datasets/unpartitioned")); } });
@Override public void run() { partitioned.getPartitionView(URI.create( "file:/tmp/datasets")); } });
@Override public void run() { partitioned.getPartitionView(URI.create("")); } });
URI relative = ((FileSystemPartitionView<E>) src).getRelativeLocation(); PartitionView<E> dest = relative != null ? getPartitionView(relative) : unbounded;
@Test public void testCoveringPartitions() { Iterable<PartitionView<TestRecord>> partitions = unpartitioned .getCoveringPartitions(); Assert.assertEquals("Should have a single partition view at the root", unpartitioned.getPartitionView(URI.create( "file:/tmp/datasets/unpartitioned")), Iterables.getOnlyElement(partitions)); partitions = partitioned.getCoveringPartitions(); Set<PartitionView<TestRecord>> expected = Sets.newHashSet(); expected.add(partitioned.getPartitionView(URI.create( "file:/tmp/datasets/partitioned/id_hash=0"))); expected.add(partitioned.getPartitionView(new Path( "file:/tmp/datasets/partitioned/id_hash=1"))); expected.add(partitioned.getPartitionView(URI.create( "file:/tmp/datasets/partitioned/id_hash=2"))); expected.add(partitioned.getPartitionView(new Path( "file:/tmp/datasets/partitioned/id_hash=3"))); Assert.assertEquals("Should have a partition view for each partition", expected, Sets.newHashSet(partitions)); PartitionView<TestRecord> partition0 = partitioned.getPartitionView( URI.create("file:/tmp/datasets/partitioned/id_hash=0")); partition0.deleteAll(); expected.remove(partition0); Assert.assertEquals("Should have a partition view for each partition", expected, Sets.newHashSet(partitions)); }
@Test public void testRestrictedRead() throws IOException { FileSystemPartitionView<TestRecord> partition0 = partitioned .getPartitionView(URI.create("id_hash=0")); FileSystemPartitionView<TestRecord> partition1 = partitioned .getPartitionView(URI.create("id_hash=1")); FileSystemPartitionView<TestRecord> partition2 = partitioned .getPartitionView(URI.create("id_hash=2")); FileSystemPartitionView<TestRecord> partition3 = partitioned .getPartitionView(URI.create("id_hash=3"));
@Test public void testReplaceSinglePartition() { FileSystemPartitionView<TestRecord> partition0 = partitioned.getPartitionView( new Path("id_hash=0")); FileSystemPartitionView<TestRecord> temp0 = temporary.getPartitionView( new Path("id_hash=0")); Assert.assertTrue("Should allow replacing a single partition", partitioned.canReplace(partition0)); Assert.assertFalse( "Should not allow replacement test with a different dataset", partitioned.canReplace(temp0)); Set<String> replacementFiles = Sets.newHashSet( Iterators.transform(temp0.pathIterator(), new GetFilename())); Set<String> originalPartitionFiles = Sets.newHashSet( Iterators.transform(partition0.pathIterator(), new GetFilename())); Assert.assertEquals("Sanity check", originalPartitionFiles.size(), replacementFiles.size()); Assert.assertFalse("Sanity check", originalPartitionFiles.equals(replacementFiles)); Set<String> expectedFiles = Sets.newHashSet( Iterators.transform(partitioned.pathIterator(), new GetFilename())); expectedFiles.removeAll(originalPartitionFiles); expectedFiles.addAll(replacementFiles); partitioned.replace(partition0, temp0); Set<String> replacedFiles = Sets.newHashSet( Iterators.transform(partitioned.pathIterator(), new GetFilename())); Assert.assertEquals("Should contain the replacement files", expectedFiles, replacedFiles); }
@Test public void testRelativePaths() { FileSystemPartitionView<TestRecord> partition = partitioned .getPartitionView(URI.create("id_hash=0")); Assert.assertEquals("Should accept a relative URI", URI.create("file:/tmp/datasets/partitioned/id_hash=0"), URI.create("id_hash=0"), partition.getRelativeLocation()); partition = partitioned.getPartitionView( new Path("id_hash=0")); Assert.assertEquals("Should accept a relative Path", URI.create("id_hash=0"), partition.getRelativeLocation()); partition = partitioned.getPartitionView((URI) null); Assert.assertEquals("Should accept a null URI", URI.create("file:/tmp/datasets/partitioned"), null, partition.getRelativeLocation()); partition = partitioned.getPartitionView((Path) null); Assert.assertEquals("Should accept a null Path", URI.create("file:/tmp/datasets/partitioned"),
FileSystemPartitionView<TestRecord> temp0 = temporary.getPartitionView( new Path("id_hash=0")); new Path(partitioned.getDirectory(), "id_hash=00")); FileSystemPartitionView<TestRecord> partition0 = partitioned.getPartitionView( new Path("id-hash=0"));
@Test public void testViewForIncompleteUri() { Path path = new Path("/tmp/datasets/ns/test/y=2014/m=03"); View<GenericRecord> view = FileSystemDatasets.viewForUri( dataset, path.toString()); Assert.assertEquals("Should create correct view", view, dataset.getPartitionView(path)); Constraints expected = ((AbstractRefinableView<GenericRecord>) dataset.with("y", 2014).with("m", 3)).getConstraints(); Constraints actual = ((AbstractRefinableView<GenericRecord>) view) .getConstraints(); Assert.assertEquals("Constraints should match expected", expected, actual); }
@Test public void testViewForUri() { Path path = new Path("/tmp/datasets/ns/test/y=2014/m=03/d=14"); View<GenericRecord> view = FileSystemDatasets.viewForUri( dataset, "file:" + path); Assert.assertEquals("Should create correct view", view, dataset.getPartitionView(path)); view = FileSystemDatasets.viewForUri( dataset, path.toString()); Assert.assertEquals("Should create correct view", view, dataset.getPartitionView(path)); Constraints expected = ((AbstractRefinableView<GenericRecord>) dataset.with("y", 2014).with("m", 3).with("d", 14)).getConstraints(); Constraints actual = ((AbstractRefinableView<GenericRecord>) view) .getConstraints(); Assert.assertEquals("Constraints should match expected", expected, actual); }
@Test public void testViewForDifferentPartitionNames() { Path path = new Path("year=2014/month=3/day=14"); // like PathConversion, this uses names from the partition strategy // and will accept partitions that have a different "name=" component View<GenericRecord> view = FileSystemDatasets.viewForUri( dataset, path.toString()); Assert.assertEquals("Should create correct view", view, dataset.getPartitionView(path)); Constraints expected = ((AbstractRefinableView<GenericRecord>) dataset.with("y", 2014).with("m", 3).with("d", 14)).getConstraints(); Constraints actual = ((AbstractRefinableView<GenericRecord>) view) .getConstraints(); Assert.assertEquals("Constraints should match expected", expected, actual); }
@Test public void testViewForMissingPartitionNames() { Path path = new Path("2014/3/14"); // like PathConversion, this uses names from the partition strategy // and will accept partitions that don't have a "name=" component View<GenericRecord> view = FileSystemDatasets.viewForUri( dataset, path.toString()); Assert.assertEquals("Should create correct view", view, dataset.getPartitionView(path)); Constraints expected = ((AbstractRefinableView<GenericRecord>) dataset.with("y", 2014).with("m", 3).with("d", 14)).getConstraints(); Constraints actual = ((AbstractRefinableView<GenericRecord>) view) .getConstraints(); Assert.assertEquals("Constraints should match expected", expected, actual); }
@Test public void testViewForRelativeUri() { Path path = new Path("/tmp/datasets/ns/test/y=2014/m=03/d=14"); View<GenericRecord> view = FileSystemDatasets.viewForUri( dataset, "y=2014/m=03/d=14"); Assert.assertEquals("Should create correct view", view, dataset.getPartitionView(path)); Constraints expected = ((AbstractRefinableView<GenericRecord>) dataset.with("y", 2014).with("m", 3).with("d", 14)).getConstraints(); Constraints actual = ((AbstractRefinableView<GenericRecord>) view) .getConstraints(); Assert.assertEquals("Constraints should match expected", expected, actual); }
@Test public void testIgnoresAuthority() { Path path = new Path("/tmp/datasets/ns/test/y=2014/m=03/d=14"); View<GenericRecord> view = FileSystemDatasets.viewForUri( dataset, "file://127.0.0.1/tmp/datasets/ns/test/y=2014/m=03/d=14"); Assert.assertEquals("Should create correct view", view, dataset.getPartitionView(path)); Constraints expected = ((AbstractRefinableView<GenericRecord>) dataset.with("y", 2014).with("m", 3).with("d", 14)).getConstraints(); Constraints actual = ((AbstractRefinableView<GenericRecord>) view) .getConstraints(); Assert.assertEquals("Constraints should match expected", expected, actual); }
@Test public void testUnpartitionedReplaceDifferentStrategy() { final FileSystemPartitionView<TestRecord> partition0 = partitioned .getPartitionView(new Path("id_hash=0")); Assert.assertFalse("Should not allow replacement with a different strategy", unpartitioned.canReplace(partitioned)); Assert.assertFalse("Should not allow replacement with a different strategy", unpartitioned.canReplace(partition0)); TestHelpers.assertThrows( "Should not allow replacement with a different partition strategy", ValidationException.class, new Runnable() { @Override public void run() { unpartitioned.replace(unpartitioned, partitioned); } }); TestHelpers.assertThrows( "Should not allow replacement with a different partition strategy", ValidationException.class, new Runnable() { @Override public void run() { unpartitioned.replace(unpartitioned, partition0); } }); }