@Override public long getSize() { long size = 0; for (Iterator<Path> i = dirIterator(); i.hasNext(); ) { Path dir = i.next(); try { for (FileStatus st : fileSystem.listStatus(dir)) { size += st.getLen(); } } catch (IOException e) { throw new DatasetIOException("Cannot find size of " + dir, e); } } return size; }
public void addExistingPartitions() { if (partitionListener != null && descriptor.isPartitioned()) { for (Iterator<Path> i = dirIterator(); i.hasNext(); ) { Path partition = i.next(); LOG.info("Adding partition {}", partition); partitionListener.partitionAdded(namespace, name, partition.toString()); } } }
public Iterator<Path> getDirectoryIterator(View view) { if (view instanceof FileSystemView) { return ((FileSystemView<?>) view).dirIterator(); } else if (view instanceof FileSystemDataset) { return ((FileSystemDataset<?>) view).dirIterator(); } else { throw new DatasetException( "Underlying Dataset must be a FileSystemDataset"); } }
replacementFiles, replacedFiles); Iterator<Path> dirIterator = partitioned.dirIterator(); Path onlyDirectory = dirIterator.next(); Assert.assertFalse("Should contain only one directory",
replacementFiles, replacedFiles); Iterator<Path> dirIterator = partitioned.dirIterator(); Path onlyDirectory = dirIterator.next(); Assert.assertFalse("Should contain only one directory",
@Test public void testPathIterator_Directory() { FileSystemDataset<Record> ds = new FileSystemDataset.Builder<Record>() .namespace("ns") .name("users") .configuration(getConfiguration()) .descriptor(new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .format(format) .compressionType(compressionType) .location(testDirectory) .build()) .type(Record.class) .build(); List<Path> dirPaths = Lists.newArrayList(ds.dirIterator()); Assert.assertEquals("dirIterator for non-partitioned dataset should yield a single path.", 1, dirPaths.size()); Assert.assertEquals("dirIterator should yield absolute paths.", testDirectory, dirPaths.get(0)); }
checkTestUsers(ds, 10); List<Path> dirPaths = Lists.newArrayList(ds.dirIterator()); List<Path> leafPaths = Lists.newArrayList(partition.dirIterator()); Assert.assertEquals(1, leafPaths.size()); final Path leafPath = leafPaths.get(0);