/** * Add a file set to the container. * @param fileSet File set, expressed as a {@link org.apache.gobblin.data.management.partition.FileSet} of {@link CopyEntity}s. * @param workUnits List of {@link WorkUnit}s corresponding to this file set. * @return true if the file set was added to the container, false otherwise (i.e. has reached max size). */ public boolean addFileSet(FileSet<CopyEntity> fileSet, List<WorkUnit> workUnits) { boolean addedWorkunits = addFileSetImpl(fileSet, workUnits); if (!addedWorkunits) { this.rejectedFileSet = true; } return addedWorkunits; }
@Test public void testNonStrictBoundAdd() throws IOException { ConcurrentBoundedWorkUnitList list = new ConcurrentBoundedWorkUnitList(10, new AllEqualComparator<FileSet<CopyEntity>>(), 2.0); Assert.assertTrue(addFiles(list, "fs", 6)); Assert.assertFalse(list.hasRejectedFileSet()); Assert.assertFalse(list.isFull()); Assert.assertTrue(addFiles(list, "fs", 5)); Assert.assertFalse(list.hasRejectedFileSet()); Assert.assertTrue(list.isFull()); Assert.assertTrue(addFiles(list, "fs", 4)); Assert.assertFalse(list.hasRejectedFileSet()); Assert.assertFalse(addFiles(list, "fs", 6)); Assert.assertTrue(list.hasRejectedFileSet()); Assert.assertTrue(list.isFull()); }
@Test public void testBoundedAdd() throws IOException { ConcurrentBoundedWorkUnitList list = new ConcurrentBoundedWorkUnitList(10, new AllEqualComparator<FileSet<CopyEntity>>(), 1); Assert.assertTrue(addFiles(list, "fs", 6)); Assert.assertFalse(list.hasRejectedFileSet()); Assert.assertFalse(addFiles(list, "fs", 5)); Assert.assertTrue(list.hasRejectedFileSet()); Assert.assertTrue(addFiles(list, "fs", 4)); Assert.assertFalse(addFiles(list, "fs", 1)); }
@Test public void testPriority() throws IOException { ConcurrentBoundedWorkUnitList list = new ConcurrentBoundedWorkUnitList(10, new NameComparator(), 1); // Fill container Assert.assertTrue(addFiles(list, "z-1", 10)); Assert.assertEquals(list.getWorkUnits().size(), 10); // Reject because same priority Assert.assertFalse(addFiles(list, "z-2", 5)); // Higher priority, so accept new work units Assert.assertTrue(addFiles(list, "y-1", 5)); Assert.assertEquals(list.getWorkUnits().size(), 5); // Lower priority fits, so accept new work units Assert.assertTrue(addFiles(list, "z-3", 2)); Assert.assertEquals(list.getWorkUnits().size(), 7); // Lower priority fits, so accept new work units Assert.assertTrue(addFiles(list, "z-4", 2)); Assert.assertEquals(list.getWorkUnits().size(), 9); // Higher priority, evict lowest priority Assert.assertTrue(addFiles(list, "y-2", 4)); Assert.assertEquals(list.getWorkUnits().size(), 9); // Highest priority, evict lowest priority Assert.assertTrue(addFiles(list, "x-1", 4)); Assert.assertEquals(list.getWorkUnits().size(), 9); }
public boolean addFiles(ConcurrentBoundedWorkUnitList list, String fileSetName, int fileNumber) throws IOException { FileSet.Builder<CopyEntity> partitionBuilder = new FileSet.Builder<>(fileSetName, new DummyDataset(new Path("/path"))); List<WorkUnit> workUnits = Lists.newArrayList(); for (int i = 0; i < fileNumber; i++) { CopyEntity cf = createCopyableFile(i); partitionBuilder.add(cf); WorkUnit workUnit = new WorkUnit(); CopySource.serializeCopyEntity(workUnit, cf); workUnits.add(workUnit); } return list.addFileSet(partitionBuilder.build(), workUnits); }
/** * Add a file set to the container. * @param fileSet File set, expressed as a {@link org.apache.gobblin.data.management.partition.FileSet} of {@link CopyEntity}s. * @param workUnits List of {@link WorkUnit}s corresponding to this file set. * @return true if the file set was added to the container, false otherwise (i.e. has reached max size). */ public boolean addFileSet(FileSet<CopyEntity> fileSet, List<WorkUnit> workUnits) { boolean addedWorkunits = addFileSetImpl(fileSet, workUnits); if (!addedWorkunits) { this.rejectedFileSet = true; } return addedWorkunits; }