/** * Helper method to process the failed {@link WorkUnit}s from the previous run and add them to the a * {@link MultiWorkUnitWeightedQueue} */ private void addFailedWorkUnits(List<WorkUnit> previousWorkUnitsForRetry, MultiWorkUnitWeightedQueue multiWorkUnitWeightedQueue) { for (WorkUnit wu : previousWorkUnitsForRetry) { try { multiWorkUnitWeightedQueue.addWorkUnit(wu, this.fs.getFileStatus(new Path(wu.getProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL))).getLen()); } catch (IOException e) { Throwables.propagate(e); } LOG.info( "Will process file from previous workunit: " + wu.getProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL)); this.fileCount++; } }
/** * Test for {@link MultiWorkUnitWeightedQueue#MultiWorkUnitWeightedQueue()}. It adds a series of WorkUnits to an * instance of MultiWorkUnitWeightedQueue of checks the size of all the WorkUnits returned by * {@link MultiWorkUnitWeightedQueue#getQueueAsList()}. */ @Test public void testDefaultConstructor() { int numWorkUnits = 10; int weight = 1; MultiWorkUnitWeightedQueue multiWorkUnitWeightedQueue = new MultiWorkUnitWeightedQueue(); WorkUnit workUnit = WorkUnit.createEmpty(); for (int i = 0; i < numWorkUnits; i++) { multiWorkUnitWeightedQueue.addWorkUnit(workUnit, weight); } List<WorkUnit> multiWorkUnitWeightedQueueList = multiWorkUnitWeightedQueue.getQueueAsList(); Assert.assertEquals(multiWorkUnitWeightedQueueList.size(), numWorkUnits); MultiWorkUnit multiWorkUnit; for (WorkUnit workUnitElement : multiWorkUnitWeightedQueueList) { multiWorkUnit = (MultiWorkUnit) workUnitElement; Assert.assertEquals(multiWorkUnit.getWorkUnits().size(), 1); } }
/** * Test for {@link MultiWorkUnitWeightedQueue#MultiWorkUnitWeightedQueue(int)}. It sets a limit on the maximum number * of MultiWorkUnits that can be created, adds a series of WorkUnits to the list, and checks the results of * {@link MultiWorkUnitWeightedQueue#getQueueAsList()} to ensure each MultiWorkUnit created is of proper length. */ @Test public void testWithQueueSizeLimit() { int maxMultiWorkUnits = 10; int numWorkUnits = 100; int weight = 1; MultiWorkUnitWeightedQueue multiWorkUnitWeightedQueue = new MultiWorkUnitWeightedQueue(maxMultiWorkUnits); WorkUnit workUnit = WorkUnit.createEmpty(); for (int i = 0; i < numWorkUnits; i++) { multiWorkUnitWeightedQueue.addWorkUnit(workUnit, weight); } MultiWorkUnit multiWorkUnit; for (WorkUnit workUnitElement : multiWorkUnitWeightedQueue.getQueueAsList()) { multiWorkUnit = (MultiWorkUnit) workUnitElement; Assert.assertEquals(multiWorkUnit.getWorkUnits().size(), numWorkUnits / maxMultiWorkUnits); } } }
multiWorkUnitWeightedQueue.addWorkUnit(singleWorkUnit, file.getFileSize());
/** * Helper method to process the failed {@link WorkUnit}s from the previous run and add them to the a * {@link MultiWorkUnitWeightedQueue} */ private void addFailedWorkUnits(List<WorkUnit> previousWorkUnitsForRetry, MultiWorkUnitWeightedQueue multiWorkUnitWeightedQueue) { for (WorkUnit wu : previousWorkUnitsForRetry) { try { multiWorkUnitWeightedQueue.addWorkUnit(wu, this.fs.getFileStatus(new Path(wu.getProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL))).getLen()); } catch (IOException e) { Throwables.propagate(e); } LOG.info( "Will process file from previous workunit: " + wu.getProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL)); this.fileCount++; } }
multiWorkUnitWeightedQueue.addWorkUnit(singleWorkUnit, file.getFileSize());