/** * Test for {@link MultiWorkUnitWeightedQueue#MultiWorkUnitWeightedQueue(int)}. It sets a limit on the maximum number * of MultiWorkUnits that can be created, adds a series of WorkUnits to the list, and checks the results of * {@link MultiWorkUnitWeightedQueue#getQueueAsList()} to ensure each MultiWorkUnit created is of proper length. */ @Test public void testWithQueueSizeLimit() { int maxMultiWorkUnits = 10; int numWorkUnits = 100; int weight = 1; MultiWorkUnitWeightedQueue multiWorkUnitWeightedQueue = new MultiWorkUnitWeightedQueue(maxMultiWorkUnits); WorkUnit workUnit = WorkUnit.createEmpty(); for (int i = 0; i < numWorkUnits; i++) { multiWorkUnitWeightedQueue.addWorkUnit(workUnit, weight); } MultiWorkUnit multiWorkUnit; for (WorkUnit workUnitElement : multiWorkUnitWeightedQueue.getQueueAsList()) { multiWorkUnit = (MultiWorkUnit) workUnitElement; Assert.assertEquals(multiWorkUnit.getWorkUnits().size(), numWorkUnits / maxMultiWorkUnits); } } }
@Override public List<WorkUnit> getWorkunits(SourceState state) { DateTimeFormatter formatter = DateTimeFormat.fullDateTime(); // Initialize all instance variables for this object init(state); LOG.info("Will pull data from " + formatter.print(this.lowWaterMark) + " until " + this.maxFilesPerJob + " files have been processed, or until there is no more data to consume"); LOG.info("Creating workunits"); // Weighted MultiWorkUnitWeightedQueue, the job will add new WorkUnits to the queue along with a weight for each // WorkUnit. The queue will take care of balancing the WorkUnits amongst a set number of MultiWorkUnits MultiWorkUnitWeightedQueue multiWorkUnitWeightedQueue = new MultiWorkUnitWeightedQueue(this.maxWorkUnitsPerJob); // Add failed work units from the previous execution addFailedWorkUnits(getPreviousWorkUnitsForRetry(this.sourceState), multiWorkUnitWeightedQueue); // If the file count has not exceeded maxFilesPerJob then start adding new WorkUnits to for this job if (this.fileCount >= this.maxFilesPerJob) { LOG.info( "The number of work units from previous job has already reached the upper limit, no more workunits will be made"); return multiWorkUnitWeightedQueue.getQueueAsList(); } addNewWorkUnits(multiWorkUnitWeightedQueue); return multiWorkUnitWeightedQueue.getQueueAsList(); }
/** * Helper method to process the failed {@link WorkUnit}s from the previous run and add them to the a * {@link MultiWorkUnitWeightedQueue} */ private void addFailedWorkUnits(List<WorkUnit> previousWorkUnitsForRetry, MultiWorkUnitWeightedQueue multiWorkUnitWeightedQueue) { for (WorkUnit wu : previousWorkUnitsForRetry) { try { multiWorkUnitWeightedQueue.addWorkUnit(wu, this.fs.getFileStatus(new Path(wu.getProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL))).getLen()); } catch (IOException e) { Throwables.propagate(e); } LOG.info( "Will process file from previous workunit: " + wu.getProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL)); this.fileCount++; } }
@Override public List<WorkUnit> getWorkunits(SourceState state) { DateTimeFormatter formatter = DateTimeFormat.fullDateTime(); // Initialize all instance variables for this object init(state); LOG.info("Will pull data from " + formatter.print(this.lowWaterMark) + " until " + this.maxFilesPerJob + " files have been processed, or until there is no more data to consume"); LOG.info("Creating workunits"); // Weighted MultiWorkUnitWeightedQueue, the job will add new WorkUnits to the queue along with a weight for each // WorkUnit. The queue will take care of balancing the WorkUnits amongst a set number of MultiWorkUnits MultiWorkUnitWeightedQueue multiWorkUnitWeightedQueue = new MultiWorkUnitWeightedQueue(this.maxWorkUnitsPerJob); // Add failed work units from the previous execution addFailedWorkUnits(getPreviousWorkUnitsForRetry(this.sourceState), multiWorkUnitWeightedQueue); // If the file count has not exceeded maxFilesPerJob then start adding new WorkUnits to for this job if (this.fileCount >= this.maxFilesPerJob) { LOG.info( "The number of work units from previous job has already reached the upper limit, no more workunits will be made"); return multiWorkUnitWeightedQueue.getQueueAsList(); } addNewWorkUnits(multiWorkUnitWeightedQueue); return multiWorkUnitWeightedQueue.getQueueAsList(); }
multiWorkUnitWeightedQueue.addWorkUnit(singleWorkUnit, file.getFileSize());
/** * Test for {@link MultiWorkUnitWeightedQueue#MultiWorkUnitWeightedQueue()}. It adds a series of WorkUnits to an * instance of MultiWorkUnitWeightedQueue of checks the size of all the WorkUnits returned by * {@link MultiWorkUnitWeightedQueue#getQueueAsList()}. */ @Test public void testDefaultConstructor() { int numWorkUnits = 10; int weight = 1; MultiWorkUnitWeightedQueue multiWorkUnitWeightedQueue = new MultiWorkUnitWeightedQueue(); WorkUnit workUnit = WorkUnit.createEmpty(); for (int i = 0; i < numWorkUnits; i++) { multiWorkUnitWeightedQueue.addWorkUnit(workUnit, weight); } List<WorkUnit> multiWorkUnitWeightedQueueList = multiWorkUnitWeightedQueue.getQueueAsList(); Assert.assertEquals(multiWorkUnitWeightedQueueList.size(), numWorkUnits); MultiWorkUnit multiWorkUnit; for (WorkUnit workUnitElement : multiWorkUnitWeightedQueueList) { multiWorkUnit = (MultiWorkUnit) workUnitElement; Assert.assertEquals(multiWorkUnit.getWorkUnits().size(), 1); } }
/** * Helper method to process the failed {@link WorkUnit}s from the previous run and add them to the a * {@link MultiWorkUnitWeightedQueue} */ private void addFailedWorkUnits(List<WorkUnit> previousWorkUnitsForRetry, MultiWorkUnitWeightedQueue multiWorkUnitWeightedQueue) { for (WorkUnit wu : previousWorkUnitsForRetry) { try { multiWorkUnitWeightedQueue.addWorkUnit(wu, this.fs.getFileStatus(new Path(wu.getProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL))).getLen()); } catch (IOException e) { Throwables.propagate(e); } LOG.info( "Will process file from previous workunit: " + wu.getProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL)); this.fileCount++; } }
multiWorkUnitWeightedQueue.addWorkUnit(singleWorkUnit, file.getFileSize());