@Override public List<WorkUnit> getWorkunits(SourceState state) { DateTimeFormatter formatter = DateTimeFormat.fullDateTime(); // Initialize all instance variables for this object init(state); LOG.info("Will pull data from " + formatter.print(this.lowWaterMark) + " until " + this.maxFilesPerJob + " files have been processed, or until there is no more data to consume"); LOG.info("Creating workunits"); // Weighted MultiWorkUnitWeightedQueue, the job will add new WorkUnits to the queue along with a weight for each // WorkUnit. The queue will take care of balancing the WorkUnits amongst a set number of MultiWorkUnits MultiWorkUnitWeightedQueue multiWorkUnitWeightedQueue = new MultiWorkUnitWeightedQueue(this.maxWorkUnitsPerJob); // Add failed work units from the previous execution addFailedWorkUnits(getPreviousWorkUnitsForRetry(this.sourceState), multiWorkUnitWeightedQueue); // If the file count has not exceeded maxFilesPerJob then start adding new WorkUnits to for this job if (this.fileCount >= this.maxFilesPerJob) { LOG.info( "The number of work units from previous job has already reached the upper limit, no more workunits will be made"); return multiWorkUnitWeightedQueue.getQueueAsList(); } addNewWorkUnits(multiWorkUnitWeightedQueue); return multiWorkUnitWeightedQueue.getQueueAsList(); }
/** * Test for {@link MultiWorkUnitWeightedQueue#MultiWorkUnitWeightedQueue()}. It adds a series of WorkUnits to an * instance of MultiWorkUnitWeightedQueue of checks the size of all the WorkUnits returned by * {@link MultiWorkUnitWeightedQueue#getQueueAsList()}. */ @Test public void testDefaultConstructor() { int numWorkUnits = 10; int weight = 1; MultiWorkUnitWeightedQueue multiWorkUnitWeightedQueue = new MultiWorkUnitWeightedQueue(); WorkUnit workUnit = WorkUnit.createEmpty(); for (int i = 0; i < numWorkUnits; i++) { multiWorkUnitWeightedQueue.addWorkUnit(workUnit, weight); } List<WorkUnit> multiWorkUnitWeightedQueueList = multiWorkUnitWeightedQueue.getQueueAsList(); Assert.assertEquals(multiWorkUnitWeightedQueueList.size(), numWorkUnits); MultiWorkUnit multiWorkUnit; for (WorkUnit workUnitElement : multiWorkUnitWeightedQueueList) { multiWorkUnit = (MultiWorkUnit) workUnitElement; Assert.assertEquals(multiWorkUnit.getWorkUnits().size(), 1); } }
/** * Test for {@link MultiWorkUnitWeightedQueue#MultiWorkUnitWeightedQueue(int)}. It sets a limit on the maximum number * of MultiWorkUnits that can be created, adds a series of WorkUnits to the list, and checks the results of * {@link MultiWorkUnitWeightedQueue#getQueueAsList()} to ensure each MultiWorkUnit created is of proper length. */ @Test public void testWithQueueSizeLimit() { int maxMultiWorkUnits = 10; int numWorkUnits = 100; int weight = 1; MultiWorkUnitWeightedQueue multiWorkUnitWeightedQueue = new MultiWorkUnitWeightedQueue(maxMultiWorkUnits); WorkUnit workUnit = WorkUnit.createEmpty(); for (int i = 0; i < numWorkUnits; i++) { multiWorkUnitWeightedQueue.addWorkUnit(workUnit, weight); } MultiWorkUnit multiWorkUnit; for (WorkUnit workUnitElement : multiWorkUnitWeightedQueue.getQueueAsList()) { multiWorkUnit = (MultiWorkUnit) workUnitElement; Assert.assertEquals(multiWorkUnit.getWorkUnits().size(), numWorkUnits / maxMultiWorkUnits); } } }
@Override public List<WorkUnit> getWorkunits(SourceState state) { DateTimeFormatter formatter = DateTimeFormat.fullDateTime(); // Initialize all instance variables for this object init(state); LOG.info("Will pull data from " + formatter.print(this.lowWaterMark) + " until " + this.maxFilesPerJob + " files have been processed, or until there is no more data to consume"); LOG.info("Creating workunits"); // Weighted MultiWorkUnitWeightedQueue, the job will add new WorkUnits to the queue along with a weight for each // WorkUnit. The queue will take care of balancing the WorkUnits amongst a set number of MultiWorkUnits MultiWorkUnitWeightedQueue multiWorkUnitWeightedQueue = new MultiWorkUnitWeightedQueue(this.maxWorkUnitsPerJob); // Add failed work units from the previous execution addFailedWorkUnits(getPreviousWorkUnitsForRetry(this.sourceState), multiWorkUnitWeightedQueue); // If the file count has not exceeded maxFilesPerJob then start adding new WorkUnits to for this job if (this.fileCount >= this.maxFilesPerJob) { LOG.info( "The number of work units from previous job has already reached the upper limit, no more workunits will be made"); return multiWorkUnitWeightedQueue.getQueueAsList(); } addNewWorkUnits(multiWorkUnitWeightedQueue); return multiWorkUnitWeightedQueue.getQueueAsList(); }