/** * Pack the list of {@code WorkUnit}s into {@code MultiWorkUnit}s. * * TODO: this is currently a simple round-robin packing. More sophisticated bin packing may be necessary * if the round-robin approach leads to mapper skew. */ private static List<WorkUnit> pack(List<WorkUnit> workUnits, int numOfMultiWorkunits) { Preconditions.checkArgument(numOfMultiWorkunits > 0); if (workUnits.size() <= numOfMultiWorkunits) { return workUnits; } List<WorkUnit> result = Lists.newArrayListWithCapacity(numOfMultiWorkunits); for (int i = 0; i < numOfMultiWorkunits; i++) { result.add(MultiWorkUnit.createEmpty()); } for (int i = 0; i < workUnits.size(); i++) { ((MultiWorkUnit) result.get(i % numOfMultiWorkunits)).addWorkUnit(workUnits.get(i)); } return result; }
private static void addToMultiWorkUnit(MultiWorkUnit multiWorkUnit, WorkUnit workUnit, long weight) { multiWorkUnit.addWorkUnit(workUnit); setMultiWorkUnitWeight(multiWorkUnit, getMultiWorkUnitWeight(multiWorkUnit) + weight); }
protected static void addWorkUnitToMultiWorkUnit(WorkUnit workUnit, MultiWorkUnit multiWorkUnit) { multiWorkUnit.addWorkUnit(workUnit); double size = multiWorkUnit.getPropAsDouble(ESTIMATED_WORKUNIT_SIZE, 0.0); multiWorkUnit.setProp(ESTIMATED_WORKUNIT_SIZE, size + getWorkUnitEstSize(workUnit)); }
private WorkUnit createWorkUnit(String... names) { if (names.length == 1) { WorkUnit workUnit = new WorkUnit(); workUnit.setProp(WORK_UNIT_NAME, names[0]); return workUnit; } MultiWorkUnit mwu = new MultiWorkUnit(); for (String name : names) { mwu.addWorkUnit(createWorkUnit(name)); } return mwu; }
@BeforeClass public void setUp() { this.multiWorkUnit = new MultiWorkUnit(); WorkUnit workUnit1 = WorkUnit.createEmpty(); workUnit1.setHighWaterMark(1000); workUnit1.setLowWaterMark(0); workUnit1.setProp("k1", "v1"); this.multiWorkUnit.addWorkUnit(workUnit1); WorkUnit workUnit2 = WorkUnit.createEmpty(); workUnit2.setHighWaterMark(2000); workUnit2.setLowWaterMark(1001); workUnit2.setProp("k2", "v2"); this.multiWorkUnit.addWorkUnit(workUnit2); }
/** * Pack the list of {@code WorkUnit}s into {@code MultiWorkUnit}s. * * TODO: this is currently a simple round-robin packing. More sophisticated bin packing may be necessary * if the round-robin approach leads to mapper skew. */ private static List<WorkUnit> pack(List<WorkUnit> workUnits, int numOfMultiWorkunits) { Preconditions.checkArgument(numOfMultiWorkunits > 0); if (workUnits.size() <= numOfMultiWorkunits) { return workUnits; } List<WorkUnit> result = Lists.newArrayListWithCapacity(numOfMultiWorkunits); for (int i = 0; i < numOfMultiWorkunits; i++) { result.add(MultiWorkUnit.createEmpty()); } for (int i = 0; i < workUnits.size(); i++) { ((MultiWorkUnit) result.get(i % numOfMultiWorkunits)).addWorkUnit(workUnits.get(i)); } return result; }
private static void addToMultiWorkUnit(MultiWorkUnit multiWorkUnit, WorkUnit workUnit, long weight) { multiWorkUnit.addWorkUnit(workUnit); setMultiWorkUnitWeight(multiWorkUnit, getMultiWorkUnitWeight(multiWorkUnit) + weight); }
protected static void addWorkUnitToMultiWorkUnit(WorkUnit workUnit, MultiWorkUnit multiWorkUnit) { multiWorkUnit.addWorkUnit(workUnit); double size = multiWorkUnit.getPropAsDouble(ESTIMATED_WORKUNIT_SIZE, 0.0); multiWorkUnit.setProp(ESTIMATED_WORKUNIT_SIZE, size + getWorkUnitEstSize(workUnit)); }