reduceDriver.withInput(key5, values5); List<Pair<IntWritable, LongWritable>> result = reduceDriver.run();
@Test public void testReducer() throws Exception { reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready"); CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); BufferedMeasureCodec codec = new BufferedMeasureCodec(cubeDesc.getMeasures()); Text key1 = new Text("72010ustech"); List<Text> values1 = new ArrayList<Text>(); values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 100)); values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 200)); values1.add(newValueText(codec, "10", "10", "10", 1, 300)); Text key2 = new Text("1tech"); List<Text> values2 = new ArrayList<Text>(); values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 500)); values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 1000)); Text key3 = new Text("0"); List<Text> values3 = new ArrayList<Text>(); values3.add(newValueText(codec, "146.52", "146.52", "146.52", 0, 0)); reduceDriver.withInput(key1, values1); reduceDriver.withInput(key2, values2); reduceDriver.withInput(key3, values3); List<Pair<Text, Text>> result = reduceDriver.run(); Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "45.43", "10", "20.34", 3, 600)); Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "35.43", "15.09", "20.34", 2, 1500)); Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "146.52", "146.52", "146.52", 0, 0)); assertEquals(3, result.size()); assertTrue(result.contains(p1)); assertTrue(result.contains(p2)); assertTrue(result.contains(p3)); }
reduceDriver.withInput(key3, values3); List<Pair<Text, Text>> result = reduceDriver.run();
@Test public void test() throws IOException { List<Text> data = new ArrayList<Text>(); for (int i = 0; i < 1001; i++) { data.add(new Text(String.valueOf(i))); } for (Text t : data) { reduceDriver.addInput(t, new ArrayList<LongWritable>()); } reduceDriver.getConfiguration().set(BatchConstants.REGION_NUMBER, "2"); List<Pair<Text, LongWritable>> result = reduceDriver.run(); assertEquals(2, result.size()); for (Pair<Text, LongWritable> p : result) { System.out.println(p.getFirst()); } }
reduceDriver.withInput(key5, values5); List<Pair<IntWritable, LongWritable>> result = reduceDriver.run();
@Test public void testReducer() throws Exception { reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready"); CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); MeasureCodec codec = new MeasureCodec(cubeDesc.getMeasures()); Text key1 = new Text("72010ustech"); List<Text> values1 = new ArrayList<Text>(); values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1)); values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1)); values1.add(newValueText(codec, "10", "10", "10", 1)); Text key2 = new Text("1tech"); List<Text> values2 = new ArrayList<Text>(); values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1)); values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1)); Text key3 = new Text("0"); List<Text> values3 = new ArrayList<Text>(); values3.add(newValueText(codec, "146.52", "146.52", "146.52", 4)); reduceDriver.withInput(key1, values1); reduceDriver.withInput(key2, values2); reduceDriver.withInput(key3, values3); List<Pair<Text, Text>> result = reduceDriver.run(); Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "45.43", "10", "20.34", 3)); Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "35.43", "15.09", "20.34", 2)); Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "146.52", "146.52", "146.52", 4)); assertEquals(3, result.size()); assertTrue(result.contains(p1)); assertTrue(result.contains(p2)); assertTrue(result.contains(p3)); }
reduceOutputs.addAll(reduceDriver.run());
public List<Pair<K3, V3>> run() throws IOException { List<Pair<K2, V2>> mapOutputs = new ArrayList<Pair<K2, V2>>(); // run map component for (Pair<K1, V1> input : inputList) { LOG.debug("Mapping input " + input.toString() + ")"); mapOutputs.addAll(new MapDriver<K1, V1, K2, V2>(myMapper).withInput( input).withCounters(getCounters()).withConfiguration(configuration).run()); } List<Pair<K2, List<V2>>> reduceInputs = shuffle(mapOutputs); List<Pair<K3, V3>> reduceOutputs = new ArrayList<Pair<K3, V3>>(); for (Pair<K2, List<V2>> input : reduceInputs) { K2 inputKey = input.getFirst(); List<V2> inputValues = input.getSecond(); StringBuilder sb = new StringBuilder(); formatValueList(inputValues, sb); LOG.debug("Reducing input (" + inputKey.toString() + ", " + sb.toString() + ")"); reduceOutputs.addAll(new ReduceDriver<K2, V2, K3, V3>(myReducer) .withCounters(getCounters()).withConfiguration(configuration) .withInputKey(inputKey).withInputValues(inputValues).run()); } return reduceOutputs; }
@Test public void testHypercubeReducer() throws IOException { ReduceDriver<IntWritable, CentroidWritable, IntWritable, CentroidWritable> reduceDriver = ReduceDriver.newReduceDriver(new StreamingKMeansReducer()); Configuration configuration = reduceDriver.getConfiguration(); configure(configuration); System.out.printf("%s reducer test\n", configuration.get(StreamingKMeansDriver.SEARCHER_CLASS_OPTION)); StreamingKMeans clusterer = new StreamingKMeans(StreamingKMeansUtilsMR .searcherFromConfiguration(configuration), (1 << NUM_DIMENSIONS) * (int)Math.log(NUM_DATA_POINTS), DISTANCE_CUTOFF); long start = System.currentTimeMillis(); clusterer.cluster(syntheticData.getFirst()); long end = System.currentTimeMillis(); System.out.printf("%f [s]\n", (end - start) / 1000.0); List<CentroidWritable> reducerInputs = Lists.newArrayList(); int postMapperTotalWeight = 0; for (Centroid intermediateCentroid : clusterer) { reducerInputs.add(new CentroidWritable(intermediateCentroid)); postMapperTotalWeight += intermediateCentroid.getWeight(); } reduceDriver.addInput(new IntWritable(0), reducerInputs); List<org.apache.hadoop.mrunit.types.Pair<IntWritable, CentroidWritable>> results = reduceDriver.run(); testReducerResults(postMapperTotalWeight, results); }