@SuppressWarnings("deprecation") @Test @Ignore("not maintaining") public void testMapper2() throws IOException { mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name); mapDriver.addInput(new Text("52010tech"), new Text("35.432")); List<Pair<RowKeyWritable, KeyValue>> result = mapDriver.run(); assertEquals(2, result.size()); byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 }; ImmutableBytesWritable key = new ImmutableBytesWritable(bytes); Pair<RowKeyWritable, KeyValue> p1 = result.get(0); Pair<RowKeyWritable, KeyValue> p2 = result.get(1); assertEquals(key, p1.getFirst()); assertEquals("cf1", new String(p1.getSecond().getFamily(), StandardCharsets.UTF_8)); assertEquals("usd_amt", new String(p1.getSecond().getQualifier(), StandardCharsets.UTF_8)); assertEquals("35.43", new String(p1.getSecond().getValue(), StandardCharsets.UTF_8)); assertEquals(key, p2.getFirst()); assertEquals("cf1", new String(p2.getSecond().getFamily(), StandardCharsets.UTF_8)); assertEquals("item_count", new String(p2.getSecond().getQualifier(), StandardCharsets.UTF_8)); assertEquals("2", new String(p2.getSecond().getValue(), StandardCharsets.UTF_8)); } }
@Test public void test() throws IOException { List<Text> data = new ArrayList<Text>(); for (int i = 0; i < 1001; i++) { data.add(new Text(String.valueOf(i))); } for (Text t : data) { mapDriver.addInput(t, new Text("abc")); } mapDriver.getConfiguration().set(BatchConstants.MAPPER_SAMPLE_NUMBER, "100"); List<Pair<Text, LongWritable>> result = mapDriver.run(); assertEquals(100, result.size()); for (Pair<Text, LongWritable> p : result) { System.out.println(p.getFirst()); } }
@SuppressWarnings("unchecked") @Test public void testMapperWithoutHeader() throws IOException { Text inputKey1 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey2 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey3 = new Text(new byte[] { 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey4 = new Text(new byte[] { 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey5 = new Text(new byte[] { 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey6 = new Text(new byte[] { 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey7 = new Text(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); mapDriver.addInput(inputKey1, new Text("abc")); mapDriver.addInput(inputKey2, new Text("abc")); mapDriver.addInput(inputKey3, new Text("abc")); mapDriver.addInput(inputKey4, new Text("abc")); mapDriver.addInput(inputKey5, new Text("abc")); mapDriver.addInput(inputKey6, new Text("abc")); mapDriver.addInput(inputKey7, new Text("abc")); List<Pair<Text, LongWritable>> result = mapDriver.run(); assertEquals(1, result.size()); byte[] key1 = result.get(0).getFirst().getBytes(); LongWritable value1 = result.get(0).getSecond(); assertArrayEquals(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }, key1); assertEquals(147, value1.get()); }
@SuppressWarnings("unchecked") @Test public void testMapperWithHeader() throws IOException { Text inputKey1 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey2 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey3 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey4 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey5 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey6 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey7 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); mapDriver.addInput(inputKey1, new Text("abc")); mapDriver.addInput(inputKey2, new Text("abc")); mapDriver.addInput(inputKey3, new Text("abc")); mapDriver.addInput(inputKey4, new Text("abc")); mapDriver.addInput(inputKey5, new Text("abc")); mapDriver.addInput(inputKey6, new Text("abc")); mapDriver.addInput(inputKey7, new Text("abc")); List<Pair<Text, LongWritable>> result = mapDriver.run(); assertEquals(1, result.size()); byte[] key1 = result.get(0).getFirst().getBytes(); LongWritable value1 = result.get(0).getSecond(); assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }, key1); assertEquals(273, value1.get()); }
@SuppressWarnings("unchecked") @Test public void testMapperOnComma() throws IOException { mapDriver.clearInput(); LongWritable inputKey1 = new LongWritable(1); LongWritable inputKey2 = new LongWritable(2); LongWritable inputKey3 = new LongWritable(3); LongWritable inputKey4 = new LongWritable(4); LongWritable inputKey5 = new LongWritable(5); LongWritable inputKey6 = new LongWritable(6); LongWritable inputKey7 = new LongWritable(7); mapDriver.addInput(inputKey1, new Text()); mapDriver.addInput(inputKey2, new Text(strArr)); mapDriver.addInput(inputKey3, new Text(strArr)); mapDriver.addInput(inputKey4, new Text(strArr)); mapDriver.addInput(inputKey5, new Text(strArr)); mapDriver.addInput(inputKey6, new Text(strArr)); mapDriver.addInput(inputKey7, new Text(strArr)); List<Pair<IntWritable, BytesWritable>> result = mapDriver.run(); assertEquals(9, result.size()); int key1 = result.get(0).getFirst().get(); BytesWritable value1 = result.get(0).getSecond(); byte[] bytes = value1.getBytes(); HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(); hllc.readRegisters(ByteBuffer.wrap(bytes)); System.out.println("ab\177ab".length()); assertTrue(key1 > 0); assertEquals(1, hllc.getCountEstimate()); }
@SuppressWarnings({ "unchecked" }) @Test @Ignore public void testMapperOn177() throws IOException { mapDriver.clearInput(); File file = new File("src/test/resources/data/test_cal_dt/part-r-00000"); FileReader reader = new FileReader(file); BufferedReader breader = new BufferedReader(reader); String s = breader.readLine(); int i = 0; while (s != null) { LongWritable inputKey = new LongWritable(i++); mapDriver.addInput(inputKey, new Text(s)); s = breader.readLine(); } // breader.close(); List<Pair<IntWritable, BytesWritable>> result = mapDriver.run(); breader.close(); assertEquals(9, result.size()); int key1 = result.get(0).getFirst().get(); BytesWritable value1 = result.get(0).getSecond(); byte[] bytes = value1.getBytes(); HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(); hllc.readRegisters(ByteBuffer.wrap(bytes)); assertTrue(key1 > 0); assertEquals(8, hllc.getCountEstimate()); }
@Test public void testMapperWithHeader() throws Exception { String cubeName = "test_kylin_cube_with_slr_1_new_segment"; String segmentName = "20130331080000_20131212080000"; mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName); mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName); // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, // metadata); mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrancesWomenAuction15123456789132.33")); List<Pair<Text, Text>> result = mapDriver.run(); CubeManager cubeMgr = CubeManager.getInstance(getTestConfig()); CubeInstance cube = cubeMgr.getCube(cubeName); assertEquals(1, result.size()); Text rowkey = result.get(0).getFirst(); byte[] key = rowkey.getBytes(); byte[] header = Bytes.head(key, 26); byte[] sellerId = Bytes.tail(header, 18); byte[] cuboidId = Bytes.head(header, 8); byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26); RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment()); decoder.decode(key); assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, Women, Auction, 0, 15]", decoder.getValues().toString()); assertTrue(Bytes.toString(sellerId).startsWith("123456789")); assertEquals(511, Bytes.toLong(cuboidId)); assertEquals(22, restKey.length); verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "132.33", "132.33", "132.33"); }
@SuppressWarnings("deprecation") @Test @Ignore("not maintaining") public void testMapper2() throws IOException { mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name); mapDriver.addInput(new Text("52010tech"), new Text("35.432")); List<Pair<ImmutableBytesWritable, KeyValue>> result = mapDriver.run(); assertEquals(2, result.size()); byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 }; ImmutableBytesWritable key = new ImmutableBytesWritable(bytes); Pair<ImmutableBytesWritable, KeyValue> p1 = result.get(0); Pair<ImmutableBytesWritable, KeyValue> p2 = result.get(1); assertEquals(key, p1.getFirst()); assertEquals("cf1", new String(p1.getSecond().getFamily())); assertEquals("usd_amt", new String(p1.getSecond().getQualifier())); assertEquals("35.43", new String(p1.getSecond().getValue())); assertEquals(key, p2.getFirst()); assertEquals("cf1", new String(p2.getSecond().getFamily())); assertEquals("item_count", new String(p2.getSecond().getQualifier())); assertEquals("2", new String(p2.getSecond().getValue())); } }
List<Pair<Text, Text>> result = mapDriver.run();
public List<Pair<K3, V3>> run() throws IOException { List<Pair<K2, V2>> mapOutputs = new ArrayList<Pair<K2, V2>>(); // run map component for (Pair<K1, V1> input : inputList) { LOG.debug("Mapping input " + input.toString() + ")"); mapOutputs.addAll(new MapDriver<K1, V1, K2, V2>(myMapper).withInput( input).withCounters(getCounters()).withConfiguration(configuration).run()); } List<Pair<K2, List<V2>>> reduceInputs = shuffle(mapOutputs); List<Pair<K3, V3>> reduceOutputs = new ArrayList<Pair<K3, V3>>(); for (Pair<K2, List<V2>> input : reduceInputs) { K2 inputKey = input.getFirst(); List<V2> inputValues = input.getSecond(); StringBuilder sb = new StringBuilder(); formatValueList(inputValues, sb); LOG.debug("Reducing input (" + inputKey.toString() + ", " + sb.toString() + ")"); reduceOutputs.addAll(new ReduceDriver<K2, V2, K3, V3>(myReducer) .withCounters(getCounters()).withConfiguration(configuration) .withInputKey(inputKey).withInputValues(inputValues).run()); } return reduceOutputs; }
@Test public void shouldNotWriteMetricsByDefault() throws IOException { driver.setInput(new LongWritable(1), record); driver.run(); Multimap<BulkIngestKey,Value> written = TestContextWriter.getWritten(); // two fields mutations + LOAD_DATE + ORIG_FILE + RAW_FILE assertEquals(5, written.size()); for (Map.Entry<BulkIngestKey,Value> entry : written.entries()) { assertEquals(SimpleDataTypeHandler.TABLE, entry.getKey().getTableName()); } }
@Test public void shouldHandleNullRawData() throws IOException { // some RecordReaders may null out raw data entirely because they pass data to their // handlers in other ways. Verify that the EventMapper can handle this case. record.setRawData(null); driver.setInput(new LongWritable(1), record); driver.run(); Multimap<BulkIngestKey,Value> written = TestContextWriter.getWritten(); // two fields mutations + LOAD_DATE + ORIG_FILE + RAW_FILE assertEquals(5, written.size()); }
@Test public void shouldNotWriteRawFile() throws IOException { record.setRawFileName(""); driver.setInput(new LongWritable(1), record); driver.run(); Multimap<BulkIngestKey,Value> written = TestContextWriter.getWritten(); // two fields mutations + LOAD_DATE + ORIG_FILE assertEquals(4, written.size()); }
mapDriver.addAll(inputs.get(mapper)); mapDriver.withMapInputPath(getMapInputPath(mapper)); outputs.addAll(mapDriver.run());
@Override public List<Pair<K3, V3>> run() throws IOException { try { preRunChecks(myMapper, myReducer); initDistributedCache(); List<Pair<K2, V2>> mapOutputs = new ArrayList<Pair<K2, V2>>(); // run map component LOG.debug("Starting map phase with mapper: " + myMapper); mapOutputs.addAll(MapDriver.newMapDriver(myMapper) .withCounters(getCounters()).withConfiguration(getConfiguration()) .withAll(inputList).withMapInputPath(getMapInputPath()).run()); if (myCombiner != null) { // User has specified a combiner. Run this and replace the mapper // outputs // with the result of the combiner. LOG.debug("Starting combine phase with combiner: " + myCombiner); mapOutputs = new ReducePhaseRunner<K2, V2, K2, V2>(inputFormatClass, getConfiguration(), counters, getOutputSerializationConfiguration(), outputFormatClass) .runReduce(sortAndGroup(mapOutputs), myCombiner); } // Run the reduce phase. LOG.debug("Starting reduce phase with reducer: " + myReducer); return new ReducePhaseRunner<K2, V2, K3, V3>(inputFormatClass, getConfiguration(), counters, getOutputSerializationConfiguration(), outputFormatClass).runReduce(sortAndGroup(mapOutputs), myReducer); } finally { cleanupDistributedCache(); } }
@Test public void testHypercubeMapper() throws IOException { MapDriver<Writable, VectorWritable, IntWritable, CentroidWritable> mapDriver = MapDriver.newMapDriver(new StreamingKMeansMapper()); configure(mapDriver.getConfiguration()); System.out.printf("%s mapper test\n", mapDriver.getConfiguration().get(StreamingKMeansDriver.SEARCHER_CLASS_OPTION)); for (Centroid datapoint : syntheticData.getFirst()) { mapDriver.addInput(new IntWritable(0), new VectorWritable(datapoint)); } List<org.apache.hadoop.mrunit.types.Pair<IntWritable,CentroidWritable>> results = mapDriver.run(); BruteSearch resultSearcher = new BruteSearch(new SquaredEuclideanDistanceMeasure()); for (org.apache.hadoop.mrunit.types.Pair<IntWritable, CentroidWritable> result : results) { resultSearcher.add(result.getSecond().getCentroid()); } System.out.printf("Clustered the data into %d clusters\n", results.size()); for (Vector mean : syntheticData.getSecond()) { WeightedThing<Vector> closest = resultSearcher.search(mean, 1).get(0); assertTrue("Weight " + closest.getWeight() + " not less than 0.5", closest.getWeight() < 0.5); } }
for (org.apache.hadoop.mrunit.types.Pair<IntWritable, CentroidWritable> pair : mapDriver.run()) { mapperCentroids.add(pair.getSecond().getCentroid());