mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrances\\NAuction15123456789\\N")); List<Pair<Text, Text>> result = mapDriver.run();
@Test public void testMapperWithHeader() throws Exception { String cubeName = "test_kylin_cube_with_slr_1_new_segment"; String segmentName = "20130331080000_20131212080000"; mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName); mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName); // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, // metadata); mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrancesWomenAuction15123456789132.33")); List<Pair<Text, Text>> result = mapDriver.run(); CubeManager cubeMgr = CubeManager.getInstance(getTestConfig()); CubeInstance cube = cubeMgr.getCube(cubeName); assertEquals(1, result.size()); Text rowkey = result.get(0).getFirst(); byte[] key = rowkey.getBytes(); byte[] header = Bytes.head(key, 26); byte[] sellerId = Bytes.tail(header, 18); byte[] cuboidId = Bytes.head(header, 8); byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26); RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment()); decoder.decode(key); assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, Women, Auction, 0, 15]", decoder.getValues().toString()); assertTrue(Bytes.toString(sellerId).startsWith("123456789")); assertEquals(511, Bytes.toLong(cuboidId)); assertEquals(22, restKey.length); verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "132.33", "132.33", "132.33"); }
public List<Pair<K3, V3>> run() throws IOException { List<Pair<K2, V2>> mapOutputs = new ArrayList<Pair<K2, V2>>(); // run map component for (Pair<K1, V1> input : inputList) { LOG.debug("Mapping input " + input.toString() + ")"); mapOutputs.addAll(new MapDriver<K1, V1, K2, V2>(myMapper).withInput( input).withCounters(getCounters()).withConfiguration(configuration).run()); } List<Pair<K2, List<V2>>> reduceInputs = shuffle(mapOutputs); List<Pair<K3, V3>> reduceOutputs = new ArrayList<Pair<K3, V3>>(); for (Pair<K2, List<V2>> input : reduceInputs) { K2 inputKey = input.getFirst(); List<V2> inputValues = input.getSecond(); StringBuilder sb = new StringBuilder(); formatValueList(inputValues, sb); LOG.debug("Reducing input (" + inputKey.toString() + ", " + sb.toString() + ")"); reduceOutputs.addAll(new ReduceDriver<K2, V2, K3, V3>(myReducer) .withCounters(getCounters()).withConfiguration(configuration) .withInputKey(inputKey).withInputValues(inputValues).run()); } return reduceOutputs; }
/** * Tests quads to triples conversion * * @throws IOException */ @Test public void triples_to_quads_mapper_01() throws IOException { MapDriver<LongWritable, TripleWritable, LongWritable, QuadWritable> driver = this.getMapDriver(); Triple t = new Triple(NodeFactory.createURI("http://s"), NodeFactory.createURI("http://p"), NodeFactory.createLiteral("test")); Quad q = new Quad(t.getSubject(), t); driver.withInput(new Pair<LongWritable, TripleWritable>(new LongWritable(1), new TripleWritable(t))).withOutput( new Pair<LongWritable, QuadWritable>(new LongWritable(1), new QuadWritable(q))); driver.runTest(); }
/** * Tests quads to triples conversion * * @throws IOException */ @Test public void triples_to_quads_mapper_01() throws IOException { MapDriver<LongWritable, TripleWritable, LongWritable, QuadWritable> driver = this.getMapDriver(); Triple t = new Triple(NodeFactory.createURI("http://s"), NodeFactory.createURI("http://p"), NodeFactory.createLiteral("test")); Quad q = new Quad(Quad.defaultGraphNodeGenerated, t); driver.withInput(new Pair<LongWritable, TripleWritable>(new LongWritable(1), new TripleWritable(t))).withOutput( new Pair<LongWritable, QuadWritable>(new LongWritable(1), new QuadWritable(q))); driver.runTest(); }
/** * Tests quads to triples conversion * * @throws IOException */ @Test public void quads_to_triples_mapper_01() throws IOException { MapDriver<LongWritable, QuadWritable, LongWritable, TripleWritable> driver = this.getMapDriver(); Triple t = new Triple(NodeFactory.createURI("http://s"), NodeFactory.createURI("http://p"), NodeFactory.createLiteral("test")); Quad q = new Quad(Quad.defaultGraphNodeGenerated, t); driver.withInput(new Pair<LongWritable, QuadWritable>(new LongWritable(1), new QuadWritable(q))).withOutput( new Pair<LongWritable, TripleWritable>(new LongWritable(1), new TripleWritable(t))); driver.runTest(); }