@Before public void setUp() { ColumnCardinalityReducer reducer = new ColumnCardinalityReducer(); reduceDriver = ReduceDriver.newReduceDriver(reducer); }
@Before public void setUp() { CubeHFileMapper mapper = new CubeHFileMapper(); mapDriver = MapDriver.newMapDriver(mapper); }
values5.add(new BytesWritable(getBytes(strArr + " xxxx"))); reduceDriver.withInput(key1, values1); reduceDriver.withInput(key2, values2); reduceDriver.withInput(key3, values3); reduceDriver.withInput(key4, values4); reduceDriver.withInput(key5, values5); List<Pair<IntWritable, LongWritable>> result = reduceDriver.run();
@Test public void testReducer() throws Exception { reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready"); CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); BufferedMeasureCodec codec = new BufferedMeasureCodec(cubeDesc.getMeasures()); Text key1 = new Text("72010ustech"); List<Text> values1 = new ArrayList<Text>(); values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 100)); values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 200)); values1.add(newValueText(codec, "10", "10", "10", 1, 300)); Text key2 = new Text("1tech"); List<Text> values2 = new ArrayList<Text>(); values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 500)); values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 1000)); Text key3 = new Text("0"); List<Text> values3 = new ArrayList<Text>(); values3.add(newValueText(codec, "146.52", "146.52", "146.52", 0, 0)); reduceDriver.withInput(key1, values1); reduceDriver.withInput(key2, values2); reduceDriver.withInput(key3, values3); List<Pair<Text, Text>> result = reduceDriver.run(); Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "45.43", "10", "20.34", 3, 600)); Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "35.43", "15.09", "20.34", 2, 1500)); Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "146.52", "146.52", "146.52", 0, 0)); assertEquals(3, result.size()); assertTrue(result.contains(p1)); assertTrue(result.contains(p2)); assertTrue(result.contains(p3)); }
@SuppressWarnings("deprecation") @Test @Ignore("not maintaining") public void testMapper2() throws IOException { mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name); mapDriver.addInput(new Text("52010tech"), new Text("35.432")); List<Pair<RowKeyWritable, KeyValue>> result = mapDriver.run(); assertEquals(2, result.size()); byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 }; ImmutableBytesWritable key = new ImmutableBytesWritable(bytes); Pair<RowKeyWritable, KeyValue> p1 = result.get(0); Pair<RowKeyWritable, KeyValue> p2 = result.get(1); assertEquals(key, p1.getFirst()); assertEquals("cf1", new String(p1.getSecond().getFamily(), StandardCharsets.UTF_8)); assertEquals("usd_amt", new String(p1.getSecond().getQualifier(), StandardCharsets.UTF_8)); assertEquals("35.43", new String(p1.getSecond().getValue(), StandardCharsets.UTF_8)); assertEquals(key, p2.getFirst()); assertEquals("cf1", new String(p2.getSecond().getFamily(), StandardCharsets.UTF_8)); assertEquals("item_count", new String(p2.getSecond().getQualifier(), StandardCharsets.UTF_8)); assertEquals("2", new String(p2.getSecond().getValue(), StandardCharsets.UTF_8)); } }
mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName); mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID); Pair<Text, Text> input1 = new Pair<Text, Text>(new Text(key), new Text(value)); mapReduceDriver.addInput(input1); List<Pair<Text, Text>> result = mapReduceDriver.run();
@SuppressWarnings("unchecked") @Test public void testMapperWithHeader() throws IOException { Text inputKey1 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey2 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey3 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey4 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey5 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey6 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey7 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); mapDriver.addInput(inputKey1, new Text("abc")); mapDriver.addInput(inputKey2, new Text("abc")); mapDriver.addInput(inputKey3, new Text("abc")); mapDriver.addInput(inputKey4, new Text("abc")); mapDriver.addInput(inputKey5, new Text("abc")); mapDriver.addInput(inputKey6, new Text("abc")); mapDriver.addInput(inputKey7, new Text("abc")); List<Pair<Text, LongWritable>> result = mapDriver.run(); assertEquals(1, result.size()); byte[] key1 = result.get(0).getFirst().getBytes(); LongWritable value1 = result.get(0).getSecond(); assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }, key1); assertEquals(273, value1.get()); }
@Before public void setUp() throws Exception { createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("./meta")); FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta")); NDCuboidMapper mapper = new NDCuboidMapper(); CuboidReducer reducer = new CuboidReducer(); mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer); }
@Test public void testReducerOnlyAggrInBaseCuboid() throws Exception { reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready"); reduceDriver.getConfiguration().setInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, 1); values3.add(newValueText(codec, "146.52", "146.52", "146.52", 0, 0)); reduceDriver.withInput(key1, values1); reduceDriver.withInput(key2, values2); reduceDriver.withInput(key3, values3); List<Pair<Text, Text>> result = reduceDriver.run();
@Before public void setUp() throws Exception { createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("../job/meta")); FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("../job/meta")); CuboidReducer reducer = new CuboidReducer(); reduceDriver = ReduceDriver.newReduceDriver(reducer); }
@SuppressWarnings({ "rawtypes", "unchecked" }) @Before public void setUp() { ColumnCardinalityMapper mapper = new ColumnCardinalityMapper(); mapDriver = MapDriver.newMapDriver(mapper); }
@Before public void setUp() throws Exception { createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("../job/meta")); FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta")); NDCuboidMapper mapper = new NDCuboidMapper(); CuboidReducer reducer = new CuboidReducer(); mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer); }
@Before @SuppressWarnings({ "rawtypes", "unchecked" }) public void setUp() { RandomKeyDistributionReducer reducer = new RandomKeyDistributionReducer(); reduceDriver = ReduceDriver.newReduceDriver(reducer); }
@Before @SuppressWarnings({ "rawtypes", "unchecked" }) public void setUp() { RandomKeyDistributionMapper mapper = new RandomKeyDistributionMapper(); mapDriver = MapDriver.newMapDriver(mapper); }
@Before public void setUp() { RangeKeyDistributionReducer reducer = new RangeKeyDistributionReducer(); reduceDriver = ReduceDriver.newReduceDriver(reducer); }
@Before public void setUp() { RangeKeyDistributionMapper mapper = new RangeKeyDistributionMapper(); mapDriver = MapDriver.newMapDriver(mapper); }
@Before public void setUp() { ColumnCardinalityReducer reducer = new ColumnCardinalityReducer(); reduceDriver = ReduceDriver.newReduceDriver(reducer); }
@Before public void setUp() { CubeHFileMapper mapper = new CubeHFileMapper(); mapDriver = MapDriver.newMapDriver(mapper); }
@Before public void setUp() throws Exception { createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("../job/meta")); FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta")); CuboidReducer reducer = new CuboidReducer(); reduceDriver = ReduceDriver.newReduceDriver(reducer); }
@Before public void setUp() throws Exception { createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("../job/meta")); FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta")); BaseCuboidMapper<Text> mapper = new BaseCuboidMapper<Text>(); mapDriver = MapDriver.newMapDriver(mapper); }