@Before public void setUp() { CubeHFileMapper mapper = new CubeHFileMapper(); mapDriver = MapDriver.newMapDriver(mapper); }
@Before @SuppressWarnings({ "rawtypes", "unchecked" }) public void setUp() { RandomKeyDistributionMapper mapper = new RandomKeyDistributionMapper(); mapDriver = MapDriver.newMapDriver(mapper); }
@SuppressWarnings({ "rawtypes", "unchecked" }) @Before public void setUp() { ColumnCardinalityMapper mapper = new ColumnCardinalityMapper(); mapDriver = MapDriver.newMapDriver(mapper); }
@Before public void setUp() { RangeKeyDistributionMapper mapper = new RangeKeyDistributionMapper(); mapDriver = MapDriver.newMapDriver(mapper); }
@Before public void setUp() { CubeHFileMapper mapper = new CubeHFileMapper(); mapDriver = MapDriver.newMapDriver(mapper); }
@Before public void setUp() throws Exception { createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("../job/meta")); FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta")); BaseCuboidMapper<Text> mapper = new BaseCuboidMapper<Text>(); mapDriver = MapDriver.newMapDriver(mapper); }
@Override public MapDriver initializeTestDriver(Mapper mapper, Reducer reducer) { return MapDriver.newMapDriver(mapper); } }
mapDriver = MapDriver.newMapDriver(mapper);
MapDriver mapDriver = MapDriver.newMapDriver(mapper); mapDriver.setCounters(counters); mapDriver.setConfiguration(getConfiguration());
@Override public List<Pair<K3, V3>> run() throws IOException { try { preRunChecks(myMapper, myReducer); initDistributedCache(); List<Pair<K2, V2>> mapOutputs = new ArrayList<Pair<K2, V2>>(); // run map component LOG.debug("Starting map phase with mapper: " + myMapper); mapOutputs.addAll(MapDriver.newMapDriver(myMapper) .withCounters(getCounters()).withConfiguration(getConfiguration()) .withAll(inputList).withMapInputPath(getMapInputPath()).run()); if (myCombiner != null) { // User has specified a combiner. Run this and replace the mapper // outputs // with the result of the combiner. LOG.debug("Starting combine phase with combiner: " + myCombiner); mapOutputs = new ReducePhaseRunner<K2, V2, K2, V2>(inputFormatClass, getConfiguration(), counters, getOutputSerializationConfiguration(), outputFormatClass) .runReduce(sortAndGroup(mapOutputs), myCombiner); } // Run the reduce phase. LOG.debug("Starting reduce phase with reducer: " + myReducer); return new ReducePhaseRunner<K2, V2, K3, V3>(inputFormatClass, getConfiguration(), counters, getOutputSerializationConfiguration(), outputFormatClass).runReduce(sortAndGroup(mapOutputs), myReducer); } finally { cleanupDistributedCache(); } }
@Test public void testHypercubeMapper() throws IOException { MapDriver<Writable, VectorWritable, IntWritable, CentroidWritable> mapDriver = MapDriver.newMapDriver(new StreamingKMeansMapper()); configure(mapDriver.getConfiguration()); System.out.printf("%s mapper test\n", mapDriver.getConfiguration().get(StreamingKMeansDriver.SEARCHER_CLASS_OPTION)); for (Centroid datapoint : syntheticData.getFirst()) { mapDriver.addInput(new IntWritable(0), new VectorWritable(datapoint)); } List<org.apache.hadoop.mrunit.types.Pair<IntWritable,CentroidWritable>> results = mapDriver.run(); BruteSearch resultSearcher = new BruteSearch(new SquaredEuclideanDistanceMeasure()); for (org.apache.hadoop.mrunit.types.Pair<IntWritable, CentroidWritable> result : results) { resultSearcher.add(result.getSecond().getCentroid()); } System.out.printf("Clustered the data into %d clusters\n", results.size()); for (Vector mean : syntheticData.getSecond()) { WeightedThing<Vector> closest = resultSearcher.search(mean, 1).get(0); assertTrue("Weight " + closest.getWeight() + " not less than 0.5", closest.getWeight() < 0.5); } }
MapDriver.newMapDriver(new StreamingKMeansMapper()); Configuration configuration = mapDriver.getConfiguration(); configure(configuration);