/** * Returns a new MapDriver without having to specify the generic types on the * right hand side of the object create statement. * * @param mapper * passed to MapDriver constructor * @return new MapDriver */ public static <K1, V1, K2, V2> MapDriver<K1, V1, K2, V2> newMapDriver( final Mapper<K1, V1, K2, V2> mapper) { return new MapDriver<K1, V1, K2, V2>(mapper); } }
/** * Returns a new MapDriver without having to specify the generic types on the * right hand side of the object create statement. * * @return new MapDriver */ public static <K1, V1, K2, V2> MapDriver<K1, V1, K2, V2> newMapDriver() { return new MapDriver<K1, V1, K2, V2>(); }
/** * Gets a map driver that can be used to create a test case * * @return Map driver */ protected MapDriver<TKeyIn, TValueIn, TKeyOut, TValueOut> getMapDriver() { MapDriver<TKeyIn, TValueIn, TKeyOut, TValueOut> driver = new MapDriver<TKeyIn, TValueIn, TKeyOut, TValueOut>( this.getInstance()); this.configureDriver(driver); return driver; }
public List<Pair<K3, V3>> run() throws IOException { List<Pair<K2, V2>> mapOutputs = new ArrayList<Pair<K2, V2>>(); // run map component for (Pair<K1, V1> input : inputList) { LOG.debug("Mapping input " + input.toString() + ")"); mapOutputs.addAll(new MapDriver<K1, V1, K2, V2>(myMapper).withInput( input).withCounters(getCounters()).withConfiguration(configuration).run()); } List<Pair<K2, List<V2>>> reduceInputs = shuffle(mapOutputs); List<Pair<K3, V3>> reduceOutputs = new ArrayList<Pair<K3, V3>>(); for (Pair<K2, List<V2>> input : reduceInputs) { K2 inputKey = input.getFirst(); List<V2> inputValues = input.getSecond(); StringBuilder sb = new StringBuilder(); formatValueList(inputValues, sb); LOG.debug("Reducing input (" + inputKey.toString() + ", " + sb.toString() + ")"); reduceOutputs.addAll(new ReduceDriver<K2, V2, K3, V3>(myReducer) .withCounters(getCounters()).withConfiguration(configuration) .withInputKey(inputKey).withInputValues(inputValues).run()); } return reduceOutputs; }
@Before public void setUp() { long eventTime = System.currentTimeMillis(); EventMapper<LongWritable,RawRecordContainer,BulkIngestKey,Value> mapper = new EventMapper<>(); driver = new MapDriver<>(mapper); conf = driver.getConfiguration(); conf.setClass(EventMapper.CONTEXT_WRITER_CLASS, TestContextWriter.class, ContextWriter.class); Type type = new Type("file", null, null, new String[] {SimpleDataTypeHandler.class.getName()}, 10, null); TypeRegistry registry = TypeRegistry.getInstance(conf); registry.put(type.typeName(), type); Multimap<String,NormalizedContentInterface> fields = HashMultimap.create(); fields.put("fileExtension", new BaseNormalizedContent("fileExtension", "gz")); fields.put("lastModified", new BaseNormalizedContent("lastModified", "2016-01-01")); SimpleDataTypeHelper.registerFields(fields); record = new SimpleRawRecord(); record.setRawFileTimestamp(eventTime); record.setDataType(type); record.setDate(eventTime); record.setRawFileName("/some/filename"); record.setRawData("some data".getBytes()); record.generateId(null); }