Refine search
static Tuple2<String, CallLog[]> fetchResultFromRequest(ObjectMapper mapper, Tuple2<String, ContentExchange> signExchange) { String sign = signExchange._1(); ContentExchange exchange = signExchange._2(); return new Tuple2(sign, readExchangeCallLog(mapper, exchange)); } static Tuple2<String, ContentExchange> createRequestForSign(String sign, HttpClient client) throws Exception {
public Tuple2<String, Integer> call(Tuple2<String, Integer> callSignCount) { String sign = callSignCount._1(); String country = lookupCountry(sign, signPrefixes.value()); return new Tuple2(country, callSignCount._2()); }}).reduceByKey(new SumInts()); countryContactCounts.saveAsTextFile(outputDir + "/countries.txt");
@Override protected void processNextRecord(Tuple2<HiveKey, V> inputRecord) throws IOException { HiveKey key = inputRecord._1(); V value = inputRecord._2(); if (value instanceof Iterable) { @SuppressWarnings("unchecked") Iterable<BytesWritable> values = (Iterable<BytesWritable>)value; reduceRecordHandler.<BytesWritable>processRow(key, values.iterator()); } else { reduceRecordHandler.processRow(key, value); } }
@Override public void call(Iterator<Tuple2<String,Tuple2<float[],Collection<String>>>> it) { if (it.hasNext()) { try (TopicProducer<String, String> producer = new TopicProducerImpl<>(updateBroker, topic, true)) { it.forEachRemaining(keyAndVectorAndItems -> { String id = keyAndVectorAndItems._1(); Tuple2<float[],Collection<String>> vectorAndItems = keyAndVectorAndItems._2(); float[] vector = vectorAndItems._1(); Collection<String> knowItemIDs = vectorAndItems._2(); Collection<?> data = knowItemIDs.isEmpty() ? Arrays.asList(whichMatrix, id, vector) : Arrays.asList(whichMatrix, id, vector, knowItemIDs); producer.send("UP", TextUtils.joinJSON(data)); }); } } }
@Override public void call(Iterator<Tuple2<String,float[]>> it) { if (it.hasNext()) { try (TopicProducer<String,String> producer = new TopicProducerImpl<>(updateBroker, topic, true)) { it.forEachRemaining(keyAndVector -> { String id = keyAndVector._1(); float[] vector = keyAndVector._2(); producer.send("UP", TextUtils.joinJSON(Arrays.asList(whichMatrix, id, vector))); }); } } }
@Override protected void processNextRecord(Tuple2<HiveKey, V> inputRecord) throws IOException { HiveKey key = inputRecord._1(); V value = inputRecord._2(); if (value instanceof Iterable) { @SuppressWarnings("unchecked") Iterable<BytesWritable> values = (Iterable<BytesWritable>)value; reduceRecordHandler.<BytesWritable>processRow(key, values.iterator()); } else { reduceRecordHandler.processRow(key, value); } }
private static void saveFeaturesRDD(JavaPairRDD<Integer,float[]> features, Path path, Broadcast<? extends Map<Integer,String>> bIndexToID) { log.info("Saving features RDD to {}", path); features.map(keyAndVector -> { String id = bIndexToID.value().get(keyAndVector._1()); float[] vector = keyAndVector._2(); return TextUtils.joinJSON(Arrays.asList(id, vector)); }).saveAsTextFile(path.toString(), GzipCodec.class); }
/** * Computes root mean squared error of {@link Rating#rating()} versus predicted value. */ static double rmse(MatrixFactorizationModel mfModel, JavaRDD<Rating> testData) { JavaPairRDD<Tuple2<Integer,Integer>,Double> testUserProductValues = testData.mapToPair(rating -> new Tuple2<>(new Tuple2<>(rating.user(), rating.product()), rating.rating())); @SuppressWarnings("unchecked") RDD<Tuple2<Object,Object>> testUserProducts = (RDD<Tuple2<Object,Object>>) (RDD<?>) testUserProductValues.keys().rdd(); JavaRDD<Rating> predictions = testData.wrapRDD(mfModel.predict(testUserProducts)); double mse = predictions.mapToPair( rating -> new Tuple2<>(new Tuple2<>(rating.user(), rating.product()), rating.rating()) ).join(testUserProductValues).values().mapToDouble(valuePrediction -> { double diff = valuePrediction._1() - valuePrediction._2(); return diff * diff; }).mean(); return Math.sqrt(mse); }
@Test public void testFunction() { WritableToValueFunction<String,String> function = new WritableToValueFunction<>(String.class, String.class, Text.class, Text.class); Tuple2<Writable,Writable> in = new Tuple2<>(new Text("bizz"), new Text("buzz")); Tuple2<String,String> out = function.call(in); assertEquals("bizz", out._1()); assertEquals("buzz", out._2()); }
@Test public void testNullKey() { ValueToWritableFunction<String,String> function = new ValueToWritableFunction<>(String.class, String.class, Text.class, Text.class); Tuple2<Writable,Writable> keyMessage = function.call(new Tuple2<>(null, "foo")); assertEquals(new Text(), keyMessage._1()); assertEquals(new Text("foo"), keyMessage._2()); keyMessage = function.call(new Tuple2<>("foo", null)); assertEquals(new Text("foo"), keyMessage._1()); assertEquals(new Text(), keyMessage._2()); }