private String toUpdateJSON(String matrix, String ID, float[] vector, String otherID) { List<?> args; if (noKnownItems) { args = Arrays.asList(matrix, ID, vector); } else { args = Arrays.asList(matrix, ID, vector, Collections.singletonList(otherID)); } return TextUtils.joinJSON(args); }
@Override public void call(Iterator<Tuple2<String,float[]>> it) { if (it.hasNext()) { try (TopicProducer<String,String> producer = new TopicProducerImpl<>(updateBroker, topic, true)) { it.forEachRemaining(keyAndVector -> { String id = keyAndVector._1(); float[] vector = keyAndVector._2(); producer.send("UP", TextUtils.joinJSON(Arrays.asList(whichMatrix, id, vector))); }); } } }
private static void saveFeaturesRDD(JavaPairRDD<Integer,float[]> features, Path path, Broadcast<? extends Map<Integer,String>> bIndexToID) { log.info("Saving features RDD to {}", path); features.map(keyAndVector -> { String id = bIndexToID.value().get(keyAndVector._1()); float[] vector = keyAndVector._2(); return TextUtils.joinJSON(Arrays.asList(id, vector)); }).saveAsTextFile(path.toString(), GzipCodec.class); }
@Override public void call(Iterator<Tuple2<String,Tuple2<float[],Collection<String>>>> it) { if (it.hasNext()) { try (TopicProducer<String, String> producer = new TopicProducerImpl<>(updateBroker, topic, true)) { it.forEachRemaining(keyAndVectorAndItems -> { String id = keyAndVectorAndItems._1(); Tuple2<float[],Collection<String>> vectorAndItems = keyAndVectorAndItems._2(); float[] vector = vectorAndItems._1(); Collection<String> knowItemIDs = vectorAndItems._2(); Collection<?> data = knowItemIDs.isEmpty() ? Arrays.asList(whichMatrix, id, vector) : Arrays.asList(whichMatrix, id, vector, knowItemIDs); producer.send("UP", TextUtils.joinJSON(data)); }); } } }
@Test public void testJSONList() { List<Object> list = new ArrayList<>(); list.add("foo"); list.add(2); assertEquals("[\"A\",[\"foo\",2],\"B\"]", TextUtils.joinJSON(Arrays.asList("A", list, "B"))); }
@Test public void testJSONMap() { Map<Object,Object> map = new HashMap<>(); map.put(1, "bar"); map.put("foo", 2); assertEquals("[\"A\",{\"1\":\"bar\",\"foo\":2},\"B\"]", TextUtils.joinJSON(Arrays.asList("A", map, "B"))); }
@Override public Pair<String, String> generate(int id, RandomGenerator random) { return new Pair<>(Integer.toString(id), TextUtils.joinJSON(arrayToList(UPDATE_POINTS[id % UPDATE_POINTS.length]))); }
@Override public Pair<String,String> generate(int id, RandomGenerator random) { if (id == 0) { PMML pmml = RDFPMMLUtilsTest.buildDummyClassificationModel(); return new Pair<>("MODEL", PMMLUtils.toString(pmml)); } else { String nodeID = "r" + ((id % 2 == 0) ? '-' : '+'); Map<Integer,Integer> counts = new HashMap<>(); counts.put(0, 1); counts.put(1, 2); return new Pair<>("UP", TextUtils.joinJSON(Arrays.asList(0, nodeID, counts))); } }
@Override public Pair<String,String> generate(int id, RandomGenerator random) { if (id == 0) { PMML pmml = KMeansPMMLUtilsTest.buildDummyClusteringModel(); return new Pair<>("MODEL", PMMLUtils.toString(pmml)); } else { List<?> data = Arrays.asList(id % 3, Arrays.asList(id, id), id); return new Pair<>("UP", TextUtils.joinJSON(data)); } }
return TextUtils.joinJSON(Arrays.asList(treeID, nodeID, p._2())); }).collect(Collectors.toList()); String nodeID = p._1().getSecond(); DoubleSummaryStatistics stats = p._2(); return TextUtils.joinJSON(Arrays.asList(treeID, nodeID, stats.getAverage(), stats.getCount())); }).collect(Collectors.toList());
return TextUtils.joinJSON(Arrays.asList(clusterID, clusterInfo.getCenter(), clusterInfo.getCount())); }).collect(Collectors.toList());
@Override public Pair<String,String> generate(int id, RandomGenerator random) { if (id % 10 == 0) { PMML pmml = PMMLUtils.buildSkeletonPMML(); AppPMMLUtils.addExtension(pmml, "features", 2); AppPMMLUtils.addExtension(pmml, "implicit", true); AppPMMLUtils.addExtensionContent(pmml, "XIDs", X.keySet()); AppPMMLUtils.addExtensionContent(pmml, "YIDs", Y.keySet()); return new Pair<>("MODEL", PMMLUtils.toString(pmml)); } else { int xOrYID = id % 10; String xOrYIDString = ALSUtilsTest.idToStringID(id); String message; boolean isX = xOrYID >= 6; if (isX) { message = TextUtils.joinJSON(Arrays.asList( "X", xOrYIDString, X.get(xOrYIDString), A.get(xOrYIDString))); } else { message = TextUtils.joinJSON(Arrays.asList( "Y", xOrYIDString, Y.get(xOrYIDString), At.get(xOrYIDString))); } return new Pair<>("UP", message); } }
private String toUpdateJSON(String matrix, String ID, float[] vector, String otherID) { List<?> args; if (noKnownItems) { args = Arrays.asList(matrix, ID, vector); } else { args = Arrays.asList(matrix, ID, vector, Collections.singletonList(otherID)); } return TextUtils.joinJSON(args); }
return TextUtils.joinJSON(Arrays.asList(treeID, nodeID, p._2())); }).collect(Collectors.toList()); String nodeID = p._1().getSecond(); DoubleSummaryStatistics stats = p._2(); return TextUtils.joinJSON(Arrays.asList(treeID, nodeID, stats.getAverage(), stats.getCount())); }).collect(Collectors.toList());
return TextUtils.joinJSON(Arrays.asList(clusterID, clusterInfo.getCenter(), clusterInfo.getCount())); }).collect(Collectors.toList());