@SuppressWarnings("unchecked") @Test public void testLeftOuterJoin() { List<List<Tuple2<String, String>>> stringStringKVStream1 = Arrays.asList( Arrays.asList(new Tuple2<>("california", "dodgers"), new Tuple2<>("new york", "yankees")), Arrays.asList(new Tuple2<>("california", "sharks") )); List<List<Tuple2<String, String>>> stringStringKVStream2 = Arrays.asList( Arrays.asList(new Tuple2<>("california", "giants") ), Arrays.asList(new Tuple2<>("new york", "islanders") ) ); List<List<Long>> expected = Arrays.asList(Arrays.asList(2L), Arrays.asList(1L)); JavaDStream<Tuple2<String, String>> stream1 = JavaTestUtils.attachTestInputStream( ssc, stringStringKVStream1, 1); JavaPairDStream<String, String> pairStream1 = JavaPairDStream.fromJavaDStream(stream1); JavaDStream<Tuple2<String, String>> stream2 = JavaTestUtils.attachTestInputStream( ssc, stringStringKVStream2, 1); JavaPairDStream<String, String> pairStream2 = JavaPairDStream.fromJavaDStream(stream2); JavaPairDStream<String, Tuple2<String, Optional<String>>> joined = pairStream1.leftOuterJoin(pairStream2); JavaDStream<Long> counted = joined.count(); JavaTestUtils.attachTestOutputStream(counted); List<List<Long>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testLeftOuterJoin() { List<List<Tuple2<String, String>>> stringStringKVStream1 = Arrays.asList( Arrays.asList(new Tuple2<>("california", "dodgers"), new Tuple2<>("new york", "yankees")), Arrays.asList(new Tuple2<>("california", "sharks") )); List<List<Tuple2<String, String>>> stringStringKVStream2 = Arrays.asList( Arrays.asList(new Tuple2<>("california", "giants") ), Arrays.asList(new Tuple2<>("new york", "islanders") ) ); List<List<Long>> expected = Arrays.asList(Arrays.asList(2L), Arrays.asList(1L)); JavaDStream<Tuple2<String, String>> stream1 = JavaTestUtils.attachTestInputStream( ssc, stringStringKVStream1, 1); JavaPairDStream<String, String> pairStream1 = JavaPairDStream.fromJavaDStream(stream1); JavaDStream<Tuple2<String, String>> stream2 = JavaTestUtils.attachTestInputStream( ssc, stringStringKVStream2, 1); JavaPairDStream<String, String> pairStream2 = JavaPairDStream.fromJavaDStream(stream2); JavaPairDStream<String, Tuple2<String, Optional<String>>> joined = pairStream1.leftOuterJoin(pairStream2); JavaDStream<Long> counted = joined.count(); JavaTestUtils.attachTestOutputStream(counted); List<List<Long>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@Test @SuppressWarnings("unchecked") public void javaAPI() { List<LabeledPoint> trainingBatch = Arrays.asList( new LabeledPoint(1.0, Vectors.dense(1.0)), new LabeledPoint(0.0, Vectors.dense(0.0))); JavaDStream<LabeledPoint> training = attachTestInputStream(ssc, Arrays.asList(trainingBatch, trainingBatch), 2); List<Tuple2<Integer, Vector>> testBatch = Arrays.asList( new Tuple2<>(10, Vectors.dense(1.0)), new Tuple2<>(11, Vectors.dense(0.0))); JavaPairDStream<Integer, Vector> test = JavaPairDStream.fromJavaDStream( attachTestInputStream(ssc, Arrays.asList(testBatch, testBatch), 2)); StreamingLinearRegressionWithSGD slr = new StreamingLinearRegressionWithSGD() .setNumIterations(2) .setInitialWeights(Vectors.dense(0.0)); slr.trainOn(training); JavaPairDStream<Integer, Double> prediction = slr.predictOnValues(test); attachTestOutputStream(prediction.count()); runStreams(ssc, 2, 2); } }
@Test @SuppressWarnings("unchecked") public void javaAPI() { List<LabeledPoint> trainingBatch = Arrays.asList( new LabeledPoint(1.0, Vectors.dense(1.0)), new LabeledPoint(0.0, Vectors.dense(0.0))); JavaDStream<LabeledPoint> training = attachTestInputStream(ssc, Arrays.asList(trainingBatch, trainingBatch), 2); List<Tuple2<Integer, Vector>> testBatch = Arrays.asList( new Tuple2<>(10, Vectors.dense(1.0)), new Tuple2<>(11, Vectors.dense(0.0))); JavaPairDStream<Integer, Vector> test = JavaPairDStream.fromJavaDStream( attachTestInputStream(ssc, Arrays.asList(testBatch, testBatch), 2)); StreamingLogisticRegressionWithSGD slr = new StreamingLogisticRegressionWithSGD() .setNumIterations(2) .setInitialWeights(Vectors.dense(0.0)); slr.trainOn(training); JavaPairDStream<Integer, Double> prediction = slr.predictOnValues(test); attachTestOutputStream(prediction.count()); runStreams(ssc, 2, 2); } }
@Test @SuppressWarnings("unchecked") public void javaAPI() { List<Vector> trainingBatch = Arrays.asList( Vectors.dense(1.0), Vectors.dense(0.0)); JavaDStream<Vector> training = attachTestInputStream(ssc, Arrays.asList(trainingBatch, trainingBatch), 2); List<Tuple2<Integer, Vector>> testBatch = Arrays.asList( new Tuple2<>(10, Vectors.dense(1.0)), new Tuple2<>(11, Vectors.dense(0.0))); JavaPairDStream<Integer, Vector> test = JavaPairDStream.fromJavaDStream( attachTestInputStream(ssc, Arrays.asList(testBatch, testBatch), 2)); StreamingKMeans skmeans = new StreamingKMeans() .setK(1) .setDecayFactor(1.0) .setInitialCenters(new Vector[]{Vectors.dense(1.0)}, new double[]{0.0}); skmeans.trainOn(training); JavaPairDStream<Integer, Integer> prediction = skmeans.predictOnValues(test); attachTestOutputStream(prediction.count()); runStreams(ssc, 2, 2); } }
@Test @SuppressWarnings("unchecked") public void javaAPI() { List<Vector> trainingBatch = Arrays.asList( Vectors.dense(1.0), Vectors.dense(0.0)); JavaDStream<Vector> training = attachTestInputStream(ssc, Arrays.asList(trainingBatch, trainingBatch), 2); List<Tuple2<Integer, Vector>> testBatch = Arrays.asList( new Tuple2<>(10, Vectors.dense(1.0)), new Tuple2<>(11, Vectors.dense(0.0))); JavaPairDStream<Integer, Vector> test = JavaPairDStream.fromJavaDStream( attachTestInputStream(ssc, Arrays.asList(testBatch, testBatch), 2)); StreamingKMeans skmeans = new StreamingKMeans() .setK(1) .setDecayFactor(1.0) .setInitialCenters(new Vector[]{Vectors.dense(1.0)}, new double[]{0.0}); skmeans.trainOn(training); JavaPairDStream<Integer, Integer> prediction = skmeans.predictOnValues(test); attachTestOutputStream(prediction.count()); runStreams(ssc, 2, 2); } }
@Test @SuppressWarnings("unchecked") public void javaAPI() { List<Vector> trainingBatch = Arrays.asList( Vectors.dense(1.0), Vectors.dense(0.0)); JavaDStream<Vector> training = attachTestInputStream(ssc, Arrays.asList(trainingBatch, trainingBatch), 2); List<Tuple2<Integer, Vector>> testBatch = Arrays.asList( new Tuple2<>(10, Vectors.dense(1.0)), new Tuple2<>(11, Vectors.dense(0.0))); JavaPairDStream<Integer, Vector> test = JavaPairDStream.fromJavaDStream( attachTestInputStream(ssc, Arrays.asList(testBatch, testBatch), 2)); StreamingKMeans skmeans = new StreamingKMeans() .setK(1) .setDecayFactor(1.0) .setInitialCenters(new Vector[]{Vectors.dense(1.0)}, new double[]{0.0}); skmeans.trainOn(training); JavaPairDStream<Integer, Integer> prediction = skmeans.predictOnValues(test); attachTestOutputStream(prediction.count()); runStreams(ssc, 2, 2); } }
@Test @SuppressWarnings("unchecked") public void javaAPI() { List<LabeledPoint> trainingBatch = Arrays.asList( new LabeledPoint(1.0, Vectors.dense(1.0)), new LabeledPoint(0.0, Vectors.dense(0.0))); JavaDStream<LabeledPoint> training = attachTestInputStream(ssc, Arrays.asList(trainingBatch, trainingBatch), 2); List<Tuple2<Integer, Vector>> testBatch = Arrays.asList( new Tuple2<>(10, Vectors.dense(1.0)), new Tuple2<>(11, Vectors.dense(0.0))); JavaPairDStream<Integer, Vector> test = JavaPairDStream.fromJavaDStream( attachTestInputStream(ssc, Arrays.asList(testBatch, testBatch), 2)); StreamingLogisticRegressionWithSGD slr = new StreamingLogisticRegressionWithSGD() .setNumIterations(2) .setInitialWeights(Vectors.dense(0.0)); slr.trainOn(training); JavaPairDStream<Integer, Double> prediction = slr.predictOnValues(test); attachTestOutputStream(prediction.count()); runStreams(ssc, 2, 2); } }
@Test @SuppressWarnings("unchecked") public void javaAPI() { List<LabeledPoint> trainingBatch = Arrays.asList( new LabeledPoint(1.0, Vectors.dense(1.0)), new LabeledPoint(0.0, Vectors.dense(0.0))); JavaDStream<LabeledPoint> training = attachTestInputStream(ssc, Arrays.asList(trainingBatch, trainingBatch), 2); List<Tuple2<Integer, Vector>> testBatch = Arrays.asList( new Tuple2<>(10, Vectors.dense(1.0)), new Tuple2<>(11, Vectors.dense(0.0))); JavaPairDStream<Integer, Vector> test = JavaPairDStream.fromJavaDStream( attachTestInputStream(ssc, Arrays.asList(testBatch, testBatch), 2)); StreamingLinearRegressionWithSGD slr = new StreamingLinearRegressionWithSGD() .setNumIterations(2) .setInitialWeights(Vectors.dense(0.0)); slr.trainOn(training); JavaPairDStream<Integer, Double> prediction = slr.predictOnValues(test); attachTestOutputStream(prediction.count()); runStreams(ssc, 2, 2); } }
@Test @SuppressWarnings("unchecked") public void javaAPI() { List<LabeledPoint> trainingBatch = Arrays.asList( new LabeledPoint(1.0, Vectors.dense(1.0)), new LabeledPoint(0.0, Vectors.dense(0.0))); JavaDStream<LabeledPoint> training = attachTestInputStream(ssc, Arrays.asList(trainingBatch, trainingBatch), 2); List<Tuple2<Integer, Vector>> testBatch = Arrays.asList( new Tuple2<>(10, Vectors.dense(1.0)), new Tuple2<>(11, Vectors.dense(0.0))); JavaPairDStream<Integer, Vector> test = JavaPairDStream.fromJavaDStream( attachTestInputStream(ssc, Arrays.asList(testBatch, testBatch), 2)); StreamingLogisticRegressionWithSGD slr = new StreamingLogisticRegressionWithSGD() .setNumIterations(2) .setInitialWeights(Vectors.dense(0.0)); slr.trainOn(training); JavaPairDStream<Integer, Double> prediction = slr.predictOnValues(test); attachTestOutputStream(prediction.count()); runStreams(ssc, 2, 2); } }
@Test @SuppressWarnings("unchecked") public void javaAPI() { List<LabeledPoint> trainingBatch = Arrays.asList( new LabeledPoint(1.0, Vectors.dense(1.0)), new LabeledPoint(0.0, Vectors.dense(0.0))); JavaDStream<LabeledPoint> training = attachTestInputStream(ssc, Arrays.asList(trainingBatch, trainingBatch), 2); List<Tuple2<Integer, Vector>> testBatch = Arrays.asList( new Tuple2<>(10, Vectors.dense(1.0)), new Tuple2<>(11, Vectors.dense(0.0))); JavaPairDStream<Integer, Vector> test = JavaPairDStream.fromJavaDStream( attachTestInputStream(ssc, Arrays.asList(testBatch, testBatch), 2)); StreamingLinearRegressionWithSGD slr = new StreamingLinearRegressionWithSGD() .setNumIterations(2) .setInitialWeights(Vectors.dense(0.0)); slr.trainOn(training); JavaPairDStream<Integer, Double> prediction = slr.predictOnValues(test); attachTestOutputStream(prediction.count()); runStreams(ssc, 2, 2); } }