throw new IllegalStateException("No saved network parameters at " + f.getAbsolutePath()); Arrays.sort(content); testDataPaths.cache();
trainingData.cache(); // Cache data since Logistic Regression is an iterative algorithm.
parsedTestRDD.cache();
@Test public void testJavaJdbcRDD() throws Exception { JavaRDD<Integer> rdd = JdbcRDD.create( sc, () -> DriverManager.getConnection("jdbc:derby:target/JavaJdbcRDDSuiteDb"), "SELECT DATA FROM FOO WHERE ? <= ID AND ID <= ?", 1, 100, 1, r -> r.getInt(1) ).cache(); Assert.assertEquals(100, rdd.count()); Assert.assertEquals(Integer.valueOf(10100), rdd.reduce((i1, i2) -> i1 + i2)); } }
@Test public void testJavaJdbcRDD() throws Exception { JavaRDD<Integer> rdd = JdbcRDD.create( sc, () -> DriverManager.getConnection("jdbc:derby:target/JavaJdbcRDDSuiteDb"), "SELECT DATA FROM FOO WHERE ? <= ID AND ID <= ?", 1, 100, 1, r -> r.getInt(1) ).cache(); Assert.assertEquals(100, rdd.count()); Assert.assertEquals(Integer.valueOf(10100), rdd.reduce((i1, i2) -> i1 + i2)); } }
@Test public void testJavaJdbcRDD() throws Exception { JavaRDD<Integer> rdd = JdbcRDD.create( sc, () -> DriverManager.getConnection("jdbc:derby:target/JavaJdbcRDDSuiteDb"), "SELECT DATA FROM FOO WHERE ? <= ID AND ID <= ?", 1, 100, 1, r -> r.getInt(1) ).cache(); Assert.assertEquals(100, rdd.count()); Assert.assertEquals(Integer.valueOf(10100), rdd.reduce((i1, i2) -> i1 + i2)); } }
@Test public void testGetPersistentRDDs() { java.util.Map<Integer, JavaRDD<?>> cachedRddsMap = sc.getPersistentRDDs(); assertTrue(cachedRddsMap.isEmpty()); JavaRDD<String> rdd1 = sc.parallelize(Arrays.asList("a", "b")).setName("RDD1").cache(); JavaRDD<String> rdd2 = sc.parallelize(Arrays.asList("c", "d")).setName("RDD2").cache(); cachedRddsMap = sc.getPersistentRDDs(); assertEquals(2, cachedRddsMap.size()); assertEquals("RDD1", cachedRddsMap.get(0).name()); assertEquals("RDD2", cachedRddsMap.get(1).name()); }
@Test public void testGetPersistentRDDs() { java.util.Map<Integer, JavaRDD<?>> cachedRddsMap = sc.getPersistentRDDs(); assertTrue(cachedRddsMap.isEmpty()); JavaRDD<String> rdd1 = sc.parallelize(Arrays.asList("a", "b")).setName("RDD1").cache(); JavaRDD<String> rdd2 = sc.parallelize(Arrays.asList("c", "d")).setName("RDD2").cache(); cachedRddsMap = sc.getPersistentRDDs(); assertEquals(2, cachedRddsMap.size()); assertEquals("RDD1", cachedRddsMap.get(0).name()); assertEquals("RDD2", cachedRddsMap.get(1).name()); }
@Test public void testGetPersistentRDDs() { java.util.Map<Integer, JavaRDD<?>> cachedRddsMap = sc.getPersistentRDDs(); assertTrue(cachedRddsMap.isEmpty()); JavaRDD<String> rdd1 = sc.parallelize(Arrays.asList("a", "b")).setName("RDD1").cache(); JavaRDD<String> rdd2 = sc.parallelize(Arrays.asList("c", "d")).setName("RDD2").cache(); cachedRddsMap = sc.getPersistentRDDs(); assertEquals(2, cachedRddsMap.size()); assertEquals("RDD1", cachedRddsMap.get(0).name()); assertEquals("RDD2", cachedRddsMap.get(1).name()); }
parsedRDD.cache();
@Test public void map() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5)); JavaDoubleRDD doubles = rdd.mapToDouble(x -> 1.0 * x).cache(); doubles.collect(); JavaPairRDD<Integer, Integer> pairs = rdd.mapToPair(x -> new Tuple2<>(x, x)) .cache(); pairs.collect(); JavaRDD<String> strings = rdd.map(Object::toString).cache(); strings.collect(); }
@Test public void map() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5)); JavaDoubleRDD doubles = rdd.mapToDouble(Integer::doubleValue).cache(); doubles.collect(); JavaPairRDD<Integer, Integer> pairs = rdd.mapToPair(x -> new Tuple2<>(x, x)).cache(); pairs.collect(); JavaRDD<String> strings = rdd.map(Object::toString).cache(); strings.collect(); }
@Test public void map() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5)); JavaDoubleRDD doubles = rdd.mapToDouble(Integer::doubleValue).cache(); doubles.collect(); JavaPairRDD<Integer, Integer> pairs = rdd.mapToPair(x -> new Tuple2<>(x, x)).cache(); pairs.collect(); JavaRDD<String> strings = rdd.map(Object::toString).cache(); strings.collect(); }
@Test public void map() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5)); JavaDoubleRDD doubles = rdd.mapToDouble(x -> 1.0 * x).cache(); doubles.collect(); JavaPairRDD<Integer, Integer> pairs = rdd.mapToPair(x -> new Tuple2<>(x, x)) .cache(); pairs.collect(); JavaRDD<String> strings = rdd.map(Object::toString).cache(); strings.collect(); }
@Test public void map() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5)); JavaDoubleRDD doubles = rdd.mapToDouble(Integer::doubleValue).cache(); doubles.collect(); JavaPairRDD<Integer, Integer> pairs = rdd.mapToPair(x -> new Tuple2<>(x, x)).cache(); pairs.collect(); JavaRDD<String> strings = rdd.map(Object::toString).cache(); strings.collect(); }
@Test public void map() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5)); JavaDoubleRDD doubles = rdd.mapToDouble(x -> 1.0 * x).cache(); doubles.collect(); JavaPairRDD<Integer, Integer> pairs = rdd.mapToPair(x -> new Tuple2<>(x, x)) .cache(); pairs.collect(); JavaRDD<String> strings = rdd.map(Object::toString).cache(); strings.collect(); }
processed.cache();
static JavaRDD<Vector> getFeatureizedData(String wikiData, JavaSparkContext context) { JavaRDD<Vector> data = context.textFile(wikiData).map(new Function<String, Vector>() { @Override public Vector call(String arg0) throws Exception { return Util.buildVector(arg0, "\t"); } }).cache(); return data; }