public static void main(String[] args) throws Exception { String master; if (args.length > 0) { master = args[0]; } else { master = "local"; } JavaSparkContext sc = new JavaSparkContext( master, "basicmap", System.getenv("SPARK_HOME"), System.getenv("JARS")); JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4)); Integer result = rdd.fold(0, new Function2<Integer, Integer, Integer>() { public Integer call(Integer x, Integer y) { return x + y;}}); System.out.println(result); } }
@Test public void fold() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4)); int sum = rdd.fold(0, new AddInts()); assertEquals(10, sum); }
@Test public void fold() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4)); int sum = rdd.fold(0, new AddInts()); assertEquals(10, sum); }
@Test public void fold() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4)); int sum = rdd.fold(0, new AddInts()); assertEquals(10, sum); }
@Test public void foldReduce() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13)); Function2<Integer, Integer, Integer> add = (a, b) -> a + b; int sum = rdd.fold(0, add); Assert.assertEquals(33, sum); sum = rdd.reduce(add); Assert.assertEquals(33, sum); }
@Test public void foldReduce() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13)); Function2<Integer, Integer, Integer> add = (a, b) -> a + b; int sum = rdd.fold(0, add); assertEquals(33, sum); sum = rdd.reduce(add); assertEquals(33, sum); }
@Test public void foldReduce() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13)); Function2<Integer, Integer, Integer> add = (a, b) -> a + b; int sum = rdd.fold(0, add); Assert.assertEquals(33, sum); sum = rdd.reduce(add); Assert.assertEquals(33, sum); }
@Test public void foldReduce() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13)); Function2<Integer, Integer, Integer> add = (a, b) -> a + b; int sum = rdd.fold(0, add); Assert.assertEquals(33, sum); sum = rdd.reduce(add); Assert.assertEquals(33, sum); }
@Test public void foldReduce() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13)); Function2<Integer, Integer, Integer> add = (a, b) -> a + b; int sum = rdd.fold(0, add); assertEquals(33, sum); sum = rdd.reduce(add); assertEquals(33, sum); }
@Test public void foldReduce() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 1, 2, 3, 5, 8, 13)); Function2<Integer, Integer, Integer> add = (a, b) -> a + b; int sum = rdd.fold(0, add); assertEquals(33, sum); sum = rdd.reduce(add); assertEquals(33, sum); }
result = nextRDD.map(Traverser::bulk).fold(0L, (a, b) -> a + b); else if (endStep instanceof SumGlobalStep) { result = nextRDD.isEmpty() ? null : nextRDD .map(traverser -> NumberHelper.mul(traverser.bulk(), (Number) traverser.get())) .fold(0, NumberHelper::add); } else if (endStep instanceof MeanGlobalStep) { result = nextRDD.isEmpty() ? null : nextRDD .map(traverser -> new MeanGlobalStep.MeanNumber((Number) traverser.get(), traverser.bulk())) .fold(MeanNumberSupplier.instance().get(), MeanGlobalStep.MeanNumber::add) .getFinal(); } else if (endStep instanceof MinGlobalStep) { result = nextRDD.isEmpty() ? null : nextRDD .map(traverser -> (Comparable) traverser.get()) .fold(Double.NaN, NumberHelper::min); } else if (endStep instanceof MaxGlobalStep) { result = nextRDD.isEmpty() ? null : nextRDD .map(traverser -> (Comparable) traverser.get()) .fold(Double.NaN, NumberHelper::max); } else if (endStep instanceof FoldStep) { final BinaryOperator biOperator = endStep.getBiOperator(); }).fold(endStep.getSeedSupplier().get(), biOperator::apply); } else if (endStep instanceof GroupStep) { final GroupStep.GroupBiOperator<Object, Object> biOperator = (GroupStep.GroupBiOperator) endStep.getBiOperator(); final GroupStep<Object, Object, Object> clone = (GroupStep) endStep.clone(); return IteratorUtils.map(partitions, clone::projectTraverser); }).fold(((GroupStep<Object, Object, Object>) endStep).getSeedSupplier().get(), biOperator::apply)); } else if (endStep instanceof GroupCountStep) {
@Override public T fold(T zeroValue, @NonNull SerializableBinaryOperator<T> operator) { return rdd.fold(zeroValue, (t, u) -> { Configurator.INSTANCE.configure(configBroadcast.value()); return operator.apply(t, u); }); }
result = nextRDD.map(Traverser::bulk).fold(0L, (a, b) -> a + b); else if (endStep instanceof SumGlobalStep) { result = nextRDD.isEmpty() ? null : nextRDD .map(traverser -> NumberHelper.mul(traverser.bulk(), (Number) traverser.get())) .fold(0, NumberHelper::add); } else if (endStep instanceof MeanGlobalStep) { result = nextRDD.isEmpty() ? null : nextRDD .map(traverser -> new MeanGlobalStep.MeanNumber((Number) traverser.get(), traverser.bulk())) .fold(MeanNumberSupplier.instance().get(), MeanGlobalStep.MeanNumber::add) .getFinal(); } else if (endStep instanceof MinGlobalStep) { result = nextRDD.isEmpty() ? null : nextRDD .map(traverser -> (Comparable) traverser.get()) .fold(Double.NaN, NumberHelper::min); } else if (endStep instanceof MaxGlobalStep) { result = nextRDD.isEmpty() ? null : nextRDD .map(traverser -> (Comparable) traverser.get()) .fold(Double.NaN, NumberHelper::max); } else if (endStep instanceof FoldStep) { final BinaryOperator biOperator = endStep.getBiOperator(); }).fold(endStep.getSeedSupplier().get(), biOperator::apply); } else if (endStep instanceof GroupStep) { final GroupStep.GroupBiOperator<Object, Object> biOperator = (GroupStep.GroupBiOperator) endStep.getBiOperator(); final GroupStep<Object, Object, Object> clone = (GroupStep) endStep.clone(); return IteratorUtils.map(partitions, clone::projectTraverser); }).fold(((GroupStep<Object, Object, Object>) endStep).getSeedSupplier().get(), biOperator::apply)); } else if (endStep instanceof GroupCountStep) {