/** * Implementation which splits based solely on time. It will return approximately * the earliest {@link #getTestFraction()} of input, ordered by timestamp, as new training * data and the rest as test data. */ @Override protected Pair<JavaRDD<String>,JavaRDD<String>> splitNewDataToTrainTest(JavaRDD<String> newData) { // Rough approximation; assumes timestamps are fairly evenly distributed StatCounter maxMin = newData.mapToDouble(line -> MLFunctions.TO_TIMESTAMP_FN.call(line).doubleValue()).stats(); long minTime = (long) maxMin.min(); long maxTime = (long) maxMin.max(); log.info("New data timestamp range: {} - {}", minTime, maxTime); long approxTestTrainBoundary = (long) (maxTime - getTestFraction() * (maxTime - minTime)); log.info("Splitting at timestamp {}", approxTestTrainBoundary); JavaRDD<String> newTrainData = newData.filter( line -> MLFunctions.TO_TIMESTAMP_FN.call(line) < approxTestTrainBoundary); JavaRDD<String> testData = newData.filter( line -> MLFunctions.TO_TIMESTAMP_FN.call(line) >= approxTestTrainBoundary); return new Pair<>(newTrainData, testData); }
public int getMaxValueSeen() { return (int) counter.max(); };
public long getMaxValueSeen() { return (long) counter.max(); };
public int getMaxValueSeen() { return (int) counter.max(); };
public double getMaxValueSeen() { return counter.max(); };
public int getMaxLengthSeen() { return (int) counter.max(); };
public long getMaxValueSeen() { return (long) counter.max(); };
public double getMaxValueSeen() { return counter.max(); };
public int getMaxLengthSeen() { return (int) counter.max(); };
public static void main(String[] args) { //Sample test data - All numbers from 1 to 99999 List<Double> testData = IntStream.range(1, 100000).mapToDouble(d -> d).collect(ArrayList::new, ArrayList::add, ArrayList::addAll); JavaDoubleRDD rdd = sc.parallelizeDoubles(testData); LOGGER.info("Mean: " + rdd.mean()); //For efficiency, use StatCounter if more than one stats are required. StatCounter statCounter = rdd.stats(); LOGGER.info("Using StatCounter"); LOGGER.info("Count: " + statCounter.count()); LOGGER.info("Min: " + statCounter.min()); LOGGER.info("Max: " + statCounter.max()); LOGGER.info("Sum: " + statCounter.sum()); LOGGER.info("Mean: " + statCounter.mean()); LOGGER.info("Variance: " + statCounter.variance()); LOGGER.info("Stdev: " + statCounter.stdev()); } }