protected void complete(final MapReduce<?, ?, OK, OV, ?> mapReduce) { if (mapReduce.getReduceKeySort().isPresent()) { final Comparator<OK> comparator = mapReduce.getReduceKeySort().get(); final List<KeyValue<OK, OV>> list = new ArrayList<>(this.reduceQueue); Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator)); this.reduceQueue.clear(); this.reduceQueue.addAll(list); } } }
protected void complete(final MapReduce<?, ?, OK, OV, ?> mapReduce) { if (mapReduce.getReduceKeySort().isPresent()) { final Comparator<OK> comparator = mapReduce.getReduceKeySort().get(); final List<KeyValue<OK, OV>> list = new ArrayList<>(this.reduceQueue); list.sort(Comparator.comparing(KeyValue::getKey, comparator)); this.reduceQueue.clear(); this.reduceQueue.addAll(list); } } }
protected void complete(final MapReduce<?, ?, OK, OV, ?> mapReduce) { if (mapReduce.getReduceKeySort().isPresent()) { final Comparator<OK> comparator = mapReduce.getReduceKeySort().get(); final List<KeyValue<OK, OV>> list = new ArrayList<>(this.reduceQueue); Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator)); this.reduceQueue.clear(); this.reduceQueue.addAll(list); } } }
public static <K, V, OK, OV> JavaPairRDD<OK, OV> executeReduce( final JavaPairRDD<K, V> mapOrCombineRDD, final MapReduce<K, V, OK, OV, ?> mapReduce, final Configuration graphComputerConfiguration) { JavaPairRDD<OK, OV> reduceRDD = mapOrCombineRDD.groupByKey().mapPartitionsToPair(partitionIterator -> { KryoShimServiceLoader.applyConfiguration(graphComputerConfiguration); return new ReduceIterator<>(MapReduce.<MapReduce<K, V, OK, OV, ?>>createMapReduce(HadoopGraph.open(graphComputerConfiguration), graphComputerConfiguration), partitionIterator); }); if (mapReduce.getReduceKeySort().isPresent()) reduceRDD = reduceRDD.sortByKey(mapReduce.getReduceKeySort().get(), true, 1); return reduceRDD; } }
@Override public void setConf(final Configuration configuration) { super.setConf(configuration); final org.apache.commons.configuration.Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(configuration); this.comparator = MapReduce.<MapReduce<?,?,?,?,?>>createMapReduce(HadoopGraph.open(apacheConfiguration),apacheConfiguration).getReduceKeySort().get(); } }
final Optional<Comparator<?>> reduceSort = mapReduce.getReduceKeySort(); newConfiguration.setClass(Constants.GREMLIN_HADOOP_MAP_REDUCE_CLASS, mapReduce.getClass(), MapReduce.class); final Job job = Job.getInstance(newConfiguration, mapReduce.toString());
protected void complete(MapReduce<?, ?, OK, OV, ?> mapReduce) { if (mapReduce.getReduceKeySort().isPresent()) { Comparator<OK> comparator = mapReduce.getReduceKeySort().get(); List<KeyValue<OK, OV>> list = new ArrayList<>(reduceQueue); Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator)); reduceQueue.clear(); reduceQueue.addAll(list); } } }
protected void complete(final MapReduce<?, ?, OK, OV, ?> mapReduce) { if (mapReduce.getReduceKeySort().isPresent()) { final Comparator<OK> comparator = mapReduce.getReduceKeySort().get(); final List<KeyValue<OK, OV>> list = new ArrayList<>(this.reduceQueue); Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator)); this.reduceQueue.clear(); this.reduceQueue.addAll(list); } } }
protected void complete(final MapReduce<?, ?, OK, OV, ?> mapReduce) { if (mapReduce.getReduceKeySort().isPresent()) { final Comparator<OK> comparator = mapReduce.getReduceKeySort().get(); final List<KeyValue<OK, OV>> list = new ArrayList<>(this.reduceQueue); Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator)); this.reduceQueue.clear(); this.reduceQueue.addAll(list); } } }
protected void complete(final MapReduce<?, ?, OK, OV, ?> mapReduce) { if (mapReduce.getReduceKeySort().isPresent()) { final Comparator<OK> comparator = mapReduce.getReduceKeySort().get(); final List<KeyValue<OK, OV>> list = new ArrayList<>(this.reduceQueue); Collections.sort(list, Comparator.comparing(KeyValue::getKey, comparator)); this.reduceQueue.clear(); this.reduceQueue.addAll(list); } } }
public static <K, V, OK, OV> JavaPairRDD<OK, OV> executeReduce( final JavaPairRDD<K, V> mapOrCombineRDD, final MapReduce<K, V, OK, OV, ?> mapReduce, final Configuration graphComputerConfiguration) { JavaPairRDD<OK, OV> reduceRDD = mapOrCombineRDD.groupByKey().mapPartitionsToPair(partitionIterator -> { KryoShimServiceLoader.applyConfiguration(graphComputerConfiguration); return new ReduceIterator<>(MapReduce.<MapReduce<K, V, OK, OV, ?>>createMapReduce(HadoopGraph.open(graphComputerConfiguration), graphComputerConfiguration), partitionIterator); }); if (mapReduce.getReduceKeySort().isPresent()) reduceRDD = reduceRDD.sortByKey(mapReduce.getReduceKeySort().get(), true, 1); return reduceRDD; } }
public static <K, V, OK, OV> JavaPairRDD<OK, OV> executeReduce( final JavaPairRDD<K, V> mapOrCombineRDD, final MapReduce<K, V, OK, OV, ?> mapReduce, final Configuration graphComputerConfiguration) { JavaPairRDD<OK, OV> reduceRDD = mapOrCombineRDD.groupByKey().mapPartitionsToPair(partitionIterator -> { KryoShimServiceLoader.applyConfiguration(graphComputerConfiguration); return new ReduceIterator<>(MapReduce.<MapReduce<K, V, OK, OV, ?>>createMapReduce(HadoopGraph.open(graphComputerConfiguration), graphComputerConfiguration), partitionIterator); }); if (mapReduce.getReduceKeySort().isPresent()){ reduceRDD = reduceRDD.sortByKey(mapReduce.getReduceKeySort().get(), true, 1);} return reduceRDD; } }
@Override public void setConf(final Configuration configuration) { super.setConf(configuration); final org.apache.commons.configuration.Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(configuration); this.comparator = MapReduce.<MapReduce<?,?,?,?,?>>createMapReduce(HadoopGraph.open(apacheConfiguration),apacheConfiguration).getReduceKeySort().get(); } }
final Optional<Comparator<?>> reduceSort = mapReduce.getReduceKeySort(); newConfiguration.setClass(Constants.GREMLIN_HADOOP_MAP_REDUCE_CLASS, mapReduce.getClass(), MapReduce.class); final Job job = Job.getInstance(newConfiguration, mapReduce.toString());