private RawComparator<?> rawComparator() { if (cmp == null) { try { Job job = new Job(ctxt.getConfiguration()); ptype.configureShuffle(job, options); cmp = ReflectionUtils.newInstance(options.getGroupingComparatorClass(), job.getConfiguration()); } catch (IOException e) { throw new CrunchRuntimeException("Error configuring grouping comparator", e); } } return cmp; }
private RawComparator<?> rawComparator() { if (cmp == null) { try { Job job = new Job(ctxt.getConfiguration()); ptype.configureShuffle(job, options); cmp = ReflectionUtils.newInstance(options.getGroupingComparatorClass(), job.getConfiguration()); } catch (IOException e) { throw new CrunchRuntimeException("Error configuring grouping comparator", e); } } return cmp; }
public static <S, T> Shuffler<S, T> create(PType<S> keyType, GroupingOptions options, Pipeline pipeline) { Map<S, Collection<T>> map = getMapForKeyType(keyType); if (options != null) { if (Pair.class.equals(keyType.getTypeClass()) && options.getGroupingComparatorClass() != null) { PType<?> pairKey = keyType.getSubTypes().get(0); return new SecondarySortShuffler(getMapForKeyType(pairKey)); } else if (options.getSortComparatorClass() != null) { RawComparator<S> rc = ReflectionUtils.newInstance(options.getSortComparatorClass(), pipeline.getConfiguration()); map = new TreeMap<S, Collection<T>>(rc); } } return new MapShuffler<S, T>(map); }
public static <S, T> Shuffler<S, T> create(PType<S> keyType, GroupingOptions options, Pipeline pipeline) { Map<Object, Collection<T>> map = getMapForKeyType(keyType); if (options != null) { Job job; try { job = new Job(pipeline.getConfiguration()); } catch (IOException e) { throw new IllegalStateException("Could not create Job instance", e); } options.configure(job); if (Pair.class.equals(keyType.getTypeClass()) && options.getGroupingComparatorClass() != null) { PType<?> pairKey = keyType.getSubTypes().get(0); return new SecondarySortShuffler(getMapForKeyType(pairKey)); } else if (options.getSortComparatorClass() != null) { RawComparator rc = ReflectionUtils.newInstance( options.getSortComparatorClass(), job.getConfiguration()); map = new TreeMap<Object, Collection<T>>(rc); return new MapShuffler<S, T>(map, keyType); } } return new MapShuffler<S, T>(map); }
groupedRDD = groupedRDD.sortByKey(scmp); if (groupingOptions.getGroupingComparatorClass() != null) { groupedRDD = groupedRDD.mapPartitionsToPair( new ReduceGroupingFunction(groupingOptions, ptype, runtime.getRuntimeContext()));
groupedRDD = groupedRDD.sortByKey(scmp); if (groupingOptions.getGroupingComparatorClass() != null) { groupedRDD = groupedRDD.mapPartitionsToPair( new ReduceGroupingFunction(groupingOptions, ptype, runtime.getRuntimeContext()));