@Explain(displayName = "Number of Partitions") public String getNumPartitions() { return Integer.toString(prop.getNumPartitions()); }
@Explain(displayName = "Number of Partitions") public String getNumPartitions() { return Integer.toString(prop.getNumPartitions()); }
private ShuffleTran generate(SparkPlan sparkPlan, SparkEdgeProperty edge, boolean toCache, String name, BaseWork work) { Preconditions.checkArgument(!edge.isShuffleNone(), "AssertionError: SHUFFLE_NONE should only be used for UnionWork."); SparkShuffler shuffler; if (edge.isMRShuffle()) { shuffler = new SortByShuffler(false, sparkPlan, shuffleSerializer); } else if (edge.isShuffleSort()) { shuffler = new SortByShuffler(true, sparkPlan, shuffleSerializer); } else { shuffler = new GroupByShuffler(); } return new ShuffleTran(sparkPlan, shuffler, edge.getNumPartitions(), toCache, name, edge, work); }
private ShuffleTran generate(SparkPlan sparkPlan, SparkEdgeProperty edge, boolean toCache) { Preconditions.checkArgument(!edge.isShuffleNone(), "AssertionError: SHUFFLE_NONE should only be used for UnionWork."); SparkShuffler shuffler; if (edge.isMRShuffle()) { shuffler = new SortByShuffler(false, sparkPlan); } else if (edge.isShuffleSort()) { shuffler = new SortByShuffler(true, sparkPlan); } else { boolean useSparkGroupBy = jobConf.getBoolean("hive.spark.use.groupby.shuffle", true); if (!useSparkGroupBy) { LOG.info("hive.spark.use.groupby.shuffle is off. Use repartitin shuffle instead."); } shuffler = useSparkGroupBy ? new GroupByShuffler() : new RepartitionShuffler(); } return new ShuffleTran(sparkPlan, shuffler, edge.getNumPartitions(), toCache); }
@Explain(displayName = "Number of Partitions") public String getNumPartitions() { return Integer.toString(prop.getNumPartitions()); }
private ShuffleTran generate(SparkPlan sparkPlan, SparkEdgeProperty edge, boolean toCache) { Preconditions.checkArgument(!edge.isShuffleNone(), "AssertionError: SHUFFLE_NONE should only be used for UnionWork."); SparkShuffler shuffler; if (edge.isMRShuffle()) { shuffler = new SortByShuffler(false); } else if (edge.isShuffleSort()) { shuffler = new SortByShuffler(true); } else { shuffler = new GroupByShuffler(); } return new ShuffleTran(sparkPlan, shuffler, edge.getNumPartitions(), toCache); }