@Explain(displayName = "Shuffle Type") public String getShuffleType() { if (isShuffleNone()) { return "NONE"; } StringBuilder sb = new StringBuilder(); if (isShuffleGroup()) { sb.append("GROUP"); } if (isMRShuffle()) { if (sb.length() != 0) { sb.append(" "); } sb.append("PARTITION-LEVEL SORT"); } if (isShuffleSort()) { if (sb.length() != 0) { sb.append(" "); } sb.append("SORT"); } return sb.toString(); }
@Explain(displayName = "Shuffle Type") public String getShuffleType() { if (isShuffleNone()) { return "NONE"; } StringBuilder sb = new StringBuilder(); if (isShuffleGroup()) { sb.append("GROUP"); } if (isMRShuffle()) { if (sb.length() != 0) { sb.append(" "); } sb.append("PARTITION-LEVEL SORT"); } if (isShuffleSort()) { if (sb.length() != 0) { sb.append(" "); } sb.append("SORT"); } return sb.toString(); }
private ShuffleTran generate(SparkPlan sparkPlan, SparkEdgeProperty edge, boolean toCache, String name, BaseWork work) { Preconditions.checkArgument(!edge.isShuffleNone(), "AssertionError: SHUFFLE_NONE should only be used for UnionWork."); SparkShuffler shuffler; if (edge.isMRShuffle()) { shuffler = new SortByShuffler(false, sparkPlan, shuffleSerializer); } else if (edge.isShuffleSort()) { shuffler = new SortByShuffler(true, sparkPlan, shuffleSerializer); } else { shuffler = new GroupByShuffler(); } return new ShuffleTran(sparkPlan, shuffler, edge.getNumPartitions(), toCache, name, edge, work); }
private ShuffleTran generate(SparkPlan sparkPlan, SparkEdgeProperty edge, boolean toCache) { Preconditions.checkArgument(!edge.isShuffleNone(), "AssertionError: SHUFFLE_NONE should only be used for UnionWork."); SparkShuffler shuffler; if (edge.isMRShuffle()) { shuffler = new SortByShuffler(false, sparkPlan); } else if (edge.isShuffleSort()) { shuffler = new SortByShuffler(true, sparkPlan); } else { boolean useSparkGroupBy = jobConf.getBoolean("hive.spark.use.groupby.shuffle", true); if (!useSparkGroupBy) { LOG.info("hive.spark.use.groupby.shuffle is off. Use repartitin shuffle instead."); } shuffler = useSparkGroupBy ? new GroupByShuffler() : new RepartitionShuffler(); } return new ShuffleTran(sparkPlan, shuffler, edge.getNumPartitions(), toCache); }
if (edgeProperty.isShuffleNone() && !sortOrder.isEmpty()) { if ((reduceSink.getConf().getPartitionCols() == null || reduceSink.getConf().getPartitionCols().isEmpty() if (edgeProperty.isShuffleNone()) { if (!useSparkGroupBy) { LOG.info("hive.spark.use.groupby.shuffle is off. Use repartition shuffle instead.");
if (edgeProperty.isShuffleNone() && !sortOrder.isEmpty()) { if ((reduceSink.getConf().getPartitionCols() == null || reduceSink.getConf().getPartitionCols().isEmpty() if (edgeProperty.isShuffleNone()) { edgeProperty.setShuffleGroup();
@Explain(displayName = "Shuffle Type") public String getShuffleType() { if (isShuffleNone()) { return "NONE"; } StringBuilder sb = new StringBuilder(); if (isShuffleGroup()) { sb.append("GROUP"); } if (isMRShuffle()) { if (sb.length() != 0) { sb.append(" "); } sb.append("PARTITION-LEVEL SORT"); } if (isShuffleSort()) { if (sb.length() != 0) { sb.append(" "); } sb.append("SORT"); } return sb.toString(); }
private ShuffleTran generate(SparkPlan sparkPlan, SparkEdgeProperty edge, boolean toCache) { Preconditions.checkArgument(!edge.isShuffleNone(), "AssertionError: SHUFFLE_NONE should only be used for UnionWork."); SparkShuffler shuffler; if (edge.isMRShuffle()) { shuffler = new SortByShuffler(false); } else if (edge.isShuffleSort()) { shuffler = new SortByShuffler(true); } else { shuffler = new GroupByShuffler(); } return new ShuffleTran(sparkPlan, shuffler, edge.getNumPartitions(), toCache); }
if (edgeProperty.isShuffleNone() && !sortOrder.isEmpty()) { if ((reduceSink.getConf().getPartitionCols() == null || reduceSink.getConf().getPartitionCols().isEmpty() if (edgeProperty.isShuffleNone()) { edgeProperty.setShuffleGroup();