private static <T> GroupingOptions buildGroupingOptions(Configuration conf, PTypeFamily tf, PType<T> ptype, Order order) { Builder builder = GroupingOptions.builder(); if (order == Order.DESCENDING) { if (tf == WritableTypeFamily.getInstance()) { builder.sortComparatorClass(ReverseWritableComparator.class); } else if (tf == AvroTypeFamily.getInstance()) { AvroType<T> avroType = (AvroType<T>) ptype; Schema schema = avroType.getSchema(); conf.set("crunch.schema", schema.toString()); builder.sortComparatorClass(ReverseAvroComparator.class); } else { throw new RuntimeException("Unrecognized type family: " + tf); } } return builder.build(); }
private static <T> GroupingOptions buildGroupingOptions(Configuration conf, PTypeFamily tf, PType<T> ptype, Order order) { Builder builder = GroupingOptions.builder(); if (order == Order.DESCENDING) { if (tf == WritableTypeFamily.getInstance()) { builder.sortComparatorClass(ReverseWritableComparator.class); } else if (tf == AvroTypeFamily.getInstance()) { AvroType<T> avroType = (AvroType<T>) ptype; Schema schema = avroType.getSchema(); conf.set("crunch.schema", schema.toString()); builder.sortComparatorClass(ReverseAvroComparator.class); } else { throw new RuntimeException("Unrecognized type family: " + tf); } } // TODO:CRUNCH-23: Intermediate Fix for release 1. More elaborate fix is // required check JIRA for details. builder.numReducers(1); return builder.build(); }
private static <K, V> GroupingOptions buildGroupingOptions(PTable<K, V> ptable, Configuration conf, int numReducers, Order order) { PType<K> ptype = ptable.getKeyType(); PTypeFamily tf = ptable.getTypeFamily(); GroupingOptions.Builder builder = GroupingOptions.builder(); if (order == Order.DESCENDING) { if (tf == WritableTypeFamily.getInstance()) { builder.sortComparatorClass(ReverseWritableComparator.class); } else if (tf == AvroTypeFamily.getInstance()) { AvroType<K> avroType = (AvroType<K>) ptype; Schema schema = avroType.getSchema(); builder.conf("crunch.schema", schema.toString()); builder.sortComparatorClass(ReverseAvroComparator.class); } else { throw new RuntimeException("Unrecognized type family: " + tf); } } else if (tf == AvroTypeFamily.getInstance()) { builder.conf("crunch.schema", ((AvroType<K>) ptype).getSchema().toString()); } builder.requireSortedKeys(); configureReducers(builder, ptable, conf, numReducers); return builder.build(); }
GroupingOptions options = GroupingOptions.builder() .partitionerClass(TotalOrderPartitioner.class) .sortComparatorClass(KeyValueComparator.class) .conf(TotalOrderPartitioner.PARTITIONER_PATH, partitionFile.toString()) .numReducers(splitPoints.size() + 1)
GroupingOptions options = GroupingOptions.builder() .partitionerClass(TotalOrderPartitioner.class) .sortComparatorClass(KeyValueComparator.class) .conf(TotalOrderPartitioner.PARTITIONER_PATH, partitionFile.toString()) .numReducers(splitPoints.size() + 1)
private static <K, V> GroupingOptions buildGroupingOptions(PTable<K, V> ptable, Configuration conf, int numReducers, ColumnOrder[] columnOrders) { PTypeFamily tf = ptable.getTypeFamily(); PType<K> keyType = ptable.getKeyType(); GroupingOptions.Builder builder = GroupingOptions.builder(); if (tf == WritableTypeFamily.getInstance()) { if (columnOrders.length == 1 && columnOrders[0].order == Order.DESCENDING) { builder.sortComparatorClass(ReverseWritableComparator.class); } else { WritableType[] wt = new WritableType[columnOrders.length]; for (int i = 0; i < wt.length; i++) { wt[i] = (WritableType) keyType.getSubTypes().get(i); } TupleWritableComparator.configureOrdering(conf, wt, columnOrders); builder.sortComparatorClass(TupleWritableComparator.class); } } else if (tf == AvroTypeFamily.getInstance()) { AvroType<K> avroType = (AvroType<K>) keyType; Schema schema = avroType.getSchema(); builder.conf("crunch.schema", schema.toString()); if (columnOrders.length == 1 && columnOrders[0].order == Order.DESCENDING) { builder.sortComparatorClass(ReverseAvroComparator.class); } } else { throw new RuntimeException("Unrecognized type family: " + tf); } builder.requireSortedKeys(); configureReducers(builder, ptable, conf, numReducers); return builder.build(); }
private static <T> GroupingOptions buildGroupingOptions(Configuration conf, PTypeFamily tf, PType<T> ptype, ColumnOrder[] columnOrders) { Builder builder = GroupingOptions.builder(); if (tf == WritableTypeFamily.getInstance()) { TupleWritableComparator.configureOrdering(conf, columnOrders); builder.sortComparatorClass(TupleWritableComparator.class); } else if (tf == AvroTypeFamily.getInstance()) { TupleAvroComparator.configureOrdering(conf, columnOrders, ptype); builder.sortComparatorClass(TupleAvroComparator.class); } else { throw new RuntimeException("Unrecognized type family: " + tf); } // TODO:CRUNCH-23: Intermediate Fix for release 1. More elaborate fix is // required check JIRA for details. builder.numReducers(1); return builder.build(); }
private static <T> GroupingOptions buildGroupingOptions(Configuration conf, PTypeFamily tf, PType<T> ptype, ColumnOrder[] columnOrders) { Builder builder = GroupingOptions.builder(); if (tf == WritableTypeFamily.getInstance()) { TupleWritableComparator.configureOrdering(conf, columnOrders); builder.sortComparatorClass(TupleWritableComparator.class); } else if (tf == AvroTypeFamily.getInstance()) { TupleAvroComparator.configureOrdering(conf, columnOrders, ptype); builder.sortComparatorClass(TupleAvroComparator.class); } else { throw new RuntimeException("Unrecognized type family: " + tf); } return builder.build(); }