@Test public void combineByKey() { JavaRDD<Integer> originalRDD = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5, 6)); Function<Integer, Integer> keyFunction = v1 -> v1 % 3; Function<Integer, Integer> createCombinerFunction = v1 -> v1; Function2<Integer, Integer, Integer> mergeValueFunction = (v1, v2) -> v1 + v2; JavaPairRDD<Integer, Integer> combinedRDD = originalRDD.keyBy(keyFunction) .combineByKey(createCombinerFunction, mergeValueFunction, mergeValueFunction); Map<Integer, Integer> results = combinedRDD.collectAsMap(); ImmutableMap<Integer, Integer> expected = ImmutableMap.of(0, 9, 1, 5, 2, 7); assertEquals(expected, results); Partitioner defaultPartitioner = Partitioner.defaultPartitioner( combinedRDD.rdd(), JavaConverters.collectionAsScalaIterableConverter( Collections.<RDD<?>>emptyList()).asScala().toSeq()); combinedRDD = originalRDD.keyBy(keyFunction) .combineByKey( createCombinerFunction, mergeValueFunction, mergeValueFunction, defaultPartitioner, false, new KryoSerializer(new SparkConf())); results = combinedRDD.collectAsMap(); assertEquals(expected, results); }
@Test public void combineByKey() { JavaRDD<Integer> originalRDD = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5, 6)); Function<Integer, Integer> keyFunction = v1 -> v1 % 3; Function<Integer, Integer> createCombinerFunction = v1 -> v1; Function2<Integer, Integer, Integer> mergeValueFunction = (v1, v2) -> v1 + v2; JavaPairRDD<Integer, Integer> combinedRDD = originalRDD.keyBy(keyFunction) .combineByKey(createCombinerFunction, mergeValueFunction, mergeValueFunction); Map<Integer, Integer> results = combinedRDD.collectAsMap(); ImmutableMap<Integer, Integer> expected = ImmutableMap.of(0, 9, 1, 5, 2, 7); assertEquals(expected, results); Partitioner defaultPartitioner = Partitioner.defaultPartitioner( combinedRDD.rdd(), JavaConverters.collectionAsScalaIterableConverter( Collections.<RDD<?>>emptyList()).asScala().toSeq()); combinedRDD = originalRDD.keyBy(keyFunction) .combineByKey( createCombinerFunction, mergeValueFunction, mergeValueFunction, defaultPartitioner, false, new KryoSerializer(new SparkConf())); results = combinedRDD.collectAsMap(); assertEquals(expected, results); }
@Test public void combineByKey() { JavaRDD<Integer> originalRDD = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5, 6)); Function<Integer, Integer> keyFunction = v1 -> v1 % 3; Function<Integer, Integer> createCombinerFunction = v1 -> v1; Function2<Integer, Integer, Integer> mergeValueFunction = (v1, v2) -> v1 + v2; JavaPairRDD<Integer, Integer> combinedRDD = originalRDD.keyBy(keyFunction) .combineByKey(createCombinerFunction, mergeValueFunction, mergeValueFunction); Map<Integer, Integer> results = combinedRDD.collectAsMap(); ImmutableMap<Integer, Integer> expected = ImmutableMap.of(0, 9, 1, 5, 2, 7); assertEquals(expected, results); Partitioner defaultPartitioner = Partitioner.defaultPartitioner( combinedRDD.rdd(), JavaConverters.collectionAsScalaIterableConverter( Collections.<RDD<?>>emptyList()).asScala().toSeq()); combinedRDD = originalRDD.keyBy(keyFunction) .combineByKey( createCombinerFunction, mergeValueFunction, mergeValueFunction, defaultPartitioner, false, new KryoSerializer(new SparkConf())); results = combinedRDD.collectAsMap(); assertEquals(expected, results); }
/** * Select a preferred language, given the list of candidates. * * Will select the preferred language, based on what languages are available, or return the default language if * none of the candidates are available. * * @param candidates The candidate languages * @return The preferred language */ public Lang preferred(Collection<Lang> candidates) { return new Lang(langs.preferred((scala.collection.Seq) JavaConversions.collectionAsScalaIterable(candidates).toSeq())); } }
/** * Get a messages context appropriate for the given candidates. * * Will select a language from the candidates, based on the languages available, and fallback to the default language * if none of the candidates are available. */ public Messages preferred(Collection<Lang> candidates) { Seq<Lang> cs = JavaConversions.collectionAsScalaIterable(candidates).toSeq(); play.api.i18n.Messages msgs = messages.preferred((Seq) cs); return new Messages(new Lang(msgs.lang()), this); }
/** * Get allowable parent types * @param nodeOfThisType Node of this type * @return Sequence of allowable parent types */ @Override public Seq<UserTypeVal> getAllowableParentTypes(Node<UserStub> nodeOfThisType) { LinkedList<UserTypeVal> list = new LinkedList<>(); list.add(UserType.ADMIN); return ScalaInJavaHelper.linkedListToScalaIterable(list).toSeq(); }
/** * Get allowable child types * @param nodeOfThisType Node of this type * @return Sequence of allowable child types */ @Override public Seq<UserTypeVal> getAllowableChildTypes(Node<UserStub> nodeOfThisType) { LinkedList<UserTypeVal> list = new LinkedList<>(); list.add(UserType.SOCIAL_NETWORK_EMPLOYEE); list.add(UserType.PUBLIC_USER); return ScalaInJavaHelper.linkedListToScalaIterable(list).toSeq(); }
/** * Get allowable child types * @param nodeOfThisType Node of this type * @return Sequence of allowable child types */ @Override public Seq<UserTypeVal> getAllowableChildTypes(Node<UserStub> nodeOfThisType) { LinkedList<UserTypeVal> list = new LinkedList<>(); list.add(UserType.PUBLIC_USER); return ScalaInJavaHelper.linkedListToScalaIterable(list).toSeq(); }
/** * Get allowable parent types * @param nodeOfThisType Node of this type * @return Sequence of allowable parent types */ @Override public Seq<UserTypeVal> getAllowableParentTypes(Node<UserStub> nodeOfThisType) { LinkedList<UserTypeVal> list = new LinkedList<>(); list.add(UserType.ADMIN); list.add(UserType.SOCIAL_NETWORK_EMPLOYEE); return ScalaInJavaHelper.linkedListToScalaIterable(list).toSeq(); }
/** * Get allowable child types * @param nodeOfThisType Node of this type * @return Sequence of allowable child types */ @Override public Seq<UserTypeVal> getAllowableChildTypes(Node<UserStub> nodeOfThisType) { LinkedList<UserTypeVal> list = new LinkedList<>(); list.add(UserType.ADMIN); list.add(UserType.SOCIAL_NETWORK_EMPLOYEE); list.add(UserType.PUBLIC_USER); return ScalaInJavaHelper.linkedListToScalaIterable(list).toSeq(); }
/** * Get allowable parent types * @param nodeOfThisType Node of this type * @return Sequence of allowable parent types */ @Override public Seq<UserTypeVal> getAllowableParentTypes(Node<UserStub> nodeOfThisType) { LinkedList<UserTypeVal> list = new LinkedList<>(); list.add(UserType.ADMIN); list.add(UserType.SOCIAL_NETWORK_EMPLOYEE); list.add(UserType.PUBLIC_USER); return ScalaInJavaHelper.linkedListToScalaIterable(list).toSeq(); }
/** * @see Offer$#choose(scala.collection.Seq) */ public static <T> Offer<T> choose(Collection<Offer<T>> offers) { scala.collection.Seq<Offer<T>> scalaSeq = JavaConverters.collectionAsScalaIterableConverter(offers).asScala().toSeq(); return Offer$.MODULE$.choose(scalaSeq); }
private static void printCurrentAssignment(ZkUtils zkUtils, List<String> specifiedTopics) { Seq<String> topics = specifiedTopics != null ? JavaConversions.iterableAsScalaIterable(specifiedTopics).toSeq() : zkUtils.getAllTopics(); System.out.println("CURRENT ASSIGNMENT:"); System.out.println( zkUtils.formatAsReassignmentJson(zkUtils.getReplicaAssignmentForTopics( topics))); }
/** * @see Offer$#choose(scala.collection.Seq) */ public static <T> Offer<T> choose(Collection<Offer<T>> offers) { scala.collection.Seq<Offer<T>> scalaSeq = JavaConverters.collectionAsScalaIterableConverter(offers).asScala().toSeq(); return Offer$.MODULE$.choose(scalaSeq); }
@Override public Dataset<Row> derive(Map<String, Dataset<Row>> dependencies) throws Exception { dependencyCheck(dependencies); Dataset<Row> sourceStep = dependencies.get(stepName); if (useIncludeFields){ if (!Arrays.asList(sourceStep.columns()).containsAll(includeFields)){ throw new RuntimeException("Columns specified in " + INCLUDE_FIELDS + " are not found in input dependency schema \n" + "Available columns: " + Arrays.toString(sourceStep.columns())); } String firstCol = includeFields.get(0); includeFields.remove(0); return sourceStep.select(firstCol, includeFields.toArray(new String[0])); } else { if (!Arrays.asList(sourceStep.columns()).containsAll(excludeFields)){ throw new RuntimeException("Columns specified in " + EXCLUDE_FIELDS + " are not found in input dependency schema \n" + "Available columns: " + Arrays.toString(sourceStep.columns())); } return sourceStep.drop(JavaConverters.collectionAsScalaIterableConverter(excludeFields).asScala().toSeq()); } }
JavaConversions.collectionAsScalaIterable(specifiedTopics).toSeq() : zkUtils.getAllTopics();
/** * Creates {@link SparkConf} with {@link org.apache.spark.serializer.KryoSerializer} along with * registering default/user-input serializable classes and user-input Avro Schemas. * Once {@link SparkContext} is created, we can no longer register serialization classes and Avro schemas. */ public SparkConf createSparkConf(@NonNull final SparkArgs sparkArgs) { /** * By custom registering classes the full class name of each object * is not stored during serialization which reduces storage space. */ final SparkConf sparkConf = new SparkConf(); sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); final List<Class> serializableClasses = getDefaultSerializableClasses(); serializableClasses.addAll(sparkArgs.getUserSerializationClasses()); sparkConf.registerKryoClasses(serializableClasses.toArray(new Class[0])); if (sparkArgs.getAvroSchemas().isPresent()) { sparkConf.registerAvroSchemas( JavaConverters .iterableAsScalaIterableConverter(sparkArgs.getAvroSchemas().get()) .asScala() .toSeq()); } // override spark properties final Map<String, String> sparkProps = sparkArgs.getOverrideSparkProperties(); for (Entry<String, String> entry : sparkProps.entrySet()) { log.info("Setting spark key:val {} : {}", entry.getKey(), entry.getValue()); sparkConf.set(entry.getKey(), entry.getValue()); } return sparkConf; }
JavaConverters.iterableAsScalaIterableConverter(avroSchemas.get()) .asScala() .toSeq());