@Override public boolean persist(List listEntity, EntityMetadata m, SparkClient sparkClient) { try { Seq s = scala.collection.JavaConversions.asScalaBuffer(listEntity).toList(); ClassTag tag = scala.reflect.ClassTag$.MODULE$.apply(m.getEntityClazz()); JavaRDD personRDD = sparkClient.sparkContext.parallelize(s, 1, tag).toJavaRDD(); CassandraJavaUtil.javaFunctions(personRDD) .writerBuilder(m.getSchema(), m.getTableName(), CassandraJavaUtil.mapToRow(m.getEntityClazz())) .saveToCassandra(); return true; } catch (Exception e) { throw new KunderaException("Cannot persist object(s)", e); } }
public static ColumnRef[] toSelectableColumnRefs(String... columnNames) { ColumnName[] refs = new ColumnName[columnNames.length]; for (int i = 0; i < columnNames.length; i++) { refs[i] = column(columnNames[i]); } return refs; }
/** * Creates a row writer factory which maps objects of a given class type to columns in table. * <p/> * The given source class is considered to follow the Java bean style convention. It should have a * public, no-args constructor, as well as a set of getters and setters. The names of the attributes * must follow the names of columns in the table (however, different conventions are allowed - see * the guide for more info). The default mapping of attributes to column names can be changed by * providing a custom map of attribute-column mappings for the pairs which do not follow the general * convention. * <p/> * The method uses {@link JavaBeanColumnMapper} as the column mapper. If another column mapper has to * be used, see {@link #mapToRow(Class, ColumnMapper)} method. */ public static <T> RowWriterFactory<T> mapToRow(Class<T> sourceClass, Pair... columnNameMappings) { return mapToRow(sourceClass, convertToMap(columnNameMappings)); }
@Override public void registerTable(EntityMetadata m, SparkClient sparkClient) { SparkContextJavaFunctions functions = CassandraJavaUtil.javaFunctions(sparkClient.sparkContext); Class clazz = m.getEntityClazz(); JavaRDD cassandraRowsRDD = functions.cassandraTable(m.getSchema(), m.getTableName(), CassandraJavaUtil.mapRowTo(clazz)); sparkClient.sqlContext.createDataFrame(cassandraRowsRDD, clazz).registerTempTable(m.getTableName()); }
/** * Creates a RowWriterFactory instance for writing tuples with given parameter types. */ public static <A, B, C, D, E> RowWriterFactory<Tuple5<A, B, C, D, E>> mapTupleToRow( Class<A> a, Class<B> b, Class<C> c, Class<D> d, Class<E> e ) { return mapToRow(Tuple5.class, tuple5ColumnMapper(a, b, c, d, e)); }
) { final TypeTags.TypeTag<Tuple7<A, B, C, D, E, F, G>> tupleTT = typeTag(Tuple7.class, typeTag(a), typeTag(b), typeTag(c), typeTag(d), typeTag(e), typeTag(f), typeTag(g) ); return new ClassBasedRowReaderFactory<>(tupleTT, tuple7ColumnMapper(a, b, c, d, e, f, g)); ) { final TypeTags.TypeTag<Tuple8<A, B, C, D, E, F, G, H>> tupleTT = typeTag(Tuple8.class, typeTag(a), typeTag(b), typeTag(c), typeTag(d), typeTag(e), typeTag(f), typeTag(g), typeTag(h) ); return new ClassBasedRowReaderFactory<>(tupleTT, tuple8ColumnMapper(a, b, c, d, e, f, g, h)); ) { final TypeTags.TypeTag<Tuple9<A, B, C, D, E, F, G, H, I>> tupleTT =
/** * Constructs a row reader factory which maps an entire row to an object of a specified type. * <p/> * The given target class is considered to follow the Java bean style convention. It should have a * public, no-args constructor, as well as a set of getters and setters. The names of the attributes * must follow the names of columns in the table (however, different conventions are allowed - see * the guide for more info). The default mapping of attributes to column names can be changed by * providing a custom map of attribute-column mappings for the pairs which do not follow the general * convention. * <p/> * The method uses {@link JavaBeanColumnMapper} as the column mapper. If another column mapper has to * be used, see {@link #mapRowTo(Class, ColumnMapper)} method. */ public static <T> RowReaderFactory<T> mapRowTo(Class<T> targetClass, Pair... columnMappings) { TypeTags.TypeTag<T> tt = typeTag(targetClass); ColumnMapper<T> mapper = javaBeanColumnMapper(safeClassTag(targetClass), convertToMap(columnMappings)); return new ClassBasedRowReaderFactory<>(tt, mapper); }
/** * Applies a function to each item, and groups consecutive items having the same value together. * Contrary to `groupBy`, items from the same group must be already next to each other in the * original collection. Works locally on each partition, so items from different partitions will * never be placed in the same group. */ public <K> JavaPairRDD<K, Iterable<R>> spanBy(Function<R, K> f, ClassTag<K> keyClassTag) { return CassandraJavaUtil.javaFunctions(rdd()).spanBy(f, keyClassTag); }
/** * Creates a RowWriterFactory instance for writing tuples with given parameter types. */ public static <A, B, C, D, E, F, G, H, I, J> RowWriterFactory<Tuple10<A, B, C, D, E, F, G, H, I, J>> mapTupleToRow( Class<A> a, Class<B> b, Class<C> c, Class<D> d, Class<E> e, Class<F> f, Class<G> g, Class<H> h, Class<I> i, Class<J> j ) { return mapToRow(Tuple10.class, tuple10ColumnMapper(a, b, c, d, e, f, g, h, i, j)); }
/** * Creates a RowWriterFactory instance for writing tuples with given parameter types. */ public static <A, B, C, D, E, F, G, H, I, J, K> RowWriterFactory<Tuple11<A, B, C, D, E, F, G, H, I, J, K>> mapTupleToRow( Class<A> a, Class<B> b, Class<C> c, Class<D> d, Class<E> e, Class<F> f, Class<G> g, Class<H> h, Class<I> i, Class<J> j, Class<K> k ) { return mapToRow(Tuple11.class, tuple11ColumnMapper(a, b, c, d, e, f, g, h, i, j, k)); }
/** * Creates a RowWriterFactory instance for writing tuples with given parameter types. */ public static <A, B> RowWriterFactory<Tuple2<A, B>> mapTupleToRow( Class<A> a, Class<B> b ) { return mapToRow(Tuple2.class, tuple2ColumnMapper(a, b)); }
/** * Creates a RowWriterFactory instance for writing tuples with given parameter types. */ public static <A> RowWriterFactory<Tuple1<A>> mapTupleToRow( Class<A> a ) { return mapToRow(Tuple1.class, tuple1ColumnMapper(a)); }
/** * Creates a RowWriterFactory instance for writing tuples with given parameter types. */ public static <A, B, C> RowWriterFactory<Tuple3<A, B, C>> mapTupleToRow( Class<A> a, Class<B> b, Class<C> c ) { return mapToRow(Tuple3.class, tuple3ColumnMapper(a, b, c)); }
/** * Creates a row writer factory which maps objects of a given class type to columns in table. * <p/> * The given source class is considered to follow the Java bean style convention. It should have a * public, no-args constructor, as well as a set of getters and setters. The names of the attributes * must follow the names of columns in the table (however, different conventions are allowed - see * the guide for more info). The default mapping of attributes to column names can be changed by * providing a custom map of attribute-column mappings for the pairs which do not follow the general * convention. * <p/> * Another version of this method {@link #mapToRow(Class, Pair[])} can be considered much more handy * for inline invocations. * <p/> * The method uses {@link JavaBeanColumnMapper} as the column mapper. If another column mapper has to * be used, see {@link #mapToRow(Class, ColumnMapper)} method. */ public static <T> RowWriterFactory<T> mapToRow( Class<T> sourceClass, Map<String, String> columnNameMappings ) { ColumnMapper<T> mapper = javaBeanColumnMapper(safeClassTag(sourceClass), columnNameMappings); return mapToRow(sourceClass, mapper); }
/** * @see {@link #keyBy(RowReaderFactory, ClassTag, ColumnRef...)} */ public <K> CassandraJavaPairRDD<K, R> keyBy( RowReaderFactory<K> rrf, Class<K> keyClass, String... columnNames) { ColumnRef[] columnRefs = toSelectableColumnRefs(columnNames); return keyBy(rrf, JavaApiHelper.getClassTag(keyClass), columnRefs); }
/** * Constructs a row reader factory which maps an entire row to an object of a specified type. * <p/> * The given target class is considered to follow the Java bean style convention. It should have a * public, no-args constructor, as well as a set of getters and setters. The names of the attributes * must follow the names of columns in the table (however, different conventions are allowed - see * the guide for more info). The default mapping of attributes to column names can be overridden by * providing a custom map of attribute-column mappings for the pairs which do not follow the general * convention. * <p/> * Another version of this method {@link #mapRowTo(Class, Pair[])} can be considered much more handy * for inline invocations. * <p/> * The method uses {@link JavaBeanColumnMapper} as the column mapper. If another column mapper has to * be used, see {@link #mapRowTo(Class, ColumnMapper)} method. */ public static <T> RowReaderFactory<T> mapRowTo(Class<T> targetClass, Map<String, String> columnMapping) { TypeTags.TypeTag<T> tt = typeTag(targetClass); ColumnMapper<T> mapper = javaBeanColumnMapper(safeClassTag(targetClass), columnMapping); return new ClassBasedRowReaderFactory<>(tt, mapper); }
/** * Retrieves a type converter for the given class type. * <p/> * This is a short-hand method for calling {@link #typeConverter(TypeTags.TypeTag)} with an argument * obtained by {@link #safeTypeTag(Class)}. */ public static <T> TypeConverter<T> safeTypeConverter(Class<T> main) { return TypeConverter$.MODULE$.forType(safeTypeTag(main)); }
/** * Groups items with the same key, assuming the items with the same key are next to each other in the * collection. It does not perform shuffle, therefore it is much faster than using much more * universal Spark RDD `groupByKey`. For this method to be useful with Cassandra tables, the key must * represent a prefix of the primary key, containing at least the partition key of the Cassandra * table. */ public JavaPairRDD<K, Collection<V>> spanByKey(ClassTag<K> keyClassTag) { ClassTag<Tuple2<K, Collection<V>>> tupleClassTag = classTag(Tuple2.class); ClassTag<Collection<V>> vClassTag = classTag(Collection.class); RDD<Tuple2<K, Collection<V>>> newRDD = pairRDDFunctions.spanByKey() .map(JavaApiHelper.<K, V, Seq<V>>valuesAsJavaCollection(), tupleClassTag); return new JavaPairRDD<>(newRDD, keyClassTag, vClassTag); } }
public CassandraJoinJavaRDD(CassandraJoinRDD<K, V> rdd, Class<K> keyClass, Class<V> valueClass) { super(rdd, safeClassTag(keyClass), safeClassTag(valueClass)); }
) { final TypeTags.TypeTag<Tuple7<A, B, C, D, E, F, G>> tupleTT = typeTag(Tuple7.class, typeTag(a), typeTag(b), typeTag(c), typeTag(d), typeTag(e), typeTag(f), typeTag(g) ); return new ClassBasedRowReaderFactory<>(tupleTT, tuple7ColumnMapper(a, b, c, d, e, f, g)); ) { final TypeTags.TypeTag<Tuple8<A, B, C, D, E, F, G, H>> tupleTT = typeTag(Tuple8.class, typeTag(a), typeTag(b), typeTag(c), typeTag(d), typeTag(e), typeTag(f), typeTag(g), typeTag(h) ); return new ClassBasedRowReaderFactory<>(tupleTT, tuple8ColumnMapper(a, b, c, d, e, f, g, h)); ) { final TypeTags.TypeTag<Tuple9<A, B, C, D, E, F, G, H, I>> tupleTT =