final RDDWrapper<CassandraPayload> payloadWrapper = this.converter.map(data);
/** * This constructor gives the option to only convert certain fields from the schema * @param inputSchema * @param conf * @param fieldsToConvert * @param requiredFields */ public CassandraSinkDataConverter(@NonNull final Schema inputSchema, @NonNull final Configuration conf, @NonNull final Optional<Set<String>> fieldsToConvert, @NonNull final List<String> requiredFields, @NonNull final TimestampInfo timestampInfo, @NonNull final ErrorExtractor errorExtractor) { super(conf, errorExtractor); if (fieldsToConvert.isPresent()) { validate(fieldsToConvert.get(), requiredFields); } this.inputSchemaJson = inputSchema.toString(); this.requiredFields = Collections.unmodifiableList(requiredFields); this.fieldsToConvert = fieldsToConvert; this.timestampInfo = timestampInfo; }
final JavaRDD<AvroPayload> payloadRDD = this.jsc.get().parallelize(records); final CassandraSinkDataConverter csdc = new CassandraSinkDataConverter(avroSchema, new Configuration(), Optional.absent(), TimestampInfo.generateEmptyTimestampInfo(), new ErrorExtractor()); final JavaRDD<CassandraPayload> cassRDD = csdc.map(payloadRDD).getData(); final List<CassandraPayload> payloads = cassRDD.collect();
private void testWriteAllFieldsMockDataToCassandra(boolean addLongTimestamp) { final JavaRDD<AvroPayload> testData = AvroPayloadUtil.generateTestData(this.jsc.get(), 100, StringTypes.EMPTY); final List<String> schemaFields = AvroPayloadUtil.getSchemaFields(); final List<String> partitionKeys = Collections.singletonList(schemaFields.get(0)); final List<ClusterKey> clusteringKeys = Collections.singletonList( new ClusterKey(schemaFields.get(1), ClusterKey.Order.DESC)); final List<String> requiredFields = Arrays.asList(schemaFields.get(0), schemaFields.get(1)); final Optional<String> timestamp = addLongTimestamp ? Optional.of(TEST_TIMESTAMP) : Optional.absent(); final TimestampInfo tsInfo = new TimestampInfo(timestamp, true); final CassandraSinkDataConverter dataconverter = new CassandraSinkDataConverter(AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY), new Configuration(), Optional.of(new HashSet<>(schemaFields)), requiredFields, tsInfo, new ErrorExtractor()); final CassandraSchemaConverter schemaConverter = new CassandraSchemaConverter(KEY_SPACE, TABLE, tsInfo, Optional.absent()); final CassandraSchema schema = schemaConverter.convertToExternalSchema( AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY)); final Optional<Long> ttl = Optional.of(10000L); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager(schema, partitionKeys, clusteringKeys, ttl); final CassandraSinkConfiguration conf = initializeConfiguration(false, addLongTimestamp); final CassandraSSTableSink sink = new CassandraSSTableSink(dataconverter, schemaManager, conf); sink.write(testData); validateCassandraTable(100, false, addLongTimestamp); }
new CassandraSinkDataConverter( avroSchema, new Configuration(),
new CassandraSinkDataConverter(avroSchema, new Configuration(), Optional.of(new HashSet<>(schemaFields)),