public static TimestampInfo generateEmptyTimestampInfo() { return new TimestampInfo(Optional.absent(), false); } }
public CassandraSchemaConverter(@NotEmpty final String keySpace, @NotEmpty final String tableName, @NonNull final Optional<Set<String>> filteredFields) { this(keySpace, tableName, TimestampInfo.generateEmptyTimestampInfo(), filteredFields); }
if (this.timestampInfo.hasTimestamp()) { final ByteBuffer bb = this.timestampInfo.isSaveAsLongType() ? LongType.instance.decompose(Long.parseLong(this.timestampInfo.getTimestamp().get())) : ByteBufferUtil.wrap(this.timestampInfo.getTimestamp().get()); insertStatement.value(SchemaUtil.DISPERSAL_TIMESTAMP, bb);
@Override public CassandraSchema convertToExternalSchema(final Schema commonSchema) { // todo T936057 - Need to handle more complex schemas (i.e Record inside record) final CassandraSchema cs = new CassandraSchema(this.keySpace, this.tableName); for (final Schema.Field field : commonSchema.getFields()) { // Cassandra does not support field names starting with _ if (this.shouldIncludeField(field.name())) { final String cassFieldType = CassandraSchemaField.convertFromAvroType(field.schema()); cs.addField(new CassandraSchemaField(field.name(), cassFieldType)); } } if (this.tsInfo.hasTimestamp()) { if (this.tsInfo.isSaveAsLongType()) { cs.addField(new CassandraSchemaField(SchemaUtil.DISPERSAL_TIMESTAMP, CassandraSchemaField.LONG_TYPE)); } else { cs.addField(new CassandraSchemaField(SchemaUtil.DISPERSAL_TIMESTAMP, CassandraSchemaField.STRING_TYPE)); } } return cs; }
final TimestampInfo tsInfo = new TimestampInfo(timestamp, false);
if (this.timestampInfo.hasTimestamp()) { final ByteBuffer bb = this.timestampInfo.isSaveAsLongType() ? LongType.instance.decompose(Long.parseLong(this.timestampInfo.getTimestamp().get())) : ByteBufferUtil.wrap(this.timestampInfo.getTimestamp().get()); row.addField( new CassandraDataField(ByteBufferUtil.wrap(SchemaUtil.DISPERSAL_TIMESTAMP), bb));
Optional.of(new HashSet<>(schemaFields)), requiredFields, TimestampInfo.generateEmptyTimestampInfo(), new ErrorExtractor());
private void testWriteAllFieldsMockDataToCassandra(boolean addLongTimestamp) { final JavaRDD<AvroPayload> testData = AvroPayloadUtil.generateTestData(this.jsc.get(), 100, StringTypes.EMPTY); final List<String> schemaFields = AvroPayloadUtil.getSchemaFields(); final List<String> partitionKeys = Collections.singletonList(schemaFields.get(0)); final List<ClusterKey> clusteringKeys = Collections.singletonList( new ClusterKey(schemaFields.get(1), ClusterKey.Order.DESC)); final List<String> requiredFields = Arrays.asList(schemaFields.get(0), schemaFields.get(1)); final Optional<String> timestamp = addLongTimestamp ? Optional.of(TEST_TIMESTAMP) : Optional.absent(); final TimestampInfo tsInfo = new TimestampInfo(timestamp, true); final CassandraSinkDataConverter dataconverter = new CassandraSinkDataConverter(AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY), new Configuration(), Optional.of(new HashSet<>(schemaFields)), requiredFields, tsInfo, new ErrorExtractor()); final CassandraSchemaConverter schemaConverter = new CassandraSchemaConverter(KEY_SPACE, TABLE, tsInfo, Optional.absent()); final CassandraSchema schema = schemaConverter.convertToExternalSchema( AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY)); final Optional<Long> ttl = Optional.of(10000L); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager(schema, partitionKeys, clusteringKeys, ttl); final CassandraSinkConfiguration conf = initializeConfiguration(false, addLongTimestamp); final CassandraSSTableSink sink = new CassandraSSTableSink(dataconverter, schemaManager, conf); sink.write(testData); validateCassandraTable(100, false, addLongTimestamp); }
Optional.of(new HashSet<>(schemaFields)), requiredFields, TimestampInfo.generateEmptyTimestampInfo(), new ErrorExtractor());
private void testWriteAllFieldsMockDataToCassandra(boolean addLongTimestamp) { final JavaRDD<AvroPayload> testData = AvroPayloadUtil.generateTestData(this.jsc.get(), 100, StringTypes.EMPTY); final List<String> schemaFields = AvroPayloadUtil.getSchemaFields(); final List<String> partitionKeys = Collections.singletonList(schemaFields.get(0)); final List<ClusterKey> clusteringKeys = Collections.singletonList( new ClusterKey(schemaFields.get(1), ClusterKey.Order.DESC)); final List<String> requiredFields = Arrays.asList(schemaFields.get(0), schemaFields.get(1)); final Optional<String> timestamp = addLongTimestamp ? Optional.of(TEST_TIMESTAMP) : Optional.absent(); final TimestampInfo tsInfo = new TimestampInfo(timestamp, true); final CassandraSinkCQLDataConverter converter = new CassandraSinkCQLDataConverter(AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY), new Configuration(), Optional.of(new HashSet<>(schemaFields)), requiredFields, tsInfo, new ErrorExtractor()); final CassandraSchemaConverter schemaConverter = new CassandraSchemaConverter(KEY_SPACE, TABLE, tsInfo, Optional.absent()); final CassandraSchema schema = schemaConverter.convertToExternalSchema( AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY)); final Optional<Long> ttl = Optional.of(10000L); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager(schema, partitionKeys, clusteringKeys, ttl); final CassandraSinkConfiguration conf = initializeConfiguration(false, addLongTimestamp); final CassandraClientSink sink = new CassandraClientSink(converter, schemaManager, conf); sink.write(testData); validateCassandraTable(100, false, addLongTimestamp); }
Optional.absent(), Collections.EMPTY_LIST, TimestampInfo.generateEmptyTimestampInfo(), new ErrorExtractor());
final TimestampInfo tsInfo = new TimestampInfo(timestamp, false);
@Test public void testConvertCommonToCassandraSchemaWithStringTimestamp() { final Schema record = SchemaBuilder.record("commonSchema") .fields() .name("field0").type().intType().noDefault() .endRecord(); final TimestampInfo tsInfo = new TimestampInfo(Optional.of("10000"), false); final CassandraSchemaConverter converter = new CassandraSchemaConverter("testKeyspace", "testTableName", tsInfo, Optional.absent()); final CassandraSchema cassSchema = converter.convertToExternalSchema(record); Assert.assertTrue(cassSchema.getFields().size() == 2); final CassandraSchemaField intField = cassSchema.getFields().get(0); Assert.assertEquals("field0", intField.getFieldName()); Assert.assertEquals(CassandraSchemaField.convertFromAvroType( SchemaTestUtil.getSchema(CassandraSchemaField.INT_TYPE)), intField.getType()); final CassandraSchemaField timestampField = cassSchema.getFields().get(1); Assert.assertEquals(SchemaUtil.DISPERSAL_TIMESTAMP, timestampField.getFieldName()); Assert.assertEquals(CassandraSchemaField.convertFromAvroType( SchemaTestUtil.getSchema(CassandraSchemaField.STRING_TYPE)), timestampField.getType()); }