@Override public CassandraSchema convertToExternalSchema(final Schema commonSchema) { // todo T936057 - Need to handle more complex schemas (i.e Record inside record) final CassandraSchema cs = new CassandraSchema(this.keySpace, this.tableName); for (final Schema.Field field : commonSchema.getFields()) { // Cassandra does not support field names starting with _ if (this.shouldIncludeField(field.name())) { final String cassFieldType = CassandraSchemaField.convertFromAvroType(field.schema()); cs.addField(new CassandraSchemaField(field.name(), cassFieldType)); } } if (this.tsInfo.hasTimestamp()) { if (this.tsInfo.isSaveAsLongType()) { cs.addField(new CassandraSchemaField(SchemaUtil.DISPERSAL_TIMESTAMP, CassandraSchemaField.LONG_TYPE)); } else { cs.addField(new CassandraSchemaField(SchemaUtil.DISPERSAL_TIMESTAMP, CassandraSchemaField.STRING_TYPE)); } } return cs; }
/** * @return * Insert statement used for sstable loading using the fields and ttl value */ public String generateInsertStmt() { final String fields = this.schema.getFields() .stream() .map(f -> StringTypes.SPACE + f.getFieldName()) .collect(Collectors.joining(",")); final String values = this.schema.getFields() .stream() .map(f -> "?") .collect(Collectors.joining(",")); final String ttlStr = this.ttl.isPresent() ? "USING TTL " + this.ttl.get().toString() : StringTypes.EMPTY; return String.format("INSERT INTO %s.%s ( %s ) VALUES ( %s ) %s", this.schema.getKeySpace(), this.schema.getTableName(), fields, values, ttlStr); }
/** * @return * Returns all the column names for a specific Cassandra table */ public String getColumnNamesFromTableQuery() { return String.format("SELECT %s FROM %s WHERE %s = '%s' " + "AND table_name = '%s'", COLUMN_NAME, SYSTEM_SCHEMA_COLS, KEYSPACE_NAME, this.schema.getKeySpace(), this.schema.getTableName()); }
@Test(expected = IllegalStateException.class) public void testPartitionKeyIsNotInFieldNames() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Collections.singletonList("non_field_name"), Collections.EMPTY_LIST); Assert.fail(); }
/** * @return * Returns a comma separated list of all the fields in the schema */ private String generateFieldsSyntax() { final List<String> fields = this.schema.getFields().stream() .map(field -> field.toString()).collect(Collectors.toList()); return joiner.join(fields); }
@Test(expected = IllegalStateException.class) public void testEmptyTableName() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, StringTypes.EMPTY, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Collections.EMPTY_LIST); Assert.fail(); }
@Test public void testConvertCommonToCassandraSchemaWithFilteredFields() { final Schema record = SchemaBuilder.record("commonSchema") .fields() .name("field0").type().intType().noDefault() .name("field1").type().doubleType().noDefault() .name("field2").type().stringType().noDefault() .name("field3").type().booleanType().noDefault() .endRecord(); final CassandraSchemaConverter converter = new CassandraSchemaConverter("testKeyspace", "testTableName", Optional.of(new HashSet<>(Arrays.asList("field2", "field3")))); final CassandraSchema cassSchema = converter.convertToExternalSchema(record); Assert.assertTrue(cassSchema.getFields().size() == 2); Assert.assertEquals("field2", cassSchema.getFields().get(0).getFieldName()); Assert.assertEquals("field3", cassSchema.getFields().get(1).getFieldName()); Assert.assertEquals(CassandraSchemaField.convertFromAvroType( SchemaTestUtil.getSchema(CassandraSchemaField.STRING_TYPE)), cassSchema.getFields().get(0).getType()); Assert.assertEquals(CassandraSchemaField.convertFromAvroType( SchemaTestUtil.getSchema(CassandraSchemaField.BOOLEAN_TYPE)), cassSchema.getFields().get(1).getType()); } }
/** * @return * Returns all the column names for a specific column family */ public String getColumnNamesFromColumnFamilyQuery() { return String.format("SELECT %s FROM %s WHERE %s = '%s' " + "AND columnfamily_name = '%s'", COLUMN_NAME, SYSTEM_SCHEMA_COLS, KEYSPACE_NAME, this.schema.getKeySpace(), this.schema.getTableName()); }
@Test(expected = IllegalStateException.class) public void testFieldsIsNull() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, null); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Collections.EMPTY_LIST); Assert.fail(); }
/** * The schema is the source of truth here and if any of the schema field names aren't found * in the existing columns we need to alter that table and update the schema * * @param existingColumns * @return List of strings, one for each column that needs to be added */ public List<String> generateAlterTableStmt(final List<String> existingColumns) { final List<String> fieldNames = this.schema.getFields().stream() .map(field -> field.getFieldName()).collect(Collectors.toList()); log.info("Existing field names in schema: {}", Arrays.toString(fieldNames.toArray())); final List<String> missingCols = fieldNames.stream() .filter(field -> !existingColumns.contains(field)) .collect(Collectors.toList()); log.info("Missing columns (if any): {}", Arrays.toString(missingCols.toArray())); return this.schema.getFields().stream() .filter(field -> missingCols.contains(field.getFieldName())) .map(field -> String.format("ALTER TABLE %s.%s ADD %s", this.schema.getKeySpace(), this.schema.getTableName(), field.toString())) .collect(Collectors.toList()); }
@Test public void testConvertCommonToCassandraSchemaWithStringTimestamp() { final Schema record = SchemaBuilder.record("commonSchema") .fields() .name("field0").type().intType().noDefault() .endRecord(); final TimestampInfo tsInfo = new TimestampInfo(Optional.of("10000"), false); final CassandraSchemaConverter converter = new CassandraSchemaConverter("testKeyspace", "testTableName", tsInfo, Optional.absent()); final CassandraSchema cassSchema = converter.convertToExternalSchema(record); Assert.assertTrue(cassSchema.getFields().size() == 2); final CassandraSchemaField intField = cassSchema.getFields().get(0); Assert.assertEquals("field0", intField.getFieldName()); Assert.assertEquals(CassandraSchemaField.convertFromAvroType( SchemaTestUtil.getSchema(CassandraSchemaField.INT_TYPE)), intField.getType()); final CassandraSchemaField timestampField = cassSchema.getFields().get(1); Assert.assertEquals(SchemaUtil.DISPERSAL_TIMESTAMP, timestampField.getFieldName()); Assert.assertEquals(CassandraSchemaField.convertFromAvroType( SchemaTestUtil.getSchema(CassandraSchemaField.STRING_TYPE)), timestampField.getType()); }
public String generateCreateTableStmt() { final String fields = generateFieldsSyntax(); final String primaryKeyStmt = generatePrimaryKeySyntax(); final String clusteringOrder = generateClusteringOrderSyntax(); final String createStatement = String.format("CREATE TABLE IF NOT EXISTS %s.%s (%s, %s) %s", this.schema.getKeySpace(), this.schema.getTableName(), fields, primaryKeyStmt, clusteringOrder); log.info("Generated table schema is {}", createStatement); return createStatement; }
@Test(expected = IllegalStateException.class) public void testFieldsIsEmpty() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, Collections.EMPTY_LIST); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Collections.EMPTY_LIST); Assert.fail(); }
Preconditions.checkState(!Strings.isNullOrEmpty(this.schema.getKeySpace()), "Keyspace is missing"); Preconditions.checkState(!Strings.isNullOrEmpty(this.schema.getTableName()), "Table name is missing"); Preconditions.checkState(this.schema.getFields() != null && !this.schema.getFields().isEmpty(), "Schema fields missing"); Preconditions.checkState(this.partitionKeys != null && !this.partitionKeys.isEmpty(), this.schema.getFields() .stream() .map(f -> f.getFieldName().toLowerCase())
@Test public void testConvertCommonToCassandraSchemaRemovesFieldStartingWithUnderscore() { final Schema record = SchemaBuilder.record("commonSchema") .fields() .name("_field0").type().intType().noDefault() .name("_field1").type().doubleType().noDefault() .name("_field2").type().stringType().noDefault() .name("field3").type().booleanType().noDefault() .endRecord(); final CassandraSchemaConverter converter = new CassandraSchemaConverter("testKeyspace", "testTableName", Optional.absent()); final CassandraSchema cassSchema = converter.convertToExternalSchema(record); Assert.assertTrue(cassSchema.getFields().size() == 1); Assert.assertEquals("field3", cassSchema.getFields().get(0).getFieldName()); Assert.assertEquals(CassandraSchemaField.convertFromAvroType( SchemaTestUtil.getSchema(CassandraSchemaField.BOOLEAN_TYPE)), cassSchema.getFields().get(0).getType()); }
@Test(expected = IllegalStateException.class) public void testPartitionKeysIsEmpty() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Collections.EMPTY_LIST, Collections.EMPTY_LIST); Assert.fail(); }
@Test(expected = IllegalStateException.class) public void tesClusterKeyIsNotInFieldNames() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Collections.singletonList("country_code"), Collections.singletonList(new ClusterKey("non_cluster_field_name", ClusterKey.Order.ASC))); Assert.fail(); }
@Test public void testAlterTableWithOneMultipleColumns() { createBasicTable(); final List<CassandraSchemaField> joinedList = Lists.newArrayList(Iterables.concat(fields, newFields)); final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, joinedList); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("city", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); final List<String> alterTableQueries = schemaManager.generateAlterTableStmt(fields.stream().map(f -> f.getFieldName()).collect(Collectors.toList())); Assert.assertTrue(alterTableQueries.size() == 2); Assert.assertEquals("ALTER TABLE marmaray.crossfit_gyms ADD new_field1 text", alterTableQueries.get(0)); Assert.assertEquals("ALTER TABLE marmaray.crossfit_gyms ADD new_field2 text", alterTableQueries.get(1)); try (final Session session = getSession()) { alterTableQueries.stream().forEach(query -> session.execute(query)); validateAlterTable(session, newFields.stream().map(f -> f.getFieldName()).collect(Collectors.toList())); } }
@Test public void testAlterTableWithOneNewColumn() { createBasicTable(); final List<CassandraSchemaField> joinedList = Lists.newArrayList(Iterables.concat(fields, Collections.singletonList(new CassandraSchemaField("new_field1", "text")))); final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, joinedList); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("city", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); final List<String> alterTableQueries = schemaManager.generateAlterTableStmt(fields.stream().map(f -> f.getFieldName()).collect(Collectors.toList())); Assert.assertTrue(alterTableQueries.size() == 1); Assert.assertEquals("ALTER TABLE marmaray.crossfit_gyms ADD new_field1 text", alterTableQueries.get(0)); try (final Session session = getSession()) { alterTableQueries.stream().forEach(query -> session.execute(query)); validateAlterTable(session, Collections.singletonList("new_field1")); } }