@Test public void shouldTreatNullAsNull() throws JsonProcessingException { final Map<String, Object> row = new HashMap<>(); row.put("ordertime", null); row.put("@orderid", null); row.put("itemid", null); row.put("orderunits", null); row.put("arrayCol", new Double[]{0.0, null}); row.put("mapCol", null); final GenericRow expected = new GenericRow(Arrays.asList(null, null, null, null, new Double[]{0.0, null}, null)); final GenericRow genericRow = ksqlJsonDeserializer.deserialize( "", objectMapper.writeValueAsBytes(row)); assertThat(genericRow, equalTo(expected)); }
@Test public void shouldDeserializeJsonCorrectly() throws JsonProcessingException { final Map<String, Object> orderRow = new HashMap<>(); orderRow.put("ordertime", 1511897796092L); orderRow.put("@orderid", 1L); orderRow.put("itemid", "Item_1"); orderRow.put("orderunits", 10.0); orderRow.put("arraycol", new Double[]{10.0, 20.0}); orderRow.put("mapcol", Collections.singletonMap("key1", 10.0)); final byte[] jsonBytes = objectMapper.writeValueAsBytes(orderRow); final GenericRow genericRow = ksqlJsonDeserializer.deserialize("", jsonBytes); assertThat(genericRow.getColumns().size(), equalTo(6)); assertThat(genericRow.getColumns().get(0), equalTo(1511897796092L)); assertThat(genericRow.getColumns().get(1), equalTo(1L)); assertThat(genericRow.getColumns().get(2), equalTo("Item_1")); assertThat(genericRow.getColumns().get(3), equalTo(10.0)); }
@Test public void shouldDeserializeEvenWithMissingFields() throws JsonProcessingException { final Map<String, Object> orderRow = new HashMap<>(); orderRow.put("ordertime", 1511897796092L); orderRow.put("@orderid", 1L); orderRow.put("itemid", "Item_1"); orderRow.put("orderunits", 10.0); final byte[] jsonBytes = objectMapper.writeValueAsBytes(orderRow); final GenericRow genericRow = ksqlJsonDeserializer.deserialize("", jsonBytes); assertThat(genericRow.getColumns().size(), equalTo(6)); assertThat(genericRow.getColumns().get(0), equalTo(1511897796092L)); assertThat(genericRow.getColumns().get(1), equalTo(1L)); assertThat(genericRow.getColumns().get(2), equalTo("Item_1")); assertThat(genericRow.getColumns().get(3), equalTo(10.0)); assertThat(genericRow.getColumns().get(4), is(nullValue())); assertThat(genericRow.getColumns().get(5), is(nullValue())); }
@Test public void shouldCreateJsonStringForStructIfDefinedAsVarchar() throws JsonProcessingException { final Schema schema = SchemaBuilder.struct() .field("itemid".toUpperCase(), Schema.OPTIONAL_STRING_SCHEMA) .build(); final KsqlJsonDeserializer deserializer = new KsqlJsonDeserializer( schema, false, recordLogger); final GenericRow expected = new GenericRow(Collections.singletonList( "{\"CATEGORY\":{\"ID\":2,\"NAME\":\"Food\"},\"ITEMID\":6,\"NAME\":\"Item_6\"}")); final GenericRow genericRow = deserializer.deserialize("", "{\"itemid\":{\"CATEGORY\":{\"ID\":2,\"NAME\":\"Food\"},\"ITEMID\":6,\"NAME\":\"Item_6\"}}".getBytes(StandardCharsets.UTF_8)); assertThat(genericRow, equalTo(expected)); }
@Test public void shouldLogDeserializationErrors() { // When: Throwable cause = null; final byte[] data = "{foo".getBytes(StandardCharsets.UTF_8); try { ksqlJsonDeserializer.deserialize("", data); fail("deserialize should have thrown"); } catch (final SerializationException e) { cause = e.getCause(); } // Then: SerdeTestUtils.shouldLogError( recordLogger, ProcessingLogMessageFactory.deserializationErrorMsg(cause, Optional.ofNullable(data)).get()); } }
@Test public void shouldDeserializeJsonCorrectlyWithRedundantFields() throws JsonProcessingException { final Map<String, Object> orderRow = new HashMap<>(); orderRow.put("ordertime", 1511897796092L); orderRow.put("@orderid", 1L); orderRow.put("itemid", "Item_1"); orderRow.put("orderunits", 10.0); orderRow.put("arraycol", new Double[]{10.0, 20.0}); orderRow.put("mapcol", Collections.singletonMap("key1", 10.0)); final byte[] jsonBytes = objectMapper.writeValueAsBytes(orderRow); final Schema newOrderSchema = SchemaBuilder.struct() .field("ordertime".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("orderid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("itemid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_STRING_SCHEMA) .field("orderunits".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_FLOAT64_SCHEMA) .build(); final KsqlJsonDeserializer ksqlJsonDeserializer = new KsqlJsonDeserializer( newOrderSchema, false, recordLogger); final GenericRow genericRow = ksqlJsonDeserializer.deserialize("", jsonBytes); assertThat(genericRow.getColumns().size(), equalTo(4)); assertThat(genericRow.getColumns().get(0), equalTo(1511897796092L)); assertThat(genericRow.getColumns().get(1), equalTo(1L)); assertThat(genericRow.getColumns().get(2), equalTo("Item_1")); assertThat(genericRow.getColumns().get(3), equalTo(10.0)); }