private void printSourceDescription(final SourceDescription source) { writer().println(String.format("%-20s : %s", "Name", source.getName())); if (!source.isExtended()) { printSchema(source.getFields(), source.getKey()); writer().println( "For runtime statistics and query details run: DESCRIBE EXTENDED <Stream,Table>;"); return; } writer().println(String.format("%-20s : %s", "Type", source.getType())); printTopicInfo(source); writer().println(""); printSchema(source.getFields(), source.getKey()); printWriteQueries(source); writer().println(String.format( "%n%-20s%n%s", "Local runtime statistics", "------------------------" )); writer().println(source.getStatistics()); writer().println(source.getErrorStats()); writer().println(String.format( "(%s)", "Statistics of the local KSQL server interaction with the Kafka topic " + source.getTopic() )); }
private void printTopicInfo(final SourceDescription source) { final String timestamp = source.getTimestamp().isEmpty() ? "Not set - using <ROWTIME>" : source.getTimestamp(); writer().println(String.format("%-20s : %s", "Key field", source.getKey())); writer().println(String.format("%-20s : %s", "Key format", "STRING")); writer().println(String.format("%-20s : %s", "Timestamp field", timestamp)); writer().println(String.format("%-20s : %s", "Value format", source.getFormat())); if (!source.getTopic().isEmpty()) { writer().println(String.format( "%-20s : %s (partitions: %d, replication: %d)", "Kafka topic", source.getTopic(), source.getPartitions(), source.getReplication() )); } }
@Override public boolean equals(final Object o) { if (this == o) { return true; } if (!(o instanceof SourceDescription)) { return false; } final SourceDescription that = (SourceDescription) o; if (!Objects.equals(name, that.name)) { return false; } if (!Objects.equals(fields, that.fields)) { return false; } if (!Objects.equals(extended, that.extended)) { return false; } if (!Objects.equals(type, that.type)) { return false; } if (!Objects.equals(format, that.format)) { return false; } return equals2(that); }
@Test public void shouldReturnStatsBasedOnKafkaTopic() { // Given: final String kafkaTopicName = "kafka"; final StructuredDataSource dataSource = buildDataSource(kafkaTopicName); consumerCollector.onConsume(buildRecords(kafkaTopicName)); StreamsErrorCollector.recordError(APP_ID, kafkaTopicName); // When final SourceDescription sourceDescription = new SourceDescription( dataSource, true, "json", Collections.emptyList(), Collections.emptyList(), null); // Then: assertThat( sourceDescription.getStatistics(), containsString(ConsumerCollector.CONSUMER_TOTAL_MESSAGES)); assertThat( sourceDescription.getErrorStats(), containsString(StreamsErrorCollector.CONSUMER_FAILED_MESSAGES)); } }
@Test public void shouldPrintTopicDescribeExtended() throws IOException { final KsqlEntityList entityList = new KsqlEntityList(ImmutableList.of( new SourceDescriptionEntity( "e", new SourceDescription( "TestSource", Collections.emptyList(), Collections.emptyList(), buildTestSchema(2), DataSource.DataSourceType.KTABLE.getKqlType(), "key", "2000-01-01", "stats", "errors", true, "avro", "kadka-topic", 2, 1)))); console.printKsqlEntityList(entityList); final String output = terminal.getOutputString(); if (console.getOutputFormat() == OutputFormat.JSON) { assertThat(output, containsString("\"topic\" : \"kadka-topic\"")); } else { assertThat(output, containsString("Kafka topic : kadka-topic (partitions: 2, replication: 1)")); } }
dataSource.getKafkaTopicName(), extended && topicClient != null ? getPartitions( topicClient, dataSource.getKafkaTopicName() ), extended && topicClient != null ? getReplication( topicClient, dataSource.getKafkaTopicName()
@Test public void shouldDescribeStatement() { // Given: final List<RunningQuery> queries = createRunningQueries( "CREATE STREAM described_stream AS SELECT * FROM test_stream;" + "CREATE STREAM down_stream AS SELECT * FROM described_stream;", Collections.emptyMap()); // When: final SourceDescriptionEntity description = makeSingleRequest( "DESCRIBE DESCRIBED_STREAM;", SourceDescriptionEntity.class); // Then: final SourceDescription expectedDescription = new SourceDescription( ksqlEngine.getMetaStore().getSource("DESCRIBED_STREAM"), false, "JSON", Collections.singletonList(queries.get(1)), Collections.singletonList(queries.get(0)), null); assertThat(description.getSourceDescription(), is(expectedDescription)); }
private SourceDescription describeSource( final String name, final boolean extended ) { final StructuredDataSource dataSource = ksqlEngine.getMetaStore().getSource(name); if (dataSource == null) { throw new KsqlException(String.format( "Could not find STREAM/TABLE '%s' in the Metastore", name )); } return new SourceDescription( dataSource, extended, dataSource.getKsqlTopic().getKsqlTopicSerDe().getSerDe().name(), getQueries(q -> q.getSourceNames().contains(dataSource.getName())), getQueries(q -> q.getSinkNames().contains(dataSource.getName())), serviceContext.getTopicClient() ); }
@Test public void shouldShowStreamsExtended() { // Given: final Schema schema = SchemaBuilder.struct() .field("FIELD1", Schema.OPTIONAL_BOOLEAN_SCHEMA) .field("FIELD2", Schema.OPTIONAL_STRING_SCHEMA); ensureSource( DataSource.DataSourceType.KSTREAM, "new_stream", "new_topic", "new_ksql_topic", schema); // When: final SourceDescriptionList descriptionList = makeSingleRequest( "SHOW STREAMS EXTENDED;", SourceDescriptionList.class); // Then: assertThat(descriptionList.getSourceDescriptions(), containsInAnyOrder( new SourceDescription( ksqlEngine.getMetaStore().getSource("TEST_STREAM"), true, "JSON", Collections.emptyList(), Collections.emptyList(), kafkaTopicClient), new SourceDescription( ksqlEngine.getMetaStore().getSource("new_stream"), true, "JSON", Collections.emptyList(), Collections.emptyList(), kafkaTopicClient))); }
@Test public void shouldShowTablesExtended() { // Given: final Schema schema = SchemaBuilder.struct() .field("FIELD1", Schema.OPTIONAL_BOOLEAN_SCHEMA) .field("FIELD2", Schema.OPTIONAL_STRING_SCHEMA); ensureSource( DataSource.DataSourceType.KTABLE, "new_table", "new_topic", "new_ksql_topic", schema); // When: final SourceDescriptionList descriptionList = makeSingleRequest( "SHOW TABLES EXTENDED;", SourceDescriptionList.class); // Then: assertThat(descriptionList.getSourceDescriptions(), containsInAnyOrder( new SourceDescription( ksqlEngine.getMetaStore().getSource("TEST_TABLE"), true, "JSON", Collections.emptyList(), Collections.emptyList(), kafkaTopicClient), new SourceDescription( ksqlEngine.getMetaStore().getSource("new_table"), true, "JSON", Collections.emptyList(), Collections.emptyList(), kafkaTopicClient))); }
new SourceDescriptionEntity( "e", new SourceDescription( "TestSource", Collections.emptyList(), Collections.emptyList(), buildTestSchema(i), DataSource.DataSourceType.KTABLE.getKqlType(), "key", "2000-01-01", "stats",