private void checkForReservedToken( final Expression expression, final String alias, final String reservedToken) { if (alias.equalsIgnoreCase(reservedToken)) { final String text = expression.toString(); if (!text.substring(text.indexOf(".") + 1).equalsIgnoreCase(reservedToken)) { throw new ParseFailedException(reservedToken + " is a reserved token for implicit column. " + "You cannot use it as an alias for a column."); } } }
@Test public void testCreateTableWithTopic() { final String queryStr = "CREATE TABLE users (usertime bigint, userid varchar, regionid varchar, gender varchar) WITH (registered_topic = 'users_topic', key='userid', statestore='user_statestore');"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); final StatementRewriter statementRewriter = new StatementRewriter(); final Statement rewrittenStatement = (Statement) statementRewriter.process(statement, null); assertThat("testRegisterTopic failed.", rewrittenStatement instanceof CreateTable); final CreateTable createTable = (CreateTable)rewrittenStatement; assertThat(createTable.getName().toString(), equalTo("USERS")); assertThat(createTable.getElements().size(), equalTo(4)); assertThat(createTable.getElements().get(0).getName(), equalTo("USERTIME")); assertThat(createTable.getProperties().get(DdlConfig.TOPIC_NAME_PROPERTY).toString(), equalTo("'users_topic'")); }
@Test public void shouldNotCreateFunctionCallIfNotNeeded() { final String simpleQuery = "SELECT orderid FROM orders;"; final Statement statement = KSQL_PARSER.buildAst(simpleQuery, metaStore).get(0).getStatement(); final QuerySpecification querySpecification = getQuerySpecification(statement); assertThat(querySpecification.getSelect().getSelectItems().size(), equalTo(1)); final Expression col0 = ((SingleColumn) querySpecification.getSelect().getSelectItems().get(0)) .getExpression(); assertThat(col0, instanceOf(DereferenceExpression.class)); assertThat(col0.toString(), equalTo("ORDERS.ORDERID")); }
@Test public void shouldCreateCorrectFunctionCallExpressionWithSubscript() { final String simpleQuery = "SELECT arraycol[0]->name as n0, mapcol['key']->name as n1 FROM nested_stream;"; final Statement statement = KSQL_PARSER.buildAst(simpleQuery, metaStore).get(0).getStatement(); final QuerySpecification querySpecification = getQuerySpecification(statement); assertThat(querySpecification.getSelect().getSelectItems().size(), equalTo(2)); final Expression col0 = ((SingleColumn) querySpecification.getSelect().getSelectItems().get(0)) .getExpression(); final Expression col1 = ((SingleColumn) querySpecification.getSelect().getSelectItems().get(1)) .getExpression(); assertThat(col0, instanceOf(FunctionCall.class)); assertThat(col1, instanceOf(FunctionCall.class)); assertThat(col0.toString(), equalTo("FETCH_FIELD_FROM_STRUCT(NESTED_STREAM.ARRAYCOL[0], 'NAME')")); assertThat(col1.toString(), equalTo("FETCH_FIELD_FROM_STRUCT(NESTED_STREAM.MAPCOL['key'], 'NAME')")); }
@Test public void testCreateStreamWithTopic() { final String queryStr = "CREATE STREAM orders (ordertime bigint, orderid varchar, itemid varchar, orderunits " + "double) WITH (registered_topic = 'orders_topic' , key='ordertime');"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); final StatementRewriter statementRewriter = new StatementRewriter(); final Statement rewrittenStatement = (Statement) statementRewriter.process(statement, null); assertThat(rewrittenStatement, instanceOf(CreateStream.class)); final CreateStream createStream = (CreateStream)rewrittenStatement; assertThat(createStream.getName().toString(), equalTo("ORDERS")); assertThat(createStream.getElements().size(), equalTo(4)); assertThat(createStream.getElements().get(0).getName(), equalTo("ORDERTIME")); assertThat(createStream.getProperties().get(DdlConfig.TOPIC_NAME_PROPERTY).toString(), equalTo("'orders_topic'")); }
@Test public void shouldBuildSearchedCaseStatement() { // Given: final String statementString = "CREATE STREAM S AS SELECT CASE WHEN orderunits < 10 THEN 'small' WHEN orderunits < 100 THEN 'medium' ELSE 'large' END FROM orders;"; // When: final Statement statement = KSQL_PARSER.buildAst(statementString, metaStore).get(0) .getStatement(); // Then: final SearchedCaseExpression searchedCaseExpression = getSearchedCaseExpressionFromCsas(statement); assertThat(searchedCaseExpression.getWhenClauses().size(), equalTo(2)); assertThat(searchedCaseExpression.getWhenClauses().get(0).getOperand().toString(), equalTo("(ORDERS.ORDERUNITS < 10)")); assertThat(searchedCaseExpression.getWhenClauses().get(0).getResult().toString(), equalTo("'small'")); assertThat(searchedCaseExpression.getWhenClauses().get(1).getOperand().toString(), equalTo("(ORDERS.ORDERUNITS < 100)")); assertThat(searchedCaseExpression.getWhenClauses().get(1).getResult().toString(), equalTo("'medium'")); assertTrue(searchedCaseExpression.getDefaultValue().isPresent()); assertThat(searchedCaseExpression.getDefaultValue().get().toString(), equalTo("'large'")); }
@Test public void shouldCreateCorrectFunctionCallExpression() { final String simpleQuery = "SELECT iteminfo->category->name, address->state FROM orders;"; final Statement statement = KSQL_PARSER.buildAst(simpleQuery, metaStore).get(0).getStatement(); final QuerySpecification querySpecification = getQuerySpecification(statement); assertThat(querySpecification.getSelect().getSelectItems().size(), equalTo(2)); final Expression col0 = ((SingleColumn) querySpecification.getSelect().getSelectItems().get(0)) .getExpression(); final Expression col1 = ((SingleColumn) querySpecification.getSelect().getSelectItems().get(1)) .getExpression(); assertThat(col0, instanceOf(FunctionCall.class)); assertThat(col1, instanceOf(FunctionCall.class)); assertThat(col0.toString(), equalTo( "FETCH_FIELD_FROM_STRUCT(FETCH_FIELD_FROM_STRUCT(ORDERS.ITEMINFO, 'CATEGORY'), 'NAME')")); assertThat(col1.toString(), equalTo("FETCH_FIELD_FROM_STRUCT(ORDERS.ADDRESS, 'STATE')")); }
@Test public void testBinaryExpression() { final String queryStr = "SELECT col0+10, col2, col3-col1 FROM test1;"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); final StatementRewriter statementRewriter = new StatementRewriter(); final Statement rewrittenStatement = (Statement) statementRewriter.process(statement, null); assertThat(rewrittenStatement, instanceOf(Query.class)); final Query query = (Query) rewrittenStatement; assertThat(query.getQueryBody(), instanceOf(QuerySpecification.class)); final QuerySpecification querySpecification = (QuerySpecification)query.getQueryBody(); final SingleColumn column0 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(0); assertThat(column0.getAlias().get(), equalTo("KSQL_COL_0")); assertThat(column0.getExpression().toString(), equalTo("(TEST1.COL0 + 10)")); }
@Test public void testProjection() { final String queryStr = "SELECT col0, col2, col3 FROM test1;"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); final StatementRewriter statementRewriter = new StatementRewriter(); final Statement rewrittenStatement = (Statement) statementRewriter.process(statement, null); assertThat(rewrittenStatement, instanceOf(Query.class)); final Query query = (Query) rewrittenStatement; assertThat(query.getQueryBody(), instanceOf(QuerySpecification.class)); final QuerySpecification querySpecification = (QuerySpecification)query.getQueryBody(); assertThat(querySpecification.getSelect().getSelectItems().size() , equalTo(3)); assertThat(querySpecification.getSelect().getSelectItems().get(0), instanceOf(SingleColumn.class)); final SingleColumn column0 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(0); assertThat(column0.getAlias().get(), equalTo("COL0")); assertThat(column0.getExpression().toString(), equalTo("TEST1.COL0")); }
@Test public void testBooleanExpression() { final String queryStr = "SELECT col0 = 10, col2, col3 > col1 FROM test1;"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); final StatementRewriter statementRewriter = new StatementRewriter(); final Statement rewrittenStatement = (Statement) statementRewriter.process(statement, null); assertThat(rewrittenStatement, instanceOf(Query.class)); final Query query = (Query) rewrittenStatement; assertThat(query.getQueryBody(), instanceOf(QuerySpecification.class)); final QuerySpecification querySpecification = (QuerySpecification)query.getQueryBody(); final SingleColumn column0 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(0); assertThat(column0.getAlias().get(), equalTo("KSQL_COL_0")); assertThat(column0.getExpression().toString(), equalTo("(TEST1.COL0 = 10)")); assertThat(column0.getExpression(), instanceOf(ComparisonExpression.class)); }
@Test public void testCreateTableWithTopic() { final String queryStr = "CREATE TABLE users (usertime bigint, userid varchar, regionid varchar, gender varchar) WITH (registered_topic = 'users_topic', key='userid', statestore='user_statestore');"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); Assert.assertTrue("testRegisterTopic failed.", statement instanceof CreateTable); final CreateTable createTable = (CreateTable)statement; Assert.assertTrue("testCreateTable failed.", createTable.getName().toString().equalsIgnoreCase("USERS")); Assert.assertTrue("testCreateTable failed.", createTable.getElements().size() == 4); Assert.assertTrue("testCreateTable failed.", createTable.getElements().get(0).getName().toString().equalsIgnoreCase("usertime")); Assert.assertTrue("testCreateTable failed.", createTable.getProperties().get(DdlConfig.TOPIC_NAME_PROPERTY).toString().equalsIgnoreCase("'users_topic'")); }
@Test public void testCreateStreamAsSelect() { final String queryStr = "CREATE STREAM bigorders_json WITH (value_format = 'json', " + "kafka_topic='bigorders_topic') AS SELECT * FROM orders WHERE orderunits > 5 ;"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); final StatementRewriter statementRewriter = new StatementRewriter(); final Statement rewrittenStatement = (Statement) statementRewriter.process(statement, null); assertThat("testCreateStreamAsSelect failed.", rewrittenStatement instanceof CreateStreamAsSelect); final CreateStreamAsSelect createStreamAsSelect = (CreateStreamAsSelect)rewrittenStatement; assertThat(createStreamAsSelect.getName().toString(), equalTo("BIGORDERS_JSON")); assertThat(createStreamAsSelect.getQuery().getQueryBody(), instanceOf(QuerySpecification.class)); final QuerySpecification querySpecification = (QuerySpecification) createStreamAsSelect.getQuery().getQueryBody(); assertThat(querySpecification.getSelect().getSelectItems().size(), equalTo(8)); assertThat(querySpecification.getWhere().get().toString(), equalTo("(ORDERS.ORDERUNITS > 5)")); assertThat(((AliasedRelation)querySpecification.getFrom()).getAlias(), equalTo("ORDERS")); }
@Test public void testCreateStreamWithTopic() { final String queryStr = "CREATE STREAM orders (ordertime bigint, orderid varchar, itemid varchar, orderunits " + "double) WITH (registered_topic = 'orders_topic' , key='ordertime');"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); Assert.assertTrue("testCreateStream failed.", statement instanceof CreateStream); final CreateStream createStream = (CreateStream)statement; Assert.assertTrue("testCreateStream failed.", createStream.getName().toString().equalsIgnoreCase("ORDERS")); Assert.assertTrue("testCreateStream failed.", createStream.getElements().size() == 4); Assert.assertTrue("testCreateStream failed.", createStream.getElements().get(0).getName().toString().equalsIgnoreCase("ordertime")); Assert.assertTrue("testCreateStream failed.", createStream.getProperties().get(DdlConfig.TOPIC_NAME_PROPERTY).toString().equalsIgnoreCase("'orders_topic'")); }
@Test public void testRegisterTopic() { final String queryStr = "REGISTER TOPIC orders_topic WITH (value_format = 'avro', " + "avroschemafile='/Users/hojjat/avro_order_schema.avro',kafka_topic='orders_topic');"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); Assert.assertTrue("testRegisterTopic failed.", statement instanceof RegisterTopic); final RegisterTopic registerTopic = (RegisterTopic)statement; Assert.assertTrue("testRegisterTopic failed.", registerTopic .getName().toString().equalsIgnoreCase("ORDERS_TOPIC")); Assert.assertTrue("testRegisterTopic failed.", registerTopic.getProperties().size() == 3); Assert.assertTrue("testRegisterTopic failed.", registerTopic.getProperties().get(DdlConfig.VALUE_FORMAT_PROPERTY).toString().equalsIgnoreCase("'avro'")); }
@Test public void testBooleanExpression() { final String queryStr = "SELECT col0 = 10, col2, col3 > col1 FROM test1;"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); Assert.assertTrue("testBooleanExpression fails", statement instanceof Query); final Query query = (Query) statement; Assert.assertTrue("testProjection fails", query.getQueryBody() instanceof QuerySpecification); final QuerySpecification querySpecification = (QuerySpecification)query.getQueryBody(); final SingleColumn column0 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(0); Assert.assertTrue("testBooleanExpression fails", column0.getAlias().get().equalsIgnoreCase("KSQL_COL_0")); Assert.assertTrue("testBooleanExpression fails", column0.getExpression().toString().equalsIgnoreCase("(TEST1.COL0 = 10)")); }
@Test public void testBinaryExpression() { final String queryStr = "SELECT col0+10, col2, col3-col1 FROM test1;"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); Assert.assertTrue("testBinaryExpression fails", statement instanceof Query); final Query query = (Query) statement; Assert.assertTrue("testBinaryExpression fails", query.getQueryBody() instanceof QuerySpecification); final QuerySpecification querySpecification = (QuerySpecification)query.getQueryBody(); final SingleColumn column0 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(0); Assert.assertTrue("testBinaryExpression fails", column0.getAlias().get().equalsIgnoreCase("KSQL_COL_0")); Assert.assertTrue("testBinaryExpression fails", column0.getExpression().toString().equalsIgnoreCase("(TEST1.COL0 + 10)")); }
@Test public void testCreateStreamAsSelect() { final String queryStr = "CREATE STREAM bigorders_json WITH (value_format = 'json', " + "kafka_topic='bigorders_topic') AS SELECT * FROM orders WHERE orderunits > 5 ;"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); assertThat( statement, instanceOf(CreateStreamAsSelect.class)); final CreateStreamAsSelect createStreamAsSelect = (CreateStreamAsSelect)statement; assertThat(createStreamAsSelect.getName().toString().toLowerCase(), equalTo("bigorders_json")); assertThat(createStreamAsSelect.getQuery().getQueryBody(), instanceOf(QuerySpecification.class)); final QuerySpecification querySpecification = (QuerySpecification) createStreamAsSelect.getQuery().getQueryBody(); assertThat(querySpecification.getSelect().getSelectItems().size(), equalTo(8)); assertThat(querySpecification.getWhere().get().toString().toUpperCase(), equalTo("(ORDERS.ORDERUNITS > 5)")); assertThat(((AliasedRelation)querySpecification.getFrom()).getAlias().toUpperCase(), equalTo("ORDERS")); }
@Test public void testProjection() { final String queryStr = "SELECT col0, col2, col3 FROM test1;"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); Assert.assertTrue("testProjection fails", statement instanceof Query); final Query query = (Query) statement; Assert.assertTrue("testProjection fails", query.getQueryBody() instanceof QuerySpecification); final QuerySpecification querySpecification = (QuerySpecification)query.getQueryBody(); Assert.assertTrue("testProjection fails", querySpecification.getSelect().getSelectItems().size() == 3); Assert.assertTrue("testProjection fails", querySpecification.getSelect().getSelectItems().get(0) instanceof SingleColumn); final SingleColumn column0 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(0); Assert.assertTrue("testProjection fails", column0.getAlias().get().equalsIgnoreCase("COL0")); Assert.assertTrue("testProjection fails", column0.getExpression().toString().equalsIgnoreCase("TEST1.COL0")); }
@Test public void testSelectTumblingWindow() { final String queryStr = "select itemid, sum(orderunits) from orders window TUMBLING ( size 30 second) where orderunits > 5 group by itemid;"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); Assert.assertTrue("testSelectTumblingWindow failed.", statement instanceof Query); final Query query = (Query) statement; Assert.assertTrue("testSelectTumblingWindow failed.", query.getQueryBody() instanceof QuerySpecification); final QuerySpecification querySpecification = (QuerySpecification) query.getQueryBody(); Assert.assertTrue("testCreateTable failed.", querySpecification.getSelect().getSelectItems ().size() == 2); Assert.assertTrue("testSelectTumblingWindow failed.", querySpecification.getWhere().get().toString().equalsIgnoreCase("(ORDERS.ORDERUNITS > 5)")); Assert.assertTrue("testSelectTumblingWindow failed.", ((AliasedRelation)querySpecification.getFrom()).getAlias().equalsIgnoreCase("ORDERS")); Assert.assertTrue("testSelectTumblingWindow failed.", querySpecification .getWindowExpression().isPresent()); Assert.assertTrue("testSelectTumblingWindow failed.", querySpecification .getWindowExpression().get().toString().equalsIgnoreCase(" WINDOW STREAMWINDOW TUMBLING ( SIZE 30 SECONDS ) ")); }