@Override public int hashCode() { return value.hashCode(); } }
@Test public void shouldSetParentsCorrectlyForQueryWithUDF() { final Statement statement = getAstWithParent("SELECT lcase(col1), concat(col2,'hello'), floor(abs(col3)) FROM test1 t1;"); final Query query = (Query) statement; final QuerySpecification querySpecification = (QuerySpecification) query.getQueryBody(); final SingleColumn firstSelectItem = (SingleColumn) querySpecification.getSelect().getSelectItems().get(0); final FunctionCall functionCall = (FunctionCall) firstSelectItem.getExpression(); assertThat(functionCall.getParent().get(), equalTo(firstSelectItem)); assertThat(functionCall.getArguments().get(0).getParent().get(), equalTo(functionCall)); }
@Test public void shouldSetParentsCorrectlyForCreateStreamAsSelect() { final Statement statement = getAstWithParent("CREATE STREAM bigorders_json WITH (value_format = 'json', " + "kafka_topic='bigorders_topic') AS SELECT * FROM orders WHERE orderunits > 5 ;"); final CreateStreamAsSelect createStreamAsSelect = (CreateStreamAsSelect) statement; final Query query = createStreamAsSelect.getQuery(); final QuerySpecification querySpecification = (QuerySpecification) query.getQueryBody(); assertThat(querySpecification.getWhere().get().getParent().get(), equalTo(querySpecification)); assertThat(querySpecification.getFrom().getParent().get(), equalTo(querySpecification)); }
private void checkForReservedToken( final Expression expression, final String alias, final String reservedToken) { if (alias.equalsIgnoreCase(reservedToken)) { final String text = expression.toString(); if (!text.substring(text.indexOf(".") + 1).equalsIgnoreCase(reservedToken)) { throw new ParseFailedException(reservedToken + " is a reserved token for implicit column. " + "You cannot use it as an alias for a column."); } } }
@Override public int hashCode() { return value.hashCode(); } }
public Optional<String> getAvroSchemaIfAvroTopic( final SqlBaseParser.RegisterTopicContext registerTopicContext ) { final AstBuilder astBuilder = new AstBuilder(null); final RegisterTopic registerTopic = (RegisterTopic) astBuilder.visitRegisterTopic(registerTopicContext); if (registerTopic.getProperties().get(DdlConfig.VALUE_FORMAT_PROPERTY) == null) { throw new KsqlException("VALUE_FORMAT is not set for the topic."); } if (registerTopic.getProperties().get(DdlConfig.VALUE_FORMAT_PROPERTY).toString() .equalsIgnoreCase("'AVRO'")) { if (registerTopic.getProperties().containsKey(DdlConfig.AVRO_SCHEMA_FILE)) { final String avroSchema = getAvroSchema(AstBuilder.unquote( registerTopic.getProperties().get(DdlConfig.AVRO_SCHEMA_FILE).toString(), "'") ); return Optional.of(avroSchema); } else { throw new KsqlException( "You need to provide avro schema file path for topics in avro format."); } } return Optional.empty(); }
@Override public int hashCode() { return value.hashCode(); } }
@Test public void testCreateTableWithTopic() { final String queryStr = "CREATE TABLE users (usertime bigint, userid varchar, regionid varchar, gender varchar) WITH (registered_topic = 'users_topic', key='userid', statestore='user_statestore');"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); final StatementRewriter statementRewriter = new StatementRewriter(); final Statement rewrittenStatement = (Statement) statementRewriter.process(statement, null); assertThat("testRegisterTopic failed.", rewrittenStatement instanceof CreateTable); final CreateTable createTable = (CreateTable)rewrittenStatement; assertThat(createTable.getName().toString(), equalTo("USERS")); assertThat(createTable.getElements().size(), equalTo(4)); assertThat(createTable.getElements().get(0).getName(), equalTo("USERTIME")); assertThat(createTable.getProperties().get(DdlConfig.TOPIC_NAME_PROPERTY).toString(), equalTo("'users_topic'")); }
@Test public void testCreateStream() throws Exception { final String queryStr = "CREATE STREAM orders (ordertime bigint, orderid varchar, itemid varchar, orderunits " + "double) WITH (value_format = 'avro', " + "avroschemafile='/Users/hojjat/avro_order_schema.avro',kafka_topic='orders_topic');"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); final StatementRewriter statementRewriter = new StatementRewriter(); final Statement rewrittenStatement = (Statement) statementRewriter.process(statement, null); assertThat(rewrittenStatement, instanceOf(CreateStream.class)); final CreateStream createStream = (CreateStream)rewrittenStatement; assertThat(createStream.getName().toString(), equalTo("ORDERS")); assertThat(createStream.getElements().size(), equalTo(4)); assertThat(createStream.getElements().get(0).getName(), equalTo("ORDERTIME")); assertThat(createStream.getProperties().get(DdlConfig.KAFKA_TOPIC_NAME_PROPERTY).toString(), equalTo("'orders_topic'")); assertThat(createStream.getProperties().get(DdlConfig .VALUE_FORMAT_PROPERTY).toString(), equalTo("'avro'")); assertThat(createStream.getProperties().get(DdlConfig.AVRO_SCHEMA_FILE).toString(), equalTo("'/Users/hojjat/avro_order_schema.avro'")); }
@Test public void testCreateTable() { final String queryStr = "CREATE TABLE users (usertime bigint, userid varchar, regionid varchar, gender varchar) " + "WITH (kafka_topic = 'users_topic', value_format='json', key = 'userid');"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); final StatementRewriter statementRewriter = new StatementRewriter(); final Statement rewrittenStatement = (Statement) statementRewriter.process(statement, null); assertThat("testRegisterTopic failed.", rewrittenStatement instanceof CreateTable); final CreateTable createTable = (CreateTable)rewrittenStatement; assertThat(createTable.getName().toString(), equalTo("USERS")); assertThat(createTable.getElements().size(), equalTo(4)); assertThat(createTable.getElements().get(0).getName(), equalTo("USERTIME")); assertThat(createTable.getProperties().get(DdlConfig.KAFKA_TOPIC_NAME_PROPERTY) .toString(), equalTo("'users_topic'")); assertThat(createTable.getProperties().get(DdlConfig.VALUE_FORMAT_PROPERTY) .toString(), equalTo("'json'")); }
@Test public void shouldNotCreateFunctionCallIfNotNeeded() { final String simpleQuery = "SELECT orderid FROM orders;"; final Statement statement = KSQL_PARSER.buildAst(simpleQuery, metaStore).get(0).getStatement(); final QuerySpecification querySpecification = getQuerySpecification(statement); assertThat(querySpecification.getSelect().getSelectItems().size(), equalTo(1)); final Expression col0 = ((SingleColumn) querySpecification.getSelect().getSelectItems().get(0)) .getExpression(); assertThat(col0, instanceOf(DereferenceExpression.class)); assertThat(col0.toString(), equalTo("ORDERS.ORDERID")); }
@Test public void shouldCreateCorrectFunctionCallExpressionWithSubscript() { final String simpleQuery = "SELECT arraycol[0]->name as n0, mapcol['key']->name as n1 FROM nested_stream;"; final Statement statement = KSQL_PARSER.buildAst(simpleQuery, metaStore).get(0).getStatement(); final QuerySpecification querySpecification = getQuerySpecification(statement); assertThat(querySpecification.getSelect().getSelectItems().size(), equalTo(2)); final Expression col0 = ((SingleColumn) querySpecification.getSelect().getSelectItems().get(0)) .getExpression(); final Expression col1 = ((SingleColumn) querySpecification.getSelect().getSelectItems().get(1)) .getExpression(); assertThat(col0, instanceOf(FunctionCall.class)); assertThat(col1, instanceOf(FunctionCall.class)); assertThat(col0.toString(), equalTo("FETCH_FIELD_FROM_STRUCT(NESTED_STREAM.ARRAYCOL[0], 'NAME')")); assertThat(col1.toString(), equalTo("FETCH_FIELD_FROM_STRUCT(NESTED_STREAM.MAPCOL['key'], 'NAME')")); }
@Test public void testCreateStreamWithTopic() { final String queryStr = "CREATE STREAM orders (ordertime bigint, orderid varchar, itemid varchar, orderunits " + "double) WITH (registered_topic = 'orders_topic' , key='ordertime');"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); final StatementRewriter statementRewriter = new StatementRewriter(); final Statement rewrittenStatement = (Statement) statementRewriter.process(statement, null); assertThat(rewrittenStatement, instanceOf(CreateStream.class)); final CreateStream createStream = (CreateStream)rewrittenStatement; assertThat(createStream.getName().toString(), equalTo("ORDERS")); assertThat(createStream.getElements().size(), equalTo(4)); assertThat(createStream.getElements().get(0).getName(), equalTo("ORDERTIME")); assertThat(createStream.getProperties().get(DdlConfig.TOPIC_NAME_PROPERTY).toString(), equalTo("'orders_topic'")); }
@Test public void shouldBuildSearchedCaseStatement() { // Given: final String statementString = "CREATE STREAM S AS SELECT CASE WHEN orderunits < 10 THEN 'small' WHEN orderunits < 100 THEN 'medium' ELSE 'large' END FROM orders;"; // When: final Statement statement = KSQL_PARSER.buildAst(statementString, metaStore).get(0) .getStatement(); // Then: final SearchedCaseExpression searchedCaseExpression = getSearchedCaseExpressionFromCsas(statement); assertThat(searchedCaseExpression.getWhenClauses().size(), equalTo(2)); assertThat(searchedCaseExpression.getWhenClauses().get(0).getOperand().toString(), equalTo("(ORDERS.ORDERUNITS < 10)")); assertThat(searchedCaseExpression.getWhenClauses().get(0).getResult().toString(), equalTo("'small'")); assertThat(searchedCaseExpression.getWhenClauses().get(1).getOperand().toString(), equalTo("(ORDERS.ORDERUNITS < 100)")); assertThat(searchedCaseExpression.getWhenClauses().get(1).getResult().toString(), equalTo("'medium'")); assertTrue(searchedCaseExpression.getDefaultValue().isPresent()); assertThat(searchedCaseExpression.getDefaultValue().get().toString(), equalTo("'large'")); }
@Test public void shouldCreateCorrectFunctionCallExpression() { final String simpleQuery = "SELECT iteminfo->category->name, address->state FROM orders;"; final Statement statement = KSQL_PARSER.buildAst(simpleQuery, metaStore).get(0).getStatement(); final QuerySpecification querySpecification = getQuerySpecification(statement); assertThat(querySpecification.getSelect().getSelectItems().size(), equalTo(2)); final Expression col0 = ((SingleColumn) querySpecification.getSelect().getSelectItems().get(0)) .getExpression(); final Expression col1 = ((SingleColumn) querySpecification.getSelect().getSelectItems().get(1)) .getExpression(); assertThat(col0, instanceOf(FunctionCall.class)); assertThat(col1, instanceOf(FunctionCall.class)); assertThat(col0.toString(), equalTo( "FETCH_FIELD_FROM_STRUCT(FETCH_FIELD_FROM_STRUCT(ORDERS.ITEMINFO, 'CATEGORY'), 'NAME')")); assertThat(col1.toString(), equalTo("FETCH_FIELD_FROM_STRUCT(ORDERS.ADDRESS, 'STATE')")); }
@Test public void shouldCreateCorrectFunctionCallExpressionWithSubscriptWithExpressionIndex() { final String simpleQuery = "SELECT arraycol[CAST (item->id AS INTEGER)]->name as n0, mapcol['key']->name as n1 FROM nested_stream;"; final Statement statement = KSQL_PARSER.buildAst(simpleQuery, metaStore).get(0).getStatement(); final QuerySpecification querySpecification = getQuerySpecification(statement); assertThat(querySpecification.getSelect().getSelectItems().size(), equalTo(2)); final Expression col0 = ((SingleColumn) querySpecification.getSelect().getSelectItems().get(0)) .getExpression(); final Expression col1 = ((SingleColumn) querySpecification.getSelect().getSelectItems().get(1)) .getExpression(); assertThat(col0, instanceOf(FunctionCall.class)); assertThat(col1, instanceOf(FunctionCall.class)); assertThat(col0.toString(), equalTo( "FETCH_FIELD_FROM_STRUCT(NESTED_STREAM.ARRAYCOL[CAST(FETCH_FIELD_FROM_STRUCT(NESTED_STREAM.ITEM, 'ID') AS INTEGER)], 'NAME')")); assertThat(col1.toString(), equalTo("FETCH_FIELD_FROM_STRUCT(NESTED_STREAM.MAPCOL['key'], 'NAME')")); }
@Test public void testUDF() { final String queryStr = "SELECT lcase(col1), concat(col2,'hello'), floor(abs(col3)) FROM test1 t1;"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); final StatementRewriter statementRewriter = new StatementRewriter(); final Statement rewrittenStatement = (Statement) statementRewriter.process(statement, null); assertThat(rewrittenStatement, instanceOf(Query.class)); final Query query = (Query) rewrittenStatement; assertThat(query.getQueryBody(), instanceOf(QuerySpecification.class)); final QuerySpecification querySpecification = (QuerySpecification)query.getQueryBody(); final SingleColumn column0 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(0); assertThat(column0.getAlias().get(), equalTo("KSQL_COL_0")); assertThat(column0.getExpression().toString(), equalTo("LCASE(T1.COL1)")); final SingleColumn column1 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(1); assertThat(column1.getAlias().get(), equalTo("KSQL_COL_1")); assertThat(column1.getExpression().toString(), equalTo("CONCAT(T1.COL2, 'hello')")); final SingleColumn column2 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(2); assertThat(column2.getAlias().get(), equalTo("KSQL_COL_2")); assertThat(column2.getExpression().toString(), equalTo("FLOOR(ABS(T1.COL3))")); }
@Test public void testBooleanLogicalExpression() { final String queryStr = "SELECT 10, col2, 'test', 2.5, true, -5 FROM test1 WHERE col1 = 10 AND col2 LIKE 'val' OR col4 > 2.6 ;"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); final StatementRewriter statementRewriter = new StatementRewriter(); final Statement rewrittenStatement = (Statement) statementRewriter.process(statement, null); assertThat(rewrittenStatement, instanceOf(Query.class)); final Query query = (Query) rewrittenStatement; assertThat(query.getQueryBody(), instanceOf(QuerySpecification.class)); final QuerySpecification querySpecification = (QuerySpecification)query.getQueryBody(); final SingleColumn column0 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(0); assertThat(column0.getAlias().get(), equalTo("KSQL_COL_0")); assertThat(column0.getExpression().toString(), equalTo("10")); final SingleColumn column1 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(1); assertThat(column1.getAlias().get(), equalTo("COL2")); assertThat(column1.getExpression().toString(), equalTo("TEST1.COL2")); final SingleColumn column2 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(2); assertThat(column2.getAlias().get(), equalTo("KSQL_COL_2")); assertThat(column2.getExpression().toString(), equalTo("'test'")); }
@Test public void testLiterals() { final String queryStr = "SELECT 10, col2, 'test', 2.5, true, -5 FROM test1;"; final Statement statement = KSQL_PARSER.buildAst(queryStr, metaStore).get(0).getStatement(); final StatementRewriter statementRewriter = new StatementRewriter(); final Statement rewrittenStatement = (Statement) statementRewriter.process(statement, null); assertThat(rewrittenStatement, instanceOf(Query.class)); final Query query = (Query) rewrittenStatement; assertThat(query.getQueryBody(), instanceOf(QuerySpecification.class)); final QuerySpecification querySpecification = (QuerySpecification)query.getQueryBody(); final SingleColumn column0 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(0); assertThat(column0.getAlias().get(), equalTo("KSQL_COL_0")); assertThat(column0.getExpression().toString(), equalTo("10")); final SingleColumn column1 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(1); assertThat(column1.getAlias().get(), equalTo("COL2")); assertThat(column1.getExpression().toString(), equalTo("TEST1.COL2")); final SingleColumn column2 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(2); assertThat(column2.getAlias().get(), equalTo("KSQL_COL_2")); assertThat(column2.getExpression().toString(), equalTo("'test'")); final SingleColumn column3 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(3); assertThat(column3.getAlias().get(), equalTo("KSQL_COL_3")); assertThat(column3.getExpression().toString(), equalTo("2.5")); final SingleColumn column4 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(4); assertThat(column4.getAlias().get(), equalTo("KSQL_COL_4")); assertThat(column4.getExpression().toString(), equalTo("true")); final SingleColumn column5 = (SingleColumn)querySpecification.getSelect().getSelectItems().get(5); assertThat(column5.getAlias().get(), equalTo("KSQL_COL_5")); assertThat(column5.getExpression().toString(), equalTo("-5")); }