How to use

getTokens

method

in

org.antlr.v4.runtime.CommonTokenStream

  private void extractComments() {
    for (Token token : tokenStream.getTokens()) {
      if (token.getChannel() != Token.HIDDEN_CHANNEL) {
        continue;
      }
      if (ListenerUtil.isSingleLineComment(token)) {
        singleLineComments.add(token);
      }
      if (ListenerUtil.isMultilineComment(token)) {
        multilineComments.add(token);
      }
    }
  }
}

List<Token> allTokens = tokens.getTokens();
if (stop != null && allTokens != null && !allTokens.isEmpty()) {
  Token last = allTokens.get(allTokens.size() - 1);

List<Token> tokens = tokenStream.getTokens();

@Override
public boolean hasNext() {
  if (direction < 0)
    return tokens != null && tokenIndex >= 0;
  else
    return tokens != null && tokenIndex < tokens.getTokens().size();
}

@Override
public boolean hasNext() {
  if (direction < 0)
    return tokens != null && tokenIndex >= 0;
  else
    return tokens != null && tokenIndex < tokens.getTokens().size();
}

/**
 * Dumps all tokens to console.
 */
public void printLexerTokens() {
  for (Token token : this.tokens.getTokens()) {
    System.out.print("'" + token.getText() + "<" + token.getType() + ">' ");
  }
}

@Override
public Token next() {
  if (!hasNext()) {
    throw new NoSuchElementException();
  }
  if (tokens != null && tokenIndex >= 0) {
    Token retval = tokens.getTokens().get(tokenIndex);
    tokenIndex += direction;
    return retval;
  }
  return null;
}

@Override
public Token next() {
  if (!hasNext()) {
    throw new NoSuchElementException();
  }
  if (tokens != null && tokenIndex >= 0) {
    Token retval = tokens.getTokens().get(tokenIndex);
    tokenIndex += direction;
    return retval;
  }
  return null;
}

/**
 * 在语法解析器可以定位到错误单词的基础下获取错误单词
 */
private String getOffendingSymbolWithHint(Recognizer<?, ?> recognizer, Object offendingSymbol)
{
  Token token = (Token)offendingSymbol;
  String tokenText = token.getText();
  if (tokenText.equals(SYMBOL_EOF))
  {
    List<Token> allTokens = ((org.antlr.v4.runtime.CommonTokenStream)recognizer.getInputStream()).getTokens();
    int tokensCount = allTokens.size();
    return (tokensCount < MIN_SIZE_FOR_TOKENS) ? "" : allTokens.get(tokensCount - MIN_SIZE_FOR_TOKENS)
      .getText();
  }
  return tokenText;
}
/**

public static List<CommonToken> copy(CommonTokenStream tokens) {
  List<CommonToken> copy = new ArrayList<>();
  tokens.fill();
  for (Token t : tokens.getTokens()) {
    copy.add(new CommonToken(t));
  }
  return copy;
}

public static void printLex(InputStream in) throws Exception {
  Java7Lexer lex = new Java7Lexer(new ANTLRInputStream(in));
  CommonTokenStream tokens = new CommonTokenStream(lex);
  tokens.fill();
  for (Token token : tokens.getTokens()) {
    System.out.println(token.getType() + " " + token.getText());
  }
}

public static void printLex(InputStream in) throws Exception {
  Java7Lexer lex = new Java7Lexer(new ANTLRInputStream(in));
  CommonTokenStream tokens = new CommonTokenStream(lex);
  tokens.fill();
  for (Token token : tokens.getTokens()) {
    System.out.println(token.getType() + " " + token.getText());
  }
}

protected Queue<Token> getQueue(String input) {
 DSWrapperLexer lexer = dsWrapperLexerThreadLocal.get();
 lexer.setInputStream(new ANTLRInputStream(input));
 CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
 commonTokenStream.fill();
 List<Token> tokens = commonTokenStream.getTokens();
 if (tokens.isEmpty()) {
  throw new RuntimeException("Could not parse: " + input);
 }
 // this is sensitive to the grammar in DSQuery.g4. We could just use the visitor but doing so
 // means we need to be creating the AST and instead we could just use the lexer. in any case,
 // we don't expect the graphite format to change anytime soon.
 // filter all EOF tokens first.
 Queue<Token> queue = tokens.stream().filter(t -> t.getType() != Lexer.EOF).collect(
   Collectors.toCollection(ArrayDeque::new));
 return queue;
}

/**
 * Print the token stream to the logger.
 *
 * @param tokens to print
 */
public static void printTokens(CommonTokenStream tokens) {
  if (log.isDebugEnabled()) {
    List tokenList = tokens.getTokens();
    StringWriter writer = new StringWriter();
    PrintWriter printer = new PrintWriter(writer);
    for (int i = 0; i < tokens.size(); i++) {
      Token t = (Token) tokenList.get(i);
      String text = t.getText();
      if (text.trim().length() == 0) {
        printer.print("'" + text + "'");
      } else {
        printer.print(text);
      }
      printer.print('[');
      printer.print(t.getType());
      printer.print(']');
      printer.print(" ");
    }
    printer.println();
    log.debug("Tokens: " + writer.toString());
  }
}

protected Queue<Token> getQueue(String input) {
 DSWrapperLexer lexer = dsWrapperLexerThreadLocal.get();
 lexer.setInputStream(new ANTLRInputStream(input));
 CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
 commonTokenStream.fill();
 List<Token> tokens = commonTokenStream.getTokens();
 if (tokens.isEmpty()) {
  throw new RuntimeException("Could not parse: " + input);
 }
 // this is sensitive to the grammar in DSQuery.g4. We could just use the visitor but doing so
 // means we need to be creating the AST and instead we could just use the lexer. in any case,
 // we don't expect the graphite format to change anytime soon.
 // filter all EOF tokens first.
 Queue<Token> queue = tokens.stream().filter(t -> t.getType() != Lexer.EOF).collect(
   Collectors.toCollection(ArrayDeque::new));
 return queue;
}

public CodeBuffTokenStream(CommonTokenStream stream) {
  super(stream.getTokenSource());
  this.fetchedEOF = false;
  for (Token t : stream.getTokens()) {
    tokens.add(new CommonToken(t));
  }
  reset();
}

public float getWSEditDistance() throws Exception {
  List<Token> wsTokens = filter(originalTokens.getTokens(),
                 t -> t.getText().matches("\\s+")); // only count whitespace
  String originalWS = tokenText(wsTokens);
  String formattedOutput = getOutput();
  CommonTokenStream formatted_tokens = tokenize(formattedOutput, corpus.language.lexerClass);
  wsTokens = filter(formatted_tokens.getTokens(),
           t -> t.getText().matches("\\s+"));
  String formattedWS = tokenText(wsTokens);
  float editDistance = normalizedLevenshteinDistance(originalWS, formattedWS);
  return editDistance;
}

public static Token getTokenUnderCursor(CommonTokenStream tokens, int offset) {
  Comparator<Token> cmp = new Comparator<Token>() {
    @Override
    public int compare(Token a, Token b) {
      if ( a.getStopIndex() < b.getStartIndex() ) return -1;
      if ( a.getStartIndex() > b.getStopIndex() ) return 1;
      return 0;
    }
  };
  if ( offset<0 || offset >= tokens.getTokenSource().getInputStream().size() ) return null;
  CommonToken key = new CommonToken(Token.INVALID_TYPE, "");
  key.setStartIndex(offset);
  key.setStopIndex(offset);
  List<Token> tokenList = tokens.getTokens();
  Token tokenUnderCursor = null;
  int i = Collections.binarySearch(tokenList, key, cmp);
  if ( i>=0 ) tokenUnderCursor = tokenList.get(i);
  return tokenUnderCursor;
}

@Test
public void testLexer() throws IOException{
  this.createTokenStream().getTokens();
}
 

@Test
public void test(){
  KalangCompiler kc = new KalangCompiler(){
    @Override
    public CodeGenerator createCodeGenerator(CompilationUnit compilationUnit) {
      return new Ast2JavaStub();
    }
  };
  kc.addSource("Test", "class{  }","Test.kl");
  kc.compile();
  CompilationUnit unit = kc.getCompilationUnit("Test");
  assert unit != null;
  CommonTokenStream ts = unit.getTokenStream();
  //the tokens contains tokens in all channels
  List<Token> tokens = ts.getTokens();
  assertEquals(5, ts.size());
  testTokenNavigator(tokens.toArray(new Token[0]),unit.getAstBuilder().getParseTree());
}