private void tokenizeSource() { this.tokens.clear(); Scanner scanner = new Scanner(true, false, false/* nls */, CompilerOptions.versionToJdkLevel(this.sourceLevel), null/* taskTags */, null/* taskPriorities */, false/* taskCaseSensitive */); scanner.setSource(this.sourceArray); while (true) { try { int tokenType = scanner.getNextToken(); if (tokenType == TokenNameEOF) break; Token token = Token.fromCurrent(scanner, tokenType); this.tokens.add(token); } catch (InvalidInputException e) { Token token = Token.fromCurrent(scanner, TokenNameNotAToken); this.tokens.add(token); } } }
private void tokenizeSource(int kind) { this.tokens.clear(); Scanner scanner = new Scanner(true, false, false/* nls */, CompilerOptions.versionToJdkLevel(this.sourceLevel), null/* taskTags */, null/* taskPriorities */, false/* taskCaseSensitive */); scanner.setSource(this.sourceArray); scanner.fakeInModule = (kind & K_MODULE_INFO) != 0; while (true) { try { int tokenType = scanner.getNextToken(); if (tokenType == TokenNameEOF) break; Token token = Token.fromCurrent(scanner, tokenType); this.tokens.add(token); } catch (InvalidInputException e) { Token token = Token.fromCurrent(scanner, TokenNameNotAToken); this.tokens.add(token); } } }
private void tokenizeSource(int kind) { this.tokens.clear(); Scanner scanner = new Scanner(true, false, false/* nls */, CompilerOptions.versionToJdkLevel(this.sourceLevel), null/* taskTags */, null/* taskPriorities */, false/* taskCaseSensitive */); scanner.setSource(this.sourceArray); scanner.fakeInModule = (kind & K_MODULE_INFO) != 0; while (true) { try { int tokenType = scanner.getNextToken(); if (tokenType == TokenNameEOF) break; Token token = Token.fromCurrent(scanner, tokenType); this.tokens.add(token); } catch (InvalidInputException e) { Token token = Token.fromCurrent(scanner, TokenNameNotAToken); this.tokens.add(token); } } }