Refine search
@Override public void tokenize(SourceCode tokens, Tokens tokenEntries) { List<String> code = tokens.getCode(); for (int i = 0; i < code.size(); i++) { String currentLine = code.get(i); for (int j = 0; j < currentLine.length(); j++) { char tok = currentLine.charAt(j); if (!Character.isWhitespace(tok) && tok != '{' && tok != '}' && tok != ';') { tokenEntries.add(new TokenEntry(String.valueOf(tok), tokens.getFileName(), i + 1)); } } } tokenEntries.add(TokenEntry.getEOF()); } }
private JavaTokenFilter createTokenFilter(final SourceCode sourceCode) { final StringBuilder stringBuilder = sourceCode.getCodeBuffer(); // Note that Java version is irrelevant for tokenizing final LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JavaLanguageModule.NAME) .getVersion("1.4").getLanguageVersionHandler(); final TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), new StringReader(stringBuilder.toString())); return new JavaTokenFilter(tokenMgr, ignoreAnnotations); }
public List<TokensLine> chunk(String fileName, Reader fileReader) { SourceCode sourceCode = new SourceCode(new FileCodeLoaderWithoutCache(fileName, fileReader)); Tokens tokens = new Tokens(); TokenEntry.clearImages(); try { tokenizer.tokenize(sourceCode, tokens); } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new RuntimeException(e); } TokenEntry.clearImages(); return convert(tokens.getTokens()); }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { final String fileName = sourceCode.getFileName(); final JavaTokenFilter tokenFilter = createTokenFilter(sourceCode); final ConstructorDetector constructorDetector = new ConstructorDetector(ignoreIdentifiers); Token currentToken = (Token) tokenFilter.getNextToken(); while (currentToken != null) { processToken(tokenEntries, fileName, currentToken, constructorDetector); currentToken = (Token) tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); }
private void addAndThrowLexicalError(SourceCode sourceCode) throws IOException { configuration.tokenizer().tokenize(sourceCode, tokens); listener.addedFile(1, new File(sourceCode.getFileName())); source.put(sourceCode.getFileName(), sourceCode); }
private void addAndSkipLexicalErrors(SourceCode sourceCode) throws IOException { TokenEntry.State savedTokenEntry = new TokenEntry.State(tokens.getTokens()); try { addAndThrowLexicalError(sourceCode); } catch (TokenMgrError e) { System.err.println("Skipping " + sourceCode.getFileName() + ". Reason: " + e.getMessage()); tokens.getTokens().clear(); tokens.getTokens().addAll(savedTokenEntry.restore()); } }
/* default */ static CharStream getCharStreamFromSourceCode(final SourceCode sourceCode) { StringBuilder buffer = sourceCode.getCodeBuffer(); return CharStreams.fromString(buffer.toString()); } }
public SourceCode sourceCodeFor(Reader reader, String sourceCodeName) { return new SourceCode(new SourceCode.ReaderCodeLoader(reader, sourceCodeName)); }
public SourceCode(URL pageURL) throws IOException { this.source = getSource(pageURL); } public String getSource() { return source; } private String getSource(URL url) throws IOException { URLConnection spoof = url.openConnection(); StringBuffer sb = new StringBuffer(); spoof.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; H010818)"); BufferedReader in = new BufferedReader(new InputStreamReader( spoof.getInputStream())); String strLine = ""; while ((strLine = in.readLine()) != null) { sb.append(strLine); } return sb.toString(); } public static void main(String[] args) throws IOException { SourceCode s = new SourceCode(new URL("https://www.google.co.in/")); System.out.println(s.getSource()); }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { final String fileName = sourceCode.getFileName(); final JavaTokenFilter tokenFilter = createTokenFilter(sourceCode); final ConstructorDetector constructorDetector = new ConstructorDetector(ignoreIdentifiers); Token currentToken = (Token) tokenFilter.getNextToken(); while (currentToken != null) { processToken(tokenEntries, fileName, currentToken, constructorDetector); currentToken = (Token) tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); }
@Override protected AntlrTokenManager getLexerForSource(SourceCode sourceCode) { CharStream charStream = AntlrTokenizer.getCharStreamFromSourceCode(sourceCode); final Lexer lexer = new Kotlin(charStream); final AntlrTokenManager tokenManager = new AntlrTokenManager(lexer, sourceCode.getFileName()) { @Override public Object getNextToken() { AntlrToken nextToken; boolean done = false; do { nextToken = (AntlrToken) super.getNextToken(); analyzeTokenStart(nextToken); if (!nextToken.isHidden() && nextToken.getType() != Kotlin.NL && !isDiscarding()) { done = true; } analyzeTokenEnd(nextToken); } while (!done && nextToken.getType() != Token.EOF); return nextToken; } }; return tokenManager; }
private void addAndSkipLexicalErrors(SourceCode sourceCode) throws IOException { TokenEntry.State savedTokenEntry = new TokenEntry.State(tokens.getTokens()); try { addAndThrowLexicalError(sourceCode); } catch (TokenMgrError e) { System.err.println("Skipping " + sourceCode.getFileName() + ". Reason: " + e.getMessage()); tokens.getTokens().clear(); tokens.getTokens().addAll(savedTokenEntry.restore()); } }
/* default */ static CharStream getCharStreamFromSourceCode(final SourceCode sourceCode) { StringBuilder buffer = sourceCode.getCodeBuffer(); return CharStreams.fromString(buffer.toString()); } }
public SourceCode sourceCodeFor(File file) { return new SourceCode(new SourceCode.FileCodeLoader(file, getSourceEncoding().name())); }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder sb = sourceCode.getCodeBuffer(); try (BufferedReader reader = new BufferedReader(new CharArrayReader(sb.toString().toCharArray()))) { int lineNumber = 1; String line = reader.readLine(); while (line != null) { StringTokenizer tokenizer = new StringTokenizer(line, TOKENS, true); while (tokenizer.hasMoreTokens()) { String token = tokenizer.nextToken(); if (!" ".equals(token) && !"\t".equals(token)) { tokenEntries.add(new TokenEntry(token, sourceCode.getFileName(), lineNumber)); } } // advance iteration variables line = reader.readLine(); lineNumber++; } } catch (IOException ignored) { ignored.printStackTrace(); } finally { tokenEntries.add(TokenEntry.getEOF()); } } }
@Override public final void tokenize(SourceCode source, Tokens cpdTokens) { String fileName = source.getFileName(); ChannelDispatcher.Builder lexerBuilder = ChannelDispatcher.builder(); lexerBuilder.addChannel(CommentChannel.JSP_COMMENT); lexerBuilder.addChannel(CommentChannel.HTML_COMMENT); lexerBuilder.addChannel(CommentChannel.C_COMMENT); lexerBuilder.addChannel(CommentChannel.CPP_COMMENT); lexerBuilder.addChannel(new WordChannel(fileName)); lexerBuilder.addChannel(new LiteralChannel(fileName)); lexerBuilder.addChannel(new BlackHoleChannel()); ChannelDispatcher<Tokens> lexer = lexerBuilder.build(); try { lexer.consume(new CodeReader(new FileReader(new File(fileName))), cpdTokens); cpdTokens.add(TokenEntry.getEOF()); } catch (FileNotFoundException e) { LOG.error("Unable to open file : " + fileName, e); } }
@Override protected AntlrTokenManager getLexerForSource(final SourceCode sourceCode) { CharStream charStream = AntlrTokenizer.getCharStreamFromSourceCode(sourceCode); return new AntlrTokenManager(new SwiftLexer(charStream), sourceCode.getFileName()); } }
private JavaTokenFilter createTokenFilter(final SourceCode sourceCode) { final StringBuilder stringBuilder = sourceCode.getCodeBuffer(); // Note that Java version is irrelevant for tokenizing final LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JavaLanguageModule.NAME) .getVersion("1.4").getLanguageVersionHandler(); final TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), new StringReader(stringBuilder.toString())); return new JavaTokenFilter(tokenMgr, ignoreAnnotations); }
public List<TokensLine> chunk(File file) { SourceCode sourceCode = new SourceCode(new FileCodeLoaderWithoutCache(file, encoding)); Tokens tokens = new Tokens(); TokenEntry.clearImages(); try { tokenizer.tokenize(sourceCode, tokens); } catch (IOException e) { throw Throwables.propagate(e); } TokenEntry.clearImages(); return convert(tokens.getTokens()); }
public static Class<?> compile(ClassLoader parent, String className, String sourceCodeInText) throws Exception { SourceCode sourceCode = new SourceCode(className, sourceCodeInText); CompiledCode compiledCode = new CompiledCode(className); Iterable<? extends JavaFileObject> compilationUnits = Arrays.asList(sourceCode); DynamicClassLoader cl = new DynamicClassLoader(parent); ExtendedStandardJavaFileManager fileManager = new ExtendedStandard( javac.getStandardFileManager(null, null, null), compiledCode, cl); // set the classpath List<String> options = new ArrayList<String>(); options.add("-classpath"); StringBuilder sb = new StringBuilder(); Enumeration<URL> resources = parent.getResources("/"); while (resources.hasMoreElements()) { URL url = resources.nextElement(); sb.append(url.getFile()).append(File.pathSeparator); } options.add(sb.toString()); // execute the compiler Boolean call = javac.getTask(null, fileManager, null, options, null, compilationUnits).call(); if (call) { return cl.loadClass(className); } return null; }