@Override public void tokenize(SourceCode tokens, Tokens tokenEntries) { List<String> code = tokens.getCode(); for (int i = 0; i < code.size(); i++) { String currentLine = code.get(i); for (int j = 0; j < currentLine.length(); j++) { char tok = currentLine.charAt(j); if (!Character.isWhitespace(tok) && tok != '{' && tok != '}' && tok != ';') { tokenEntries.add(new TokenEntry(String.valueOf(tok), tokens.getFileName(), i + 1)); } } } tokenEntries.add(TokenEntry.getEOF()); } }
@Override public void tokenize(SourceCode tokens, Tokens tokenEntries) { code = tokens.getCode(); for (lineNumber = 0; lineNumber < code.size(); lineNumber++) { currentLine = code.get(lineNumber); int loc = 0; while (loc < currentLine.length()) { StringBuilder token = new StringBuilder(); loc = getTokenFromLine(token, loc); if (token.length() > 0 && !isIgnorableString(token.toString())) { if (downcaseString) { token = new StringBuilder(token.toString().toLowerCase(Locale.ROOT)); } // need to re-think how to link this // if ( CPD.debugEnable ) { // System.out.println("Token added:" + token.toString()); // } tokenEntries.add(new TokenEntry(token.toString(), tokens.getFileName(), lineNumber + 1)); } } } tokenEntries.add(TokenEntry.getEOF()); }
@Override public void tokenize(SourceCode tokens, Tokens tokenEntries) { List<String> code = tokens.getCode(); for (int i = 0; i < code.size(); i++) { String currentLine = code.get(i); for (int j = 0; j < currentLine.length(); j++) { char tok = currentLine.charAt(j); if (!Character.isWhitespace(tok) && tok != '{' && tok != '}' && tok != ';') { tokenEntries.add(new TokenEntry(String.valueOf(tok), tokens.getFileName(), i + 1)); } } } tokenEntries.add(TokenEntry.getEOF()); } }
public void tokenize(SourceCode tokens, Tokens tokenEntries) { this.code = tokens.getCode();
@Override public void tokenize(SourceCode tokens, Tokens tokenEntries) { code = tokens.getCode(); for (lineNumber = 0; lineNumber < code.size(); lineNumber++) { currentLine = code.get(lineNumber); int loc = 0; while (loc < currentLine.length()) { StringBuilder token = new StringBuilder(); loc = getTokenFromLine(token, loc); if (token.length() > 0 && !isIgnorableString(token.toString())) { if (downcaseString) { token = new StringBuilder(token.toString().toLowerCase(Locale.ROOT)); } // need to re-think how to link this // if ( CPD.debugEnable ) { // System.out.println("Token added:" + token.toString()); // } tokenEntries.add(new TokenEntry(token.toString(), tokens.getFileName(), lineNumber + 1)); } } } tokenEntries.add(TokenEntry.getEOF()); }