if (!(force || notEmpty || advance)) return false; String actual = "'" + first(notNullize(builder.getTokenText(), "null"), MAX_ERROR_TOKEN_TEXT, true) + "'"; String message = expectedText + (builder.eof() ? "unexpected end of file" : notEmpty ? "got " + actual : actual + " unexpected"); if (advance) {
@Override public String text() { return builder.getTokenText(); }
@SuppressWarnings({"StringConcatenationInsideStringBufferAppend", "UnusedDeclaration"}) private void logPos() { StringBuilder sb = new StringBuilder(); sb.append("\nmyLexPosition=" + myLexPosition + "/" + myShrunkSequenceSize); if (myLexPosition < myShrunkSequenceSize) { final MyShiftedToken token = myShrunkSequence.get(myLexPosition); sb.append("\nshrunk:" + token.shrunkStart + "," + token.shrunkEnd); sb.append("\nreal:" + token.realStart + "," + token.realEnd); sb.append("\nTT:" + getTokenText()); } sb.append("\ndelegate:"); sb.append("eof=" + myDelegate.eof()); if (!myDelegate.eof()) { //noinspection ConstantConditions sb.append("\nposition:" + myDelegate.getCurrentOffset() + "," + (myDelegate.getCurrentOffset() + myDelegate.getTokenText().length())); sb.append("\nTT:" + myDelegate.getTokenText()); } LOG.info(sb.toString()); }
public static boolean parseLabelDeclaration(PsiBuilder b, @SuppressWarnings("unused") int l) { if (b.lookAhead(1) == COLON && b.getTokenType() != RESERVED_SUB) { String tokenText = b.getTokenText(); if (tokenText != null && IDENTIFIER_PATTERN.matcher(tokenText).matches()) { b.advanceLexer(); b.advanceLexer(); return true; } } return false; }
public static boolean separator(PsiBuilder builder, int tokenType) { if (builder.getTokenType() == CsvTypes.COMMA) { PsiFile currentFile = builder.getUserDataUnprotected(FileContextUtil.CONTAINING_FILE_KEY); return builder.getTokenText().equals( CsvCodeStyleSettings.getCurrentSeparator(builder.getProject(), currentFile != null ? currentFile.getLanguage() : null) ); } return false; }
public static boolean smartRemapAndAdvance(PsiBuilder builder, String expectedTokenText, IElementType expectedTokenType, IElementType newTokenType) { IElementType current = builder.getTokenType(); if (current == newTokenType) { // already remapped, probably due to reverting an earlier parse result builder.advanceLexer(); } else if (expectedTokenText.equals(builder.getTokenText()) && current == expectedTokenType) { builder.remapCurrentToken(newTokenType); builder.advanceLexer(); } else { builder.error("unexpected token"); return false; } return true; }
/** * Parses use parameters with package processor if it's possible. If not, uses default parsing logic. */ public static boolean parseUseParameters(@NotNull PsiBuilder b, int l, @NotNull Parser defaultParser) { if (b.getTokenType() == PACKAGE) { String packageName = b.getTokenText(); if (StringUtil.isEmpty(packageName)) { return false; } PerlPackageProcessor packageProcessor = PerlPackageProcessorEP.EP.findSingle(packageName); if (packageProcessor != null) { assert b instanceof PerlBuilder; PsiBuilder.Marker m = b.mark(); if (packageProcessor.parseUseParameters((PerlBuilder)b, l)) { m.drop(); return true; } m.rollbackTo(); } } return defaultParser.parse(b, l); }
@Override public Token nextToken() { TokenIElementType ideaTType = (TokenIElementType)builder.getTokenType(); int type; if ( ideaTType==null ) { type = Token.EOF; } else { type = ideaTType.getType(); } int channel = Token.DEFAULT_CHANNEL; Pair<TokenSource, CharStream> source = new Pair<TokenSource, CharStream>(this, null); String text = builder.getTokenText(); int start = builder.getCurrentOffset(); int length = text != null ? text.length() : 0; int stop = start + length - 1; // PsiBuilder doesn't provide line, column info int line = 0; int charPositionInLine = 0; Token t = factory.create(source, type, text, channel, start, stop, line, charPositionInLine); builder.advanceLexer(); // System.out.println("TOKEN: "+t); return t; }
if (!force && empty && !advance) return false; String actual = trim(builder.getTokenText()); if (isEmpty(actual)) { sb.append(empty ? "unmatched input" : " expected");
String text = builder.getTokenText(); int start = builder.getCurrentOffset(); int length = text != null ? text.length() : 0;
@Override protected boolean generateNodeCall(PsiBuilder builder, int level, BnfRule rule, @Nullable BnfExpression node, String nextName, Map<String, GeneratedParserUtilBase.Parser> externalArguments) { int tokenStartOffset = builder.getCurrentOffset(); int initialOffset = builder.rawLookup(-1) == TokenType.WHITE_SPACE ? builder.rawTokenTypeStart(-1) : builder.getCurrentOffset(); String tokenText = builder.getTokenText(); int tokenEndOffset = tokenText == null? tokenStartOffset : tokenStartOffset + tokenText.length(); boolean result = super.generateNodeCall(builder, level, rule, node, nextName, externalArguments); builder.getCurrentOffset(); // advance to the next token first int finalOffset = builder.rawLookup(-1) == TokenType.WHITE_SPACE ? builder.rawTokenTypeStart(-1) : builder.getCurrentOffset(); if (node != null) { if (result && initialOffset <= caretOffset && finalOffset > caretOffset || !result && initialOffset <= caretOffset && tokenEndOffset > caretOffset) { boolean inWhitespace = isTokenExpression(node) && initialOffset <= caretOffset && tokenStartOffset > caretOffset; if (!processor.process(node, result && !inWhitespace)) { throw new ProcessCanceledException(); } } } return result; }
final Token.IdType i = current().type; if (i.equals(tok("identifier")) || i.equals(tok("byte")) || i.equals(tok("short")) || i.equals(tok("int")) || i.equals(tok("float")) || i.equals(tok("double")) || i.equals(tok("real"))) { final String tokenText = builder.getTokenText(); final Token t = advance(); if (t.type.equals(tok("identifier"))) if (currentIs(tok("identifier")) && builder.getTokenText().equals("ptr")) { advance();
if (!typePrefix(m)) return false; } else if (i.equals(tok("identifier"))) { switch (builder.getTokenText()) { case "offsetof": case "seg": {