protected void warn_balanced(int c, boolean spaceSeen, String op, String syn) { if (!isLexState(last_state, EXPR_CLASS|EXPR_DOT|EXPR_FNAME|EXPR_ENDFN) && spaceSeen && !Character.isWhitespace(c)) { ambiguousOperator(op, syn); } }
protected void magicCommentEncoding(ByteList encoding) { if (!comment_at_top()) return; setEncoding(encoding); }
public boolean tokadd_mbchar(int first_byte, ByteList buffer) { int length = precise_mbclen(); if (length <= 0) { compile_error("invalid multibyte char (" + getEncoding() + ")"); return false; } tokAdd(first_byte, buffer); // add first byte since we have it. lex_p += length - 1; // we already read first byte so advance pointer for remainder if (length > 1) tokCopy(length - 1, buffer); // copy next n bytes over. return true; }
protected int numberLiteralSuffix(int mask) throws IOException { int c = nextc(); if (c == 'i') return (mask & SUFFIX_I) != 0 ? mask & SUFFIX_I : 0; if (c == 'r') { int result = 0; if ((mask & SUFFIX_R) != 0) result |= (mask & SUFFIX_R); if (peek('i') && (mask & SUFFIX_I) != 0) { c = nextc(); result |= (mask & SUFFIX_I); } return result; } if (c == '.') { int c2 = nextc(); if (Character.isDigit(c2)) { compile_error("unexpected fraction part after numeric literal"); do { // Ripper does not stop so we follow MRI here and read over next word... c2 = nextc(); } while (isIdentifierChar(c2)); } else { pushback(c2); } } pushback(c); return 0; }
public void setRegexpEncoding(Ruby runtime, ByteList value, RegexpOptions options) { Encoding optionsEncoding = options.setup(runtime); // Change encoding to one specified by regexp options as long as the string is compatible. if (optionsEncoding != null) { if (optionsEncoding != value.getEncoding() && !is7BitASCII(value)) { mismatchedRegexpEncodingError(optionsEncoding, value.getEncoding()); } value.setEncoding(optionsEncoding); } else if (options.isEncodingNone()) { if (value.getEncoding() != ASCII8BIT_ENCODING && !is7BitASCII(value)) { mismatchedRegexpEncodingError(optionsEncoding, value.getEncoding()); } value.setEncoding(ASCII8BIT_ENCODING); } else if (getEncoding() == USASCII_ENCODING) { if (!is7BitASCII(value)) { value.setEncoding(USASCII_ENCODING); // This will raise later } else { value.setEncoding(ASCII8BIT_ENCODING); } } }
/** * This differs from MRI in a few ways. This version does not apply value to a separate token buffer. * It is for use when we know we will not be omitting or including ant non-syntactical characters. Use * tokadd_mbchar(int, ByteList) if the string differs from actual source. Secondly, this returns a boolean * instead of the first byte passed. MRI only used the return value as a success/failure code to return * EOF. * * Because this version does not use a separate token buffer we only just increment lex_p. When we reach * end of the token it will just get the bytes directly from source directly. */ public boolean tokadd_mbchar(int first_byte) { int length = precise_mbclen(); if (length <= 0) { compile_error("invalid multibyte char (" + getEncoding() + ")"); return false; } else if (length > 1) { tokenCR = StringSupport.CR_VALID; } lex_p += length - 1; // we already read first byte so advance pointer for remainder return true; }
public void checkRegexpFragment(Ruby runtime, ByteList value, RegexpOptions options) { setRegexpEncoding(runtime, value, options); ThreadContext context = runtime.getCurrentContext(); IRubyObject $ex = context.getErrorInfo(); try { RubyRegexp.preprocessCheck(runtime, value); } catch (RaiseException re) { context.setErrorInfo($ex); compile_error(re.getMessage()); } }
public String createTokenString(int start) { return createAsEncodedString(lexb.getUnsafeBytes(), lexb.begin() + start, lex_p - start, getEncoding()); }
public String createAsEncodedString(byte[] bytes, int start, int length, Encoding encoding) { // FIXME: We should be able to move some faster non-exception cache using Encoding.isDefined try { Charset charset = EncodingUtils.charsetForEncoding(getEncoding()); if (charset != null) { if (charset == RubyEncoding.UTF8) { return RubyEncoding.decodeUTF8(bytes, start, length); } else { return new String(bytes, start, length, charset); } } } catch (UnsupportedCharsetException e) {} return new String(bytes, start, length); }
@Deprecated public void validateFormalIdentifier(String identifier) { char first = identifier.charAt(0); if (Character.isUpperCase(first)) { compile_error("formal argument cannot be a constant"); } switch(first) { case '@': if (identifier.charAt(1) == '@') { compile_error("formal argument cannot be a class variable"); } else { compile_error("formal argument cannot be an instance variable"); } break; case '$': compile_error("formal argument cannot be a global variable"); break; default: // This mechanism feels a tad dicey but at this point we are dealing with a valid // method name at least so we should not need to check the entire string... char last = identifier.charAt(identifier.length() - 1); if (last == '=' || last == '?' || last == '!') { compile_error("formal argument must be local variable"); } } }
@JRubyMethod(meta = true) public static IRubyObject dedent_string(ThreadContext context, IRubyObject self, IRubyObject _input, IRubyObject _width) { RubyString input = _input.convertToString(); int wid = _width.convertToInteger().getIntValue(); input.modify19(); int col = LexingCommon.dedent_string(input.getByteList(), wid); return context.runtime.newFixnum(col); }
public void newtok(boolean unreadOnce) { tokline = getPosition(); // We assume all idents are 7BIT until they aren't. tokenCR = StringSupport.CR_7BIT; tokp = lex_p - (unreadOnce ? 1 : 0); // We use tokp of ripper to mark beginning of tokens. }
public ISourcePosition getPosition() { if (tokline != null && ruby_sourceline == tokline.getLine()) return tokline; return new SimpleSourcePosition(getFile(), ruby_sourceline); }
public String createTokenString() { return createTokenString(tokp); }
protected int numberLiteralSuffix(int mask) throws IOException { int c = nextc(); if (c == 'i') return (mask & SUFFIX_I) != 0 ? mask & SUFFIX_I : 0; if (c == 'r') { int result = 0; if ((mask & SUFFIX_R) != 0) result |= (mask & SUFFIX_R); if (peek('i') && (mask & SUFFIX_I) != 0) { c = nextc(); result |= (mask & SUFFIX_I); } return result; } if (c == '.') { int c2 = nextc(); if (Character.isDigit(c2)) { compile_error("unexpected fraction part after numeric literal"); do { // Ripper does not stop so we follow MRI here and read over next word... c2 = nextc(); } while (isIdentifierChar(c2)); } else { pushback(c2); } } pushback(c); return 0; }
public void setRegexpEncoding(Ruby runtime, ByteList value, RegexpOptions options) { Encoding optionsEncoding = options.setup(runtime); // Change encoding to one specified by regexp options as long as the string is compatible. if (optionsEncoding != null) { if (optionsEncoding != value.getEncoding() && !is7BitASCII(value)) { mismatchedRegexpEncodingError(optionsEncoding, value.getEncoding()); } value.setEncoding(optionsEncoding); } else if (options.isEncodingNone()) { if (value.getEncoding() != ASCII8BIT_ENCODING && !is7BitASCII(value)) { mismatchedRegexpEncodingError(optionsEncoding, value.getEncoding()); } value.setEncoding(ASCII8BIT_ENCODING); } else if (getEncoding() == USASCII_ENCODING) { if (!is7BitASCII(value)) { value.setEncoding(USASCII_ENCODING); // This will raise later } else { value.setEncoding(ASCII8BIT_ENCODING); } } }
/** * This differs from MRI in a few ways. This version does not apply value to a separate token buffer. * It is for use when we know we will not be omitting or including ant non-syntactical characters. Use * tokadd_mbchar(int, ByteList) if the string differs from actual source. Secondly, this returns a boolean * instead of the first byte passed. MRI only used the return value as a success/failure code to return * EOF. * * Because this version does not use a separate token buffer we only just increment lex_p. When we reach * end of the token it will just get the bytes directly from source directly. */ public boolean tokadd_mbchar(int first_byte) { int length = precise_mbclen(); if (length <= 0) { compile_error("invalid multibyte char (" + getEncoding() + ")"); return false; } else if (length > 1) { tokenCR = StringSupport.CR_VALID; } lex_p += length - 1; // we already read first byte so advance pointer for remainder return true; }
public void checkRegexpFragment(Ruby runtime, ByteList value, RegexpOptions options) { setRegexpEncoding(runtime, value, options); ThreadContext context = runtime.getCurrentContext(); IRubyObject $ex = context.getErrorInfo(); try { RubyRegexp.preprocessCheck(runtime, value); } catch (RaiseException re) { context.setErrorInfo($ex); compile_error(re.getMessage()); } }
public String createTokenString(int start) { return createAsEncodedString(lexb.getUnsafeBytes(), lexb.begin() + start, lex_p - start, getEncoding()); }
public String createAsEncodedString(byte[] bytes, int start, int length, Encoding encoding) { // FIXME: We should be able to move some faster non-exception cache using Encoding.isDefined try { Charset charset = EncodingUtils.charsetForEncoding(getEncoding()); if (charset != null) { if (charset == RubyEncoding.UTF8) { return RubyEncoding.decodeUTF8(bytes, start, length); } else { return new String(bytes, start, length, charset); } } } catch (UnsupportedCharsetException e) {} return new String(bytes, start, length); }