/** * This differs from MRI in a few ways. This version does not apply value to a separate token buffer. * It is for use when we know we will not be omitting or including ant non-syntactical characters. Use * tokadd_mbchar(int, ByteList) if the string differs from actual source. Secondly, this returns a boolean * instead of the first byte passed. MRI only used the return value as a success/failure code to return * EOF. * * Because this version does not use a separate token buffer we only just increment lex_p. When we reach * end of the token it will just get the bytes directly from source directly. */ public boolean tokadd_mbchar(int first_byte) { int length = precise_mbclen(); if (length <= 0) { compile_error("invalid multibyte char (" + getEncoding() + ")"); return false; } else if (length > 1) { tokenCR = StringSupport.CR_VALID; } lex_p += length - 1; // we already read first byte so advance pointer for remainder return true; }
/** * This differs from MRI in a few ways. This version does not apply value to a separate token buffer. * It is for use when we know we will not be omitting or including ant non-syntactical characters. Use * tokadd_mbchar(int, ByteList) if the string differs from actual source. Secondly, this returns a boolean * instead of the first byte passed. MRI only used the return value as a success/failure code to return * EOF. * * Because this version does not use a separate token buffer we only just increment lex_p. When we reach * end of the token it will just get the bytes directly from source directly. */ public boolean tokadd_mbchar(int first_byte) { int length = precise_mbclen(); if (length <= 0) { compile_error("invalid multibyte char (" + getEncoding() + ")"); return false; } else if (length > 1) { tokenCR = StringSupport.CR_VALID; } lex_p += length - 1; // we already read first byte so advance pointer for remainder return true; }
public boolean tokadd_mbchar(int first_byte, ByteList buffer) { int length = precise_mbclen(); if (length <= 0) { compile_error("invalid multibyte char (" + getEncoding() + ")"); return false; } tokAdd(first_byte, buffer); // add first byte since we have it. lex_p += length - 1; // we already read first byte so advance pointer for remainder if (length > 1) tokCopy(length - 1, buffer); // copy next n bytes over. return true; }
public boolean tokadd_mbchar(int first_byte, ByteList buffer) { int length = precise_mbclen(); if (length <= 0) { compile_error("invalid multibyte char (" + getEncoding() + ")"); return false; } tokAdd(first_byte, buffer); // add first byte since we have it. lex_p += length - 1; // we already read first byte so advance pointer for remainder if (length > 1) tokCopy(length - 1, buffer); // copy next n bytes over. return true; }