/** * write the string as tag+length, with length being the number of UTF-8 bytes */ public void writeStr(CharSequence s) throws IOException { if (s == null) { writeTag(NULL); return; } int end = s.length(); int maxSize = end * ByteUtils.MAX_UTF8_BYTES_PER_CHAR; if (maxSize <= MAX_UTF8_SIZE_FOR_ARRAY_GROW_STRATEGY) { if (bytes == null || bytes.length < maxSize) bytes = new byte[maxSize]; int sz = ByteUtils.UTF16toUTF8(s, 0, end, bytes, 0); writeTag(STR, sz); daos.write(bytes, 0, sz); } else { // double pass logic for large strings, see SOLR-7971 int sz = ByteUtils.calcUTF16toUTF8Length(s, 0, end); writeTag(STR, sz); if (bytes == null || bytes.length < 8192) bytes = new byte[8192]; ByteUtils.writeUTF16toUTF8(s, 0, end, daos, bytes); } }
/** Convert UTF8 bytes into a String */ public static String UTF8toUTF16(byte[] utf8, int offset, int len) { char[] out = new char[len]; int n = UTF8toUTF16(utf8, offset, len, out, 0); return new String(out,0,n); }
public static byte[] toUTF8(CharArr out) { byte[] arr = new byte[out.size() * 3]; int nBytes = ByteUtils.UTF16toUTF8(out, 0, out.size(), arr, 0); return Arrays.copyOf(arr, nBytes); }
public static byte[] toUTF8(CharArr out) { byte[] arr = new byte[out.size() << 2]; // is 4x the real worst-case upper-bound? int nBytes = ByteUtils.UTF16toUTF8(out, 0, out.size(), arr, 0); return Arrays.copyOf(arr, nBytes); }
/** Convert UTF8 bytes into a String */ public static String UTF8toUTF16(byte[] utf8, int offset, int len) { char[] out = new char[len]; int n = UTF8toUTF16(utf8, offset, len, out, 0); return new String(out,0,n); }
/** * write the string as tag+length, with length being the number of UTF-8 bytes */ public void writeStr(String s) throws IOException { if (s == null) { writeTag(NULL); return; } int end = s.length(); int maxSize = end * 4; if (bytes == null || bytes.length < maxSize) bytes = new byte[maxSize]; int sz = ByteUtils.UTF16toUTF8(s, 0, end, bytes, 0); writeTag(STR, sz); daos.write(bytes, 0, sz); }
public String get(StringBytes b) { String result = cache.get(b); if (result == null) { //make a copy because the buffer received may be changed later by the caller StringBytes copy = new StringBytes(Arrays.copyOfRange(b.bytes, b.offset, b.offset + b.length), 0, b.length); CharArr arr = new CharArr(); ByteUtils.UTF8toUTF16(b.bytes, b.offset, b.length, arr); result = arr.toString(); cache.put(copy, result); } return result; } }
public String get(StringBytes b) { String result = cache.get(b); if (result == null) { //make a copy because the buffer received may be changed later by the caller StringBytes copy = new StringBytes(Arrays.copyOfRange(b.bytes, b.offset, b.offset + b.length), 0, b.length); CharArr arr = new CharArr(); ByteUtils.UTF8toUTF16(b.bytes, b.offset, b.length, arr); result = arr.toString(); cache.put(copy, result); } return result; } }
/** Convert UTF8 bytes into UTF16 characters. */ public static void UTF8toUTF16(byte[] utf8, int offset, int len, CharArr out) { // TODO: do in chunks if the input is large out.reserve(len); int n = UTF8toUTF16(utf8, offset, len, out.getArray(), out.getEnd()); out.setEnd(out.getEnd() + n); }
public static Object fromJSON(byte[] utf8) { // convert directly from bytes to chars // and parse directly from that instead of going through // intermediate strings or readers CharArr chars = new CharArr(); ByteUtils.UTF8toUTF16(utf8, 0, utf8.length, chars); JSONParser parser = new JSONParser(chars.getArray(), chars.getStart(), chars.length()); try { return ObjectBuilder.getVal(parser); } catch (IOException e) { throw new RuntimeException(e); // should never happen w/o using real IO } }
/** Convert UTF8 bytes into UTF16 characters. */ public static void UTF8toUTF16(byte[] utf8, int offset, int len, CharArr out) { // TODO: do in chunks if the input is large out.reserve(len); int n = UTF8toUTF16(utf8, offset, len, out.getArray(), out.getEnd()); out.setEnd(out.getEnd() + n); }
public static Object fromJSON(byte[] utf8) { // convert directly from bytes to chars // and parse directly from that instead of going through // intermediate strings or readers CharArr chars = new CharArr(); ByteUtils.UTF8toUTF16(utf8, 0, utf8.length, chars); JSONParser parser = new JSONParser(chars.getArray(), chars.getStart(), chars.length()); parser.setFlags(parser.getFlags() | JSONParser.ALLOW_MISSING_COLON_COMMA_BEFORE_OBJECT | JSONParser.OPTIONAL_OUTER_BRACES); try { return STANDARDOBJBUILDER.apply(parser).getVal(parser); } catch (IOException e) { throw new RuntimeException(e); // should never happen w/o using real IO } }
public String readStr(DataInputInputStream dis, StringCache stringCache) throws IOException { int sz = readSize(dis); if (bytes == null || bytes.length < sz) bytes = new byte[sz]; dis.readFully(bytes, 0, sz); if (stringCache != null) { return stringCache.get(bytesRef.reset(bytes, 0, sz)); } else { arr.reset(); ByteUtils.UTF8toUTF16(bytes, 0, sz, arr); return arr.toString(); } }
public String readStr(DataInputInputStream dis, StringCache stringCache) throws IOException { int sz = readSize(dis); if (bytes == null || bytes.length < sz) bytes = new byte[sz]; dis.readFully(bytes, 0, sz); if (stringCache != null) { return stringCache.get(bytesRef.reset(bytes, 0, sz)); } else { arr.reset(); ByteUtils.UTF8toUTF16(bytes, 0, sz, arr); return arr.toString(); } }