Refine search
// Create the encoder and decoder for ISO-8859-1 Charset charset = Charset.forName("ISO-8859-1"); CharsetDecoder decoder = charset.newDecoder(); CharsetEncoder encoder = charset.newEncoder(); try { // Convert a string to ISO-LATIN-1 bytes in a ByteBuffer // The new ByteBuffer is ready to be read. ByteBuffer bbuf = encoder.encode(CharBuffer.wrap("a string")); // Convert ISO-LATIN-1 bytes in a ByteBuffer to a character ByteBuffer and then to a string. // The new ByteBuffer is ready to be read. CharBuffer cbuf = decoder.decode(bbuf); String s = cbuf.toString(); } catch (CharacterCodingException e) { }
Assert.notNull(charset, "Charset must not be null"); if (charSequence.length() != 0) { CharsetEncoder charsetEncoder = charset.newEncoder() .onMalformedInput(CodingErrorAction.REPLACE) .onUnmappableCharacter(CodingErrorAction.REPLACE); CharBuffer inBuffer = CharBuffer.wrap(charSequence); int estimatedSize = (int) (inBuffer.remaining() * charsetEncoder.averageBytesPerChar()); ByteBuffer outBuffer = ensureCapacity(estimatedSize) .asByteBuffer(writePosition(), writableByteCount()); while (true) { CoderResult cr = (inBuffer.hasRemaining() ? charsetEncoder.encode(inBuffer, outBuffer, true) : CoderResult.UNDERFLOW); if (cr.isUnderflow()) { cr = charsetEncoder.flush(outBuffer); if (cr.isUnderflow()) { break; if (cr.isOverflow()) { writePosition(outBuffer.position()); int maximumSize = (int) (inBuffer.remaining() * charsetEncoder.maxBytesPerChar()); ensureCapacity(maximumSize); outBuffer = asByteBuffer(writePosition(), writableByteCount());
/** * @see org.apache.tools.zip.ZipEncoding#canEncode(java.lang.String) */ public boolean canEncode(final String name) { final CharsetEncoder enc = this.charset.newEncoder(); enc.onMalformedInput(CodingErrorAction.REPORT); enc.onUnmappableCharacter(CodingErrorAction.REPORT); return enc.canEncode(name); }
/** * Creates a new input stream that will encode the characters from {@code reader} into bytes using * the given character set. Malformed input and unmappable characters will be replaced. * * @param reader input source * @param charset character set used for encoding chars to bytes * @param bufferSize size of internal input and output buffers * @throws IllegalArgumentException if bufferSize is non-positive */ ReaderInputStream(Reader reader, Charset charset, int bufferSize) { this( reader, charset .newEncoder() .onMalformedInput(CodingErrorAction.REPLACE) .onUnmappableCharacter(CodingErrorAction.REPLACE), bufferSize); }
/** * Create a copy of {@code value} into this instance using the encoding type of {@code charset}. * The copy will start at index {@code start} and copy {@code length} bytes. */ public AsciiString(CharSequence value, Charset charset, int start, int length) { CharBuffer cbuf = CharBuffer.wrap(value, start, start + length); CharsetEncoder encoder = CharsetUtil.encoder(charset); ByteBuffer nativeBuffer = ByteBuffer.allocate((int) (encoder.maxBytesPerChar() * length)); encoder.encode(cbuf, nativeBuffer, true); final int offset = nativeBuffer.arrayOffset(); this.value = Arrays.copyOfRange(nativeBuffer.array(), offset, offset + nativeBuffer.position()); this.offset = 0; this.length = this.value.length; }
/** * Fills the internal char buffer from the reader. * * @throws IOException * If an I/O error occurs */ private void fillBuffer() throws IOException { if (!endOfInput && (lastCoderResult == null || lastCoderResult.isUnderflow())) { encoderIn.compact(); final int position = encoderIn.position(); // We don't use Reader#read(CharBuffer) here because it is more efficient // to write directly to the underlying char array (the default implementation // copies data to a temporary char array). final int c = reader.read(encoderIn.array(), position, encoderIn.remaining()); if (c == EOF) { endOfInput = true; } else { encoderIn.position(position+c); } encoderIn.flip(); } encoderOut.compact(); lastCoderResult = encoder.encode(encoderIn, encoderOut, endOfInput); encoderOut.flip(); }
public ByteBuffer getBytes(String charsetName) { if (!isEnd) { throw new IllegalStateException("The conact has not been end."); } final CharsetEncoder encoder = Charset.forName(charsetName).newEncoder().onMalformedInput( CodingErrorAction.REPLACE).onUnmappableCharacter(CodingErrorAction.REPLACE); final CharBuffer cb = CharBuffer.wrap(this.value, 0, this.value.length); try { encoder.reset(); final int len = (int) (this.length * (double) encoder.maxBytesPerChar()); final ByteBuffer bb = ByteBuffer.allocate(len); CoderResult cr = encoder.encode(cb, bb, true); if (!cr.isUnderflow()) { cr.throwException(); } cr = encoder.flush(bb); if (!cr.isUnderflow()) { cr.throwException(); } return bb; } catch (Exception x) { throw new Error(x); } }
@Override public Object evaluate(DeferredObject[] arguments) throws HiveException { String value = PrimitiveObjectInspectorUtils.getString(arguments[0].get(), stringOI); if (value == null) { return null; } ByteBuffer encoded; if (encoder != null){ try { encoded = encoder.encode(CharBuffer.wrap(value)); } catch (CharacterCodingException e) { throw new HiveException(e); } } else { encoded = Charset.forName( PrimitiveObjectInspectorUtils.getString(arguments[1].get(), charsetOI)).encode(value); } result.setSize(encoded.limit()); encoded.get(result.getBytes(), 0, encoded.limit()); return result; }
final Charset inputCharset = Charset.forName(context.getProperty(INPUT_CHARSET).evaluateAttributeExpressions(flowFile).getValue()); final Charset outputCharset = Charset.forName(context.getProperty(OUTPUT_CHARSET).evaluateAttributeExpressions(flowFile).getValue()); final CharBuffer charBuffer = CharBuffer.allocate(MAX_BUFFER_SIZE); final CharsetDecoder decoder = inputCharset.newDecoder(); decoder.onMalformedInput(CodingErrorAction.REPLACE); decoder.onUnmappableCharacter(CodingErrorAction.REPLACE); decoder.replaceWith("?"); final CharsetEncoder encoder = outputCharset.newEncoder(); encoder.onMalformedInput(CodingErrorAction.REPLACE); encoder.onUnmappableCharacter(CodingErrorAction.REPLACE); encoder.replaceWith("?".getBytes(outputCharset));
private void fillBuffer() throws IOException { int len = (int) Math.min(charBuffer.capacity() - charBuffer.position(), remaining); if (len > 0) { len = reader.read(charBuffer.array(), charBuffer.position(), len); } if (len > 0) { remaining -= len; } else { len = 0; remaining = 0; } length += len; charBuffer.limit(charBuffer.position() + len); charBuffer.rewind(); byteBuffer = ByteBuffer.allocate(Constants.IO_BUFFER_SIZE); boolean end = remaining == 0; encoder.encode(charBuffer, byteBuffer, end); if (end && byteBuffer.position() == 0) { // EOF byteBuffer = null; return; } byteBuffer.flip(); charBuffer.compact(); charBuffer.flip(); charBuffer.position(charBuffer.limit()); }
byte[] byteArray = ...; String charsetName = ...; ByteBuffer buffer = ByteBuffer.wrap(byteArray, dstBegin, dstEnd - dstBegin); CharsetEncoder encoder = Charset.forName(charsetName).newEncoder(); CoderResult result = encoder.encode(CharBuffer.wrap(string), buffer, true); if (!result.isUnderflow()) { result.throwException(); }
private void advance() throws IOException { assert !endOfInput; assert !bbuf.hasRemaining() : "advance() should be called when output byte buffer is empty. bbuf: " + bbuf + ", as string: " + bbuf.asCharBuffer().toString(); assert cbuf.remaining() < 2; // given that bbuf.capacity = 3 x cbuf.capacity, the only time that we should have a // remaining char is if the last char read was the 1st half of a surrogate pair if (cbuf.remaining() == 0) { cbuf.clear(); } else { cbuf.compact(); } int n = reader.read(cbuf); // read #1 cbuf.flip(); CoderResult result; endOfInput = n == -1; bbuf.clear(); result = encoder.encode(cbuf, bbuf, endOfInput); checkEncodeResult(result); if (endOfInput) { result = encoder.flush(bbuf); checkEncodeResult(result); } bbuf.flip(); }
buf.flip(); this.byteBuf = new byte[1]; // single byte this.charBuf = CharBuffer.allocate(2); // two chars for surrogate pairs charBuf.flip(); this.fileSize = file.length(); this.decoder = charset.newDecoder(); this.position = 0; this.syncPosition = 0; if (charset.name().startsWith("UTF-8")) { } else if (charset.name().startsWith("UTF-16")) { this.maxCharWidth = (int) Math.ceil(charset.newEncoder().maxBytesPerChar()); "Unexpected value for decode error policy: " + decodeErrorPolicy); decoder.onMalformedInput(errorAction); decoder.onUnmappableCharacter(errorAction);
/** * Create a text response with known length. */ public static Response newFixedLengthResponse(IStatus status, String mimeType, String txt) { ContentType contentType = new ContentType(mimeType); if (txt == null) { return newFixedLengthResponse(status, mimeType, new ByteArrayInputStream(new byte[0]), 0); } else { byte[] bytes; try { CharsetEncoder newEncoder = Charset.forName(contentType.getEncoding()).newEncoder(); if (!newEncoder.canEncode(txt)) { contentType = contentType.tryUTF8(); } bytes = txt.getBytes(contentType.getEncoding()); } catch (UnsupportedEncodingException e) { NanoHTTPD.LOG.log(Level.SEVERE, "encoding problem, responding nothing", e); bytes = new byte[0]; } return newFixedLengthResponse(status, contentType.getContentTypeHeader(), new ByteArrayInputStream(bytes), bytes.length); } }
try { while (byteBuffer.hasRemaining()) { while (charBuffer.hasRemaining()) { final CoderResult result = encoder.encode(charBuffer, byteBuffer, false); if (result.isOverflow()) { return true; if (result.isUnderflow()) { filled = true; break; if (result.isError()) { if (result.isMalformed()) { throw msg.malformedInput(); charBuffer.compact(); try { final int cnt = reader.read(charBuffer); if (cnt == -1) { return filled; charBuffer.flip();
final CharsetEncoder enc = this.charset.newEncoder(); enc.onMalformedInput(CodingErrorAction.REPORT); enc.onUnmappableCharacter(CodingErrorAction.REPORT); final CharBuffer cb = CharBuffer.wrap(name); ByteBuffer out = ByteBuffer.allocate(name.length() + (name.length() + 1) / 2); while (cb.remaining() > 0) { final CoderResult res = enc.encode(cb, out, true); if (res.isUnmappable() || res.isMalformed()) { if (res.length() * 6 > out.remaining()) { out = ZipEncodingHelper.growBuffer(out, out.position() + res.length() * 6); for (int i = 0; i < res.length(); ++i) { ZipEncodingHelper.appendSurrogate(out, cb.get()); } else if (res.isUnderflow()) { enc.flush(out); break;
boolean utf16 = encoder.charset().name().startsWith("UTF-16"); CharBuffer in = CharBuffer.wrap(val); limit(end); encoder.reset(); if (in.hasRemaining()) { cr = encoder.encode(in, buf(), true); } else { cr = encoder.flush(buf()); if (cr.isUnderflow() || cr.isOverflow()) { break; cr.throwException();
response.setCharacterEncoding(OUT_ENCODING); CharBuffer cb = inCharset.newDecoder().decode(ByteBuffer.wrap(responseDraft, 0, responseDraft.length)); String s = cb.toString(); int size = cb.length(); CharBuffer cb2 = CharBuffer.wrap(csq); CharsetEncoder encoder = outCharset.newEncoder(); encoder.encode(cb2, bb, false); encoder.encode(cb, bb, false); bb.flip(); byte[] b = new byte[bb.limit()];
@Override public ManagedHttpClientConnection create(HttpRoute route, ConnectionConfig config) { final ConnectionConfig cconfig = config != null ? config : ConnectionConfig.DEFAULT; CharsetDecoder chardecoder = null; CharsetEncoder charencoder = null; final Charset charset = cconfig.getCharset(); final CodingErrorAction malformedInputAction = cconfig.getMalformedInputAction() != null ? cconfig.getMalformedInputAction() : CodingErrorAction.REPORT; final CodingErrorAction unmappableInputAction = cconfig.getUnmappableInputAction() != null ? cconfig.getUnmappableInputAction() : CodingErrorAction.REPORT; if (charset != null) { chardecoder = charset.newDecoder(); chardecoder.onMalformedInput(malformedInputAction); chardecoder.onUnmappableCharacter(unmappableInputAction); charencoder = charset.newEncoder(); charencoder.onMalformedInput(malformedInputAction); charencoder.onUnmappableCharacter(unmappableInputAction); } return new RecordingHttpClientConnection(DEFAULT_BUFSIZE, DEFAULT_BUFSIZE, chardecoder, charencoder, cconfig.getMessageConstraints(), null, null, DefaultHttpRequestWriterFactory.INSTANCE, DefaultHttpResponseParserFactory.INSTANCE); } };