Refine search
ByteStream.Output serializeStream = new ByteStream.Output(); LazyUtils.writePrimitiveUTF8(serializeStream, writable, poi, false, (byte) 0, null); startRow = new byte[serializeStream.getLength()]; System.arraycopy(serializeStream.getData(), 0, startRow, 0, serializeStream.getLength()); return startRow; return Bytes.toBytes(((ShortWritable) writable).get()); case STRING: return Bytes.toBytes(((Text) writable).toString()); case BYTE: return Bytes.toBytes(((ByteWritable) writable).get());
protected final boolean isNull( Text nullSequence, byte[] bytes, int fieldByteBegin, int fieldLength) { // Test the length first so in most cases we avoid doing a byte[] // comparison. return fieldLength < 0 || (fieldLength == nullSequence.getLength() && LazyUtils.compare(bytes, fieldByteBegin, fieldLength, nullSequence.getBytes(), 0, nullSequence.getLength()) == 0); }
int result = 0, end = start + length; while (offset < end) { int digit = LazyUtils.digit(bytes[offset++], radix); if (digit == -1) { if (bytes[offset-1] == separator) { throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length)); throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length)); throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length)); int digit = LazyUtils.digit(bytes[offset++], radix); if (digit == -1) { throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length)); result = -result; if (result < 0) { throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
switch (objInspector.getCategory()) { case PRIMITIVE: LazyUtils.writePrimitiveUTF8(out, obj, (PrimitiveObjectInspector) objInspector, escaped, escapeChar, needsEscape); return; case LIST: separator = (char) LazyUtils.getSeparator(separators, level); ListObjectInspector loi = (ListObjectInspector) objInspector; list = loi.getList(obj); ObjectInspector eoi = loi.getListElementObjectInspector(); if (list == null) { out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); } else { for (int i = 0; i < list.size(); i++) { if (i > 0) { out.write(separator); separator = (char) LazyUtils.getSeparator(separators, level); char keyValueSeparator = (char) LazyUtils.getSeparator(separators, level + 1); separator = (char) LazyUtils.getSeparator(separators, level); StructObjectInspector soi = (StructObjectInspector) objInspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); separator = (char) LazyUtils.getSeparator(separators, level); UnionObjectInspector uoi = (UnionObjectInspector) objInspector; List<? extends ObjectInspector> ois = uoi.getObjectInspectors();
@Override public void init(ByteArrayRef bytes, int start, int length) { if (oi.isEscaped()) { Text textData = data.getTextValue(); // This is doing a lot of copying here, this could be improved by enforcing length // at the same time as escaping rather than as separate steps. LazyUtils.copyAndEscapeStringDataToText(bytes.getData(), start, length, oi.getEscapeChar(),textData); data.set(textData.toString(), maxLength); isNull = false; } else { String byteData = null; try { byteData = Text.decode(bytes.getData(), start, length); data.set(byteData, maxLength); isNull = false; } catch (CharacterCodingException e) { isNull = true; LOG.debug("Data not in the HiveChar data type range so converted to null.", e); } } }
/** * Attempts to construct the binary value from the given inspector. Falls back to UTF8 encoding * when the value cannot be coerced into binary. * * @return Binary value when possible, utf8 otherwise * @throws IOException */ protected Text getBinaryValue(ConstantObjectInspector objInspector) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); if (objInspector instanceof PrimitiveObjectInspector) { LazyUtils.writePrimitive(out, objInspector.getWritableConstantValue(), (PrimitiveObjectInspector) objInspector); } else { return getUtf8Value(objInspector); } out.close(); return new Text(out.toByteArray()); } }
throws IOException, SerDeException { if (obj == null) { out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); return; byte[] toWrite = new byte[bw.getLength()]; System.arraycopy(bw.getBytes(), 0, toWrite, 0, bw.getLength()); out.write(toWrite, 0, toWrite.length); } else { LazyUtils.writePrimitiveUTF8(out, obj, oi, escaped, escapeChar, needsEscape); ObjectInspector eoi = loi.getListElementObjectInspector(); if (list == null) { out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); } else { for (int i = 0; i < list.size(); i++) {
@Override public void init(ByteArrayRef bytes, int start, int length) { if (oi.isEscaped()) { byte escapeChar = oi.getEscapeChar(); byte[] inputBytes = bytes.getData(); LazyUtils.copyAndEscapeStringDataToText(inputBytes, start, length, escapeChar, data); } else { // if the data is not escaped, simply copy the data. data.set(bytes.getData(), start, length); } isNull = false; }
ByteBuffer b = Text.encode(String.valueOf(f)); out.write(b.array(), 0, b.limit()); break; ByteBuffer b = Text.encode(String.valueOf(d)); out.write(b.array(), 0, b.limit()); break; writeEscaped(out, t.getBytes(), 0, t.getLength(), escaped, escapeChar, needsEscape); break; HiveCharWritable hc = ((HiveCharObjectInspector) oi).getPrimitiveWritableObject(o); Text t = hc.getPaddedValue(); writeEscaped(out, t.getBytes(), 0, t.getLength(), escaped, escapeChar, needsEscape); break; HiveVarcharWritable hc = ((HiveVarcharObjectInspector)oi).getPrimitiveWritableObject(o); Text t = hc.getTextValue(); writeEscaped(out, t.getBytes(), 0, t.getLength(), escaped, escapeChar, needsEscape); break;
@Override protected int getLength(ObjectInspector objectInspector, ByteArrayRef cachedByteArrayRef, int start, int fieldLen) { if (fieldLen == lengthNullSequence) { byte[] data = cachedByteArrayRef.getData(); if (LazyUtils.compare(data, start, fieldLen, nullSequence.getBytes(), 0, lengthNullSequence) == 0) { return -1; } } return fieldLen; }
@Override public void writeHiveVarchar(HiveVarchar hiveVarchar) throws IOException { beginPrimitive(); ByteBuffer b = Text.encode(hiveVarchar.getValue()); LazyUtils.writeEscaped(output, b.array(), 0, b.limit(), isEscaped, escapeChar, needsEscape); finishPrimitive(); }
LazyUtils.writePrimitiveUTF8(ss, obj, (PrimitiveObjectInspector) objInspector, escaped, escapeChar, needsEscape); return true; case LIST: for (int i = 0; i < list.size(); i++) { if (i > 0) { ss.write(separator); first = false; } else { ss.write(sep); ss.write(keyValueSeparator); serialize(currentValue, voi, level + 2, ss);
separators[0] = getByte(schema.getProperty(FIELD_DELIM, schema.getProperty(SERIALIZATION_FORMAT)), DEFAULT_SEPARATORS[0]); separators[1] = getByte(schema.getProperty(COLLECTION_DELIM), DEFAULT_SEPARATORS[1]); separators[2] = getByte(schema.getProperty(MAPKEY_DELIM), DEFAULT_SEPARATORS[2]); Byte escapeByte = null; if (escapeProperty != null) { escapeByte = getByte(escapeProperty, (byte) '\\');
offset++; if (length == 1) { throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
private byte[] serialize(Object obj, ObjectInspector objInspector, int level, boolean writeBinary) throws IOException { output.reset(); if (objInspector.getCategory() == ObjectInspector.Category.PRIMITIVE && writeBinary) { LazyUtils.writePrimitive(output, obj, (PrimitiveObjectInspector) objInspector); } else { if (!serialize(obj, objInspector, level, output)) { return null; } } return output.toByteArray(); }
@Override public void writeString(byte[] v, int start, int length) throws IOException { beginPrimitive(); LazyUtils.writeEscaped(output, v, start, length, isEscaped, escapeChar, needsEscape); finishPrimitive(); }
@Override public void init(ByteArrayRef bytes, int start, int length) { if (oi.isEscaped()) { Text textData = data.getTextValue(); // This is doing a lot of copying here, this could be improved by enforcing length // at the same time as escaping rather than as separate steps. LazyUtils.copyAndEscapeStringDataToText(bytes.getData(), start, length, oi.getEscapeChar(),textData); data.set(textData.toString(), maxLength); isNull = false; } else { try { String byteData = null; byteData = Text.decode(bytes.getData(), start, length); data.set(byteData, maxLength); isNull = false; } catch (CharacterCodingException e) { isNull = true; LOG.debug("Data not in the HiveVarchar data type range so converted to null.", e); } } }
/** * Attempts to construct the binary value from the given inspector. Falls back to UTF8 encoding * when the value cannot be coerced into binary. * * @return Binary value when possible, utf8 otherwise * @throws IOException */ protected Text getBinaryValue(ConstantObjectInspector objInspector) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); if (objInspector instanceof PrimitiveObjectInspector) { LazyUtils.writePrimitive(out, objInspector.getWritableConstantValue(), (PrimitiveObjectInspector) objInspector); } else { return getUtf8Value(objInspector); } out.close(); return new Text(out.toByteArray()); } }