throws SerDeException { if (objInspector.getCategory() != Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); serializeStream.reset(); serializedSize = 0; serializeStream.write(serdeParams.getSeparators()[0]); ObjectInspector foi = fields.get(i).getFieldObjectInspector(); Object f = (list == null ? null : list.get(i)); throw new SerDeException("Error: expecting " + declaredFields.size() + " but asking for field " + i + "\n" + "data=" + obj + "\n" + "tableType=" + serdeParams.getRowTypeInfo().toString() + "\n" .set(serializeStream.getData(), 0, serializeStream.getLength()); serializedSize = serializeStream.getLength(); lastOperationSerialize = true; lastOperationDeserialize = false;
ByteStream.Output serializeStream = new ByteStream.Output(); LazyUtils.writePrimitiveUTF8(serializeStream, writable, poi, false, (byte) 0, null); startRow = new byte[serializeStream.getLength()]; System.arraycopy(serializeStream.getData(), 0, startRow, 0, serializeStream.getLength()); return startRow; return Bytes.toBytes(((ShortWritable) writable).get()); case STRING: return Bytes.toBytes(((Text) writable).toString()); case BYTE: return Bytes.toBytes(((ByteWritable) writable).get());
@Override public void beginStruct(List fieldValues) { final Field current = new Field(STRUCT); beginComplex(current); current.fieldCount = fieldValues.size(); if (!skipLengthPrefix) { // 1/ reserve spaces for the byte size of the struct // which is a integer and takes four bytes current.byteSizeStart = output.getLength(); output.reserve(4); current.start = output.getLength(); } current.nullOffset = output.getLength(); }
throws IOException, SerDeException { if (obj == null) { out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); return; byte[] toWrite = new byte[bw.getLength()]; System.arraycopy(bw.getBytes(), 0, toWrite, 0, bw.getLength()); out.write(toWrite, 0, toWrite.length); } else { LazyUtils.writePrimitiveUTF8(out, obj, oi, escaped, escapeChar, needsEscape); ObjectInspector eoi = loi.getListElementObjectInspector(); if (list == null) { out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); } else { for (int i = 0; i < list.size(); i++) { if (i > 0) { out.write(separator); Map<?, ?> map = moi.getMap(obj); if (map == null) { out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); } else { boolean first = true; first = false; } else { out.write(separator);
throws SerDeException { StructObjectInspector soi = (StructObjectInspector) objInspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); List<Object> list = soi.getStructFieldsDataAsList(obj); if (fields.size() != numColumns) { throw new SerDeException("Cannot serialize the object because there are " + fields.size() + " fields but the table has " + numColumns + " columns."); serializeStream.reset(); serializeStream.write(fieldDelimited.getBytes(), 0, fieldDelimited.getBytes().length); ObjectInspector fieldOI = fields.get(c).getFieldObjectInspector(); serdeParams.getNeedsEscape()); } catch (IOException e) { throw new SerDeException(e); serializeCache.set(serializeStream.getData(), 0, serializeStream.getLength()); return serializeCache;
Output buffer = new Output(); BinarySortableSerializeWrite binarySortableSerializeWrite = new BinarySortableSerializeWrite(fieldCount); if (constant instanceof Text) { Text text = (Text) constant; bytes = text.getBytes(); binarySortableSerializeWrite.writeString(bytes, 0, text.getLength()); } else { throw new HiveException("Unexpected constant String type " + serializedInChildren[i] = Arrays.copyOfRange(buffer.getData(), 0, buffer.getLength());
@Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { output.reset(); StructObjectInspector soi = (StructObjectInspector) objInspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < columnNames.size(); i++) { serialize(output, soi.getStructFieldData(obj, fields.get(i)), fields.get(i).getFieldObjectInspector(), columnSortOrderIsDesc[i], columnNullMarker[i], columnNotNullMarker[i]); } serializeBytesWritable.set(output.getData(), 0, output.getLength()); return serializeBytesWritable; }
Object o = keySerDe.deserialize(key); StructObjectInspector soi = (StructObjectInspector)keySerDe.getObjectInspector(); List<? extends StructField> fields = soi.getAllStructFieldRefs(); Object[] data = new Object[fields.size()]; List<ObjectInspector> fois = new ArrayList<ObjectInspector>(fields.size()); for (int i = 0; i < fields.size(); i++) { data[i] = soi.getStructFieldData(o, fields.get(i)); fois.add(fields.get(i).getFieldObjectInspector()); Output output = new Output(); boolean[] sortableSortOrders = new boolean[fields.size()]; Arrays.fill(sortableSortOrders, false); BinarySortableSerDe.serializeStruct(output, data, fois, sortableSortOrders, columnNullMarker, columnNotNullMarker); hasTag = (output.getLength() != b.getLength()); if (hasTag) { LOG.error("Tag found in keys and will be removed. This should not happen."); if (output.getLength() != (b.getLength() - 1)) { throw new SerDeException( "Unexpected tag: " + b.getLength() + " reserialized to " + output.getLength());
@Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { if (objInspector.getCategory() != Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); } // serialize the row as BytesWritable serializeByteStream.reset(); rowSerializer.serializeValue(serializeByteStream, obj, objInspector, true, nullMapKey); serializeBytesWritable.set(serializeByteStream.getData(), 0, serializeByteStream.getLength()); // Stats bookkeeping serializedSize = serializeByteStream.getLength(); lastOperationSerialize = true; lastOperationDeserialize = false; return serializeBytesWritable; }
@Override public byte[] serializeKey(Object object, StructField field) throws IOException { ObjectInspector inspector = field.getFieldObjectInspector(); if (inspector.getCategory() != ObjectInspector.Category.STRUCT) { throw new IllegalStateException("invalid type value " + inspector.getTypeName()); } output.reset(); for (Object element : ((StructObjectInspector)inspector).getStructFieldsDataAsList(object)) { output.write(toBinary(String.valueOf(element).getBytes(), FIXED_LENGTH, false, false)); } return output.getLength() > 0 ? output.toByteArray() : null; }
private boolean serialize(Object obj, ObjectInspector objInspector, int level, ByteStream.Output ss) throws IOException { switch (objInspector.getCategory()) { case PRIMITIVE: LazyUtils.writePrimitiveUTF8(ss, obj, (PrimitiveObjectInspector) objInspector, escaped, escapeChar, needsEscape); for (int i = 0; i < list.size(); i++) { if (i > 0) { ss.write(separator); first = false; } else { ss.write(sep); ss.write(keyValueSeparator); serialize(currentValue, voi, level + 2, ss); for (int i = 0; i < list.size(); i++) { if (i > 0) { ss.write(sep); serialize(currentItem, fields.get(i).getFieldObjectInspector(), level + 1, ss); throw new RuntimeException("Unknown category type: " + objInspector.getCategory());
/** * Serialize an Accumulo rowid */ protected byte[] serializeRowId(Object rowId, StructField rowIdField, ColumnMapping rowIdMapping) throws IOException { if (rowId == null) { throw new IOException("Accumulo rowId cannot be NULL"); } // Reset the buffer we're going to use output.reset(); ObjectInspector rowIdFieldOI = rowIdField.getFieldObjectInspector(); String rowIdMappingType = rowIdMapping.getColumnType(); TypeInfo rowIdTypeInfo = TypeInfoUtils.getTypeInfoFromTypeString(rowIdMappingType); if (!rowIdFieldOI.getCategory().equals(ObjectInspector.Category.PRIMITIVE) && rowIdTypeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { // we always serialize the String type using the escaped algorithm for LazyString writeString(output, SerDeUtils.getJSONString(rowId, rowIdFieldOI), PrimitiveObjectInspectorFactory.javaStringObjectInspector); return output.toByteArray(); } // use the serialization option switch to write primitive values as either a variable // length UTF8 string or a fixed width bytes if serializing in binary format getSerializedValue(rowIdFieldOI, rowId, output, rowIdMapping); return output.toByteArray(); }
private byte[] serialize(Object obj, ObjectInspector objInspector, int level, boolean writeBinary) throws IOException { output.reset(); if (objInspector.getCategory() == ObjectInspector.Category.PRIMITIVE && writeBinary) { LazyUtils.writePrimitive(output, obj, (PrimitiveObjectInspector) objInspector); } else { if (!serialize(obj, objInspector, level, output)) { return null; } } return output.toByteArray(); }
TProtocolFactory protFactory = TReflectionUtils .getProtocolFactoryByName(protoName); bos_ = new ByteStream.Output(); bis_ = new ByteStream.Input(); tios = new TIOStreamTransport(bis_, bos_); throw new SerDeException("Could not lookup table type " + type_name + " in this ddl: " + ddl); } catch (Exception e) { System.err.println(StringUtils.stringifyException(e)); throw new SerDeException(e);
public VectorKeySeriesSerializedImpl(T serializeWrite) { super(); this.serializeWrite = serializeWrite; output = new Output(); serializedKeyLengths = new int[VectorizedRowBatch.DEFAULT_SIZE]; }