public static ByteArrayRef newByteArrayRef(String encodedMap) { ByteArrayRef bytes = new ByteArrayRef(); bytes.setData(encodedMap.getBytes(StandardCharsets.US_ASCII)); return bytes; } }
@Override public Object doDeserialize(Writable blob) throws SerDeException { if (byteArrayRef == null) { byteArrayRef = new ByteArrayRef(); } // we use the default field delimiter('\1') to replace the multiple-char field delimiter // but we cannot use it to parse the row since column data can contain '\1' as well String rowStr; if (blob instanceof BytesWritable) { BytesWritable b = (BytesWritable) blob; rowStr = new String(b.getBytes()); } else if (blob instanceof Text) { Text rowText = (Text) blob; rowStr = rowText.toString(); } else { throw new SerDeException(getClass() + ": expects either BytesWritable or Text object!"); } byteArrayRef.setData(rowStr.replaceAll(Pattern.quote(fieldDelimited), "\1").getBytes()); cachedLazyStruct.init(byteArrayRef, 0, byteArrayRef.getData().length); // use the multi-char delimiter to parse the lazy struct cachedLazyStruct.parseMultiDelimit(rowStr.getBytes(), fieldDelimited.getBytes()); return cachedLazyStruct; }
/** * Convert the given object to a lazy object using the given {@link ObjectInspector} * * @param obj Object to be converted to a {@link LazyObject} * @param oi ObjectInspector used for the conversion * @return the created {@link LazyObject lazy object} * */ private LazyObject<? extends ObjectInspector> toLazyPrimitiveObject(Object obj, ObjectInspector oi) { if (obj == null) { return null; } LazyObject<? extends ObjectInspector> lazyObject = LazyFactory.createLazyObject(oi); ByteArrayRef ref = new ByteArrayRef(); String objAsString = obj.toString().trim(); ref.setData(objAsString.getBytes()); // initialize the lazy object lazyObject.init(ref, 0, ref.getData().length); return lazyObject; }
/** * Create an initialize a {@link LazyObject} with the given bytes for the given fieldID. * * @param fieldID field for which the object is to be created * @param bytes value with which the object is to be initialized with * @return initialized {@link LazyObject} * */ public LazyObject<? extends ObjectInspector> toLazyObject(int fieldID, byte[] bytes) { ObjectInspector fieldOI = oi.getAllStructFieldRefs().get(fieldID).getFieldObjectInspector(); LazyObject<? extends ObjectInspector> lazyObject = LazyFactory.createLazyObject(fieldOI); ByteArrayRef ref = new ByteArrayRef(); ref.setData(bytes); // initialize the lazy object lazyObject.init(ref, 0, ref.getData().length); return lazyObject; } }
/** * Create an initialize a {@link LazyObject} with the given bytes for the given fieldID. * * @param fieldID * field for which the object is to be created * @param bytes * value with which the object is to be initialized with * @return initialized {@link LazyObject} * */ public LazyObject<? extends ObjectInspector> toLazyObject(int fieldID, byte[] bytes) { ObjectInspector fieldOI = oi.getAllStructFieldRefs().get(fieldID).getFieldObjectInspector(); LazyObject<? extends ObjectInspector> lazyObject = LazyFactory .createLazyObject(fieldOI); ByteArrayRef ref = new ByteArrayRef(); ref.setData(bytes); // initialize the lazy object lazyObject.init(ref, 0, ref.getData().length); return lazyObject; }
/** * Deserialize a row from the Writable to a LazyObject. * * @param field * the Writable that contains the data * @return The deserialized row Object. * @see org.apache.hadoop.hive.serde2.AbstractSerDe#deserialize(Writable) */ @Override public Object doDeserialize(Writable field) throws SerDeException { if (byteArrayRef == null) { byteArrayRef = new ByteArrayRef(); } BinaryComparable b = (BinaryComparable) field; byteArrayRef.setData(b.getBytes()); cachedLazyStruct.init(byteArrayRef, 0, b.getLength()); lastOperationSerialize = false; lastOperationDeserialize = true; return cachedLazyStruct; }
/** * Deserialize a table record to a lazybinary struct. */ @Override public Object deserialize(Writable field) throws SerDeException { if (byteArrayRef == null) { byteArrayRef = new ByteArrayRef(); } BinaryComparable b = (BinaryComparable) field; if (b.getLength() == 0) { return null; } byteArrayRef.setData(b.getBytes()); cachedLazyBinaryStruct.init(byteArrayRef, 0, b.getLength()); lastOperationSerialize = false; lastOperationDeserialize = true; return cachedLazyBinaryStruct; }
private List<Object> unpack(WriteBuffers.ByteSegmentRef ref) throws HiveException { if (ref.getLength() == 0) { return EMPTY_LIST; // shortcut, 0 length means no fields } uselessIndirection.setData(ref.getBytes()); valueStruct.init(uselessIndirection, (int)ref.getOffset(), ref.getLength()); List<Object> result; if (!needsComplexObjectFixup) { // Good performance for common case where small table has no complex objects. result = valueStruct.getFieldsAsList(); } else { // Convert the complex LazyBinary objects to standard (Java) objects so downstream // operators like FileSinkOperator can serialize complex objects in the form they expect // (i.e. Java objects). result = MapJoinBytesTableContainer.getComplexFieldsAsList( valueStruct, complexObjectArrayBuffer, internalValueOi); } return result; }
private List<Object> unpack(WriteBuffers.ByteSegmentRef ref) throws HiveException { if (ref.getLength() == 0) { return EMPTY_LIST; // shortcut, 0 length means no fields } uselessIndirection.setData(ref.getBytes()); valueStruct.init(uselessIndirection, (int)ref.getOffset(), ref.getLength()); List<Object> result; if (!needsComplexObjectFixup) { // Good performance for common case where small table has no complex objects. result = valueStruct.getFieldsAsList(); } else { // Convert the complex LazyBinary objects to standard (Java) objects so downstream // operators like FileSinkOperator can serialize complex objects in the form they expect // (i.e. Java objects). result = getComplexFieldsAsList( valueStruct, complexObjectArrayBuffer, internalValueOi); } return result; }
private List<Object> unpack(WriteBuffers.ByteSegmentRef ref) throws HiveException { if (ref.getLength() == 0) { return EMPTY_LIST; // shortcut, 0 length means no fields } uselessIndirection.setData(ref.getBytes()); valueStruct.init(uselessIndirection, (int)ref.getOffset(), ref.getLength()); List<Object> result; if (!needsComplexObjectFixup) { // Good performance for common case where small table has no complex objects. result = valueStruct.getFieldsAsList(); } else { // Convert the complex LazyBinary objects to standard (Java) objects so downstream // operators like FileSinkOperator can serialize complex objects in the form they expect // (i.e. Java objects). result = MapJoinBytesTableContainer.getComplexFieldsAsList( valueStruct, complexObjectArrayBuffer, internalValueOi); } return result; }
private List<Object> unpack(WriteBuffers.ByteSegmentRef ref) throws HiveException { if (ref.getLength() == 0) { return EMPTY_LIST; // shortcut, 0 length means no fields } uselessIndirection.setData(ref.getBytes()); valueStruct.init(uselessIndirection, (int)ref.getOffset(), ref.getLength()); List<Object> result; if (!needsComplexObjectFixup) { // Good performance for common case where small table has no complex objects. result = valueStruct.getFieldsAsList(); } else { // Convert the complex LazyBinary objects to standard (Java) objects so downstream // operators like FileSinkOperator can serialize complex objects in the form they expect // (i.e. Java objects). result = getComplexFieldsAsList( valueStruct, complexObjectArrayBuffer, internalValueOi); } return result; }
cachedByteArrayRef.setData(rawBytesField.getData()); } catch (IOException e) { throw new RuntimeException(e);
/** * Create an initialize a {@link LazyObject} with the given bytes for the given fieldID. * * @param fieldID field for which the object is to be created * @param bytes value with which the object is to be initialized with * * @return initialized {@link LazyObject} * */ @Override public LazyObject<? extends ObjectInspector> toLazyObject(int fieldID, byte[] bytes) { ObjectInspector fieldOI = oi.getAllStructFieldRefs().get(fieldID).getFieldObjectInspector(); LazyObject<? extends ObjectInspector> lazyObject = LazyFactory.createLazyObject(fieldOI); ByteArrayRef ref = new ByteArrayRef(); ref.setData(bytes); // initialize the lazy object lazyObject.init(ref, 0, ref.getData().length); return lazyObject; } }
} else { ByteArrayRef ref = new ByteArrayRef(); ref.setData(bytes); fields[fieldID].init(ref, 0, bytes.length);
@Nonnull public static LazyString lazyString(@Nonnull final String str, @Nonnull final LazyStringObjectInspector oi) { LazyString lazy = new LazyString(oi); ByteArrayRef ref = new ByteArrayRef(); byte[] data = str.getBytes(StandardCharsets.UTF_8); ref.setData(data); lazy.init(ref, 0, data.length); return lazy; }
public static ByteArrayRef newByteArrayRef(String encodedMap) { ByteArrayRef bytes = new ByteArrayRef(); bytes.setData(encodedMap.getBytes(StandardCharsets.US_ASCII)); return bytes; } }
public static ByteArrayRef newByteArrayRef(String encodedMap) { ByteArrayRef bytes = new ByteArrayRef(); bytes.setData(encodedMap.getBytes(StandardCharsets.US_ASCII)); return bytes; } }
private List<Object> uppack(WriteBuffers.ByteSegmentRef ref) throws HiveException { if (ref.getLength() == 0) { return EMPTY_LIST; // shortcut, 0 length means no fields } uselessIndirection.setData(ref.getBytes()); valueStruct.init(uselessIndirection, (int)ref.getOffset(), ref.getLength()); return valueStruct.getFieldsAsList(); // TODO: should we unset bytes after that? }
private List<Object> uppack(WriteBuffers.ByteSegmentRef ref) throws HiveException { if (ref.getLength() == 0) { return EMPTY_LIST; // shortcut, 0 length means no fields } uselessIndirection.setData(ref.getBytes()); valueStruct.init(uselessIndirection, (int)ref.getOffset(), ref.getLength()); return valueStruct.getFieldsAsList(); // TODO: should we unset bytes after that? }