Refine search
@Override public Object doDeserialize(Writable blob) throws SerDeException { if (byteArrayRef == null) { byteArrayRef = new ByteArrayRef(); } // we use the default field delimiter('\1') to replace the multiple-char field delimiter // but we cannot use it to parse the row since column data can contain '\1' as well String rowStr; if (blob instanceof BytesWritable) { BytesWritable b = (BytesWritable) blob; rowStr = new String(b.getBytes()); } else if (blob instanceof Text) { Text rowText = (Text) blob; rowStr = rowText.toString(); } else { throw new SerDeException(getClass() + ": expects either BytesWritable or Text object!"); } byteArrayRef.setData(rowStr.replaceAll(Pattern.quote(fieldDelimited), "\1").getBytes()); cachedLazyStruct.init(byteArrayRef, 0, byteArrayRef.getData().length); // use the multi-char delimiter to parse the lazy struct cachedLazyStruct.parseMultiDelimit(rowStr.getBytes(), fieldDelimited.getBytes()); return cachedLazyStruct; }
public static ByteArrayRef newByteArrayRef(String encodedMap) { ByteArrayRef bytes = new ByteArrayRef(); bytes.setData(encodedMap.getBytes(StandardCharsets.US_ASCII)); return bytes; } }
/** * Convert the given object to a lazy object using the given {@link ObjectInspector} * * @param obj Object to be converted to a {@link LazyObject} * @param oi ObjectInspector used for the conversion * @return the created {@link LazyObject lazy object} * */ private LazyObject<? extends ObjectInspector> toLazyPrimitiveObject(Object obj, ObjectInspector oi) { if (obj == null) { return null; } LazyObject<? extends ObjectInspector> lazyObject = LazyFactory.createLazyObject(oi); ByteArrayRef ref = new ByteArrayRef(); String objAsString = obj.toString().trim(); ref.setData(objAsString.getBytes()); // initialize the lazy object lazyObject.init(ref, 0, ref.getData().length); return lazyObject; }
@Override public void init(ByteArrayRef bytes, int start, int length) { int value = 0; try { in = new ByteStream.Input(bytes.getData(), start, length); din = new DataInputStream(in); value = din.readInt(); data.set(value); isNull = false; } catch (IOException e) { isNull = true; } finally { try { din.close(); } catch (IOException e) { // swallow exception } try { in.close(); } catch (IOException e) { // swallow exception } } } }
@Override public void init(ByteArrayRef bytes, int start, int length) { long value = 0; try { in = new ByteStream.Input(bytes.getData(), start, length); din = new DataInputStream(in); value = din.readLong(); data.set(value); isNull = false; } catch (IOException e) { isNull = true; } finally { try { din.close(); } catch (IOException e) { // swallow exception } try { in.close(); } catch (IOException e) { // swallow exception } } } }
@Override public void init(ByteArrayRef bytes, int start, int length) { short value = 0; try { in = new ByteStream.Input(bytes.getData(), start, length); din = new DataInputStream(in); value = din.readShort(); data.set(value); isNull = false; } catch (Exception e) { isNull = true; } finally { try { din.close(); } catch (IOException e) { // swallow exception } try { in.close(); } catch (IOException e) { // swallow exception } } } }
@Override public void init(ByteArrayRef bytes, int start, int length) { boolean value = false; try { in = new ByteStream.Input(bytes.getData(), start, length); din = new DataInputStream(in); value = din.readBoolean(); data.set(value); isNull = false; } catch (IOException e) { isNull = true; } finally { try { din.close(); } catch (IOException e) { // swallow exception } try { in.close(); } catch (IOException e) { // swallow exception } } } }
@Override public void init(ByteArrayRef bytes, int start, int length) { float value = 0.0F; try { in = new ByteStream.Input(bytes.getData(), start, length); din = new DataInputStream(in); value = din.readFloat(); data.set(value); isNull = false; } catch (IOException e) { isNull = true; } finally { try { din.close(); } catch (IOException e) { // swallow exception } try { in.close(); } catch (IOException e) { // swallow exception } } } }
@Override public void init(ByteArrayRef bytes, int start, int length) { byte value = 0; try { in = new ByteStream.Input(bytes.getData(), start, length); din = new DataInputStream(in); value = din.readByte(); data.set(value); isNull = false; } catch (Exception e) { isNull = true; } finally { try { din.close(); } catch (IOException e) { // swallow exception } try { in.close(); } catch (IOException e) { // swallow exception } } } }
@Override public void init(ByteArrayRef bytes, int start, int length) { double value = 0.0; try { in = new ByteStream.Input(bytes.getData(), start, length); din = new DataInputStream(in); value = din.readDouble(); data.set(value); isNull = false; } catch (IOException e) { isNull = true; } finally { try { din.close(); } catch (IOException e) { // swallow exception } try { in.close(); } catch (IOException e) { // swallow exception } } } }
protected final boolean isNull( Text nullSequence, byte[] bytes, int fieldByteBegin, int fieldLength) { // Test the length first so in most cases we avoid doing a byte[] // comparison. return fieldLength < 0 || (fieldLength == nullSequence.getLength() && LazyUtils.compare(bytes, fieldByteBegin, fieldLength, nullSequence.getBytes(), 0, nullSequence.getLength()) == 0); }
@Override public void init(ByteArrayRef bytes, int start, int length) { if (oi.isEscaped()) { Text textData = data.getTextValue(); // This is doing a lot of copying here, this could be improved by enforcing length // at the same time as escaping rather than as separate steps. LazyUtils.copyAndEscapeStringDataToText(bytes.getData(), start, length, oi.getEscapeChar(),textData); data.set(textData.toString(), maxLength); isNull = false; } else { String byteData = null; try { byteData = Text.decode(bytes.getData(), start, length); data.set(byteData, maxLength); isNull = false; } catch (CharacterCodingException e) { isNull = true; LOG.debug("Data not in the HiveChar data type range so converted to null.", e); } } }
@Override public void init(ByteArrayRef bytes, int start, int length) { if (oi.isEscaped()) { byte escapeChar = oi.getEscapeChar(); byte[] inputBytes = bytes.getData(); LazyUtils.copyAndEscapeStringDataToText(inputBytes, start, length, escapeChar, data); } else { // if the data is not escaped, simply copy the data. data.set(bytes.getData(), start, length); } isNull = false; }
@Override protected int getLength(ObjectInspector objectInspector, ByteArrayRef cachedByteArrayRef, int start, int fieldLen) { if (fieldLen == lengthNullSequence) { byte[] data = cachedByteArrayRef.getData(); if (LazyUtils.compare(data, start, fieldLen, nullSequence.getBytes(), 0, lengthNullSequence) == 0) { return -1; } } return fieldLen; }
totalVariableSize += ((LazyPrimitive<LazyStringObjectInspector, Text>) key). getWritableObject().getLength(); } else if (key instanceof String) { totalVariableSize += ((String) key).length(); } else if (key instanceof Text) { totalVariableSize += ((Text) key).getLength(); } else if (key instanceof LazyBinary) { totalVariableSize += totalVariableSize += ((BytesWritable) key).getLength(); } else if (key instanceof ByteArrayRef) { totalVariableSize += ((ByteArrayRef) key).getData().length;
public void logExceptionMessage(ByteArrayRef bytes, int start, int length, String dataType) { try { if(LOG.isDebugEnabled()) { String byteData = Text.decode(bytes.getData(), start, length); LOG.debug("Data not in the " + dataType + " data type range so converted to null. Given data is :" + byteData, new Exception("For debugging purposes")); } } catch (CharacterCodingException e1) { LOG.debug("Data not in the " + dataType + " data type range so converted to null.", e1); } }
@Override public void init(ByteArrayRef bytes, int start, int length) { String byteData = null; if (!LazyUtils.isNumberMaybe(bytes.getData(), start, length)) { isNull = true; return; } try { byteData = Text.decode(bytes.getData(), start, length); data.set(Double.parseDouble(byteData)); isNull = false; } catch (NumberFormatException e) { isNull = true; LOG.debug("Data not in the Double data type range so converted to null. Given data is :" + byteData, e); } catch (CharacterCodingException e) { isNull = true; LOG.debug("Data not in the Double data type range so converted to null.", e); } }
/** * Initializes LazyDate object by interpreting the input bytes as a SQL date string. * * @param bytes * @param start * @param length */ @Override public void init(ByteArrayRef bytes, int start, int length) { String s = null; if (!LazyUtils.isDateMaybe(bytes.getData(), start, length)) { isNull = true; return; } try { s = Text.decode(bytes.getData(), start, length); data.set(Date.valueOf(s)); isNull = false; } catch (Exception e) { isNull = true; logExceptionMessage(bytes, start, length, "DATE"); } }
public FieldInfo(LazyObjectBase lazyObject, boolean fieldSkipped, ObjectInspector oi) { field = lazyObject; cachedByteArrayRef = new ByteArrayRef(); objectInspector = oi; if (fieldSkipped) { this.fieldSkipped = true; inited = true; } else { inited = false; } }