Refine search
public static void assertMapDecode(String encodedMap, Map<? extends Object, ? extends Object> expectedMap) { LazyMap lazyMap = (LazyMap) createLazyObject(getLazySimpleMapObjectInspector( LAZY_STRING_OBJECT_INSPECTOR, getLazyStringObjectInspector(false, (byte) 0), (byte) 2, (byte) 3, new Text("\\N"), false, (byte) 0)); lazyMap.init(newByteArrayRef(encodedMap), 0, encodedMap.length()); Map<Object, Object> map = lazyMap.getMap(); assertEquals(map, expectedMap); }
@Override public ObjectInspector createValueObjectInspector(TypeInfo type) throws SerDeException { return LazyFactory.createLazyObjectInspector(type, 1, serdeParams, ObjectInspectorOptions.JAVA); }
LazyFactory.createLazyPrimitiveClass( (PrimitiveObjectInspector) lazyMoi.getMapKeyObjectInspector(), binaryStorage.get(0)); ByteArrayRef keyRef = new ByteArrayRef(); if (qualPrefix!=null && hideQualPrefix){ keyRef.setData(Bytes.tail(e.getKey(), e.getKey().length-qualPrefix.length)); //cut prefix from hive's map key }else{ keyRef.setData(e.getKey()); //for non-prefix maps key.init(keyRef, 0, keyRef.getData().length); LazyFactory.createLazyObject(lazyMoi.getMapValueObjectInspector(), binaryStorage.get(1));
/** * Create a lazy primitive object instance given a primitive object inspector based on it's * type. It takes a boolean switch to decide whether to return a binary or standard variant * of the lazy object. * * @param poi PrimitiveObjectInspector * @param typeBinary a switch to return either a LazyPrimtive class or it's binary * companion * @return LazyPrimitive<? extends ObjectInspector, ? extends Writable> */ public static LazyPrimitive<? extends ObjectInspector, ? extends Writable> createLazyPrimitiveClass(PrimitiveObjectInspector poi, boolean typeBinary) { if (typeBinary) { return createLazyPrimitiveBinaryClass(poi); } else { return createLazyPrimitiveClass(poi); } }
/** * Creates a LazyObject based on the LazyObjectInspector. Will create binary variants for * primitive objects when the switch <code>typeBinary</code> is specified as true. * * @param oi ObjectInspector * @param typeBinary Boolean value used as switch to return variants of LazyPrimitive * objects which are initialized from a binary format for the data. * @return LazyObject<? extends ObjectInspector> */ public static LazyObject<? extends ObjectInspector> createLazyObject(ObjectInspector oi, boolean typeBinary) { if (oi.getCategory() == Category.PRIMITIVE) { return createLazyPrimitiveClass((PrimitiveObjectInspector) oi, typeBinary); } else { return createLazyObject(oi); } }
/** * Initialize the SerDe given the parameters. serialization.format: separator * char or byte code (only supports byte-value up to 127) columns: * ","-separated column names columns.types: ",", ":", or ";"-separated column * types * * @see org.apache.hadoop.hive.serde2.AbstractSerDe#initialize(Configuration, Properties) */ @Override public void initialize(Configuration job, Properties tbl) throws SerDeException { super.initialize(job, tbl); serdeParams = new LazySerDeParameters(job, tbl, getClass().getName()); // Create the ObjectInspectors for the fields cachedObjectInspector = LazyFactory.createLazyStructInspector(serdeParams .getColumnNames(), serdeParams.getColumnTypes(), new LazyObjectInspectorParametersImpl(serdeParams)); cachedLazyStruct = (LazyStruct) LazyFactory .createLazyObject(cachedObjectInspector); serializedSize = 0; stats = new SerDeStats(); lastOperationSerialize = false; lastOperationDeserialize = false; }
Text cf = new Text(columnMapping.getColumnFamily()); for (ColumnTuple tuple : sourceRow.getTuples()) { String cq = tuple.getCq().toString(); if (!cf.equals(tuple.getCf()) || !cq.startsWith(columnMapping.getColumnQualifierPrefix())) { .createLazyPrimitiveClass((PrimitiveObjectInspector) lazyMoi.getMapKeyObjectInspector(), ColumnEncoding.BINARY == columnMapping.getKeyEncoding()); ByteArrayRef keyRef = new ByteArrayRef(); keyRef.setData(cq.getBytes(Charsets.UTF_8)); key.init(keyRef, 0, keyRef.getData().length); LazyObject<?> value = LazyFactory.createLazyObject(lazyMoi.getMapValueObjectInspector(), ColumnEncoding.BINARY == columnMapping.getValueEncoding());
@Test public void testInitColumnPrefix() throws Exception { Text nullSequence = new Text("\\N"); ObjectInspector oi = LazyFactory.createLazyObjectInspector( TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(0), new byte[]{(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0); assertNotNull(b.getMapValueElement(new Text(col1))); assertNotNull(b.getMapValueElement(new Text(col2)));
List<String> fieldNames = Arrays.asList( new String[]{"key", "a", "b", "c", "d"}); Text nullSequence = new Text("\\N"); String hbaseColsMapping = ":key,cfa:a,cfa:b,cfb:,cfc:d"; ObjectInspector oi = LazyFactory.createLazyStructInspector( fieldNames, fieldTypeInfos,
/** * Get the value object with the index without checking parsed. * * @param index * The index into the array starting from 0 */ private LazyObject uncheckedGetValue(int index) { Text nullSequence = oi.getNullSequence(); int valueIBegin = keyEnd[index] + 1; int valueILength = keyStart[index + 1] - 1 - valueIBegin; if (valueILength < 0 || ((valueILength == nullSequence.getLength()) && 0 == LazyUtils .compare(bytes.getData(), valueIBegin, valueILength, nullSequence .getBytes(), 0, nullSequence.getLength()))) { return null; } if (!valueInited[index]) { valueInited[index] = true; if (valueObjects[index] == null) { valueObjects[index] = LazyFactory .createLazyObject(((MapObjectInspector) oi) .getMapValueObjectInspector()); } valueObjects[index].init(bytes, valueIBegin, valueILength); } return valueObjects[index]; }
/** * Get the key object with the index without checking parsed. * * @param index * The index into the array starting from 0 */ private LazyPrimitive<?, ?> uncheckedGetKey(int index) { Text nullSequence = oi.getNullSequence(); int keyIBegin = keyStart[index]; int keyILength = keyEnd[index] - keyStart[index]; if (keyILength < 0 || ((keyILength == nullSequence.getLength()) && 0 == LazyUtils.compare( bytes.getData(), keyIBegin, keyILength, nullSequence.getBytes(), 0, nullSequence.getLength()))) { return null; } if (!keyInited[index]) { keyInited[index] = true; if (keyObjects[index] == null) { // Keys are always primitive keyObjects[index] = LazyFactory .createLazyPrimitiveClass((PrimitiveObjectInspector) ((MapObjectInspector) oi) .getMapKeyObjectInspector()); } keyObjects[index].init(bytes, keyIBegin, keyILength); } return keyObjects[index]; }
this.lengthNullSequence = nullSequence.getLength(); LazyFactory.createLazyObject(fieldRefs.get(i) .getFieldObjectInspector()), !notSkippedColumnIDs.contains(i));
private void parseObjectColumn(int column, byte[] bytes, int start, int length) { boolean wasNull; if (length == "\\N".length() && bytes[start] == '\\' && bytes[start + 1] == 'N') { wasNull = true; } else { LazyObject<? extends ObjectInspector> lazyObject = LazyFactory.createLazyObject(fieldInspectors[column]); ByteArrayRef byteArrayRef = new ByteArrayRef(); byteArrayRef.setData(bytes); lazyObject.init(byteArrayRef, start, length); objects[column] = getBlockObject(types[column], lazyObject.getObject(), fieldInspectors[column]); wasNull = false; } nulls[column] = wasNull; } @Override
@Override protected LazyObjectBase createLazyObjectBase(ObjectInspector objectInspector) { return LazyFactory.createLazyObject(objectInspector); } }
byte[] bytes = this.bytes.getData(); field = LazyFactory.createLazyObject(oi.getObjectInspectors().get(tag)); fieldInited = false; parsed = true;
/** * Create a hierarchical ObjectInspector for LazyStruct with the given * columnNames and columnTypeInfos. * * @param lastColumnTakesRest * whether the last column of the struct should take the rest of the * row if there are extra fields. * @throws SerDeException * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, * boolean, byte) */ @Deprecated public static ObjectInspector createLazyStructInspector( List<String> columnNames, List<TypeInfo> typeInfos, byte[] separators, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, byte escapeChar) throws SerDeException { return createLazyStructInspector(columnNames, typeInfos, separators, nullSequence, lastColumnTakesRest, escaped, escapeChar, false); }
/** * Create a hierarchical LazyObject based on the given typeInfo. */ public static LazyObject<? extends ObjectInspector> createLazyObject(ObjectInspector oi) { ObjectInspector.Category c = oi.getCategory(); switch (c) { case PRIMITIVE: return createLazyPrimitiveClass((PrimitiveObjectInspector) oi); case MAP: return new LazyMap((LazyMapObjectInspector) oi); case LIST: return new LazyArray((LazyListObjectInspector) oi); case STRUCT: return new LazyStruct((LazySimpleStructObjectInspector) oi); case UNION: return new LazyUnion((LazyUnionObjectInspector) oi); } throw new RuntimeException("Hive LazySerDe Internal error."); }
/** * Create a hierarchical ObjectInspector for ColumnarStruct with the given * columnNames and columnTypeInfos. * @throws SerDeException * * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, * boolean, byte) */ @Deprecated public static ObjectInspector createColumnarStructInspector( List<String> columnNames, List<TypeInfo> columnTypes, byte[] separators, Text nullSequence, boolean escaped, byte escapeChar) throws SerDeException { LazyObjectInspectorParametersImpl lazyParams = new LazyObjectInspectorParametersImpl( escaped, escapeChar, false, null, separators, nullSequence); return createColumnarStructInspector(columnNames, columnTypes, lazyParams); }
Text cf = new Text(columnMapping.getColumnFamily()); for (ColumnTuple tuple : sourceRow.getTuples()) { String cq = tuple.getCq().toString(); if (!cf.equals(tuple.getCf()) || !cq.startsWith(columnMapping.getColumnQualifierPrefix())) { .createLazyPrimitiveClass((PrimitiveObjectInspector) lazyMoi.getMapKeyObjectInspector(), ColumnEncoding.BINARY == columnMapping.getKeyEncoding()); ByteArrayRef keyRef = new ByteArrayRef(); keyRef.setData(cq.getBytes(Charsets.UTF_8)); key.init(keyRef, 0, keyRef.getData().length); LazyObject<?> value = LazyFactory.createLazyObject(lazyMoi.getMapValueObjectInspector(), ColumnEncoding.BINARY == columnMapping.getValueEncoding());
Text nullSequence = new Text("\\N"); ObjectInspector oi = LazyFactory.createLazyObjectInspector( TypeInfoUtils.getTypeInfosFromTypeString("map<int,string>").get(0), new byte[]{(byte)1, (byte)2}, 0, nullSequence, false, (byte)0); new Text("def"), ((LazyString)b.getMapValueElement( new IntWritable(2))).getWritableObject()); new Text("0"), ((LazyString)b.getMapValueElement( new IntWritable(0))).getWritableObject());