@Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { if(partial == null) { return; } List<DoubleWritable> partialHistogram = (List<DoubleWritable>) loi.getList(partial); DoubleObjectInspector doi = (DoubleObjectInspector)loi.getListElementObjectInspector(); StdAgg myagg = (StdAgg) agg; myagg.histogram.merge(partialHistogram, doi); }
private List getListObject(ListObjectInspector listObjectInspector, Object listObject) { if (listObjectInspector.getListLength(listObject) < 0) { return null; } List<?> objectList = listObjectInspector.getList(listObject); List<?> list = null; ObjectInspector child = listObjectInspector.getListElementObjectInspector(); switch (child.getCategory()) { case PRIMITIVE: final PrimitiveObjectInspector primitiveObjectInspector = (PrimitiveObjectInspector) child; list = objectList.stream() .map(input -> coercePrimitiveObject(primitiveObjectInspector, input)) .collect(Collectors.toList()); break; default: break; } return list; }
@Override public void process(Object[] o) throws HiveException { List<?> list = listOI.getList(o[0]); for (Object r : list) { forwardObj[0] = r; forwardObj[1] = r; forward(forwardObj); } }
@Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { if(partial == null) { return; } List<DoubleWritable> partialHistogram = (List<DoubleWritable>) loi.getList(partial); DoubleObjectInspector doi = (DoubleObjectInspector)loi.getListElementObjectInspector(); StdAgg myagg = (StdAgg) agg; myagg.histogram.merge(partialHistogram, doi); }
@Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { MkArrayAggregationBuffer myagg = (MkArrayAggregationBuffer) agg; List<Object> partialResult = (List<Object>) internalMergeOI.getList(partial); if (partialResult != null) { for(Object i : partialResult) { putIntoCollection(i, myagg); } } }
private Object serializeList(ListTypeInfo typeInfo, ListObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException { List<?> list = fieldOI.getList(structFieldData); List<Object> deserialized = new GenericData.Array<Object>(list.size(), schema); TypeInfo listElementTypeInfo = typeInfo.getListElementTypeInfo(); ObjectInspector listElementObjectInspector = fieldOI.getListElementObjectInspector(); Schema elementType = schema.getElementType(); for(int i = 0; i < list.size(); i++) { deserialized.add(i, serialize(listElementTypeInfo, listElementObjectInspector, list.get(i), elementType)); } return deserialized; }
@Override public void process(Object[] os) throws HiveException { List<?> list = li.getList(os[0]); if (list != null && !list.isEmpty()) { for (Object row : list.toArray()) { forward(row); } } }
@Override public Object evaluate(DeferredObject[] arguments) throws HiveException { if (arguments[0].get() == null) { return null; } Object array = arguments[0].get(); ListObjectInspector arrayOI = (ListObjectInspector) argumentOIs[0]; List retArray = (List) arrayOI.getList(array); final ObjectInspector valInspector = arrayOI.getListElementObjectInspector(); Collections.sort(retArray, new Comparator() { @Override public int compare(Object o1, Object o2) { return ObjectInspectorUtils.compare(o1, valInspector, o2, valInspector); } }); ret.clear(); for (int i = 0; i < retArray.size(); i++) { ret.add(converters[0].convert(retArray.get(i))); } return ret; }
@Override public void process(Object[] o) throws HiveException { List<?> list = listOI.getList(o[0]); if(list == null) { return; } for (int i = 0; i < list.size(); i++) { Object r = list.get(i); forwardObj[0] = new Integer(i); forwardObj[1] = r; forward(forwardObj); } }
@Override public Object evaluate(DeferredObject[] arguments) throws HiveException { if (arguments[0].get() == null) { return null; } Object array = arguments[0].get(); ListObjectInspector arrayOI = (ListObjectInspector) argumentOIs[0]; List retArray = (List) arrayOI.getList(array); final ObjectInspector valInspector = arrayOI.getListElementObjectInspector(); Collections.sort(retArray, new Comparator() { @Override public int compare(Object o1, Object o2) { return ObjectInspectorUtils.compare(o1, valInspector, o2, valInspector); } }); ret.clear(); for (int i = 0; i < retArray.size(); i++) { ret.add(converters[0].convert(retArray.get(i))); } return ret; }
@Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { MkArrayAggregationBuffer myagg = (MkArrayAggregationBuffer) agg; List<Object> partialResult = (ArrayList<Object>) internalMergeOI.getList(partial); if (partialResult != null) { for(Object i : partialResult) { putIntoCollection(i, myagg); } } }
private static Block serializeList(Type type, BlockBuilder builder, Object object, ListObjectInspector inspector) { List<?> list = inspector.getList(object); if (list == null) { requireNonNull(builder, "parent builder is null").appendNull(); return null; } List<Type> typeParameters = type.getTypeParameters(); checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter"); Type elementType = typeParameters.get(0); ObjectInspector elementInspector = inspector.getListElementObjectInspector(); BlockBuilder currentBuilder; if (builder != null) { currentBuilder = builder.beginBlockEntry(); } else { currentBuilder = elementType.createBlockBuilder(null, list.size()); } for (Object element : list) { serializeObject(elementType, currentBuilder, element, elementInspector); } if (builder != null) { builder.closeEntry(); return null; } else { Block resultBlock = currentBuilder.build(); return resultBlock; } }
@Override public void process(Object[] os) throws HiveException { List<?> list = li.getList(os[0]); if (list != null && !list.isEmpty()) { for (Object row : list.toArray()) { forward(row); } } }
private void writeSingleLevelArray(final Object value, final ListObjectInspector inspector, final GroupType type) { // Get the internal array structure Type elementType = type.getType(0); recordConsumer.startGroup(); List<?> arrayValues = inspector.getList(value); if (!arrayValues.isEmpty()) { recordConsumer.startField(elementType.getName(), 0); ObjectInspector elementInspector = inspector.getListElementObjectInspector(); for (Object element : arrayValues) { if (element == null) { throw new IllegalArgumentException("Array elements are requires in given schema definition"); } writeValue(element, elementInspector, elementType); } recordConsumer.endField(elementType.getName(), 0); } recordConsumer.endGroup(); }
@Override public void process(Object[] o) throws HiveException { List<?> list = listOI.getList(o[0]); if(list == null) { return; } for (int i = 0; i < list.size(); i++) { Object r = list.get(i); forwardObj[0] = new Integer(i); forwardObj[1] = r; forward(forwardObj); } }
recordConsumer.startGroup(); List<?> arrayValues = inspector.getList(value); if (!arrayValues.isEmpty()) { recordConsumer.startField(repeatedType.getName(), 0);
@Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { if(partial == null) { return; } NGramAggBuf myagg = (NGramAggBuf) agg; List partialNGrams = (List) loi.getList(partial); int n = Integer.parseInt(partialNGrams.get(partialNGrams.size()-1).toString()); // A value of 0 for n indicates that the mapper processed data that does not meet // filter criteria, so merge() should be NO-OP. if (n == 0) { return; } if(myagg.n > 0 && myagg.n != n) { throw new HiveException(getClass().getSimpleName() + ": mismatch in value for 'n'" + ", which usually is caused by a non-constant expression. Found '"+n+"' and '" + myagg.n + "'."); } myagg.n = n; partialNGrams.remove(partialNGrams.size()-1); myagg.nge.merge(partialNGrams); }
@Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { if(partial == null) { return; } NGramAggBuf myagg = (NGramAggBuf) agg; List partialNGrams = (List) loi.getList(partial); int n = Integer.parseInt(partialNGrams.get(partialNGrams.size()-1).toString()); // A value of 0 for n indicates that the mapper processed data that does not meet // filter criteria, so merge() should be NO-OP. if (n == 0) { return; } if(myagg.n > 0 && myagg.n != n) { throw new HiveException(getClass().getSimpleName() + ": mismatch in value for 'n'" + ", which usually is caused by a non-constant expression. Found '"+n+"' and '" + myagg.n + "'."); } myagg.n = n; partialNGrams.remove(partialNGrams.size()-1); myagg.nge.merge(partialNGrams); }
@Override protected Object _evaluate(Object row, int version) throws HiveException { // Get the result in leftInspectableObject Object left = leftEvaluator.evaluate(row, version); if (expr.getIsList()) { List<?> list = ((ListObjectInspector) leftInspector).getList(left); if (list == null) { return null; } else { cachedList.clear(); for (int i = 0; i < list.size(); i++) { cachedList.add(structObjectInspector.getStructFieldData(list.get(i), field)); } return cachedList; } } else { return structObjectInspector.getStructFieldData(left, field); } }
@Override protected Object _evaluate(Object row, int version) throws HiveException { // Get the result in leftInspectableObject Object left = leftEvaluator.evaluate(row, version); if (expr.getIsList()) { List<?> list = ((ListObjectInspector) leftInspector).getList(left); if (list == null) { return null; } else { cachedList.clear(); for (int i = 0; i < list.size(); i++) { cachedList.add(structObjectInspector.getStructFieldData(list.get(i), field)); } return cachedList; } } else { return structObjectInspector.getStructFieldData(left, field); } }