@Override public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException, IllegalAccessException { StringObjectInspector poi = (StringObjectInspector) oi; if (oprot instanceof WriteTextProtocol) { ((WriteTextProtocol) oprot).writeText(poi.getPrimitiveWritableObject(o)); } else { oprot.writeString(poi.getPrimitiveJavaObject(o)); } }
@Override public Object getTransformedWritable(DeferredObject object) throws HiveException { String value = columnType.getPrimitiveJavaObject(object.get()); if(value != null) { String transformedValue = transformer.transform(value); if(transformedValue != null) { writable.set(transformedValue); return writable; } } return null; } }
@Override public void write(Object value) { String v = inspector.getPrimitiveJavaObject(value); recordConsumer.addBinary(Binary.fromString(v)); } }
@Override public Object getTransformedWritable(DeferredObject object) throws HiveException { String value = columnType.getPrimitiveJavaObject(object.get()); if(value != null) { String transformedValue = transformer.transform(value); if(transformedValue != null) { writable.set(transformedValue); return writable; } } return null; } }
@Override public void write(Object value) { String v = inspector.getPrimitiveJavaObject(value); recordConsumer.addBinary(Binary.fromString(v)); } }
@Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { final StructObjectInspector outputRowOI = (StructObjectInspector) objInspector; final List<? extends StructField> outputFieldRefs = outputRowOI.getAllStructFieldRefs(); if (outputFieldRefs.size() != numCols) { throw new SerDeException("Cannot serialize the object because there are " + outputFieldRefs.size() + " fields but the table has " + numCols + " columns."); } // Get all data out. for (int c = 0; c < numCols; c++) { final Object field = outputRowOI.getStructFieldData(obj, outputFieldRefs.get(c)); final ObjectInspector fieldOI = outputFieldRefs.get(c).getFieldObjectInspector(); // The data must be of type String final StringObjectInspector fieldStringOI = (StringObjectInspector) fieldOI; // Convert the field to Java class String, because objects of String type // can be stored in String, Text, or some other classes. outputFields[c] = fieldStringOI.getPrimitiveJavaObject(field); } final StringWriter writer = new StringWriter(); final CSVWriter csv = newWriter(writer, separatorChar, quoteChar, escapeChar); try { csv.writeNext(outputFields); csv.close(); return new Text(writer.toString()); } catch (final IOException ioe) { throw new SerDeException(ioe); } }
/** * init exec */ public void initExec(DeferredObject[] arguments) throws HiveException { exec = new Exec(); exec.enterGlobalScope(); String query = queryOI.getPrimitiveJavaObject(arguments[0].get()); String[] args = { "-e", query, "-trace" }; try { exec.setUdfRun(true); exec.init(args); } catch (Exception e) { throw new HiveException(e.getMessage()); } }
@Override public void process(Object[] arguments) throws HiveException { String query = stringOI.getPrimitiveJavaObject(arguments[0]); int num = intOI.get(arguments[1]); PlanFragment fragment = createPlanFragment(query, num); TezWork tezWork = fragment.work; Schema schema = fragment.schema; try { for (InputSplit s : getSplits(jc, num, tezWork, schema)) { Object[] os = new Object[1]; bos.reset(); s.write(dos); byte[] frozen = bos.toByteArray(); os[0] = frozen; forward(os); } } catch (Exception e) { throw new HiveException(e); } }
public boolean areEqual(Object o0, Object o1) { if (o0 == null && o1 == null) { return true; } else if (o0 == null || o1 == null) { return false; } switch (compareType) { case COMPARE_TEXT: return (soi0.getPrimitiveWritableObject(o0).equals( soi1.getPrimitiveWritableObject(o1))); case COMPARE_INT: return (ioi0.get(o0) == ioi1.get(o1)); case COMPARE_LONG: return (loi0.get(o0) == loi1.get(o1)); case COMPARE_BYTE: return (byoi0.get(o0) == byoi1.get(o1)); case COMPARE_BOOL: return (boi0.get(o0) == boi1.get(o1)); case COMPARE_STRING: return (soi0.getPrimitiveJavaObject(o0).equals( soi1.getPrimitiveJavaObject(o1))); default: return (ObjectInspectorUtils.compare( o0, oi0, o1, oi1) == 0); } } }
@Override public void process(Object[] arguments) throws HiveException { String query = stringOI.getPrimitiveJavaObject(arguments[0]); int num = intOI.get(arguments[1]);
private void unpackDoubleStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) throws UnsupportedDoubleException { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setNumDVs(v); } else if (fName.equals("max")) { double d = ((DoubleObjectInspector) oi).get(o); if (Double.isInfinite(d) || Double.isNaN(d)) { throw new UnsupportedDoubleException(); } statsObj.getStatsData().getDoubleStats().setHighValue(d); } else if (fName.equals("min")) { double d = ((DoubleObjectInspector) oi).get(o); if (Double.isInfinite(d) || Double.isNaN(d)) { throw new UnsupportedDoubleException(); } statsObj.getStatsData().getDoubleStats().setLowValue(d); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; String v = ((StringObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDoubleStats().setBitVectors(v);; } }
private void unpackStringStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setNumDVs(v); } else if (fName.equals("avglength")) { double d = ((DoubleObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setAvgColLen(d); } else if (fName.equals("maxlength")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setMaxColLen(v); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; String v = ((StringObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getStringStats().setBitVectors(v);; } }
private void unpackLongStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setNumDVs(v); } else if (fName.equals("max")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setHighValue(v); } else if (fName.equals("min")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setLowValue(v); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; String v = ((StringObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getLongStats().setBitVectors(v);; } }
/** * Set parameters for the current call */ void setParameters(DeferredObject[] arguments) throws HiveException { for (int i = 1; i < arguments.length; i++) { String name = ":" + i; if (argumentsOI[i] instanceof StringObjectInspector) { String value = ((StringObjectInspector)argumentsOI[i]).getPrimitiveJavaObject(arguments[i].get()); if (value != null) { exec.setVariable(name, value); } } else if (argumentsOI[i] instanceof IntObjectInspector) { Integer value = (Integer)((IntObjectInspector)argumentsOI[i]).getPrimitiveJavaObject(arguments[i].get()); if (value != null) { exec.setVariable(name, new Var(new Long(value))); } } else if (argumentsOI[i] instanceof LongObjectInspector) { Long value = (Long)((LongObjectInspector)argumentsOI[i]).getPrimitiveJavaObject(arguments[i].get()); if (value != null) { exec.setVariable(name, new Var(value)); } } else { exec.setVariableToNull(name); } } }
private void unpackDecimalStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDecimalStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDecimalStats().setNumDVs(v); } else if (fName.equals("max")) { HiveDecimal d = ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDecimalStats().setHighValue(convertToThriftDecimal(d)); } else if (fName.equals("min")) { HiveDecimal d = ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDecimalStats().setLowValue(convertToThriftDecimal(d)); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; String v = ((StringObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDecimalStats().setBitVectors(v);; } }
private void unpackDateStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumDVs(v); } else if (fName.equals("max")) { DateWritable v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setHighValue(new Date(v.getDays())); } else if (fName.equals("min")) { DateWritable v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setLowValue(new Date(v.getDays())); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; String v = ((StringObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDateStats().setBitVectors(v);; } }
String s = soi.getPrimitiveJavaObject(o).trim(); return TimestampTZUtil.parseOrNull(trimNanoTimestamp(s), timeZone);
@Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { if (partial != null) { NumericStatsAgg myagg = (NumericStatsAgg) agg; if (myagg.numDV == null) { Object partialValue = soi.getStructFieldData(partial, numBitVectorsField); int numVectors = numBitVectorsFieldOI.get(partialValue); if (numVectors <= 0) { return; } myagg.initNDVEstimator(numVectors); } // Update min if min is lesser than the smallest value seen so far Object minValue = soi.getStructFieldData(partial, minField); myagg.updateMin(minValue, minFieldOI); // Update max if max is greater than the largest value seen so far Object maxValue = soi.getStructFieldData(partial, maxField); myagg.updateMax(maxValue, maxFieldOI); // Update the null counter Object countNull = soi.getStructFieldData(partial, countNullsField); myagg.countNulls += countNullsFieldOI.get(countNull); // Merge numDistinctValue Estimators Object numDistinct = soi.getStructFieldData(partial, ndvField); String v = ndvFieldOI.getPrimitiveJavaObject(numDistinct); NumDistinctValueEstimator o = new NumDistinctValueEstimator(v, myagg.numDV.getnumBitVectors()); myagg.numDV.mergeEstimators(o); } } }
return; case STRING: type.writeSlice(builder, Slices.utf8Slice(((StringObjectInspector) inspector).getPrimitiveJavaObject(object))); return; case VARCHAR:
break; case STRING: String v = ((StringObjectInspector) inspector).getPrimitiveJavaObject(value); recordConsumer.addBinary(Binary.fromString(v)); break;