public StringHelper(PrimitiveCategory type) throws UDFArgumentException { this.type = type; switch (type) { case STRING: returnValue = new Text(); break; case CHAR: returnValue = new HiveCharWritable(); break; case VARCHAR: returnValue = new HiveVarcharWritable(); break; default: throw new UDFArgumentException("Unexpected non-string type " + type); } }
public Text getStrippedValue() { // A lot of these methods could be done more efficiently by operating on the Text value // directly, rather than converting to HiveChar. return new Text(getHiveChar().getStrippedValue()); }
/** * Write CHAR(N). * The representation of char in Teradata binary format is: * the byte number to read is based on the [charLength] * [bytePerChar] <- totalLength, * bytePerChar is decided by the charset: LATAIN charset is 2 bytes per char and UNICODE charset is 3 bytes per char. * the null char will use space to pad. * * @param writable the writable * @param length the byte n * @throws IOException the io exception */ public void writeChar(HiveCharWritable writable, int length) throws IOException { if (writable == null) { String pad = join("", Collections.nCopies(length, " ")); write(pad.getBytes("UTF8")); return; } Text t = writable.getStrippedValue(); int contentLength = t.getLength(); write(t.getBytes(), 0, contentLength); if (length - contentLength < 0) { throw new IOException(format("The byte num %s of HiveCharWritable is more than the byte num %s we can hold. " + "The content of HiveCharWritable is %s", contentLength, length, writable.getPaddedValue())); } if (length > contentLength) { String pad = join("", Collections.nCopies(length - contentLength, " ")); write(pad.getBytes("UTF8")); } }
public void enforceMaxLength(int maxLength) { if (getCharacterLength()!=maxLength) set(getHiveChar(), maxLength); }
public void set(HiveCharWritable val, int maxLength) { set(val.getHiveChar(), maxLength); }
private HiveCharWritable getWritableWithParams(HiveCharWritable val) { HiveCharWritable newValue = new HiveCharWritable(); newValue.set(val, getMaxLength()); return newValue; }
return null; Text t = new Text(); return null; case BOOLEAN: t.set(((BooleanObjectInspector) inputOI).get(input) ? trueBytes : falseBytes); return t; .getStrippedValue()); .toString()); t.set(((DateObjectInspector) inputOI).getPrimitiveWritableObject(input).toString()); return t; .getPrimitiveWritableObject(input).toString()); return t; case BINARY: if (binaryOI.preferWritable()) { BytesWritable bytes = binaryOI.getPrimitiveWritableObject(input); t.set(bytes.getBytes(), 0, bytes.getLength()); t.set(((HiveDecimalObjectInspector) inputOI).getPrimitiveWritableObject(input).toString()); return t; default:
@Test public void testCharVarcharArgs() throws HiveException { GenericUDFPrintf udf = new GenericUDFPrintf(); ObjectInspector[] inputOIs = { PrimitiveObjectInspectorFactory.writableStringObjectInspector, PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(TypeInfoFactory.getCharTypeInfo(5)), PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(TypeInfoFactory.getVarcharTypeInfo(7)) }; HiveCharWritable argChar = new HiveCharWritable(); argChar.set("hello"); HiveVarcharWritable argVarchar = new HiveVarcharWritable(); argVarchar.set("world"); DeferredObject[] args = { new DeferredJavaObject(new Text("1st: %s, 2nd: %s")), new DeferredJavaObject(argChar), new DeferredJavaObject(argVarchar) }; PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs); Assert.assertEquals(PrimitiveObjectInspectorFactory.writableStringObjectInspector, oi); Text res = (Text) udf.evaluate(args); Assert.assertEquals("1st: hello, 2nd: world", res.toString()); }
return new DoubleWritable(0); case BINARY: return new BytesWritable(ArrayUtils.EMPTY_BYTE_ARRAY); case STRING: return new Text(ArrayUtils.EMPTY_BYTE_ARRAY); case VARCHAR: return new HiveVarcharWritable(new HiveVarchar(StringUtils.EMPTY, -1)); case CHAR: return new HiveCharWritable(new HiveChar(StringUtils.EMPTY, -1)); case DECIMAL: return new HiveDecimalWritable(); case INTERVAL_YEAR_MONTH: return new HiveIntervalYearMonthWritable();
HiveDecimalWritable vDecimal = ((HiveDecimalObjectInspector)inputOI). getPrimitiveWritableObject(parameters[0]); int startIdx = vDecimal.toBytes(scratchBuffer); bf.addBytes(scratchBuffer, startIdx, scratchBuffer.length - startIdx); break; DateWritable vDate = ((DateObjectInspector)inputOI). getPrimitiveWritableObject(parameters[0]); bf.addLong(vDate.getDays()); break; case TIMESTAMP: case CHAR: Text vChar = ((HiveCharObjectInspector)inputOI). getPrimitiveWritableObject(parameters[0]).getStrippedValue(); bf.addBytes(vChar.getBytes(), 0, vChar.getLength()); break; case VARCHAR: Text vVarChar = ((HiveVarcharObjectInspector)inputOI). getPrimitiveWritableObject(parameters[0]).getTextValue(); bf.addBytes(vVarChar.getBytes(), 0, vVarChar.getLength()); break; case STRING:
Text t; if (value instanceof String) { t = new Text((String) value); } else { t = ((WritableStringObjectInspector) outputOI).getPrimitiveWritableObject(value); bv.setVal(i, t.getBytes(), 0, t.getLength()); } else if (outputOI instanceof WritableHiveCharObjectInspector) { WritableHiveCharObjectInspector writableHiveCharObjectOI = (WritableHiveCharObjectInspector) outputOI; hiveCharWritable = writableHiveCharObjectOI.getPrimitiveWritableObject(value); Text t = hiveCharWritable.getTextValue(); hiveVarcharWritable = writableHiveVarcharObjectOI.getPrimitiveWritableObject(value); Text t = hiveVarcharWritable.getTextValue(); StringExpr.truncate(bv, i, t.getBytes(), 0, t.getLength(), maxLength); ts = ((WritableDateObjectInspector) outputOI).getPrimitiveJavaObject(value); long l = DateWritable.dateToDays(ts); lv.vector[i] = l; } else if (outputOI instanceof WritableBooleanObjectInspector) {
@Override public String toString() { return getPaddedValue().toString(); } }
Text r = reuse == null ? new Text() : (Text) reuse; return deserializeText(buffer, invert, r); reuse == null ? new HiveCharWritable() : (HiveCharWritable) reuse; deserializeText(buffer, invert, r.getTextValue()); r.enforceMaxLength(getCharacterMaxLength(type)); return r; reuse == null ? new HiveVarcharWritable() : (HiveVarcharWritable) reuse; deserializeText(buffer, invert, r.getTextValue()); r.enforceMaxLength(getCharacterMaxLength(type)); return r; BytesWritable bw = new BytesWritable() ; bw.set(buffer.getData(), start, length); } else { bw.set(buffer.getData(), start, length); HiveDecimalWritable bdw = (reuse == null ? new HiveDecimalWritable() : (HiveDecimalWritable) reuse); bdw.set(bd); return bdw;
private Object parseAsPrimitive(JsonNode value, TypeInfo typeInfo) throws SerDeException { switch (TypeInfoFactory.getPrimitiveTypeInfo(typeInfo.getTypeName()).getPrimitiveCategory()) { case TIMESTAMP: TimestampWritable timestampWritable = new TimestampWritable(); timestampWritable.setTime(TS_PARSER.get().parseMillis(value.textValue())); return timestampWritable; return (new DoubleWritable(value.doubleValue())); case DECIMAL: return (new HiveDecimalWritable(HiveDecimal.create(value.decimalValue()))); case CHAR: return (new HiveCharWritable(new HiveChar(value.textValue(), ((CharTypeInfo) typeInfo).getLength()))); case VARCHAR: return (new HiveVarcharWritable(new HiveVarchar(value.textValue(), ((CharTypeInfo) typeInfo).getLength()))); case STRING: return (new Text(value.textValue())); case BOOLEAN: return (new BooleanWritable(value.isBoolean() ? value.booleanValue() : Boolean.valueOf(value.textValue())));
return new LongWritable(0); case TIMESTAMP: return new TimestampWritable(new Timestamp(0)); case DATE: return new DateWritable(new Date(0)); case FLOAT: return new FloatWritable(0); return new DoubleWritable(0); case BINARY: return new BytesWritable(ArrayUtils.EMPTY_BYTE_ARRAY); case STRING: return new Text(ArrayUtils.EMPTY_BYTE_ARRAY); case VARCHAR: return new HiveVarcharWritable(new HiveVarchar(StringUtils.EMPTY, -1)); case CHAR: return new HiveCharWritable(new HiveChar(StringUtils.EMPTY, -1)); case DECIMAL: return new HiveDecimalWritable(); case INTERVAL_YEAR_MONTH: return new HiveIntervalYearMonthWritable();
case TIMESTAMP: ((TimestampColumnVector) batch.cols[projectionColumnNum]).set( batchIndex, ((TimestampWritable) object).getTimestamp()); break; case DATE: ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] = ((DateWritable) object).getDays(); break; case FLOAT: batchIndex, bw.getBytes(), 0, bw.getLength()); batchIndex, tw.getBytes(), 0, tw.getLength()); hiveVarchar = (HiveVarchar) object; } else { hiveVarchar = ((HiveVarcharWritable) object).getHiveVarchar(); hiveChar = (HiveChar) object; } else { hiveChar = ((HiveCharWritable) object).getHiveChar();
BytesWritable blob = ((BinaryObjectInspector) inspector) .getPrimitiveWritableObject(obj); vector.setVal(rowId, blob.getBytes(), 0, blob.getLength()); break; Text blob = ((StringObjectInspector) inspector) .getPrimitiveWritableObject(obj); vector.setVal(rowId, blob.getBytes(), 0, blob.getLength()); break; BytesColumnVector vector = (BytesColumnVector) column; Text blob = ((HiveVarcharObjectInspector) inspector) .getPrimitiveWritableObject(obj).getTextValue(); vector.setVal(rowId, blob.getBytes(), 0, blob.getLength()); break; BytesColumnVector vector = (BytesColumnVector) column; Text blob = ((HiveCharObjectInspector) inspector) .getPrimitiveWritableObject(obj).getTextValue(); vector.setVal(rowId, blob.getBytes(), 0, blob.getLength()); break; LongColumnVector vector = (LongColumnVector) column; vector.vector[rowId] = ((DateObjectInspector) inspector) .getPrimitiveWritableObject(obj).getDays(); break;
actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes()); actualValue = new SqlDate(((DateWritable) actualValue).getDays()); actualValue = ((HiveCharWritable) actualValue).getPaddedValue().toString(); HiveDecimalWritable writable = (HiveDecimalWritable) actualValue; BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale()); actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale()); actualValue = sqlTimestampOf((timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), SESSION);
public static Writable convertDatum2Writable(Datum value) { switch(value.kind()) { case INT1: return new ByteWritable(value.asByte()); case INT2: return new ShortWritable(value.asInt2()); case INT4: return new IntWritable(value.asInt4()); case INT8: return new LongWritable(value.asInt8()); case FLOAT4: return new FloatWritable(value.asFloat4()); case FLOAT8: return new DoubleWritable(value.asFloat8()); // NOTE: value should be DateDatum case DATE: return new DateWritable(value.asInt4() - DateTimeConstants.UNIX_EPOCH_JDATE); // NOTE: value should be TimestampDatum case TIMESTAMP: TimestampWritable result = new TimestampWritable(); result.setTime(DateTimeUtil.julianTimeToJavaTime(value.asInt8())); return result; case CHAR: { String str = value.asChars(); return new HiveCharWritable(new HiveChar(str, str.length())); } case TEXT: return new Text(value.asChars()); case VARBINARY: return new BytesWritable(value.asByteArray()); case NULL_TYPE: return null; } throw new TajoRuntimeException(new NotImplementedException(TypeStringEncoder.encode(value.type()))); }
ByteBuffer b = Text.encode(String.valueOf(f)); out.write(b.array(), 0, b.limit()); break; ByteBuffer b = Text.encode(String.valueOf(d)); out.write(b.array(), 0, b.limit()); break; writeEscaped(out, t.getBytes(), 0, t.getLength(), escaped, escapeChar, needsEscape); break; Text t = hc.getPaddedValue(); writeEscaped(out, t.getBytes(), 0, t.getLength(), escaped, escapeChar, needsEscape); Text t = hc.getTextValue(); writeEscaped(out, t.getBytes(), 0, t.getLength(), escaped, escapeChar, needsEscape); byte[] toEncode = new byte[bw.getLength()]; System.arraycopy(bw.getBytes(), 0,toEncode, 0, bw.getLength()); byte[] toWrite = Base64.encodeBase64(toEncode); out.write(toWrite, 0, toWrite.length);