@Override public Tuple getNext() throws IOException { try { HCatRecord hr = (HCatRecord) (reader.nextKeyValue() ? reader.getCurrentValue() : null); Tuple t = PigHCatUtil.transformToTuple(hr, outputSchema); // TODO : we were discussing an iter interface, and also a LazyTuple // change this when plans for that solidifies. return t; } catch (ExecException e) { int errCode = 6018; String errMsg = "Error while reading input"; throw new ExecException(errMsg, errCode, PigException.REMOTE_ENVIRONMENT, e); } catch (Exception eOther) { int errCode = 6018; String errMsg = "Error converting read value to tuple"; throw new ExecException(errMsg, errCode, PigException.REMOTE_ENVIRONMENT, eOther); } }
return extractString(bytes, startIndex, endIndex, true); } else if (fs.type == DataType.BYTEARRAY) { return new DataByteArray(bytes, startIndex, endIndex+1); return ToDate.extractDateTime(val); } else if (fs.type == DataType.BIGINTEGER) { return new BigInteger(val); } else if (fs.type == DataType.BIGDECIMAL) { return new BigDecimal(val); } else { throw new ExecException("Can't deserialize type: " + DataType.findTypeName(fs.type));
return BigInteger.valueOf(((Integer)o).longValue()); return BigInteger.valueOf(((Long)o).longValue()); return BigInteger.valueOf(((Float)o).longValue()); return new BigInteger(((DataByteArray)o).toString()); return ((BigDecimal)o).toBigInteger(); default: int errCode = 1071; String msg = "Cannot convert a " + findTypeName(o) + " to a BigInteger."; throw new ExecException(msg, errCode, PigException.INPUT); int errCode = 1074; String msg = "Problem with formatting. Could not convert " + o + " to BigInteger."; throw new ExecException(msg, errCode, PigException.INPUT, nfe); } catch (Exception e) { int errCode = 2054; String msg = "Internal error. Could not convert " + o + " to BigInteger."; throw new ExecException(msg, errCode, PigException.BUG);
return BigDecimal.valueOf(((Integer)o).longValue()); return BigDecimal.valueOf(((Long)o).longValue()); return BigDecimal.valueOf(((Float)o).doubleValue()); return new BigDecimal(((DataByteArray)o).toString()); default: int errCode = 1071; String msg = "Cannot convert a " + findTypeName(o) + " to a BigDecimal."; throw new ExecException(msg, errCode, PigException.INPUT); int errCode = 1074; String msg = "Problem with formatting. Could not convert " + o + " to BigDecimal."; throw new ExecException(msg, errCode, PigException.INPUT, nfe); } catch (Exception e) { int errCode = 2054; String msg = "Internal error. Could not convert " + o + " to BigDecimal."; throw new ExecException(msg, errCode, PigException.BUG);
return ((DataByteArray)o).toString(); return ((BigInteger)o).toString(); return ((BigDecimal)o).toString(); default: int errCode = 1071; String msg = "Cannot convert a " + findTypeName(o) + " to a String"; throw new ExecException(msg, errCode, PigException.INPUT); int errCode = 2054; String msg = "Internal error. Could not convert " + o + " to String."; throw new ExecException(msg, errCode, PigException.BUG);
protected Number add(Number a, Number b, byte dataType) throws ExecException { switch(dataType) { case DataType.DOUBLE: return Double.valueOf((Double) a + (Double) b); case DataType.INTEGER: return Integer.valueOf((Integer) a + (Integer) b); case DataType.LONG: return Long.valueOf((Long) a + (Long) b); case DataType.FLOAT: return Float.valueOf((Float) a + (Float) b); case DataType.BIGINTEGER: return ((BigInteger) a).add((BigInteger) b); case DataType.BIGDECIMAL: return ((BigDecimal) a).add((BigDecimal) b); default: throw new ExecException("called on unsupported Number class " + DataType.findTypeName(dataType)); } }
return null; case BYTEARRAY: return new DateTime(((DataByteArray) o).toString()); case CHARARRAY: return new DateTime(((Double) o).longValue()); case BIGINTEGER: return new DateTime(((BigInteger) o).longValue()); case BIGDECIMAL: return new DateTime(((BigDecimal) o).longValue()); case DATETIME: return (DateTime) o; default: int errCode = 1071; String msg = "Cannot convert a " + findTypeName(o) + " to a DateTime"; throw new ExecException(msg, errCode, PigException.INPUT); int errCode = 1074; String msg = "Problem with formatting. Could not convert " + o + " to DateTime."; throw new ExecException(msg, errCode, PigException.INPUT, nfe); } catch (Exception e) { int errCode = 2054; String msg = "Internal error. Could not convert " + o + " to DateTime."; throw new ExecException(msg, errCode, PigException.BUG);
protected Number multiply(Number a, Number b, byte dataType) throws ExecException { switch(dataType) { case DataType.DOUBLE: return Double.valueOf((Double) a * (Double) b); case DataType.INTEGER: return Integer.valueOf((Integer) a * (Integer) b); case DataType.LONG: return Long.valueOf((Long) a * (Long) b); case DataType.FLOAT: return Float.valueOf((Float) a * (Float) b); case DataType.BIGINTEGER: return ((BigInteger) a).multiply((BigInteger) b); case DataType.BIGDECIMAL: return ((BigDecimal) a).multiply((BigDecimal) b); default: throw new ExecException("called on unsupported Number class " + DataType.findTypeName(dataType)); } }
return Integer.valueOf(((DataByteArray)o).toString()); return Integer.valueOf(((BigInteger)o).intValue()); return Integer.valueOf(((BigDecimal)o).intValue()); default: int errCode = 1071; String msg = "Cannot convert a " + findTypeName(o) + " to an Integer"; throw new ExecException(msg, errCode, PigException.INPUT); int errCode = 1074; String msg = "Problem with formatting. Could not convert " + o + " to Integer."; throw new ExecException(msg, errCode, PigException.INPUT, nfe); } catch (Exception e) { int errCode = 2054; String msg = "Internal error. Could not convert " + o + " to Integer."; throw new ExecException(msg, errCode, PigException.BUG);
protected Number subtract(Number a, Number b, byte dataType) throws ExecException { switch(dataType) { case DataType.DOUBLE: return Double.valueOf((Double) a - (Double) b); case DataType.INTEGER: return Integer.valueOf((Integer) a - (Integer) b); case DataType.LONG: return Long.valueOf((Long) a - (Long) b); case DataType.FLOAT: return Float.valueOf((Float) a - (Float) b); case DataType.BIGINTEGER: return ((BigInteger) a).subtract((BigInteger) b); case DataType.BIGDECIMAL: return ((BigDecimal) a).subtract((BigDecimal) b); default: throw new ExecException("called on unsupported Number class " + DataType.findTypeName(dataType)); } }
protected boolean equalsZero(Number a, byte dataType) throws ExecException { switch (dataType) { case DataType.DOUBLE: return ((Double) a).equals(0.0); case DataType.INTEGER: return ((Integer) a).equals(0); case DataType.LONG: return ((Long) a).equals(0L); case DataType.FLOAT: return ((Float) a).equals(0.0f); case DataType.BIGINTEGER: return BigInteger.ZERO.equals((BigInteger) a); case DataType.BIGDECIMAL: return BigDecimal.ZERO.equals((BigDecimal) a); default: throw new ExecException("Called on unsupported Number class " + DataType.findTypeName(dataType)); } }
return Double.valueOf(((DataByteArray)o).toString()); return Double.valueOf(((BigInteger)o).doubleValue()); return Double.valueOf(((BigDecimal)o).doubleValue()); default: int errCode = 1071; String msg = "Cannot convert a " + findTypeName(o) + " to a Double"; throw new ExecException(msg, errCode, PigException.INPUT); int errCode = 1074; String msg = "Problem with formatting. Could not convert " + o + " to Double."; throw new ExecException(msg, errCode, PigException.INPUT, nfe); } catch (Exception e) { int errCode = 2054; String msg = "Internal error. Could not convert " + o + " to Double."; throw new ExecException(msg, errCode, PigException.BUG);
realType = DataType.findType(res.result); res.result = DataType.toDateTime(res.result, realType); } catch (ClassCastException cce) { realType = DataType.findType(res.result); res.result = DataType.toDateTime(res.result, realType); res.result = caster.bytesToDateTime(dba.get()); } else { int errCode = 1075; String msg = unknownByteArrayErrorMessage + "datetime for " + this.getOriginalLocations(); throw new ExecException(msg, errCode, PigException.INPUT); Result res = in.getNextBigInteger(); if (res.returnStatus == POStatus.STATUS_OK && res.result != null) { res.result = new DateTime(((BigInteger) res.result).longValue()); Result res = in.getNextBigDecimal(); if (res.returnStatus == POStatus.STATUS_OK && res.result != null) { res.result = new DateTime(((BigDecimal) res.result).longValue());
return Float.valueOf(((DataByteArray)o).toString()); return Float.valueOf(((BigInteger)o).floatValue()); return Float.valueOf(((BigDecimal)o).floatValue()); default: int errCode = 1071; String msg = "Cannot convert a " + findTypeName(o) + " to a Float"; throw new ExecException(msg, errCode, PigException.INPUT); int errCode = 1074; String msg = "Problem with formatting. Could not convert " + o + " to Float."; throw new ExecException(msg, errCode, PigException.INPUT, nfe); } catch (Exception e) { int errCode = 2054; String msg = "Internal error. Could not convert " + o + " to Float."; throw new ExecException(msg, errCode, PigException.BUG);
protected Number divide(Number a, Number b, byte dataType) throws ExecException { switch (dataType) { case DataType.DOUBLE: return Double.valueOf((Double) a / (Double) b); case DataType.INTEGER: return Integer.valueOf((Integer) a / (Integer) b); case DataType.LONG: return Long.valueOf((Long) a / (Long) b); case DataType.FLOAT: return Float.valueOf((Float) a / (Float) b); case DataType.BIGINTEGER: return ((BigInteger) a).divide((BigInteger) b); case DataType.BIGDECIMAL: return bigDecimalDivideWithScale(a, b); default: throw new ExecException("called on unsupported Number class " + DataType.findTypeName(dataType)); } }
realType = DataType.findType(res.result); try { res.result = DataType.toFloat(res.result, realType); } catch (ClassCastException cce) { realType = DataType.findType(res.result); res.result = DataType.toFloat(res.result, realType); res.result = caster.bytesToFloat(dba.get()); } else { int errCode = 1075; String msg = unknownByteArrayErrorMessage + "float for " + this.getOriginalLocations(); throw new ExecException(msg, errCode, PigException.INPUT); Result res = in.getNextBigInteger(); if (res.returnStatus == POStatus.STATUS_OK && res.result != null) { res.result = Float.valueOf(((BigInteger)res.result).floatValue()); Result res = in.getNextBigDecimal(); if (res.returnStatus == POStatus.STATUS_OK && res.result != null) { res.result = Float.valueOf(((BigDecimal)res.result).floatValue());
@Override public Tuple exec(Tuple input) throws IOException { try { // input is a bag with one tuple containing // the column we are trying to max on DataBag bg = (DataBag) input.get(0); String s = null; if(bg.iterator().hasNext()) { Tuple tp = bg.iterator().next(); s = (String)(tp.get(0)); } return tfact.newTuple(s); } catch (ExecException ee) { throw ee; } catch (Exception e) { int errCode = 2106; String msg = "Error while computing max in " + this.getClass().getSimpleName(); throw new ExecException(msg, errCode, PigException.BUG, e); } } }
protected Number mod(Number a, Number b, byte dataType) throws ExecException { switch(dataType) { case DataType.INTEGER: return Integer.valueOf((Integer) a % (Integer) b); case DataType.LONG: return Long.valueOf((Long) a % (Long) b); case DataType.BIGINTEGER: return ((BigInteger)a).mod((BigInteger)b); default: throw new ExecException("called on unsupported Number class " + DataType.findTypeName(dataType)); } }
@Override public void putNext(Tuple t) throws IOException { if(t.size() != 2) { throw new ExecException("Output tuple has wrong size: is " + t.size() + ", should be 2"); } byte[] keyBytes = ((DataByteArray) t.get(0)).get(); byte[] valueBytes = ((DataByteArray) t.get(1)).get(); if (keyBytes == null || valueBytes == null) { throw new ExecException("Output tuple contains null"); } ArrayList<byte[]> alk = new ArrayList<byte[]>(); alk.add(keyBytes); NullableTuple key = new NullableTuple(TupleFactory.getInstance().newTuple(alk)); ArrayList<byte[]> alv = new ArrayList<byte[]>(); alv.add(valueBytes); NullableTuple val = new NullableTuple(TupleFactory.getInstance().newTuple(alv)); try { writer.write(key, val); } catch (InterruptedException e) { throw new IOException(e); } } }
final TupleFactory tf = TupleFactory.getInstance(); Map<String, Object> distMap = (Map<String, Object>) t.get(0); DataBag partitionList = (DataBag) distMap.get(PartitionSkewedKeys.PARTITION_LIST); Iterator<Tuple> it = partitionList.iterator(); while (it.hasNext()) { Tuple idxTuple = it.next(); Integer maxIndex = (Integer) idxTuple.get(idxTuple.size() - 1); Integer minIndex = (Integer) idxTuple.get(idxTuple.size() - 2); Tuple keyTuple = tf.newTuple(); for (int i = 0; i < idxTuple.size() - 2; i++) { keyTuple.append(idxTuple.get(i)); log.warn(e.getMessage());