@Override public Object addToArray(Object array, List<Object> value) { return TupleFactory.getInstance().newTupleNoCopy(value); }
@Override protected Object binaryValue(byte[] value) { return new DataByteArray(value); }
private static Tuple transformToTuple(List<?> objList, HCatSchema hs) throws Exception { if (objList == null) { return null; } Tuple t = tupFac.newTuple(objList.size()); List<HCatFieldSchema> subFields = hs.getFields(); for (int i = 0; i < subFields.size(); i++) { t.set(i, extractPigObject(objList.get(i), subFields.get(i))); } return t; }
private PigTuple createTuple(Object obj, ResourceSchema schema) { PigTuple tuple = new PigTuple(schema); tuple.setTuple(TupleFactory.getInstance().newTuple(obj)); return tuple; }
@Test public void testBagWithAnonTuple() { String expected = "{\"bag_0\":[[{\"val_0\":\"xxx\",\"val_1\":\"yyy\"}]," + "[{\"val_0\":\"xxx\",\"val_1\":\"yyy\"}]," + "[{\"val_0\":\"xxx\",\"val_1\":\"yyy\"}]]}"; Tuple tuple = TupleFactory.getInstance().newTuple(Arrays.asList(new String[] { "xxx", "yyy" })); assertThat(pigTypeToJson(createTuple(new DefaultDataBag(Arrays.asList(new Tuple[] { tuple, tuple, tuple })), createSchema("{t:(chararray, chararray)}"))), is(expected)); }
private static DataBag transformToBag(List<?> list, HCatFieldSchema hfs) throws Exception { if (list == null) { return null; } HCatFieldSchema elementSubFieldSchema = hfs.getArrayElementSchema().getFields().get(0); DataBag db = new DefaultDataBag(); for (Object o : list) { Tuple tuple; if (elementSubFieldSchema.getType() == Type.STRUCT) { tuple = transformToTuple((List<?>) o, elementSubFieldSchema); } else { // bags always contain tuples tuple = tupFac.newTuple(extractPigObject(o, elementSubFieldSchema)); } db.add(tuple); } return db; }
/** * Checks to see if the given field is a schema-less Map that has values. * @return true if Map has no schema but has values (mixed schema map). false if not a Map or if Map is just empty. */ private boolean isPopulatedMixedValueMap(ResourceFieldSchema schema, int field, Tuple object) { if (schema.getType() != DataType.MAP) { // Can't be a mixed value map if it's not a map at all. return false; } try { Object fieldValue = object.get(field); Map<?, ?> map = (Map<?, ?>) fieldValue; return schema.getSchema() == null && !(map == null || map.isEmpty()); } catch (ExecException e) { throw new EsHadoopIllegalStateException(e); } }
private static void throwTypeMismatchException(byte pigDataType, List<Type> hcatRequiredType, HCatFieldSchema hcatActualField, int columnPos) throws FrontendException { if(!hcatRequiredType.contains(hcatActualField.getType())) { throw new FrontendException( "Pig '" + DataType.findTypeName(pigDataType) + "' type in column " + columnPos + "(0-based) cannot map to HCat '" + hcatActualField.getType() + "'type. Target filed must be of HCat type {" + StringUtils.join(hcatRequiredType, " or ") + "}"); } }
private PigTuple createTuple(Object obj, ResourceSchema schema) { PigTuple tuple = new PigTuple(schema); tuple.setTuple(TupleFactory.getInstance().newTuple(obj)); return tuple; }
@Test public void testNamedBag() { String expected = "{\"bag\":[[{\"first\":\"one\",\"second\":\"two\",\"third\":\"three\"}]," + "[{\"first\":\"one\",\"second\":\"two\",\"third\":\"three\"}]," + "[{\"first\":\"one\",\"second\":\"two\",\"third\":\"three\"}]]}"; Tuple tuple = TupleFactory.getInstance().newTuple(Arrays.asList(new String[] { "one", "two", "three" })); assertThat(pigTypeToJson(createTuple(new DefaultDataBag(Arrays.asList(new Tuple[] { tuple, tuple, tuple })), createSchema("bag: {t:(first:chararray, second:chararray, third: chararray)}"))), is(expected)); }
@Test public void testByteArray() { String expected = "{\"name\":\"Ynl0ZSBhcnJheQ==\"}"; assertThat(pigTypeToJson(createTuple(new DataByteArray("byte array".getBytes()), createSchema("name:bytearray"))), is(expected)); }
private PigTuple createTuple(Object obj, ResourceSchema schema) { PigTuple tuple = new PigTuple(schema); tuple.setTuple(TupleFactory.getInstance().newTuple(obj)); return tuple; } }
@Test public void testBagWithAnonTuple() { String expected = "{\"bag\":[[\"xxx\",\"yyy\"],[\"xxx\",\"yyy\"],[\"xxx\",\"yyy\"]]}"; Tuple tuple = TupleFactory.getInstance().newTuple(Arrays.asList(new String[] { "xxx", "yyy" })); assertThat((pigTypeToJson(createTuple(new DefaultDataBag(Arrays.asList(new Tuple[] { tuple, tuple, tuple })), createSchema("bag: {t:(chararray, chararray)}")))), is(expected)); }
@Test public void testNamedByteArray() { String expected = "{\"name\":\"Ynl0ZSBhcnJheQ==\"}"; assertThat(pigTypeToJson(createTuple(new DataByteArray("byte array".getBytes()), createSchema("name:bytearray"))), is(expected)); }
private PigTuple createTuple(Object obj, ResourceSchema schema) { PigTuple tuple = new PigTuple(schema); tuple.setTuple(TupleFactory.getInstance().newTuple(obj)); return tuple; } }
@Test public void testNamedBag() { String expected = "{\"bag\":[[\"one\",\"two\",\"three\"],[\"one\",\"two\",\"three\"],[\"one\",\"two\",\"three\"]]}"; Tuple tuple = TupleFactory.getInstance().newTuple(Arrays.asList(new String[] { "one", "two", "three" })); assertThat(pigTypeToJson(createTuple(new DefaultDataBag(Arrays.asList(new Tuple[] { tuple, tuple, tuple })), createSchema("bag: {t:(first:chararray, second:chararray, third: chararray)}"))), is(expected)); }
@Test public void testAnonByteArray() { String expected = "{\"val_0\":\"Ynl0ZSBhcnJheQ==\"}"; assertThat(pigTypeToJson(createTuple(new DataByteArray("byte array".getBytes()), createSchema("bytearray"))), is(expected)); }
@Test public void testNamedTupleWithMixedValues() { String expected = "{\"namedtuplewithmixedvalues\":[1,\"two\"]}"; assertThat(pigTypeToJson(createTuple(TupleFactory.getInstance().newTuple(Arrays.asList(new Object[] { 1, "two" })), createSchema("namedtuplewithmixedvalues: (first:int, second:chararray)"))), is(expected)); }
@Test public void testAnonymousTuple() { String expected = "{\"anontuple\":[\"xxx\",\"yyy\",\"zzz\"]}"; assertThat(pigTypeToJson(createTuple( TupleFactory.getInstance().newTuple(Arrays.asList(new String[] { "xxx", "yyy", "zzz" })), createSchema("anontuple: (chararray, chararray, chararray)"))), is(expected)); }
@Test public void testNamedTupleWithMixedValues() { String expected = "{\"namedtuplewithmixedvalues\":[{\"first\":1,\"second\":\"two\"}]}"; assertThat(pigTypeToJson(createTuple(TupleFactory.getInstance().newTuple(Arrays.asList(new Object[] { 1, "two" })), createSchema("namedtuplewithmixedvalues: (first:int, second:chararray)"))), is(expected)); }