static void initializeDeserializer(Deserializer deserializer, Configuration conf, HCatTableInfo info, HCatSchema schema) throws SerDeException { Properties props = getSerdeProperties(info, schema); LOG.info("Initializing " + deserializer.getClass().getName() + " with properties " + props); SerDeUtils.initializeSerDe(deserializer, conf, props, null); }
public Deserializer getDeserializer(Configuration conf, boolean ignoreError) throws Exception { Deserializer de = ReflectionUtil.newInstance( getDeserializerClass().asSubclass(Deserializer.class), conf); if (ignoreError) { SerDeUtils.initializeSerDeWithoutErrorCheck(de, conf, properties, null); } else { SerDeUtils.initializeSerDe(de, conf, properties, null); } return de; }
protected static AbstractSerDe createLazyBinarySerDe(Configuration cfg, StructObjectInspector oi, Map<String, String> serdePropsMap) throws SerDeException { serdePropsMap = serdePropsMap == null ? new LinkedHashMap<String, String>() : serdePropsMap; PTFDeserializer.addOIPropertiestoSerDePropsMap(oi, serdePropsMap); AbstractSerDe serDe = new LazyBinarySerDe(); Properties p = new Properties(); p.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS, serdePropsMap.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS)); p.setProperty( org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES, serdePropsMap.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES)); SerDeUtils.initializeSerDe(serDe, cfg, p, null); return serDe; }
public Deserializer getDeserializer(Configuration conf, boolean ignoreError) throws Exception { Deserializer de = ReflectionUtil.newInstance( getDeserializerClass().asSubclass(Deserializer.class), conf); if (ignoreError) { SerDeUtils.initializeSerDeWithoutErrorCheck(de, conf, properties, null); } else { SerDeUtils.initializeSerDe(de, conf, properties, null); } return de; }
@Override public LazySimpleSerDe createSerde() throws SerializationError { try { Properties tableProps = table.getMetadata(); tableProps.setProperty(serdeConstants.LIST_COLUMNS, Joiner.on(",").join(inputColumns)); tableProps.setProperty(serdeConstants.LIST_COLUMN_TYPES, Joiner.on(":").join(inputTypes)); tableProps.setProperty(serdeConstants.FIELD_DELIM, String.valueOf(fieldDelimiter)); tableProps.setProperty(serdeConstants.COLLECTION_DELIM, String.valueOf(collectionDelimiter)); tableProps.setProperty(serdeConstants.MAPKEY_DELIM, String.valueOf(mapKeyDelimiter)); LazySimpleSerDe serde = new LazySimpleSerDe(); SerDeUtils.initializeSerDe(serde, conf, tableProps, null); this.serde = serde; return serde; } catch (SerDeException e) { throw new SerializationError("Error initializing serde", e); } } }
@Override public AbstractSerDe createSerde() throws SerializationError { try { Properties tableProps = table.getMetadata(); tableProps.setProperty(serdeConstants.LIST_COLUMNS, Joiner.on(",").join(inputColumns)); tableProps.setProperty(serdeConstants.LIST_COLUMN_TYPES, Joiner.on(":").join(inputTypes)); NiFiRecordSerDe serde = new NiFiRecordSerDe(recordReader, log); SerDeUtils.initializeSerDe(serde, conf, tableProps, null); this.serde = serde; return serde; } catch (SerDeException e) { throw new SerializationError("Error initializing serde " + NiFiRecordSerDe.class.getName(), e); } }
static void initializeOutputSerDe(AbstractSerDe serDe, Configuration conf, OutputJobInfo jobInfo) throws SerDeException { SerDeUtils.initializeSerDe(serDe, conf, getSerdeProperties(jobInfo.getTableInfo(), jobInfo.getOutputSchema()), null); }
/** * Creates JsonSerDe * @param tbl used to create serde * @param conf used to create serde * @return * @throws SerializationError if serde could not be initialized */ private static JsonSerDe createSerde(Table tbl, HiveConf conf) throws SerializationError { try { Properties tableProps = MetaStoreUtils.getTableMetadata(tbl); JsonSerDe serde = new JsonSerDe(); SerDeUtils.initializeSerDe(serde, conf, tableProps, null); return serde; } catch (SerDeException e) { throw new SerializationError("Error initializing serde " + JsonSerDe.class.getName(), e); } }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); TableDesc tbl = this.getConf().getTbl(); try { Deserializer serde = tbl.getDeserializerClass().newInstance(); SerDeUtils.initializeSerDe(serde, hconf, tbl.getProperties(), null); this.outputObjInspector = serde.getObjectInspector(); } catch (Exception e) { LOG.error("Generating output obj inspector from dummy object error", e); e.printStackTrace(); } }
@BeforeClass public static void setupClass() throws SerDeException { cfg = new Configuration(); serDe = new LazyBinarySerDe(); Properties p = new Properties(); p.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS, COL_NAMES); p.setProperty( org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES, COL_TYPES); SerDeUtils.initializeSerDe(serDe, cfg, p, null); }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); TableDesc tbl = this.getConf().getTbl(); try { Deserializer serde = tbl.getDeserializerClass().newInstance(); SerDeUtils.initializeSerDe(serde, hconf, tbl.getProperties(), null); this.outputObjInspector = serde.getObjectInspector(); } catch (Exception e) { LOG.error("Generating output obj inspector from dummy object error", e); e.printStackTrace(); } }
/** * Return a deserializer object corresponding to the partitionDesc. */ public Deserializer getDeserializer(Configuration conf) throws Exception { Properties schema = getProperties(); String clazzName = getDeserializerClassName(); Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(clazzName) .asSubclass(Deserializer.class), conf); SerDeUtils.initializeSerDe(deserializer, conf, getTableDesc().getProperties(), schema); return deserializer; }
/** * Return a deserializer object corresponding to the partitionDesc. */ public Deserializer getDeserializer(Configuration conf) throws Exception { Properties schema = getProperties(); String clazzName = getDeserializerClassName(); Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(clazzName) .asSubclass(Deserializer.class), conf); SerDeUtils.initializeSerDe(deserializer, conf, getTableDesc().getProperties(), schema); return deserializer; }
private ParquetHiveRecord getParquetWritable(String columnNames, String columnTypes, ArrayWritable record) throws SerDeException { Properties recordProperties = new Properties(); recordProperties.setProperty("columns", columnNames); recordProperties.setProperty("columns.types", columnTypes); ParquetHiveSerDe serDe = new ParquetHiveSerDe(); SerDeUtils.initializeSerDe(serDe, new Configuration(), recordProperties, null); return new ParquetHiveRecord(serDe.deserialize(record), getObjectInspector(columnNames, columnTypes)); }
public static AbstractSerDe getSpillSerDe(byte alias, TableDesc[] spillTableDesc, JoinDesc conf, boolean noFilter) { TableDesc desc = getSpillTableDesc(alias, spillTableDesc, conf, noFilter); if (desc == null) { return null; } AbstractSerDe sd = (AbstractSerDe) ReflectionUtil.newInstance(desc.getDeserializerClass(), null); try { SerDeUtils.initializeSerDe(sd, null, desc.getProperties(), null); } catch (SerDeException e) { e.printStackTrace(); return null; } return sd; }
@Explain(displayName = "keyContext", explainLevels = { Level.DEBUG }) public String getDebugKeyContext() { MapJoinObjectSerDeContext keyContext; try { AbstractSerDe keySerde = (AbstractSerDe) ReflectionUtil.newInstance( keyTblDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(keySerde, null, keyTblDesc.getProperties(), null); keyContext = new MapJoinObjectSerDeContext(keySerde, false); } catch (SerDeException e) { return null; } return keyContext.stringify(); }
public void testUpperCaseKey() throws Exception { Configuration conf = new Configuration(); Properties props = new Properties(); props.put(serdeConstants.LIST_COLUMNS, "empid,name"); props.put(serdeConstants.LIST_COLUMN_TYPES, "int,string"); JsonSerDe rjsd = new JsonSerDe(); SerDeUtils.initializeSerDe(rjsd, conf, props, null); Text text1 = new Text("{ \"empId\" : 123, \"name\" : \"John\" } "); Text text2 = new Text("{ \"empId\" : 456, \"name\" : \"Jane\" } "); HCatRecord expected1 = new DefaultHCatRecord(Arrays.<Object>asList(123, "John")); HCatRecord expected2 = new DefaultHCatRecord(Arrays.<Object>asList(456, "Jane")); assertTrue(HCatDataCheckUtil.recordsEqual((HCatRecord)rjsd.deserialize(text1), expected1)); assertTrue(HCatDataCheckUtil.recordsEqual((HCatRecord)rjsd.deserialize(text2), expected2)); }
@Test public void testDruidObjectDeserializer() throws SerDeException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException, IOException, InterruptedException, NoSuchMethodException, InvocationTargetException { // Create, initialize, and test the SerDe DruidSerDe serDe = new DruidSerDe(); Configuration conf = new Configuration(); Properties tbl; // Mixed source (all types) tbl = createPropertiesSource(COLUMN_NAMES, COLUMN_TYPES); SerDeUtils.initializeSerDe(serDe, conf, tbl, null); deserializeObject(serDe, ROW_OBJECT_2, DRUID_WRITABLE_2); }
@Test public void testDruidObjectSerializer() throws SerDeException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException, IOException, InterruptedException, NoSuchMethodException, InvocationTargetException { // Create, initialize, and test the SerDe DruidSerDe serDe = new DruidSerDe(); Configuration conf = new Configuration(); Properties tbl; // Mixed source (all types) tbl = createPropertiesSource(COLUMN_NAMES, COLUMN_TYPES); SerDeUtils.initializeSerDe(serDe, conf, tbl, null); serializeObject(tbl, serDe, ROW_OBJECT, DRUID_WRITABLE); }
@Test public void testEmptyFile() throws Exception { Properties properties = new Properties(); properties.setProperty("columns", "x,y"); properties.setProperty("columns.types", "int:int"); HiveOutputFormat<?, ?> outFormat = new OrcOutputFormat(); org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter writer = outFormat.getHiveRecordWriter(conf, testFilePath, MyRow.class, true, properties, Reporter.NULL); writer.close(true); AbstractSerDe serde = new OrcSerde(); SerDeUtils.initializeSerDe(serde, conf, properties, null); InputFormat<?,?> in = new OrcInputFormat(); FileInputFormat.setInputPaths(conf, testFilePath.toString()); InputSplit[] splits = in.getSplits(conf, 1); assertTrue(0 == splits.length); assertEquals(null, serde.getSerDeStats()); }