public Deserializer getDeserializer(Configuration conf, boolean ignoreError) throws Exception { Deserializer de = ReflectionUtil.newInstance( getDeserializerClass().asSubclass(Deserializer.class), conf); if (ignoreError) { SerDeUtils.initializeSerDeWithoutErrorCheck(de, conf, properties, null); } else { SerDeUtils.initializeSerDe(de, conf, properties, null); } return de; }
@Override @SuppressWarnings("deprecation") public void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); serializer = (Serializer) ReflectionUtils.newInstance( conf.getTable().getDeserializerClass(), null); buffer = new DataOutputBuffer(); }
@Override @SuppressWarnings("deprecation") public void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); serializer = (Serializer) ReflectionUtils.newInstance( conf.getTable().getDeserializerClass(), null); buffer = new DataOutputBuffer(); }
public Deserializer getDeserializer(Configuration conf, boolean ignoreError) throws Exception { Deserializer de = ReflectionUtil.newInstance( getDeserializerClass().asSubclass(Deserializer.class), conf); if (ignoreError) { SerDeUtils.initializeSerDeWithoutErrorCheck(de, conf, properties, null); } else { SerDeUtils.initializeSerDe(de, conf, properties, null); } return de; }
@Override public void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); MAX_SIZE = HiveConf.getLongVar(hconf, ConfVars.TEZ_DYNAMIC_PARTITION_PRUNING_MAX_EVENT_SIZE); serializer = (Serializer) ReflectionUtils.newInstance(conf.getTable().getDeserializerClass(), null); initDataBuffer(false); }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); TableDesc tbl = this.getConf().getTbl(); try { Deserializer serde = tbl.getDeserializerClass().newInstance(); SerDeUtils.initializeSerDe(serde, hconf, tbl.getProperties(), null); this.outputObjInspector = serde.getObjectInspector(); } catch (Exception e) { LOG.error("Generating output obj inspector from dummy object error", e); e.printStackTrace(); } }
private boolean needConversion(TableDesc tableDesc, List<PartitionDesc> partDescs) { Class<?> tableSerDe = tableDesc.getDeserializerClass(); SerDeSpec spec = AnnotationUtils.getAnnotation(tableSerDe, SerDeSpec.class); if (null == spec) { // Serde may not have this optional annotation defined in which case be conservative // and say conversion is needed. return true; } String[] schemaProps = spec.schemaProps(); Properties tableProps = tableDesc.getProperties(); for (PartitionDesc partitionDesc : partDescs) { if (!tableSerDe.getName().equals(partitionDesc.getDeserializerClassName())) { return true; } Properties partProps = partitionDesc.getProperties(); for (String schemaProp : schemaProps) { if (!org.apache.commons.lang3.StringUtils.equals( tableProps.getProperty(schemaProp), partProps.getProperty(schemaProp))) { return true; } } } return false; }
@Override public void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); MAX_SIZE = HiveConf.getLongVar(hconf, ConfVars.TEZ_DYNAMIC_PARTITION_PRUNING_MAX_EVENT_SIZE); serializer = (Serializer) ReflectionUtils.newInstance(conf.getTable().getDeserializerClass(), null); initDataBuffer(false); }
public SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, String columnType, JobConf jobConf) throws SerDeException { this.skipPruning.set(false); this.partKey = partKey; this.columnName = columnName; this.columnType = columnType; deserializer = ReflectionUtils.newInstance(table.getDeserializerClass(), null); deserializer.initialize(jobConf, table.getProperties()); ObjectInspector inspector = deserializer.getObjectInspector(); LOG.debug("Type of obj insp: " + inspector.getTypeName()); soi = (StructObjectInspector) inspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); if (fields.size() > 1) { LOG.error("expecting single field in input"); } field = fields.get(0); fieldInspector = ObjectInspectorUtils.getStandardObjectInspector(field.getFieldObjectInspector()); } }
SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, String columnType, JobConf jobConf) throws SerDeException { this.partKey = partKey; this.columnName = columnName; this.columnType = columnType; deserializer = ReflectionUtils.newInstance(table.getDeserializerClass(), null); deserializer.initialize(jobConf, table.getProperties()); ObjectInspector inspector = deserializer.getObjectInspector(); if (LOG.isDebugEnabled()) { LOG.debug("Type of obj insp: " + inspector.getTypeName()); } soi = (StructObjectInspector) inspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); assert(fields.size() > 1) : "expecting single field in input"; field = fields.get(0); fieldInspector = ObjectInspectorUtils.getStandardObjectInspector(field.getFieldObjectInspector()); } }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); TableDesc tbl = this.getConf().getTbl(); try { Deserializer serde = tbl.getDeserializerClass().newInstance(); SerDeUtils.initializeSerDe(serde, hconf, tbl.getProperties(), null); this.outputObjInspector = serde.getObjectInspector(); } catch (Exception e) { LOG.error("Generating output obj inspector from dummy object error", e); e.printStackTrace(); } }
public SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, String columnType, JobConf jobConf) throws SerDeException { this.skipPruning.set(false); this.partKey = partKey; this.columnName = columnName; this.columnType = columnType; deserializer = ReflectionUtils.newInstance(table.getDeserializerClass(), null); deserializer.initialize(jobConf, table.getProperties()); ObjectInspector inspector = deserializer.getObjectInspector(); LOG.debug("Type of obj insp: " + inspector.getTypeName()); soi = (StructObjectInspector) inspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); if (fields.size() > 1) { LOG.error("expecting single field in input"); } field = fields.get(0); fieldInspector = ObjectInspectorUtils.getStandardObjectInspector(field.getFieldObjectInspector()); } }
SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, JobConf jobConf) throws SerDeException { this.partKey = partKey; this.columnName = columnName; deserializer = ReflectionUtils.newInstance(table.getDeserializerClass(), null); deserializer.initialize(jobConf, table.getProperties()); ObjectInspector inspector = deserializer.getObjectInspector(); if (LOG.isDebugEnabled()) { LOG.debug("Type of obj insp: " + inspector.getTypeName()); } soi = (StructObjectInspector) inspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); assert(fields.size() > 1) : "expecting single field in input"; field = fields.get(0); fieldInspector = ObjectInspectorUtils.getStandardObjectInspector(field.getFieldObjectInspector()); } }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); firstRow = true; statsMap.put(Counter.DESERIALIZE_ERRORS.toString(), deserialize_error_count); statsMap.put(Counter.SERIALIZE_ERRORS.toString(), serialize_error_count); try { this.hconf = hconf; scriptOutputDeserializer = conf.getScriptOutputInfo() .getDeserializerClass().newInstance(); SerDeUtils.initializeSerDe(scriptOutputDeserializer, hconf, conf.getScriptOutputInfo().getProperties(), null); scriptInputSerializer = (Serializer) conf.getScriptInputInfo() .getDeserializerClass().newInstance(); scriptInputSerializer.initialize(hconf, conf.getScriptInputInfo() .getProperties()); outputObjInspector = scriptOutputDeserializer.getObjectInspector(); } catch (Exception e) { throw new HiveException(ErrorMsg.SCRIPT_INIT_ERROR.getErrorCodedMsg(), e); } }
public static AbstractSerDe getSpillSerDe(byte alias, TableDesc[] spillTableDesc, JoinDesc conf, boolean noFilter) { TableDesc desc = getSpillTableDesc(alias, spillTableDesc, conf, noFilter); if (desc == null) { return null; } AbstractSerDe sd = (AbstractSerDe) ReflectionUtil.newInstance(desc.getDeserializerClass(), null); try { SerDeUtils.initializeSerDe(sd, null, desc.getProperties(), null); } catch (SerDeException e) { e.printStackTrace(); return null; } return sd; }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); firstRow = true; statsMap.put(Counter.DESERIALIZE_ERRORS.toString(), deserialize_error_count); statsMap.put(Counter.SERIALIZE_ERRORS.toString(), serialize_error_count); try { this.hconf = hconf; scriptOutputDeserializer = conf.getScriptOutputInfo() .getDeserializerClass().newInstance(); SerDeUtils.initializeSerDe(scriptOutputDeserializer, hconf, conf.getScriptOutputInfo().getProperties(), null); scriptInputSerializer = (Serializer) conf.getScriptInputInfo() .getDeserializerClass().newInstance(); scriptInputSerializer.initialize(hconf, conf.getScriptInputInfo() .getProperties()); outputObjInspector = scriptOutputDeserializer.getObjectInspector(); } catch (Exception e) { throw new HiveException(ErrorMsg.SCRIPT_INIT_ERROR.getErrorCodedMsg(), e); } }
@Explain(displayName = "keyContext", explainLevels = { Level.DEBUG }) public String getDebugKeyContext() { MapJoinObjectSerDeContext keyContext; try { AbstractSerDe keySerde = (AbstractSerDe) ReflectionUtil.newInstance( keyTblDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(keySerde, null, keyTblDesc.getProperties(), null); keyContext = new MapJoinObjectSerDeContext(keySerde, false); } catch (SerDeException e) { return null; } return keyContext.stringify(); }
public static AbstractSerDe getSpillSerDe(byte alias, TableDesc[] spillTableDesc, JoinDesc conf, boolean noFilter) { TableDesc desc = getSpillTableDesc(alias, spillTableDesc, conf, noFilter); if (desc == null) { return null; } AbstractSerDe sd = (AbstractSerDe) ReflectionUtil.newInstance(desc.getDeserializerClass(), null); try { SerDeUtils.initializeSerDe(sd, null, desc.getProperties(), null); } catch (SerDeException e) { e.printStackTrace(); return null; } return sd; }
public void generateMapMetaData() throws HiveException { // generate the meta data for key // index for key is -1 try { TableDesc keyTableDesc = conf.getKeyTblDesc(); AbstractSerDe keySerializer = (AbstractSerDe) ReflectionUtil.newInstance( keyTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(keySerializer, null, keyTableDesc.getProperties(), null); MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerializer, false); for (int pos = 0; pos < order.length; pos++) { if (pos == posBigTable) { continue; } TableDesc valueTableDesc; if (conf.getNoOuterJoin()) { valueTableDesc = conf.getValueTblDescs().get(pos); } else { valueTableDesc = conf.getValueFilteredTblDescs().get(pos); } AbstractSerDe valueSerDe = (AbstractSerDe) ReflectionUtil.newInstance( valueTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null); MapJoinObjectSerDeContext valueContext = new MapJoinObjectSerDeContext(valueSerDe, hasFilter(pos)); mapJoinTableSerdes[pos] = new MapJoinTableContainerSerDe(keyContext, valueContext); } } catch (SerDeException e) { throw new HiveException(e); } }
public static MapJoinTableContainerSerDe createMapJoinTableContainerSerDe(MapJoinDesc mapJoinDesc) throws SerDeException { final Byte smallTablePos = 1; TableDesc keyTableDesc = mapJoinDesc.getKeyTblDesc(); AbstractSerDe keySerializer = (AbstractSerDe) ReflectionUtil.newInstance( BinarySortableSerDe.class, null); SerDeUtils.initializeSerDe(keySerializer, null, keyTableDesc.getProperties(), null); MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerializer, false); final List<TableDesc> valueTableDescList; if (mapJoinDesc.getNoOuterJoin()) { valueTableDescList = mapJoinDesc.getValueTblDescs(); } else { valueTableDescList = mapJoinDesc.getValueFilteredTblDescs(); } TableDesc valueTableDesc = valueTableDescList.get(smallTablePos); AbstractSerDe valueSerDe = (AbstractSerDe) ReflectionUtil.newInstance( valueTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null); MapJoinObjectSerDeContext valueContext = new MapJoinObjectSerDeContext(valueSerDe, hasFilter(mapJoinDesc, smallTablePos)); MapJoinTableContainerSerDe mapJoinTableContainerSerDe = new MapJoinTableContainerSerDe(keyContext, valueContext); return mapJoinTableContainerSerDe; }