public static void configureJobConf(TableDesc tableDesc, JobConf jobConf) throws Exception { HBaseSerDeParameters serdeParams = new HBaseSerDeParameters(jobConf, tableDesc.getProperties(), HBaseSerDe.class.getName()); serdeParams.getKeyFactory().configureJobConf(tableDesc, jobConf); } }
private HBaseKeyFactory initKeyFactory(Configuration conf, Properties tbl) throws SerDeException { try { HBaseKeyFactory keyFactory = createKeyFactory(conf, tbl); if (keyFactory != null) { keyFactory.init(this, tbl); } return keyFactory; } catch (Exception e) { throw new SerDeException(e); } }
private List<HBaseValueFactory> initValueFactories(Configuration conf, Properties tbl) throws SerDeException { List<HBaseValueFactory> valueFactories = createValueFactories(conf, tbl); for (HBaseValueFactory valueFactory : valueFactories) { valueFactory.init(this, conf, tbl); } return valueFactories; }
@Override public void init(HBaseSerDeParameters hbaseParams, Configuration conf, Properties properties) throws SerDeException { this.hbaseParams = hbaseParams; this.serdeParams = hbaseParams.getSerdeParams(); this.columnMappings = hbaseParams.getColumnMappings(); this.properties = properties; this.conf = conf; }
public String toString() { return "[" + columnMappingString + ":" + getColumnNames() + ":" + getColumnTypes() + "]"; }
/** * Construct a LazyHBaseRow object with the ObjectInspector. */ public LazyHBaseRow(LazySimpleStructObjectInspector oi, HBaseSerDeParameters serdeParams) { super(oi); this.keyFactory = serdeParams.getKeyFactory(); this.valueFactories = serdeParams.getValueFactories(); this.columnsMapping = serdeParams.getColumnMappings().getColumnsMapping(); }
public HBaseRowSerializer(HBaseSerDeParameters hbaseParam) { this.hbaseParam = hbaseParam; this.keyFactory = hbaseParam.getKeyFactory(); this.serdeParam = hbaseParam.getSerdeParams(); this.separators = serdeParam.getSeparators(); this.escaped = serdeParam.isEscaped(); this.escapeChar = serdeParam.getEscapeChar(); this.needsEscape = serdeParam.getNeedsEscape(); this.keyIndex = hbaseParam.getKeyIndex(); this.timestampIndex = hbaseParam.getTimestampIndex(); this.columnMappings = hbaseParam.getColumnMappings().getColumnsMapping(); this.keyMapping = hbaseParam.getColumnMappings().getKeyMapping(); this.timestampMapping = hbaseParam.getColumnMappings().getTimestampMapping(); this.putTimestamp = hbaseParam.getPutTimestamp(); }
public static ObjectInspector createLazyHBaseStructInspector(HBaseSerDeParameters hSerdeParams, Properties tbl) throws SerDeException { List<TypeInfo> columnTypes = hSerdeParams.getColumnTypes(); ArrayList<ObjectInspector> columnObjectInspectors = new ArrayList<ObjectInspector>( columnTypes.size()); for (int i = 0; i < columnTypes.size(); i++) { if (i == hSerdeParams.getKeyIndex()) { columnObjectInspectors.add(hSerdeParams.getKeyFactory() .createKeyObjectInspector(columnTypes.get(i))); } else { columnObjectInspectors.add(hSerdeParams.getValueFactories().get(i) .createValueObjectInspector(columnTypes.get(i))); } } List<String> structFieldComments = StringUtils.isEmpty(tbl.getProperty("columns.comments")) ? new ArrayList<>(Collections.nCopies(columnTypes.size(), "")) : Arrays.asList(tbl.getProperty("columns.comments").split("\0", columnTypes.size())); return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector( hSerdeParams.getColumnNames(), columnObjectInspectors, structFieldComments, hSerdeParams.getSerdeParams().getSeparators()[0], hSerdeParams.getSerdeParams(), ObjectInspectorOptions.JAVA); } }
/** * Initialize the SerDe given parameters. * @see SerDe#initialize(Configuration, Properties) */ @Override public void initialize(Configuration conf, Properties tbl) throws SerDeException { serdeParams = new HBaseSerDeParameters(conf, tbl, getClass().getName()); cachedObjectInspector = HBaseLazyObjectFactory .createLazyHBaseStructInspector(serdeParams.getSerdeParams(), serdeParams.getKeyIndex(), serdeParams.getKeyFactory(), serdeParams.getValueFactories()); cachedHBaseRow = new LazyHBaseRow( (LazySimpleStructObjectInspector) cachedObjectInspector, serdeParams); serializer = new HBaseRowSerializer(serdeParams); if (LOG.isDebugEnabled()) { LOG.debug("HBaseSerDe initialized with : " + serdeParams); } }
@Override public LazyObjectBase createValueObject(ObjectInspector inspector) throws SerDeException { try { return (T) constructor.newInstance(inspector, properties, hbaseParams.getBaseConfiguration(), hbaseParams.getColumnMappings().getColumnsMapping()[fieldID]); } catch (Exception e) { throw new SerDeException(e); } } }
HBaseSerDe hBaseSerDe, ExprNodeDesc predicate) { ColumnMapping keyMapping = hBaseSerDe.getHBaseSerdeParam().getKeyColumnMapping(); ColumnMapping tsMapping = hBaseSerDe.getHBaseSerdeParam().getTimestampColumnMapping(); IndexPredicateAnalyzer analyzer = HiveHBaseTableInputFormat.newIndexPredicateAnalyzer( keyMapping.columnName, keyMapping.isComparable(),
@Override public T createKey(ObjectInspector inspector) throws SerDeException { try { return (T) constructor.newInstance(inspector, properties, hbaseParams.getBaseConfiguration()); } catch (Exception e) { throw new SerDeException(e); } } }
@Override public void init(HBaseSerDeParameters hbaseParam, Properties properties) throws SerDeException { this.hbaseParams = hbaseParam; this.keyMapping = hbaseParam.getKeyColumnMapping(); this.properties = properties; }
public HBaseKeyFactory getKeyFactory() { return serdeParams.getKeyFactory(); }
/** * Initialize the SerDe given parameters. * @see AbstractSerDe#initialize(Configuration, Properties) */ @Override public void initialize(Configuration conf, Properties tbl) throws SerDeException { serdeParams = new HBaseSerDeParameters(conf, tbl, getClass().getName()); cachedObjectInspector = HBaseLazyObjectFactory.createLazyHBaseStructInspector(serdeParams, tbl); cachedHBaseRow = new LazyHBaseRow( (LazySimpleStructObjectInspector) cachedObjectInspector, serdeParams); serializer = new HBaseRowSerializer(serdeParams); if (LOG.isDebugEnabled()) { LOG.debug("HBaseSerDe initialized with : " + serdeParams); } }
public HBaseRowSerializer(HBaseSerDeParameters hbaseParam) { this.hbaseParam = hbaseParam; this.keyFactory = hbaseParam.getKeyFactory(); this.serdeParam = hbaseParam.getSerdeParams(); this.separators = serdeParam.getSeparators(); this.escaped = serdeParam.isEscaped(); this.escapeChar = serdeParam.getEscapeChar(); this.needsEscape = serdeParam.getNeedsEscape(); this.keyIndex = hbaseParam.getKeyIndex(); this.timestampIndex = hbaseParam.getTimestampIndex(); this.columnMappings = hbaseParam.getColumnMappings().getColumnsMapping(); this.keyMapping = hbaseParam.getColumnMappings().getKeyMapping(); this.timestampMapping = hbaseParam.getColumnMappings().getTimestampMapping(); this.putTimestamp = hbaseParam.getPutTimestamp(); }
/** * Construct a LazyHBaseRow object with the ObjectInspector. */ public LazyHBaseRow(LazySimpleStructObjectInspector oi, HBaseSerDeParameters serdeParams) { super(oi); this.keyFactory = serdeParams.getKeyFactory(); this.valueFactories = serdeParams.getValueFactories(); this.columnsMapping = serdeParams.getColumnMappings().getColumnsMapping(); }
@Override public LazyObjectBase createValueObject(ObjectInspector inspector) throws SerDeException { try { return (T) constructor.newInstance(inspector, properties, hbaseParams.getBaseConfiguration(), hbaseParams.getColumnMappings().getColumnsMapping()[fieldID]); } catch (Exception e) { throw new SerDeException(e); } } }
@Override public void init(HBaseSerDeParameters hbaseParams, Configuration conf, Properties properties) throws SerDeException { this.hbaseParams = hbaseParams; this.serdeParams = hbaseParams.getSerdeParams(); this.columnMappings = hbaseParams.getColumnMappings(); this.properties = properties; this.conf = conf; }
public String toString() { return "[" + columnMappingString + ":" + getColumnNames() + ":" + getColumnTypes() + "]"; }