mapJoinOp.getConf().getKeyTblDesc().getProperties().setProperty( serdeConstants.SERIALIZATION_LIB, BinarySortableSerDe.class.getName());
mapJoinOp.getConf().getKeyTblDesc().getProperties().setProperty( serdeConstants.SERIALIZATION_LIB, BinarySortableSerDe.class.getName());
public void generateMapMetaData() throws HiveException { // generate the meta data for key // index for key is -1 try { TableDesc keyTableDesc = conf.getKeyTblDesc(); AbstractSerDe keySerializer = (AbstractSerDe) ReflectionUtil.newInstance( keyTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(keySerializer, null, keyTableDesc.getProperties(), null); MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerializer, false); for (int pos = 0; pos < order.length; pos++) { if (pos == posBigTable) { continue; } TableDesc valueTableDesc; if (conf.getNoOuterJoin()) { valueTableDesc = conf.getValueTblDescs().get(pos); } else { valueTableDesc = conf.getValueFilteredTblDescs().get(pos); } AbstractSerDe valueSerDe = (AbstractSerDe) ReflectionUtil.newInstance( valueTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null); MapJoinObjectSerDeContext valueContext = new MapJoinObjectSerDeContext(valueSerDe, hasFilter(pos)); mapJoinTableSerdes[pos] = new MapJoinTableContainerSerDe(keyContext, valueContext); } } catch (SerDeException e) { throw new HiveException(e); } }
public HashTableSinkDesc(MapJoinDesc clone) { this.bigKeysDirMap = clone.getBigKeysDirMap(); this.conds = clone.getConds(); this.exprs = new HashMap<Byte, List<ExprNodeDesc>>(clone.getExprs()); this.handleSkewJoin = clone.getHandleSkewJoin(); this.keyTableDesc = clone.getKeyTableDesc(); this.noOuterJoin = clone.getNoOuterJoin(); this.outputColumnNames = clone.getOutputColumnNames(); this.reversedExprs = clone.getReversedExprs(); this.skewKeyDefinition = clone.getSkewKeyDefinition(); this.skewKeysValuesTables = clone.getSkewKeysValuesTables(); this.smallKeysDirMap = clone.getSmallKeysDirMap(); this.tagOrder = clone.getTagOrder(); this.filters = new HashMap<Byte, List<ExprNodeDesc>>(clone.getFilters()); this.filterMap = clone.getFilterMap(); this.keys = new HashMap<Byte, List<ExprNodeDesc>>(clone.getKeys()); this.keyTblDesc = clone.getKeyTblDesc(); this.valueTblDescs = clone.getValueTblDescs(); this.valueTblFilteredDescs = clone.getValueFilteredTblDescs(); this.posBigTable = clone.getPosBigTable(); this.retainList = clone.getRetainList(); this.dumpFilePrefix = clone.getDumpFilePrefix(); this.bucketMapjoinContext = new BucketMapJoinContext(clone); this.hashtableMemoryUsage = clone.getHashTableMemoryUsage(); }
public HashTableSinkDesc(MapJoinDesc clone) { this.bigKeysDirMap = clone.getBigKeysDirMap(); this.conds = clone.getConds(); this.exprs = new HashMap<Byte, List<ExprNodeDesc>>(clone.getExprs()); this.handleSkewJoin = clone.getHandleSkewJoin(); this.keyTableDesc = clone.getKeyTableDesc(); this.noOuterJoin = clone.getNoOuterJoin(); this.outputColumnNames = clone.getOutputColumnNames(); this.reversedExprs = clone.getReversedExprs(); this.skewKeyDefinition = clone.getSkewKeyDefinition(); this.skewKeysValuesTables = clone.getSkewKeysValuesTables(); this.smallKeysDirMap = clone.getSmallKeysDirMap(); this.tagOrder = clone.getTagOrder(); this.filters = new HashMap<Byte, List<ExprNodeDesc>>(clone.getFilters()); this.filterMap = clone.getFilterMap(); this.keys = new HashMap<Byte, List<ExprNodeDesc>>(clone.getKeys()); this.keyTblDesc = clone.getKeyTblDesc(); this.valueTblDescs = clone.getValueTblDescs(); this.valueTblFilteredDescs = clone.getValueFilteredTblDescs(); this.posBigTable = clone.getPosBigTable(); this.retainList = clone.getRetainList(); this.dumpFilePrefix = clone.getDumpFilePrefix(); this.bucketMapjoinContext = new BucketMapJoinContext(clone); this.hashtableMemoryUsage = clone.getHashTableMemoryUsage(); }
public void generateMapMetaData() throws HiveException { // generate the meta data for key // index for key is -1 try { TableDesc keyTableDesc = conf.getKeyTblDesc(); AbstractSerDe keySerializer = (AbstractSerDe) ReflectionUtil.newInstance( keyTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(keySerializer, null, keyTableDesc.getProperties(), null); MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerializer, false); for (int pos = 0; pos < order.length; pos++) { if (pos == posBigTable) { continue; } TableDesc valueTableDesc; if (conf.getNoOuterJoin()) { valueTableDesc = conf.getValueTblDescs().get(pos); } else { valueTableDesc = conf.getValueFilteredTblDescs().get(pos); } AbstractSerDe valueSerDe = (AbstractSerDe) ReflectionUtil.newInstance( valueTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null); MapJoinObjectSerDeContext valueContext = new MapJoinObjectSerDeContext(valueSerDe, hasFilter(pos)); mapJoinTableSerdes[pos] = new MapJoinTableContainerSerDe(keyContext, valueContext); } } catch (SerDeException e) { throw new HiveException(e); } }
public static MapJoinTableContainerSerDe createMapJoinTableContainerSerDe(MapJoinDesc mapJoinDesc) throws SerDeException { final Byte smallTablePos = 1; TableDesc keyTableDesc = mapJoinDesc.getKeyTblDesc(); AbstractSerDe keySerializer = (AbstractSerDe) ReflectionUtil.newInstance( BinarySortableSerDe.class, null); SerDeUtils.initializeSerDe(keySerializer, null, keyTableDesc.getProperties(), null); MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerializer, false); final List<TableDesc> valueTableDescList; if (mapJoinDesc.getNoOuterJoin()) { valueTableDescList = mapJoinDesc.getValueTblDescs(); } else { valueTableDescList = mapJoinDesc.getValueFilteredTblDescs(); } TableDesc valueTableDesc = valueTableDescList.get(smallTablePos); AbstractSerDe valueSerDe = (AbstractSerDe) ReflectionUtil.newInstance( valueTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null); MapJoinObjectSerDeContext valueContext = new MapJoinObjectSerDeContext(valueSerDe, hasFilter(mapJoinDesc, smallTablePos)); MapJoinTableContainerSerDe mapJoinTableContainerSerDe = new MapJoinTableContainerSerDe(keyContext, valueContext); return mapJoinTableContainerSerDe; }
public void generateMapMetaData() throws HiveException, SerDeException { // generate the meta data for key // index for key is -1 TableDesc keyTableDesc = conf.getKeyTblDesc(); SerDe keySerializer = (SerDe) ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(), null); keySerializer.initialize(null, keyTableDesc.getProperties()); MapJoinMetaData.put(Integer.valueOf(metadataKeyTag), new HashTableSinkObjectCtx( ObjectInspectorUtils.getStandardObjectInspector(keySerializer.getObjectInspector(), ObjectInspectorCopyOption.WRITABLE), keySerializer, keyTableDesc, hconf)); // index for values is just alias for (int tag = 0; tag < order.length; tag++) { int alias = (int) order[tag]; if (alias == this.bigTableAlias) { continue; } TableDesc valueTableDesc = conf.getValueTblDescs().get(tag); SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(), null); valueSerDe.initialize(null, valueTableDesc.getProperties()); MapJoinMetaData.put(Integer.valueOf(alias), new HashTableSinkObjectCtx(ObjectInspectorUtils .getStandardObjectInspector(valueSerDe.getObjectInspector(), ObjectInspectorCopyOption.WRITABLE), valueSerDe, valueTableDesc, hconf)); } }
public HashTableSinkDesc(MapJoinDesc clone) { this.bigKeysDirMap = clone.getBigKeysDirMap(); this.conds = clone.getConds(); this.exprs = new HashMap<Byte, List<ExprNodeDesc>>(clone.getExprs()); this.handleSkewJoin = clone.getHandleSkewJoin(); this.keyTableDesc = clone.getKeyTableDesc(); this.noOuterJoin = clone.getNoOuterJoin(); this.outputColumnNames = clone.getOutputColumnNames(); this.reversedExprs = clone.getReversedExprs(); this.skewKeyDefinition = clone.getSkewKeyDefinition(); this.skewKeysValuesTables = clone.getSkewKeysValuesTables(); this.smallKeysDirMap = clone.getSmallKeysDirMap(); this.tagOrder = clone.getTagOrder(); this.filters = new HashMap<Byte, List<ExprNodeDesc>>(clone.getFilters()); this.filterMap = clone.getFilterMap(); this.keys = new HashMap<Byte, List<ExprNodeDesc>>(clone.getKeys()); this.keyTblDesc = clone.getKeyTblDesc(); this.valueTblDescs = clone.getValueTblDescs(); this.valueTblFilteredDescs = clone.getValueFilteredTblDescs(); this.posBigTable = clone.getPosBigTable(); this.retainList = clone.getRetainList(); this.dumpFilePrefix = clone.getDumpFilePrefix(); this.bucketMapjoinContext = new BucketMapJoinContext(clone); this.hashtableMemoryUsage = clone.getHashTableMemoryUsage(); }
public void generateMapMetaData() throws HiveException { // generate the meta data for key // index for key is -1 try { TableDesc keyTableDesc = conf.getKeyTblDesc(); SerDe keySerializer = (SerDe) ReflectionUtil.newInstance( keyTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(keySerializer, null, keyTableDesc.getProperties(), null); MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerializer, false); for (int pos = 0; pos < order.length; pos++) { if (pos == posBigTable) { continue; } TableDesc valueTableDesc; if (conf.getNoOuterJoin()) { valueTableDesc = conf.getValueTblDescs().get(pos); } else { valueTableDesc = conf.getValueFilteredTblDescs().get(pos); } SerDe valueSerDe = (SerDe) ReflectionUtil.newInstance( valueTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null); MapJoinObjectSerDeContext valueContext = new MapJoinObjectSerDeContext(valueSerDe, hasFilter(pos)); mapJoinTableSerdes[pos] = new MapJoinTableContainerSerDe(keyContext, valueContext); } } catch (SerDeException e) { throw new HiveException(e); } }
public HashTableSinkDesc(MapJoinDesc clone) { this.bigKeysDirMap = clone.getBigKeysDirMap(); this.conds = clone.getConds(); this.exprs= clone.getExprs(); this.handleSkewJoin = clone.getHandleSkewJoin(); this.keyTableDesc = clone.getKeyTableDesc(); this.noOuterJoin = clone.getNoOuterJoin(); this.outputColumnNames = clone.getOutputColumnNames(); this.reversedExprs = clone.getReversedExprs(); this.skewKeyDefinition = clone.getSkewKeyDefinition(); this.skewKeysValuesTables = clone.getSkewKeysValuesTables(); this.smallKeysDirMap = clone.getSmallKeysDirMap(); this.tagOrder = clone.getTagOrder(); this.filters = clone.getFilters(); this.keys = clone.getKeys(); this.keyTblDesc = clone.getKeyTblDesc(); this.valueTblDescs = clone.getValueTblDescs(); this.valueTblFilteredDescs = clone.getValueFilteredTblDescs(); this.posBigTable = clone.getPosBigTable(); this.retainList = clone.getRetainList(); this.bigTableAlias = clone.getBigTableAlias(); this.aliasBucketFileNameMapping = clone.getAliasBucketFileNameMapping(); this.bucketFileNameMapping = clone.getBucketFileNameMapping(); }