@Explain(displayName = "keyExpressions", explainLevels = { Level.DEBUG }) public Map<Byte, String> getKeyExpressionString() { Map<Byte, String> keyMap = new LinkedHashMap<Byte, String>(); for (Map.Entry<Byte, List<ExprNodeDesc>> k: getKeys().entrySet()) { keyMap.put(k.getKey(), k.getValue().toString()); } return keyMap; }
/** * @return the keys in string form */ @Override @Explain(displayName = "keys") public Map<String, String> getKeysString() { Map<String, String> keyMap = new LinkedHashMap<>(); for (Map.Entry<Byte, List<ExprNodeDesc>> k: getKeys().entrySet()) { keyMap.put(String.valueOf(k.getKey()), PlanUtils.getExprListString(k.getValue())); } return keyMap; }
@Override @Explain(displayName = "keys", explainLevels = { Level.USER }) public Map<Byte, String> getUserLevelExplainKeysString() { Map<Byte, String> keyMap = new LinkedHashMap<Byte, String>(); for (Map.Entry<Byte, List<ExprNodeDesc>> k: getKeys().entrySet()) { keyMap.put(k.getKey(), PlanUtils.getExprListString(k.getValue(), true)); } return keyMap; }
/** * @return the keys in string form */ @Override @Explain(displayName = "keys") public Map<Byte, String> getKeysString() { Map<Byte, String> keyMap = new LinkedHashMap<Byte, String>(); for (Map.Entry<Byte, List<ExprNodeDesc>> k: getKeys().entrySet()) { keyMap.put(k.getKey(), PlanUtils.getExprListString(k.getValue())); } return keyMap; }
@Override @Explain(displayName = "keys", explainLevels = { Level.USER }) public Map<Byte, String> getUserLevelExplainKeysString() { Map<Byte, String> keyMap = new LinkedHashMap<Byte, String>(); for (Map.Entry<Byte, List<ExprNodeDesc>> k: getKeys().entrySet()) { keyMap.put(k.getKey(), PlanUtils.getExprListString(k.getValue(), true)); } return keyMap; }
public VectorMapJoinOperator (CompilationOpContext ctx, VectorizationContext vContext, OperatorDesc conf) throws HiveException { super(ctx, vContext, conf); MapJoinDesc desc = (MapJoinDesc) conf; Map<Byte, List<ExprNodeDesc>> filterExpressions = desc.getFilters(); bigTableFilterExpressions = vContext.getVectorExpressions(filterExpressions.get(posBigTable), VectorExpressionDescriptor.Mode.FILTER); List<ExprNodeDesc> keyDesc = desc.getKeys().get(posBigTable); keyExpressions = vContext.getVectorExpressions(keyDesc); // We're only going to evaluate the big table vectorized expressions, Map<Byte, List<ExprNodeDesc>> exprs = desc.getExprs(); bigTableValueExpressions = vContext.getVectorExpressions(exprs.get(posBigTable)); }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { @SuppressWarnings("unchecked") AbstractMapJoinOperator<? extends MapJoinDesc> mjOp = (AbstractMapJoinOperator<? extends MapJoinDesc>) nd; MapJoinDesc mjDesc = mjOp.getConf(); String bigTablAlias = mjDesc.getBigTableAlias(); if ( bigTablAlias == null ) { Operator<? extends OperatorDesc> parent = null; for(Operator<? extends OperatorDesc> op : mjOp.getParentOperators() ) { if ( op instanceof TableScanOperator ) { parent = op; } } if ( parent != null) { TableScanDesc tDesc = ((TableScanOperator)parent).getConf(); bigTablAlias = tDesc.getAlias(); } } bigTablAlias = bigTablAlias == null ? "?" : bigTablAlias; List<ExprNodeDesc> joinExprs = mjDesc.getKeys().values().iterator().next(); if ( joinExprs.size() == 0 ) { warnings.add( String.format("Map Join %s[bigTable=%s] in task '%s' is a cross product", mjOp.toString(), bigTablAlias, taskName)); } return null; } }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { @SuppressWarnings("unchecked") AbstractMapJoinOperator<? extends MapJoinDesc> mjOp = (AbstractMapJoinOperator<? extends MapJoinDesc>) nd; MapJoinDesc mjDesc = mjOp.getConf(); String bigTablAlias = mjDesc.getBigTableAlias(); if ( bigTablAlias == null ) { Operator<? extends OperatorDesc> parent = null; for(Operator<? extends OperatorDesc> op : mjOp.getParentOperators() ) { if ( op instanceof TableScanOperator ) { parent = op; } } if ( parent != null) { TableScanDesc tDesc = ((TableScanOperator)parent).getConf(); bigTablAlias = tDesc.getAlias(); } } bigTablAlias = bigTablAlias == null ? "?" : bigTablAlias; List<ExprNodeDesc> joinExprs = mjDesc.getKeys().values().iterator().next(); if ( joinExprs.size() == 0 ) { warnings.add( String.format("Map Join %s[bigTable=%s] in task '%s' is a cross product", mjOp.toString(), bigTablAlias, taskName)); } return null; } }
private boolean validateMapJoinDesc(MapJoinDesc desc) { byte posBigTable = (byte) desc.getPosBigTable(); List<ExprNodeDesc> filterExprs = desc.getFilters().get(posBigTable); if (!validateExprNodeDesc(filterExprs, "Filter", VectorExpressionDescriptor.Mode.FILTER)) { return false; } List<ExprNodeDesc> keyExprs = desc.getKeys().get(posBigTable); if (!validateExprNodeDesc(keyExprs, "Key")) { return false; } List<ExprNodeDesc> valueExprs = desc.getExprs().get(posBigTable); if (!validateExprNodeDesc(valueExprs, "Value")) { return false; } Byte[] order = desc.getTagOrder(); Byte posSingleVectorMapJoinSmallTable = (order[0] == posBigTable ? order[1] : order[0]); List<ExprNodeDesc> smallTableExprs = desc.getExprs().get(posSingleVectorMapJoinSmallTable); if (!validateExprNodeDesc(smallTableExprs, "Small Table")) { return false; } if (desc.getResidualFilterExprs() != null && !desc.getResidualFilterExprs().isEmpty()) { LOG.info("Cannot vectorize outer join with complex ON clause"); return false; } return true; }
private boolean validateMapJoinDesc(MapJoinDesc desc) { byte posBigTable = (byte) desc.getPosBigTable(); List<ExprNodeDesc> filterExprs = desc.getFilters().get(posBigTable); if (!validateExprNodeDesc( filterExprs, "Filter", VectorExpressionDescriptor.Mode.FILTER, /* allowComplex */ true)) { return false; } List<ExprNodeDesc> keyExprs = desc.getKeys().get(posBigTable); if (!validateExprNodeDesc(keyExprs, "Key")) { return false; } List<ExprNodeDesc> valueExprs = desc.getExprs().get(posBigTable); if (!validateExprNodeDesc(valueExprs, "Value")) { return false; } Byte[] order = desc.getTagOrder(); Byte posSingleVectorMapJoinSmallTable = (order[0] == posBigTable ? order[1] : order[0]); List<ExprNodeDesc> smallTableExprs = desc.getExprs().get(posSingleVectorMapJoinSmallTable); if (!validateExprNodeDesc(smallTableExprs, "Small Table")) { return false; } if (desc.getResidualFilterExprs() != null && !desc.getResidualFilterExprs().isEmpty()) { setOperatorIssue("Non-equi joins not supported"); return false; } return true; }
Map<Byte, List<ExprNodeDesc>> keysMap = mapJoinOp.getConf().getKeys();
Map<Byte, List<ExprNodeDesc>> keysMap = mapJoinOp.getConf().getKeys();
/** * Check if a mapjoin can be converted to a bucket mapjoin, * and do the version if possible. */ public static void checkAndConvertBucketMapJoin(ParseContext pGraphContext, MapJoinOperator mapJoinOp, String baseBigAlias, List<String> joinAliases) throws SemanticException { BucketJoinProcCtx ctx = new BucketJoinProcCtx(pGraphContext.getConf()); BucketMapjoinProc proc = new BucketMapjoinProc(pGraphContext); Map<Byte, List<ExprNodeDesc>> keysMap = mapJoinOp.getConf().getKeys(); if (proc.checkConvertBucketMapJoin(ctx, mapJoinOp.getConf().getAliasToOpInfo(), keysMap, baseBigAlias, joinAliases)) { proc.convertMapJoinToBucketMapJoin(mapJoinOp, ctx); } } }
/** * Check if a mapjoin can be converted to a bucket mapjoin, * and do the version if possible. */ public static void checkAndConvertBucketMapJoin(ParseContext pGraphContext, MapJoinOperator mapJoinOp, String baseBigAlias, List<String> joinAliases) throws SemanticException { BucketJoinProcCtx ctx = new BucketJoinProcCtx(pGraphContext.getConf()); BucketMapjoinProc proc = new BucketMapjoinProc(pGraphContext); Map<Byte, List<ExprNodeDesc>> keysMap = mapJoinOp.getConf().getKeys(); if (proc.checkConvertBucketMapJoin(ctx, mapJoinOp.getConf().getAliasToOpInfo(), keysMap, baseBigAlias, joinAliases)) { proc.convertMapJoinToBucketMapJoin(mapJoinOp, ctx); } } }
@Override @SuppressWarnings("unchecked") protected void initializeOp(Configuration hconf) throws HiveException { if (conf.getGenJoinKeys()) { int tagLen = conf.getTagLength(); joinKeys = new List[tagLen]; JoinUtil.populateJoinKeyValue(joinKeys, conf.getKeys(), NOTSKIPBIGTABLE, hconf); joinKeysObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinKeys, inputObjInspectors,NOTSKIPBIGTABLE, tagLen); } super.initializeOp(hconf); numMapRowsRead = 0; // all other tables are small, and are cached in the hash table posBigTable = (byte) conf.getPosBigTable(); emptyList = new RowContainer<List<Object>>(1, hconf, reporter); RowContainer<List<Object>> bigPosRC = JoinUtil.getRowContainer(hconf, rowContainerStandardObjectInspectors[posBigTable], posBigTable, joinCacheSize,spillTableDesc, conf, !hasFilter(posBigTable), reporter); storage[posBigTable] = bigPosRC; }
@Override @SuppressWarnings("unchecked") protected void initializeOp(Configuration hconf) throws HiveException { if (conf.getGenJoinKeys()) { int tagLen = conf.getTagLength(); joinKeys = new List[tagLen]; JoinUtil.populateJoinKeyValue(joinKeys, conf.getKeys(), NOTSKIPBIGTABLE, hconf); joinKeysObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinKeys, inputObjInspectors,NOTSKIPBIGTABLE, tagLen); } super.initializeOp(hconf); numMapRowsRead = 0; // all other tables are small, and are cached in the hash table posBigTable = (byte) conf.getPosBigTable(); emptyList = new RowContainer<List<Object>>(1, hconf, reporter); RowContainer<List<Object>> bigPosRC = JoinUtil.getRowContainer(hconf, rowContainerStandardObjectInspectors[posBigTable], posBigTable, joinCacheSize,spillTableDesc, conf, !hasFilter(posBigTable), reporter); storage[posBigTable] = bigPosRC; }
public HashTableSinkDesc(MapJoinDesc clone) { this.bigKeysDirMap = clone.getBigKeysDirMap(); this.conds = clone.getConds(); this.exprs = new HashMap<Byte, List<ExprNodeDesc>>(clone.getExprs()); this.handleSkewJoin = clone.getHandleSkewJoin(); this.keyTableDesc = clone.getKeyTableDesc(); this.noOuterJoin = clone.getNoOuterJoin(); this.outputColumnNames = clone.getOutputColumnNames(); this.reversedExprs = clone.getReversedExprs(); this.skewKeyDefinition = clone.getSkewKeyDefinition(); this.skewKeysValuesTables = clone.getSkewKeysValuesTables(); this.smallKeysDirMap = clone.getSmallKeysDirMap(); this.tagOrder = clone.getTagOrder(); this.filters = new HashMap<Byte, List<ExprNodeDesc>>(clone.getFilters()); this.filterMap = clone.getFilterMap(); this.keys = new HashMap<Byte, List<ExprNodeDesc>>(clone.getKeys()); this.keyTblDesc = clone.getKeyTblDesc(); this.valueTblDescs = clone.getValueTblDescs(); this.valueTblFilteredDescs = clone.getValueFilteredTblDescs(); this.posBigTable = clone.getPosBigTable(); this.retainList = clone.getRetainList(); this.dumpFilePrefix = clone.getDumpFilePrefix(); this.bucketMapjoinContext = new BucketMapJoinContext(clone); this.hashtableMemoryUsage = clone.getHashTableMemoryUsage(); }
public HashTableSinkDesc(MapJoinDesc clone) { this.bigKeysDirMap = clone.getBigKeysDirMap(); this.conds = clone.getConds(); this.exprs = new HashMap<Byte, List<ExprNodeDesc>>(clone.getExprs()); this.handleSkewJoin = clone.getHandleSkewJoin(); this.keyTableDesc = clone.getKeyTableDesc(); this.noOuterJoin = clone.getNoOuterJoin(); this.outputColumnNames = clone.getOutputColumnNames(); this.reversedExprs = clone.getReversedExprs(); this.skewKeyDefinition = clone.getSkewKeyDefinition(); this.skewKeysValuesTables = clone.getSkewKeysValuesTables(); this.smallKeysDirMap = clone.getSmallKeysDirMap(); this.tagOrder = clone.getTagOrder(); this.filters = new HashMap<Byte, List<ExprNodeDesc>>(clone.getFilters()); this.filterMap = clone.getFilterMap(); this.keys = new HashMap<Byte, List<ExprNodeDesc>>(clone.getKeys()); this.keyTblDesc = clone.getKeyTblDesc(); this.valueTblDescs = clone.getValueTblDescs(); this.valueTblFilteredDescs = clone.getValueFilteredTblDescs(); this.posBigTable = clone.getPosBigTable(); this.retainList = clone.getRetainList(); this.dumpFilePrefix = clone.getDumpFilePrefix(); this.bucketMapjoinContext = new BucketMapJoinContext(clone); this.hashtableMemoryUsage = clone.getHashTableMemoryUsage(); }
tableEligibleForBucketedSortMergeJoin = tableEligibleForBucketedSortMergeJoin && isEligibleForBucketSortMergeJoin(smbJoinContext, mapJoinOp.getConf().getKeys().get((byte) pos), mapJoinOp.getConf().getAliasToOpInfo(), srcs,
vContext.getVectorExpressions(mapJoinDesc.getKeys().get(posBigTable)); vectorMapJoinInfo.setSlimmedBigTableKeyExpressions(slimmedBigTableKeyExpressions);