public VectorMapJoinOperator (CompilationOpContext ctx, VectorizationContext vContext, OperatorDesc conf) throws HiveException { super(ctx, vContext, conf); MapJoinDesc desc = (MapJoinDesc) conf; Map<Byte, List<ExprNodeDesc>> filterExpressions = desc.getFilters(); bigTableFilterExpressions = vContext.getVectorExpressions(filterExpressions.get(posBigTable), VectorExpressionDescriptor.Mode.FILTER); List<ExprNodeDesc> keyDesc = desc.getKeys().get(posBigTable); keyExpressions = vContext.getVectorExpressions(keyDesc); // We're only going to evaluate the big table vectorized expressions, Map<Byte, List<ExprNodeDesc>> exprs = desc.getExprs(); bigTableValueExpressions = vContext.getVectorExpressions(exprs.get(posBigTable)); }
final int numAliases = mapJoinDesc.getExprs().size(); List<TableDesc> valueFilteredTblDescs = mapJoinDesc.getValueFilteredTblDescs(); for (byte pos = 0; pos < numAliases; pos++) {
private boolean validateMapJoinDesc(MapJoinDesc desc) { byte posBigTable = (byte) desc.getPosBigTable(); List<ExprNodeDesc> filterExprs = desc.getFilters().get(posBigTable); if (!validateExprNodeDesc(filterExprs, "Filter", VectorExpressionDescriptor.Mode.FILTER)) { return false; } List<ExprNodeDesc> keyExprs = desc.getKeys().get(posBigTable); if (!validateExprNodeDesc(keyExprs, "Key")) { return false; } List<ExprNodeDesc> valueExprs = desc.getExprs().get(posBigTable); if (!validateExprNodeDesc(valueExprs, "Value")) { return false; } Byte[] order = desc.getTagOrder(); Byte posSingleVectorMapJoinSmallTable = (order[0] == posBigTable ? order[1] : order[0]); List<ExprNodeDesc> smallTableExprs = desc.getExprs().get(posSingleVectorMapJoinSmallTable); if (!validateExprNodeDesc(smallTableExprs, "Small Table")) { return false; } if (desc.getResidualFilterExprs() != null && !desc.getResidualFilterExprs().isEmpty()) { LOG.info("Cannot vectorize outer join with complex ON clause"); return false; } return true; }
private boolean validateMapJoinDesc(MapJoinDesc desc) { byte posBigTable = (byte) desc.getPosBigTable(); List<ExprNodeDesc> filterExprs = desc.getFilters().get(posBigTable); if (!validateExprNodeDesc( filterExprs, "Filter", VectorExpressionDescriptor.Mode.FILTER, /* allowComplex */ true)) { return false; } List<ExprNodeDesc> keyExprs = desc.getKeys().get(posBigTable); if (!validateExprNodeDesc(keyExprs, "Key")) { return false; } List<ExprNodeDesc> valueExprs = desc.getExprs().get(posBigTable); if (!validateExprNodeDesc(valueExprs, "Value")) { return false; } Byte[] order = desc.getTagOrder(); Byte posSingleVectorMapJoinSmallTable = (order[0] == posBigTable ? order[1] : order[0]); List<ExprNodeDesc> smallTableExprs = desc.getExprs().get(posSingleVectorMapJoinSmallTable); if (!validateExprNodeDesc(smallTableExprs, "Small Table")) { return false; } if (desc.getResidualFilterExprs() != null && !desc.getResidualFilterExprs().isEmpty()) { setOperatorIssue("Non-equi joins not supported"); return false; } return true; }
conf.getExprs().toString()); LOG.info(getLoggingPrefix() + " mapJoinDesc.getRetainList " + conf.getRetainList().toString());
public VectorMapJoinBaseOperator(CompilationOpContext ctx, OperatorDesc conf, VectorizationContext vContext, VectorDesc vectorDesc) throws HiveException { super(ctx); MapJoinDesc desc = (MapJoinDesc) conf; this.conf = desc; this.vContext = vContext; this.vectorDesc = (VectorMapJoinDesc) vectorDesc; order = desc.getTagOrder(); numAliases = desc.getExprs().size(); posBigTable = (byte) desc.getPosBigTable(); filterMaps = desc.getFilterMap(); noOuterJoin = desc.isNoOuterJoin(); // We are making a new output vectorized row batch. vOutContext = new VectorizationContext(getName(), desc.getOutputColumnNames(), /* vContextEnvironment */ vContext); vOutContext.setInitialTypeInfos(Arrays.asList(getOutputTypeInfos(desc))); }
public VectorMapJoinBaseOperator(CompilationOpContext ctx, VectorizationContext vContext, OperatorDesc conf) throws HiveException { super(ctx); MapJoinDesc desc = (MapJoinDesc) conf; this.conf = desc; order = desc.getTagOrder(); numAliases = desc.getExprs().size(); posBigTable = (byte) desc.getPosBigTable(); filterMaps = desc.getFilterMap(); noOuterJoin = desc.isNoOuterJoin(); // We are making a new output vectorized row batch. vOutContext = new VectorizationContext(getName(), desc.getOutputColumnNames(), /* vContextEnvironment */ vContext); }
Map<Byte, List<ExprNodeDesc>> valueExpressions = conf.getExprs(); List<ExprNodeDesc> bigTableExpressions = valueExpressions.get(posBigTable);
Map<Byte, List<ExprNodeDesc>> valueExpressions = conf.getExprs(); List<ExprNodeDesc> bigTableExpressions = valueExpressions.get(posBigTable);
public HashTableSinkDesc(MapJoinDesc clone) { this.bigKeysDirMap = clone.getBigKeysDirMap(); this.conds = clone.getConds(); this.exprs = new HashMap<Byte, List<ExprNodeDesc>>(clone.getExprs()); this.handleSkewJoin = clone.getHandleSkewJoin(); this.keyTableDesc = clone.getKeyTableDesc(); this.noOuterJoin = clone.getNoOuterJoin(); this.outputColumnNames = clone.getOutputColumnNames(); this.reversedExprs = clone.getReversedExprs(); this.skewKeyDefinition = clone.getSkewKeyDefinition(); this.skewKeysValuesTables = clone.getSkewKeysValuesTables(); this.smallKeysDirMap = clone.getSmallKeysDirMap(); this.tagOrder = clone.getTagOrder(); this.filters = new HashMap<Byte, List<ExprNodeDesc>>(clone.getFilters()); this.filterMap = clone.getFilterMap(); this.keys = new HashMap<Byte, List<ExprNodeDesc>>(clone.getKeys()); this.keyTblDesc = clone.getKeyTblDesc(); this.valueTblDescs = clone.getValueTblDescs(); this.valueTblFilteredDescs = clone.getValueFilteredTblDescs(); this.posBigTable = clone.getPosBigTable(); this.retainList = clone.getRetainList(); this.dumpFilePrefix = clone.getDumpFilePrefix(); this.bucketMapjoinContext = new BucketMapJoinContext(clone); this.hashtableMemoryUsage = clone.getHashTableMemoryUsage(); }
public HashTableSinkDesc(MapJoinDesc clone) { this.bigKeysDirMap = clone.getBigKeysDirMap(); this.conds = clone.getConds(); this.exprs = new HashMap<Byte, List<ExprNodeDesc>>(clone.getExprs()); this.handleSkewJoin = clone.getHandleSkewJoin(); this.keyTableDesc = clone.getKeyTableDesc(); this.noOuterJoin = clone.getNoOuterJoin(); this.outputColumnNames = clone.getOutputColumnNames(); this.reversedExprs = clone.getReversedExprs(); this.skewKeyDefinition = clone.getSkewKeyDefinition(); this.skewKeysValuesTables = clone.getSkewKeysValuesTables(); this.smallKeysDirMap = clone.getSmallKeysDirMap(); this.tagOrder = clone.getTagOrder(); this.filters = new HashMap<Byte, List<ExprNodeDesc>>(clone.getFilters()); this.filterMap = clone.getFilterMap(); this.keys = new HashMap<Byte, List<ExprNodeDesc>>(clone.getKeys()); this.keyTblDesc = clone.getKeyTblDesc(); this.valueTblDescs = clone.getValueTblDescs(); this.valueTblFilteredDescs = clone.getValueFilteredTblDescs(); this.posBigTable = clone.getPosBigTable(); this.retainList = clone.getRetainList(); this.dumpFilePrefix = clone.getDumpFilePrefix(); this.bucketMapjoinContext = new BucketMapJoinContext(clone); this.hashtableMemoryUsage = clone.getHashTableMemoryUsage(); }
List<ExprNodeDesc> bigTableExprs = desc.getExprs().get(posBigTable); VectorExpression[] allBigTableValueExpressions = vContext.getVectorExpressions(bigTableExprs); List<ExprNodeDesc> smallTableExprs = desc.getExprs().get(posSingleVectorMapJoinSmallTable); if (desc.getValueIndices() != null && desc.getValueIndices().get(posSingleVectorMapJoinSmallTable) != null) { smallTableIndices = desc.getValueIndices().get(posSingleVectorMapJoinSmallTable);
List<ExprNodeDesc> bigTableExprs = desc.getExprs().get(posBigTable); List<ExprNodeDesc> smallTableExprs = desc.getExprs().get(posSingleVectorMapJoinSmallTable); if (desc.getValueIndices() != null && desc.getValueIndices().get(posSingleVectorMapJoinSmallTable) != null) { smallTableIndices = desc.getValueIndices().get(posSingleVectorMapJoinSmallTable);
vectorMapJoinInfo.setSlimmedBigTableKeyExpressions(slimmedBigTableKeyExpressions); Map<Byte, List<ExprNodeDesc>> exprs = mapJoinDesc.getExprs(); VectorExpression[] slimmedBigTableValueExpressions = vContext.getVectorExpressions(exprs.get(posBigTable));
vectorMapJoinDesc.setAllBigTableKeyExpressions(allBigTableKeyExpressions); Map<Byte, List<ExprNodeDesc>> exprs = mapJoinDesc.getExprs(); VectorExpression[] allBigTableValueExpressions = vContext.getVectorExpressions(exprs.get(posBigTable));
public VectorMapJoinOperator (VectorizationContext vContext, OperatorDesc conf) throws HiveException { super(vContext, conf); MapJoinDesc desc = (MapJoinDesc) conf; Map<Byte, List<ExprNodeDesc>> filterExpressions = desc.getFilters(); bigTableFilterExpressions = vContext.getVectorExpressions(filterExpressions.get(posBigTable), VectorExpressionDescriptor.Mode.FILTER); List<ExprNodeDesc> keyDesc = desc.getKeys().get(posBigTable); keyExpressions = vContext.getVectorExpressions(keyDesc); // We're only going to evaluate the big table vectorized expressions, Map<Byte, List<ExprNodeDesc>> exprs = desc.getExprs(); bigTableValueExpressions = vContext.getVectorExpressions(exprs.get(posBigTable)); }
public VectorMapJoinBaseOperator (VectorizationContext vContext, OperatorDesc conf) throws HiveException { super(); MapJoinDesc desc = (MapJoinDesc) conf; this.conf = desc; order = desc.getTagOrder(); numAliases = desc.getExprs().size(); posBigTable = (byte) desc.getPosBigTable(); filterMaps = desc.getFilterMap(); noOuterJoin = desc.isNoOuterJoin(); // We are making a new output vectorized row batch. vOutContext = new VectorizationContext(getName(), desc.getOutputColumnNames()); }
private boolean validateMapJoinDesc(MapJoinDesc desc) { byte posBigTable = (byte) desc.getPosBigTable(); List<ExprNodeDesc> filterExprs = desc.getFilters().get(posBigTable); if (!validateExprNodeDesc(filterExprs, VectorExpressionDescriptor.Mode.FILTER)) { LOG.info("Cannot vectorize map work filter expression"); return false; } List<ExprNodeDesc> keyExprs = desc.getKeys().get(posBigTable); if (!validateExprNodeDesc(keyExprs)) { LOG.info("Cannot vectorize map work key expression"); return false; } List<ExprNodeDesc> valueExprs = desc.getExprs().get(posBigTable); if (!validateExprNodeDesc(valueExprs)) { LOG.info("Cannot vectorize map work value expression"); return false; } return true; }
public HashTableSinkDesc(MapJoinDesc clone) { this.bigKeysDirMap = clone.getBigKeysDirMap(); this.conds = clone.getConds(); this.exprs= clone.getExprs(); this.handleSkewJoin = clone.getHandleSkewJoin(); this.keyTableDesc = clone.getKeyTableDesc(); this.noOuterJoin = clone.getNoOuterJoin(); this.outputColumnNames = clone.getOutputColumnNames(); this.reversedExprs = clone.getReversedExprs(); this.skewKeyDefinition = clone.getSkewKeyDefinition(); this.skewKeysValuesTables = clone.getSkewKeysValuesTables(); this.smallKeysDirMap = clone.getSmallKeysDirMap(); this.tagOrder = clone.getTagOrder(); this.filters = clone.getFilters(); this.keys = clone.getKeys(); this.keyTblDesc = clone.getKeyTblDesc(); this.valueTblDescs = clone.getValueTblDescs(); this.valueTblFilteredDescs = clone.getValueFilteredTblDescs(); this.posBigTable = clone.getPosBigTable(); this.retainList = clone.getRetainList(); this.bigTableAlias = clone.getBigTableAlias(); this.aliasBucketFileNameMapping = clone.getAliasBucketFileNameMapping(); this.bucketFileNameMapping = clone.getBucketFileNameMapping(); }