ImmutableBitSet rightCols = RelOptUtil.InputFinder.bits(((RexCall) joinCond).getOperands().get(1)); if (leftCols.cardinality() != 1 || rightCols.cardinality() != 1 ) { return null; int nFieldsLeft = joinRel.getLeft().getRowType().getFieldList().size(); int nFieldsRight = joinRel.getRight().getRowType().getFieldList().size(); int nSysFields = joinRel.getSystemFieldList().size(); ImmutableBitSet rightFieldsBitSet = ImmutableBitSet.range(nSysFields + nFieldsLeft, nSysFields + nFieldsLeft + nFieldsRight); if (rightFieldsBitSet.contains(leftCols)) { ImmutableBitSet t = leftCols; leftCols = rightCols; leftColIdx = leftCols.nextSetBit(0) - nSysFields; rightColIdx = rightCols.nextSetBit(0) - (nSysFields + nFieldsLeft);
private Statistic buildStatistic() { if (stats != null || primaryKey == -1) { return stats; } Direction dir = primaryKeyMonotonicity == INCREASING ? ASCENDING : DESCENDING; RelFieldCollation collation = new RelFieldCollation(primaryKey, dir, NullDirection.UNSPECIFIED); return Statistics.of(fields.size(), ImmutableList.of(ImmutableBitSet.of(primaryKey)), ImmutableList.of(RelCollations.of(collation))); }
ImmutableBitSet backtrackedGBSet = ImmutableBitSet.builder().addAll(backtrackedGBList).build(); if(backtrackedGBSet.contains(key)) { if(currentKey == null || currentKey.isEmpty()) { return originalGroupSet; ImmutableBitSet.Builder builder = ImmutableBitSet.builder(); int orgCol = originalGroupSet.nth(i); if(fieldsUsed.get((orgCol)) || currentKey.get(backtrackedCol)) {
ExprsItr(ImmutableBitSet fields) { nextMapping = null; columns = new int[fields.cardinality()]; columnSets = new BitSet[fields.cardinality()]; iterationIdx = new int[fields.cardinality()]; for (int j = 0, i = fields.nextSetBit(0); i >= 0; i = fields .nextSetBit(i + 1), j++) { columns[j] = i; columnSets[j] = equivalence.get(i); iterationIdx[j] = 0; } firstCall = true; }
private boolean filterRefersToBothSidesOfJoin(RexNode filter, Join j) { boolean refersToBothSides = false; int joinNoOfProjects = j.getRowType().getFieldCount(); ImmutableBitSet filterProjs = ImmutableBitSet.FROM_BIT_SET.apply(new BitSet(joinNoOfProjects)); ImmutableBitSet allLeftProjs = filterProjs.union(ImmutableBitSet.range(0, j.getInput(0) .getRowType().getFieldCount())); ImmutableBitSet allRightProjs = filterProjs.union(ImmutableBitSet.range(j.getInput(0) .getRowType().getFieldCount(), joinNoOfProjects)); filterProjs = filterProjs.union(InputFinder.bits(filter)); if (allLeftProjs.intersects(filterProjs) && allRightProjs.intersects(filterProjs)) refersToBothSides = true; return refersToBothSides; } }
final RelDataType inputRowType = peek().getRowType(); final List<RexNode> extraNodes = projects(inputRowType); final GroupKeyImpl groupKey_ = (GroupKeyImpl) groupKey; final ImmutableBitSet groupSet = ImmutableBitSet.of(registerExpressions(extraNodes, groupKey_.nodes)); final ImmutableList<ImmutableBitSet> groupSets; if (groupKey_.nodeLists != null) { for (ImmutableList<RexNode> nodeList : groupKey_.nodeLists) { final ImmutableBitSet groupSet2 = ImmutableBitSet.of(registerExpressions(extraNodes, nodeList)); if (!groupSet.contains(groupSet2)) { throw new IllegalArgumentException("group set element " + nodeList + " must be a subset of group key"); groupSets = ImmutableList.of(groupSet); if (extraNodes.size() > inputRowType.getFieldCount()) { project(extraNodes); aggregateCall = AggregateCall.create(aggCall1.aggFunction, aggCall1.distinct, args, filterArg, groupSet.cardinality(), r, null, aggCall1.alias); } else { aggregateCall = ((AggCallImpl2) aggCall).aggregateCall; assert groupSet.contains(set);
aggregate.getGroupSet().rebuild(); ImmutableBitSet gbKeysDeleted = originalGroupSet.except(updatedGroupSet); ImmutableBitSet updatedGroupFields = ImmutableBitSet.range(originalGroupSet.cardinality()); final int updatedGroupCount = updatedGroupSet.cardinality(); setIdx = gbKeysDeleted.nextSetBit(setIdx); if(setIdx != -1) { updatedGroupFields = updatedGroupFields.clear(setIdx); setIdx++; fieldsUsed.union(updatedGroupFields); && fieldsUsed.equals(ImmutableBitSet.range(rowType.getFieldCount()))) { return result(aggregate, Mappings.createIdentity(rowType.getFieldCount())); int originalGroupCount = aggregate.getGroupSet().cardinality(); int j = originalGroupCount; int usedAggCallCount = 0; for (int i = 0; i < aggregate.getAggCallList().size(); i++) { if (fieldsUsed.get(j++)) { ++usedAggCallCount; if(!updatedGroupSet.equals(aggregate.getGroupSet())) { newGroupSets = ImmutableList.of(newGroupSet); } else { newGroupSets = ImmutableList.copyOf( Iterables.transform(aggregate.getGroupSets(),
this.joinRel = joinRel; this.isSemiJoin = isSemiJoin; nFieldsLeft = joinRel.getLeft().getRowType().getFieldList().size(); nFieldsRight = joinRel.getRight().getRowType().getFieldList().size(); nSysFields = joinRel.getSystemFieldList().size(); leftFieldsBitSet = ImmutableBitSet.range(nSysFields, nSysFields + nFieldsLeft); rightFieldsBitSet = ImmutableBitSet.range(nSysFields + nFieldsLeft, nSysFields + nFieldsLeft + nFieldsRight); allFieldsBitSet = ImmutableBitSet.range(0, nSysFields + nFieldsLeft + nFieldsRight); Mappings.TargetMapping leftMapping = Mappings.createShiftMapping( nSysFields + nFieldsLeft, nSysFields, 0, nFieldsLeft); leftChildPredicates = lPreds.accept( new RexPermuteInputsShuttle(leftMapping, joinRel.getInput(0))); exprFields.put(r.toString(), RelOptUtil.InputFinder.bits(r)); allExprsDigests.add(r.toString()); RexBuilder rexBuilder = joinRel.getCluster().getRexBuilder(); List<RexNode> exprs = RelOptUtil.conjunctions( compose(rexBuilder, ImmutableList.of(joinRel.getCondition())));
private static RelNode mergeJoin(HiveJoin join, RelNode left, RelNode right) { final RexBuilder rexBuilder = join.getCluster().getRexBuilder(); joinFilters.add(leftJoinFilters.get(i)); newInputs.addAll(left.getInputs()); } else { // The join operation in the child is not on the same keys return null; return null; ImmutableBitSet.Builder keysInInputsBuilder = ImmutableBitSet.builder(); for (int i=0; i<newInputs.size(); i++) { List<RexNode> partialCondition = joinKeyExprs.get(i); keysInInputs.intersect(ImmutableBitSet.range(numberLeftInputs)); ImmutableBitSet rightReferencedInputs = keysInInputs.intersect(ImmutableBitSet.range(numberLeftInputs, newInputs.size())); if (join.getJoinType() != JoinRelType.INNER && (leftReferencedInputs.cardinality() > 1 || rightReferencedInputs.cardinality() > 1)) { return null; int leftInput = keysInInputs.nextSetBit(0); int rightInput = keysInInputs.nextSetBit(numberLeftInputs); joinInputs.add(Pair.of(leftInput, rightInput)); joinTypes.add(join.getJoinType());
@Override public void onMatch(RelOptRuleCall call) { final Filter fil = call.rel(0); final TableScan rel = call.rel(1); // Add identity RelBuilder relBuilder = call.builder(); relBuilder.push(rel); List<RexNode> identityFields = relBuilder.fields( ImmutableBitSet.range(0, rel.getRowType().getFieldCount()).asList()); RelNode newRel = relBuilder .project(identityFields, ImmutableList.<String>of(), true) .build(); call.transformTo(fil.copy(fil.getTraitSet(), ImmutableList.of(newRel))); }
/** * Creates a LogicalAggregate that removes all duplicates from the result of * an underlying relational expression. * * @param rel underlying rel * @return rel implementing SingleValueAgg */ public static RelNode createSingleValueAggRel( RelOptCluster cluster, RelNode rel, RelFactories.AggregateFactory aggregateFactory) { // assert (rel.getRowType().getFieldCount() == 1); final int aggCallCnt = rel.getRowType().getFieldCount(); final List<AggregateCall> aggCalls = new ArrayList<>(); for (int i = 0; i < aggCallCnt; i++) { aggCalls.add( AggregateCall.create( SqlStdOperatorTable.SINGLE_VALUE, false, false, ImmutableList.of(i), -1, 0, rel, null, null)); } return aggregateFactory.createAggregate(rel, false, ImmutableBitSet.of(), null, aggCalls); }
public ImmutableList<RelCollation> collations(HiveAggregate aggregate, RelMetadataQuery mq) { // Compute collations ImmutableList.Builder<RelFieldCollation> collationListBuilder = new ImmutableList.Builder<RelFieldCollation>(); for (int pos : aggregate.getGroupSet().asList()) { final RelFieldCollation fieldCollation = new RelFieldCollation(pos); collationListBuilder.add(fieldCollation); } // Return aggregate collations return ImmutableList.of( RelCollationTraitDef.INSTANCE.canonize( new HiveRelCollation(collationListBuilder.build()))); }
LOG.debug("Matched HiveSemiJoinRule"); final RelOptCluster cluster = join.getCluster(); final RexBuilder rexBuilder = cluster.getRexBuilder(); final ImmutableBitSet rightBits = ImmutableBitSet.range(left.getRowType().getFieldCount(), join.getRowType().getFieldCount()); if (topRefs.intersects(rightBits)) { return; if (!joinInfo.rightSet().equals( ImmutableBitSet.range(aggregate.getGroupCount()))) { call.transformTo(topOperator.copy(topOperator.getTraitSet(), ImmutableList.of(left))); return; final List<Integer> aggregateKeys = aggregate.getGroupSet().asList(); for (int key : joinInfo.rightKeys) { newRightKeyBuilder.add(aggregateKeys.get(key)); RelNode topProject = call.builder().push(rightJoin).project(projects, rightJoin.getRowType().getFieldNames(), semi = call.builder().push(left).push(aggregate.getInput()).semiJoin(newCondition).build(); call.transformTo(topOperator.copy(topOperator.getTraitSet(), ImmutableList.of(semi)));
RexNode joinCond = rexBuilder.makeLiteral(true); if ((joinType != JoinRelType.LEFT) || (joinCond != rexBuilder.makeLiteral(true))) { return; if (!aggregate.getGroupSet().isEmpty()) { return; int nFields = left.getRowType().getFieldCount(); ImmutableBitSet allCols = ImmutableBitSet.range(nFields); ImmutableList.of( Pair.<RexNode, String>of(rexBuilder.makeLiteral(true), "nullIndicator"))); int nullIndicatorPos = join.getRowType().getFieldCount() - 1; cluster.getTypeFactory().createTypeWithNullability( join.getRowType().getFieldList() .get(nullIndicatorPos).getType(), ImmutableBitSet.range(groupCount); Aggregate newAggregate = (Aggregate) relBuilder.push(joinOutputProject)
final RelNode child = aggregate.getInput(); final List<RelDataTypeField> childFields = child.getRowType().getFieldList(); for (int i : aggregate.getGroupSet()) { sourceOf.put(i, projects.size()); ImmutableBitSet.range(projects.size()), null, ImmutableList.<AggregateCall>of());
final Union union = call.rel(0); final int count = union.getRowType().getFieldCount(); if (count == 1) { List<RelDataTypeField> fields = union.getRowType().getFieldList(); List<RexNode> topChildExprs = new ArrayList<>(); List<String> topChildExprsFields = new ArrayList<>(); List<RexNode> refs = new ArrayList<>(); ImmutableBitSet.Builder refsIndexBuilder = ImmutableBitSet.builder(); for (int i = 0; i < count ; i++) { RexNode expr = rexBuilder.makeInputRef(union, i); RelDataTypeField field = fields.get(i); if (constants.containsKey(expr)) { RelOptUtil.permutation(refs, union.getInput(0).getRowType()).inverse(); topChildExprs = ImmutableList.copyOf(RexUtil.apply(mapping, topChildExprs)); RelNode input = union.getInput(i); List<Pair<RexNode, String>> newChildExprs = new ArrayList<>(); for (int j = 0; j < refsIndex.cardinality(); j++ ) { int pos = refsIndex.nth(j); newChildExprs.add( Pair.<RexNode, String>of(rexBuilder.makeInputRef(input, pos), input.getRowType().getFieldList().get(pos).getName()));
public double computeMapJoinCPUCost( ImmutableList<Double> cardinalities, ImmutableBitSet streaming) { // Hash-join double cpuCost = 0.0; for (int i=0; i<cardinalities.size(); i++) { double cardinality = cardinalities.get(i); if (!streaming.get(i)) { cpuCost += cardinality; } cpuCost += cardinality * cpuUnitCost; } return cpuCost; }
private Pair<List<ImmutableBitSet>, List<ImmutableBitSet>> generateKeys() { ImmutableList.Builder<ImmutableBitSet> builder = ImmutableList.builder(); ImmutableList.Builder<ImmutableBitSet> nonNullbuilder = ImmutableList.builder(); ImmutableBitSet.Builder keys = ImmutableBitSet.builder(); for (String pkColName : pki.getColNames().values()) { int pkPos; for (pkPos = 0; pkPos < rowType.getFieldNames().size(); pkPos++) { String colName = rowType.getFieldNames().get(pkPos); if (pkColName.equals(colName)) { break; if (pkPos == rowType.getFieldNames().size()) { LOG.error("Column for primary key definition " + pkColName + " not found"); ImmutableBitSet.Builder keys = ImmutableBitSet.builder(); boolean isNonNullable = true; for (UniqueConstraintCol ukCol : ukCols) {
if (aggregate.getGroupSets().size() > 1 || aggregate.getIndicatorCount() > 0 || fieldsUsed.contains(originalGroupSet)) { RelMetadataQuery mq = aggregate.getCluster().getMetadataQuery(); if (aggregate.getGroupSet().contains(key)) { groupByUniqueKey = key; break; ImmutableBitSet nonKeyColumns = aggregate.getGroupSet().except(groupByUniqueKey); ImmutableBitSet columnsToRemove = nonKeyColumns.except(fieldsUsed); ImmutableBitSet newGroupSet = aggregate.getGroupSet().except(columnsToRemove);