private Prel newProject(List<RexNode> exprs, RelDataType rowType, Prel child) { return new ProjectPrel(cluster, traits, child, exprs, rowType); }
&& StarColumnHelper.containsStarColumnInProject(prel.getInput().getRowType(), proj.getProjects()) && prel.getRowType().getFieldNames().size() > 1) { prefixedForStar = true; RelNode child = ((Prel) prel.getInput(0)).accept(this, null); for (Pair<String, RexNode> pair : Pair.zip(prel.getRowType().getFieldNames(), proj.getProjects())) { if (pair.right instanceof RexInputRef) { String name = child.getRowType().getFieldNames().get(((RexInputRef) pair.right).getIndex()); RelDataType rowType = RexUtil.createStructType(prel.getCluster().getTypeFactory(), proj.getProjects(), fieldNames); ProjectPrel newProj = (ProjectPrel) proj.copy(proj.getTraitSet(), child, proj.getProjects(), rowType);
@Override public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException { Prel child = (Prel) this.getInput(); PhysicalOperator childPOP = child.getPhysicalOperator(creator); com.dremio.exec.physical.config.Project p = new com.dremio.exec.physical.config.Project( this.getProjectExpressions(new ParseContext(PrelUtil.getSettings(getCluster()))), childPOP); return creator.addMetadata(this, p); }
@Override public Project copy(RelTraitSet traitSet, RelNode input, List<RexNode> exps, RelDataType rowType) { return new ProjectPrel(getCluster(), traitSet, input, exps, rowType); }
@Override public Prel visitProject(ProjectPrel project, Object unused) throws RelConversionException { RelNode transformedInput = ((Prel)project.getInput()).accept(this, null); Prel accepted = (Prel) SplitUpComplexExpressions.visitProject(project, transformedInput, funcReg); if (accepted == null) { return (ProjectPrel) project.copy(project.getTraitSet(), Lists.newArrayList(transformedInput)); } else { return accepted; } }
@Override public Prel visitProject(ProjectPrel topRel, Void voidValue) throws RuntimeException { final List<RexNode> expressions = topRel.getChildExps(); final RelNode inputRel = ((Prel) topRel.getInput()).accept(this, null); throw failed(); final String inputField = topRel.getRowType().getFieldNames().get(fieldId); conversions.add(new ConversionColumn(OriginType.LITERAL, null, null, inputField, getLiteralSchema(context, value))); bottomExprs.add(literal); final String tableFieldname = originTable.getRowType().getFieldNames().get(origin.getOriginColumnOrdinal()); final String inputFieldname = topRel.getRowType().getFieldNames().get(fieldId); conversions.add(new ConversionColumn(OriginType.RAW, tableSchemaPath, tableFieldname, inputFieldname, getRawSchema(context, tableSchemaPath, tableFieldname))); bottomExprs.add(inputRef); return (Prel) topRel.copy(topRel.getTraitSet(), Collections.singletonList((inputRel))); }}); final RelDataType bottomType = factory.createStructType(bottomProjectType, topRel.getRowType().getFieldNames()); final ProjectPrel newBottomProject = new ProjectPrel(topRel.getCluster(), topRel.getTraitSet(), inputRel, bottomExprs, bottomType); return new ConvertFromJsonPrel(topRel.getCluster(), topRel.getTraitSet(), topRel.getRowType(), newBottomProject, conversions);
private Prel visit(ExchangePrel hashPrel, List<DistributionTrait.DistributionField> fields, Prel child) { final List<String> childFields = child.getRowType().getFieldNames(); // Insert Project SqlOperatorImpl with new column that will be a hash for HashToRandomExchange fields final ProjectPrel addColumnprojectPrel = HashPrelUtil.addHashProject(fields, child, null); final Prel newPrel = (Prel) hashPrel.copy(addColumnprojectPrel.getTraitSet(), Collections.<RelNode>singletonList(addColumnprojectPrel)); int validRows = newPrel.getRowType().getFieldCount() - 1; final List<RelDataTypeField> all = newPrel.getRowType().getFieldList(); final List<RexNode> keptExprs = new ArrayList<>(validRows); final RexBuilder rexBuilder = newPrel.getCluster().getRexBuilder(); for(int i = 0; i < validRows; i++){ RexNode rex = rexBuilder.makeInputRef(all.get(i).getType(), i); keptExprs.add(rex); } // remove earlier inserted Project SqlOperatorImpl - since it creates issues down the road in HashJoin RelDataType removeRowType = RexUtil.createStructType(newPrel.getCluster().getTypeFactory(), keptExprs, childFields); return new ProjectPrel(newPrel.getCluster(), newPrel.getTraitSet(), newPrel, keptExprs, removeRowType); }
protected void doMatch(RelOptRuleCall call, InfoSchemaScanPrel scan, ProjectPrel project, FilterPrel filter) { if(scan.hasFilter()) { return; } PushdownResult result = ExpressionConverter.pushdown(scan.getCluster().getRexBuilder(), scan.getRowType(), filter.getCondition()); if(result.getQuery() == null) { return; //no filter pushdown ==> No transformation. } RelNode input = new InfoSchemaScanPrel(scan.getCluster(), scan.getTraitSet(), scan.getTable(), scan.getTableMetadata(), result.getQuery(), scan.getProjectedColumns(), scan.getObservedRowcountAdjustment()); if (project != null) { input = project.copy(project.getTraitSet(), input, project.getProjects(), filter.getRowType()); } if (result.getRemainder() == null) { // Filter can be removed as all expressions in the filter are converted and pushed to scan call.transformTo(input); } else { call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of(input))); } } }
protected void doPushFilterToScan(final RelOptRuleCall call, final FilterPrel filter, final ProjectPrel project, final HBaseScanPrel scan, final RexNode condition) { final LogicalExpression conditionExp = RexToExpr.toExpr(new ParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan.getRowType(), scan.getCluster().getRexBuilder(), condition); final HBaseFilterBuilder hbaseFilterBuilder = new HBaseFilterBuilder(TableNameGetter.getTableName(scan.getTableMetadata().getName()), scan.getStartRow(), scan.getStopRow(), scan.getFilter(), conditionExp); final HBaseScanSpec newScanSpec = hbaseFilterBuilder.parseTree(); if (newScanSpec == null) { return; //no filter pushdown ==> No transformation. } Predicate<DatasetSplit> predicate = newScanSpec.getRowKeyPredicate(); TableMetadata metadata = scan.getTableMetadata(); if(predicate != null) { try { metadata = metadata.prune(predicate); } catch (NamespaceException ex) { throw Throwables.propagate(ex); } } final HBaseScanPrel newScanPrel = new HBaseScanPrel(scan.getCluster(), scan.getTraitSet(), scan.getTable(), metadata, scan.getProjectedColumns(), scan.getObservedRowcountAdjustment(), newScanSpec.getStartRow(), newScanSpec.getStopRow(), newScanSpec.getSerializedFilter()); // Depending on whether is a project in the middle, assign either scan or copy of project to childRel. final RelNode childRel = project == null ? newScanPrel : project.copy(project.getTraitSet(), ImmutableList.of((RelNode)newScanPrel));; if (hbaseFilterBuilder.isAllExpressionsConverted()) { /* * Since we could convert the entire filter condition expression into an HBase filter, * we can eliminate the filter operator altogether. */ call.transformTo(childRel); } else { call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of(childRel))); } }
@Override public Iterator<Prel> iterator() { return PrelUtil.iter(getInput()); }
@Override public PrelWithDictionaryInfo visitProject(ProjectPrel projectPrel, Void value) throws RuntimeException { assert projectPrel.getInputs().size() == 1; PrelWithDictionaryInfo newInput = ((Prel)projectPrel.getInput()).accept(this, value); if (projectPrel.getInput() == newInput.getPrel()) { return new PrelWithDictionaryInfo(projectPrel); for (int i = 0; i < projectPrel.getProjects().size(); ++i) { final RexNode expr = projectPrel.getProjects().get(i); if (!(expr instanceof RexInputRef)) { expr.accept(visitor); final GlobalDictionaryFieldInfo[] reorderedFields = new GlobalDictionaryFieldInfo[projectPrel.getRowType().getFieldCount()]; final List<RelDataTypeField> newFields = Lists.newArrayList(); final Set<Integer> dictionaryEncodedPassThroughFields = Sets.newHashSet(); for (RelDataTypeField field : projectPrel.getRowType().getFieldList()) { final RelDataType rowDataType = PrelWithDictionaryInfo.toRowDataType(newFields, projectPrel.getCluster().getTypeFactory()); final List<RexNode> newExprs = Lists.newArrayList(); int exprIndex = 0; for (RexNode expr : projectPrel.getProjects()) { if (expr instanceof RexInputRef) { newExprs.add(new RexInputRef(((RexInputRef) expr).getIndex(), rowDataType.getFieldList().get(exprIndex).getType())); (Prel)projectPrel.copy(projectPrel.getTraitSet(), newInput.getPrel(), newExprs, rowDataType), reorderedFields);
private Prel newProject(List<RexNode> exprs, RelDataType rowType, Prel child) { return new ProjectPrel(cluster, traits, child, exprs, rowType); }
private Prel addTrivialOrderedProjectPrel(Prel prel) { RelDataType t = prel.getRowType(); RexBuilder b = prel.getCluster().getRexBuilder(); List<RexNode> projections = Lists.newArrayList(); int projectCount = t.getFieldList().size(); // no point in reordering if we only have one column if (projectCount < 2) { return prel; } for (int i = 0; i < projectCount; i++) { projections.add(b.makeInputRef(prel, i)); } return new ProjectPrel(prel.getCluster(), prel.getTraitSet(), prel, projections, prel.getRowType()); }
private RelNode rename(RelNode input, List<RelDataTypeField> inputFields, List<String> outputFieldNames) { List<RexNode> exprs = Lists.newArrayList(); for (RelDataTypeField field : inputFields) { RexNode expr = input.getCluster().getRexBuilder().makeInputRef(field.getType(), field.getIndex()); exprs.add(expr); } RelDataType rowType = RexUtil.createStructType(input.getCluster().getTypeFactory(), exprs, outputFieldNames); ProjectPrel proj = new ProjectPrel(input.getCluster(), input.getTraitSet(), input, exprs, rowType); return proj; }
private Prel renameAsNecessary(RelDataType expectedRowType, Prel initialInput) { if(RelOptUtil.areRowTypesEqual(initialInput.getRowType(), expectedRowType, false) && !RelOptUtil.areRowTypesEqual(initialInput.getRowType(), expectedRowType, true)) { final List<RexNode> refs = new ArrayList<>(); final List<RelDataTypeField> fields = expectedRowType.getFieldList(); final RexBuilder rb = initialInput.getCluster().getRexBuilder(); for(int i = 0; i < expectedRowType.getFieldCount(); i++) { refs.add(rb.makeInputRef(fields.get(i).getType(), i)); } return new ProjectPrel(initialInput.getCluster(), initialInput.getTraitSet(), initialInput, refs, expectedRowType); } else { return initialInput; } }
output = new ProjectPrel(output.getCluster(), poppedProject.getTraitSet(), output, poppedProject.getChildExps(), poppedProject.getRowType());
public static ProjectPrel addHashProject(List<DistributionField> distFields, Prel input, Integer ringCount){ // Insert Project SqlOperatorImpl with new column that will be a hash for HashToRandomExchange fields final List<String> outputFieldNames = Lists.newArrayList(input.getRowType().getFieldNames()); final String fieldName = ringCount == null ? HashPrelUtil.HASH_EXPR_NAME : WriterPrel.BUCKET_NUMBER_FIELD; outputFieldNames.add(fieldName); final RexBuilder rexBuilder = input.getCluster().getRexBuilder(); final List<RelDataTypeField> childRowTypeFields = input.getRowType().getFieldList(); // create new hashed field. final HashExpressionCreatorHelper<RexNode> hashHelper = new RexNodeBasedHashExpressionCreatorHelper(rexBuilder); final List<RexNode> distFieldRefs = Lists.newArrayListWithExpectedSize(distFields.size()); for(int i = 0; i < distFields.size(); i++) { final int fieldId = distFields.get(i).getFieldId(); distFieldRefs.add(rexBuilder.makeInputRef(childRowTypeFields.get(fieldId).getType(), fieldId)); } final List <RexNode> updatedExpr = Lists.newArrayListWithExpectedSize(childRowTypeFields.size() + 1); for ( RelDataTypeField field : childRowTypeFields) { RexNode rex = rexBuilder.makeInputRef(field.getType(), field.getIndex()); updatedExpr.add(rex); } RexNode hashExpression = HashPrelUtil.createHashBasedPartitionExpression(distFieldRefs, hashHelper); if(ringCount != null){ RelDataType intType = input.getCluster().getTypeFactory().createSqlType(SqlTypeName.INTEGER); hashExpression = rexBuilder.makeCall(SqlStdOperatorTable.MOD, ImmutableList.of(hashExpression, rexBuilder.makeExactLiteral(BigDecimal.valueOf(ringCount), intType))); hashExpression = rexBuilder.makeCall(SqlStdOperatorTable.ABS, Collections.singletonList(hashExpression)); } updatedExpr.add(hashExpression); RelDataType rowType = RexUtil.createStructType(input.getCluster().getTypeFactory(), updatedExpr, outputFieldNames); ProjectPrel addColumnprojectPrel = new ProjectPrel(input.getCluster(), input.getTraitSet(), input, updatedExpr, rowType); return addColumnprojectPrel; }
@Override public Prel visitLeaf(LeafPrel scanPrel, Void value) throws RuntimeException { if (StarColumnHelper.containsStarColumn(scanPrel.getRowType()) && prefixedForStar ) { List<RexNode> exprs = Lists.newArrayList(); for (RelDataTypeField field : scanPrel.getRowType().getFieldList()) { RexNode expr = scanPrel.getCluster().getRexBuilder().makeInputRef(field.getType(), field.getIndex()); exprs.add(expr); } List<String> fieldNames = Lists.newArrayList(); long tableId = tableNumber.getAndIncrement(); for (String name : scanPrel.getRowType().getFieldNames()) { if (StarColumnHelper.isNonPrefixedStarColumn(name)) { fieldNames.add("T" + tableId + StarColumnHelper.PREFIX_DELIMITER + name); // Add prefix to * column. } else { fieldNames.add(name); // Keep regular column as it is. } } RelDataType rowType = RexUtil.createStructType(scanPrel.getCluster().getTypeFactory(), exprs, fieldNames); // insert a PAS. ProjectPrel proj = new ProjectPrel(scanPrel.getCluster(), scanPrel.getTraitSet(), scanPrel, exprs, rowType); return proj; } else { return visitPrel(scanPrel, value); } }
@Override public RelNode convertChild(ProjectRel project, RelNode rel) throws RuntimeException { DistributionTrait childDist = rel.getTraitSet().getTrait(DistributionTraitDef.INSTANCE); RelCollation childCollation = rel.getTraitSet().getTrait(RelCollationTraitDef.INSTANCE); DistributionTrait newDist = convertDist(childDist, distributionMap); RelCollation newCollation = convertRelCollation(childCollation, collationMap); RelTraitSet newProjectTraits = newTraitSet(Prel.PHYSICAL, newDist, newCollation); return new ProjectPrel(project.getCluster(), newProjectTraits, rel, project.getProjects(), project.getRowType()); }
private Prel insertProjUnderScreenOrWriter(Prel prel, RelDataType origRowType, Prel child) { ProjectPrel proj = null; List<RelNode> children = Lists.newArrayList(); List<RexNode> exprs = Lists.newArrayList(); for (int i = 0; i < origRowType.getFieldCount(); i++) { RexNode expr = child.getCluster().getRexBuilder().makeInputRef(origRowType.getFieldList().get(i).getType(), i); exprs.add(expr); } RelDataType newRowType = RexUtil.createStructType(child.getCluster().getTypeFactory(), exprs, origRowType.getFieldNames()); int fieldCount = prel.getRowType().isStruct()? prel.getRowType().getFieldCount():1; // Insert PUS/PUW : remove the prefix and keep the original field name. if (fieldCount > 1) { // // no point in allowing duplicates if we only have one column proj = new ProjectAllowDupPrel(child.getCluster(), child.getTraitSet(), child, exprs, newRowType); } else { proj = new ProjectPrel(child.getCluster(), child.getTraitSet(), child, exprs, newRowType); } children.add(proj); return (Prel) prel.copy(prel.getTraitSet(), children); } @Override