@Override public PhysicalPlan getPlan(SqlHandlerConfig config, String sql, SqlNode sqlNode) throws Exception { try{ final ConvertedRelNode convertedRelNode = PrelTransformer.validateAndConvert(config, sqlNode); final RelDataType validatedRowType = convertedRelNode.getValidatedRowType(); final RelNode queryRelNode = convertedRelNode.getConvertedNode(); final Rel drel = PrelTransformer.convertToDrel(config, queryRelNode, validatedRowType); final Pair<Prel, String> convertToPrel = PrelTransformer.convertToPrel(config, drel); final Prel prel = convertToPrel.getKey(); textPlan = convertToPrel.getValue(); final PhysicalOperator pop = PrelTransformer.convertToPop(config, prel); final PhysicalPlan plan = PrelTransformer.convertToPlan(config, pop); logger.debug("Final Physical Plan {}", textPlan); PrelTransformer.log(config, "Dremio Plan", plan, logger); return plan; }catch(Exception ex){ throw SqlExceptionHelper.coerceException(logger, sql, ex, true); } }
public static ConvertedRelNode validateAndConvert(SqlHandlerConfig config, SqlNode sqlNode, RelTransformer relTransformer) throws ForemanSetupException, RelConversionException, ValidationException { final Pair<SqlNode, RelDataType> validatedTypedSqlNode = validateNode(config, sqlNode); final SqlNode validated = validatedTypedSqlNode.getKey(); final RelNode rel = convertToRel(config, validated, relTransformer); final RelNode preprocessedRel = preprocessNode(config, rel); return new ConvertedRelNode(preprocessedRel, validatedTypedSqlNode.getValue()); }
private static RelNode convertToRel(SqlHandlerConfig config, SqlNode node, RelTransformer relTransformer) throws RelConversionException { final RelNode rel = convertToRelRootAndJdbc(config, node, relTransformer); log("INITIAL", rel, logger, null); return transform(config, PlannerType.HEP, PlannerPhase.WINDOW_REWRITE, rel, rel.getTraitSet(), true); }
final ConvertedRelNode convertedRelNode = PrelTransformer.validateAndConvert(config, innerNode); final RelDataType validatedRowType = convertedRelNode.getValidatedRowType(); final RelNode queryRelNode = convertedRelNode.getConvertedNode(); PrelTransformer.log("Calcite", queryRelNode, logger, null); drel = PrelTransformer.convertToDrel(config, queryRelNode, validatedRowType); final Pair<Prel, String> convertToPrel = PrelTransformer.convertToPrel(config, drel); final String text = convertToPrel.getValue(); return Collections.singletonList(new Explain(text));
public static ConvertedRelNode validateAndConvert(SqlHandlerConfig config, SqlNode sqlNode) throws ForemanSetupException, RelConversionException, ValidationException { return validateAndConvert(config, sqlNode, NO_OP_TRANSFORMER); }
/** * Return Dremio Logical RelNode tree for a SELECT statement, when it is executed / explained directly. * * @param relNode : root RelNode corresponds to Calcite Logical RelNode. * @param validatedRowType : the rowType for the final field names. A rename project may be placed on top of the root. * @return * @throws RelConversionException * @throws SqlUnsupportedException */ public static Rel convertToDrel(SqlHandlerConfig config, RelNode relNode, RelDataType validatedRowType) throws RelConversionException, SqlUnsupportedException { Rel convertedRelNode = convertToDrel(config, relNode); // Put a non-trivial topProject to ensure the final output field name is preserved, when necessary. convertedRelNode = addRenamedProject(config, convertedRelNode, validatedRowType); convertedRelNode = SqlHandlerUtil.storeQueryResultsIfNeeded(config.getConverter().getParserConfig(), config.getContext(), convertedRelNode); return new ScreenRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode); }
private static RelNode convertToRelRootAndJdbc(SqlHandlerConfig config, SqlNode node, RelTransformer relTransformer) throws RelConversionException { final RelNode convertible = toConvertibleRelRoot(config, node, false, relTransformer); } else { convertedNodeWithoutRexSubquery = toConvertibleRelRoot(config, node, true, relTransformer); if (!checker.canPushdownRexSubQuery()) { final RelNode injectJdbcLogical = ExpansionNode.removeFromTree(convertedNode.accept(new InjectSample(leafLimitEnabled))); final RelNode jdbcPushedPartial = transform(config, PlannerType.HEP_AC, PlannerPhase.JDBC_PUSHDOWN, injectJdbcLogical, injectJdbcLogical.getTraitSet(), false); log("Failed to pushdown RexSubquery. Applying JDBC pushdown to query with IN/EXISTS/SCALAR sub-queries converted to joins.", jdbcPushed, logger, null); final RelNode expandedWithSample = convertedNodeWithoutRexSubquery.accept(new InjectSample(leafLimitEnabled)); finalConvertedNode = transform(config,PlannerType.HEP_AC, PlannerPhase.JDBC_PUSHDOWN, expandedWithSample, expandedWithSample.getTraitSet(), false).accept(new ConvertJdbcLogicalToJdbcRel()); } else {
protected static void log(final PlannerType plannerType, final PlannerPhase phase, final RelNode node, final Logger logger, Stopwatch watch) { if (logger.isDebugEnabled()) { log(plannerType.name() + ":" + phase.description, node, logger, watch); } }
@Override public Prel visitPrel(Prel prel, Void value) throws IOException { if (prel instanceof LimitPrel) { LimitPrel limit = (LimitPrel) prel; if(isLimit0(limit.getFetch())){ PhysicalOperator op = PrelTransformer.convertToPop(config, prel); BatchSchema schema = op.getSchema(config.getContext().getFunctionRegistry()); // make sure to remove any selection vector modes since we're now the leaf node. schema = schema.clone(SelectionVectorMode.NONE); return new EmptyPrel(prel.getCluster(), prel.getTraitSet(), prel.getRowType(), schema); } } List<RelNode> children = new ArrayList<>(); for(Prel child : prel){ children.add(child.accept(this, null)); } return (Prel) prel.copy(prel.getTraitSet(), children); }
final Stopwatch watch = Stopwatch.createStarted(); final RelNode relNode = transform(config, PlannerType.VOLCANO, PlannerPhase.PHYSICAL, drel, traits, true); log(PlannerType.VOLCANO, PlannerPhase.PHYSICAL, phyRelNode, logger, watch); } catch (RelOptPlanner.CannotPlanException ex) { logger.error(ex.getMessage());
private static Rel convertToDrel( SqlHandlerConfig config, RelNode relNode, Catalog catalog, NamespaceKey key, WriterOptions options, RelDataType queryRowType, final Map<String, Object> storageOptions) throws RelConversionException, SqlUnsupportedException { Rel convertedRelNode = PrelTransformer.convertToDrel(config, relNode); // Put a non-trivial topProject to ensure the final output field name is preserved, when necessary. // Only insert project when the field count from the child is same as that of the queryRowType. convertedRelNode = new WriterRel(convertedRelNode.getCluster(), convertedRelNode.getCluster().traitSet().plus(Rel.LOGICAL), convertedRelNode, catalog.createNewTable(key, options, storageOptions), queryRowType); convertedRelNode = SqlHandlerUtil.storeQueryResultsIfNeeded(config.getConverter().getParserConfig(), config.getContext(), convertedRelNode); return new ScreenRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode); }
private RelNode getPlan(SqlHandlerConfig sqlHandlerConfig, List<String> refreshTablePath, PlanNormalizer planNormalizer) { SqlSelect select = new SqlSelect( SqlParserPos.ZERO, new SqlNodeList(SqlParserPos.ZERO), new SqlNodeList(ImmutableList.<SqlNode>of(SqlIdentifier.star(SqlParserPos.ZERO)), SqlParserPos.ZERO), new SqlIdentifier(refreshTablePath, SqlParserPos.ZERO), null, null, null, null, null, null, null ); try { ConvertedRelNode converted = PrelTransformer.validateAndConvert(sqlHandlerConfig, select, planNormalizer); return converted.getConvertedNode(); } catch (ForemanSetupException | RelConversionException | ValidationException e) { throw Throwables.propagate(SqlExceptionHelper.coerceException(logger, select.toString(), e, false)); } }
private static RelNode doTransform(SqlHandlerConfig config, final PlannerType plannerType, final PlannerPhase phase, final RelOptPlanner planner, final RelNode input, boolean log, Supplier<RelNode> toPlan) { final Stopwatch watch = Stopwatch.createStarted(); try { final RelNode output = toPlan.get(); if (log) { log(plannerType, phase, output, logger, watch); config.getObserver().planRelTransform(phase, planner, input, output, watch.elapsed(TimeUnit.MILLISECONDS)); } CALCITE_LOGGER.trace("Completed Phase: {}.", phase); return output; } catch (Throwable t) { // log our input state as oput anyway so we can ensure that we have details. try { log(plannerType, phase, input, logger, watch); config.getObserver().planRelTransform(phase, planner, input, input, watch.elapsed(TimeUnit.MILLISECONDS)); } catch (Throwable unexpected) { t.addSuppressed(unexpected); } throw t; } }
final ConvertedRelNode convertedRelNode = PrelTransformer.validateAndConvert(config, sqlCreateTable.getQuery()); final RelDataType validatedRowType = convertedRelNode.getValidatedRowType(); final RelNode queryRelNode = convertedRelNode.getConvertedNode(); PrelTransformer.log("Calcite", newTblRelNodeWithPCol, logger, null); createStorageOptionsMap(config, sqlCreateTable.getFormatOptions())); final Pair<Prel, String> convertToPrel = PrelTransformer.convertToPrel(config, drel); final Prel prel = convertToPrel.getKey(); textPlan = convertToPrel.getValue(); PhysicalOperator pop = PrelTransformer.convertToPop(config, prel); PhysicalPlan plan = PrelTransformer.convertToPlan(config, pop); PrelTransformer.log(config, "Dremio Plan", plan, logger);
ConvertedRelNode converted = PrelTransformer.validateAndConvert(sqlHandlerConfig, select, planNormalizer);
public static RelNode trimFields(final RelNode relNode, boolean shouldLog) { final Stopwatch w = Stopwatch.createStarted(); final RelFieldTrimmer trimmer = DremioFieldTrimmer.of(relNode.getCluster()); final RelNode trimmed = trimmer.trim(relNode); if(shouldLog) { log(PlannerType.HEP, PlannerPhase.FIELD_TRIMMING, trimmed, logger, w); } return trimmed; }
final Rel drel = PrelTransformer.convertToDrel(config, initial); final Set<String> fields = ImmutableSet.copyOf(drel.getRowType().getFieldNames()); final long ringCount = config.getContext().getOptions().getOption(PlannerSettings.RING_COUNT); final Pair<Prel, String> convertToPrel = PrelTransformer.convertToPrel(config, screen); final Prel prel = convertToPrel.getKey(); this.textPlan = convertToPrel.getValue(); PhysicalOperator pop = PrelTransformer.convertToPop(config, prel); PhysicalPlan plan = PrelTransformer.convertToPlan(config, pop); PrelTransformer.log(config, "Dremio Plan", plan, logger);
final ConvertedRelNode convertedRelNode = PrelTransformer.validateAndConvert(config, createView.getQuery()); final RelDataType validatedRowType = convertedRelNode.getValidatedRowType(); final RelNode queryRelNode = convertedRelNode.getConvertedNode();
final Rel drel = PrelTransformer.convertToDrel(config, initial); final Set<String> fields = ImmutableSet.copyOf(drel.getRowType().getFieldNames()); final Pair<Prel, String> convertToPrel = PrelTransformer.convertToPrel(config, screen); final Prel prel = convertToPrel.getKey(); this.textPlan = convertToPrel.getValue(); PhysicalOperator pop = PrelTransformer.convertToPop(config, prel); PhysicalPlan plan = PrelTransformer.convertToPlan(config, pop); if (logger.isTraceEnabled()) { PrelTransformer.log(config, "Dremio Plan", plan, logger);
final SqlHandlerConfig config = new SqlHandlerConfig(queryContext, converter, observer, null); final ConvertedRelNode convertedRelNode = PrelTransformer.validateAndConvert(config, node); final RelDataType validatedRowType = convertedRelNode.getValidatedRowType(); final RelNode queryRelNode = convertedRelNode.getConvertedNode(); final Rel drel = PrelTransformer.convertToDrel(config, queryRelNode, validatedRowType); final Pair<Prel, String> convertToPrel = PrelTransformer.convertToPrel(config, drel); final Prel prel = convertToPrel.getKey(); final String prePhysicaltextPlan = convertToPrel.getValue(); assertThat(prePhysicaltextPlan, CoreMatchers.containsString("EasyScan")); final PhysicalOperator pop = PrelTransformer.convertToPop(config, prel); final PhysicalPlan plan = PrelTransformer.convertToPlan(config, pop); final String postPhysicaltextPlan = plan.unparse(config.getContext().getLpPersistence().getMapper().writer());