public ExplainHandler(SqlHandlerConfig config) { super(); this.config = new SqlHandlerConfig(config.getContext(), config.getConverter(), config.getObserver(), config.getMaterializations().orNull()); }
private static RelNode toConvertibleRelRoot(SqlHandlerConfig config, final SqlNode validatedNode, boolean expand, RelTransformer relTransformer) { final Stopwatch stopwatch = Stopwatch.createStarted(); final RelRootPlus convertible = config.getConverter().toConvertibleRelRoot(validatedNode, expand); config.getObserver().planConvertedToRel(convertible.rel, stopwatch.elapsed(TimeUnit.MILLISECONDS)); final RelNode reduced = relTransformer.transform(transform(config, PlannerType.HEP, PlannerPhase.REDUCE_EXPRESSIONS, convertible.rel, convertible.rel.getTraitSet(), true)); config.getObserver().planSerializable(reduced); return reduced; }
private static Pair<SqlNode, RelDataType> validateNode(SqlHandlerConfig config, final SqlNode sqlNode) throws ValidationException, RelConversionException, ForemanSetupException { final Stopwatch stopwatch = Stopwatch.createStarted(); final SqlNode sqlNodeValidated; try { sqlNodeValidated = config.getConverter().validate(sqlNode); } catch (final Throwable ex) { throw new ValidationException("unable to validate sql node", ex); } final Pair<SqlNode, RelDataType> typedSqlNode = new Pair<>(sqlNodeValidated, config.getConverter().getOutputType(sqlNodeValidated)); // Check if the unsupported functionality is used UnsupportedOperatorsVisitor visitor = UnsupportedOperatorsVisitor.createVisitor(config.getContext()); try { sqlNodeValidated.accept(visitor); } catch (UnsupportedOperationException ex) { // If the exception due to the unsupported functionalities visitor.convertException(); // If it is not, let this exception move forward to higher logic throw ex; } config.getObserver().planValidated(typedSqlNode.getValue(), typedSqlNode.getKey(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); return typedSqlNode; }
PlanNormalizer(SqlHandlerConfig sqlHandlerConfig) { this.optionManager = sqlHandlerConfig.getContext().getOptions(); }
public SqlHandlerConfig cloneWithNewObserver(AttemptObserver replacementObserver){ return new SqlHandlerConfig(this.context, this.converter, replacementObserver, this.materializations); }
final RelNode intermediateNode = transform(config, PlannerType.VOLCANO, PlannerPhase.LOGICAL, preLog, logicalTraits, true); final Catalog catalog = config.getContext().getCatalog(); if (catalog instanceof CachingCatalog) { config.getObserver().tablesCollected(catalog.getAllRequestedTables()); final Optional<SubstitutionInfo> acceleration = findUsedMaterializations(config, drel); if (acceleration.isPresent()) { config.getObserver().planAccelerated(acceleration.get());
RelTraitSet targetTraits, boolean log) { final RuleSet rules = config.getRules(phase); final RelTraitSet toTraits = targetTraits.simplify(); final RelOptPlanner planner; SqlConverter converter = config.getConverter(); final HepPlanner hepPlanner = new DremioHepPlanner(hepPgmBldr.build(), config.getContext().getPlannerSettings(), converter.getCostFactory(), phase); hepPlanner.setExecutor(new ConstExecutor(converter.getFunctionImplementationRegistry(), converter.getFunctionContext(), converter.getSettings())); final AccelerationAwareSubstitutionProvider substitutions = config.getConverter().getSubstitutionProvider(); substitutions.setObserver(config.getObserver()); substitutions.setEnabled(phase.useMaterializations); substitutions.setPostSubstitutionTransformer( final HepProgramBuilder builder = HepProgram.builder(); builder.addMatchOrder(HepMatchOrder.ARBITRARY); builder.addRuleCollection(Lists.newArrayList(config.getRules(PlannerPhase.POST_SUBSTITUTION)));
queryContext.getScanResult()); final SqlNode node = converter.parse(sql); final SqlHandlerConfig config = new SqlHandlerConfig(queryContext, converter, observer, null); final String postPhysicaltextPlan = plan.unparse(config.getContext().getLpPersistence().getMapper().writer());
/** * Return Dremio Logical RelNode tree for a SELECT statement, when it is executed / explained directly. * * @param relNode : root RelNode corresponds to Calcite Logical RelNode. * @param validatedRowType : the rowType for the final field names. A rename project may be placed on top of the root. * @return * @throws RelConversionException * @throws SqlUnsupportedException */ public static Rel convertToDrel(SqlHandlerConfig config, RelNode relNode, RelDataType validatedRowType) throws RelConversionException, SqlUnsupportedException { Rel convertedRelNode = convertToDrel(config, relNode); // Put a non-trivial topProject to ensure the final output field name is preserved, when necessary. convertedRelNode = addRenamedProject(config, convertedRelNode, validatedRowType); convertedRelNode = SqlHandlerUtil.storeQueryResultsIfNeeded(config.getConverter().getParserConfig(), config.getContext(), convertedRelNode); return new ScreenRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode); }
Iterable<DremioTable> requestedTables = sqlHandlerConfig.getContext().getCatalog().getAllRequestedTables(); final RelSerializerFactory serializerFactory = RelSerializerFactory.getFactory(config, sqlHandlerConfig.getScanResult());
private static RelNode doTransform(SqlHandlerConfig config, final PlannerType plannerType, final PlannerPhase phase, final RelOptPlanner planner, final RelNode input, boolean log, Supplier<RelNode> toPlan) { final Stopwatch watch = Stopwatch.createStarted(); try { final RelNode output = toPlan.get(); if (log) { log(plannerType, phase, output, logger, watch); config.getObserver().planRelTransform(phase, planner, input, output, watch.elapsed(TimeUnit.MILLISECONDS)); } CALCITE_LOGGER.trace("Completed Phase: {}.", phase); return output; } catch (Throwable t) { // log our input state as oput anyway so we can ensure that we have details. try { log(plannerType, phase, input, logger, watch); config.getObserver().planRelTransform(phase, planner, input, input, watch.elapsed(TimeUnit.MILLISECONDS)); } catch (Throwable unexpected) { t.addSuppressed(unexpected); } throw t; } }
if (!config.getMaterializations().isPresent()) { return Optional.absent(); final MaterializationList table = config.getMaterializations().get(); root.accept(new StatelessRelShuttleImpl() { @Override
@Override public double plan() throws Exception { try{ final RecordingObserver recording = new RecordingObserver(); final AttemptObservers observers = AttemptObservers.of(observer, recording); observers.planStart(sql); plan = handler.getPlan(config.cloneWithNewObserver(observers), sql, sqlNode); PreparedPlan prepared = new PreparedPlan(context.getQueryId(), context.getQueryUserName(), sql, plan, recording); final Long handle = PREPARE_ID.getAndIncrement(); state = ServerPreparedStatementState.newBuilder() .setHandle(handle) .setSqlQuery(sql) .setPrepareId(context.getQueryId()) .build(); planCache.put(handle, prepared); // record a partial plan so that we can grab metadata and use it (for example during view creation of via sql). observers.planCompleted(new ExecutionPlan(plan, ImmutableList.of(), ImmutableList.of())); return 1; }catch(Exception ex){ throw SqlExceptionHelper.coerceException(logger, sql, ex, true); } }
final SqlCompactMaterialization compact = SqlNodeUtil.unwrap(sqlNode, SqlCompactMaterialization.class); if(!SystemUser.SYSTEM_USERNAME.equals(config.getContext().getQueryUserName())) { throw SqlExceptionHelper.parseError("$COMPACT REFRESH not supported.", sql, compact.getParserPosition()) .build(logger); final AttemptObserver observer = config.getObserver(); ReflectionService service = config.getContext().getAccelerationManager().unwrap(ReflectionService.class); observer.planValidated(RecordWriter.SCHEMA.toCalciteRecordType(config.getConverter().getCluster().getTypeFactory()), compact, watch.elapsed(TimeUnit.MILLISECONDS)); final long ringCount = config.getContext().getOptions().getOption(PlannerSettings.RING_COUNT); final Rel writerDrel = new WriterRel(drel.getCluster(), drel.getCluster().traitSet().plus(Rel.LOGICAL), drel, config.getContext().getCatalog().createNewTable( new NamespaceKey(ReflectionUtils.getMaterializationPath(newMaterialization)), getWriterOptions((int) ringCount, goal, fields), ImmutableMap.of()), ); final RelNode doubleWriter = SqlHandlerUtil.storeQueryResultsIfNeeded(config.getConverter().getParserConfig(), config.getContext(), writerDrel);
public ReflectionPlanNormalizer( SqlHandlerConfig sqlHandlerConfig, ReflectionGoal goal, ReflectionEntry entry, Materialization materialization, NamespaceService namespace, SabotConfig config, ReflectionSettings reflectionSettings, MaterializationStore materializationStore) { this.sqlHandlerConfig = sqlHandlerConfig; this.goal = goal; this.entry = entry; this.materialization = materialization; this.namespace = namespace; this.config = config; this.reflectionSettings = reflectionSettings; this.materializationStore = materializationStore; this.optionManager = sqlHandlerConfig.getContext().getOptions(); }
final SqlHandlerConfig config = new SqlHandlerConfig(context, converter, observer, null); NormalHandler handler = new NormalHandler(); PhysicalPlan pp = handler.getPlan(config, query.getSql(), sqlNode);
QueryContext context = config.getContext(); OptionManager queryOptions = context.getOptions(); phyRelNode = SimpleLimitExchangeRemover.apply(config.getContext().getPlannerSettings(), phyRelNode); long targetSliceSize = config.getContext().getPlannerSettings().getSliceTarget(); phyRelNode = ExcessiveExchangeIdentifier.removeExcessiveEchanges(phyRelNode, targetSliceSize); if (logger.isDebugEnabled() || config.getObserver() != null) { textPlan = PrelSequencer.setPlansWithIds(phyRelNode, SqlExplainLevel.ALL_ATTRIBUTES, config.getObserver(), finalPrelTimer.elapsed(TimeUnit.MILLISECONDS)); if (logger.isDebugEnabled()) { logger.debug(String.format("%s:\n%s", "Final Physical Transformation", textPlan)); config.getObserver().finalPrel(phyRelNode); return Pair.of(phyRelNode, textPlan);
private static Rel convertToDrel( SqlHandlerConfig config, RelNode relNode, Catalog catalog, NamespaceKey key, WriterOptions options, RelDataType queryRowType, final Map<String, Object> storageOptions) throws RelConversionException, SqlUnsupportedException { Rel convertedRelNode = PrelTransformer.convertToDrel(config, relNode); // Put a non-trivial topProject to ensure the final output field name is preserved, when necessary. // Only insert project when the field count from the child is same as that of the queryRowType. convertedRelNode = new WriterRel(convertedRelNode.getCluster(), convertedRelNode.getCluster().traitSet().plus(Rel.LOGICAL), convertedRelNode, catalog.createNewTable(key, options, storageOptions), queryRowType); convertedRelNode = SqlHandlerUtil.storeQueryResultsIfNeeded(config.getConverter().getParserConfig(), config.getContext(), convertedRelNode); return new ScreenRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode); }
final SqlRefreshReflection materialize = SqlNodeUtil.unwrap(sqlNode, SqlRefreshReflection.class); if(!SystemUser.SYSTEM_USERNAME.equals(config.getContext().getQueryUserName())) { throw SqlExceptionHelper.parseError("$MATERIALIZE not supported.", sql, materialize.getParserPosition()).build(logger); final AttemptObserver observer = config.getObserver(); ReflectionService service = config.getContext().getAccelerationManager().unwrap(ReflectionService.class); observer.planValidated(RecordWriter.SCHEMA.toCalciteRecordType(config.getConverter().getCluster().getTypeFactory()), materialize, watch.elapsed(TimeUnit.MILLISECONDS)); service.getExcludedReflectionsProvider(), namespace, new ExtendedToRelContext(config.getConverter()), config.getContext().getConfig(), reflectionSettings, materializationStore); final UserBitShared.QueryId queryId = config.getContext().getQueryId(); final AttemptId attemptId = AttemptId.of(queryId); drel, config.getContext().getCatalog().createNewTable( new NamespaceKey(tablePath), getWriterOptions(0, goal, fields), ImmutableMap.of() ), initial.getRowType()); final RelNode doubleWriter = SqlHandlerUtil.storeQueryResultsIfNeeded(config.getConverter().getParserConfig(), config.getContext(), writerDrel);
public static void log(final SqlHandlerConfig config, final String name, final PhysicalPlan plan, final Logger logger) throws JsonProcessingException { if (logger.isDebugEnabled()) { String planText = plan.unparse(config.getContext().getLpPersistence().getMapper().writer()); logger.debug(name + " : \n" + planText); } }