@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { Operator op = (Operator) nd; String opKey = op.getOperatorId(); Operator<?> found = opMap.get(opKey); if (found != null) { throw new RuntimeException("operator id reuse found: " + opKey); } opMap.put(opKey, op); return null; } }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { Operator<?> op = (Operator<?>) nd; ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx; cppCtx.getOpToConstantExprs().put(op, new HashMap<ColumnInfo, ExprNodeDesc>()); if (LOG.isDebugEnabled()) { LOG.debug("Stop propagate constants on op " + op.getOperatorId()); } return null; } }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { Operator<?> op = (Operator<?>) nd; ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx; cppCtx.getOpToConstantExprs().put(op, new HashMap<ColumnInfo, ExprNodeDesc>()); if (LOG.isDebugEnabled()) { LOG.debug("Stop propagate constants on op " + op.getOperatorId()); } return null; } }
private static void annotateRuntimeStats(Operator<? extends OperatorDesc> op, ParseContext pctx) { Long runTimeNumRows = pctx.getContext().getExplainConfig().getOpIdToRuntimeNumRows() .get(op.getOperatorId()); if (op.getConf() != null && op.getConf().getStatistics() != null && runTimeNumRows != null) { LOG.info("annotateRuntimeStats for " + op.getOperatorId()); op.getConf().getStatistics().setRunTimeNumRows(runTimeNumRows); } else { LOG.debug("skip annotateRuntimeStats for " + op.getOperatorId()); } }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { ReduceSinkOperator rs = (ReduceSinkOperator) nd; List<Operator<? extends OperatorDesc>> children = rs.getChildOperators(); List<String> outputOperators = new ArrayList<>(); for (Operator<? extends OperatorDesc> operator : children) { outputOperators.add(operator.getOperatorId()); } rs.getConf().setOutputOperators(outputOperators); return null; } }
private static void annotateRuntimeStats(Operator<? extends OperatorDesc> op, ParseContext pctx) { Long runTimeNumRows = pctx.getContext().getExplainConfig().getOpIdToRuntimeNumRows() .get(op.getOperatorId()); if (op.getConf() != null && op.getConf().getStatistics() != null && runTimeNumRows != null) { LOG.info("annotateRuntimeStats for " + op.getOperatorId()); op.getConf().getStatistics().setRunTimeNumRows(runTimeNumRows); } else { LOG.debug("skip annotateRuntimeStats for " + op.getOperatorId()); } }
statsMap.put(Counter.RECORDS_OUT_OPERATOR.name() + "_" + getOperatorId(), runTimeRowsWritable); statsMap.put(getCounterName(Counter.RECORDS_OUT_INTERMEDIATE, hconf), recordCounter); this.runTimeNumRows = 0;
private static void setRuntimeStatsDir(Operator<? extends OperatorDesc> op, ParseContext pctx) throws SemanticException { try { OperatorDesc conf = op.getConf(); if (conf != null) { LOG.info("setRuntimeStatsDir for " + op.getOperatorId()); String path = new Path(pctx.getContext().getExplainConfig().getExplainRootPath(), op.getOperatorId()).toString(); StatsPublisher statsPublisher = new FSStatsPublisher(); StatsCollectionContext runtimeStatsContext = new StatsCollectionContext(pctx.getConf()); runtimeStatsContext.setStatsTmpDir(path); if (!statsPublisher.init(runtimeStatsContext)) { LOG.error("StatsPublishing error: StatsPublisher is not initialized."); throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } conf.setRuntimeStatsTmpDir(path); } else { LOG.debug("skip setRuntimeStatsDir for " + op.getOperatorId() + " because OperatorDesc is null"); } } catch (HiveException e) { throw new SemanticException(e); } }
private static void setRuntimeStatsDir(Operator<? extends OperatorDesc> op, ParseContext pctx) throws SemanticException { try { OperatorDesc conf = op.getConf(); if (conf != null) { LOG.info("setRuntimeStatsDir for " + op.getOperatorId()); String path = new Path(pctx.getContext().getExplainConfig().getExplainRootPath(), op.getOperatorId()).toString(); StatsPublisher statsPublisher = new FSStatsPublisher(); StatsCollectionContext runtimeStatsContext = new StatsCollectionContext(pctx.getConf()); runtimeStatsContext.setStatsTmpDir(path); if (!statsPublisher.init(runtimeStatsContext)) { LOG.error("StatsPublishing error: StatsPublisher is not initialized."); throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } conf.setRuntimeStatsTmpDir(path); } else { LOG.debug("skip setRuntimeStatsDir for " + op.getOperatorId() + " because OperatorDesc is null"); } } catch (HiveException e) { throw new SemanticException(e); } }
rsGrandChild.getParentOperators().clear(); rsGrandChild.getParentOperators().add(rsParent); LOG.info("Removed " + rsToRemove.getOperatorId() + " and " + rsChild.getOperatorId() + " as it was introduced by enforce bucketing/sorting.");
rsGrandChild.getParentOperators().clear(); rsGrandChild.getParentOperators().add(rsParent); LOG.info("Removed " + rsToRemove.getOperatorId() + " and " + rsChild.getOperatorId() + " as it was introduced by enforce bucketing/sorting.");
public static <T extends OperatorDesc> Operator<T> getVectorOperator( Class<? extends Operator<?>> opClass, CompilationOpContext cContext, T conf, VectorizationContext vContext, Operator<? extends OperatorDesc> originalOp) throws HiveException { try { VectorDesc vectorDesc = ((AbstractOperatorDesc) conf).getVectorDesc(); vectorDesc.setVectorOp(opClass); Operator<T> op = (Operator<T>) opClass.getDeclaredConstructor(CompilationOpContext.class, VectorizationContext.class, OperatorDesc.class).newInstance(cContext, vContext, conf); op.setOperatorId(originalOp.getOperatorId()); if (op instanceof VectorReduceSinkOperator || op instanceof VectorReduceSinkCommonOperator) { ((ReduceSinkDesc) op.getConf()).setOutputOperators(((ReduceSinkDesc) originalOp.getConf()) .getOutputOperators()); } return op; } catch (Exception e) { e.printStackTrace(); throw new HiveException(e); } }
Map<ColumnInfo, ExprNodeDesc> constMap = opToConstantExprs.get(parent); if (constMap == null) { LOG.debug("Constant of Op " + parent.getOperatorId() + " is not found"); areAllParentsContainConstant = false; } else { LOG.debug("Constant of Op " + parent.getOperatorId() + " " + constMap);
operator.setOperatorId(op.getOperatorId()); operator.setOperatorType(op.getType()); task.addToOperatorList(operator); new org.apache.hadoop.hive.ql.plan.api.Adjacency(); entry.setAdjacencyType(AdjacencyType.CONJUNCTIVE); entry.setNode(op.getOperatorId()); for (Operator<? extends OperatorDesc> childOp : op.getChildOperators()) { entry.addToChildren(childOp.getOperatorId()); if (!opsVisited.contains(childOp)) { opsToVisit.add(childOp);
operator.setOperatorId(op.getOperatorId()); operator.setOperatorType(op.getType()); task.addToOperatorList(operator); new org.apache.hadoop.hive.ql.plan.api.Adjacency(); entry.setAdjacencyType(AdjacencyType.CONJUNCTIVE); entry.setNode(op.getOperatorId()); for (Operator<? extends OperatorDesc> childOp : op.getChildOperators()) { entry.addToChildren(childOp.getOperatorId()); if (!opsVisited.contains(childOp)) { opsToVisit.add(childOp);
if (pruningSinkOp.getOperatorId().equals(op.getOperatorId())) { return null; String marker = "SPARK_DPP_BRANCH_POINT_" + branchingOp.getOperatorId(); branchingOp.setMarker(marker); List<Operator<?>> savedChildOps = branchingOp.getChildOperators();
private void publishRunTimeStats() throws HiveException { StatsPublisher statsPublisher = new FSStatsPublisher(); StatsCollectionContext sContext = new StatsCollectionContext(hconf); sContext.setStatsTmpDir(conf.getRuntimeStatsTmpDir()); sContext.setContextSuffix(getOperatorId()); if (!statsPublisher.connect(sContext)) { LOG.error("StatsPublishing error: cannot connect to database"); throw new HiveException(ErrorMsg.STATSPUBLISHER_CONNECTION_ERROR.getErrorCodedMsg()); } String prefix = ""; Map<String, String> statsToPublish = new HashMap<String, String>(); statsToPublish.put(StatsSetupConst.RUN_TIME_ROW_COUNT, Long.toString(runTimeNumRows)); if (!statsPublisher.publishStat(prefix, statsToPublish)) { // The original exception is lost. // Not changing the interface to maintain backward compatibility throw new HiveException(ErrorMsg.STATSPUBLISHER_PUBLISHING_ERROR.getErrorCodedMsg()); } if (!statsPublisher.closeConnection(sContext)) { // The original exception is lost. // Not changing the interface to maintain backward compatibility throw new HiveException(ErrorMsg.STATSPUBLISHER_CLOSING_ERROR.getErrorCodedMsg()); } }
fsOp.getSchema().getSignature().add(ci); LOG.info("Inserted " + granularitySelOp.getOperatorId() + ", " + rsOp.getOperatorId() + " and " + backtrackSelOp.getOperatorId() + " as parent of " + fsOp.getOperatorId() + " and child of " + fsParent.getOperatorId());
LOG.info("Inserted " + granularitySelOp.getOperatorId() + ", " + rsOp.getOperatorId() + " and " + backtrackSelOp.getOperatorId() + " as parent of " + fsOp.getOperatorId() + " and child of " + fsParent.getOperatorId()); parseCtx.setReduceSinkAddedBySortedDynPartition(true); return null;
String groupName = HiveConf.getVar(conf, HiveConf.ConfVars.HIVECOUNTERGROUP); for (Operator<? extends OperatorDesc> op : baseWork.getAllOperators()) { String operatorId = op.getOperatorId(); OperatorStats operatorStats = null; String counterName = Operator.Counter.RECORDS_OUT_OPERATOR.toString() + "_" + operatorId;