/** * Returns a map of output column name to input expression map Note that * currently it returns only key columns for ReduceSink and GroupBy operators. * * @return null if the operator doesn't change columns */ public Map<String, ExprNodeDesc> getColumnExprMap() { if (this.getConf() == null) { return null; } return this.getConf().getColumnExprMap(); }
/** * Decides whether two operators are logically the same. * This can be used to merge same operators and avoid repeated computation. */ public boolean logicalEquals(Operator other) { return getClass().getName().equals(other.getClass().getName()) && (conf == other.getConf() || (conf != null && other.getConf() != null && conf.isSame(other.getConf()))); }
@Override public Object process(Node n, Stack<Node> s, NodeProcessorCtx c, Object... os) { @SuppressWarnings({ "unchecked" }) List<ExprNodeDesc> exprs = ((Operator<SelectDesc>) n).getConf().getColList(); Boolean retval = new Boolean(checkExpressions(exprs)); if (!retval) { LOG.info("Cannot run select operator [" + n + "] in llap mode"); } return new Boolean(retval); } });
@Override public Object process(Node n, Stack<Node> s, NodeProcessorCtx c, Object... os) { @SuppressWarnings("unchecked") List<AggregationDesc> aggs = ((Operator<GroupByDesc>) n).getConf().getAggregators(); Boolean retval = new Boolean(checkAggregators(aggs)); if (!retval) { LOG.info("Cannot run group by operator [" + n + "] in llap mode"); } return new Boolean(retval); } });
private OpSignature(Operator<? extends OperatorDesc> op) { sigMap = new HashMap<>(); // FIXME: consider other operator info as well..not just conf? SignatureUtils.write(sigMap, op.getConf()); }
@Override public Object process(Node n, Stack<Node> s, NodeProcessorCtx c, Object... os) { @SuppressWarnings("unchecked") List<AggregationDesc> aggs = ((Operator<GroupByDesc>) n).getConf().getAggregators(); Boolean retval = new Boolean(checkAggregators(aggs)); if (!retval) { LOG.info("Cannot run group by operator [" + n + "] in llap mode"); } return new Boolean(retval); } });
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd; if (opMap.get(op) == null) { opMap.put(op, curNum++); } out.println("[" + opMap.get(op) + "] " + op.getClass().getName() + " =p=> " + getParents(op) + " =c=> " + getChildren(op)); if (op.getConf() == null) { return null; } return null; } }
public static Operator<? extends OperatorDesc> vectorizePTFOperator( Operator<? extends OperatorDesc> ptfOp, VectorizationContext vContext, VectorPTFDesc vectorPTFDesc) throws HiveException { PTFDesc ptfDesc = (PTFDesc) ptfOp.getConf(); VectorPTFInfo vectorPTFInfo = createVectorPTFInfo(ptfOp, ptfDesc, vContext, vectorPTFDesc); vectorPTFDesc.setVectorPTFInfo(vectorPTFInfo); Class<? extends Operator<?>> opClass = VectorPTFOperator.class; return OperatorFactory.getVectorOperator( opClass, ptfOp.getCompilationOpContext(), ptfOp.getConf(), vContext, vectorPTFDesc); }
private List<VirtualColumn> getVirtualColumns(Operator<?> ts) { if (ts instanceof TableScanOperator && ts.getConf() != null) { return ((TableScanOperator)ts).getConf().getVirtualCols(); } return null; }
private void setStatistics(Operator<? extends OperatorDesc> origin, Operator<? extends OperatorDesc> clone) { clone.getConf().setStatistics(origin.getConf().getStatistics()); clone.getConf().setTraits(origin.getConf().getTraits()); if (origin.getChildOperators().size() == clone.getChildOperators().size()) { for (int i = 0; i < clone.getChildOperators().size(); i++) { setStatistics(origin.getChildOperators().get(i), clone.getChildOperators().get(i)); } } } }
private static void annotateRuntimeStats(Operator<? extends OperatorDesc> op, ParseContext pctx) { Long runTimeNumRows = pctx.getContext().getExplainConfig().getOpIdToRuntimeNumRows() .get(op.getOperatorId()); if (op.getConf() != null && op.getConf().getStatistics() != null && runTimeNumRows != null) { LOG.info("annotateRuntimeStats for " + op.getOperatorId()); op.getConf().getStatistics().setRunTimeNumRows(runTimeNumRows); } else { LOG.debug("skip annotateRuntimeStats for " + op.getOperatorId()); } }
private Operator<? extends OperatorDesc> specializeArrowFileSinkOperator( Operator<? extends OperatorDesc> op, VectorizationContext vContext, FileSinkDesc desc, VectorFileSinkDesc vectorDesc) throws HiveException { Class<? extends Operator<?>> opClass = VectorFileSinkArrowOperator.class; Operator<? extends OperatorDesc> vectorOp = null; try { vectorOp = OperatorFactory.getVectorOperator( opClass, op.getCompilationOpContext(), op.getConf(), vContext, vectorDesc); } catch (Exception e) { LOG.info("Vectorizer vectorizeOperator file sink class exception " + opClass.getSimpleName() + " exception " + e); throw new HiveException(e); } return vectorOp; }
public static void setMemoryAvailable(final List<Operator<? extends OperatorDesc>> operators, final long memoryAvailableToTask) { if (operators == null) { return; } for (Operator<? extends OperatorDesc> op : operators) { if (op.getConf() != null) { op.getConf().setMaxMemoryAvailable(memoryAvailableToTask); } if (op.getChildOperators() != null && !op.getChildOperators().isEmpty()) { setMemoryAvailable(op.getChildOperators(), memoryAvailableToTask); } } }
private void setWriteIdForSurrogateKeys(LoadTableDesc ltd, Operator input) throws SemanticException { Map<String, ExprNodeDesc> columnExprMap = input.getConf().getColumnExprMap(); if (ltd == null || columnExprMap == null) { return; } for (ExprNodeDesc desc : columnExprMap.values()) { if (desc instanceof ExprNodeGenericFuncDesc) { GenericUDF genericUDF = ((ExprNodeGenericFuncDesc)desc).getGenericUDF(); if (genericUDF instanceof GenericUDFSurrogateKey) { ((GenericUDFSurrogateKey)genericUDF).setWriteId(ltd.getWriteId()); } } } }
public static Operator<? extends OperatorDesc> vectorizeTableScanOperator( Operator<? extends OperatorDesc> tableScanOp, VectorizationContext vContext) throws HiveException { TableScanDesc tableScanDesc = (TableScanDesc) tableScanOp.getConf(); VectorTableScanDesc vectorTableScanDesc = new VectorTableScanDesc(); tableScanDesc.setVectorDesc(vectorTableScanDesc); vectorTableScanDesc.setProjectedOutputColumns( ArrayUtils.toPrimitive(vContext.getProjectedColumns().toArray(new Integer[0]))); return tableScanOp; }
private void setOperatorNotSupported(Operator<? extends OperatorDesc> op) { OperatorDesc desc = op.getConf(); Annotation note = AnnotationUtils.getAnnotation(desc.getClass(), Explain.class); if (note != null) { Explain explainNote = (Explain) note; setNodeIssue(explainNote.displayName() + " (" + op.getType() + ") not supported"); } else { setNodeIssue("Operator " + op.getType() + " not supported"); } }
private void setOperatorNotSupported(Operator<? extends OperatorDesc> op) { OperatorDesc desc = op.getConf(); Annotation note = AnnotationUtils.getAnnotation(desc.getClass(), Explain.class); if (note != null) { Explain explainNote = (Explain) note; setNodeIssue(explainNote.displayName() + " (" + op.getType() + ") not supported"); } else { setNodeIssue("Operator " + op.getType() + " not supported"); } }
public static Operator<? extends OperatorDesc> vectorizeFilterOperator( Operator<? extends OperatorDesc> filterOp, VectorizationContext vContext, VectorFilterDesc vectorFilterDesc) throws HiveException { FilterDesc filterDesc = (FilterDesc) filterOp.getConf(); ExprNodeDesc predicateExpr = filterDesc.getPredicate(); VectorExpression vectorPredicateExpr = vContext.getVectorExpression(predicateExpr, VectorExpressionDescriptor.Mode.FILTER); vectorFilterDesc.setPredicateExpression(vectorPredicateExpr); return OperatorFactory.getVectorOperator( filterOp.getCompilationOpContext(), filterDesc, vContext, vectorFilterDesc); }
private static Operator<? extends OperatorDesc> vectorizeTopNKeyOperator( Operator<? extends OperatorDesc> topNKeyOperator, VectorizationContext vContext, VectorTopNKeyDesc vectorTopNKeyDesc) throws HiveException { TopNKeyDesc topNKeyDesc = (TopNKeyDesc) topNKeyOperator.getConf(); List<ExprNodeDesc> keyColumns = topNKeyDesc.getKeyColumns(); VectorExpression[] keyExpressions = vContext.getVectorExpressions(keyColumns); vectorTopNKeyDesc.setKeyExpressions(keyExpressions); return OperatorFactory.getVectorOperator( topNKeyOperator.getCompilationOpContext(), topNKeyDesc, vContext, vectorTopNKeyDesc); }