public static <T extends OperatorDesc> Operator<T> getVectorOperator( CompilationOpContext cContext, T conf, VectorizationContext vContext, VectorDesc vectorDesc) throws HiveException { Class<T> descClass = (Class<T>) conf.getClass(); Class<?> opClass = vectorOpvec.get(descClass); if (opClass != null) { return getVectorOperator(vectorOpvec.get(descClass), cContext, conf, vContext, vectorDesc); } throw new HiveException("No vector operator for descriptor class " + descClass.getName()); }
public static <T extends OperatorDesc> Operator<T> getVectorOperator( CompilationOpContext cContext, T conf, VectorizationContext vContext, Operator<? extends OperatorDesc> originalOp) throws HiveException { Class<T> descClass = (Class<T>) conf.getClass(); Class<? extends Operator<? extends OperatorDesc>> opClass = vectorOpvec.get(descClass); if (opClass != null) { return getVectorOperator(opClass, cContext, conf, vContext, originalOp); } throw new HiveException("No vector operator for descriptor class " + descClass.getName()); }
private Operator<? extends OperatorDesc> specializeArrowFileSinkOperator( Operator<? extends OperatorDesc> op, VectorizationContext vContext, FileSinkDesc desc, VectorFileSinkDesc vectorDesc) throws HiveException { Class<? extends Operator<?>> opClass = VectorFileSinkArrowOperator.class; Operator<? extends OperatorDesc> vectorOp = null; try { vectorOp = OperatorFactory.getVectorOperator( opClass, op.getCompilationOpContext(), op.getConf(), vContext, vectorDesc); } catch (Exception e) { LOG.info("Vectorizer vectorizeOperator file sink class exception " + opClass.getSimpleName() + " exception " + e); throw new HiveException(e); } return vectorOp; }
private static Operator<? extends OperatorDesc> vectorizeTopNKeyOperator( Operator<? extends OperatorDesc> topNKeyOperator, VectorizationContext vContext, VectorTopNKeyDesc vectorTopNKeyDesc) throws HiveException { TopNKeyDesc topNKeyDesc = (TopNKeyDesc) topNKeyOperator.getConf(); List<ExprNodeDesc> keyColumns = topNKeyDesc.getKeyColumns(); VectorExpression[] keyExpressions = vContext.getVectorExpressions(keyColumns); vectorTopNKeyDesc.setKeyExpressions(keyExpressions); return OperatorFactory.getVectorOperator( topNKeyOperator.getCompilationOpContext(), topNKeyDesc, vContext, vectorTopNKeyDesc); }
public static Operator<? extends OperatorDesc> vectorizePTFOperator( Operator<? extends OperatorDesc> ptfOp, VectorizationContext vContext, VectorPTFDesc vectorPTFDesc) throws HiveException { PTFDesc ptfDesc = (PTFDesc) ptfOp.getConf(); VectorPTFInfo vectorPTFInfo = createVectorPTFInfo(ptfOp, ptfDesc, vContext, vectorPTFDesc); vectorPTFDesc.setVectorPTFInfo(vectorPTFInfo); Class<? extends Operator<?>> opClass = VectorPTFOperator.class; return OperatorFactory.getVectorOperator( opClass, ptfOp.getCompilationOpContext(), ptfOp.getConf(), vContext, vectorPTFDesc); }
public static Operator<? extends OperatorDesc> vectorizeFilterOperator( Operator<? extends OperatorDesc> filterOp, VectorizationContext vContext, VectorFilterDesc vectorFilterDesc) throws HiveException { FilterDesc filterDesc = (FilterDesc) filterOp.getConf(); ExprNodeDesc predicateExpr = filterDesc.getPredicate(); VectorExpression vectorPredicateExpr = vContext.getVectorExpression(predicateExpr, VectorExpressionDescriptor.Mode.FILTER); vectorFilterDesc.setPredicateExpression(vectorPredicateExpr); return OperatorFactory.getVectorOperator( filterOp.getCompilationOpContext(), filterDesc, vContext, vectorFilterDesc); }
vectorGroupByDesc.setProjectedOutputColumns(projectedOutputColumns); Operator<GroupByDesc> vectorOp = OperatorFactory.getVectorOperator( groupByOp.getCompilationOpContext(), groupByDesc, vContext, vectorGroupByDesc);
public static Operator<? extends OperatorDesc> vectorizeSelectOperator( Operator<? extends OperatorDesc> selectOp, VectorizationContext vContext) throws HiveException { SelectDesc selectDesc = (SelectDesc) selectOp.getConf(); VectorSelectDesc vectorSelectDesc = new VectorSelectDesc(); selectDesc.setVectorDesc(vectorSelectDesc); List<ExprNodeDesc> colList = selectDesc.getColList(); int index = 0; final int size = colList.size(); VectorExpression[] vectorSelectExprs = new VectorExpression[size]; int[] projectedOutputColumns = new int[size]; for (int i = 0; i < size; i++) { ExprNodeDesc expr = colList.get(i); VectorExpression ve = vContext.getVectorExpression(expr); projectedOutputColumns[i] = ve.getOutputColumn(); if (ve instanceof IdentityExpression) { // Suppress useless evaluation. continue; } vectorSelectExprs[index++] = ve; } if (index < size) { vectorSelectExprs = Arrays.copyOf(vectorSelectExprs, index); } vectorSelectDesc.setSelectExpressions(vectorSelectExprs); vectorSelectDesc.setProjectedOutputColumns(projectedOutputColumns); return OperatorFactory.getVectorOperator( selectOp.getCompilationOpContext(), selectDesc, vContext, selectOp); }
vectorSelectDesc.setProjectedOutputColumns(projectedOutputColumns); return OperatorFactory.getVectorOperator( selectOp.getCompilationOpContext(), selectDesc, vContext, vectorSelectDesc);
public static Operator<? extends OperatorDesc> vectorizeFilterOperator( Operator<? extends OperatorDesc> filterOp, VectorizationContext vContext) throws HiveException { FilterDesc filterDesc = (FilterDesc) filterOp.getConf(); VectorFilterDesc vectorFilterDesc = new VectorFilterDesc(); filterDesc.setVectorDesc(vectorFilterDesc); ExprNodeDesc predicateExpr = filterDesc.getPredicate(); VectorExpression vectorPredicateExpr = vContext.getVectorExpression(predicateExpr, VectorExpressionDescriptor.Mode.FILTER); vectorFilterDesc.setPredicateExpression(vectorPredicateExpr); return OperatorFactory.getVectorOperator( filterOp.getCompilationOpContext(), filterDesc, vContext, filterOp); }
public static Operator<? extends OperatorDesc> vectorizeGroupByOperator( Operator<? extends OperatorDesc> groupByOp, VectorizationContext vContext) throws HiveException { GroupByDesc groupByDesc = (GroupByDesc) groupByOp.getConf(); List<ExprNodeDesc> keysDesc = groupByDesc.getKeys(); VectorExpression[] vecKeyExpressions = vContext.getVectorExpressions(keysDesc); ArrayList<AggregationDesc> aggrDesc = groupByDesc.getAggregators(); final int size = aggrDesc.size(); VectorAggregateExpression[] vecAggregators = new VectorAggregateExpression[size]; int[] projectedOutputColumns = new int[size]; for (int i = 0; i < size; ++i) { AggregationDesc aggDesc = aggrDesc.get(i); vecAggregators[i] = vContext.getAggregatorExpression(aggDesc); // GroupBy generates a new vectorized row batch... projectedOutputColumns[i] = i; } VectorGroupByDesc vectorGroupByDesc = (VectorGroupByDesc) groupByDesc.getVectorDesc(); vectorGroupByDesc.setKeyExpressions(vecKeyExpressions); vectorGroupByDesc.setAggregators(vecAggregators); vectorGroupByDesc.setProjectedOutputColumns(projectedOutputColumns); return OperatorFactory.getVectorOperator( groupByOp.getCompilationOpContext(), groupByDesc, vContext, groupByOp); }
vectorOp = OperatorFactory.getVectorOperator( opClass, op.getCompilationOpContext(), op.getConf(), vContext, op); } catch (Exception e) {
vectorOp = OperatorFactory.getVectorOperator( opClass, op.getCompilationOpContext(), op.getConf(), vContext, vectorDesc);
vectorDesc.setVectorMapJoinInfo(vectorMapJoinInfo); vectorOp = OperatorFactory.getVectorOperator( opClass, op.getCompilationOpContext(), op.getConf(), vContext, vectorDesc); LOG.info("Vectorizer vectorizeOperator map join class " + vectorOp.getClass().getSimpleName());
vectorDesc.setVectorMapJoinInfo(vectorMapJoinInfo); vectorOp = OperatorFactory.getVectorOperator( opClass, op.getCompilationOpContext(), op.getConf(), vContext, op); LOG.info("Vectorizer vectorizeOperator map join class " + vectorOp.getClass().getSimpleName());
vectorSelectDesc.setProjectedOutputColumns(projectedColumns); Operator<SelectDesc> vectorSelectOperator = OperatorFactory.getVectorOperator( selectOperator.getCompilationOpContext(), selectDesc, vOutContext, vectorSelectDesc);
vectorOp = OperatorFactory.getVectorOperator( opClass, op.getCompilationOpContext(), op.getConf(), vContext, op); isNative = false; VectorSMBJoinDesc vectorSMBJoinDesc = new VectorSMBJoinDesc(); smbJoinSinkDesc.setVectorDesc(vectorSMBJoinDesc); vectorOp = OperatorFactory.getVectorOperator( op.getCompilationOpContext(), smbJoinSinkDesc, vContext, op); isNative = false; vectorOp = OperatorFactory.getVectorOperator( op.getCompilationOpContext(), op.getConf(), vContext, op); isNative = false; VectorFileSinkDesc vectorFileSinkDesc = new VectorFileSinkDesc(); fileSinkDesc.setVectorDesc(vectorFileSinkDesc); vectorOp = OperatorFactory.getVectorOperator( op.getCompilationOpContext(), fileSinkDesc, vContext, op); isNative = false; VectorLimitDesc vectorLimitDesc = new VectorLimitDesc(); limitDesc.setVectorDesc(vectorLimitDesc); vectorOp = OperatorFactory.getVectorOperator( op.getCompilationOpContext(), limitDesc, vContext, op); isNative = true; VectorAppMasterEventDesc vectorEventDesc = new VectorAppMasterEventDesc(); eventDesc.setVectorDesc(vectorEventDesc); vectorOp = OperatorFactory.getVectorOperator(
vectorOp = OperatorFactory.getVectorOperator( opClass, op.getCompilationOpContext(), desc, vContext, vectorMapJoinDesc); vectorOp = OperatorFactory.getVectorOperator( op.getCompilationOpContext(), smbJoinSinkDesc, vContext, vectorSMBJoinDesc); isNative = false; vectorOp = OperatorFactory.getVectorOperator( op.getCompilationOpContext(), reduceDesc, vContext, vectorReduceSinkDesc); isNative = false; } else { vectorOp = OperatorFactory.getVectorOperator( op.getCompilationOpContext(), fileSinkDesc, vContext, vectorFileSinkDesc); isNative = false; vectorOp = OperatorFactory.getVectorOperator( op.getCompilationOpContext(), limitDesc, vContext, vectorLimitDesc); isNative = true; vectorOp = OperatorFactory.getVectorOperator( op.getCompilationOpContext(), eventDesc, vContext, vectorEventDesc); isNative = true; vectorOp = OperatorFactory.getVectorOperator( op.getCompilationOpContext(), sparkHashTableSinkDesc, vContext, vectorSparkHashTableSinkDesc);
public static <T extends OperatorDesc> Operator<T> getVectorOperator(T conf, VectorizationContext vContext) throws HiveException { Class<T> descClass = (Class<T>) conf.getClass(); for (OpTuple o : vectorOpvec) { if (o.descClass == descClass) { return getVectorOperator(o.opClass, conf, vContext); } } throw new HiveException("No vector operator for descriptor class " + descClass.getName()); }
vectorOp = OperatorFactory.getVectorOperator(opClass, op.getConf(), vContext); case EXTRACT: case EVENT: vectorOp = OperatorFactory.getVectorOperator(op.getConf(), vContext); break; default: