RowSchema rowSchema = tableScanOperator.getSchema(); for (ColumnInfo c : rowSchema.getSignature()) {
private void getTableScanOperatorSchemaInfo(TableScanOperator tableScanOperator, List<String> logicalColumnNameList, List<TypeInfo> logicalTypeInfoList) { // Add all non-virtual columns to make a vectorization context for // the TableScan operator. RowSchema rowSchema = tableScanOperator.getSchema(); for (ColumnInfo c : rowSchema.getSignature()) { // Validation will later exclude vectorization of virtual columns usage (HIVE-5560). if (!isVirtualColumn(c)) { String columnName = c.getInternalName(); String typeName = c.getTypeName(); TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName); logicalColumnNameList.add(columnName); logicalTypeInfoList.add(typeInfo); } } }
RowSchema rowSchema = findTsOp(mapWork).getSchema(); final List<String> colNames = new ArrayList<String>(rowSchema.getSignature().size()); final List<TypeInfo> colTypes = new ArrayList<TypeInfo>(rowSchema.getSignature().size());
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { TableScanOperator scanOp = (TableScanOperator) nd; ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; List<FieldNode> cols = cppCtx .genColLists((Operator<? extends OperatorDesc>) nd); if (cols == null && !scanOp.getConf().isGatherStats() ) { scanOp.setNeededColumnIDs(null); return null; } cols = cols == null ? new ArrayList<FieldNode>() : cols; cppCtx.getPrunedColLists().put((Operator<? extends OperatorDesc>) nd, cols); RowSchema inputRS = scanOp.getSchema(); setupNeededColumns(scanOp, inputRS, cols); return null; } }
rowSchema = ts.getSchema(); ExprNodeGenericFuncDesc filterExpr = ts.getConf() == null ? null : ts.getConf().getFilterExpr();
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { TableScanOperator scanOp = (TableScanOperator) nd; ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; List<FieldNode> cols = cppCtx .genColLists((Operator<? extends OperatorDesc>) nd); if (cols == null && !scanOp.getConf().isGatherStats() ) { scanOp.setNeededColumnIDs(null); return null; } cols = cols == null ? new ArrayList<FieldNode>() : cols; cppCtx.getPrunedColLists().put((Operator<? extends OperatorDesc>) nd, cols); RowSchema inputRS = scanOp.getSchema(); setupNeededColumns(scanOp, inputRS, cols); return null; } }
/** * Collect table, partition and column level statistics * @param conf * - hive configuration * @param partList * - partition list * @param table * - table * @param tableScanOperator * - table scan operator * @return statistics object * @throws HiveException */ public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList partList, ColumnStatsList colStatsCache, Table table, TableScanOperator tableScanOperator) throws HiveException { // column level statistics are required only for the columns that are needed List<ColumnInfo> schema = tableScanOperator.getSchema().getSignature(); List<String> neededColumns = tableScanOperator.getNeededColumns(); List<String> referencedColumns = tableScanOperator.getReferencedColumns(); return collectStatistics(conf, partList, table, schema, neededColumns, colStatsCache, referencedColumns); }
rowSchema = ts.getSchema(); ExprNodeGenericFuncDesc filterExpr = ts.getConf() == null ? null : ts.getConf().getFilterExpr();
LOG.debug("Generated " + ts + " with row schema: [" + ts.getSchema() + "]");
/** * Collect table, partition and column level statistics * @param conf * - hive configuration * @param partList * - partition list * @param table * - table * @param tableScanOperator * - table scan operator * @return statistics object * @throws HiveException */ public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList partList, Table table, TableScanOperator tableScanOperator) throws HiveException { // column level statistics are required only for the columns that are needed List<ColumnInfo> schema = tableScanOperator.getSchema().getSignature(); List<String> neededColumns = tableScanOperator.getNeededColumns(); List<String> referencedColumns = tableScanOperator.getReferencedColumns(); return collectStatistics(conf, partList, table, schema, neededColumns, referencedColumns); }
LOG.debug("Generated " + ts + " with row schema: [" + ts.getSchema() + "]");
/** * Inserts a filter below the table scan operator. Construct the filter * from the filter expression provided. * @param tableScanOp the table scan operators * @param filterExpr the filter expression */ private void insertFilterOnTop( TableScanOperator tableScanOp, ExprNodeDesc filterExpr) { // Get the top operator and it's child, all operators have a single parent Operator<? extends OperatorDesc> currChild = tableScanOp.getChildOperators().get(0); // Create the filter Operator and update the parents and children appropriately tableScanOp.setChildOperators(null); currChild.setParentOperators(null); Operator<FilterDesc> filter = OperatorFactory.getAndMakeChild( new FilterDesc(filterExpr, false), new RowSchema(tableScanOp.getSchema().getSignature()), tableScanOp); OperatorFactory.makeChild(filter, currChild); }
/** * Inserts a filter below the table scan operator. Construct the filter * from the filter expression provided. * @param tableScanOp the table scan operators * @param filterExpr the filter expression */ private void insertFilterOnTop( TableScanOperator tableScanOp, ExprNodeDesc filterExpr) { // Get the top operator and it's child, all operators have a single parent Operator<? extends OperatorDesc> currChild = tableScanOp.getChildOperators().get(0); // Create the filter Operator and update the parents and children appropriately tableScanOp.setChildOperators(null); currChild.setParentOperators(null); Operator<FilterDesc> filter = OperatorFactory.getAndMakeChild( new FilterDesc(filterExpr, false), new RowSchema(tableScanOp.getSchema().getSignature()), tableScanOp); OperatorFactory.makeChild(filter, currChild); }
RowSchema rs = top.getSchema(); List<FieldSchema> cols = t.getAllCols(); Map<String, FieldSchema> fieldSchemaMap = new HashMap<String, FieldSchema>();
RowSchema rs = top.getSchema(); List<FieldSchema> cols = t.getAllCols(); Map<String, FieldSchema> fieldSchemaMap = new HashMap<String, FieldSchema>();
Operator<FilterDesc> newOp = OperatorFactory.get(tsOp.getCompilationOpContext(), new FilterDesc(tableScanExprNode.clone(), false), new RowSchema(tsOp.getSchema().getSignature())); tsOp.replaceChild(op, newOp); newOp.getParentOperators().add(tsOp);
RowSchema rowSchema = findTsOp(mapWork).getSchema(); final List<String> colNames = new ArrayList<String>(rowSchema.getSignature().size()); final List<TypeInfo> colTypes = new ArrayList<TypeInfo>(rowSchema.getSignature().size());
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { TableScanOperator scanOp = (TableScanOperator) nd; ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; List<String> cols = cppCtx .genColLists((Operator<? extends OperatorDesc>) nd); if (cols == null && !scanOp.getConf().isGatherStats() ) { scanOp.setNeededColumnIDs(null); return null; } cols = cols == null ? new ArrayList<String>() : cols; cppCtx.getPrunedColLists().put((Operator<? extends OperatorDesc>) nd, cols); RowSchema inputRS = scanOp.getSchema(); setupNeededColumns(scanOp, inputRS, cols); return null; } }
/** * Collect table, partition and column level statistics * @param conf * - hive configuration * @param partList * - partition list * @param table * - table * @param tableScanOperator * - table scan operator * @return statistics object * @throws HiveException */ public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList partList, Table table, TableScanOperator tableScanOperator) throws HiveException { // column level statistics are required only for the columns that are needed List<ColumnInfo> schema = tableScanOperator.getSchema().getSignature(); List<String> neededColumns = tableScanOperator.getNeededColumns(); List<String> referencedColumns = tableScanOperator.getReferencedColumns(); return collectStatistics(conf, partList, table, schema, neededColumns, referencedColumns); }
/** * Inserts a filter below the table scan operator. Construct the filter * from the filter expression provided. * @param tableScanOp the table scan operators * @param filterExpr the filter expression */ private void insertFilterOnTop( TableScanOperator tableScanOp, ExprNodeDesc filterExpr) { // Get the top operator and it's child, all operators have a single parent Operator<? extends OperatorDesc> currChild = tableScanOp.getChildOperators().get(0); // Create the filter Operator and update the parents and children appropriately tableScanOp.setChildOperators(null); currChild.setParentOperators(null); Operator<FilterDesc> filter = OperatorFactory.getAndMakeChild( new FilterDesc(filterExpr, false), new RowSchema(tableScanOp.getSchema().getSignature()), tableScanOp); OperatorFactory.makeChild(filter, currChild); }