private void determineDataColumnNums(TableScanOperator tableScanOperator, List<String> allColumnNameList, int dataColumnCount, List<Integer> dataColumnNums) { /* * The TableScanOperator's needed columns are just the data columns. */ Set<String> neededColumns = new HashSet<String>(tableScanOperator.getNeededColumns()); for (int dataColumnNum = 0; dataColumnNum < dataColumnCount; dataColumnNum++) { String columnName = allColumnNameList.get(dataColumnNum); if (neededColumns.contains(columnName)) { dataColumnNums.add(dataColumnNum); } } }
private void determineDataColumnNums(TableScanOperator tableScanOperator, List<String> allColumnNameList, int dataColumnCount, List<Integer> dataColumnNums) { /* * The TableScanOperator's needed columns are just the data columns. */ Set<String> neededColumns = new HashSet<String>(tableScanOperator.getNeededColumns()); for (int dataColumnNum = 0; dataColumnNum < dataColumnCount; dataColumnNum++) { String columnName = allColumnNameList.get(dataColumnNum); if (neededColumns.contains(columnName)) { dataColumnNums.add(dataColumnNum); } } }
@Override public Operator<? extends OperatorDesc> clone() throws CloneNotSupportedException { TableScanOperator ts = (TableScanOperator) super.clone(); ts.setNeededColumnIDs(new ArrayList<Integer>(getNeededColumnIDs())); ts.setNeededColumns(new ArrayList<String>(getNeededColumns())); ts.setReferencedColumns(new ArrayList<String>(getReferencedColumns())); return ts; }
@Override public Operator<? extends OperatorDesc> clone() throws CloneNotSupportedException { TableScanOperator ts = (TableScanOperator) super.clone(); ts.setNeededColumnIDs(new ArrayList<Integer>(getNeededColumnIDs())); ts.setNeededColumns(new ArrayList<String>(getNeededColumns())); ts.setReferencedColumns(new ArrayList<String>(getReferencedColumns())); return ts; }
List<String> prevTsOpNeededColumns = tsOp1.getNeededColumns(); List<String> tsOpNeededColumns = tsOp2.getNeededColumns(); if (prevTsOpNeededColumns.size() != tsOpNeededColumns.size()) { return false;
private boolean checkVectorizerSupportedTypes(boolean hasLlap) { for (Map.Entry<String, Operator<? extends OperatorDesc>> entry : aliasToWork.entrySet()) { final String alias = entry.getKey(); Operator<? extends OperatorDesc> op = entry.getValue(); PartitionDesc partitionDesc = aliasToPartnInfo.get(alias); if (op instanceof TableScanOperator && partitionDesc != null && partitionDesc.getTableDesc() != null) { final TableScanOperator tsOp = (TableScanOperator) op; final List<String> readColumnNames = tsOp.getNeededColumns(); final Properties props = partitionDesc.getTableDesc().getProperties(); final List<TypeInfo> typeInfos = TypeInfoUtils.getTypeInfosFromTypeString( props.getProperty(serdeConstants.LIST_COLUMN_TYPES)); final List<String> allColumnTypes = TypeInfoUtils.getTypeStringsFromTypeInfo(typeInfos); final List<String> allColumnNames = Utilities.getColumnNames(props); hasLlap = Utilities.checkVectorizerSupportedTypes(readColumnNames, allColumnNames, allColumnTypes); } } return hasLlap; }
jobClone, ts.getNeededColumnIDs(), ts.getNeededColumns(), ts.getNeededNestedColumnPaths());
tableScan.getNeededColumnIDs(), tableScan.getNeededColumns()); pushDownProjection = true;
jobClone, ts.getNeededColumnIDs(), ts.getNeededColumns(), ts.getNeededNestedColumnPaths());
/** * Collect table, partition and column level statistics * @param conf * - hive configuration * @param partList * - partition list * @param table * - table * @param tableScanOperator * - table scan operator * @return statistics object * @throws HiveException */ public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList partList, ColumnStatsList colStatsCache, Table table, TableScanOperator tableScanOperator) throws HiveException { // column level statistics are required only for the columns that are needed List<ColumnInfo> schema = tableScanOperator.getSchema().getSignature(); List<String> neededColumns = tableScanOperator.getNeededColumns(); List<String> referencedColumns = tableScanOperator.getReferencedColumns(); return collectStatistics(conf, partList, table, schema, neededColumns, colStatsCache, referencedColumns); }
tableScan.getNeededColumnIDs(), tableScan.getNeededColumns()); pushDownProjection = true;
/** * Collect table, partition and column level statistics * @param conf * - hive configuration * @param partList * - partition list * @param table * - table * @param tableScanOperator * - table scan operator * @return statistics object * @throws HiveException */ public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList partList, Table table, TableScanOperator tableScanOperator) throws HiveException { // column level statistics are required only for the columns that are needed List<ColumnInfo> schema = tableScanOperator.getSchema().getSignature(); List<String> neededColumns = tableScanOperator.getNeededColumns(); List<String> referencedColumns = tableScanOperator.getReferencedColumns(); return collectStatistics(conf, partList, table, schema, neededColumns, referencedColumns); }
jobClone, ts.getNeededColumnIDs(), ts.getNeededColumns(), ts.getNeededNestedColumnPaths());
jobConf, ts.getNeededColumnIDs(), ts.getNeededColumns(), ts.getNeededNestedColumnPaths());
jobClone, ts.getNeededColumnIDs(), ts.getNeededColumns(), ts.getNeededNestedColumnPaths());
jobConf, ts.getNeededColumnIDs(), ts.getNeededColumns(), ts.getNeededNestedColumnPaths());
job, ts.getNeededColumnIDs(), ts.getNeededColumns(), ts.getNeededNestedColumnPaths());
if (parent instanceof TableScanOperator) { TableScanOperator tsop = (TableScanOperator) parent; neededCols = tsop.getNeededColumns();
job, ts.getNeededColumnIDs(), ts.getNeededColumns(), ts.getNeededNestedColumnPaths());
if (parent instanceof TableScanOperator) { TableScanOperator tsop = (TableScanOperator) parent; neededCols = tsop.getNeededColumns();