@Override public Object clone() { List<VirtualColumn> vcs = new ArrayList<VirtualColumn>(getVirtualCols()); return new TableScanDesc(getAlias(), vcs, this.tableMetadata); }
@Override public Object clone() { List<VirtualColumn> vcs = new ArrayList<VirtualColumn>(getVirtualCols()); return new TableScanDesc(getAlias(), vcs, this.tableMetadata); }
public PrunedPartitionList getPrunedPartitions(TableScanOperator ts) throws SemanticException { return getPrunedPartitions(ts.getConf().getAlias(), ts); }
public PrunedPartitionList getPrunedPartitions(TableScanOperator ts) throws SemanticException { return getPrunedPartitions(ts.getConf().getAlias(), ts); }
public static String getOpNamePretty(Operator<?> op) { if (op instanceof TableScanOperator) { return op.toString() + " (" + ((TableScanOperator) op).getConf().getAlias() + ")"; } return op.toString(); }
private Map<String, Operator<? extends OperatorDesc>> getTopOps( Operator<? extends OperatorDesc> op) { // Must be deterministic order map for consistent q-test output across // Java versions Map<String, Operator<? extends OperatorDesc>> topOps = new LinkedHashMap<String, Operator<? extends OperatorDesc>>(); if (op.getParentOperators() == null || op.getParentOperators().size() == 0) { topOps.put(((TableScanOperator)op).getConf().getAlias(), op); } else { for (Operator<? extends OperatorDesc> parent : op.getParentOperators()) { if (parent != null) { topOps.putAll(getTopOps(parent)); } } } return topOps; }
private Map<String, Operator<? extends OperatorDesc>> getTopOps( Operator<? extends OperatorDesc> op) { // Must be deterministic order map for consistent q-test output across // Java versions Map<String, Operator<? extends OperatorDesc>> topOps = new LinkedHashMap<String, Operator<? extends OperatorDesc>>(); if (op.getParentOperators() == null || op.getParentOperators().size() == 0) { topOps.put(((TableScanOperator)op).getConf().getAlias(), op); } else { for (Operator<? extends OperatorDesc> parent : op.getParentOperators()) { if (parent != null) { topOps.putAll(getTopOps(parent)); } } } return topOps; }
public static String getOpNamePretty(Operator<?> op) { if (op instanceof TableScanOperator) { return op.toString() + " (" + ((TableScanOperator) op).getConf().getAlias() + ")"; } return op.toString(); } }
public MapWork createMapWork(GenSparkProcContext context, Operator<?> root, SparkWork sparkWork, PrunedPartitionList partitions, boolean deferSetup) throws SemanticException { Preconditions.checkArgument(root.getParentOperators().isEmpty(), "AssertionError: expected root.getParentOperators() to be empty"); MapWork mapWork = new MapWork("Map " + (++sequenceNumber)); LOG.debug("Adding map work (" + mapWork.getName() + ") for " + root); // map work starts with table scan operators Preconditions.checkArgument(root instanceof TableScanOperator, "AssertionError: expected root to be an instance of TableScanOperator, but was " + root.getClass().getName()); String alias = ((TableScanOperator) root).getConf().getAlias(); if (!deferSetup) { setupMapWork(mapWork, context, partitions,(TableScanOperator) root, alias); } // add new item to the Spark work sparkWork.add(mapWork); return mapWork; }
public void initEmptyInputChildren(List<Operator<?>> children, Configuration hconf) throws SerDeException, Exception { setChildOperators(children); Map<String, Configuration> tableNameToConf = cloneConfsForNestedColPruning(hconf); for (Operator<?> child : children) { TableScanOperator tsOp = (TableScanOperator) child; StructObjectInspector soi = null; PartitionDesc partDesc = conf.getAliasToPartnInfo().get(tsOp.getConf().getAlias()); Configuration newConf = tableNameToConf.get(partDesc.getTableDesc().getTableName()); Deserializer serde = partDesc.getTableDesc().getDeserializer(); partDesc.setProperties(partDesc.getProperties()); MapOpCtx opCtx = new MapOpCtx(tsOp.getConf().getAlias(), child, partDesc); StructObjectInspector tableRowOI = (StructObjectInspector) serde.getObjectInspector(); initObjectInspector(newConf, opCtx, tableRowOI); soi = opCtx.rowObjectInspector; child.getParentOperators().add(this); childrenOpToOI.put(child, soi); } }
@Override protected void generatePredicate(NodeProcessorCtx procCtx, FilterOperator fop, TableScanOperator top) throws SemanticException, UDFArgumentException { OpWalkerCtx owc = (OpWalkerCtx) procCtx; // Otherwise this is not a sampling predicate and we need to ExprNodeDesc predicate = fop.getConf().getPredicate(); String alias = top.getConf().getAlias(); // Generate the partition pruning predicate ExprNodeDesc ppr_pred = ExprProcFactory.genPruner(alias, predicate); // Add the pruning predicate to the table scan operator addPruningPred(owc.getOpToPartPruner(), top, ppr_pred); }
public void initEmptyInputChildren(List<Operator<?>> children, Configuration hconf) throws SerDeException, Exception { setChildOperators(children); Map<String, Configuration> tableNameToConf = cloneConfsForNestedColPruning(hconf); for (Operator<?> child : children) { TableScanOperator tsOp = (TableScanOperator) child; StructObjectInspector soi = null; PartitionDesc partDesc = conf.getAliasToPartnInfo().get(tsOp.getConf().getAlias()); Configuration newConf = tableNameToConf.get(partDesc.getTableDesc().getTableName()); Deserializer serde = partDesc.getTableDesc().getDeserializer(); partDesc.setProperties(partDesc.getProperties()); MapOpCtx opCtx = new MapOpCtx(tsOp.getConf().getAlias(), child, partDesc); StructObjectInspector tableRowOI = (StructObjectInspector) serde.getObjectInspector(); initObjectInspector(newConf, opCtx, tableRowOI); soi = opCtx.rowObjectInspector; child.getParentOperators().add(this); childrenOpToOI.put(child, soi); } }
@Override protected void generatePredicate(NodeProcessorCtx procCtx, FilterOperator fop, TableScanOperator top) throws SemanticException, UDFArgumentException { OpWalkerCtx owc = (OpWalkerCtx) procCtx; // Otherwise this is not a sampling predicate and we need to ExprNodeDesc predicate = fop.getConf().getPredicate(); String alias = top.getConf().getAlias(); // Generate the partition pruning predicate ExprNodeDesc ppr_pred = ExprProcFactory.genPruner(alias, predicate); // Add the pruning predicate to the table scan operator addPruningPred(owc.getOpToPartPruner(), top, ppr_pred); }
private static boolean findSourceColumn( LineageCtx lctx, Predicate cond, String tabAlias, String alias) { for (Map.Entry<String, TableScanOperator> topOpMap: lctx.getParseCtx().getTopOps().entrySet()) { TableScanOperator tableScanOp = topOpMap.getValue(); Table tbl = tableScanOp.getConf().getTableMetadata(); if (tbl.getTableName().equals(tabAlias) || tabAlias.equals(tableScanOp.getConf().getAlias())) { for (FieldSchema column: tbl.getCols()) { if (column.getName().equals(alias)) { TableAliasInfo table = new TableAliasInfo(); table.setTable(tbl.getTTable()); table.setAlias(tabAlias); BaseColumnInfo colInfo = new BaseColumnInfo(); colInfo.setColumn(column); colInfo.setTabAlias(table); cond.getBaseCols().add(colInfo); return true; } } } } return false; }
@Override public boolean equals(TableScanOperator op1, TableScanOperator op2) { Preconditions.checkNotNull(op1); Preconditions.checkNotNull(op2); TableScanDesc op1Conf = op1.getConf(); TableScanDesc op2Conf = op2.getConf(); if (compareString(op1Conf.getAlias(), op2Conf.getAlias()) && compareExprNodeDesc(op1Conf.getFilterExpr(), op2Conf.getFilterExpr()) && op1Conf.getRowLimit() == op2Conf.getRowLimit() && op1Conf.isGatherStats() == op2Conf.isGatherStats()) { return true; } else { return false; } } }
@SuppressWarnings("unchecked") private MapInput generateMapInput(SparkPlan sparkPlan, MapWork mapWork) throws Exception { JobConf jobConf = cloneJobConf(mapWork); Class ifClass = getInputFormat(jobConf, mapWork); sc.sc().setCallSite(CallSite.apply(mapWork.getName(), "")); JavaPairRDD<WritableComparable, Writable> hadoopRDD; if (mapWork.getNumMapTasks() != null) { jobConf.setNumMapTasks(mapWork.getNumMapTasks()); hadoopRDD = sc.hadoopRDD(jobConf, ifClass, WritableComparable.class, Writable.class, mapWork.getNumMapTasks()); } else { hadoopRDD = sc.hadoopRDD(jobConf, ifClass, WritableComparable.class, Writable.class); } boolean toCache = false/*cloneToWork.containsKey(mapWork)*/; String tables = mapWork.getAllRootOperators().stream() .filter(op -> op instanceof TableScanOperator) .map(ts -> ((TableScanDesc) ts.getConf()).getAlias()) .collect(Collectors.joining(", ")); String rddName = mapWork.getName() + " (" + tables + ", " + hadoopRDD.getNumPartitions() + (toCache ? ", cached)" : ")"); // Caching is disabled for MapInput due to HIVE-8920 MapInput result = new MapInput(sparkPlan, hadoopRDD, toCache, rddName, mapWork); return result; }
@Override protected void generatePredicate(NodeProcessorCtx procCtx, FilterOperator fop, TableScanOperator top) throws SemanticException, UDFArgumentException { LBOpWalkerCtx owc = (LBOpWalkerCtx) procCtx; // Otherwise this is not a sampling predicate and we need to ExprNodeDesc predicate = fop.getConf().getPredicate(); String alias = top.getConf().getAlias(); // Generate the list bucketing pruning predicate ExprNodeDesc lbprPred = LBExprProcFactory.genPruner(alias, predicate, owc.getPart()); /* * add list bucketing predicate to to the table scan operator */ addPruningPred(owc.getOpToPartToLBPruner(), top, lbprPred, owc.getPart()); }
@Override protected void generatePredicate(NodeProcessorCtx procCtx, FilterOperator fop, TableScanOperator top) throws SemanticException, UDFArgumentException { LBOpWalkerCtx owc = (LBOpWalkerCtx) procCtx; // Otherwise this is not a sampling predicate and we need to ExprNodeDesc predicate = fop.getConf().getPredicate(); String alias = top.getConf().getAlias(); // Generate the list bucketing pruning predicate ExprNodeDesc lbprPred = LBExprProcFactory.genPruner(alias, predicate, owc.getPart()); /* * add list bucketing predicate to to the table scan operator */ addPruningPred(owc.getOpToPartToLBPruner(), top, lbprPred, owc.getPart()); }
public MapWork createMapWork(GenTezProcContext context, Operator<?> root, TezWork tezWork, PrunedPartitionList partitions) throws SemanticException { assert root.getParentOperators().isEmpty(); MapWork mapWork = new MapWork(Utilities.MAPNAME + context.nextSequenceNumber()); LOG.debug("Adding map work (" + mapWork.getName() + ") for " + root); // map work starts with table scan operators assert root instanceof TableScanOperator; TableScanOperator ts = (TableScanOperator) root; String alias = ts.getConf().getAlias(); setupMapWork(mapWork, context, partitions, ts, alias); if (ts.getConf().getTableMetadata() != null && ts.getConf().getTableMetadata().isDummyTable()) { mapWork.setDummyTableScan(true); } if (ts.getConf().getNumBuckets() > 0) { mapWork.setIncludedBuckets(ts.getConf().getIncludedBuckets()); } // add new item to the tez work tezWork.add(mapWork); return mapWork; }
public MapWork createMapWork(GenTezProcContext context, Operator<?> root, TezWork tezWork, PrunedPartitionList partitions) throws SemanticException { assert root.getParentOperators().isEmpty(); MapWork mapWork = new MapWork(Utilities.MAPNAME + context.nextSequenceNumber()); LOG.debug("Adding map work (" + mapWork.getName() + ") for " + root); // map work starts with table scan operators assert root instanceof TableScanOperator; TableScanOperator ts = (TableScanOperator) root; String alias = ts.getConf().getAlias(); setupMapWork(mapWork, context, partitions, ts, alias); if (ts.getConf().getTableMetadata() != null && ts.getConf().getTableMetadata().isDummyTable()) { mapWork.setDummyTableScan(true); } if (ts.getConf().getNumBuckets() > 0) { mapWork.setIncludedBuckets(ts.getConf().getIncludedBuckets()); } // add new item to the tez work tezWork.add(mapWork); return mapWork; }