@Override public ILogicalOperator visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException { ArrayList<LogicalVariable> newInputList = new ArrayList<>(); newInputList.addAll(op.getVariables()); Mutable<ILogicalExpression> newSelectCondition = op.getSelectCondition() != null ? deepCopyExpressionRef(op.getSelectCondition()) : null; DataSourceScanOperator newOp = new DataSourceScanOperator(newInputList, op.getDataSource(), newSelectCondition, op.getOutputLimit()); return newOp; }
IMetadataProvider mp = context.getMetadataProvider(); IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op); List<LogicalVariable> vars = scan.getVariables(); List<LogicalVariable> projectVars = scan.getProjectVariables(); if (scan.getSelectCondition() != null) { tupleFilterFactory = context.getMetadataProvider().createTupleFilterFactory( new IOperatorSchema[] { opSchema }, typeEnv, scan.getSelectCondition().getValue(), context); scan.isProjectPushed(), scan.getMinFilterVars(), scan.getMaxFilterVars(), tupleFilterFactory, scan.getOutputLimit(), opSchema, typeEnv, context, builder.getJobSpec(), implConfig); IOperatorDescriptor opDesc = p.first; opDesc.setSourceLocation(scan.getSourceLocation()); builder.contributeHyracksOperator(scan, opDesc); if (p.second != null) { ILogicalOperator srcExchange = scan.getInputs().get(0).getValue(); builder.contributeGraphEdge(srcExchange, 0, scan, 0);
@Override public Void visitDataScanOperator(DataSourceScanOperator op, Integer indent) throws AlgebricksException { AlgebricksAppendable plan = addIndent(indent).append( "data-scan " + op.getProjectVariables() + "<-" + op.getVariables() + " <- " + op.getDataSource()); appendFilterInformation(plan, op.getMinFilterVars(), op.getMaxFilterVars()); appendSelectConditionInformation(plan, op.getSelectCondition(), indent); appendLimitInformation(plan, op.getOutputLimit()); return null; }
private boolean isScanPushable(DataSourceScanOperator op, Set<LogicalVariable> selectedVariables) { if (op.getOutputLimit() >= 0) { return false; } if (!op.getInputs().isEmpty() && op.getInputs().get(0).getValue().getOperatorTag() != LogicalOperatorTag.EMPTYTUPLESOURCE) { return false; } if (((DataSource) op.getDataSource()).getDatasourceType() != DataSource.Type.INTERNAL_DATASET) { return false; } if (!op.getScanVariables().containsAll(selectedVariables)) { return false; } return true; }
List<LogicalVariable> variables = new ArrayList<>(); variables.add(unnest.getVariable()); DataSourceScanOperator scan = new DataSourceScanOperator(variables, datasource); scan.setSourceLocation(unnest.getSourceLocation()); List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs(); scanInpList.addAll(unnest.getInputs()); opRef.setValue(scan);
ets.setExecutionMode(PARTITIONED); DataSourceScanOperator scanOperator = new DataSourceScanOperator(Collections.emptyList(), null); scanOperator.setExecutionMode(PARTITIONED); scanOperator.getInputs().add(new MutableObject<>(ets));
@Override public Void visitDataScanOperator(DataSourceScanOperator op, IOptimizationContext ctx) throws AlgebricksException { ILogicalOperator inp1 = op.getInputs().get(0).getValue(); Map<LogicalVariable, EquivalenceClass> eqClasses = getOrCreateEqClasses(op, ctx); Map<LogicalVariable, EquivalenceClass> propagatedEqClasses = getOrComputeEqClasses(inp1, ctx); eqClasses.putAll(propagatedEqClasses); ctx.putEquivalenceClassMap(op, eqClasses); List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>(getOrComputeFDs(inp1, ctx)); ctx.putFDList(op, fds); op.getDataSource().computeFDs(op.getVariables(), fds); return null; }
@Override public Boolean visitDataScanOperator(DataSourceScanOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; if (aop.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) { return Boolean.FALSE; } DataSourceScanOperator argScan = (DataSourceScanOperator) arg; if (!argScan.getDataSource().toString().equals(op.getDataSource().toString())) { return Boolean.FALSE; } DataSourceScanOperator scanOpArg = (DataSourceScanOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), scanOpArg.getVariables()) && op.getDataSource().toString().equals(scanOpArg.getDataSource().toString()); return isomorphic; }
DataSource ds = (DataSource) dataSourceScanOp.getDataSource(); if (dataset.getDatasetName().compareTo(((DatasetDataSource) ds).getDataset().getDatasetName()) == 0) { List<LogicalVariable> minFilterVars = new ArrayList<>(); dataSourceScanOp.getSourceLocation()); dataSourceScanOp.setMinFilterVars(minFilterVars); dataSourceScanOp.setMaxFilterVars(maxFilterVars); dataSourceScanOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); assignOp.getInputs().add(new MutableObject<>(dataSourceScanOp.getInputs().get(0).getValue())); dataSourceScanOp.getInputs().get(0).setValue(assignOp);
LogicalVariable recVar = scan.getVariables().get(scan.getVariables().size() - 1); if (recVar != var) { return false; DataSourceId asid = ((IDataSource<DataSourceId>) scan.getDataSource()).getId(); throw new CompilationException(ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE, scan.getSourceLocation(), asid.getDatasourceName(), asid.getDataverseName());
for (int i = 0; i < pkVars.size() - 1; i++) { for (int j = i + 1; j < pkVars.size(); j++) { IDataSource<?> leftSource = ((DataSourceScanOperator) (varOpMap.get(pkVars.get(i)))).getDataSource(); IDataSource<?> rightSource = ((DataSourceScanOperator) (varOpMap.get(pkVars.get(j)))).getDataSource(); if (leftSource.getId().toString().equals(rightSource.getId().toString())) { mapVariablesTopDown(varOpMap.get(pkVars.get(i)), varOpMap.get(pkVars.get(j)), variableMapping);
AssignOperator assign = (AssignOperator) op2; DataSourceScanOperator opNew = new DataSourceScanOperator(assign.getVariables(), ds); opNew.getInputs().addAll(assign.getInputs()); opRef2.setValue(opNew); return true;
@Override public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException { producedVariables.addAll(op.getVariables()); return null; }
inputOp.setSourceLocation(dataSourceScan.getSourceLocation()); OperatorManipulationUtil.deepCopy(dataSourceScan.getInputs().get(0).getValue()))); context.computeAndSetTypeEnvironmentForOperator(inputOp); inputOp.setExecutionMode(dataSourceScan.getExecutionMode()); } else {
@Override public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) { if (op.getAdditionalFilteringExpressions() != null) { for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) { e.getValue().getUsedVariables(usedVariables); } } if (op.getSelectCondition() != null) { Set<LogicalVariable> usedVariablesBySelect = new HashSet<>(); op.getSelectCondition().getValue().getUsedVariables(usedVariablesBySelect); usedVariablesBySelect.removeAll(op.getVariables()); usedVariables.addAll(usedVariablesBySelect); } return null; }
@Override public Void visitDataScanOperator(DataSourceScanOperator op, Pair<LogicalVariable, LogicalVariable> pair) throws AlgebricksException { List<LogicalVariable> variables = op.getVariables(); for (int i = 0; i < variables.size(); i++) { if (variables.get(i) == pair.first) { variables.set(i, pair.second); return null; } } if (op.getSelectCondition() != null) { op.getSelectCondition().getValue().substituteVar(pair.first, pair.second); } substVarTypes(op, pair); return null; }
} else if (grandparentOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) { DataSourceScanOperator dssOp = (DataSourceScanOperator) grandparentOp; inputRecord = dssOp.getVariables().get(0); inputRecordSourceLoc = dssOp.getSourceLocation(); } else if (grandparentOp.getOperatorTag() == LogicalOperatorTag.ASSIGN) { AbstractLogicalOperator greatgrandparentOp =
DataSource dataSource = (DataSource) scanOp.getDataSource(); if (dataSource.getDatasourceType() != DataSource.Type.FEED) { return false; op.getInputs().get(0).setValue(exchangeOp); exchangeOp.getInputs().add(new MutableObject<ILogicalOperator>(scanOp)); ExecutionMode em = scanOp.getExecutionMode(); exchangeOp.setExecutionMode(em); exchangeOp.computeDeliveredPhysicalProperties(context);
@Override public String visitDataScanOperator(DataSourceScanOperator op, Boolean showDetails) throws AlgebricksException { stringBuilder.setLength(0); stringBuilder.append("data-scan ").append(op.getProjectVariables()).append("<-").append(op.getVariables()) .append(" <- ").append(op.getDataSource()); appendFilterInformation(op.getMinFilterVars(), op.getMaxFilterVars()); appendSelectConditionInformation(op.getSelectCondition()); appendLimitInformation(op.getOutputLimit()); appendSchema(op, showDetails); appendAnnotations(op, showDetails); appendPhysicalOperatorInfo(op, showDetails); return stringBuilder.toString(); }
feedDataScanOutputVariables.addAll(pkVars); DataSourceScanOperator scan = new DataSourceScanOperator(feedDataScanOutputVariables, ds); scan.setSourceLocation(unnest.getSourceLocation()); List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs(); scanInpList.addAll(unnest.getInputs()); opRef.setValue(scan);