@Override public ILogicalOperator visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, Void arg) throws AlgebricksException { List<Mutable<ILogicalExpression>> newKeyExpressions = new ArrayList<>(); deepCopyExpressionRefs(newKeyExpressions, op.getPrimaryKeyExpressions()); List<Mutable<ILogicalExpression>> newLSMComponentFilterExpressions = new ArrayList<>(); deepCopyExpressionRefs(newKeyExpressions, op.getAdditionalFilteringExpressions()); InsertDeleteUpsertOperator insertDeleteOp = new InsertDeleteUpsertOperator(op.getDataSource(), deepCopyExpressionRef(op.getPayloadExpression()), newKeyExpressions, op.getOperation(), op.isBulkload()); insertDeleteOp.setAdditionalFilteringExpressions(newLSMComponentFilterExpressions); return insertDeleteOp; }
@Override public String visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, Boolean showDetails) { stringBuilder.setLength(0); String header = getIndexOpString(op.getOperation()); stringBuilder.append(header).append(str(op.getDataSource())).append(" from record: ") .append(op.getPayloadExpression().getValue().toString()); if (op.getAdditionalNonFilteringExpressions() != null) { stringBuilder.append(", meta: "); printExprList(op.getAdditionalNonFilteringExpressions()); } stringBuilder.append(" partitioned by "); printExprList(op.getPrimaryKeyExpressions()); if (op.getOperation() == Kind.UPSERT) { stringBuilder.append(" out: ([record-before-upsert:").append(op.getBeforeOpRecordVar()); if (op.getBeforeOpAdditionalNonFilteringVars() != null) { stringBuilder.append(", additional-before-upsert: ").append(op.getBeforeOpAdditionalNonFilteringVars()); } stringBuilder.append("]) "); } if (op.isBulkload()) { stringBuilder.append(" [bulkload]"); } appendSchema(op, showDetails); appendAnnotations(op, showDetails); appendPhysicalOperatorInfo(op, showDetails); return stringBuilder.toString(); }
@Override public Boolean visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; if (aop.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE_UPSERT) { return Boolean.FALSE; } InsertDeleteUpsertOperator insertOpArg = (InsertDeleteUpsertOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), insertOpArg.getSchema()); if (!op.getDataSource().equals(insertOpArg.getDataSource())) { isomorphic = false; } if (!op.getPayloadExpression().equals(insertOpArg.getPayloadExpression())) { isomorphic = false; } return isomorphic; }
boolean isBulkload = primaryIndexModificationOp.isBulkload(); ILogicalExpression newRecordExpr = primaryIndexModificationOp.getPayloadExpression().getValue(); List<Mutable<ILogicalExpression>> newMetaExprs = primaryIndexModificationOp.getAdditionalNonFilteringExpressions(); LogicalVariable newRecordVar; LogicalVariable newMetaVar = null; SourceLocation sourceLoc = primaryIndexModificationOp.getSourceLocation(); (AbstractLogicalOperator) primaryIndexModificationOp.getInputs().get(0).getValue(); newRecordVar = getRecordVar(context, inputOp, newRecordExpr, 0); if (newMetaExprs != null && !newMetaExprs.isEmpty()) { DataSource datasetSource = (DataSource) primaryIndexModificationOp.getDataSource(); MetadataProvider mp = (MetadataProvider) context.getMetadataProvider(); String dataverseName = datasetSource.getId().getDataverseName(); filteringExpressions = new ArrayList<>(); for (Mutable<ILogicalExpression> filteringExpression : primaryIndexModificationOp .getAdditionalFilteringExpressions()) { filteringExpression.getValue().getUsedVariables(filteringVars); for (LogicalVariable var : filteringVars) { if (secondaryIndexTotalCnt > 1 && primaryIndexModificationOp.isBulkload()) { if (primaryIndexModificationOp.getOperation() == Kind.INSERT || primaryIndexModificationOp.getOperation() == Kind.UPSERT || primaryIndexModificationOp.getOperation() == Kind.DELETE) { injectFieldAccessesForIndexes(context, dataset, indexes, fieldVarsForNewRecord, recType, metaType,
List<LogicalVariable> additionalFilteringKeys = null; List<LogicalVariable> additionalNonFilterVariables = null; if (opLoad.getAdditionalNonFilteringExpressions() != null) { additionalNonFilterVariables = new ArrayList<LogicalVariable>(); getKeys(opLoad.getAdditionalNonFilteringExpressions(), additionalNonFilterVariables); payload = getKeysAndLoad(opLoad.getPayloadExpression(), opLoad.getPrimaryKeyExpressions(), keys); if (opLoad.getAdditionalFilteringExpressions() != null) { additionalFilteringKeys = new ArrayList<LogicalVariable>(); getKeys(opLoad.getAdditionalFilteringExpressions(), additionalFilteringKeys); if (opLoad.isBulkload()) { op.setPhysicalOperator(new BulkloadPOperator(payload, keys, additionalFilteringKeys, additionalNonFilterVariables, opLoad.getDataSource())); } else { op.setPhysicalOperator(new InsertDeleteUpsertPOperator(payload, keys, additionalFilteringKeys, opLoad.getDataSource(), opLoad.getOperation(), additionalNonFilterVariables));
upsertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, metaExpSingletonList, InsertDeleteUpsertOperator.Kind.UPSERT, false); upsertOp.setUpsertIndicatorVar(context.newVar()); upsertOp.setUpsertIndicatorVarType(BuiltinType.ABOOLEAN); upsertOp.setPrevRecordVar(context.newVar()); upsertOp.setPrevRecordType(targetDatasource.getItemType()); upsertOp.setSourceLocation(sourceLoc); if (targetDatasource.getDataset().hasMetaPart()) { List<LogicalVariable> metaVars = new ArrayList<>(); metaVars.add(context.newVar()); upsertOp.setPrevAdditionalNonFilteringVars(metaVars); List<Object> metaTypes = new ArrayList<>(); metaTypes.add(targetDatasource.getMetaItemType()); upsertOp.setPrevAdditionalNonFilteringTypes(metaTypes); upsertOp.setPrevFilterVar(context.newVar()); upsertOp.setPrevFilterType( ((ARecordType) targetDatasource.getItemType()).getFieldType(additionalFilteringField.get(0))); additionalFilteringAssign.getInputs().clear(); additionalFilteringAssign.getInputs().add(assign.getInputs().get(0)); upsertOp.getInputs().add(new MutableObject<>(additionalFilteringAssign)); } else { upsertOp.getInputs().add(assign.getInputs().get(0)); metaAndKeysAssign.setSourceLocation(sourceLoc); topOp.getInputs().set(0, new MutableObject<>(metaAndKeysAssign)); upsertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); } else {
@Override public Void visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, Void arg) { // 1. The record variable op.getPayloadExpression().getValue().getUsedVariables(usedVariables); // 2. The primary key variables for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) { e.getValue().getUsedVariables(usedVariables); } // 3. The filters variables if (op.getAdditionalFilteringExpressions() != null) { for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) { e.getValue().getUsedVariables(usedVariables); } } // 4. The Other variables (Not key, Not payload, and Not Filter) if (op.getAdditionalNonFilteringExpressions() != null) { for (Mutable<ILogicalExpression> e : op.getAdditionalNonFilteringExpressions()) { e.getValue().getUsedVariables(usedVariables); } } return null; }
private ILogicalOperator translateDelete(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef, List<Mutable<ILogicalExpression>> varRefsForLoading, List<Mutable<ILogicalExpression>> additionalFilteringExpressions, ILogicalOperator assign, ICompiledDmlStatement stmt) throws AlgebricksException { SourceLocation sourceLoc = stmt.getSourceLocation(); if (targetDatasource.getDataset().hasMetaPart()) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, targetDatasource.getDataset().getDatasetName() + ": delete from dataset is not supported on Datasets with Meta records"); } InsertDeleteUpsertOperator deleteOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, InsertDeleteUpsertOperator.Kind.DELETE, false); deleteOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); deleteOp.getInputs().add(new MutableObject<>(assign)); deleteOp.setSourceLocation(sourceLoc); DelegateOperator leafOperator = new DelegateOperator(new CommitOperator(true)); leafOperator.getInputs().add(new MutableObject<>(deleteOp)); leafOperator.setSourceLocation(sourceLoc); return leafOperator; }
@SuppressWarnings({ "rawtypes", "unchecked" }) @Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { InsertDeleteUpsertOperator insertDeleteOp = (InsertDeleteUpsertOperator) op; assert insertDeleteOp.getOperation() == Kind.INSERT; assert insertDeleteOp.isBulkload(); IMetadataProvider mp = context.getMetadataProvider(); IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op); JobSpecification spec = builder.getJobSpec(); RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context); Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getInsertRuntime(dataSource, propagatedSchema, typeEnv, primaryKeys, payload, additionalFilteringKeys, additionalNonFilterVars, inputDesc, context, spec, true); IOperatorDescriptor opDesc = runtimeAndConstraints.first; opDesc.setSourceLocation(insertDeleteOp.getSourceLocation()); builder.contributeHyracksOperator(insertDeleteOp, opDesc); builder.contributeAlgebricksPartitionConstraint(opDesc, runtimeAndConstraints.second); ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, insertDeleteOp, 0); }
if (insertDeleteOp.getOperation() == InsertDeleteUpsertOperator.Kind.DELETE) { return false; DataSource dataSource = (DataSource) insertDeleteOperator.getDataSource(); IAType requiredRecordType = dataSource.getItemType(); insertDeleteOperator.getPayloadExpression().getValue().getUsedVariables(usedVariables); IVariableTypeEnvironment env = insertDeleteOperator.computeOutputTypeEnvironment(context); IAType inputRecordType = (IAType) env.getVarType(inputRecordVar);
if (op2.getOperatorTag() == LogicalOperatorTag.INSERT_DELETE_UPSERT) { InsertDeleteUpsertOperator insertDeleteOp = (InsertDeleteUpsertOperator) op2; if (insertDeleteOp.getOperation() == InsertDeleteUpsertOperator.Kind.DELETE) { return false; DataSource dataSource = (DataSource) insertDeleteOperator.getDataSource(); requiredRecordType = (ARecordType) dataSource.getItemType(); ILogicalExpression expr = insertDeleteOperator.getPayloadExpression().getValue(); List<LogicalVariable> payloadVars = new ArrayList<>(); expr.getUsedVariables(payloadVars);
if (!insertDeleteUpsertOperator.isBulkload()) { primaryKeyExprs = insertDeleteUpsertOperator.getPrimaryKeyExpressions(); dataset = ((DatasetDataSource) insertDeleteUpsertOperator.getDataSource()).getDataset(); break;
if (insertOp.getOperation() != Kind.INSERT && insertOp.getOperation() != Kind.UPSERT) { return false; DatasetDataSource dds = (DatasetDataSource) insertOp.getDataSource(); boolean autogenerated = ((InternalDatasetDetails) dds.getDataset().getDatasetDetails()).isAutogenerated(); if (!autogenerated) { if (((DataSource) insertOp.getDataSource()).getDatasourceType() != Type.INTERNAL_DATASET) { return false; SourceLocation insertOpSourceLoc = insertOp.getSourceLocation(); List<String> pkFieldName = ((InternalDatasetDetails) dds.getDataset().getDatasetDetails()).getPrimaryKey().get(0);
@Override public Void visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, Pair<LogicalVariable, LogicalVariable> pair) throws AlgebricksException { op.getPayloadExpression().getValue().substituteVar(pair.first, pair.second); for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) { e.getValue().substituteVar(pair.first, pair.second); } substVarTypes(op, pair); return null; }
@Override public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); if (op.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE_UPSERT) { return false; } InsertDeleteUpsertOperator insertOp = (InsertDeleteUpsertOperator) op; boolean sameDataset = checkIfInsertAndScanDatasetsSame(op, ((DatasetDataSource) insertOp.getDataSource()).getDataset().getDatasetName()); if (sameDataset) { MaterializeOperator materializeOperator = new MaterializeOperator(); materializeOperator.setSourceLocation(op.getSourceLocation()); MaterializePOperator materializePOperator = new MaterializePOperator(true); materializeOperator.setPhysicalOperator(materializePOperator); materializeOperator.getInputs() .add(new MutableObject<ILogicalOperator>(insertOp.getInputs().get(0).getValue())); context.computeAndSetTypeEnvironmentForOperator(materializeOperator); insertOp.getInputs().clear(); insertOp.getInputs().add(new MutableObject<ILogicalOperator>(materializeOperator)); context.computeAndSetTypeEnvironmentForOperator(insertOp); return true; } else { return false; } }
new AssignOperator(insertedVar, new MutableObject<>(insertOp.getPayloadExpression().getValue())); insertedVarAssignOperator.getInputs().add(insertOp.getInputs().get(0)); insertedVarAssignOperator.setSourceLocation(sourceLoc); insertOp.getInputs().set(0, new MutableObject<>(insertedVarAssignOperator));
private ILogicalOperator translateInsert(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef, List<Mutable<ILogicalExpression>> varRefsForLoading, List<Mutable<ILogicalExpression>> additionalFilteringExpressions, ILogicalOperator assign, ICompiledDmlStatement stmt) throws AlgebricksException { SourceLocation sourceLoc = stmt.getSourceLocation(); if (targetDatasource.getDataset().hasMetaPart()) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, targetDatasource.getDataset().getDatasetName() + ": insert into dataset is not supported on Datasets with Meta records"); } // Adds the insert operator. InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false); insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); insertOp.getInputs().add(new MutableObject<>(assign)); insertOp.setSourceLocation(sourceLoc); // Adds the commit operator. CompiledInsertStatement compiledInsert = (CompiledInsertStatement) stmt; Expression returnExpression = compiledInsert.getReturnExpression(); DelegateOperator rootOperator = new DelegateOperator(new CommitOperator(returnExpression == null)); rootOperator.getInputs().add(new MutableObject<>(insertOp)); rootOperator.setSourceLocation(sourceLoc); // Compiles the return expression. return processReturningExpression(rootOperator, insertOp, compiledInsert); }
@Override public Void visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, Integer indent) throws AlgebricksException { String header = getIndexOpString(op.getOperation()); addIndent(indent).append(header).append(str(op.getDataSource())).append(" from record: ") .append(op.getPayloadExpression().getValue().accept(exprVisitor, indent)); if (op.getAdditionalNonFilteringExpressions() != null) { buffer.append(", meta: "); pprintExprList(op.getAdditionalNonFilteringExpressions(), indent); } buffer.append(" partitioned by "); pprintExprList(op.getPrimaryKeyExpressions(), indent); if (op.getOperation() == Kind.UPSERT) { buffer.append(" out: ([record-before-upsert:" + op.getBeforeOpRecordVar() + ((op.getBeforeOpAdditionalNonFilteringVars() != null) ? (", additional-before-upsert: " + op.getBeforeOpAdditionalNonFilteringVars()) : "") + "]) "); } if (op.isBulkload()) { buffer.append(" [bulkload]"); } return null; }
InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, payloadRef, varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, true); insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); insertOp.setSourceLocation(sourceLoc); insertOp.getInputs().add(new MutableObject<>(additionalFilteringAssign)); } else { insertOp.getInputs().add(new MutableObject<>(assign));
@Override public Void visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, Integer indent) throws AlgebricksException { String header = "\"operator\": \"" + getIndexOpString(op.getOperation()) + "\",\n"; addIndent(indent).append(header); addIndent(indent).append(str("\"data-source\": \"" + op.getDataSource() + "\",\n")); addIndent(indent).append("\"from-record\": \"") .append(op.getPayloadExpression().getValue().accept(exprVisitor, indent) + "\""); if (op.getAdditionalNonFilteringExpressions() != null) { buffer.append(",\n\"meta\": {"); pprintExprList(op.getAdditionalNonFilteringExpressions(), 0); buffer.append("}"); pprintExprList(op.getPrimaryKeyExpressions(), 0); buffer.append("}"); if (op.getOperation() == Kind.UPSERT) { addIndent(indent).append(",\n\"out\": {\n"); addIndent(indent).append("\"record-before-upsert\": \"" + op.getBeforeOpRecordVar() + "\""); if (op.getBeforeOpAdditionalNonFilteringVars() != null) { buffer.append(",\n"); addIndent(indent) .append("\"additional-before-upsert\": \"" + op.getBeforeOpAdditionalNonFilteringVars() + "\""); if (op.isBulkload()) { buffer.append(",\n"); addIndent(indent).append("\"bulkload\": true");