public static void visit(JobSpecification spec, IOperatorDescriptorVisitor visitor) throws HyracksException { Set<OperatorDescriptorId> seen = new HashSet<>(); for (IOperatorDescriptor op : spec.getOperatorMap().values()) { visitOperator(visitor, seen, op); } }
public List<IOperatorDescriptor> getGeneratedMetaOps() { List<IOperatorDescriptor> resultOps = new ArrayList<>(); for (IOperatorDescriptor opd : jobSpec.getOperatorMap().values()) { if (opd instanceof AlgebricksMetaOperatorDescriptor) { resultOps.add(opd); } } resultOps.sort((op1, op2) -> sendsOutput(op1, op2) ? 1 : sendsOutput(op2, op1) ? -1 : 0); return resultOps; }
private void setPartitionConstraintsTopdown(OperatorDescriptorId opId, Map<IConnectorDescriptor, TargetConstraint> tgtConstraints, IOperatorDescriptor parentOp) { List<IConnectorDescriptor> opInputs = jobSpec.getOperatorInputMap().get(opId); AlgebricksPartitionConstraint opConstraint; IOperatorDescriptor opDesc = jobSpec.getOperatorMap().get(opId); if (opInputs != null) { for (IConnectorDescriptor conn : opInputs) { ConnectorDescriptorId cid = conn.getConnectorId(); org.apache.commons.lang3.tuple.Pair<org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>, org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>> p = jobSpec.getConnectorOperatorMap().get(cid); IOperatorDescriptor src = p.getLeft().getLeft(); TargetConstraint constraint = tgtConstraints.get(conn); if (constraint != null) { if (constraint == TargetConstraint.SAME_COUNT) { opConstraint = partitionConstraintMap.get(opDesc); if (partitionConstraintMap.get(src) == null) { if (opConstraint != null) { partitionConstraintMap.put(src, opConstraint); AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, src, opConstraint); } } } } // Post Order DFS setPartitionConstraintsTopdown(src.getOperatorId(), tgtConstraints, opDesc); } } }
opSchema.addAllVariables(topOpInSubplanScm); Map<OperatorDescriptorId, IOperatorDescriptor> opMap = nestedJob.getOperatorMap(); List<? extends IOperatorDescriptor> metaOps = nestedJob.getMetaOps(); if (opMap.size() != metaOps.size()) {
List<IConnectorDescriptor> opInputs = jobSpec.getOperatorInputMap().get(opId); AlgebricksPartitionConstraint opConstraint = null; IOperatorDescriptor opDesc = jobSpec.getOperatorMap().get(opId); if (opInputs != null) { for (IConnectorDescriptor conn : opInputs) {
(FeedIntakeOperatorDescriptor) intakeJob.getOperatorMap().get(new OperatorDescriptorId(0)); FeedIntakeOperatorDescriptor ingestionOp; if (firstOp.getAdaptorFactory() == null) { JobSpecification subJob = jobsList.get(iter1); operatorIdMapping.clear(); Map<OperatorDescriptorId, IOperatorDescriptor> operatorsMap = subJob.getOperatorMap(); String datasetName = feedConnections.get(iter1).getDatasetName(); FeedConnectionId feedConnectionId = new FeedConnectionId(ingestionOp.getEntityId(), datasetName); Pair<IOperatorDescriptor, Integer> leftOp = entry.getValue().getLeft(); Pair<IOperatorDescriptor, Integer> rightOp = entry.getValue().getRight(); IOperatorDescriptor leftOpDesc = jobSpec.getOperatorMap().get(leftOp.getLeft().getOperatorId()); IOperatorDescriptor rightOpDesc = jobSpec.getOperatorMap().get(rightOp.getLeft().getOperatorId()); if (leftOp.getLeft() instanceof FeedCollectOperatorDescriptor) { jobSpec.connect(new OneToOneConnectorDescriptor(jobSpec), replicateOp, iter1, leftOpDesc, case PARTITION_LOCATION: opId = ((PartitionLocationExpression) lexpr).getOperatorDescriptorId(); IOperatorDescriptor opDesc = jobSpec.getOperatorMap().get(operatorIdMapping.get(opId)); List<LocationConstraint> locations = operatorLocations.get(opDesc.getOperatorId()); if (locations == null) { IOperatorDescriptor opDesc = jobSpec.getOperatorMap().get(entry.getKey()); IOperatorDescriptor opDesc = jobSpec.getOperatorMap().get(key); if (!operatorLocations.keySet().contains(key)) { PartitionConstraintHelper.addPartitionCountConstraint(jobSpec, opDesc, value);
Assert.assertNotNull(index); jobSpecification = IndexUtil.buildSecondaryIndexLoadingJobSpec(dataset, index, metadataProvider, null); jobSpecification.getOperatorMap().values().forEach(iOperatorDescriptor -> { Assert.assertFalse(iOperatorDescriptor instanceof AbstractSorterOperatorDescriptor); }); Assert.assertNotNull(index); jobSpecification = IndexUtil.buildSecondaryIndexLoadingJobSpec(dataset, index, metadataProvider, null); final long numOfSortOperators = jobSpecification.getOperatorMap().values().stream() .filter(op -> op instanceof AbstractSorterOperatorDescriptor).count(); Assert.assertTrue(numOfSortOperators != 0);