private Map<IConnectorDescriptor, TargetConstraint> setupConnectors() throws AlgebricksException { Map<IConnectorDescriptor, TargetConstraint> tgtConstraints = new HashMap<>(); for (ILogicalOperator exchg : connectors.keySet()) { ILogicalOperator inOp = inEdges.get(exchg).get(0); ILogicalOperator outOp = outEdges.get(exchg).get(0); IOperatorDescriptor inOpDesc = findOpDescForAlgebraicOp(inOp); IOperatorDescriptor outOpDesc = findOpDescForAlgebraicOp(outOp); Pair<IConnectorDescriptor, TargetConstraint> connPair = connectors.get(exchg); IConnectorDescriptor conn = connPair.first; int producerPort = outEdges.get(inOp).indexOf(exchg); int consumerPort = inEdges.get(outOp).indexOf(exchg); jobSpec.connect(conn, inOpDesc, producerPort, outOpDesc, consumerPort); if (connPair.second != null) { tgtConstraints.put(conn, connPair.second); } } return tgtConstraints; }
private static void populateDataToRebalanceTarget(Dataset source, Dataset target, MetadataProvider metadataProvider, IHyracksClientConnection hcc) throws Exception { JobSpecification spec = new JobSpecification(); TxnId txnId = metadataProvider.getTxnIdFactory().create(); JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(txnId, true); spec.setJobletEventListenerFactory(jobEventListenerFactory); // The pipeline starter. IOperatorDescriptor starter = DatasetUtil.createDummyKeyProviderOp(spec, source, metadataProvider); // Creates primary index scan op. IOperatorDescriptor primaryScanOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, source); // Creates secondary BTree upsert op. IOperatorDescriptor upsertOp = createPrimaryIndexUpsertOp(spec, metadataProvider, source, target); // The final commit operator. IOperatorDescriptor commitOp = createUpsertCommitOp(spec, metadataProvider, target); // Connects empty-tuple-source and scan. spec.connect(new OneToOneConnectorDescriptor(spec), starter, 0, primaryScanOp, 0); // Connects scan and upsert. int numKeys = target.getPrimaryKeys().size(); int[] keys = IntStream.range(0, numKeys).toArray(); IConnectorDescriptor connectorDescriptor = new MToNPartitioningConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(keys, target.getPrimaryHashFunctionFactories(metadataProvider))); spec.connect(connectorDescriptor, primaryScanOp, 0, upsertOp, 0); // Connects upsert and sink. spec.connect(new OneToOneConnectorDescriptor(spec), upsertOp, 0, commitOp, 0); // Executes the job. JobUtils.runJob(hcc, spec, true); }
if (isOverridingKeyFieldTypes && !enforcedItemType.equals(itemType)) { sourceOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced()); spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0); metaOp.setSourceLocation(sourceLoc); spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, asterixAssignOp, 0); if (anySecondaryKeyIsNullable || isOverridingKeyFieldTypes) { spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, tokenizerOp, 0); } else { spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, tokenizerOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), tokenizerOp, 0, sortOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, invIndexBulkLoadOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), invIndexBulkLoadOp, 0, metaOp, 0); spec.addRoot(metaOp); spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
if (isOverridingKeyFieldTypes && !enforcedItemType.equals(itemType)) { sourceOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced()); spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0); metaOp.setSourceLocation(sourceLoc); spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, asterixAssignOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, processorOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), processorOp, 0, tokenizerOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), tokenizerOp, 0, sortOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, invIndexBulkLoadOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), invIndexBulkLoadOp, 0, metaOp, 0); spec.addRoot(metaOp); spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
if (isOverridingKeyFieldTypes && !enforcedItemType.equals(itemType)) { sourceOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced()); spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0); metaOp.setSourceLocation(sourceLoc); spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, asterixAssignOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, processorOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), processorOp, 0, sortOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), secondaryBulkLoadOp, 0, metaOp, 0); spec.addRoot(metaOp); spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
if (isOverridingKeyFieldTypes && !enforcedItemType.equals(itemType)) { sourceOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced()); spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, asterixAssignOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, processorOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), processorOp, 0, sortOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), secondaryBulkLoadOp, 0, metaOp, 0); spec.addRoot(metaOp); spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
private static Pair<JobSpecification, IAdapterFactory> buildFeedIntakeJobSpec(Feed feed, MetadataProvider metadataProvider, FeedPolicyAccessor policyAccessor) throws Exception { JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext()); spec.setFrameSize(metadataProvider.getApplicationContext().getCompilerProperties().getFrameSize()); IAdapterFactory adapterFactory; IOperatorDescriptor feedIngestor; AlgebricksPartitionConstraint ingesterPc; Triple<IOperatorDescriptor, AlgebricksPartitionConstraint, IAdapterFactory> t = metadataProvider.buildFeedIntakeRuntime(spec, feed, policyAccessor); feedIngestor = t.first; ingesterPc = t.second; adapterFactory = t.third; AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, feedIngestor, ingesterPc); NullSinkOperatorDescriptor nullSink = new NullSinkOperatorDescriptor(spec); AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, nullSink, ingesterPc); spec.connect(new OneToOneConnectorDescriptor(spec), feedIngestor, 0, nullSink, 0); spec.addRoot(nullSink); return Pair.of(spec, adapterFactory); }
new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory .of(UTF8StringPointable.FACTORY) })); spec.connect(scanGroupConn, wordScanner, 0, gBy, 0); } else { IBinaryComparatorFactory[] cfs = new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory .of(UTF8StringPointable.FACTORY) })); spec.connect(scanSortConn, wordScanner, 0, sorter, 0); createPartitionConstraint(spec, gBy, outSplits); OneToOneConnectorDescriptor sortGroupConn = new OneToOneConnectorDescriptor(spec); spec.connect(sortGroupConn, sorter, 0, gBy, 0); spec.connect(gbyPrinterConn, gBy, 0, writer, 0);
new FieldHashPartitionComputerFactory(new int[] { 1 }, new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) })); spec.connect(ordJoinConn, ordScanner, 0, join, 1); spec.connect(custJoinConn, custScanner, 0, join, 0); new FieldHashPartitionComputerFactory(new int[] { 6 }, new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) })); spec.connect(joinGroupConn, join, 0, gby, 0); spec.connect(endingPrinterConn, endingOp, 0, writer, 0);
public static void flushDataset(IHyracksClientConnection hcc, MetadataProvider metadataProvider, Dataset dataset) throws Exception { CompilerProperties compilerProperties = metadataProvider.getApplicationContext().getCompilerProperties(); int frameSize = compilerProperties.getFrameSize(); JobSpecification spec = new JobSpecification(frameSize); RecordDescriptor[] rDescs = new RecordDescriptor[] { new RecordDescriptor(new ISerializerDeserializer[] {}) }; AlgebricksMetaOperatorDescriptor emptySource = new AlgebricksMetaOperatorDescriptor(spec, 0, 1, new IPushRuntimeFactory[] { new EmptyTupleSourceRuntimeFactory() }, rDescs); TxnId txnId = metadataProvider.getTxnIdFactory().create(); FlushDatasetOperatorDescriptor flushOperator = new FlushDatasetOperatorDescriptor(spec, txnId, dataset.getDatasetId()); spec.connect(new OneToOneConnectorDescriptor(spec), emptySource, 0, flushOperator, 0); Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider.getSplitProviderAndConstraints(dataset, dataset.getDatasetName()); AlgebricksPartitionConstraint primaryPartitionConstraint = primarySplitsAndConstraint.second; AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, emptySource, primaryPartitionConstraint); JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(txnId, true); spec.setJobletEventListenerFactory(jobEventListenerFactory); JobUtils.runJob(hcc, spec, true); }
new FieldHashPartitionComputerFactory(keys, new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) })); spec.connect(scanGroupConn, wordScanner, 0, gBy, 0); } else { IBinaryComparatorFactory[] cfs = new FieldHashPartitionComputerFactory(keys, new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) })); spec.connect(scanSortConn, wordScanner, 0, sorter, 0); createPartitionConstraint(spec, gBy, outSplits); OneToOneConnectorDescriptor sortGroupConn = new OneToOneConnectorDescriptor(spec); spec.connect(sortGroupConn, sorter, 0, gBy, 0); spec.connect(gbyPrinterConn, gBy, 0, writer, 0);
JobHelper.createPartitionConstraint(spec, printer, splitNCs); spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, secondarySearchOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), secondarySearchOp, 0, primarySearchOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), primarySearchOp, 0, printer, 0);
PartitionConstraintHelper.addAbsoluteLocationConstraint(jobSpec, writeOperator, HyracksUtils.NC1_ID); jobSpec.connect(new OneToOneConnectorDescriptor(jobSpec), readOperator, 0, sortOperator, 0); jobSpec.connect( new MToNPartitioningMergingConnectorDescriptor(jobSpec, new FieldHashPartitionComputerFactory(new int[] { 0 },
PartitionConstraintHelper.addAbsoluteLocationConstraint(jobSpec, writeOperator, HyracksUtils.NC1_ID); jobSpec.connect(new OneToOneConnectorDescriptor(jobSpec), readOperator, 0, sortOperator, 0); jobSpec.connect( new MToNPartitioningMergingConnectorDescriptor(jobSpec, new FieldHashPartitionComputerFactory(new int[] { 0 },