public static JobSpecification createJobSpecification(ICcApplicationContext appCtx) { CompilerProperties compilerProperties = appCtx.getCompilerProperties(); int frameSize = compilerProperties.getFrameSize(); return new JobSpecification(frameSize); } }
private JobSpecification compilePlanImpl(ILogicalPlan plan, boolean isNestedPlan, IOperatorSchema outerPlanSchema, IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException { JobSpecification spec = new JobSpecification(context.getFrameSize()); if (jobEventListenerFactory != null) { spec.setJobletEventListenerFactory(jobEventListenerFactory); } List<ILogicalOperator> rootOps = new ArrayList<>(); JobBuilder builder = new JobBuilder(spec, context.getClusterLocations()); for (Mutable<ILogicalOperator> opRef : plan.getRoots()) { compileOpRef(opRef, spec, builder, outerPlanSchema); rootOps.add(opRef.getValue()); } reviseEdges(builder); operatorVisitedToParents.clear(); builder.buildSpec(rootOps); spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy()); // Do not do activity cluster planning because it is slow on large clusters spec.setUseConnectorPolicyForScheduling(false); if (isNestedPlan) { spec.setMetaOps(builder.getGeneratedMetaOps()); } return spec; }
private static JobSpecification combineIntakeCollectJobs(MetadataProvider metadataProvider, Feed feed, JobSpecification intakeJob, List<JobSpecification> jobsList, List<FeedConnection> feedConnections, String[] intakeLocations) throws AlgebricksException, HyracksDataException { JobSpecification jobSpec = new JobSpecification(intakeJob.getFrameSize());
private static void populateDataToRebalanceTarget(Dataset source, Dataset target, MetadataProvider metadataProvider, IHyracksClientConnection hcc) throws Exception { JobSpecification spec = new JobSpecification(); TxnId txnId = metadataProvider.getTxnIdFactory().create(); JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(txnId, true); spec.setJobletEventListenerFactory(jobEventListenerFactory); // The pipeline starter. IOperatorDescriptor starter = DatasetUtil.createDummyKeyProviderOp(spec, source, metadataProvider); // Creates primary index scan op. IOperatorDescriptor primaryScanOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, source); // Creates secondary BTree upsert op. IOperatorDescriptor upsertOp = createPrimaryIndexUpsertOp(spec, metadataProvider, source, target); // The final commit operator. IOperatorDescriptor commitOp = createUpsertCommitOp(spec, metadataProvider, target); // Connects empty-tuple-source and scan. spec.connect(new OneToOneConnectorDescriptor(spec), starter, 0, primaryScanOp, 0); // Connects scan and upsert. int numKeys = target.getPrimaryKeys().size(); int[] keys = IntStream.range(0, numKeys).toArray(); IConnectorDescriptor connectorDescriptor = new MToNPartitioningConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(keys, target.getPrimaryHashFunctionFactories(metadataProvider))); spec.connect(connectorDescriptor, primaryScanOp, 0, upsertOp, 0); // Connects upsert and sink. spec.connect(new OneToOneConnectorDescriptor(spec), upsertOp, 0, commitOp, 0); // Executes the job. JobUtils.runJob(hcc, spec, true); }
@Test public void testJobLocations() { final String nc1 = "nc1"; final String nc2 = "nc2"; final NodeJobTracker nodeJobTracker = new NodeJobTracker(); nodeJobTracker.notifyNodeJoin(nc1, null); nodeJobTracker.notifyNodeJoin(nc2, null); final JobSpecification jobSpec = new JobSpecification(); // add only nc1 to the job locations final ConstantExpression nc1Location = new ConstantExpression(nc1); final LValueConstraintExpression lValueMock = Mockito.mock(LValueConstraintExpression.class); jobSpec.getUserConstraints().add(new Constraint(lValueMock, nc1Location)); final String[] clusterLocation = new String[] { nc1, nc2 }; final AlgebricksAbsolutePartitionConstraint jobLocations = APIFramework.getJobLocations(jobSpec, nodeJobTracker, new AlgebricksAbsolutePartitionConstraint(clusterLocation)); // ensure nc2 wasn't included Assert.assertEquals(1, jobLocations.getLocations().length); Assert.assertEquals(nc1, jobLocations.getLocations()[0]); } }
private static JobSpecification createJob(FileSplit[] ordersSplits, FileSplit[] outputSplit, String memBufferAlg, int frameLimit, int frameSize, int limit, boolean usingHeapSorter) { JobSpecification spec = new JobSpecification();
public static void flushDataset(IHyracksClientConnection hcc, MetadataProvider metadataProvider, Dataset dataset) throws Exception { CompilerProperties compilerProperties = metadataProvider.getApplicationContext().getCompilerProperties(); int frameSize = compilerProperties.getFrameSize(); JobSpecification spec = new JobSpecification(frameSize); RecordDescriptor[] rDescs = new RecordDescriptor[] { new RecordDescriptor(new ISerializerDeserializer[] {}) }; AlgebricksMetaOperatorDescriptor emptySource = new AlgebricksMetaOperatorDescriptor(spec, 0, 1, new IPushRuntimeFactory[] { new EmptyTupleSourceRuntimeFactory() }, rDescs); TxnId txnId = metadataProvider.getTxnIdFactory().create(); FlushDatasetOperatorDescriptor flushOperator = new FlushDatasetOperatorDescriptor(spec, txnId, dataset.getDatasetId()); spec.connect(new OneToOneConnectorDescriptor(spec), emptySource, 0, flushOperator, 0); Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider.getSplitProviderAndConstraints(dataset, dataset.getDatasetName()); AlgebricksPartitionConstraint primaryPartitionConstraint = primarySplitsAndConstraint.second; AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, emptySource, primaryPartitionConstraint); JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(txnId, true); spec.setJobletEventListenerFactory(jobEventListenerFactory); JobUtils.runJob(hcc, spec, true); }
private static JobSpecification createJob(Options options) { JobSpecification spec = new JobSpecification(options.frameSize);
private static JobSpecification createJob(Options options) { JobSpecification spec = new JobSpecification(options.frameSize);
private static JobSpecification createJob(Options options) throws HyracksDataException { JobSpecification spec = new JobSpecification(options.frameSize);
JobSpecification jobSpec = new JobSpecification(); RecordDescriptor recordDesc = new RecordDescriptor(new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer() });
private static JobSpecification createJob(FileSplit[] inSplits, FileSplit[] outSplits, int htSize, long fileSize, int frameLimit, int frameSize, String alg, boolean outPlain) { JobSpecification spec = new JobSpecification(frameSize); IFileSplitProvider splitsProvider = new ConstantFileSplitProvider(inSplits);
private static JobSpecification createJob(FileSplit[] inSplits, FileSplit[] outSplits, String algo, int htSize, int sbSize, String format, int frameSize) { JobSpecification spec = new JobSpecification(frameSize);
private static JobSpecification createJob(Options options) { JobSpecification spec = new JobSpecification(options.frameSize);
double graceFactor, int memSize, int tableSize, boolean hasGroupBy, int frameSize) throws HyracksDataException { JobSpecification spec = new JobSpecification(frameSize);
public IPushRuntime getFullScanPipeline(IFrameWriter countOp, IHyracksTaskContext ctx, Dataset dataset, IAType[] primaryKeyTypes, ARecordType recordType, ARecordType metaType, NoMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, int[] filterFields, int[] primaryKeyIndexes, List<Integer> primaryKeyIndicators, StorageComponentProvider storageComponentProvider) throws HyracksDataException, AlgebricksException { IPushRuntime emptyTupleOp = new EmptyTupleSourceRuntimeFactory().createPushRuntime(ctx)[0]; JobSpecification spec = new JobSpecification(); PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset, primaryKeyTypes, recordType, metaType, mergePolicyFactory, mergePolicyProperties, filterFields, primaryKeyIndexes, primaryKeyIndicators); IIndexDataflowHelperFactory indexDataflowHelperFactory = new IndexDataflowHelperFactory( storageComponentProvider.getStorageManager(), primaryIndexInfo.getFileSplitProvider()); BTreeSearchOperatorDescriptor searchOpDesc = new BTreeSearchOperatorDescriptor(spec, primaryIndexInfo.rDesc, null, null, true, true, indexDataflowHelperFactory, false, false, null, NoOpOperationCallbackFactory.INSTANCE, filterFields, filterFields, false); BTreeSearchOperatorNodePushable searchOp = searchOpDesc.createPushRuntime(ctx, primaryIndexInfo.getSearchRecordDescriptorProvider(), ctx.getTaskAttemptId().getTaskId().getPartition(), 1); emptyTupleOp.setOutputFrameWriter(0, searchOp, primaryIndexInfo.getSearchRecordDescriptorProvider().getInputRecordDescriptor(null, 0)); searchOp.setOutputFrameWriter(0, countOp, primaryIndexInfo.rDesc); return emptyTupleOp; }
private static JobSpecification createJob(FileSplit[] inSplits, FileSplit[] outSplits, String algo, int htSize, int frameLimit, String format, int frameSize) { JobSpecification spec = new JobSpecification(frameSize);
private static JobSpecification createJob(Options options) throws HyracksDataException { JobSpecification spec = new JobSpecification(options.frameSize);
JobSpecification jobSpec = new JobSpecification(); RecordDescriptor recordDesc = new RecordDescriptor(new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer() });
private static JobSpecification createJob(FileSplit[] inSplits, FileSplit[] outSplits, int htSize, int sbSize, int frameSize, boolean sortOutput, int alg, boolean outPlain) { JobSpecification spec = new JobSpecification(frameSize); IFileSplitProvider splitsProvider = new ConstantFileSplitProvider(inSplits);