public static RecordDescriptor createFileIndexRecordDescriptor() { return new RecordDescriptor(createExternalFileTupleFieldsSerdes(), EXTERNAL_FILE_INDEX_TYPE_TRAITS); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { EmptyTupleSourceRuntimeFactory runtime = new EmptyTupleSourceRuntimeFactory(); runtime.setSourceLocation(op.getSourceLocation()); RecordDescriptor recDesc = new RecordDescriptor(new ISerializerDeserializer[] {}); builder.contributeMicroOperator(op, runtime, recDesc); }
public IRecordDescriptorProvider getSearchRecordDescriptorProvider() { ITypeTraits[] primaryKeyTypeTraits = new ITypeTraits[primaryKeyTypes.length]; ISerializerDeserializer<?>[] primaryKeySerdes = new ISerializerDeserializer<?>[primaryKeyTypes.length]; for (int i = 0; i < primaryKeyTypes.length; i++) { primaryKeyTypeTraits[i] = TypeTraitProvider.INSTANCE.getTypeTrait(primaryKeyTypes[i]); primaryKeySerdes[i] = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(primaryKeyTypes[i]); } RecordDescriptor searcgRecDesc = new RecordDescriptor(primaryKeySerdes, primaryKeyTypeTraits); IRecordDescriptorProvider rDescProvider = Mockito.mock(IRecordDescriptorProvider.class); Mockito.when(rDescProvider.getInputRecordDescriptor(Mockito.any(), Mockito.anyInt())) .thenReturn(searcgRecDesc); return rDescProvider; }
public RecordDescriptor getSearchOutputDesc(IAType[] keyTypes, ARecordType recordType, ARecordType metaType) { int primaryIndexNumOfTupleFields = keyTypes.length + (1 + ((metaType == null) ? 0 : 1)); ITypeTraits[] primaryIndexTypeTraits = createPrimaryIndexTypeTraits(primaryIndexNumOfTupleFields, keyTypes, recordType, metaType); ISerializerDeserializer<?>[] primaryIndexSerdes = createPrimaryIndexSerdes(primaryIndexNumOfTupleFields, keyTypes, recordType, metaType); return new RecordDescriptor(primaryIndexSerdes, primaryIndexTypeTraits); }
@Override public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<String> dataSource, List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed, List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig) throws AlgebricksException { VXQueryCollectionDataSource ds = (VXQueryCollectionDataSource) dataSource; if (sourceFileMap != null) { final int len = ds.getPartitions().length; String[] collectionPartitions = new String[len]; for (int i = 0; i < len; ++i) { String partition = ds.getPartitions()[i]; File mapped = sourceFileMap.get(partition); collectionPartitions[i] = mapped != null ? mapped.toString() : partition; } ds.setPartitions(collectionPartitions); } RecordDescriptor rDesc = new RecordDescriptor(new ISerializerDeserializer[opSchema.getSize()]); IOperatorDescriptor scanner = new VXQueryCollectionOperatorDescriptor(jobSpec, ds, rDesc, this.hdfsConf, this.nodeControllerInfos); AlgebricksPartitionConstraint constraint = getClusterLocations(nodeList, ds.getPartitionCount()); return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(scanner, constraint); }
@SuppressWarnings("rawtypes") public static RecordDescriptor mkRecordDescriptor(IVariableTypeEnvironment env, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException { ISerializerDeserializer[] fields = new ISerializerDeserializer[opSchema.getSize()]; ITypeTraits[] typeTraits = new ITypeTraits[opSchema.getSize()]; ISerializerDeserializerProvider sdp = context.getSerializerDeserializerProvider(); ITypeTraitProvider ttp = context.getTypeTraitProvider(); int i = 0; for (LogicalVariable var : opSchema) { Object t = env.getVarType(var); if (t == null) { LOGGER.warn("No type for variable " + var); } fields[i] = sdp.getSerializerDeserializer(t); typeTraits[i] = ttp.getTypeTrait(t); i++; } return new RecordDescriptor(fields, typeTraits); }
protected RecordDescriptor getTaggedRecordDescriptor(RecordDescriptor recDescriptor) { ISerializerDeserializer[] fields = new ISerializerDeserializer[recDescriptor.getFields().length + NUM_TAG_FIELDS]; ITypeTraits[] traits = null; if (recDescriptor.getTypeTraits() != null) { traits = new ITypeTraits[recDescriptor.getTypeTraits().length + NUM_TAG_FIELDS]; } //component position field fields[COMPONENT_POS_OFFSET] = IntegerSerializerDeserializer.INSTANCE; if (traits != null) { traits[COMPONENT_POS_OFFSET] = IntegerPointable.TYPE_TRAITS; } //anti-matter field fields[ANTI_MATTER_OFFSET] = BooleanSerializerDeserializer.INSTANCE; if (traits != null) { traits[ANTI_MATTER_OFFSET] = BooleanPointable.TYPE_TRAITS; } for (int i = NUM_TAG_FIELDS; i < fields.length; i++) { fields[i] = recDescriptor.getFields()[i - NUM_TAG_FIELDS]; if (traits != null && i < traits.length) { traits[i] = recDescriptor.getTypeTraits()[i - NUM_TAG_FIELDS]; } } return new RecordDescriptor(fields, traits); }
protected ExternalScanOperatorDescriptor createExternalIndexingOp(JobSpecification spec) throws AlgebricksException { // A record + primary keys ISerializerDeserializer[] serdes = new ISerializerDeserializer[1 + numPrimaryKeys]; ITypeTraits[] typeTraits = new ITypeTraits[1 + numPrimaryKeys]; // payload serde and type traits for the record slot serdes[0] = payloadSerde; typeTraits[0] = TypeTraitProvider.INSTANCE.getTypeTrait(itemType); // serdes and type traits for rid fields for (int i = 1; i < serdes.length; i++) { serdes[i] = IndexingConstants.getSerializerDeserializer(i - 1); typeTraits[i] = IndexingConstants.getTypeTraits(i - 1); } // output record desc RecordDescriptor indexerDesc = new RecordDescriptor(serdes, typeTraits); // Create the operator and its partition constraits Pair<ExternalScanOperatorDescriptor, AlgebricksPartitionConstraint> indexingOpAndConstraints; try { indexingOpAndConstraints = ExternalIndexingOperations.createExternalIndexingOp(spec, metadataProvider, dataset, itemType, indexerDesc, externalFiles, sourceLoc); } catch (Exception e) { throw new AlgebricksException(e); } AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, indexingOpAndConstraints.first, indexingOpAndConstraints.second); // Set the primary partition constraints to this partition constraints primaryPartitionConstraint = indexingOpAndConstraints.second; return indexingOpAndConstraints.first; }
new RecordDescriptor(serdes.toArray(new ISerializerDeserializer[serdes.size()])); FeedPolicyEntity feedPolicy = (FeedPolicyEntity) getProperties().get(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY);
private void setPrimaryRecDescAndComparators() throws AlgebricksException { List<List<String>> partitioningKeys = dataset.getPrimaryKeys(); ISerializerDeserializer[] primaryRecFields = new ISerializerDeserializer[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)]; ITypeTraits[] primaryTypeTraits = new ITypeTraits[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)]; primaryComparatorFactories = new IBinaryComparatorFactory[numPrimaryKeys]; primaryBloomFilterKeyFields = new int[numPrimaryKeys]; ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider(); List<Integer> indicators = null; if (dataset.hasMetaPart()) { indicators = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator(); } for (int i = 0; i < numPrimaryKeys; i++) { IAType keyType = (indicators == null || indicators.get(i) == 0) ? itemType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i)); primaryRecFields[i] = serdeProvider.getSerializerDeserializer(keyType); primaryComparatorFactories[i] = BinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType, true); primaryTypeTraits[i] = TypeTraitProvider.INSTANCE.getTypeTrait(keyType); primaryBloomFilterKeyFields[i] = i; } primaryRecFields[numPrimaryKeys] = payloadSerde; primaryTypeTraits[numPrimaryKeys] = TypeTraitProvider.INSTANCE.getTypeTrait(itemType); if (dataset.hasMetaPart()) { primaryRecFields[numPrimaryKeys + 1] = payloadSerde; primaryTypeTraits[numPrimaryKeys + 1] = TypeTraitProvider.INSTANCE.getTypeTrait(metaType); } primaryRecDesc = new RecordDescriptor(primaryRecFields, primaryTypeTraits); }
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildExternalDatasetDataScannerRuntime( JobSpecification jobSpec, IAType itemType, IAdapterFactory adapterFactory) throws AlgebricksException { if (itemType.getTypeTag() != ATypeTag.OBJECT) { throw new AlgebricksException("Can only scan datasets of records."); } ISerializerDeserializer<?> payloadSerde = getDataFormat().getSerdeProvider().getSerializerDeserializer(itemType); RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] { payloadSerde }); ExternalScanOperatorDescriptor dataScanner = new ExternalScanOperatorDescriptor(jobSpec, scannerDesc, adapterFactory); AlgebricksPartitionConstraint constraint; try { constraint = adapterFactory.getPartitionConstraint(); } catch (Exception e) { throw new AlgebricksException(e); } return new Pair<>(dataScanner, constraint); }
primaryTypeTraits[numPrimaryKeys + 1] = TypeTraitProvider.INSTANCE.getTypeTrait(itemType); return new RecordDescriptor(primaryRecFields, primaryTypeTraits);
public SecondaryIndexInfo(PrimaryIndexInfo primaryIndexInfo, Index secondaryIndex) { this.primaryIndexInfo = primaryIndexInfo; this.secondaryIndex = secondaryIndex; List<String> nodes = Collections.singletonList(ExecutionTestUtil.integrationUtil.ncs[0].getId()); CcApplicationContext appCtx = (CcApplicationContext) ExecutionTestUtil.integrationUtil.cc.getApplicationContext(); FileSplit[] splits = SplitsAndConstraintsUtil.getIndexSplits(appCtx.getClusterStateManager(), primaryIndexInfo.dataset, secondaryIndex.getIndexName(), nodes); fileSplitProvider = new ConstantFileSplitProvider(splits); secondaryIndexTypeTraits = createSecondaryIndexTypeTraits(primaryIndexInfo.recordType, primaryIndexInfo.metaType, primaryIndexInfo.primaryKeyTypes, secondaryIndex.getKeyFieldTypes().toArray(new IAType[secondaryIndex.getKeyFieldTypes().size()])); secondaryIndexSerdes = createSecondaryIndexSerdes(primaryIndexInfo.recordType, primaryIndexInfo.metaType, primaryIndexInfo.primaryKeyTypes, secondaryIndex.getKeyFieldTypes().toArray(new IAType[secondaryIndex.getKeyFieldTypes().size()])); rDesc = new RecordDescriptor(secondaryIndexSerdes, secondaryIndexTypeTraits); insertFieldsPermutations = new int[secondaryIndexTypeTraits.length]; for (int i = 0; i < insertFieldsPermutations.length; i++) { insertFieldsPermutations[i] = i; } primaryKeyIndexes = new int[primaryIndexInfo.primaryKeyIndexes.length]; for (int i = 0; i < primaryKeyIndexes.length; i++) { primaryKeyIndexes[i] = i + secondaryIndex.getKeyFieldNames().size(); } }
primaryIndexSerdes = createPrimaryIndexSerdes(primaryIndexNumOfTupleFields, primaryKeyTypes, recordType, metaType); rDesc = new RecordDescriptor(primaryIndexSerdes, primaryIndexTypeTraits); primaryIndexInsertFieldsPermutations = new int[primaryIndexNumOfTupleFields]; for (int i = 0; i < primaryIndexNumOfTupleFields; i++) {
RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers); ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
RecordDescriptor recDesc = new RecordDescriptor(new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(), IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(), }); RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
public static void flushDataset(IHyracksClientConnection hcc, MetadataProvider metadataProvider, Dataset dataset) throws Exception { CompilerProperties compilerProperties = metadataProvider.getApplicationContext().getCompilerProperties(); int frameSize = compilerProperties.getFrameSize(); JobSpecification spec = new JobSpecification(frameSize); RecordDescriptor[] rDescs = new RecordDescriptor[] { new RecordDescriptor(new ISerializerDeserializer[] {}) }; AlgebricksMetaOperatorDescriptor emptySource = new AlgebricksMetaOperatorDescriptor(spec, 0, 1, new IPushRuntimeFactory[] { new EmptyTupleSourceRuntimeFactory() }, rDescs); TxnId txnId = metadataProvider.getTxnIdFactory().create(); FlushDatasetOperatorDescriptor flushOperator = new FlushDatasetOperatorDescriptor(spec, txnId, dataset.getDatasetId()); spec.connect(new OneToOneConnectorDescriptor(spec), emptySource, 0, flushOperator, 0); Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider.getSplitProviderAndConstraints(dataset, dataset.getDatasetName()); AlgebricksPartitionConstraint primaryPartitionConstraint = primarySplitsAndConstraint.second; AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, emptySource, primaryPartitionConstraint); JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(txnId, true); spec.setJobletEventListenerFactory(jobEventListenerFactory); JobUtils.runJob(hcc, spec, true); }
JobSpecification jobSpec = new JobSpecification(); RecordDescriptor recordDesc = new RecordDescriptor(new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer() });
outputSerDes[j + f] = inputRecordDesc.getFields()[j]; return new RecordDescriptor(outputSerDes, outputTypeTraits);
JobSpecification jobSpec = new JobSpecification(); RecordDescriptor recordDesc = new RecordDescriptor(new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer() });