@SuppressWarnings("rawtypes") public static RecordDescriptor mkRecordDescriptor(IVariableTypeEnvironment env, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException { ISerializerDeserializer[] fields = new ISerializerDeserializer[opSchema.getSize()]; ITypeTraits[] typeTraits = new ITypeTraits[opSchema.getSize()]; ISerializerDeserializerProvider sdp = context.getSerializerDeserializerProvider(); ITypeTraitProvider ttp = context.getTypeTraitProvider(); int i = 0; for (LogicalVariable var : opSchema) { Object t = env.getVarType(var); if (t == null) { LOGGER.warn("No type for variable " + var); } fields[i] = sdp.getSerializerDeserializer(t); typeTraits[i] = ttp.getTypeTrait(t); i++; } return new RecordDescriptor(fields, typeTraits); }
metadataProvider.getDataFormat().getSerdeProvider().getSerializerDeserializer(feedOutputType); ArrayList<ISerializerDeserializer> serdes = new ArrayList<>(); serdes.add(payloadSerde);
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildExternalDatasetDataScannerRuntime( JobSpecification jobSpec, IAType itemType, IAdapterFactory adapterFactory) throws AlgebricksException { if (itemType.getTypeTag() != ATypeTag.OBJECT) { throw new AlgebricksException("Can only scan datasets of records."); } ISerializerDeserializer<?> payloadSerde = getDataFormat().getSerdeProvider().getSerializerDeserializer(itemType); RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] { payloadSerde }); ExternalScanOperatorDescriptor dataScanner = new ExternalScanOperatorDescriptor(jobSpec, scannerDesc, adapterFactory); AlgebricksPartitionConstraint constraint; try { constraint = adapterFactory.getPartitionConstraint(); } catch (Exception e) { throw new AlgebricksException(e); } return new Pair<>(dataScanner, constraint); }
private void setPrimaryRecDescAndComparators() throws AlgebricksException { List<List<String>> partitioningKeys = dataset.getPrimaryKeys(); ISerializerDeserializer[] primaryRecFields = new ISerializerDeserializer[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)]; ITypeTraits[] primaryTypeTraits = new ITypeTraits[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)]; primaryComparatorFactories = new IBinaryComparatorFactory[numPrimaryKeys]; primaryBloomFilterKeyFields = new int[numPrimaryKeys]; ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider(); List<Integer> indicators = null; if (dataset.hasMetaPart()) { indicators = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator(); } for (int i = 0; i < numPrimaryKeys; i++) { IAType keyType = (indicators == null || indicators.get(i) == 0) ? itemType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i)); primaryRecFields[i] = serdeProvider.getSerializerDeserializer(keyType); primaryComparatorFactories[i] = BinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType, true); primaryTypeTraits[i] = TypeTraitProvider.INSTANCE.getTypeTrait(keyType); primaryBloomFilterKeyFields[i] = i; } primaryRecFields[numPrimaryKeys] = payloadSerde; primaryTypeTraits[numPrimaryKeys] = TypeTraitProvider.INSTANCE.getTypeTrait(itemType); if (dataset.hasMetaPart()) { primaryRecFields[numPrimaryKeys + 1] = payloadSerde; primaryTypeTraits[numPrimaryKeys + 1] = TypeTraitProvider.INSTANCE.getTypeTrait(metaType); } primaryRecDesc = new RecordDescriptor(primaryRecFields, primaryTypeTraits); }
IAType keyType = keyTypePair.first; anySecondaryKeyIsNullable = anySecondaryKeyIsNullable || keyTypePair.second; ISerializerDeserializer keySerde = serdeProvider.getSerializerDeserializer(keyType); secondaryRecFields[i] = keySerde; secondaryComparatorFactories[i] = comparatorFactoryProvider.getBinaryComparatorFactory(keyType, true); enforcedRecFields[numPrimaryKeys] = serdeProvider.getSerializerDeserializer(itemType); enforcedTypeTraits[numPrimaryKeys] = typeTraitProvider.getTypeTrait(itemType); if (dataset.hasMetaPart()) { enforcedRecFields[numPrimaryKeys + 1] = serdeProvider.getSerializerDeserializer(metaType); enforcedTypeTraits[numPrimaryKeys + 1] = typeTraitProvider.getTypeTrait(metaType); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType); IAType type = keyTypePair.first; ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type); secondaryRecFields[numPrimaryKeys + numSecondaryKeys] = serde; enforcedRecFields[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)] = serde;
(indicators == null || indicators.get(i) == 0) ? itemType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i)); primaryRecFields[i] = serdeProvider.getSerializerDeserializer(keyType); primaryTypeTraits[i] = TypeTraitProvider.INSTANCE.getTypeTrait(keyType);
IAType keyType = keyTypePair.first; anySecondaryKeyIsNullable = anySecondaryKeyIsNullable || keyTypePair.second; ISerializerDeserializer keySerde = serdeProvider.getSerializerDeserializer(keyType); secondaryRecFields[i] = keySerde; secondaryComparatorFactories[i] = comparatorFactoryProvider.getBinaryComparatorFactory(keyType, true); enforcedRecFields[numPrimaryKeys] = serdeProvider.getSerializerDeserializer(itemType); enforcedTypeTraits[numPrimaryKeys] = typeTraitProvider.getTypeTrait(itemType); if (dataset.hasMetaPart()) { enforcedRecFields[numPrimaryKeys + 1] = serdeProvider.getSerializerDeserializer(metaType); enforcedTypeTraits[numPrimaryKeys + 1] = typeTraitProvider.getTypeTrait(metaType); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType); IAType type = keyTypePair.first; ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type); secondaryRecFields[numPrimaryKeys + numSecondaryKeys] = serde; enforcedRecFields[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)] = serde;
secondaryKeyType = keyTypePair.first; anySecondaryKeyIsNullable = anySecondaryKeyIsNullable || keyTypePair.second; ISerializerDeserializer keySerde = serdeProvider.getSerializerDeserializer(secondaryKeyType); secondaryRecFields[0] = keySerde; secondaryTypeTraits[0] = typeTraitProvider.getTypeTrait(secondaryKeyType); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType); IAType type = keyTypePair.first; ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type); secondaryRecFields[numPrimaryKeys + numSecondaryKeys] = serde; enforcedTypeTraits[0] = primaryRecDesc.getTypeTraits()[0]; enforcedRecFields[numPrimaryKeys] = serdeProvider.getSerializerDeserializer(itemType); enforcedRecDesc = new RecordDescriptor(enforcedRecFields, enforcedTypeTraits); ITypeTraits[] tokenKeyPairTypeTraits = new ITypeTraits[numTokenKeyPairFields]; tokenKeyPairComparatorFactories = new IBinaryComparatorFactory[numTokenKeyPairFields]; tokenKeyPairFields[0] = serdeProvider.getSerializerDeserializer(secondaryKeyType); tokenKeyPairTypeTraits[0] = tokenTypeTraits[0]; tokenKeyPairComparatorFactories[0] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType);
secondaryKeyType = keyTypePair.first; anySecondaryKeyIsNullable = anySecondaryKeyIsNullable || keyTypePair.second; ISerializerDeserializer keySerde = serdeProvider.getSerializerDeserializer(secondaryKeyType); secondaryRecFields[0] = keySerde; secondaryTypeTraits[0] = typeTraitProvider.getTypeTrait(secondaryKeyType); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType); IAType type = keyTypePair.first; ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type); secondaryRecFields[numPrimaryKeys + numSecondaryKeys] = serde; enforcedTypeTraits[0] = primaryRecDesc.getTypeTraits()[0]; enforcedRecFields[numPrimaryKeys] = serdeProvider.getSerializerDeserializer(itemType); enforcedRecDesc = new RecordDescriptor(enforcedRecFields, enforcedTypeTraits); ITypeTraits[] tokenKeyPairTypeTraits = new ITypeTraits[numTokenKeyPairFields]; tokenKeyPairComparatorFactories = new IBinaryComparatorFactory[numTokenKeyPairFields]; tokenKeyPairFields[0] = serdeProvider.getSerializerDeserializer(secondaryKeyType); tokenKeyPairTypeTraits[0] = tokenTypeTraits[0]; tokenKeyPairComparatorFactories[0] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType);
outputSerDes[f] = dataFormat.getSerdeProvider().getSerializerDeserializer(BuiltinType.ABOOLEAN); outputTypeTraits[f] = dataFormat.getTypeTraitProvider().getTypeTrait(BuiltinType.ABOOLEAN); f++; outputSerDes[f] = dataFormat.getSerdeProvider().getSerializerDeserializer(itemType); outputTypeTraits[f] = dataFormat.getTypeTraitProvider().getTypeTrait(itemType); f++; outputSerDes[f] = dataFormat.getSerdeProvider().getSerializerDeserializer(metaItemType); outputTypeTraits[f] = dataFormat.getTypeTraitProvider().getTypeTrait(metaItemType); f++; outputTypeTraits[f] = dataFormat.getTypeTraitProvider().getTypeTrait(itemType.getFieldTypes()[fieldIdx]); outputSerDes[f] = dataFormat.getSerdeProvider().getSerializerDeserializer(itemType.getFieldTypes()[fieldIdx]); f++;
tokenKeyPairFields[tokenOffset] = serdeProvider.getSerializerDeserializer(secondaryKeyType); tokenKeyPairTypeTraits[tokenOffset] = tokenTypeTraits[0]; tokenOffset++;
outputSerDes[f] = FormatUtils.getDefaultFormat().getSerdeProvider().getSerializerDeserializer(itemType); f++; outputSerDes[f] = FormatUtils.getDefaultFormat().getSerdeProvider().getSerializerDeserializer(metaItemType); outputTypeTraits[f] = FormatUtils.getDefaultFormat().getTypeTraitProvider().getTypeTrait(metaItemType); f++; .getTypeTrait(itemType.getFieldTypes()[fieldIdx]); outputSerDes[f] = FormatUtils.getDefaultFormat().getSerdeProvider() .getSerializerDeserializer(itemType.getFieldTypes()[fieldIdx]); f++;
jobGenCtx.getSerializerDeserializerProvider().getSerializerDeserializer(returnType); bbis.setByteBuffer(ByteBuffer.wrap(p.getByteArray(), p.getStartOffset(), p.getLength()), 0); IAObject o = (IAObject) serde.deserialize(dis);