@Override public INormalizedKeyProperties getNormalizedKeyProperties() { return nkcf.getNormalizedKeyProperties(); }
@Override public INormalizedKeyComputer createNormalizedKeyComputer() { final INormalizedKeyComputer nkc = nkcf.createNormalizedKeyComputer(); return new INormalizedKeyComputer() { @Override public void normalize(byte[] bytes, int start, int length, int[] normalizedKeys, int keyStart) { // start +1, length -1 is because in ASTERIX data format, there is always a type tag before the value nkc.normalize(bytes, start + 1, length - 1, normalizedKeys, keyStart); } @Override public INormalizedKeyProperties getNormalizedKeyProperties() { return nkc.getNormalizedKeyProperties(); } }; }
firstKeyNormalizerFactory != null ? firstKeyNormalizerFactory.createNormalizedKeyComputer() : null; this.normalizedKeyDecisive = firstKeyNormalizerFactory != null ? firstKeyNormalizerFactory.getNormalizedKeyProperties().isDecisive() && compareFields[0].length == 1 : false;
this.nkcs[i] = normalizedKeyComputerFactories[i].createNormalizedKeyComputer(); this.normalizedKeyLength[i] = normalizedKeyComputerFactories[i].getNormalizedKeyProperties().getNormalizedKeyLength(); runningNormalizedKeyTotalLength += this.normalizedKeyLength[i];
this.nkcs[i] = keyNormalizerFactories[i].createNormalizedKeyComputer(); this.normalizedKeyLength[i] = keyNormalizerFactories[i].getNormalizedKeyProperties().getNormalizedKeyLength(); runningNormalizedKeyTotalLength += this.normalizedKeyLength[i];
@Override public INormalizedKeyProperties getNormalizedKeyProperties() { return nkcf.getNormalizedKeyProperties(); }
@Override public INormalizedKeyComputer createNormalizedKeyComputer() { final INormalizedKeyComputer nkc = nkcf.createNormalizedKeyComputer(); return new INormalizedKeyComputer() { @Override public void normalize(byte[] bytes, int start, int length, int[] normalizedKeys, int keyStart) { nkc.normalize(bytes, start + 1, length - 1, normalizedKeys, keyStart); for (int i = 0; i < normalizedKeyLength; i++) { int key = normalizedKeys[keyStart + i]; normalizedKeys[keyStart + i] = ~key; } } @Override public INormalizedKeyProperties getNormalizedKeyProperties() { return nkc.getNormalizedKeyProperties(); } }; }
public static int getDecisivePrefixLength(INormalizedKeyComputerFactory[] keyNormalizerFactories) { if (keyNormalizerFactories == null) { return 0; } for (int i = 0; i < keyNormalizerFactories.length; i++) { if (!keyNormalizerFactories[i].getNormalizedKeyProperties().isDecisive()) { return i; } } return keyNormalizerFactories.length; }
@Override public INormalizedKeyComputer createNormalizedKeyComputer() { final INormalizedKeyComputer nkc = int64NormalizerFactory.createNormalizedKeyComputer(); return new INormalizedKeyComputer() { @Override public void normalize(byte[] bytes, int start, int length, int[] normalizedKeys, int keyStart) { // normalize msb nkc.normalize(bytes, start, length, normalizedKeys, keyStart); // normalize lsb nkc.normalize(bytes, start + Long.BYTES, length - Long.BYTES, normalizedKeys, keyStart + int64NormalizedKeyLength); } @Override public INormalizedKeyProperties getNormalizedKeyProperties() { return PROPERTIES; } }; }
public AWrappedDescNormalizedKeyComputerFactory(INormalizedKeyComputerFactory nkcf) { this.nkcf = nkcf; this.normalizedKeyLength = nkcf.getNormalizedKeyProperties().getNormalizedKeyLength(); }
public ExternalGroupWriteOperatorNodePushable(IHyracksTaskContext ctx, Object stateId, ISpillableTableFactory spillableTableFactory, RecordDescriptor partialAggRecordDesc, RecordDescriptor outRecordDesc, int framesLimit, int[] groupFields, INormalizedKeyComputerFactory nmkFactory, IBinaryComparatorFactory[] comparatorFactories, IAggregatorDescriptorFactory aggregatorFactory) { this.ctx = ctx; this.stateId = stateId; this.spillableTableFactory = spillableTableFactory; this.frameLimit = framesLimit; this.nmkComputer = nmkFactory == null ? null : nmkFactory.createNormalizedKeyComputer(); this.partialAggRecordDesc = partialAggRecordDesc; this.outRecordDesc = outRecordDesc; this.mergeAggregatorFactory = aggregatorFactory; //create merge group fields int numGroupFields = groupFields.length; mergeGroupFields = new int[numGroupFields]; for (int i = 0; i < numGroupFields; i++) { mergeGroupFields[i] = i; } //setup comparators for grouping groupByComparators = new IBinaryComparator[Math.min(mergeGroupFields.length, comparatorFactories.length)]; for (int i = 0; i < groupByComparators.length; i++) { groupByComparators[i] = comparatorFactories[i].createBinaryComparator(); } }
public AUUIDNormalizedKeyComputerFactory() { int64NormalizerFactory = new Integer64NormalizedKeyComputerFactory(); int64NormalizedKeyLength = int64NormalizerFactory.getNormalizedKeyProperties().getNormalizedKeyLength(); }
public ExternalGroupBuildOperatorNodePushable(IHyracksTaskContext ctx, Object stateId, int tableSize, long fileSize, int[] keyFields, int framesLimit, IBinaryComparatorFactory[] comparatorFactories, INormalizedKeyComputerFactory firstNormalizerFactory, IAggregatorDescriptorFactory aggregatorFactory, RecordDescriptor inRecordDescriptor, RecordDescriptor outRecordDescriptor, ISpillableTableFactory spillableTableFactory) { this.ctx = ctx; this.stateId = stateId; this.framesLimit = framesLimit; this.aggregatorFactory = aggregatorFactory; this.keyFields = keyFields; this.comparators = new IBinaryComparator[comparatorFactories.length]; for (int i = 0; i < comparatorFactories.length; ++i) { comparators[i] = comparatorFactories[i].createBinaryComparator(); } this.firstNormalizerComputer = firstNormalizerFactory == null ? null : firstNormalizerFactory.createNormalizedKeyComputer(); this.spillableTableFactory = spillableTableFactory; this.inRecordDescriptor = inRecordDescriptor; this.outRecordDescriptor = outRecordDescriptor; this.tableSize = tableSize; this.fileSize = fileSize; }
@Override public IPartitionCollector createPartitionCollector(IHyracksTaskContext ctx, RecordDescriptor recordDesc, int index, int nProducerPartitions, int nConsumerPartitions) throws HyracksDataException { IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length]; for (int i = 0; i < comparatorFactories.length; ++i) { comparators[i] = comparatorFactories[i].createBinaryComparator(); } INormalizedKeyComputer nmkComputer = nkcFactory == null ? null : nkcFactory.createNormalizedKeyComputer(); IPartitionBatchManager pbm = new NonDeterministicPartitionBatchManager(nProducerPartitions); IFrameReader sortMergeFrameReader = new SortMergeFrameReader(ctx, nProducerPartitions, nProducerPartitions, sortFields, comparators, nmkComputer, recordDesc, pbm); BitSet expectedPartitions = new BitSet(); expectedPartitions.set(0, nProducerPartitions); return new PartitionCollector(ctx, getConnectorId(), index, expectedPartitions, sortMergeFrameReader, pbm); } }
@Override public void initialize() throws HyracksDataException { SortTaskState state = (SortTaskState) ctx .getStateObject(new TaskId(new ActivityId(getOperatorId(), SORT_ACTIVITY_ID), partition)); List<GeneratedRunFileReader> runs = state.generatedRunFileReaders; ISorter sorter = state.sorter; IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length]; for (int i = 0; i < comparatorFactories.length; ++i) { comparators[i] = comparatorFactories[i].createBinaryComparator(); } INormalizedKeyComputer nmkComputer = keyNormalizerFactories == null ? null : keyNormalizerFactories[0].createNormalizedKeyComputer(); AbstractExternalSortRunMerger merger = getSortRunMerger(ctx, recordDescProvider, writer, sorter, runs, comparators, nmkComputer, framesLimit); merger.process(); } };