protected static IBinaryComparator[] createBinaryComparators(IBinaryComparatorFactory[] factories) { IBinaryComparator[] comparators = new IBinaryComparator[factories.length]; for (int i = 0; i < factories.length; i++) { comparators[i] = factories[i].createBinaryComparator(); } return comparators; }
@Override public IBinaryComparator createBinaryComparator() { final IBinaryComparator bc = factory.createBinaryComparator(); if (ascending) { return new ABinaryComparator() { @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) throws HyracksDataException { return bc.compare(b1, s1 + 1, l1 - 1, b2, s2 + 1, l2 - 1); } }; } else { return new ABinaryComparator() { @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) throws HyracksDataException { return -bc.compare(b1, s1 + 1, l1 - 1, b2, s2 + 1, l2 - 1); } }; } }
@Override protected IFrameWriter getFlushableFrameWriter(RunFileWriter writer) throws HyracksDataException { //create group-by comparators IBinaryComparator[] comparators = new IBinaryComparator[Math.min(groupFields.length, comparatorFactories.length)]; for (int i = 0; i < comparators.length; i++) { comparators[i] = comparatorFactories[i].createBinaryComparator(); } return new PreclusteredGroupWriter(ctx, groupFields, comparators, aggregatorFactory, this.inRecordDesc, this.outRecordDesc, writer, true); } }
@Override public ITuplePairComparator createTuplePairComparator(IHyracksTaskContext ctx) { IBinaryComparator[] binaryComparators = new IBinaryComparator[binaryComparatorFactories.length]; for (int i = 0; i < binaryComparators.length; i++) { binaryComparators[i] = binaryComparatorFactories[i].createBinaryComparator(); } return new JoinMultiComparator(binaryComparators, keysLeft, keysRight); } }
@Override public ITuplePairComparator createTuplePairComparator(IHyracksTaskContext ctx) { return new JoinComparator(bFactory.createBinaryComparator(), pos0, pos1); }
public static IBinaryComparator serdeToComparator(ISerializerDeserializer serde) { IBinaryComparatorFactory f = serdeToComparatorFactory(serde); return f.createBinaryComparator(); }
public ExternalGroupBuildOperatorNodePushable(IHyracksTaskContext ctx, Object stateId, int tableSize, long fileSize, int[] keyFields, int framesLimit, IBinaryComparatorFactory[] comparatorFactories, INormalizedKeyComputerFactory firstNormalizerFactory, IAggregatorDescriptorFactory aggregatorFactory, RecordDescriptor inRecordDescriptor, RecordDescriptor outRecordDescriptor, ISpillableTableFactory spillableTableFactory) { this.ctx = ctx; this.stateId = stateId; this.framesLimit = framesLimit; this.aggregatorFactory = aggregatorFactory; this.keyFields = keyFields; this.comparators = new IBinaryComparator[comparatorFactories.length]; for (int i = 0; i < comparatorFactories.length; ++i) { comparators[i] = comparatorFactories[i].createBinaryComparator(); } this.firstNormalizerComputer = firstNormalizerFactory == null ? null : firstNormalizerFactory.createNormalizedKeyComputer(); this.spillableTableFactory = spillableTableFactory; this.inRecordDescriptor = inRecordDescriptor; this.outRecordDescriptor = outRecordDescriptor; this.tableSize = tableSize; this.fileSize = fileSize; }
public ExternalGroupWriteOperatorNodePushable(IHyracksTaskContext ctx, Object stateId, ISpillableTableFactory spillableTableFactory, RecordDescriptor partialAggRecordDesc, RecordDescriptor outRecordDesc, int framesLimit, int[] groupFields, INormalizedKeyComputerFactory nmkFactory, IBinaryComparatorFactory[] comparatorFactories, IAggregatorDescriptorFactory aggregatorFactory) { this.ctx = ctx; this.stateId = stateId; this.spillableTableFactory = spillableTableFactory; this.frameLimit = framesLimit; this.nmkComputer = nmkFactory == null ? null : nmkFactory.createNormalizedKeyComputer(); this.partialAggRecordDesc = partialAggRecordDesc; this.outRecordDesc = outRecordDesc; this.mergeAggregatorFactory = aggregatorFactory; //create merge group fields int numGroupFields = groupFields.length; mergeGroupFields = new int[numGroupFields]; for (int i = 0; i < numGroupFields; i++) { mergeGroupFields[i] = i; } //setup comparators for grouping groupByComparators = new IBinaryComparator[Math.min(mergeGroupFields.length, comparatorFactories.length)]; for (int i = 0; i < groupByComparators.length; i++) { groupByComparators[i] = comparatorFactories[i].createBinaryComparator(); } }
@Override public ILinearizeComparator createBinaryComparator() { final ILinearizeComparator bc = (ILinearizeComparator) factory.createBinaryComparator(); final int dimension = bc.getDimensions(); if (ascending) { return new ILinearizeComparator() { @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) throws HyracksDataException { return bc.compare(b1, s1 + 1, l1, b2, s2 + 1, l2); } @Override public int getDimensions() { return dimension; } }; } else { return new ILinearizeComparator() { @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) throws HyracksDataException { return -bc.compare(b1, s1 + 1, l1, b2, s2 + 1, l2); } @Override public int getDimensions() { return dimension; } }; } }
public static MultiComparator create(IBinaryComparatorFactory[] cmpFactories) { IBinaryComparator[] cmps = new IBinaryComparator[cmpFactories.length]; for (int i = 0; i < cmpFactories.length; i++) { cmps[i] = cmpFactories[i].createBinaryComparator(); } if (cmps.length == 1) { return new SingleComparator(cmps); } else { return new MultiComparator(cmps); } }
public static MultiComparator create(IBinaryComparatorFactory[] cmpFactories, int startIndex, int numCmps) { IBinaryComparator[] cmps = new IBinaryComparator[numCmps]; for (int i = startIndex; i < startIndex + numCmps; i++) { cmps[i] = cmpFactories[i].createBinaryComparator(); } if (cmps.length == 1) { return new SingleComparator(cmps); } else { return new MultiComparator(cmps); } }
@Override public void open() throws HyracksDataException { final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length]; for (int i = 0; i < comparatorFactories.length; ++i) { comparators[i] = comparatorFactories[i].createBinaryComparator(); } pgw = new PreclusteredGroupWriter(ctx, groupFields, comparators, aggregatorFactory, inRecordDescriptor, outRecordDescriptor, writer, false, groupAll, frameLimit); pgw.open(); }
public static MultiComparator create(IBinaryComparatorFactory[]... cmpFactories) { int size = 0; for (int i = 0; i < cmpFactories.length; i++) { size += cmpFactories[i].length; } IBinaryComparator[] cmps = new IBinaryComparator[size]; int x = 0; for (int i = 0; i < cmpFactories.length; i++) { for (int j = 0; j < cmpFactories[i].length; j++) { cmps[x++] = cmpFactories[i][j].createBinaryComparator(); } } if (cmps.length == 1) { return new SingleComparator(cmps); } else { return new MultiComparator(cmps); } } }
@Override public IPartitionCollector createPartitionCollector(IHyracksTaskContext ctx, RecordDescriptor recordDesc, int index, int nProducerPartitions, int nConsumerPartitions) throws HyracksDataException { IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length]; for (int i = 0; i < comparatorFactories.length; ++i) { comparators[i] = comparatorFactories[i].createBinaryComparator(); } INormalizedKeyComputer nmkComputer = nkcFactory == null ? null : nkcFactory.createNormalizedKeyComputer(); IPartitionBatchManager pbm = new NonDeterministicPartitionBatchManager(nProducerPartitions); IFrameReader sortMergeFrameReader = new SortMergeFrameReader(ctx, nProducerPartitions, nProducerPartitions, sortFields, comparators, nmkComputer, recordDesc, pbm); BitSet expectedPartitions = new BitSet(); expectedPartitions.set(0, nProducerPartitions); return new PartitionCollector(ctx, getConnectorId(), index, expectedPartitions, sortMergeFrameReader, pbm); } }
public static MultiComparator getSearchMultiComparator(IBinaryComparatorFactory[] cmpFactories, ITupleReference searchKey) { if (searchKey == null || cmpFactories.length == searchKey.getFieldCount()) { return MultiComparator.create(cmpFactories); } IBinaryComparator[] newCmps = new IBinaryComparator[searchKey.getFieldCount()]; for (int i = 0; i < searchKey.getFieldCount(); i++) { newCmps[i] = cmpFactories[i].createBinaryComparator(); } return new MultiComparator(newCmps); }
public static MultiComparator getSearchMultiComparator(IBinaryComparatorFactory[] cmpFactories, ITupleReference searchKey) { if (searchKey == null || cmpFactories.length == searchKey.getFieldCount()) { return MultiComparator.create(cmpFactories); } IBinaryComparator[] newCmps = new IBinaryComparator[searchKey.getFieldCount()]; for (int i = 0; i < searchKey.getFieldCount(); i++) { newCmps[i] = cmpFactories[i].createBinaryComparator(); } return new MultiComparator(newCmps); }
@Override public void initialize() throws HyracksDataException { SortTaskState state = (SortTaskState) ctx .getStateObject(new TaskId(new ActivityId(getOperatorId(), SORT_ACTIVITY_ID), partition)); List<GeneratedRunFileReader> runs = state.generatedRunFileReaders; ISorter sorter = state.sorter; IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length]; for (int i = 0; i < comparatorFactories.length; ++i) { comparators[i] = comparatorFactories[i].createBinaryComparator(); } INormalizedKeyComputer nmkComputer = keyNormalizerFactories == null ? null : keyNormalizerFactories[0].createNormalizedKeyComputer(); AbstractExternalSortRunMerger merger = getSortRunMerger(ctx, recordDescProvider, writer, sorter, runs, comparators, nmkComputer, framesLimit); merger.process(); } };
public static void verifyRangeOrder(RangeMap rangeMap, boolean ascending) throws CompilationException { // TODO Add support for composite fields. int fieldIndex = 0; int fieldType = rangeMap.getTag(0, 0); BinaryComparatorFactoryProvider comparatorFactory = BinaryComparatorFactoryProvider.INSTANCE; IBinaryComparatorFactory bcf = comparatorFactory.getBinaryComparatorFactory(ATypeTag.VALUE_TYPE_MAPPING[fieldType], ascending); IBinaryComparator comparator = bcf.createBinaryComparator(); int c = 0; for (int split = 1; split < rangeMap.getSplitCount(); ++split) { if (fieldType != rangeMap.getTag(fieldIndex, split)) { throw new CompilationException("Range field contains more than a single type of items (" + fieldType + " and " + rangeMap.getTag(fieldIndex, split) + ")."); } int previousSplit = split - 1; try { c = comparator.compare(rangeMap.getByteArray(), rangeMap.getStartOffset(fieldIndex, previousSplit), rangeMap.getLength(fieldIndex, previousSplit), rangeMap.getByteArray(), rangeMap.getStartOffset(fieldIndex, split), rangeMap.getLength(fieldIndex, split)); } catch (HyracksDataException e) { throw new CompilationException(e); } if (c >= 0) { throw new CompilationException("Range fields are not in sorted order."); } } } }
private <T> void searchIndex(TxnId txnId, IMetadataIndex index, ITupleReference searchKey, IValueExtractor<T> valueExtractor, List<T> results) throws AlgebricksException, HyracksDataException, RemoteException { IBinaryComparatorFactory[] comparatorFactories = index.getKeyBinaryComparatorFactory(); if (index.getFile() == null) { throw new AlgebricksException("No file for Index " + index.getDataverseName() + "." + index.getIndexName()); } String resourceName = index.getFile().getRelativePath(); IIndex indexInstance = datasetLifecycleManager.get(resourceName); datasetLifecycleManager.open(resourceName); IIndexAccessor indexAccessor = indexInstance.createAccessor(NoOpIndexAccessParameters.INSTANCE); try { IBinaryComparator[] searchCmps = null; MultiComparator searchCmp = null; if (searchKey != null) { searchCmps = new IBinaryComparator[searchKey.getFieldCount()]; for (int i = 0; i < searchKey.getFieldCount(); i++) { searchCmps[i] = comparatorFactories[i].createBinaryComparator(); } searchCmp = new MultiComparator(searchCmps); } RangePredicate rangePred = new RangePredicate(searchKey, searchKey, true, true, searchCmp, searchCmp); search(indexAccessor, rangePred, results, valueExtractor, txnId); } finally { indexAccessor.destroy(); } datasetLifecycleManager.close(resourceName); }
protected void addFilterField(IIndexTestContext ctx, MutablePair<ITupleReference, ITupleReference> minMax) throws HyracksDataException { //Duplicate the PK field as a filter field at the end of the tuple to be inserted. int filterField = ctx.getFieldCount(); ITupleReference currTuple = ctx.getTuple(); ArrayTupleBuilder filterBuilder = new ArrayTupleBuilder(1); filterBuilder.addField(currTuple.getFieldData(filterField), currTuple.getFieldStart(filterField), currTuple.getFieldLength(filterField)); IBinaryComparator comparator = ctx.getComparatorFactories()[0].createBinaryComparator(); ArrayTupleReference filterOnlyTuple = new ArrayTupleReference(); filterOnlyTuple.reset(filterBuilder.getFieldEndOffsets(), filterBuilder.getByteArray()); if (minMax == null) { minMax = MutablePair.of(filterOnlyTuple, filterOnlyTuple); } else if (compareFilterTuples(minMax.getLeft(), filterOnlyTuple, comparator) > 0) { minMax.setLeft(filterOnlyTuple); } else if (compareFilterTuples(minMax.getRight(), filterOnlyTuple, comparator) < 0) { minMax.setRight(filterOnlyTuple); } }