public void add(BytesRef utf8) { if (closed) throw new IllegalStateException(); buffer.append(utf8); }
/** * sugar for {@link #iterator(Comparator)} with a <code>null</code> comparator */ public BytesRefIterator iterator() { return iterator(null); }
/** Read in a single partition of data */ int readPartition(ByteSequencesReader reader) throws IOException { long start = System.currentTimeMillis(); final BytesRef scratch = new BytesRef(); while ((scratch.bytes = reader.read()) != null) { scratch.length = scratch.bytes.length; buffer.append(scratch); // Account for the created objects. // (buffer slots do not account to buffer size.) if (ramBufferSize.bytes < buffer.bytesUsed()) { break; } } sortInfo.readTime += (System.currentTimeMillis() - start); return buffer.size(); }
/** Sort a single partition in-memory. */ protected File sortPartition(int len) throws IOException { BytesRefList data = this.buffer; File tempFile = File.createTempFile("sort", "partition", tempDirectory); long start = System.currentTimeMillis(); sortInfo.sortTime += (System.currentTimeMillis() - start); final ByteSequencesWriter out = new ByteSequencesWriter(tempFile); BytesRef spare; try { BytesRefIterator iter = buffer.iterator(comparator); while((spare = iter.next()) != null) { assert spare.length <= Short.MAX_VALUE; out.write(spare); } out.close(); // Clean up the buffer for the next partition. data.clear(); return tempFile; } finally { IOUtils.close(out); } }
@Override protected void setPivot(int i) { final int ord = orderdEntries[i]; get(pivot, ord); }
public UnsortedTermFreqIteratorWrapper(TermFreqIterator source) throws IOException { super(source); ords = new int[entries.size()]; Random random = new Random(); for (int i = 0; i < ords.length; i++) { ords[i] = i; } for (int i = 0; i < ords.length; i++) { int randomPosition = random.nextInt(ords.length); int temp = ords[i]; ords[i] = ords[randomPosition]; ords[randomPosition] = temp; } }
public BytesRef next() throws IOException { if (pos < size) { return get(spare, ords == null ? pos++ : ords[pos++]); } return null; }
private int[] sort(final Comparator<BytesRef> comp) { final int[] orderdEntries = new int[size()]; for (int i = 0; i < orderdEntries.length; i++) { orderdEntries[i] = i;
public BufferingTermFreqIteratorWrapper(TermFreqIterator source) throws IOException { this.comp = source.getComparator(); BytesRef spare; int freqIndex = 0; while((spare = source.next()) != null) { entries.append(spare); if (freqIndex >= freqs.length) { freqs = ArrayUtil.grow(freqs, freqs.length+1); } freqs[freqIndex++] = source.weight(); } }
public BytesRefIterator iterator() { closed = true; return buffer.iterator(comparator); }
@Override protected int comparePivot(int j) { final int ord = orderdEntries[j]; return comp.compare(pivot, get(scratch2, ord)); }