assert this == blocks.get(0); assert scratchBytes.getFilePointer() == 0; scratchBytes.writeVLong(encodeOutput(fp, hasTerms, isFloor)); if (isFloor) { scratchBytes.writeVInt(blocks.size()-1); for (int i=1;i<blocks.size();i++) { PendingBlock sub = blocks.get(i); scratchBytes.writeByte((byte) sub.floorLeadByte); assert sub.fp > fp; scratchBytes.writeVLong((sub.fp - fp) << 1 | (sub.hasTerms ? 1 : 0)); final byte[] bytes = new byte[(int) scratchBytes.getFilePointer()]; assert bytes.length > 0; scratchBytes.writeTo(bytes, 0); indexBuilder.add(Util.toIntsRef(prefix, scratchIntsRef), new BytesRef(bytes, 0, bytes.length)); scratchBytes.reset();
try (RAMOutputStream out = new RAMOutputStream()) { String lastTermField = null; String lastUpdateField = null; code |= 4; out.writeVInt(code); out.writeVInt(update.docIDUpto); if (termField.equals(lastTermField) == false) { out.writeString(termField); lastTermField = termField; out.writeString(updateField); lastUpdateField = updateField; out.writeBytes(update.term.bytes().bytes, update.term.bytes().offset, update.term.bytes().length); if (update.hasValue()) { update.writeTo(out); byte[] bytes = new byte[(int) out.getFilePointer()]; out.writeTo(bytes, 0); return bytes;
@Override public void close() throws IOException { flush(); }
/** Appends the current contents of writeBuffer as another block on the growing in-memory file */ private int appendBlock(RAMOutputStream writeBuffer, List<byte[]> blocks) throws IOException { int pos = Math.toIntExact(writeBuffer.getFilePointer()); byte[] bytes = new byte[pos]; writeBuffer.writeTo(bytes, 0); writeBuffer.reset(); blocks.add(bytes); return pos; }
/** * Writes the buffered skip lists to the given output. * * @param output the IndexOutput the skip lists shall be written to * @return the pointer the skip list starts */ public long writeSkip(IndexOutput output) throws IOException { long skipPointer = output.getFilePointer(); //System.out.println("skipper.writeSkip fp=" + skipPointer); if (skipBuffer == null || skipBuffer.length == 0) return skipPointer; for (int level = numberOfSkipLevels - 1; level > 0; level--) { long length = skipBuffer[level].getFilePointer(); if (length > 0) { output.writeVLong(length); skipBuffer[level].writeTo(output); } } skipBuffer[0].writeTo(output); return skipPointer; } }
writeSkipData(level, skipBuffer[level]); long newChildPointer = skipBuffer[level].getFilePointer(); skipBuffer[level].writeVLong(childPointer);
static String buildScrollId(AtomicArray<? extends SearchPhaseResult> searchPhaseResults) throws IOException { try (RAMOutputStream out = new RAMOutputStream()) { out.writeString(searchPhaseResults.length() == 1 ? ParsedScrollId.QUERY_AND_FETCH_TYPE : ParsedScrollId.QUERY_THEN_FETCH_TYPE); out.writeVInt(searchPhaseResults.asList().size()); for (SearchPhaseResult searchPhaseResult : searchPhaseResults.asList()) { out.writeLong(searchPhaseResult.getRequestId()); SearchShardTarget searchShardTarget = searchPhaseResult.getSearchShardTarget(); if (searchShardTarget.getClusterAlias() != null) { out.writeString(RemoteClusterAware.buildRemoteIndexName(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId())); } else { out.writeString(searchShardTarget.getNodeId()); } } byte[] bytes = new byte[(int) out.getFilePointer()]; out.writeTo(bytes, 0); return Base64.getUrlEncoder().encodeToString(bytes); } }
private void flushTermsDictBlock(RAMOutputStream headerBuffer, RAMOutputStream bytesBuffer, int suffixDeltas[]) throws IOException { boolean twoByte = false; for (int i = 1; i < suffixDeltas.length; i++) { if (suffixDeltas[i] > 254) { twoByte = true; } } if (twoByte) { headerBuffer.writeByte((byte)255); for (int i = 1; i < suffixDeltas.length; i++) { headerBuffer.writeShort((short) suffixDeltas[i]); } } else { for (int i = 1; i < suffixDeltas.length; i++) { headerBuffer.writeByte((byte) suffixDeltas[i]); } } headerBuffer.writeTo(data); headerBuffer.reset(); bytesBuffer.writeTo(data); bytesBuffer.reset(); }
RAMOutputStream addressBuffer = new RAMOutputStream(); MonotonicBlockPackedWriter termAddresses = new MonotonicBlockPackedWriter(addressBuffer, MONOTONIC_BLOCK_SIZE); RAMOutputStream bytesBuffer = new RAMOutputStream(); RAMOutputStream headerBuffer = new RAMOutputStream(); BytesRefBuilder lastTerm = new BytesRefBuilder(); lastTerm.grow(maxLength); termAddresses.add(data.getFilePointer() - startFP); headerBuffer.writeVInt(v.length); headerBuffer.writeBytes(v.bytes, v.offset, v.length); lastTerm.copyBytes(v); } else { bytesBuffer.writeByte((byte) sharedPrefix); bytesBuffer.writeBytes(v.bytes, v.offset + sharedPrefix, v.length - sharedPrefix); addressBuffer.writeTo(data); addressBuffer = null; termAddresses = null;
/** add a term. This fully consumes in the incoming {@link BytesRef}. */ public void add(String field, BytesRef bytes) { assert lastTerm.equals(new Term("")) || new Term(field, bytes).compareTo(lastTerm) > 0; try { final int prefix; if (size > 0 && field.equals(lastTerm.field)) { // same field as the last term prefix = StringHelper.bytesDifference(lastTerm.bytes, bytes); output.writeVInt(prefix << 1); } else { // field change prefix = 0; output.writeVInt(1); output.writeString(field); } int suffix = bytes.length - prefix; output.writeVInt(suffix); output.writeBytes(bytes.bytes, bytes.offset + prefix, suffix); lastTermBytes.copyBytes(bytes); lastTerm.bytes = lastTermBytes.get(); lastTerm.field = field; size += 1; } catch (IOException e) { throw new RuntimeException(e); } }
RAMOutputStream ros = new RAMOutputStream(); outputs.writeFinalOutput(emptyOutput, ros); byte[] emptyOutputBytes = new byte[(int) ros.getFilePointer()]; ros.writeTo(emptyOutputBytes, 0);
meta.writeInt(Lucene70DocValuesFormat.TERMS_DICT_BLOCK_SHIFT); RAMOutputStream addressBuffer = new RAMOutputStream(); meta.writeInt(DIRECT_MONOTONIC_BLOCK_SHIFT); long numBlocks = (size + Lucene70DocValuesFormat.TERMS_DICT_BLOCK_MASK) >>> Lucene70DocValuesFormat.TERMS_DICT_BLOCK_SHIFT; meta.writeLong(data.getFilePointer() - start); start = data.getFilePointer(); addressBuffer.writeTo(data); meta.writeLong(start); meta.writeLong(data.getFilePointer() - start);
/** Allocates internal skip buffers. */ protected void init() { skipBuffer = new RAMOutputStream[numberOfSkipLevels]; for (int i = 0; i < numberOfSkipLevels; i++) { skipBuffer[i] = new RAMOutputStream(); } }
/** Creates new buffers or empties the existing ones */ protected void resetSkip() { if (skipBuffer == null) { init(); } else { for (int i = 0; i < skipBuffer.length; i++) { skipBuffer[i].reset(); } } }
public void finishTerm(BytesRef text, BlockTermState state) throws IOException { if (numTerms > 0 && numTerms % SKIP_INTERVAL == 0) { bufferSkip(); } // write term meta data into fst final long longs[] = new long[longsSize]; final long delta = state.totalTermFreq - state.docFreq; if (state.totalTermFreq > 0) { if (delta == 0) { statsOut.writeVInt(state.docFreq<<1|1); } else { statsOut.writeVInt(state.docFreq<<1); statsOut.writeVLong(state.totalTermFreq-state.docFreq); } } else { statsOut.writeVInt(state.docFreq); } postingsWriter.encodeTerm(longs, metaBytesOut, fieldInfo, state, true); for (int i = 0; i < longsSize; i++) { metaLongsOut.writeVLong(longs[i] - lastLongs[i]); lastLongs[i] = longs[i]; } metaLongsOut.writeVLong(metaBytesOut.getFilePointer() - lastMetaBytesFP); builder.add(Util.toIntsRef(text, scratchTerm), numTerms); numTerms++; lastMetaBytesFP = metaBytesOut.getFilePointer(); }
public void seek(long pos) throws IOException { // set the file length in case we seek back // and flush() has not been called yet setFileLength(); if (pos < bufferStart || pos >= bufferStart + bufferLength) { currentBufferIndex = (int) (pos / BUFFER_SIZE); switchCurrentBuffer(); } bufferPosition = (int) (pos % BUFFER_SIZE); }
synchronized void free(PerDoc perDoc) { assert freeCount < docFreeList.length; assert 0 == perDoc.numStoredFields; assert 0 == perDoc.fdt.length(); assert 0 == perDoc.fdt.getFilePointer(); docFreeList[freeCount++] = perDoc; }
void flushDocument(int numStoredFields, RAMOutputStream buffer) throws IOException { indexStream.writeLong(fieldsStream.getFilePointer()); fieldsStream.writeVInt(numStoredFields); buffer.writeTo(fieldsStream); }
if (payloadLen != lastPayloadLen) { lastPayloadLen = payloadLen; buffer.writeVInt((delta<<1)|1); buffer.writeVInt(payloadLen); } else { buffer.writeVInt(delta<<1); buffer.writeVInt(delta); int offsetLength = endOffset - startOffset; if (offsetLength != lastOffsetLength) { buffer.writeVInt(offsetDelta << 1 | 1); buffer.writeVInt(offsetLength); } else { buffer.writeVInt(offsetDelta << 1); buffer.writeBytes(payload.bytes, payload.offset, payloadLen);