/** * Attempts to backtrack thru the string after encountering a dead end * at some given position. Returns false if no more possible strings * can match. * * @param position current position in the input String * @return {@code position >= 0} if more possible solutions exist for the DFA */ private int backtrack(int position) { while (position-- > 0) { int nextChar = seekBytesRef.byteAt(position) & 0xff; // if a character is 0xff it's a dead-end too, // because there is no higher character in binary sort order. if (nextChar++ != 0xff) { seekBytesRef.setByteAt(position, (byte) nextChar); seekBytesRef.setLength(position+1); return position; } } return -1; /* all solutions exhausted */ } }
/** Just converts IntsRef to BytesRef; you must ensure the * int values fit into a byte. */ public static BytesRef toBytesRef(IntsRef input, BytesRefBuilder scratch) { scratch.grow(input.length); for(int i=0;i<input.length;i++) { int value = input.ints[i+input.offset]; // NOTE: we allow -128 to 255 assert value >= Byte.MIN_VALUE && value <= 255: "value " + value + " doesn't fit into byte"; scratch.setByteAt(i, (byte) value); } scratch.setLength(input.length); return scratch.get(); }
@Override protected BytesRef changeToken(BytesRef in) { int upto = 0; for (int i = 0; i < in.length; i++) { byte b = in.bytes[in.offset + i]; if (b == sepLabel) { spare.grow(upto + 2); spare.setByteAt(upto++, sepLabel); spare.setByteAt(upto++, b); } else { spare.grow(upto + 1); spare.setByteAt(upto++, b); } } spare.setLength(upto); return spare.get(); } }
@Override protected BytesRef changeToken(BytesRef in) { int upto = 0; for(int i=0;i<in.length;i++) { byte b = in.bytes[in.offset+i]; if (b == (byte) sepLabel) { spare.grow(upto+2); spare.setByteAt(upto++, (byte) sepLabel); spare.setByteAt(upto++, b); } else { spare.grow(upto+1); spare.setByteAt(upto++, b); } } spare.setLength(upto); return spare.get(); } }
term.setByteAt(targetUpto, (byte) targetLabel); arc = nextArc;
scratch.setByteAt(scratch.length()-1, (byte) mid); if (iterator.seekCeil(scratch.get()) == TermsEnum.SeekStatus.END) {
public void finishTerm(long defaultWeight) throws IOException { ArrayUtil.timSort(surfaceFormsAndPayload, 0, count); int deduplicator = 0; analyzed.append((byte) 0); analyzed.setLength(analyzed.length() + 1); analyzed.grow(analyzed.length()); for (int i = 0; i < count; i++) { analyzed.setByteAt(analyzed.length() - 1, (byte) deduplicator++); Util.toIntsRef(analyzed.get(), scratchInts); SurfaceFormAndPayload candiate = surfaceFormsAndPayload[i]; long cost = candiate.weight == -1 ? encodeWeight(Math.min(Integer.MAX_VALUE, defaultWeight)) : candiate.weight; builder.add(scratchInts.get(), outputs.newPair(cost, candiate.payload)); } seenSurfaceForms.clear(); count = 0; }
@Override protected BytesRef changeToken(BytesRef in) { int upto = 0; for (int i = 0; i < in.length; i++) { byte b = in.bytes[in.offset + i]; if (b == (byte) sepLabel) { spare.grow(upto + 2); spare.setByteAt(upto++, (byte) sepLabel); spare.setByteAt(upto++, b); } else { spare.grow(upto + 1); spare.setByteAt(upto++, b); } } spare.setLength(upto); return spare.get(); } }
@Override protected BytesRef changeToken(BytesRef in) { int upto = 0; for(int i=0;i<in.length;i++) { byte b = in.bytes[in.offset+i]; if (b == (byte) sepLabel) { spare.grow(upto+2); spare.setByteAt(upto++, (byte) sepLabel); spare.setByteAt(upto++, b); } else { spare.grow(upto+1); spare.setByteAt(upto++, b); } } spare.setLength(upto); return spare.get(); } }
public static void readLine(DataInput in, BytesRefBuilder scratch) throws IOException { int upto = 0; while(true) { byte b = in.readByte(); scratch.grow(1+upto); if (b == ESCAPE) { scratch.setByteAt(upto++, in.readByte()); } else { if (b == NEWLINE) { break; } else { scratch.setByteAt(upto++, b); } } } scratch.setLength(upto); }
/** Just converts IntsRef to BytesRef; you must ensure the * int values fit into a byte. */ public static BytesRef toBytesRef(IntsRef input, BytesRefBuilder scratch) { scratch.grow(input.length); for(int i=0;i<input.length;i++) { int value = input.ints[i+input.offset]; // NOTE: we allow -128 to 255 assert value >= Byte.MIN_VALUE && value <= 255: "value " + value + " doesn't fit into byte"; scratch.setByteAt(i, (byte) value); } scratch.setLength(input.length); return scratch.get(); }
/** Just converts IntsRef to BytesRef; you must ensure the * int values fit into a byte. */ public static BytesRef toBytesRef(IntsRef input, BytesRefBuilder scratch) { scratch.grow(input.length); for(int i=0;i<input.length;i++) { int value = input.ints[i+input.offset]; // NOTE: we allow -128 to 255 assert value >= Byte.MIN_VALUE && value <= 255: "value " + value + " doesn't fit into byte"; scratch.setByteAt(i, (byte) value); } scratch.setLength(input.length); return scratch.get(); }
/** Just converts IntsRef to BytesRef; you must ensure the * int values fit into a byte. */ public static BytesRef toBytesRef(IntsRef input, BytesRefBuilder scratch) { scratch.grow(input.length); for(int i=0;i<input.length;i++) { int value = input.ints[i+input.offset]; // NOTE: we allow -128 to 255 assert value >= Byte.MIN_VALUE && value <= 255: "value " + value + " doesn't fit into byte"; scratch.setByteAt(i, (byte) value); } scratch.setLength(input.length); return scratch.get(); }
public void finishTerm(long defaultWeight) throws IOException { ArrayUtil.timSort(surfaceFormsAndPayload, 0, count); int deduplicator = 0; analyzed.append((byte) 0); analyzed.setLength(analyzed.length() + 1); analyzed.grow(analyzed.length()); for (int i = 0; i < count; i++) { analyzed.setByteAt(analyzed.length() - 1, (byte) deduplicator++); Util.toIntsRef(analyzed.get(), scratchInts); SurfaceFormAndPayload candiate = surfaceFormsAndPayload[i]; long cost = candiate.weight == -1 ? encodeWeight(Math.min(Integer.MAX_VALUE, defaultWeight)) : candiate.weight; builder.add(scratchInts.get(), outputs.newPair(cost, candiate.payload)); } seenSurfaceForms.clear(); count = 0; }
public void finishTerm(long defaultWeight) throws IOException { ArrayUtil.timSort(surfaceFormsAndPayload, 0, count); int deduplicator = 0; analyzed.append((byte) 0); analyzed.setLength(analyzed.length() + 1); analyzed.grow(analyzed.length()); for (int i = 0; i < count; i++) { analyzed.setByteAt(analyzed.length() - 1, (byte) deduplicator++); Util.toIntsRef(analyzed.get(), scratchInts); SurfaceFormAndPayload candiate = surfaceFormsAndPayload[i]; long cost = candiate.weight == -1 ? encodeWeight(Math.min(Integer.MAX_VALUE, defaultWeight)) : candiate.weight; builder.add(scratchInts.get(), outputs.newPair(cost, candiate.payload)); } seenSurfaceForms.clear(); count = 0; }
public void finishTerm(long defaultWeight) throws IOException { ArrayUtil.timSort(surfaceFormsAndPayload, 0, count); int deduplicator = 0; analyzed.append((byte) 0); analyzed.setLength(analyzed.length() + 1); analyzed.grow(analyzed.length()); for (int i = 0; i < count; i++) { analyzed.setByteAt(analyzed.length() - 1, (byte) deduplicator++); Util.toIntsRef(analyzed.get(), scratchInts); SurfaceFormAndPayload candiate = surfaceFormsAndPayload[i]; long cost = candiate.weight == -1 ? encodeWeight(Math.min(Integer.MAX_VALUE, defaultWeight)) : candiate.weight; builder.add(scratchInts.get(), outputs.newPair(cost, candiate.payload)); } seenSurfaceForms.clear(); count = 0; }
public void finishTerm(long defaultWeight) throws IOException { ArrayUtil.timSort(surfaceFormsAndPayload, 0, count); int deduplicator = 0; analyzed.append((byte) 0); analyzed.setLength(analyzed.length() + 1); analyzed.grow(analyzed.length()); for (int i = 0; i < count; i++) { analyzed.setByteAt(analyzed.length() - 1, (byte) deduplicator++); Util.toIntsRef(analyzed.get(), scratchInts); SurfaceFormAndPayload candiate = surfaceFormsAndPayload[i]; long cost = candiate.weight == -1 ? encodeWeight(Math.min(Integer.MAX_VALUE, defaultWeight)) : candiate.weight; builder.add(scratchInts.get(), outputs.newPair(cost, candiate.payload)); } seenSurfaceForms.clear(); count = 0; }