long tailOffset = Ref.getOffset(ref); writeBuffers.setUnsafeReadPoint(recOffset); int valueLength = (int)writeBuffers.unsafeReadVLong(), keyLength = (int)writeBuffers.unsafeReadVLong(); long ptrOffset = writeBuffers.getUnsafeReadPoint(); if (Ref.hasList(ref)) { byteIntervals.put(recOffset, (int)(ptrOffset + 5 - recOffset));
long tailOffset = Ref.getOffset(ref); writeBuffers.setReadPoint(recOffset); int valueLength = (int)writeBuffers.readVLong(), keyLength = (int)writeBuffers.readVLong(); long ptrOffset = writeBuffers.getReadPoint(); if (Ref.hasList(ref)) { byteIntervals.put(recOffset, (int)(ptrOffset + 5 - recOffset));
refs[slot] = Ref.makeFirstRef(tailOffset, stateByte, hashCode, startingHashBitCount); ++keysAssigned; } else { long tailOffset = writeValueAndLength(kv); addRecordToList(lrPtrOffset, tailOffset); byte oldStateByte = Ref.getStateByte(ref); byte stateByte = kv.updateStateByte(oldStateByte); if (oldStateByte != stateByte) { ref = Ref.setStateByte(ref, stateByte); refs[slot] = Ref.setListFlag(ref);
int hashCode = (int)writeBuffers.readNByteLong(Ref.getOffset(oldRef) - writeBuffers.readVLong() - writeBuffers.readVLong() - 4, 4); int probeSteps = relocateKeyRef(newRefs, oldRef, hashCode);
/** * Verifies that the key matches a requisite key. * @param cmpOffset The offset to the key to compare with. * @param cmpLength The length of the key to compare with. * @param ref The ref that can be used to retrieve the candidate key. * @param hashCode * @return -1 if the key referenced by ref is different than the one referenced by cmp... * 0 if the keys match, and there's only one value for this key (no list). * Offset if the keys match, and there are multiple values for this key (a list). */ private boolean isSameKey(long cmpOffset, int cmpLength, long ref, int hashCode) { if (!compareHashBits(ref, hashCode)) { return false; // Hash bits in ref don't match. } writeBuffers.setReadPoint(getFirstRecordLengthsOffset(ref, null)); int valueLength = (int)writeBuffers.readVLong(), keyLength = (int)writeBuffers.readVLong(); if (keyLength != cmpLength) { return false; } long keyOffset = Ref.getOffset(ref) - (valueLength + keyLength); // There's full hash code stored in front of the key. We could check that first. If keyLength // is <= 4 it obviously doesn't make sense, less bytes to check in a key. Then, if there's a // match, we check it in vain. But what is the proportion of matches? For writes it could be 0 // if all keys are unique, for reads we hope it's really high. Then if there's a mismatch what // probability is there that key mismatches in <4 bytes (so just checking the key is faster)? // We assume the latter is pretty high, so we don't check for now. return writeBuffers.isEqual(cmpOffset, cmpLength, keyOffset, keyLength); }
/** * Finds a key. Values can be read with the supplied result object. * * Important Note: The caller is expected to pre-allocate the hashMapResult and not * share it among other threads. * * @param key Key buffer. * @param offset the offset to the key in the buffer * @param hashMapResult The object to fill in that can read the values. * @return The state byte. */ public byte getValueResult(byte[] key, int offset, int length, Result hashMapResult) { hashMapResult.forget(); WriteBuffers.Position readPos = hashMapResult.getReadPos(); // First, find first record for the key. long ref = findKeyRefToRead(key, offset, length, readPos); if (ref == 0) { return 0; } boolean hasList = Ref.hasList(ref); // This relies on findKeyRefToRead doing key equality check and leaving read ptr where needed. long offsetAfterListRecordKeyLen = hasList ? writeBuffers.getReadPoint(readPos) : 0; hashMapResult.set(this, Ref.getOffset(ref), hasList, offsetAfterListRecordKeyLen); return Ref.getStateByte(ref); }
/** * @param ref The ref. * @return The offset to list record pointer; list record is created if it doesn't exist. */ private long createOrGetListRecord(long ref) { if (Ref.hasList(ref)) { // LOG.info("Found list record at " + writeBuffers.getReadPoint()); return writeBuffers.getReadPoint(); // Assumes we are here after key compare. } long firstTailOffset = Ref.getOffset(ref); // LOG.info("First tail offset to create list record is " + firstTailOffset); // Determine the length of storage for value and key lengths of the first record. writeBuffers.setReadPoint(firstTailOffset); writeBuffers.skipVLong(); writeBuffers.skipVLong(); int lengthsLength = (int)(writeBuffers.getReadPoint() - firstTailOffset); // Create the list record, copy first record value/key lengths there. writeBuffers.writeBytes(firstTailOffset, lengthsLength); long lrPtrOffset = writeBuffers.getWritePoint(); // LOG.info("Creating list record: copying " + lengthsLength + ", lrPtrOffset " + lrPtrOffset); // Reserve 5 bytes for writeValueRecord to fill. There might be junk there so null them. writeBuffers.write(FIVE_ZEROES); // Link the list record to the first element. writeBuffers.writeFiveByteULong(firstTailOffset, lrPtrOffset - lengthsLength - firstTailOffset); return lrPtrOffset; }
/** * Same as {@link #isSameKey(long, int, long, int)} but for externally stored key. */ private boolean isSameKey(byte[] key, int offset, int length, long ref, int hashCode, WriteBuffers.Position readPos) { if (!compareHashBits(ref, hashCode)) { return false; // Hash bits don't match. } writeBuffers.setReadPoint(getFirstRecordLengthsOffset(ref, readPos), readPos); int valueLength = (int)writeBuffers.readVLong(readPos), keyLength = (int)writeBuffers.readVLong(readPos); long keyOffset = Ref.getOffset(ref) - (valueLength + keyLength); // See the comment in the other isSameKey if (offset == 0) { return writeBuffers.isEqual(key, length, keyOffset, keyLength); } else { return writeBuffers.isEqual(key, offset, length, keyOffset, keyLength); } }
/** * @param ref Reference. * @return The offset to value and key length vlongs of the first record referenced by ref. */ private long getFirstRecordLengthsOffset(long ref, WriteBuffers.Position readPos) { long tailOffset = Ref.getOffset(ref); if (Ref.hasList(ref)) { long relativeOffset = (readPos == null) ? writeBuffers.readNByteLong(tailOffset, 5) : writeBuffers.readNByteLong(tailOffset, 5, readPos); tailOffset += relativeOffset; } return tailOffset; }
private boolean compareHashBits(long ref, int hashCode) { long fakeRef = Ref.makeFirstRef(0, (byte)0, hashCode, startingHashBitCount); return (Ref.getHashBits(fakeRef) == Ref.getHashBits(ref)); }
public static int getNthHashBit(long ref, int skippedBits, int position) { long hashPart = getHashBits(ref) << skippedBits; // Original hashcode, with 0-d low bits. return (int)(hashPart & (1 << (position - 1))); }
private static String dumpRef(long ref) { return StringUtils.leftPad(Long.toBinaryString(ref), 64, "0") + " o=" + Ref.getOffset(ref) + " s=" + Ref.getStateByte(ref) + " l=" + Ref.hasList(ref) + " h=" + Long.toBinaryString(Ref.getHashBits(ref)); }
long tailOffset = Ref.getOffset(ref); writeBuffers.setUnsafeReadPoint(recOffset); int valueLength = (int)writeBuffers.unsafeReadVLong(), keyLength = (int)writeBuffers.unsafeReadVLong(); long ptrOffset = writeBuffers.getUnsafeReadPoint(); if (Ref.hasList(ref)) { byteIntervals.put(recOffset, (int)(ptrOffset + 5 - recOffset));
int hashCode = (int)writeBuffers.unsafeReadNByteLong(Ref.getOffset(oldRef) - writeBuffers.unsafeReadVLong() - writeBuffers.unsafeReadVLong() - 4, 4); int probeSteps = relocateKeyRef(newRefs, oldRef, hashCode);
refs[slot] = Ref.makeFirstRef(tailOffset, stateByte, hashCode, startingHashBitCount); ++keysAssigned; } else { long tailOffset = writeValueAndLength(kv); addRecordToList(lrPtrOffset, tailOffset); byte oldStateByte = Ref.getStateByte(ref); byte stateByte = kv.updateStateByte(oldStateByte); if (oldStateByte != stateByte) { ref = Ref.setStateByte(ref, stateByte); refs[slot] = Ref.setListFlag(ref);
int valueLength = (int) writeBuffers.readVLong(readPos); int keyLength = (int) writeBuffers.readVLong(readPos); long keyOffset = Ref.getOffset(ref) - (valueLength + keyLength); boolean hasList = Ref.hasList(ref); long offsetAfterListRecordKeyLen = hasList ? writeBuffers.getReadPoint(readPos) : 0; hashMapResult.set(this, Ref.getOffset(ref), hasList, offsetAfterListRecordKeyLen);
/** * Verifies that the key matches a requisite key. * @param cmpOffset The offset to the key to compare with. * @param cmpLength The length of the key to compare with. * @param ref The ref that can be used to retrieve the candidate key. * @param hashCode * @return -1 if the key referenced by ref is different than the one referenced by cmp... * 0 if the keys match, and there's only one value for this key (no list). * Offset if the keys match, and there are multiple values for this key (a list). */ private boolean isSameKey(long cmpOffset, int cmpLength, long ref, int hashCode) { if (!compareHashBits(ref, hashCode)) { return false; // Hash bits in ref don't match. } writeBuffers.setUnsafeReadPoint(getFirstRecordLengthsOffset(ref, null)); int valueLength = (int)writeBuffers.unsafeReadVLong(), keyLength = (int)writeBuffers.unsafeReadVLong(); if (keyLength != cmpLength) { return false; } long keyOffset = Ref.getOffset(ref) - (valueLength + keyLength); // There's full hash code stored in front of the key. We could check that first. If keyLength // is <= 4 it obviously doesn't make sense, less bytes to check in a key. Then, if there's a // match, we check it in vain. But what is the proportion of matches? For writes it could be 0 // if all keys are unique, for reads we hope it's really high. Then if there's a mismatch what // probability is there that key mismatches in <4 bytes (so just checking the key is faster)? // We assume the latter is pretty high, so we don't check for now. return writeBuffers.isEqual(cmpOffset, cmpLength, keyOffset, keyLength); }
int hashCode = (int)writeBuffers.unsafeReadNByteLong(Ref.getOffset(oldRef) - writeBuffers.unsafeReadVLong() - writeBuffers.unsafeReadVLong() - 4, 4); int probeSteps = relocateKeyRef(newRefs, oldRef, hashCode);
/** * @param ref The ref. * @return The offset to list record pointer; list record is created if it doesn't exist. */ private long createOrGetListRecord(long ref) { if (Ref.hasList(ref)) { // LOG.info("Found list record at " + writeBuffers.getReadPoint()); return writeBuffers.getUnsafeReadPoint(); // Assumes we are here after key compare. } long firstTailOffset = Ref.getOffset(ref); // LOG.info("First tail offset to create list record is " + firstTailOffset); // Determine the length of storage for value and key lengths of the first record. writeBuffers.setUnsafeReadPoint(firstTailOffset); writeBuffers.unsafeSkipVLong(); writeBuffers.unsafeSkipVLong(); int lengthsLength = (int)(writeBuffers.getUnsafeReadPoint() - firstTailOffset); // Create the list record, copy first record value/key lengths there. writeBuffers.writeBytes(firstTailOffset, lengthsLength); long lrPtrOffset = writeBuffers.getWritePoint(); // LOG.info("Creating list record: copying " + lengthsLength + ", lrPtrOffset " + lrPtrOffset); // Reserve 5 bytes for writeValueRecord to fill. There might be junk there so null them. writeBuffers.write(FIVE_ZEROES); // Link the list record to the first element. writeBuffers.writeFiveByteULong(firstTailOffset, lrPtrOffset - lengthsLength - firstTailOffset); return lrPtrOffset; }
refs[slot] = Ref.makeFirstRef(tailOffset, stateByte, hashCode, startingHashBitCount); ++keysAssigned; } else { long tailOffset = writeValueAndLength(kv); addRecordToList(lrPtrOffset, tailOffset); byte oldStateByte = Ref.getStateByte(ref); byte stateByte = kv.updateStateByte(oldStateByte); if (oldStateByte != stateByte) { ref = Ref.setStateByte(ref, stateByte); refs[slot] = Ref.setListFlag(ref);