@Override public long hashRow(int position, Page page) { long result = 0; for (int i = 0; i < hashChannels.size(); i++) { int hashChannel = hashChannels.get(i); Type type = types.get(hashChannel); Block block = page.getBlock(i); result = result * 31 + TypeUtils.hashPosition(type, block, position); } return result; }
@UsedByGeneratedCode public static long hash(MethodHandle keyHashCodeFunction, MethodHandle valueHashCodeFunction, Type keyType, Type valueType, Block block) { long result = 0; for (int position = 0; position < block.getPositionCount(); position += 2) { result += hashPosition(keyHashCodeFunction, keyType, block, position) ^ hashPosition(valueHashCodeFunction, valueType, block, position + 1); } return result; } }
public long hashPosition(int position, IntFunction<Block> blockProvider) { long result = HashGenerationOptimizer.INITIAL_HASH_VALUE; for (int i = 0; i < hashChannels.length; i++) { Type type = hashChannelTypes.get(i); result = CombineHashFunction.getHash(result, TypeUtils.hashPosition(type, blockProvider.apply(hashChannels[i]), position)); } return result; }
@Override public long hashPosition(int blockIndex, int position) { if (precomputedHashChannel != null) { return BIGINT.getLong(precomputedHashChannel.get(blockIndex), position); } long result = 0; for (int hashChannel : hashChannels) { Type type = types.get(hashChannel); Block block = channels.get(hashChannel).get(blockIndex); result = result * 31 + TypeUtils.hashPosition(type, block, position); } return result; }
private int getHashPosition(int position) { int hashPosition = getMaskedHash(hashPosition(type, block, position)); while (true) { if (positionByHash[hashPosition] == EMPTY_SLOT) { return hashPosition; } else if (positionEqualsPosition(type, block, positionByHash[hashPosition], block, position)) { return hashPosition; } hashPosition = getMaskedHash(hashPosition + 1); } }
private int getHashPositionOfKey(Block key, int position) { int hashPosition = getMaskedHash(hashPosition(keyType, key, position)); while (true) { if (keyPositionByHash[hashPosition] == EMPTY_SLOT) { return hashPosition; } else if (positionEqualsPosition(keyType, keyBlockBuilder, keyPositionByHash[hashPosition], key, position)) { return hashPosition; } hashPosition = getMaskedHash(hashPosition + 1); } }
@Test public void testUniqueness() { assertEquals(valueStore.addAndGetPosition(type, block, 0, TypeUtils.hashPosition(type, block, 0)), 0); assertEquals(valueStore.addAndGetPosition(type, block, 1, TypeUtils.hashPosition(type, block, 1)), 1); assertEquals(valueStore.addAndGetPosition(type, block, 2, TypeUtils.hashPosition(type, block, 2)), 2); assertEquals(valueStore.addAndGetPosition(type, block, 1, TypeUtils.hashPosition(type, block, 1)), 1); assertEquals(valueStore.addAndGetPosition(type, block, 3, TypeUtils.hashPosition(type, block, 1)), 3); }
@Test public void testTriggerRehash() { long hash0 = TypeUtils.hashPosition(type, block, 0); long hash1 = TypeUtils.hashPosition(type, block, 1); long hash2 = TypeUtils.hashPosition(type, block, 2); assertEquals(valueStoreSmall.addAndGetPosition(type, block, 0, hash0), 0); assertEquals(valueStoreSmall.addAndGetPosition(type, block, 1, hash1), 1); // triggers rehash and hash1 will end up in position 3 assertEquals(valueStoreSmall.addAndGetPosition(type, block, 2, hash2), 2); // this is just to make sure we trigger rehash code positions should be the same assertTrue(valueStoreSmall.getRehashCount() > 0); assertEquals(valueStoreSmall.addAndGetPosition(type, block, 0, hash0), 0); assertEquals(valueStoreSmall.addAndGetPosition(type, block, 1, hash1), 1); assertEquals(valueStoreSmall.addAndGetPosition(type, block, 2, hash2), 2); } }
/** * Get slot position of element at {@code position} of {@code block} */ private int getHashPositionOfElement(Block block, int position) { int hashPosition = getMaskedHash(hashPosition(elementType, block, position)); while (true) { int blockPosition = blockPositionByHash.get(hashPosition); // Doesn't have this element if (blockPosition == EMPTY_SLOT) { return hashPosition; } // Already has this element else if (positionEqualsPosition(elementType, elementBlock, blockPosition, block, position)) { return hashPosition; } hashPosition = getMaskedHash(hashPosition + 1); } }
@Override public void add(int position, Block block, long count) { int hashPosition = getBucketId(TypeUtils.hashPosition(type, block, position), mask); // look for an empty slot or a slot containing this key while (true) { if (hashPositions.get(hashPosition) == -1) { break; } if (type.equalTo(block, position, values, hashPositions.get(hashPosition))) { counts.add(hashPositions.get(hashPosition), count); return; } // increment position and mask to handle wrap around hashPosition = (hashPosition + 1) & mask; } addNewGroup(hashPosition, position, block, count); }
private void rehash() { long newCapacityLong = hashCapacity * 2L; if (newCapacityLong > Integer.MAX_VALUE) { throw new PrestoException(GENERIC_INSUFFICIENT_RESOURCES, "Size of hash table cannot exceed 1 billion entries"); } int newCapacity = (int) newCapacityLong; int newMask = newCapacity - 1; IntBigArray newHashPositions = new IntBigArray(-1); newHashPositions.ensureCapacity(newCapacity); for (int i = 0; i < values.getPositionCount(); i++) { // find an empty slot for the address int hashPosition = getBucketId(TypeUtils.hashPosition(type, values, i), newMask); while (newHashPositions.get(hashPosition) != -1) { hashPosition = (hashPosition + 1) & newMask; } // record the mapping newHashPositions.set(hashPosition, i); } hashCapacity = newCapacity; mask = newMask; maxFill = calculateMaxFill(newCapacity); hashPositions = newHashPositions; this.counts.ensureCapacity(maxFill); }
assertEquals(hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition), hashPosition(VARCHAR, leftBlock, leftBlockPosition));
long valueHash = murmurHash3(TypeUtils.hashPosition(type, block, position)); long groupIdHash = murmurHash3(groupId); long valueAndGroupHash = combineGroupAndValueHash(groupIdHash, valueHash);
assertEquals(hashPosition(type, block, position), expectedHash);
@TypeParameter("T") @SqlType(StandardTypes.BIGINT) public static long anotherHash( @OperatorDependency(operator = HASH_CODE, returnType = StandardTypes.BIGINT, argumentTypes = "T") MethodHandle hashFunction, @TypeParameter("T") Type type, @SqlType("array(T)") Block block) { int hash = 0; for (int i = 0; i < block.getPositionCount(); i++) { checkElementNotNull(block.isNull(i), ARRAY_NULL_ELEMENT_MSG); hash = (int) getHash(hash, hashPosition(hashFunction, type, block, i)); } return hash; } }
protected void assertPositionEquals(Block block, int position, Object expectedStackValue, Object expectedObjectValue) { long hash = 0; if (type.isComparable()) { hash = hashPosition(type, block, position); } assertPositionValue(block, position, expectedStackValue, hash, expectedObjectValue); assertPositionValue(block.getSingleValueBlock(position), 0, expectedStackValue, hash, expectedObjectValue); assertPositionValue(block.getRegion(position, 1), 0, expectedStackValue, hash, expectedObjectValue); assertPositionValue(block.getRegion(0, position + 1), position, expectedStackValue, hash, expectedObjectValue); assertPositionValue(block.getRegion(position, block.getPositionCount() - position), 0, expectedStackValue, hash, expectedObjectValue); BlockBuilder blockBuilder = type.createBlockBuilder(null, 1); type.appendTo(block, position, blockBuilder); assertPositionValue(blockBuilder.build(), 0, expectedStackValue, hash, expectedObjectValue); }
public static long hash(MethodHandle keyHashCodeFunction, MethodHandle valueHashCodeFunction, Type keyType, Type valueType, Block block) { long result = 0; for (int position = 0; position < block.getPositionCount(); position += 2) { result += hashPosition(keyHashCodeFunction, keyType, block, position); result += hashPosition(valueHashCodeFunction, valueType, block, position + 1); } return result; } }
@Override public int hashRow(int position, Block... blocks) { int result = 0; for (int i = 0; i < hashChannels.size(); i++) { int hashChannel = hashChannels.get(i); Type type = types.get(hashChannel); Block block = blocks[i]; result = result * 31 + TypeUtils.hashPosition(type, block, position); } return result; }
@Override public int hash(Block block, int position) { Block mapBlock = getObject(block, position); int result = 0; for (int i = 0; i < mapBlock.getPositionCount(); i += 2) { result += hashPosition(keyType, mapBlock, i); result += hashPosition(valueType, mapBlock, i + 1); } return result; }
@Override public int hashPosition(int position, Page page) { Block[] blocks = page.getBlocks(); int result = HashGenerationOptimizer.INITIAL_HASH_VALUE; for (int i = 0; i < hashChannels.length; i++) { Type type = hashChannelTypes.get(i); result = (int) CombineHashFunction.getHash(result, TypeUtils.hashPosition(type, blocks[hashChannels[i]], position)); } return result; }