@Override public long getEstimatedMemorySize() { long size = 0; // adding 16KB constant memory for stringCommon as the rabit hole is deep to implement // MemoryEstimate interface, also it is constant overhead size += (16 * 1024L); return super.getEstimatedMemorySize() + size; } }
public void verifyNonMatched(VectorMapJoinFastBytesHashMap map, Random random) throws HiveException { int mapSize = map.size(); if (mapSize != count) { TestCase.fail("map.size() does not match expected count"); MatchTracker matchTracker = map.createMatchTracker(); boolean[] nonMatched = new boolean[mapSize]; int nonMatchedCount = 0; map.createNonMatchedIterator(matchTracker); nonMatchedIterator.init();
@Override public JoinUtil.JoinResult lookup(byte[] keyBytes, int keyStart, int keyLength, VectorMapJoinHashMapResult hashMapResult) { VectorMapJoinFastValueStore.HashMapResult optimizedHashMapResult = (VectorMapJoinFastValueStore.HashMapResult) hashMapResult; optimizedHashMapResult.forget(); long hashCode = HashCodeUtil.murmurHash(keyBytes, keyStart, keyLength); long valueRefWord = findReadSlot(keyBytes, keyStart, keyLength, hashCode, hashMapResult.getReadPos()); JoinUtil.JoinResult joinResult; if (valueRefWord == -1) { joinResult = JoinUtil.JoinResult.NOMATCH; } else { // LOG.debug("VectorMapJoinFastBytesHashMap lookup hashCode " + Long.toHexString(hashCode) + " valueRefWord " + Long.toHexString(valueRefWord) + " (valueStore != null) " + (valueStore != null)); optimizedHashMapResult.set(valueStore, valueRefWord); joinResult = JoinUtil.JoinResult.MATCH; } optimizedHashMapResult.setJoinResult(joinResult); return joinResult; }
private void verifyOne(VectorMapJoinFastBytesHashMap map, int index, MatchTracker matchTracker) { FastBytesHashMapElement element = array[index]; byte[] key = element.getKey(); List<byte[]> values = element.getValues(); VectorMapJoinHashMapResult hashMapResult = map.createHashMapResult(); JoinUtil.JoinResult joinResult = map.lookup(key, 0, key.length, hashMapResult, matchTracker); if (joinResult != JoinUtil.JoinResult.MATCH) { assertTrue(false); } verifyHashMapValues(hashMapResult, values); }
@Override public JoinUtil.JoinResult lookup(byte[] keyBytes, int keyStart, int keyLength, VectorMapJoinHashMapResult hashMapResult) { VectorMapJoinFastBytesHashMapStore.HashMapResult fastHashMapResult = (VectorMapJoinFastBytesHashMapStore.HashMapResult) hashMapResult; fastHashMapResult.forget(); long hashCode = HashCodeUtil.murmurHash(keyBytes, keyStart, keyLength); doHashMapMatch( keyBytes, keyStart, keyLength, hashCode, fastHashMapResult); return fastHashMapResult.joinResult(); }
public void verify(VectorMapJoinFastBytesHashMap map) { int mapSize = map.size(); if (mapSize != count) { TestCase.fail("map.size() does not match expected count"); } for (int index = 0; index < count; index++) { verifyOne(map, index, null); } }
public void add(byte[] keyBytes, int keyStart, int keyLength, BytesWritable currentValue) { expandAndRehash();
@Override public JoinUtil.JoinResult lookup(byte[] keyBytes, int keyStart, int keyLength, VectorMapJoinHashMapResult hashMapResult) { VectorMapJoinFastValueStore.HashMapResult optimizedHashMapResult = (VectorMapJoinFastValueStore.HashMapResult) hashMapResult; optimizedHashMapResult.forget(); long hashCode = VectorMapJoinFastBytesHashUtil.hashKey(keyBytes, keyStart, keyLength); long valueRefWord = findReadSlot(keyBytes, keyStart, keyLength, hashCode); JoinUtil.JoinResult joinResult; if (valueRefWord == -1) { joinResult = JoinUtil.JoinResult.NOMATCH; } else { // LOG.debug("VectorMapJoinFastBytesHashMap lookup hashCode " + Long.toHexString(hashCode) + " valueRefWord " + Long.toHexString(valueRefWord) + " (valueStore != null) " + (valueStore != null)); optimizedHashMapResult.set(valueStore, valueRefWord); joinResult = JoinUtil.JoinResult.MATCH; } optimizedHashMapResult.setJoinResult(joinResult); return joinResult; }
@Override public JoinUtil.JoinResult lookup(byte[] keyBytes, int keyStart, int keyLength, VectorMapJoinHashMapResult hashMapResult, MatchTracker matchTracker) { VectorMapJoinFastBytesHashMapStore.HashMapResult fastHashMapResult = (VectorMapJoinFastBytesHashMapStore.HashMapResult) hashMapResult; fastHashMapResult.forget(); long hashCode = HashCodeUtil.murmurHash(keyBytes, keyStart, keyLength); final int slot = doHashMapMatch( keyBytes, keyStart, keyLength, hashCode, fastHashMapResult); if (slot != -1 && matchTracker != null) { matchTracker.trackMatch(slot); } return fastHashMapResult.joinResult(); }
@Override public long getEstimatedMemorySize() { return super.getEstimatedMemorySize(); } }