public static void calculateBytesArrayHashCodes(byte[][] bytesArrays, int[] starts, int[] lengths, int[] valueSelected, int[] hashCodes, final int count) { for (int i = 0; i < count; i++) { int batchIndex = valueSelected[i]; hashCodes[i] = murmurHash(bytesArrays[batchIndex], starts[batchIndex], lengths[batchIndex]); } }
@Override public void setHashKey() { hashcode = isNull0 ? nullHashcode : HashCodeUtil.calculateLongHashCode(longValue0); }
@Override public void setHashKey() { if (isNull0 || isNull1) { hashcode = (isNull0 && isNull1 ? twoNullHashcode : (isNull0 ? null0Hashcode ^ HashCodeUtil.calculateLongHashCode(longValue1) : HashCodeUtil.calculateLongHashCode(longValue0) ^ null1Hashcode)); } else { hashcode = HashCodeUtil.calculateTwoLongHashCode(longValue0, longValue1); } }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); Preconditions.checkState(!isEmptyKey); // Create all nulls key. try { Output nullKeyOutput = new Output(); keyBinarySortableSerializeWrite.set(nullKeyOutput); for (int i = 0; i < reduceSinkKeyColumnMap.length; i++) { keyBinarySortableSerializeWrite.writeNull(); } int nullBytesLength = nullKeyOutput.getLength(); nullBytes = new byte[nullBytesLength]; System.arraycopy(nullKeyOutput.getData(), 0, nullBytes, 0, nullBytesLength); nullKeyHashCode = HashCodeUtil.calculateBytesHashCode(nullBytes, 0, nullBytesLength); } catch (Exception e) { throw new HiveException(e); } }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); // Create all nulls key. try { Output nullKeyOutput = new Output(); keyBinarySortableSerializeWrite.set(nullKeyOutput); for (int i = 0; i < reduceSinkKeyColumnMap.length; i++) { keyBinarySortableSerializeWrite.writeNull(); } int nullBytesLength = nullKeyOutput.getLength(); nullBytes = new byte[nullBytesLength]; System.arraycopy(nullKeyOutput.getData(), 0, nullBytes, 0, nullBytesLength); nullKeyHashCode = HashCodeUtil.calculateBytesHashCode(nullBytes, 0, nullBytesLength); } catch (Exception e) { throw new HiveException(e); } }
public static int calculateBytesHashCode(byte[] keyBytes, int keyStart, int keyLength) { return murmurHash(keyBytes, keyStart, keyLength); }
public static void calculateLongArrayHashCodes(long[] longs, int[] hashCodes, final int count) { for (int v = 0; v < count; v++) { hashCodes[v] = (int) calculateLongHashCode(longs[v]); } }
public int hashCode(byte[] key, int offset, int length) { return HashCodeUtil.murmurHash(key, offset, length); }
long hashCode = HashCodeUtil.calculateLongHashCode(tableKey); int intHashCode = (int) hashCode; int newSlot = intHashCode & newLogicalHashBucketMask;
@Override public int getHashFromKey() throws SerDeException { byte[] keyBytes = key.getBytes(); int keyLength = key.getLength(); return HashCodeUtil.murmurHash(keyBytes, 0, keyLength); }
long hashCode = HashCodeUtil.calculateLongHashCode(tableKey); int intHashCode = (int) hashCode; int newSlot = intHashCode & newLogicalHashBucketMask;
@Override public int getHashFromKey() throws SerDeException { byte[] keyBytes = key.getBytes(); int keyLength = key.getLength(); return HashCodeUtil.murmurHash(keyBytes, 0, keyLength); } }
long hashCode = HashCodeUtil.calculateLongHashCode(key); int intHashCode = (int) hashCode; int slot = (intHashCode & logicalHashBucketMask);
/** * Batch compute the hash codes for all the serialized keys. * * NOTE: MAJOR MAJOR ASSUMPTION: * We assume that HashCodeUtil.murmurHash produces the same result * as MurmurHash.hash with seed = 0 (the method used by ReduceSinkOperator for * UNIFORM distribution). */ protected void computeSerializedHashCodes() { int offset = 0; int keyLength; byte[] bytes = output.getData(); for (int i = 0; i < nonNullKeyCount; i++) { keyLength = serializedKeyLengths[i]; hashCodes[i] = HashCodeUtil.murmurHash(bytes, offset, keyLength); offset += keyLength; } }
long hashCode = HashCodeUtil.calculateLongHashCode(key); int intHashCode = (int) hashCode; int slot = (intHashCode & logicalHashBucketMask);
public int hashCode(long offset, int length, Position readPos) { setReadPoint(offset, readPos); if (isAllInOneReadBuffer(length, readPos)) { int result = HashCodeUtil.murmurHash(readPos.buffer, readPos.offset, length); readPos.offset += length; return result; } // Rare case of buffer boundary. Unfortunately we'd have to copy some bytes. byte[] bytes = new byte[length]; int destOffset = 0; while (destOffset < length) { ponderNextBufferToRead(readPos); int toRead = Math.min(length - destOffset, wbSize - readPos.offset); System.arraycopy(readPos.buffer, readPos.offset, bytes, destOffset, toRead); readPos.offset += toRead; destOffset += toRead; } return HashCodeUtil.murmurHash(bytes, 0, bytes.length); }
@Override public JoinResult contains(long key, VectorMapJoinHashSetResult hashSetResult) { VectorMapJoinFastHashSet.HashSetResult optimizedHashSetResult = (VectorMapJoinFastHashSet.HashSetResult) hashSetResult; optimizedHashSetResult.forget(); long hashCode = HashCodeUtil.calculateLongHashCode(key); long existance = findReadSlot(key, hashCode); JoinUtil.JoinResult joinResult; if (existance == -1) { joinResult = JoinUtil.JoinResult.NOMATCH; } else { joinResult = JoinUtil.JoinResult.MATCH; } optimizedHashSetResult.setJoinResult(joinResult); return joinResult; }
@Override public int getHashFromKey() throws SerDeException { if (!(key instanceof BinaryComparable)) { throw new SerDeException("Unexpected type " + key.getClass().getCanonicalName()); } sanityCheckKeyForTag(); BinaryComparable b = (BinaryComparable)key; return HashCodeUtil.murmurHash(b.getBytes(), 0, b.getLength() - (hasTag ? 1 : 0)); }
@Override public JoinResult contains(long key, VectorMapJoinHashSetResult hashSetResult) { VectorMapJoinFastHashSet.HashSetResult optimizedHashSetResult = (VectorMapJoinFastHashSet.HashSetResult) hashSetResult; optimizedHashSetResult.forget(); long hashCode = HashCodeUtil.calculateLongHashCode(key); int pairIndex = findReadSlot(key, hashCode); JoinUtil.JoinResult joinResult; if (pairIndex == -1) { joinResult = JoinUtil.JoinResult.NOMATCH; } else { /* * NOTE: Support for trackMatched not needed yet for Set. if (matchTracker != null) { matchTracker.trackMatch(pairIndex / 2); } */ joinResult = JoinUtil.JoinResult.MATCH; } optimizedHashSetResult.setJoinResult(joinResult); return joinResult; }
@Override public int getHashFromKey() throws SerDeException { if (!(key instanceof BinaryComparable)) { throw new SerDeException("Unexpected type " + key.getClass().getCanonicalName()); } sanityCheckKeyForTag(); BinaryComparable b = (BinaryComparable)key; return HashCodeUtil.murmurHash(b.getBytes(), 0, b.getLength() - (hasTag ? 1 : 0)); }