public void addRowGroup(int rowCount) { longSlice.setInt(0, rowCount); rowGroupHash.update(longBuffer, 0, Integer.BYTES); }
private WriteChecksumBuilder(List<Type> types) { this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); ImmutableList.Builder<XxHash64> columnHashes = ImmutableList.builder(); for (Type ignored : types) { columnHashes.add(new XxHash64()); } this.columnHashes = columnHashes.build(); }
private static long updateBody(long seed, Object base, long address, int length) { long v1 = seed + PRIME64_1 + PRIME64_2; long v2 = seed + PRIME64_2; long v3 = seed; long v4 = seed - PRIME64_1; int remaining = length; while (remaining >= 32) { v1 = mix(v1, unsafe.getLong(base, address)); v2 = mix(v2, unsafe.getLong(base, address + 8)); v3 = mix(v3, unsafe.getLong(base, address + 16)); v4 = mix(v4, unsafe.getLong(base, address + 24)); address += 32; remaining -= 32; } long hash = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18); hash = update(hash, v1); hash = update(hash, v2); hash = update(hash, v3); hash = update(hash, v4); return hash; }
public long hash() { long hash; if (bodyLength > 0) { hash = computeBody(); } else { hash = seed + PRIME64_5; } hash += bodyLength + bufferSize; return updateTail(hash, buffer, BUFFER_ADDRESS, 0, bufferSize); }
private static long updateBody(long seed, Object base, long address, int length) { long v1 = seed + PRIME64_1 + PRIME64_2; long v2 = seed + PRIME64_2; long v3 = seed; long v4 = seed - PRIME64_1; int remaining = length; while (remaining >= 32) { v1 = mix(v1, unsafe.getLong(base, address)); v2 = mix(v2, unsafe.getLong(base, address + 8)); v3 = mix(v3, unsafe.getLong(base, address + 16)); v4 = mix(v4, unsafe.getLong(base, address + 24)); address += 32; remaining -= 32; } long hash = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18); hash = update(hash, v1); hash = update(hash, v2); hash = update(hash, v3); hash = update(hash, v4); return hash; }
public long hash() { long hash; if (bodyLength > 0) { hash = computeBody(); } else { hash = seed + PRIME64_5; } hash += bodyLength + bufferSize; return updateTail(hash, buffer, BUFFER_ADDRESS, 0, bufferSize); }
@Override public long checksum() { return XxHash64.hash(Slices.wrappedIntArray(positionLinks)); } };
@Test public void testEmpty() throws Exception { long expected = 0xEF46DB3751D8E999L; assertEquals(hash(EMPTY_SLICE), expected); assertEquals(hash(EMPTY_SLICE, 0, 0), expected); assertEquals(hash(0, EMPTY_SLICE), expected); assertEquals(hash(0, EMPTY_SLICE, 0, 0), expected); assertEquals(new XxHash64().update(EMPTY_SLICE).hash(), expected); assertEquals(new XxHash64().update(EMPTY_SLICE, 0, 0).hash(), expected); assertEquals(new XxHash64().update(EMPTY_BYTES).hash(), expected); assertEquals(new XxHash64().update(EMPTY_BYTES, 0, 0).hash(), expected); assertEquals( new XxHash64() .update(EMPTY_BYTES) .update(EMPTY_BYTES, 0, 0) .update(EMPTY_SLICE) .update(EMPTY_SLICE, 0, 0) .hash(), expected); }
public void addStripe(int rowCount) { longSlice.setInt(0, rowCount); stripeHash.update(longBuffer, 0, Integer.BYTES); }
private WriteChecksumBuilder(List<Type> types) { this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); ImmutableList.Builder<XxHash64> columnHashes = ImmutableList.builder(); for (Type ignored : types) { columnHashes.add(new XxHash64()); } this.columnHashes = columnHashes.build(); }
public WriteChecksum build() { return new WriteChecksum( totalRowCount, rowGroupHash.hash(), columnHashes.stream() .map(XxHash64::hash) .collect(toList())); } }
@Test public void testEmpty() throws Exception { long expected = 0xEF46DB3751D8E999L; assertEquals(hash(EMPTY_SLICE), expected); assertEquals(hash(EMPTY_SLICE, 0, 0), expected); assertEquals(hash(0, EMPTY_SLICE), expected); assertEquals(hash(0, EMPTY_SLICE, 0, 0), expected); assertEquals(new XxHash64().update(EMPTY_SLICE).hash(), expected); assertEquals(new XxHash64().update(EMPTY_SLICE, 0, 0).hash(), expected); assertEquals(new XxHash64().update(EMPTY_BYTES).hash(), expected); assertEquals(new XxHash64().update(EMPTY_BYTES, 0, 0).hash(), expected); assertEquals( new XxHash64() .update(EMPTY_BYTES) .update(EMPTY_BYTES, 0, 0) .update(EMPTY_SLICE) .update(EMPTY_SLICE, 0, 0) .hash(), expected); }
public void addPage(Page page) { requireNonNull(page, "page is null"); checkArgument(page.getChannelCount() == columnHashes.size(), "invalid page"); for (int channel = 0; channel < columnHashes.size(); channel++) { Type type = types.get(channel); Block block = page.getBlock(channel); XxHash64 xxHash64 = columnHashes.get(channel); for (int position = 0; position < block.getPositionCount(); position++) { long hash = hashPositionSkipNullMapKeys(type, block, position); longSlice.setLong(0, hash); xxHash64.update(longBuffer); } } totalRowCount += page.getPositionCount(); }
private WriteChecksumBuilder(List<Type> types) { this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); ImmutableList.Builder<XxHash64> columnHashes = ImmutableList.builder(); for (Type ignored : types) { columnHashes.add(new XxHash64()); } this.columnHashes = columnHashes.build(); }
private static int processRawHash(long rawHash) { // mix the bits so we don't use the same hash used to distribute between stages return (int) XxHash64.hash(Long.reverse(rawHash)); } }
private static void assertHash(long seed, Slice data, int length, long expected) throws IOException { assertEquals(hash(seed, data, 0, length), expected); assertEquals(hash(seed, data.slice(0, length)), expected); assertEquals(new XxHash64(seed).update(data.slice(0, length)).hash(), expected); assertEquals(new XxHash64(seed).update(data, 0, length).hash(), expected); assertEquals(new XxHash64(seed).update(data.getBytes(0, length)).hash(), expected); assertEquals(new XxHash64(seed).update(data.getBytes(), 0, length).hash(), expected); assertEquals(hash(seed, new ByteArrayInputStream(data.getBytes(0, length))), expected); for (int chunkSize = 1; chunkSize <= length; chunkSize++) { XxHash64 hash = new XxHash64(seed); for (int i = 0; i < length; i += chunkSize) { int updateSize = min(length - i, chunkSize); hash.update(data.slice(i, updateSize)); assertEquals(hash.hash(), hash(seed, data, 0, i + updateSize)); } assertEquals(hash.hash(), expected); } }
public void addPage(Page page) { requireNonNull(page, "page is null"); checkArgument(page.getChannelCount() == columnHashes.size(), "invalid page"); totalRowCount += page.getPositionCount(); for (int channel = 0; channel < columnHashes.size(); channel++) { Type type = types.get(channel); Block block = page.getBlock(channel); XxHash64 xxHash64 = columnHashes.get(channel); for (int position = 0; position < block.getPositionCount(); position++) { long hash = hashPositionSkipNullMapKeys(type, block, position); longSlice.setLong(0, hash); xxHash64.update(longBuffer); } } }