if (NativeCrc32.isAvailable()) { NativeCrc32.calculateChunkedSums(bytesPerChecksum, type.id, checksums, data); return;
/** * Implementation of chunked calculation specifically on byte arrays. This * is to avoid the copy when dealing with ByteBuffers that have array backing. */ public void calculateChunkedSums( byte[] data, int dataOffset, int dataLength, byte[] sums, int sumsOffset) { if (type.size == 0) return; if (NativeCrc32.isAvailable()) { NativeCrc32.calculateChunkedSumsByteArray(bytesPerChecksum, type.id, sums, sumsOffset, data, dataOffset, dataLength); return; } int remaining = dataLength; while (remaining > 0) { int n = Math.min(remaining, bytesPerChecksum); summer.reset(); summer.update(data, dataOffset, n); dataOffset += n; remaining -= n; long calculated = summer.getValue(); sums[sumsOffset++] = (byte) (calculated >> 24); sums[sumsOffset++] = (byte) (calculated >> 16); sums[sumsOffset++] = (byte) (calculated >> 8); sums[sumsOffset++] = (byte) (calculated); } }
final int crcsOffset = checksums.arrayOffset() + checksums.position(); if (NativeCrc32.isAvailable()) { NativeCrc32.verifyChunkedSumsByteArray(bytesPerChecksum, type.id, checksums.array(), crcsOffset, data.array(), dataOffset, data.remaining(), fileName, basePos); if (NativeCrc32.isAvailable() && data.isDirect()) { NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data, fileName, basePos); } else {
if (type.size == 0) return; if (NativeCrc32.isAvailable()) { NativeCrc32.verifyChunkedSumsByteArray(bytesPerChecksum, type.id, checksums, checksumsOff, data, dataOff, dataLen, fileName, basePos); return;
return; if (NativeCrc32.isAvailable()) { NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data, fileName, basePos); return;
public static void calculateChunkedSums(int bytesPerSum, int checksumType, ByteBuffer sums, ByteBuffer data) { nativeComputeChunkedSums(bytesPerSum, checksumType, sums, sums.position(), data, data.position(), data.remaining(), "", 0, false); }
public static void calculateChunkedSumsByteArray(int bytesPerSum, int checksumType, byte[] sums, int sumsOffset, byte[] data, int dataOffset, int dataLength) { nativeComputeChunkedSumsByteArray(bytesPerSum, checksumType, sums, sumsOffset, data, dataOffset, dataLength, "", 0, false); }
@Test public void testCalculateChunkedSumsByteArraySuccess() throws ChecksumException { allocateArrayByteBuffers(); fillDataAndValidChecksums(); NativeCrc32.calculateChunkedSumsByteArray(bytesPerChecksum, checksumType.id, checksums.array(), checksums.position(), data.array(), data.position(), data.remaining()); }
@Test public void testVerifyChunkedSumsByteArraySuccess() throws ChecksumException { allocateArrayByteBuffers(); fillDataAndValidChecksums(); NativeCrc32.verifyChunkedSumsByteArray(bytesPerChecksum, checksumType.id, checksums.array(), checksums.position(), data.array(), data.position(), data.remaining(), fileName, BASE_POSITION); }
@Test public void testVerifyChunkedSumsSuccess() throws ChecksumException { allocateDirectByteBuffers(); fillDataAndValidChecksums(); NativeCrc32.verifyChunkedSums(bytesPerChecksum, checksumType.id, checksums, data, fileName, BASE_POSITION); }
@Test public void testCalculateChunkedSumsFail() throws ChecksumException { allocateDirectByteBuffers(); fillDataAndInvalidChecksums(); NativeCrc32.calculateChunkedSums(bytesPerChecksum, checksumType.id, checksums, data); }
@Before public void setup() { assumeTrue(NativeCrc32.isAvailable()); assertEquals( "These tests assume they can write a checksum value as a 4-byte int.", 4, checksumType.size); Configuration conf = new Configuration(); bytesPerChecksum = conf.getInt(IO_BYTES_PER_CHECKSUM_KEY, IO_BYTES_PER_CHECKSUM_DEFAULT); fileName = this.getClass().getSimpleName(); checksum = DataChecksum.newDataChecksum(checksumType, bytesPerChecksum); }
/** * Verify the given buffers of data and checksums, and throw an exception * if any checksum is invalid. The buffers given to this function should * have their position initially at the start of the data, and their limit * set at the end of the data. The position, limit, and mark are not * modified. * * @param bytesPerSum the chunk size (eg 512 bytes) * @param checksumType the DataChecksum type constant * @param sums the DirectByteBuffer pointing at the beginning of the * stored checksums * @param data the DirectByteBuffer pointing at the beginning of the * data to check * @param basePos the position in the file where the data buffer starts * @param fileName the name of the file being verified * @throws ChecksumException if there is an invalid checksum */ public static void verifyChunkedSums(int bytesPerSum, int checksumType, ByteBuffer sums, ByteBuffer data, String fileName, long basePos) throws ChecksumException { nativeVerifyChunkedSums(bytesPerSum, checksumType, sums, sums.position(), data, data.position(), data.remaining(), fileName, basePos); }
if (type.size == 0) return; if (NativeCrc32.isAvailable()) { NativeCrc32.verifyChunkedSumsByteArray(bytesPerChecksum, type.id, checksums, checksumsOff, data, dataOff, dataLen, fileName, basePos); return;
return; if (NativeCrc32.isAvailable()) { NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data, fileName, basePos); return;
/** * Verify the given buffers of data and checksums, and throw an exception * if any checksum is invalid. The buffers given to this function should * have their position initially at the start of the data, and their limit * set at the end of the data. The position, limit, and mark are not * modified. * * @param bytesPerSum the chunk size (eg 512 bytes) * @param checksumType the DataChecksum type constant (NULL is not supported) * @param sums the DirectByteBuffer pointing at the beginning of the * stored checksums * @param data the DirectByteBuffer pointing at the beginning of the * data to check * @param basePos the position in the file where the data buffer starts * @param fileName the name of the file being verified * @throws ChecksumException if there is an invalid checksum */ public static void verifyChunkedSums(int bytesPerSum, int checksumType, ByteBuffer sums, ByteBuffer data, String fileName, long basePos) throws ChecksumException { nativeComputeChunkedSums(bytesPerSum, checksumType, sums, sums.position(), data, data.position(), data.remaining(), fileName, basePos, true); }
public static void verifyChunkedSumsByteArray(int bytesPerSum, int checksumType, byte[] sums, int sumsOffset, byte[] data, int dataOffset, int dataLength, String fileName, long basePos) throws ChecksumException { nativeComputeChunkedSumsByteArray(bytesPerSum, checksumType, sums, sumsOffset, data, dataOffset, dataLength, fileName, basePos, true); }
@Test public void testCalculateChunkedSumsByteArrayFail() throws ChecksumException { allocateArrayByteBuffers(); fillDataAndInvalidChecksums(); NativeCrc32.calculateChunkedSumsByteArray(bytesPerChecksum, checksumType.id, checksums.array(), checksums.position(), data.array(), data.position(), data.remaining()); }
@Test public void testVerifyChunkedSumsByteArraySuccess() throws ChecksumException { allocateArrayByteBuffers(); fillDataAndValidChecksums(); NativeCrc32.verifyChunkedSumsByteArray(bytesPerChecksum, checksumType.id, checksums.array(), checksums.position(), data.array(), data.position(), data.remaining(), fileName, BASE_POSITION); }
@Test public void testVerifyChunkedSumsSuccess() throws ChecksumException { allocateDirectByteBuffers(); fillDataAndValidChecksums(); NativeCrc32.verifyChunkedSums(bytesPerChecksum, checksumType.id, checksums, data, fileName, BASE_POSITION); }