@Override public UploadPartResult uploadPart(String key, String uploadId, int partNumber, File inputFile, long length) throws IOException { final UploadPartRequest uploadRequest = s3accessHelper.newUploadPartRequest( key, uploadId, partNumber, MathUtils.checkedDownCast(length), null, inputFile, 0L); return s3accessHelper.uploadPart(uploadRequest); }
/** * Creates a new table with a capacity tailored to the given expected number of elements. * * @param expectedNumberOfElements The number of elements to tailor the capacity to. */ public KeyMap(int expectedNumberOfElements) { if (expectedNumberOfElements < 0) { throw new IllegalArgumentException("Invalid capacity: " + expectedNumberOfElements); } // round up to the next power or two // guard against too small capacity and integer overflows int capacity = Integer.highestOneBit(expectedNumberOfElements) << 1; capacity = capacity >= 0 ? Math.max(MIN_CAPACITY, capacity) : MAX_CAPACITY; // this also acts as a sanity check log2size = MathUtils.log2strict(capacity); shift = FULL_BIT_RANGE - log2size; table = allocateTable(capacity); rehashThreshold = getRehashThreshold(capacity); }
public XORShiftRandom(long input) { super(input); this.seed = MathUtils.murmurHash((int) input) ^ MathUtils.murmurHash((int) (input >>> 32)); }
"Number of task slots must be at least one."); final int pageSize = checkedDownCast(MemorySize.parse(configuration.getString(TaskManagerOptions.MEMORY_SEGMENT_SIZE)).getBytes()); checkConfigParameter(MathUtils.isPowerOf2(pageSize), pageSize, TaskManagerOptions.MEMORY_SEGMENT_SIZE.key(), "Memory segment size must be a power of 2.");
@Test public void testRoundDownToPowerOf2() { assertEquals(0, MathUtils.roundDownToPowerOf2(0)); assertEquals(1, MathUtils.roundDownToPowerOf2(1)); assertEquals(2, MathUtils.roundDownToPowerOf2(2)); assertEquals(2, MathUtils.roundDownToPowerOf2(3)); assertEquals(4, MathUtils.roundDownToPowerOf2(4)); assertEquals(4, MathUtils.roundDownToPowerOf2(5)); assertEquals(4, MathUtils.roundDownToPowerOf2(6)); assertEquals(4, MathUtils.roundDownToPowerOf2(7)); assertEquals(8, MathUtils.roundDownToPowerOf2(8)); assertEquals(8, MathUtils.roundDownToPowerOf2(9)); assertEquals(8, MathUtils.roundDownToPowerOf2(15)); assertEquals(16, MathUtils.roundDownToPowerOf2(16)); assertEquals(16, MathUtils.roundDownToPowerOf2(17)); assertEquals(16, MathUtils.roundDownToPowerOf2(31)); assertEquals(32, MathUtils.roundDownToPowerOf2(32)); assertEquals(32, MathUtils.roundDownToPowerOf2(33)); assertEquals(32, MathUtils.roundDownToPowerOf2(42)); assertEquals(32, MathUtils.roundDownToPowerOf2(63)); assertEquals(64, MathUtils.roundDownToPowerOf2(64)); assertEquals(64, MathUtils.roundDownToPowerOf2(125)); assertEquals(16384, MathUtils.roundDownToPowerOf2(25654)); assertEquals(33554432, MathUtils.roundDownToPowerOf2(34366363)); assertEquals(33554432, MathUtils.roundDownToPowerOf2(63463463)); assertEquals(1073741824, MathUtils.roundDownToPowerOf2(1852987883)); assertEquals(1073741824, MathUtils.roundDownToPowerOf2(Integer.MAX_VALUE)); }
@Test public void testRoundUpToPowerOf2() { assertEquals(0, MathUtils.roundUpToPowerOfTwo(0)); assertEquals(1, MathUtils.roundUpToPowerOfTwo(1)); assertEquals(2, MathUtils.roundUpToPowerOfTwo(2)); assertEquals(4, MathUtils.roundUpToPowerOfTwo(3)); assertEquals(4, MathUtils.roundUpToPowerOfTwo(4)); assertEquals(8, MathUtils.roundUpToPowerOfTwo(5)); assertEquals(8, MathUtils.roundUpToPowerOfTwo(6)); assertEquals(8, MathUtils.roundUpToPowerOfTwo(7)); assertEquals(8, MathUtils.roundUpToPowerOfTwo(8)); assertEquals(16, MathUtils.roundUpToPowerOfTwo(9)); assertEquals(16, MathUtils.roundUpToPowerOfTwo(15)); assertEquals(16, MathUtils.roundUpToPowerOfTwo(16)); assertEquals(32, MathUtils.roundUpToPowerOfTwo(17)); assertEquals(32, MathUtils.roundUpToPowerOfTwo(31)); assertEquals(32, MathUtils.roundUpToPowerOfTwo(32)); assertEquals(64, MathUtils.roundUpToPowerOfTwo(33)); assertEquals(64, MathUtils.roundUpToPowerOfTwo(42)); assertEquals(64, MathUtils.roundUpToPowerOfTwo(63)); assertEquals(64, MathUtils.roundUpToPowerOfTwo(64)); assertEquals(128, MathUtils.roundUpToPowerOfTwo(125)); assertEquals(32768, MathUtils.roundUpToPowerOfTwo(25654)); assertEquals(67108864, MathUtils.roundUpToPowerOfTwo(34366363)); assertEquals(67108864, MathUtils.roundUpToPowerOfTwo(67108863)); assertEquals(67108864, MathUtils.roundUpToPowerOfTwo(67108864)); assertEquals(0x40000000, MathUtils.roundUpToPowerOfTwo(0x3FFFFFFE)); assertEquals(0x40000000, MathUtils.roundUpToPowerOfTwo(0x3FFFFFFF)); assertEquals(0x40000000, MathUtils.roundUpToPowerOfTwo(0x40000000)); }
checkArgument(MathUtils.isPowerOf2(segmentSize)); this.segmentSizeBits = MathUtils.log2strict(segmentSize); this.segmentSizeMask = segmentSize - 1;
@Test public void testPowerOfTwo() { assertTrue(MathUtils.isPowerOf2(1)); assertTrue(MathUtils.isPowerOf2(2)); assertTrue(MathUtils.isPowerOf2(4)); assertTrue(MathUtils.isPowerOf2(8)); assertTrue(MathUtils.isPowerOf2(32768)); assertTrue(MathUtils.isPowerOf2(65536)); assertTrue(MathUtils.isPowerOf2(1 << 30)); assertTrue(MathUtils.isPowerOf2(1L + Integer.MAX_VALUE)); assertTrue(MathUtils.isPowerOf2(1L << 41)); assertTrue(MathUtils.isPowerOf2(1L << 62)); assertFalse(MathUtils.isPowerOf2(3)); assertFalse(MathUtils.isPowerOf2(5)); assertFalse(MathUtils.isPowerOf2(567923)); assertFalse(MathUtils.isPowerOf2(Integer.MAX_VALUE)); assertFalse(MathUtils.isPowerOf2(Long.MAX_VALUE)); }
/** * This function hashes an integer value. * * <p>It is crucial to use different hash functions to partition data across machines and the internal partitioning of * data structures. This hash function is intended for partitioning across machines. * * @param code The integer to be hashed. * @return The non-negative hash code for the integer. */ public static int murmurHash(int code) { code *= 0xcc9e2d51; code = Integer.rotateLeft(code, 15); code *= 0x1b873593; code = Integer.rotateLeft(code, 13); code = code * 5 + 0xe6546b64; code ^= 4; code = bitMix(code); if (code >= 0) { return code; } else if (code != Integer.MIN_VALUE) { return -code; } else { return 0; } }
/** * The level parameter is needed so that we can have different hash functions when we recursively apply * the partitioning, so that the working set eventually fits into memory. */ public static int hash(int code, int level) { final int rotation = level * 11; code = Integer.rotateLeft(code, rotation); return MathUtils.jenkinsHash(code); }
@Override public int hashCode() { return MathUtils.longToIntWithBitMixing(start + end); }
"Number of task slots must be at least one."); final int pageSize = checkedDownCast(MemorySize.parse(configuration.getString(TaskManagerOptions.MEMORY_SEGMENT_SIZE)).getBytes()); checkConfigParameter(MathUtils.isPowerOf2(pageSize), pageSize, TaskManagerOptions.MEMORY_SEGMENT_SIZE.key(), "Memory segment size must be a power of 2.");
private static int getBucketBuffersByRowCount(long rowCount, int maxSegs, int segmentSize) { int minNumBuckets = (int) Math.ceil((rowCount / 0.5)); Preconditions.checkArgument(segmentSize % 16 == 0); return MathUtils.roundDownToPowerOf2((int) Math.max(1, Math.min(maxSegs, Math.ceil(((double) minNumBuckets) * 16 / segmentSize)))); }
/** * Computes a default maximum parallelism from the operator parallelism. This is used in case the user has not * explicitly configured a maximum parallelism to still allow a certain degree of scale-up. * * @param operatorParallelism the operator parallelism as basis for computation. * @return the computed default maximum parallelism. */ public static int computeDefaultMaxParallelism(int operatorParallelism) { checkParallelismPreconditions(operatorParallelism); return Math.min( Math.max( MathUtils.roundUpToPowerOfTwo(operatorParallelism + (operatorParallelism / 2)), DEFAULT_LOWER_BOUND_MAX_PARALLELISM), UPPER_BOUND_MAX_PARALLELISM); }
private void setNewBuckets(MemorySegment[] buckets, int numBuckets, int threshold) { this.buckets = buckets; checkArgument(MathUtils.isPowerOf2(numBuckets)); this.numBuckets = numBuckets; this.numBucketsMask = numBuckets - 1; this.overflowSegments = new MemorySegment[2]; this.numOverflowSegments = 0; this.nextOverflowBucket = 0; this.threshold = threshold; }
/** * Helper function that creates and scrambles a composite hash for key and namespace. */ private static int compositeHash(Object key, Object namespace) { // create composite key through XOR, then apply some bit-mixing for better distribution of skewed keys. return MathUtils.bitMix(key.hashCode() ^ namespace.hashCode()); }
/** * The level parameter is needed so that we can have different hash functions when we recursively apply * the partitioning, so that the working set eventually fits into memory. */ public static int hash(int code, int level) { final int rotation = level * 11; code = Integer.rotateLeft(code, rotation); return MathUtils.jenkinsHash(code); }
@Override public int hashCode() { return MathUtils.longToIntWithBitMixing(start + end); }
@Override public IntValue translate(LongValue value, IntValue reuse) throws Exception { if (reuse == null) { reuse = new IntValue(); } reuse.setValue(MathUtils.checkedDownCast(value.getValue())); return reuse; } }
@Override public Tuple2<Integer, Integer> map(Integer value) throws Exception { if (key == -1){ key = MathUtils.murmurHash(value) % numKeys; } return new Tuple2<>(key, value); } }).split(new OutputSelector<Tuple2<Integer, Integer>>() {