public void loadNext() { packedRecordPointer.set(pointerArray.get(position)); position++; } }
public void loadNext() { packedRecordPointer.set(pointerArray.get(position)); position++; } }
@Override public PackedRecordPointer getKey(LongArray data, int pos, PackedRecordPointer reuse) { reuse.set(data.get(pos)); return reuse; }
@Override public PackedRecordPointer getKey(LongArray data, int pos, PackedRecordPointer reuse) { reuse.set(data.get(pos)); return reuse; }
public void loadNext() { packedRecordPointer.set(pointerArray.get(position)); position++; } }
@Override public PackedRecordPointer getKey(LongArray data, int pos, PackedRecordPointer reuse) { reuse.set(data.get(pos)); return reuse; }
@Test public void maximumPartitionIdCanBeEncoded() { PackedRecordPointer packedPointer = new PackedRecordPointer(); packedPointer.set(PackedRecordPointer.packPointer(0, MAXIMUM_PARTITION_ID)); assertEquals(MAXIMUM_PARTITION_ID, packedPointer.getPartitionId()); }
@Test public void partitionIdsGreaterThanMaximumPartitionIdWillOverflowOrTriggerError() { PackedRecordPointer packedPointer = new PackedRecordPointer(); try { // Pointers greater than the maximum partition ID will overflow or trigger an assertion error packedPointer.set(PackedRecordPointer.packPointer(0, MAXIMUM_PARTITION_ID + 1)); assertFalse(MAXIMUM_PARTITION_ID + 1 == packedPointer.getPartitionId()); } catch (AssertionError e ) { // pass } }
@Test public void partitionIdsGreaterThanMaximumPartitionIdWillOverflowOrTriggerError() { PackedRecordPointer packedPointer = new PackedRecordPointer(); try { // Pointers greater than the maximum partition ID will overflow or trigger an assertion error packedPointer.set(PackedRecordPointer.packPointer(0, MAXIMUM_PARTITION_ID + 1)); assertFalse(MAXIMUM_PARTITION_ID + 1 == packedPointer.getPartitionId()); } catch (AssertionError e ) { // pass } }
@Test public void partitionIdsGreaterThanMaximumPartitionIdWillOverflowOrTriggerError() { PackedRecordPointer packedPointer = new PackedRecordPointer(); try { // Pointers greater than the maximum partition ID will overflow or trigger an assertion error packedPointer.set(PackedRecordPointer.packPointer(0, MAXIMUM_PARTITION_ID + 1)); assertFalse(MAXIMUM_PARTITION_ID + 1 == packedPointer.getPartitionId()); } catch (AssertionError e ) { // pass } }
@Test public void maximumPartitionIdCanBeEncoded() { PackedRecordPointer packedPointer = new PackedRecordPointer(); packedPointer.set(PackedRecordPointer.packPointer(0, MAXIMUM_PARTITION_ID)); assertEquals(MAXIMUM_PARTITION_ID, packedPointer.getPartitionId()); }
@Test public void maximumPartitionIdCanBeEncoded() { PackedRecordPointer packedPointer = new PackedRecordPointer(); packedPointer.set(PackedRecordPointer.packPointer(0, MAXIMUM_PARTITION_ID)); assertEquals(MAXIMUM_PARTITION_ID, packedPointer.getPartitionId()); }
@Test public void maximumOffsetInPageCanBeEncoded() { PackedRecordPointer packedPointer = new PackedRecordPointer(); long address = TaskMemoryManager.encodePageNumberAndOffset(0, MAXIMUM_PAGE_SIZE_BYTES - 1); packedPointer.set(PackedRecordPointer.packPointer(address, 0)); assertEquals(address, packedPointer.getRecordPointer()); }
@Test public void offsetsPastMaxOffsetInPageWillOverflow() { PackedRecordPointer packedPointer = new PackedRecordPointer(); long address = TaskMemoryManager.encodePageNumberAndOffset(0, MAXIMUM_PAGE_SIZE_BYTES); packedPointer.set(PackedRecordPointer.packPointer(address, 0)); assertEquals(0, packedPointer.getRecordPointer()); } }
@Test public void offsetsPastMaxOffsetInPageWillOverflow() { PackedRecordPointer packedPointer = new PackedRecordPointer(); long address = TaskMemoryManager.encodePageNumberAndOffset(0, MAXIMUM_PAGE_SIZE_BYTES); packedPointer.set(PackedRecordPointer.packPointer(address, 0)); assertEquals(0, packedPointer.getRecordPointer()); } }
@Test public void maximumOffsetInPageCanBeEncoded() { PackedRecordPointer packedPointer = new PackedRecordPointer(); long address = TaskMemoryManager.encodePageNumberAndOffset(0, MAXIMUM_PAGE_SIZE_BYTES - 1); packedPointer.set(PackedRecordPointer.packPointer(address, 0)); assertEquals(address, packedPointer.getRecordPointer()); }
@Test public void maximumOffsetInPageCanBeEncoded() { PackedRecordPointer packedPointer = new PackedRecordPointer(); long address = TaskMemoryManager.encodePageNumberAndOffset(0, MAXIMUM_PAGE_SIZE_BYTES - 1); packedPointer.set(PackedRecordPointer.packPointer(address, 0)); assertEquals(address, packedPointer.getRecordPointer()); }
@Test public void offsetsPastMaxOffsetInPageWillOverflow() { PackedRecordPointer packedPointer = new PackedRecordPointer(); long address = TaskMemoryManager.encodePageNumberAndOffset(0, MAXIMUM_PAGE_SIZE_BYTES); packedPointer.set(PackedRecordPointer.packPointer(address, 0)); assertEquals(0, packedPointer.getRecordPointer()); } }
@Test public void heap() throws IOException { final SparkConf conf = new SparkConf().set("spark.memory.offHeap.enabled", "false"); final TaskMemoryManager memoryManager = new TaskMemoryManager(new TestMemoryManager(conf), 0); final MemoryConsumer c = new TestMemoryConsumer(memoryManager, MemoryMode.ON_HEAP); final MemoryBlock page0 = memoryManager.allocatePage(128, c); final MemoryBlock page1 = memoryManager.allocatePage(128, c); final long addressInPage1 = memoryManager.encodePageNumberAndOffset(page1, page1.getBaseOffset() + 42); PackedRecordPointer packedPointer = new PackedRecordPointer(); packedPointer.set(PackedRecordPointer.packPointer(addressInPage1, 360)); assertEquals(360, packedPointer.getPartitionId()); final long recordPointer = packedPointer.getRecordPointer(); assertEquals(1, TaskMemoryManager.decodePageNumber(recordPointer)); assertEquals(page1.getBaseOffset() + 42, memoryManager.getOffsetInPage(recordPointer)); assertEquals(addressInPage1, recordPointer); memoryManager.cleanUpAllAllocatedMemory(); }
@Test public void heap() throws IOException { final SparkConf conf = new SparkConf().set("spark.memory.offHeap.enabled", "false"); final TaskMemoryManager memoryManager = new TaskMemoryManager(new TestMemoryManager(conf), 0); final MemoryConsumer c = new TestMemoryConsumer(memoryManager, MemoryMode.ON_HEAP); final MemoryBlock page0 = memoryManager.allocatePage(128, c); final MemoryBlock page1 = memoryManager.allocatePage(128, c); final long addressInPage1 = memoryManager.encodePageNumberAndOffset(page1, page1.getBaseOffset() + 42); PackedRecordPointer packedPointer = new PackedRecordPointer(); packedPointer.set(PackedRecordPointer.packPointer(addressInPage1, 360)); assertEquals(360, packedPointer.getPartitionId()); final long recordPointer = packedPointer.getRecordPointer(); assertEquals(1, TaskMemoryManager.decodePageNumber(recordPointer)); assertEquals(page1.getBaseOffset() + 42, memoryManager.getOffsetInPage(recordPointer)); assertEquals(addressInPage1, recordPointer); memoryManager.cleanUpAllAllocatedMemory(); }