long v1 = rand.nextLong(); long v2 = rand.nextLong(); appendRow(batch, makeKeyRow(k1, k2), makeValueRow(v1, v2)); expectedK1[i] = k1; expectedK2[i] = k2;
long v1 = rand.nextLong(); long v2 = rand.nextLong(); appendRow(batch, makeKeyRow(k1, k2), makeValueRow(v1, v2)); expectedK1[i] = k1; expectedK2[i] = k2;
@Test public void failureToAllocateFirstPage() throws Exception { memoryManager.limit(1024); RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { UnsafeRow key = makeKeyRow(1, "A"); UnsafeRow value = makeValueRow(11, 11); UnsafeRow ret = appendRow(batch, key, value); Assert.assertNull(ret); Assert.assertFalse(batch.rowIterator().next()); } finally { batch.close(); } }
@Test public void failureToAllocateFirstPage() throws Exception { memoryManager.limit(1024); RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { UnsafeRow key = makeKeyRow(1, "A"); UnsafeRow value = makeValueRow(11, 11); UnsafeRow ret = appendRow(batch, key, value); Assert.assertNull(ret); Assert.assertFalse(batch.rowIterator().next()); } finally { batch.close(); } }
@Test public void failureToAllocateFirstPage() throws Exception { memoryManager.limit(1024); RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { UnsafeRow key = makeKeyRow(1, "A"); UnsafeRow value = makeValueRow(11, 11); UnsafeRow ret = appendRow(batch, key, value); Assert.assertNull(ret); Assert.assertFalse(batch.rowIterator().next()); } finally { batch.close(); } }
@Test public void setUpdateAndRetrieve() { RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); Assert.assertEquals(1, batch.numRows()); UnsafeRow retrievedValue = batch.getValueRow(0); updateValueRow(retrievedValue, 2, 2); UnsafeRow retrievedValue2 = batch.getValueRow(0); Assert.assertTrue(checkValue(retrievedValue2, 2, 2)); } finally { batch.close(); } }
@Test public void setUpdateAndRetrieve() { RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); Assert.assertEquals(1, batch.numRows()); UnsafeRow retrievedValue = batch.getValueRow(0); updateValueRow(retrievedValue, 2, 2); UnsafeRow retrievedValue2 = batch.getValueRow(0); Assert.assertTrue(checkValue(retrievedValue2, 2, 2)); } finally { batch.close(); } }
@Test public void setUpdateAndRetrieve() { RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); Assert.assertEquals(1, batch.numRows()); UnsafeRow retrievedValue = batch.getValueRow(0); updateValueRow(retrievedValue, 2, 2); UnsafeRow retrievedValue2 = batch.getValueRow(0); Assert.assertTrue(checkValue(retrievedValue2, 2, 2)); } finally { batch.close(); } }
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { UnsafeRow ret1 = appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); Assert.assertTrue(checkValue(ret1, 1, 1)); UnsafeRow ret2 = appendRow(batch, makeKeyRow(2, "B"), makeValueRow(2, 2)); Assert.assertTrue(checkValue(ret2, 2, 2)); UnsafeRow ret3 = appendRow(batch, makeKeyRow(3, "C"), makeValueRow(3, 3)); Assert.assertTrue(checkValue(ret3, 3, 3)); Assert.assertEquals(3, batch.numRows());
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { UnsafeRow ret1 = appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); Assert.assertTrue(checkValue(ret1, 1, 1)); UnsafeRow ret2 = appendRow(batch, makeKeyRow(2, "B"), makeValueRow(2, 2)); Assert.assertTrue(checkValue(ret2, 2, 2)); UnsafeRow ret3 = appendRow(batch, makeKeyRow(3, "C"), makeValueRow(3, 3)); Assert.assertTrue(checkValue(ret3, 3, 3)); Assert.assertEquals(3, batch.numRows());
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); appendRow(batch, makeKeyRow(2, "B"), makeValueRow(2, 2)); appendRow(batch, makeKeyRow(3, "C"), makeValueRow(3, 3)); Assert.assertEquals(3, batch.numRows()); org.apache.spark.unsafe.KVIterator<UnsafeRow, UnsafeRow> iterator
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); appendRow(batch, makeKeyRow(2, "B"), makeValueRow(2, 2)); appendRow(batch, makeKeyRow(3, "C"), makeValueRow(3, 3)); Assert.assertEquals(3, batch.numRows()); org.apache.spark.unsafe.KVIterator<UnsafeRow, UnsafeRow> iterator
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(1, "A"), makeValueRow(1, 1)); appendRow(batch, makeKeyRow(2, "B"), makeValueRow(2, 2)); appendRow(batch, makeKeyRow(3, "C"), makeValueRow(3, 3)); Assert.assertEquals(3, batch.numRows()); org.apache.spark.unsafe.KVIterator<UnsafeRow, UnsafeRow> iterator
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(11, 11), makeValueRow(1, 1)); appendRow(batch, makeKeyRow(22, 22), makeValueRow(2, 2)); appendRow(batch, makeKeyRow(33, 33), makeValueRow(3, 3)); UnsafeRow retrievedKey1 = batch.getKeyRow(0); Assert.assertTrue(checkKey(retrievedKey1, 11, 11));
@Test public void appendRowUntilExceedingCapacity() throws Exception { RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, 10); try { UnsafeRow key = makeKeyRow(1, "A"); UnsafeRow value = makeValueRow(1, 1); for (int i = 0; i < 10; i++) { appendRow(batch, key, value); } UnsafeRow ret = appendRow(batch, key, value); Assert.assertEquals(batch.numRows(), 10); Assert.assertNull(ret); org.apache.spark.unsafe.KVIterator<UnsafeRow, UnsafeRow> iterator = batch.rowIterator(); for (int i = 0; i < 10; i++) { Assert.assertTrue(iterator.next()); UnsafeRow key1 = iterator.getKey(); UnsafeRow value1 = iterator.getValue(); Assert.assertTrue(checkKey(key1, 1, "A")); Assert.assertTrue(checkValue(value1, 1, 1)); } Assert.assertFalse(iterator.next()); } finally { batch.close(); } }
valueSchema, taskMemoryManager, DEFAULT_CAPACITY); try { appendRow(batch, makeKeyRow(11, 11), makeValueRow(1, 1)); appendRow(batch, makeKeyRow(22, 22), makeValueRow(2, 2)); appendRow(batch, makeKeyRow(33, 33), makeValueRow(3, 3)); UnsafeRow retrievedKey1 = batch.getKeyRow(0); Assert.assertTrue(checkKey(retrievedKey1, 11, 11));
@Test public void appendRowUntilExceedingCapacity() throws Exception { RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, 10); try { UnsafeRow key = makeKeyRow(1, "A"); UnsafeRow value = makeValueRow(1, 1); for (int i = 0; i < 10; i++) { appendRow(batch, key, value); } UnsafeRow ret = appendRow(batch, key, value); Assert.assertEquals(batch.numRows(), 10); Assert.assertNull(ret); org.apache.spark.unsafe.KVIterator<UnsafeRow, UnsafeRow> iterator = batch.rowIterator(); for (int i = 0; i < 10; i++) { Assert.assertTrue(iterator.next()); UnsafeRow key1 = iterator.getKey(); UnsafeRow value1 = iterator.getValue(); Assert.assertTrue(checkKey(key1, 1, "A")); Assert.assertTrue(checkValue(value1, 1, 1)); } Assert.assertFalse(iterator.next()); } finally { batch.close(); } }
@Test public void appendRowUntilExceedingCapacity() throws Exception { RowBasedKeyValueBatch batch = RowBasedKeyValueBatch.allocate(keySchema, valueSchema, taskMemoryManager, 10); try { UnsafeRow key = makeKeyRow(1, "A"); UnsafeRow value = makeValueRow(1, 1); for (int i = 0; i < 10; i++) { appendRow(batch, key, value); } UnsafeRow ret = appendRow(batch, key, value); Assert.assertEquals(batch.numRows(), 10); Assert.assertNull(ret); org.apache.spark.unsafe.KVIterator<UnsafeRow, UnsafeRow> iterator = batch.rowIterator(); for (int i = 0; i < 10; i++) { Assert.assertTrue(iterator.next()); UnsafeRow key1 = iterator.getKey(); UnsafeRow value1 = iterator.getValue(); Assert.assertTrue(checkKey(key1, 1, "A")); Assert.assertTrue(checkValue(value1, 1, 1)); } Assert.assertFalse(iterator.next()); } finally { batch.close(); } }
UnsafeRow key = makeKeyRow(1, "A"); UnsafeRow value = makeValueRow(1, 1); int recordLength = 8 + key.getSizeInBytes() + value.getSizeInBytes() + 8;
UnsafeRow key = makeKeyRow(1, "A"); UnsafeRow value = makeValueRow(1, 1); int recordLength = 8 + key.getSizeInBytes() + value.getSizeInBytes() + 8;