@Ignore @Test public void testOutOfBounds() throws Exception { random = new Random(42662); int HIGHEST_INT_POWER_OF_2 = 1073741824; boolean error = false; try { // The c'tor should throw the error VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, HIGHEST_INT_POWER_OF_2, LOAD_FACTOR, MODERATE_WB_SIZE, -1); } catch (MapJoinMemoryExhaustionError e) { error = true; } assert error; }
@VisibleForTesting public void testPutRow(byte[] currentKey, byte[] currentValue) throws HiveException, IOException { if (testKeyBytesWritable == null) { testKeyBytesWritable = new BytesWritable(); testValueBytesWritable = new BytesWritable(); } testKeyBytesWritable.set(currentKey, 0, currentKey.length); testValueBytesWritable.set(currentValue, 0, currentValue.length); putRow(testKeyBytesWritable, testValueBytesWritable); }
@Test public void testGetNonExistent() throws Exception { random = new Random(1002); VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); byte[] key1 = new byte[random.nextInt(MAX_KEY_LENGTH)]; random.nextBytes(key1); byte[] value = new byte[random.nextInt(MAX_VALUE_LENGTH)]; random.nextBytes(value); map.testPutRow(key1, value); verifyTable.add(key1, value); verifyTable.verify(map); byte[] key2 = new byte[random.nextInt(MAX_KEY_LENGTH)]; random.nextBytes(key2); VectorMapJoinHashMapResult hashMapResult = map.createHashMapResult(); JoinUtil.JoinResult joinResult = map.lookup(key2, 0, key2.length, hashMapResult); assertTrue(joinResult == JoinUtil.JoinResult.NOMATCH); assertTrue(!hashMapResult.hasRows()); map.testPutRow(key2, value); verifyTable.add(key2, value); verifyTable.verify(map); byte[] key3 = new byte[random.nextInt(MAX_KEY_LENGTH)]; random.nextBytes(key3); hashMapResult = map.createHashMapResult(); joinResult = map.lookup(key3, 0, key3.length, hashMapResult); assertTrue(joinResult == JoinUtil.JoinResult.NOMATCH); assertTrue(!hashMapResult.hasRows()); }
@Test public void testOneKey() throws Exception { random = new Random(82733); VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); byte[] key = new byte[random.nextInt(MAX_KEY_LENGTH)]; random.nextBytes(key); byte[] value = new byte[random.nextInt(MAX_VALUE_LENGTH)]; random.nextBytes(value); map.testPutRow(key, value); verifyTable.add(key, value); // Second value. value = new byte[random.nextInt(MAX_VALUE_LENGTH)]; random.nextBytes(value); map.testPutRow(key, value); verifyTable.add(key, value); // Third value. value = new byte[random.nextInt(MAX_VALUE_LENGTH)]; random.nextBytes(value); map.testPutRow(key, value); verifyTable.add(key, value); verifyTable.verifyNonMatched(map, random); }
hashMapResult = stringHashMap.createHashMapResult(); joinResult = stringHashMap.lookup(keyBytes, 0, keyBytes.length, hashMapResult); if (joinResult != JoinUtil.JoinResult.MATCH) { assertTrue(false);
map.testPutRow(key, value); verifyTable.add(key, value); } else { byte[] randomExistingKey = verifyTable.addRandomExisting(value, random); map.testPutRow(randomExistingKey, value);
@Test public void testMultipleKeysSingleValue() throws Exception { random = new Random(29383); VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); int keyCount = 100 + random.nextInt(1000); for (int i = 0; i < keyCount; i++) { byte[] key = new byte[random.nextInt(MAX_KEY_LENGTH)]; random.nextBytes(key); if (!verifyTable.contains(key)) { // Unique keys for this test. break; } byte[] value = new byte[random.nextInt(MAX_VALUE_LENGTH)]; random.nextBytes(value); map.testPutRow(key, value); verifyTable.add(key, value); verifyTable.verify(map); } }
map.testPutRow(key, value); verifyTable.add(key, value); map.testPutRow(randomExistingKey, value);
@Ignore @Test public void testOutOfBounds() throws Exception { random = new Random(42662); int HIGHEST_INT_POWER_OF_2 = 1073741824; boolean error = false; try { // The c'tor should throw the error VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, HIGHEST_INT_POWER_OF_2, LOAD_FACTOR, MODERATE_WB_SIZE, -1); } catch (MapJoinMemoryExhaustionError e) { error = true; } assert error; }
@Test public void testMultipleKeysSingleValue() throws Exception { random = new Random(29383); VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); int keyCount = 100 + random.nextInt(1000); for (int i = 0; i < keyCount; i++) { byte[] key = new byte[random.nextInt(MAX_KEY_LENGTH)]; random.nextBytes(key); if (!verifyTable.contains(key)) { // Unique keys for this test. break; } byte[] value = new byte[random.nextInt(MAX_VALUE_LENGTH)]; random.nextBytes(value); map.testPutRow(key, value); verifyTable.add(key, value); } verifyTable.verifyNonMatched(map, random); }
new VectorMapJoinFastMultiKeyHashMap(false,CAPACITY, 1f, WB_SIZE, -1); random.nextBytes(value); map.testPutRow(key, value); verifyTable.add(key, value); VectorMapJoinHashMapResult hashMapResult = map.createHashMapResult(); JoinUtil.JoinResult joinResult = map.lookup(anotherKey, 0, anotherKey.length, hashMapResult); assertTrue(joinResult == JoinUtil.JoinResult.NOMATCH);
@VisibleForTesting public void testPutRow(byte[] currentKey, byte[] currentValue) throws HiveException, IOException { if (testKeyBytesWritable == null) { testKeyBytesWritable = new BytesWritable(); testValueBytesWritable = new BytesWritable(); } testKeyBytesWritable.set(currentKey, 0, currentKey.length); testValueBytesWritable.set(currentValue, 0, currentValue.length); putRow(testKeyBytesWritable, testValueBytesWritable); }
@Test public void testLargeAndExpand() throws Exception { random = new Random(21111); // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false,MODERATE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); int keyCount = 1000; addAndVerifyMultipleKeyMultipleValue(keyCount, map, verifyTable); }
@Test public void testOneKey() throws Exception { random = new Random(82733); VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); byte[] key = new byte[random.nextInt(MAX_KEY_LENGTH)]; random.nextBytes(key); byte[] value = new byte[random.nextInt(MAX_VALUE_LENGTH)]; random.nextBytes(value); map.testPutRow(key, value); verifyTable.add(key, value); verifyTable.verify(map); // Second value. value = new byte[random.nextInt(MAX_VALUE_LENGTH)]; random.nextBytes(value); map.testPutRow(key, value); verifyTable.add(key, value); verifyTable.verify(map); // Third value. value = new byte[random.nextInt(MAX_VALUE_LENGTH)]; random.nextBytes(value); map.testPutRow(key, value); verifyTable.add(key, value); verifyTable.verify(map); }
@Test public void testMultipleKeysMultipleValue() throws Exception { random = new Random(9332); // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false,LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); int keyCount = 1000; addAndVerifyMultipleKeyMultipleValue(keyCount, map, verifyTable); }
@Test public void testExpand() throws Exception { random = new Random(99221); // Start with capacity 1; make sure we expand on every put. VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap(false,1, 0.0000001f, WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); for (int i = 0; i < 18; ++i) { byte[] key; while (true) { key = new byte[random.nextInt(MAX_KEY_LENGTH)]; random.nextBytes(key); if (!verifyTable.contains(key)) { // Unique keys for this test. break; } } byte[] value = new byte[random.nextInt(MAX_VALUE_LENGTH)]; random.nextBytes(value); map.testPutRow(key, value); verifyTable.add(key, value); // verifyTable.verify(map); } verifyTable.verify(map); // assertEquals(1 << 18, map.getCapacity()); }
@Test public void testReallyBig() throws Exception { random = new Random(42662); // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, LARGE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); int keyCount = 100; addAndVerifyMultipleKeyMultipleValue(keyCount, map, verifyTable); } }
@Test public void testReallyBig() throws Exception { random = new Random(42662); // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false,LARGE_CAPACITY, LOAD_FACTOR, MODERATE_WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); int keyCount = 1000000; addAndVerifyMultipleKeyMultipleValue(keyCount, map, verifyTable); }
@Test public void testMultipleKeysMultipleValue() throws Exception { random = new Random(9332); // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false,LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastBytesHashMap verifyTable = new VerifyFastBytesHashMap(); int keyCount = 100; addAndVerifyMultipleKeyMultipleValue(keyCount, map, verifyTable); }
@Test public void testMultiKeyRowsClipped2() throws Exception { random = new Random(7403); // Use a large capacity that doesn't require expansion, yet. VectorMapJoinFastMultiKeyHashMap map = new VectorMapJoinFastMultiKeyHashMap( false, LARGE_CAPACITY, LOAD_FACTOR, LARGE_WB_SIZE, -1); VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap(); VectorRandomRowSource valueSource = new VectorRandomRowSource(); valueSource.init( random, VectorRandomRowSource.SupportedTypes.ALL, 4, /* allowNulls */ false, /* isUnicodeOk */ false); int rowCount = 1000; Object[][] rows = valueSource.randomRows(rowCount); addAndVerifyRows(valueSource, rows, map, HashTableKeyType.MULTI_KEY, verifyTable, new String[] { "varchar(20)", "varchar(40)" }, /* doClipping */ true, /* useExactBytes */ false); }