public HashPartition(int initialCapacity, float loadFactor, int wbSize, long maxProbeSize, boolean createHashMap, String spillLocalDirs) { if (createHashMap) { // Probe space should be at least equal to the size of our designated wbSize maxProbeSize = Math.max(maxProbeSize, wbSize); hashMap = new BytesBytesMultiHashMap(initialCapacity, loadFactor, wbSize, maxProbeSize); } else { hashMapSpilledOnCreation = true; hashMapOnDisk = true; } this.spillLocalDirs = spillLocalDirs; this.initialCapacity = initialCapacity; this.loadFactor = loadFactor; this.wbSize = wbSize; }
public HashPartition(int initialCapacity, float loadFactor, int wbSize, long maxProbeSize, boolean createHashMap, String spillLocalDirs) { if (createHashMap) { // Probe space should be at least equal to the size of our designated wbSize maxProbeSize = Math.max(maxProbeSize, wbSize); hashMap = new BytesBytesMultiHashMap(initialCapacity, loadFactor, wbSize, maxProbeSize); } else { hashMapSpilledOnCreation = true; hashMapOnDisk = true; } this.spillLocalDirs = spillLocalDirs; this.initialCapacity = initialCapacity; this.loadFactor = loadFactor; this.wbSize = wbSize; }
private MapJoinBytesTableContainer(float keyCountAdj, int threshold, float loadFactor, int wbSize, MapJoinObjectSerDeContext valCtx, long keyCount, long memUsage) throws SerDeException { int newThreshold = HashMapWrapper.calculateTableSize( keyCountAdj, threshold, loadFactor, keyCount); hashMap = new BytesBytesMultiHashMap(newThreshold, loadFactor, wbSize, memUsage); directWriteHelper = new DirectKeyValueWriter(); }
private MapJoinBytesTableContainer(float keyCountAdj, int threshold, float loadFactor, int wbSize, MapJoinObjectSerDeContext valCtx, long keyCount, long memUsage) throws SerDeException { int newThreshold = HashMapWrapper.calculateTableSize( keyCountAdj, threshold, loadFactor, keyCount); hashMap = new BytesBytesMultiHashMap(newThreshold, loadFactor, wbSize, memUsage); directWriteHelper = new DirectKeyValueWriter(); }
public BytesBytesMultiHashMap getHashMapFromDisk(int rowCount) throws IOException, ClassNotFoundException { if (hashMapSpilledOnCreation) { return new BytesBytesMultiHashMap(rowCount, loadFactor, wbSize, -1); } else { InputStream inputStream = Files.newInputStream(hashMapLocalPath); com.esotericsoftware.kryo.io.Input input = new com.esotericsoftware.kryo.io.Input(inputStream); Kryo kryo = SerializationUtilities.borrowKryo(); BytesBytesMultiHashMap restoredHashMap = null; try { restoredHashMap = kryo.readObject(input, BytesBytesMultiHashMap.class); } finally { SerializationUtilities.releaseKryo(kryo); } if (rowCount > 0) { restoredHashMap.expandAndRehashToTarget(rowCount); } // some bookkeeping rowsOnDisk = 0; hashMapOnDisk = false; input.close(); inputStream.close(); Files.delete(hashMapLocalPath); return restoredHashMap; } }
@Test public void testCapacityValidation() { BytesBytesMultiHashMap map = new BytesBytesMultiHashMap(CAPACITY, LOAD_FACTOR, WB_SIZE); assertEquals(CAPACITY, map.getCapacity()); map = new BytesBytesMultiHashMap(9, LOAD_FACTOR, WB_SIZE); assertEquals(16, map.getCapacity()); // Verify the scenario when maxProbeSize is a very small value, it doesn't fail BytesBytesMultiHashMap map1 = new BytesBytesMultiHashMap(1024, (float) 0.75, 524288, 1); }
public BytesBytesMultiHashMap getHashMapFromDisk(int rowCount) throws IOException, ClassNotFoundException { if (hashMapSpilledOnCreation) { return new BytesBytesMultiHashMap(rowCount, loadFactor, wbSize, -1); } else { InputStream inputStream = Files.newInputStream(hashMapLocalPath); com.esotericsoftware.kryo.io.Input input = new com.esotericsoftware.kryo.io.Input(inputStream); Kryo kryo = SerializationUtilities.borrowKryo(); BytesBytesMultiHashMap restoredHashMap = null; try { restoredHashMap = kryo.readObject(input, BytesBytesMultiHashMap.class); } finally { SerializationUtilities.releaseKryo(kryo); } if (rowCount > 0) { restoredHashMap.expandAndRehashToTarget(rowCount); } // some bookkeeping rowsOnDisk = 0; hashMapOnDisk = false; input.close(); inputStream.close(); Files.delete(hashMapLocalPath); return restoredHashMap; } }
@Test public void testExpand() throws Exception { // Start with capacity 1; make sure we expand on every put. BytesBytesMultiHashMap map = new BytesBytesMultiHashMap(1, 0.0000001f, WB_SIZE); UniqueKeysKvSource kv = new UniqueKeysKvSource(); for (int i = 0; i < 18; ++i) { map.put(kv, -1); for (int j = 0; j <= i; ++j) { verifyHashMapResult(map, kv.keys.get(j), kv.values.get(j)); } } assertEquals(1 << 18, map.getCapacity()); }
@Test public void testPutWithFullMap() throws Exception { // Make sure the map does not expand; should be able to find space. BytesBytesMultiHashMap map = new BytesBytesMultiHashMap(CAPACITY, 1f, WB_SIZE); UniqueKeysKvSource kv = new UniqueKeysKvSource(); for (int i = 0; i < CAPACITY; ++i) { map.put(kv, -1); } for (int i = 0; i < kv.keys.size(); ++i) { verifyHashMapResult(map, kv.keys.get(i), kv.values.get(i)); } assertEquals(CAPACITY, map.getCapacity()); // Get of non-existent key should terminate.. BytesBytesMultiHashMap.Result hashMapResult = new BytesBytesMultiHashMap.Result(); map.getValueResult(new byte[0], 0, 0, hashMapResult, null); }
@Test public void testPutGetMultiple() throws Exception { BytesBytesMultiHashMap map = new BytesBytesMultiHashMap(CAPACITY, LOAD_FACTOR, WB_SIZE); RandomKvSource kv = new RandomKvSource(0, 100); map.put(kv, -1); verifyHashMapResult(map, kv.getLastKey(), kv.getLastValue()); FixedKeyKvSource kv2 = new FixedKeyKvSource(kv.getLastKey(), 0, 100); kv2.values.add(kv.getLastValue()); for (int i = 0; i < 3; ++i) { map.put(kv2, -1); verifyHashMapResult(map, kv2.key, kv2.values.toArray(new byte[kv2.values.size()][])); } }
@Test public void testPutGetOne() throws Exception { BytesBytesMultiHashMap map = new BytesBytesMultiHashMap(CAPACITY, LOAD_FACTOR, WB_SIZE); RandomKvSource kv = new RandomKvSource(0, 0); map.put(kv, -1); verifyHashMapResult(map, kv.getLastKey(), kv.getLastValue()); kv = new RandomKvSource(10, 100); map.put(kv, -1); verifyHashMapResult(map, kv.getLastKey(), kv.getLastValue()); }
@Test public void testGetNonExistent() throws Exception { BytesBytesMultiHashMap map = new BytesBytesMultiHashMap(CAPACITY, LOAD_FACTOR, WB_SIZE); RandomKvSource kv = new RandomKvSource(1, 100); map.put(kv, -1); byte[] key = kv.getLastKey(); key[0] = (byte)(key[0] + 1); FixedKeyKvSource kv2 = new FixedKeyKvSource(kv.getLastKey(), 0, 100); map.put(kv2, -1); key[0] = (byte)(key[0] + 1); BytesBytesMultiHashMap.Result hashMapResult = new BytesBytesMultiHashMap.Result(); map.getValueResult(key, 0, key.length, hashMapResult, null); assertTrue(!hashMapResult.hasRows()); map.getValueResult(key, 0, 0, hashMapResult, null); assertTrue(!hashMapResult.hasRows()); }
public HashPartition(int threshold, float loadFactor, int wbSize, long memUsage, boolean createHashMap) { if (createHashMap) { hashMap = new BytesBytesMultiHashMap(threshold, loadFactor, wbSize, memUsage); } else { hashMapSpilledOnCreation = true; hashMapOnDisk = true; } this.threshold = threshold; this.loadFactor = loadFactor; this.wbSize = wbSize; }
private MapJoinBytesTableContainer(float keyCountAdj, int threshold, float loadFactor, int wbSize, MapJoinObjectSerDeContext valCtx, long keyCount, long memUsage) throws SerDeException { int newThreshold = HashMapWrapper.calculateTableSize( keyCountAdj, threshold, loadFactor, keyCount); hashMap = new BytesBytesMultiHashMap(newThreshold, loadFactor, wbSize, memUsage); directWriteHelper = new DirectKeyValueWriter(); }
public BytesBytesMultiHashMap getHashMapFromDisk(int initialCapacity) throws IOException, ClassNotFoundException { if (hashMapSpilledOnCreation) { return new BytesBytesMultiHashMap(Math.max(threshold, initialCapacity) , loadFactor, wbSize, -1); } else { InputStream inputStream = Files.newInputStream(hashMapLocalPath); com.esotericsoftware.kryo.io.Input input = new com.esotericsoftware.kryo.io.Input(inputStream); Kryo kryo = Utilities.runtimeSerializationKryo.get(); BytesBytesMultiHashMap restoredHashMap = kryo.readObject(input, BytesBytesMultiHashMap.class); if (initialCapacity > 0) { restoredHashMap.expandAndRehashToTarget(initialCapacity); } input.close(); inputStream.close(); Files.delete(hashMapLocalPath); return restoredHashMap; } }