private MapJoinBytesTableContainer(float keyCountAdj, int threshold, float loadFactor, int wbSize, MapJoinObjectSerDeContext valCtx, long keyCount, long memUsage) throws SerDeException { int newThreshold = HashMapWrapper.calculateTableSize( keyCountAdj, threshold, loadFactor, keyCount); hashMap = new BytesBytesMultiHashMap(newThreshold, loadFactor, wbSize, memUsage); directWriteHelper = new DirectKeyValueWriter(); }
public int size() { if (isHashMapOnDisk()) { // Rows are in a combination of the on-disk hashmap and the sidefile return rowsOnDisk + (sidefileKVContainer != null ? sidefileKVContainer.size() : 0); } else { // All rows should be in the in-memory hashmap return hashMap.size(); } } }
@Override public MapJoinKey putRow(Writable currentKey, Writable currentValue) throws SerDeException, HiveException { MapJoinKey key = MapJoinKey.read(output, keyContext, currentKey); FlatRowContainer values = (FlatRowContainer)get(key); if (values == null) { values = new FlatRowContainer(); put(key, values); } values.add(valueContext, (BytesWritable)currentValue); return key; }
/** * Postpone processing the big table row temporarily by spilling it to a row container * @param hybridHtContainer Hybrid hashtable container * @param row big table row */ protected void spillBigTableRow(MapJoinTableContainer hybridHtContainer, Object row) throws HiveException { HybridHashTableContainer ht = (HybridHashTableContainer) hybridHtContainer; int partitionId = ht.getToSpillPartitionId(); HashPartition hp = ht.getHashPartitions()[partitionId]; ObjectContainer bigTable = hp.getMatchfileObjContainer(); bigTable.add(row); }
@Override public Row next() throws HiveException { boolean endOfCurrBlock = endOfCurrentReadBlock(); if ( endOfCurrBlock ) { currentReadBlockStartRow += getCurrentReadBlockSize(); } return super.next(); }
@Before public void setup() throws Exception { rowContainer = new MapJoinEagerRowContainer(); rowContainer.addRow(VALUE); container = new HashMapWrapper(); } @Test
@Test public void testPutGetOne() throws Exception { BytesBytesMultiHashMap map = new BytesBytesMultiHashMap(CAPACITY, LOAD_FACTOR, WB_SIZE); RandomKvSource kv = new RandomKvSource(0, 0); map.put(kv, -1); verifyHashMapResult(map, kv.getLastKey(), kv.getLastValue()); kv = new RandomKvSource(10, 100); map.put(kv, -1); verifyHashMapResult(map, kv.getLastKey(), kv.getLastValue()); }
public JoinUtil.JoinResult setDirect(byte[] bytes, int offset, int length, BytesBytesMultiHashMap.Result hashMapResult, MatchTracker matchTracker) { aliasFilter = hashMap.getValueResult(bytes, offset, length, hashMapResult, matchTracker); dummyRow = null; if (hashMapResult.hasRows()) { return JoinUtil.JoinResult.MATCH; } else { aliasFilter = (byte) 0xff; return JoinUtil.JoinResult.NOMATCH; } } }
@Override public void put(Writable currentKey, Writable currentValue) throws SerDeException { directWriteHelper.setKeyValue(currentKey, currentValue); hashMap.put(directWriteHelper, -1); }
@SuppressWarnings("deprecation") @Override public MapJoinKey putRow(Writable currentKey, Writable currentValue) throws SerDeException { writeHelper.setKeyValue(currentKey, currentValue); hashMap.put(writeHelper, -1); return null; // there's no key to return }
@Override public List<Object> subList(int fromIndex, int toIndex) { checkSingleRow(); return new ReadOnlySubList(fromIndex, toIndex - fromIndex); }
@SuppressWarnings("deprecation") @Override public MapJoinKey putRow(Writable currentKey, Writable currentValue) throws SerDeException, HiveException, IOException { writeHelper.setKeyValue(currentKey, currentValue); return internalPutRow(writeHelper, currentKey, currentValue); }
@Override public void put(Writable currentKey, Writable currentValue) throws SerDeException, IOException { directWriteHelper.setKeyValue(currentKey, currentValue); internalPutRow(directWriteHelper, currentKey, currentValue); }
@Override public AbstractRowContainer.RowIterator<List<Object>> rowIter() throws HiveException { if (array.length == rowLength) { // optimize for common case - just one row for a key, container acts as iterator return this; } return rowLength > 0 ? new RowIterator() : new EmptyRowIterator(-rowLength); }
@Override public boolean isNext() { // If another non-matched key is found, the key bytes will be referenced by keyRef, and // our ReusableRowContainer's BytesBytesMultiHashMap.Result will reference the value rows. currentIndex = hashMap.findNextNonMatched( currentIndex, keyRef, currentValue.getHashMapResult(), matchTracker); return (currentIndex != -1); }
@Override public MatchTracker createMatchTracker() { return MatchTracker.create(hashMap.getNumHashBuckets()); }