public static void persistDummyTable(ObjectOutputStream out) throws IOException { MapJoinPersistableTableContainer tableContainer = new HashMapWrapper(); out.writeUTF(tableContainer.getClass().getName()); out.writeObject(tableContainer.getMetaData()); out.writeInt(tableContainer.size()); }
@Override public MapJoinKey putRow(Writable currentKey, Writable currentValue) throws SerDeException, HiveException { MapJoinKey key = MapJoinKey.read(output, keyContext, currentKey); FlatRowContainer values = (FlatRowContainer)get(key); if (values == null) { values = new FlatRowContainer(); put(key, values); } values.add(valueContext, (BytesWritable)currentValue); return key; }
private HashMapWrapper(float keyCountAdj, int threshold, float loadFactor, long keyCount) { super(createConstructorMetaData(threshold, loadFactor)); threshold = calculateTableSize(keyCountAdj, threshold, loadFactor, keyCount); mHash = new HashMap<MapJoinKey, MapJoinRowContainer>(threshold, loadFactor); }
@Test public void testContainerBasics() throws Exception { container.put(KEY1, rowContainer); container.put(KEY2, rowContainer); container.put(KEY3, rowContainer); container.put(KEY4, rowContainer); Assert.assertEquals(4, container.size()); Map<MapJoinKey, MapJoinRowContainer> localContainer = new HashMap<MapJoinKey, MapJoinRowContainer>(); for(Entry<MapJoinKey, MapJoinRowContainer> entry : container.entrySet()) { localContainer.put(entry.getKey(), entry.getValue()); } Utilities.testEquality(container.get(KEY1), localContainer.get(KEY1)); Utilities.testEquality(container.get(KEY2), localContainer.get(KEY2)); Utilities.testEquality(container.get(KEY3), localContainer.get(KEY3)); Utilities.testEquality(container.get(KEY4), localContainer.get(KEY4)); container.clear(); Assert.assertEquals(0, container.size()); Assert.assertTrue(container.entrySet().isEmpty()); } }
private MapJoinBytesTableContainer(float keyCountAdj, int threshold, float loadFactor, int wbSize, MapJoinObjectSerDeContext valCtx, long keyCount, long memUsage) throws SerDeException { int newThreshold = HashMapWrapper.calculateTableSize( keyCountAdj, threshold, loadFactor, keyCount); hashMap = new BytesBytesMultiHashMap(newThreshold, loadFactor, wbSize, memUsage); directWriteHelper = new DirectKeyValueWriter(); }
new HashMapWrapper<AbstractMapJoinKey, MapJoinObjectValue>(); File file = new File(path.toUri().getPath()); fs.create(path); long fileLength = hashTable.flushMemoryCacheToPersistent(file); console.printInfo(Utilities.now() + "\tUpload 1 File to: " + tmpURIPath + " File size: " + fileLength); hashTable.close();
if (pos.intValue() != tag) { MapJoinObjectValue o = mapJoinTables.get(pos).get(key); MapJoinRowContainer<ArrayList<Object>> rowContainer = rowContainerMap.get(pos);
@Override public long getEstimatedMemorySize() { // TODO: Key and Values are Object[] which can be eagerly deserialized or lazily deserialized. To accurately // estimate the entry size, every possible Objects in Key, Value should implement MemoryEstimate interface which // is very intrusive. So assuming default entry size here. return size() * DEFAULT_HASHMAP_ENTRY_SIZE; }
private MapJoinBytesTableContainer(float keyCountAdj, int threshold, float loadFactor, int wbSize, MapJoinObjectSerDeContext valCtx, long keyCount, long memUsage) throws SerDeException { int newThreshold = HashMapWrapper.calculateTableSize( keyCountAdj, threshold, loadFactor, keyCount); hashMap = new BytesBytesMultiHashMap(newThreshold, loadFactor, wbSize, memUsage); directWriteHelper = new DirectKeyValueWriter(); }
public static void persistDummyTable(ObjectOutputStream out) throws IOException { MapJoinPersistableTableContainer tableContainer = new HashMapWrapper(); out.writeUTF(tableContainer.getClass().getName()); out.writeObject(tableContainer.getMetaData()); out.writeInt(tableContainer.size()); }
@Override public MapJoinKey putRow(Writable currentKey, Writable currentValue) throws SerDeException, HiveException { MapJoinKey key = MapJoinKey.read(output, keyContext, currentKey); FlatRowContainer values = (FlatRowContainer)get(key); if (values == null) { values = new FlatRowContainer(); put(key, values); } values.add(valueContext, (BytesWritable)currentValue); return key; }
private HashMapWrapper(float keyCountAdj, int threshold, float loadFactor, long keyCount) { super(createConstructorMetaData(threshold, loadFactor)); threshold = calculateTableSize(keyCountAdj, threshold, loadFactor, keyCount); mHash = new HashMap<MapJoinKey, MapJoinRowContainer>(threshold, loadFactor); }
public VectorMapJoinFastTableContainer(MapJoinDesc desc, Configuration hconf, long estimatedKeyCount) throws SerDeException { this.desc = desc; this.hconf = hconf; keyCountAdj = HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT); threshold = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD); loadFactor = HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR); wbSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEWBSIZE); this.estimatedKeyCount = estimatedKeyCount; int newThreshold = HashMapWrapper.calculateTableSize( keyCountAdj, threshold, loadFactor, estimatedKeyCount); // LOG.debug("VectorMapJoinFastTableContainer load newThreshold " + newThreshold); vectorMapJoinFastHashTable = createHashTable(newThreshold); }
private static MapJoinTableContainer getDefaultEmptyContainer(MapJoinObjectSerDeContext keyCtx, MapJoinObjectSerDeContext valCtx) throws SerDeException { MapJoinTableContainer container = new HashMapWrapper(); container.setSerde(keyCtx, valCtx); container.seal(); return container; } }
@Override public MapJoinKey putRow(MapJoinObjectSerDeContext keyContext, Writable currentKey, MapJoinObjectSerDeContext valueContext, Writable currentValue) throws SerDeException, HiveException { MapJoinKey key = MapJoinKey.read(output, keyContext, currentKey); FlatRowContainer values = (FlatRowContainer)get(key); if (values == null) { values = new FlatRowContainer(); put(key, values); } values.add(valueContext, (BytesWritable)currentValue); return key; }
private HashMapWrapper(float keyCountAdj, int threshold, float loadFactor, long keyCount) { super(createConstructorMetaData(threshold, loadFactor)); threshold = calculateTableSize(keyCountAdj, threshold, loadFactor, keyCount); mHash = new HashMap<MapJoinKey, MapJoinRowContainer>(threshold, loadFactor); }
public VectorMapJoinFastTableContainer(MapJoinDesc desc, Configuration hconf, long estimatedKeyCount) throws SerDeException { this.desc = desc; this.hconf = hconf; keyCountAdj = HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT); threshold = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD); loadFactor = HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR); wbSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEWBSIZE); this.estimatedKeyCount = estimatedKeyCount; // LOG.info("VectorMapJoinFastTableContainer load keyCountAdj " + keyCountAdj); // LOG.info("VectorMapJoinFastTableContainer load threshold " + threshold); // LOG.info("VectorMapJoinFastTableContainer load loadFactor " + loadFactor); // LOG.info("VectorMapJoinFastTableContainer load wbSize " + wbSize); int newThreshold = HashMapWrapper.calculateTableSize( keyCountAdj, threshold, loadFactor, estimatedKeyCount); // LOG.debug("VectorMapJoinFastTableContainer load newThreshold " + newThreshold); vectorMapJoinFastHashTable = createHashTable(newThreshold); }
private static MapJoinTableContainer getDefaultEmptyContainer(Configuration hconf, MapJoinObjectSerDeContext keyCtx, MapJoinObjectSerDeContext valCtx) throws SerDeException { boolean useOptimizedContainer = HiveConf.getBoolVar( hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE); if (useOptimizedContainer) { return new MapJoinBytesTableContainer(hconf, valCtx, -1, 0); } MapJoinTableContainer container = new HashMapWrapper(); container.setSerde(keyCtx, valCtx); container.seal(); return container; } }
directWriteHelper = new MapJoinBytesTableContainer.DirectKeyValueWriter(); int newKeyCount = HashMapWrapper.calculateTableSize( keyCountAdj, threshold, loadFactor, keyCount);