private void clearAllTableContainers() { if (mapJoinTables != null) { for (MapJoinTableContainer tableContainer : mapJoinTables) { if (tableContainer != null) { tableContainer.clear(); } } } }
private void createMatchTracker(MapJoinTableContainer smallTable) { ReusableGetAdaptor hashMapRowGetter = smallTable.createGetter(null); matchTracker = ((ReusableGetAdaptorDirectAccess) hashMapRowGetter).createMatchTracker(); Preconditions.checkState(matchTracker != null); }
cacheKey, tableContainer.getClass().getSimpleName(), pos); tableContainer.setSerde(keyCtx, valCtx); while (kvReader.next()) { tableContainer.putRow((Writable) kvReader.getCurrentKey(), (Writable) kvReader.getCurrentValue()); numEntries++; if (doMemCheck && (numEntries % memoryMonitorInfo.getMemoryCheckInterval() == 0)) { final long estMemUsage = tableContainer.getEstimatedMemorySize(); if (estMemUsage > effectiveThreshold) { String msg = "Hash table loading exceeded memory limits for input: " + inputName + tableContainer.seal(); mapJoinTables[pos] = tableContainer; if (doMemCheck) { LOG.info("Finished loading hash table for input: {} cacheKey: {} numEntries: {} estimatedMemoryUsage: {}", inputName, cacheKey, numEntries, tableContainer.getEstimatedMemorySize()); } else { LOG.info("Finished loading hash table for input: {} cacheKey: {} numEntries: {}", inputName, cacheKey,
tableContainer.setSerde(keyCtx, valCtx); while (kvReader.next()) { tableContainer.putRow( (Writable)kvReader.getCurrentKey(), (Writable)kvReader.getCurrentValue()); tableContainer.seal(); LOG.info("Finished loading hashtable using " + tableContainer.getClass() + ". Small table position: " + pos); mapJoinTables[pos] = tableContainer;
public static VectorMapJoinOptimizedHashTable createHashTable(MapJoinDesc desc, MapJoinTableContainer mapJoinTableContainer) { MapJoinKey refKey = mapJoinTableContainer.getAnyKey(); ReusableGetAdaptor hashMapRowGetter = mapJoinTableContainer.createGetter(refKey);
if (pos != conf.getPosBigTable()) { MapJoinTableContainer container = mapJoinTables[pos]; if (container != null && container.getKey() != null) { SmallTableCache.cache(container.getKey(), container); for (MapJoinTableContainer container : mapJoinTables) { if (container != null) { spilled = spilled || container.hasSpill(); container.dumpMetrics();
for (MapJoinTableContainer container : mapJoinTables) { if (container != null) { spilled = spilled || container.hasSpill(); container.dumpMetrics();
protected MapJoinKey getRefKey(byte alias) { // We assume that since we are joining on the same key, all tables would have either // optimized or non-optimized key; hence, we can pass any key in any table as reference. // We do it so that MJKB could determine whether it can use optimized keys. for (byte pos = 0; pos < order.length; pos++) { if (pos == alias) continue; MapJoinKey refKey = mapJoinTables[pos].getAnyKey(); if (refKey != null) return refKey; } return null; // All join tables have 0 keys, doesn't matter what we generate. }
@Override public void closeOp(boolean aborted) throws HiveException { super.closeOp(aborted); for (MapJoinTableContainer tableContainer : mapJoinTables) { if (tableContainer != null) { tableContainer.dumpMetrics(); } } if (!aborted) { if (outputBatch.size > 0) { flushOutput(); } } }
for (MapJoinTableContainer container : pair.getLeft()) { if (container != null) { spilled = spilled || container.hasSpill();
MapJoinTestConfig.createMapJoinTableContainerSerDe(mapJoinDesc); mapJoinTableContainer.setSerde( mapJoinTableContainerSerDe.getKeyContext(), mapJoinTableContainerSerDe.getValueContext());
@Override public int size() { return originalTableContainer.size(); }
public static VectorMapJoinOptimizedHashTable createHashTable(MapJoinDesc desc, MapJoinTableContainer mapJoinTableContainer) { MapJoinKey refKey = mapJoinTableContainer.getAnyKey(); ReusableGetAdaptor hashMapRowGetter = mapJoinTableContainer.createGetter(refKey);
for (MapJoinTableContainer container : mapJoinTables) { if (container != null) { spilled = spilled || container.hasSpill(); container.dumpMetrics();
protected MapJoinKey getRefKey(byte alias) { // We assume that since we are joining on the same key, all tables would have either // optimized or non-optimized key; hence, we can pass any key in any table as reference. // We do it so that MJKB could determine whether it can use optimized keys. for (byte pos = 0; pos < order.length; pos++) { if (pos == alias) continue; MapJoinKey refKey = mapJoinTables[pos].getAnyKey(); if (refKey != null) return refKey; } return null; // All join tables have 0 keys, doesn't matter what we generate. }
@Override public void closeOp(boolean aborted) throws HiveException { super.closeOp(aborted); for (MapJoinTableContainer tableContainer : mapJoinTables) { if (tableContainer != null) { tableContainer.dumpMetrics(); } } if (!aborted && 0 < outputBatch.size) { flushOutput(); } }