@VisibleForTesting SmallTableLocalCache(Ticker ticker) { cleanupService = Executors.newScheduledThreadPool(INTEGER_ONE, new ThreadFactoryBuilder().setNameFormat("SmallTableCache Cleanup Thread").setDaemon(true).build()); cacheL1 = CacheBuilder.newBuilder().expireAfterAccess(L1_CACHE_EXPIRE_DURATION, TimeUnit.SECONDS) .ticker(ticker).build(); cacheL2 = CacheBuilder.newBuilder().softValues().build(); cleanupService.scheduleAtFixedRate(() -> { cleanup(); }, INTEGER_ZERO, MAINTENANCE_THREAD_CLEANUP_PERIOD, TimeUnit.SECONDS); }
/** * Check if this is a new query. If so, clean up the cache * that is for the previous query, and reset the current query id. */ public static void initialize(Configuration conf) { String currentQueryId = conf.get(HiveConf.ConfVars.HIVEQUERYID.varname); if (!currentQueryId.equals(queryId)) { if (TABLE_CONTAINER_CACHE.size() != 0) { synchronized (TABLE_CONTAINER_CACHE) { if (!currentQueryId.equals(queryId) && TABLE_CONTAINER_CACHE.size() != 0) { TABLE_CONTAINER_CACHE.clear((path, tableContainer) -> tableContainer.clear()); LOG.debug("Cleaned up small table cache for query {}", queryId); } } } queryId = currentQueryId; } }
@Test public void testL2IsNotClearedIfTheItemIsInL1() throws ExecutionException { FakeTicker ticker = new FakeTicker(); cache = new SmallTableCache.SmallTableLocalCache<>(ticker); cache.get(KEY, () -> { counter.incrementAndGet(); return new String(TEST_VALUE_1); }); forceOOMToClearSoftValues(); ticker.advance(60, TimeUnit.SECONDS); cache.cleanup(); String res = cache.get(KEY, () -> { counter.incrementAndGet(); return new String(TEST_VALUE_2); }); assertEquals(TEST_VALUE_1, res); assertEquals(1, counter.get()); assertEquals(1, cache.size()); }
@Test public void testL2Miss() throws ExecutionException { FakeTicker ticker = new FakeTicker(); cache = new SmallTableCache.SmallTableLocalCache<>(ticker); cache.get(KEY, () -> { counter.incrementAndGet(); return new String(TEST_VALUE_1); }); ticker.advance(60, TimeUnit.SECONDS); cache.cleanup(); forceOOMToClearSoftValues(); String res = cache.get(KEY, () -> { counter.incrementAndGet(); return new String(TEST_VALUE_2); }); assertEquals(TEST_VALUE_2, res); assertEquals(2, counter.get()); assertEquals(1, cache.size()); }
@Test public void testL2Hit() throws ExecutionException { FakeTicker ticker = new FakeTicker(); cache = new SmallTableCache.SmallTableLocalCache<>(ticker); cache.get(KEY, () -> { counter.incrementAndGet(); return new String(TEST_VALUE_1); }); ticker.advance(60, TimeUnit.SECONDS); String res = cache.get(KEY, () -> { counter.incrementAndGet(); return new String(TEST_VALUE_2); }); assertEquals(TEST_VALUE_1, res); assertEquals(1, counter.get()); assertEquals(1, cache.size()); }
@Test public void testL1Hit() throws ExecutionException { cache.get(KEY, () -> { counter.incrementAndGet(); return new String(TEST_VALUE_1); }); String res = cache.get(KEY, () -> { counter.incrementAndGet(); return new String(TEST_VALUE_2); }); assertEquals(TEST_VALUE_1, res); assertEquals(1, counter.get()); assertEquals(1, cache.size()); }
public static void cache(String key, MapJoinTableContainer tableContainer) { TABLE_CONTAINER_CACHE.put(key, tableContainer); }
@Test public void testPutL2() throws ExecutionException { FakeTicker ticker = new FakeTicker(); cache = new SmallTableCache.SmallTableLocalCache<>(ticker); cache.put(KEY, new String(TEST_VALUE_1)); String res = cache.get(KEY, () -> { counter.incrementAndGet(); return new String(TEST_VALUE_2); }); ticker.advance(60, TimeUnit.SECONDS); cache.cleanup(); assertEquals(TEST_VALUE_1, res); assertEquals(0, counter.get()); }
public static MapJoinTableContainer get(String key, Callable<MapJoinTableContainer> valueLoader) throws ExecutionException { return TABLE_CONTAINER_CACHE.get(key, valueLoader); }
@Before public void setUp() { this.cache = new SmallTableCache.SmallTableLocalCache<>(); this.counter = new AtomicInteger(0); }