protected void unbindStatsMBeans(IndexStatsMBean mBean) { statsMBeans.remove(mBean.getName()); }
@Override public SimpleToken lock(String asyncIndexerLane) { IndexStatsMBean mbean = getIndexStatsMBean(asyncIndexerLane); if (IndexStatsMBean.STATUS_RUNNING.equals(mbean.getStatus())){ log.info("Aborting current indexing run of async indexer for lane [{}]", asyncIndexerLane); } mbean.abortAndPause(); retry(mbean, TIMEOUT_SECONDS, 1000); log.info("Aborted and paused async indexer for lane [{}]", asyncIndexerLane); return new SimpleToken(asyncIndexerLane); }
@Override public void unlock(SimpleToken token) { getIndexStatsMBean(token.laneName).resume(); log.info("Resumed async indexer for lane [{}]", token.laneName); }
private void printAsyncIndexInfo(PrintWriter pw) { List<String> asyncLanes = ImmutableList.copyOf(asyncIndexInfoService.getAsyncLanes()); String title = "Async Indexers State"; printTitle(pw, title); pw.printf("Number of async indexer lanes : %d%n", asyncLanes.size()); pw.println(); for (String lane : asyncLanes) { pw.println(lane); AsyncIndexInfo info = asyncIndexInfoService.getInfo(lane); if (info != null) { pw.printf(" Last indexed to : %s%n", formatTime(info.getLastIndexedTo())); IndexStatsMBean stats = info.getStatsMBean(); if (stats != null) { pw.printf(" Status : %s%n", stats.getStatus()); pw.printf(" Failing : %s%n", stats.isFailing()); pw.printf(" Paused : %s%n", stats.isPaused()); if (stats.isFailing()) { pw.printf(" Failing since : %s%n", stats.getFailingSince()); pw.printf(" Latest error : %s%n", stats.getLatestError()); } } pw.println(); } } }
try { if (bean != null) { beanStatus = bean.getStatus(); } else { return false; LOG.warn("Exception during getting status for {}. Ignoring this indexer lane", bean.getName(), e); return false; long newExecCnt = indexLaneBean.getTotalExecutionCount(); String beanStatus = indexLaneBean.getStatus(); LOG.info("Lane {} has moved - oldExecCnt {}, newExecCnt {}", indexLaneBean.getName(), oldExecCnt, newExecCnt); } else if (clock.getTime() - start > TimeUnit.MINUTES.toMillis(2)) { LOG.warn("Timed out while waiting for running index lane executions"); break; } else { LOG.info("Lane {} still has execution count {}. Waiting....", indexLaneBean.getName(), newExecCnt);
private void retry(IndexStatsMBean mbean, int timeoutSeconds, int intervalBetweenTriesMsec) { long timeout = clock.getTime() + timeoutSeconds * 1000L; while (clock.getTime() < timeout) { try { if (!IndexStatsMBean.STATUS_RUNNING.equals(mbean.getStatus())) { return; } } catch (Exception e) { throw new RuntimeException(e); } try { int delta = (int) (timeout - clock.getTime() / 1000); log.info("Async indexer for lane [{}] found to be running. Would wait for {} seconds " + "more for it to stop", mbean.getName(), delta); Thread.sleep(intervalBetweenTriesMsec); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return; } } throw new IllegalStateException("RetryLoop failed, condition is false after " + timeoutSeconds + " seconds"); } }
@Test public void lockBasics() throws Exception{ AbortingIndexerLock lock = new AbortingIndexerLock(infoService); when(statsMBean.getStatus()).thenReturn(IndexStatsMBean.STATUS_DONE); SimpleToken lockToken = lock.lock("async"); assertNotNull(lockToken); verify(statsMBean).abortAndPause(); lock.unlock(lockToken); verify(statsMBean).resume(); }
private void printAsyncIndexInfo(PrintWriter pw) { List<String> asyncLanes = ImmutableList.copyOf(asyncIndexInfoService.getAsyncLanes()); String title = "Async Indexers State"; printTitle(pw, title); pw.printf("Number of async indexer lanes : %d%n", asyncLanes.size()); pw.println(); for (String lane : asyncLanes) { pw.println(lane); AsyncIndexInfo info = asyncIndexInfoService.getInfo(lane); if (info != null) { pw.printf(" Last indexed to : %s%n", formatTime(info.getLastIndexedTo())); IndexStatsMBean stats = info.getStatsMBean(); if (stats != null) { pw.printf(" Status : %s%n", stats.getStatus()); pw.printf(" Failing : %s%n", stats.isFailing()); pw.printf(" Paused : %s%n", stats.isPaused()); if (stats.isFailing()) { pw.printf(" Failing since : %s%n", stats.getFailingSince()); pw.printf(" Latest error : %s%n", stats.getLatestError()); } } pw.println(); } } }
try { if (bean != null) { beanStatus = bean.getStatus(); } else { return false; LOG.warn("Exception during getting status for {}. Ignoring this indexer lane", bean.getName(), e); return false; long newExecCnt = indexLaneBean.getTotalExecutionCount(); String beanStatus = indexLaneBean.getStatus(); LOG.info("Lane {} has moved - oldExecCnt {}, newExecCnt {}", indexLaneBean.getName(), oldExecCnt, newExecCnt); } else if (clock.getTime() - start > TimeUnit.MINUTES.toMillis(2)) { LOG.warn("Timed out while waiting for running index lane executions"); break; } else { LOG.info("Lane {} still has execution count {}. Waiting....", indexLaneBean.getName(), newExecCnt);
private void retry(IndexStatsMBean mbean, int timeoutSeconds, int intervalBetweenTriesMsec) { long timeout = clock.getTime() + timeoutSeconds * 1000L; while (clock.getTime() < timeout) { try { if (!IndexStatsMBean.STATUS_RUNNING.equals(mbean.getStatus())) { return; } } catch (Exception e) { throw new RuntimeException(e); } try { int delta = (int) (timeout - clock.getTime() / 1000); log.info("Async indexer for lane [{}] found to be running. Would wait for {} seconds " + "more for it to stop", mbean.getName(), delta); Thread.sleep(intervalBetweenTriesMsec); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return; } } throw new IllegalStateException("RetryLoop failed, condition is false after " + timeoutSeconds + " seconds"); } }
private void printAsyncIndexInfo(PrintWriter pw) { List<String> asyncLanes = ImmutableList.copyOf(asyncIndexInfoService.getAsyncLanes()); String title = "Async Indexers State"; printTitle(pw, title); pw.printf("Number of async indexer lanes : %d%n", asyncLanes.size()); pw.println(); for (String lane : asyncLanes) { pw.println(lane); AsyncIndexInfo info = asyncIndexInfoService.getInfo(lane); if (info != null) { pw.printf(" Last indexed to : %s%n", formatTime(info.getLastIndexedTo())); IndexStatsMBean stats = info.getStatsMBean(); if (stats != null) { pw.printf(" Status : %s%n", stats.getStatus()); pw.printf(" Failing : %s%n", stats.isFailing()); pw.printf(" Paused : %s%n", stats.isPaused()); if (stats.isFailing()) { pw.printf(" Failing since : %s%n", stats.getFailingSince()); pw.printf(" Latest error : %s%n", stats.getLatestError()); } } pw.println(); } } }
static AsyncIndexInfoService getAsyncIndexInfoService(List<IndexMBeanInfoSupplier> infoSuppliers) { AsyncIndexInfoService service = mock(AsyncIndexInfoService.class); List<String> asyncLanes = Lists.newArrayList(); for (IndexMBeanInfoSupplier info : infoSuppliers) { String lane = info.getName(); IndexStatsMBean bean = mock(IndexStatsMBean.class); when(bean.getName()).thenReturn(lane); when(bean.getStatus()).then(mockObj -> info.getStatus()); when(bean.getTotalExecutionCount()).then(mockObj -> info.getExecCnt()); when(service.getInfo(lane)).then(mockObj -> new AsyncIndexInfo( lane, 1324L, 4567L, STATUS_RUNNING.equals(info.getStatus()), bean )); asyncLanes.add(lane); } when(service.getAsyncLanes()).thenReturn(asyncLanes); return service; }
@Override public SimpleToken lock(String asyncIndexerLane) { IndexStatsMBean mbean = getIndexStatsMBean(asyncIndexerLane); if (IndexStatsMBean.STATUS_RUNNING.equals(mbean.getStatus())){ log.info("Aborting current indexing run of async indexer for lane [{}]", asyncIndexerLane); } mbean.abortAndPause(); retry(mbean, TIMEOUT_SECONDS, 1000); log.info("Aborted and paused async indexer for lane [{}]", asyncIndexerLane); return new SimpleToken(asyncIndexerLane); }
private void retry(IndexStatsMBean mbean, int timeoutSeconds, int intervalBetweenTriesMsec) { long timeout = clock.getTime() + timeoutSeconds * 1000L; while (clock.getTime() < timeout) { try { if (!IndexStatsMBean.STATUS_RUNNING.equals(mbean.getStatus())) { return; } } catch (Exception e) { throw new RuntimeException(e); } try { int delta = (int) (timeout - clock.getTime() / 1000); log.info("Async indexer for lane [{}] found to be running. Would wait for {} seconds " + "more for it to stop", mbean.getName(), delta); Thread.sleep(intervalBetweenTriesMsec); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return; } } throw new IllegalStateException("RetryLoop failed, condition is false after " + timeoutSeconds + " seconds"); } }
protected void unbindStatsMBeans(IndexStatsMBean mBean) { statsMBeans.remove(mBean.getName()); }
@Override public void unlock(SimpleToken token) { getIndexStatsMBean(token.laneName).resume(); log.info("Resumed async indexer for lane [{}]", token.laneName); }
@Override public SimpleToken lock(String asyncIndexerLane) { IndexStatsMBean mbean = getIndexStatsMBean(asyncIndexerLane); if (IndexStatsMBean.STATUS_RUNNING.equals(mbean.getStatus())){ log.info("Aborting current indexing run of async indexer for lane [{}]", asyncIndexerLane); } mbean.abortAndPause(); retry(mbean, TIMEOUT_SECONDS, 1000); log.info("Aborted and paused async indexer for lane [{}]", asyncIndexerLane); return new SimpleToken(asyncIndexerLane); }
@Reference(name = "statsMBeans", policy = ReferencePolicy.DYNAMIC, cardinality = ReferenceCardinality.MULTIPLE, policyOption = ReferencePolicyOption.GREEDY, service = IndexStatsMBean.class ) protected void bindStatsMBeans(IndexStatsMBean mBean) { statsMBeans.put(mBean.getName(), mBean); }
@Override public void unlock(SimpleToken token) { getIndexStatsMBean(token.laneName).resume(); log.info("Resumed async indexer for lane [{}]", token.laneName); }
@Test public void lockWithRetry() throws Exception{ AbortingIndexerLock lock = new AbortingIndexerLock(infoService); when(statsMBean.getStatus()) .thenReturn(IndexStatsMBean.STATUS_RUNNING) .thenReturn(IndexStatsMBean.STATUS_RUNNING) .thenReturn(IndexStatsMBean.STATUS_DONE); SimpleToken lockToken = lock.lock("async"); assertNotNull(lockToken); verify(statsMBean).abortAndPause(); verify(statsMBean, times(3)).getStatus(); }