public void reconnectNode(EVCacheNodeImpl evcNode ) { final long upTime = System.currentTimeMillis() - evcNode.getCreateTime(); if (log.isDebugEnabled()) log.debug("Reconnecting node : " + evcNode + "; UpTime : " + upTime); if(upTime > 30000) { //not more than once every 30 seconds : TODO make this configurable EVCacheMetricsFactory.getCounter(appName + "-RECONNECT", evcNode.getBaseTags()).increment(); evcNode.setConnectTime(System.currentTimeMillis()); mconn.queueReconnect(evcNode); } } private final String BULK_OPERATION_STRING = "BulkOperation";
private boolean ensureWriteQueueSize(MemcachedNode node, String key) throws EVCacheException { if (node instanceof EVCacheNodeImpl) { final EVCacheNodeImpl evcNode = (EVCacheNodeImpl) node; int i = 0; while (true) { final int size = evcNode.getWriteQueueSize(); final boolean canAddToOpQueue = size < maxWriteQueueSize; if (log.isDebugEnabled()) log.debug("App : " + appName + "; zone : " + zone + "; key : " + key + "; WriteQSize : " + size); if (canAddToOpQueue) break; EVCacheMetricsFactory.getCounter("EVCacheClient-" + appName + "-WRITE_BLOCK", evcNode.getBaseTags()).increment(); try { Thread.sleep(writeBlock.get()); } catch (InterruptedException e) { throw new EVCacheException("Thread was Interrupted", e); } if(i++ > 3) { EVCacheMetricsFactory.getCounter("EVCacheClient-" + appName + "-INACTIVE_NODE", evcNode.getBaseTags()).increment(); if (log.isDebugEnabled()) log.debug("Node : " + evcNode + " for app : " + appName + "; zone : " + zone + " is not active. Will Fail Fast and the write will be dropped for key : " + key); evcNode.shutdown(); return false; } } } return true; }
@Override public void receivedStatus(OperationStatus s) { rv.set(new Long(s.isSuccess() ? s.getMessage() : "-1")); final MemcachedNode node = getEVCacheNode(key); if(node instanceof EVCacheNodeImpl) { getCounter("MutateOperation-"+ s.getStatusCode().name(), ((EVCacheNodeImpl)node).getBaseTags()).increment(); } else { getCounter("MutateOperation-"+ s.getStatusCode().name(), BasicTagList.of("HOST", node.getSocketAddress().toString())).increment(); } }
private boolean validateNode(String key, boolean _throwException) throws EVCacheException, EVCacheConnectException { final MemcachedNode node = evcacheMemcachedClient.getEVCacheNode(key); // First check if the node is active if (node instanceof EVCacheNodeImpl) { final EVCacheNodeImpl evcNode = (EVCacheNodeImpl) node; if (!evcNode.isAvailable()) { EVCacheMetricsFactory.getCounter("EVCacheClient-" + appName + "-INACTIVE_NODE", evcNode.getBaseTags()).increment(); if (log.isDebugEnabled()) log.debug("Node : " + node + " for app : " + appName + "; zone : " + zone + " is not active. Will Fail Fast so that we can fallback to Other Zone if available."); if (_throwException) throw new EVCacheConnectException("Connection for Node : " + node + " for app : " + appName + "; zone : " + zone + " is not active"); return false; } final int size = evcNode.getReadQueueSize(); final boolean canAddToOpQueue = size < maxReadQueueSize.get(); if (log.isDebugEnabled()) log.debug("Current Read Queue Size - " + size + " for app " + appName + " & zone " + zone + " and node : " + evcNode); if (!canAddToOpQueue) { EVCacheMetricsFactory.getCounter(appName + "-READ_QUEUE_FULL", evcNode.getBaseTags()).increment(); if (log.isDebugEnabled()) log.debug("Read Queue Full for Node : " + node + "; app : " + appName + "; zone : " + zone + "; Current Size : " + size + "; Max Size : " + maxReadQueueSize.get()); if (_throwException) throw new EVCacheReadQueueException("Read Queue Full for Node : " + node + "; app : " + appName + "; zone : " + zone + "; Current Size : " + size + "; Max Size : " + maxReadQueueSize.get()); return false; } } return true; }
private Collection<String> validateReadQueueSize(Collection<String> canonicalKeys) throws EVCacheException { if (evcacheMemcachedClient.getNodeLocator() == null) return canonicalKeys; final Collection<String> retKeys = new ArrayList<>(canonicalKeys.size()); for (String key : canonicalKeys) { final MemcachedNode node = evcacheMemcachedClient.getNodeLocator().getPrimary(key); if (node instanceof EVCacheNodeImpl) { final EVCacheNodeImpl evcNode = (EVCacheNodeImpl) node; if (!evcNode.isAvailable()) { continue; } final int size = evcNode.getReadQueueSize(); final boolean canAddToOpQueue = size < (maxReadQueueSize.get() * 2); // if (log.isDebugEnabled()) log.debug("Bulk Current Read Queue // Size - " + size + " for app " + appName + " & zone " + zone + // " ; node " + node); if (!canAddToOpQueue) { EVCacheMetricsFactory.getCounter(appName + "-READ_QUEUE_FULL", evcNode.getBaseTags()).increment(); if (log.isDebugEnabled()) log.debug("Read Queue Full on Bulk Operation for app : " + appName + "; zone : " + zone + "; Current Size : " + size + "; Max Size : " + maxReadQueueSize.get() * 2); } else { retKeys.add(key); } } } return retKeys; }
TagList tags = null; if(op.getHandlingNode() instanceof EVCacheNodeImpl) { tags = ((EVCacheNodeImpl)op.getHandlingNode()).getBaseTags(); } else { tags = BasicTagList.of(DataSourceType.COUNTER); TagList tags = null; if(op.getHandlingNode() instanceof EVCacheNodeImpl) { tags = ((EVCacheNodeImpl)op.getHandlingNode()).getBaseTags(); } else { tags = BasicTagList.of(DataSourceType.COUNTER);
TagList tags = null; if(op.getHandlingNode() instanceof EVCacheNodeImpl) { tags = ((EVCacheNodeImpl)op.getHandlingNode()).getBaseTags(); } else { tags = BasicTagList.of(DataSourceType.COUNTER); TagList tags = null; if(op.getHandlingNode() instanceof EVCacheNodeImpl) { tags = ((EVCacheNodeImpl)op.getHandlingNode()).getBaseTags(); } else { tags = BasicTagList.of(DataSourceType.COUNTER);
private void updateQueueStats() { for (ServerGroup serverGroup : memcachedInstancesByServerGroup.keySet()) { List<EVCacheClient> clients = memcachedInstancesByServerGroup.get(serverGroup); for(EVCacheClient client : clients) { final int wSize = client.getWriteQueueLength(); EVCacheMetricsFactory.getLongGauge("EVCacheClientPool-WriteQueueSize", client.getTagList()).set(Long.valueOf(wSize)); final int rSize = client.getReadQueueLength(); EVCacheMetricsFactory.getLongGauge("EVCacheClientPool-ReadQueueSize", client.getTagList()).set(Long.valueOf(rSize)); if(refreshConnectionOnReadQueueFull.get()) { final Collection<MemcachedNode> allNodes = client.getNodeLocator().getAll(); for (MemcachedNode node : allNodes) { if (node instanceof EVCacheNodeImpl) { final EVCacheNodeImpl evcNode = ((EVCacheNodeImpl) node); if(evcNode.getReadQueueSize() >= refreshConnectionOnReadQueueFullSize.get().intValue()) { EVCacheMetricsFactory.getCounter("EVCacheClientPool-REFRESH_ON_QUEUE_FULL", evcNode.getBaseTags()).increment(); client.getEVCacheMemcachedClient().reconnectNode(evcNode); } } } } } } }
TagList tags = null; if(op.getHandlingNode() instanceof EVCacheNodeImpl) { tags = ((EVCacheNodeImpl)op.getHandlingNode()).getBaseTags(); } else { tags = BasicTagList.of(DataSourceType.COUNTER); TagList tags = null; if(op.getHandlingNode() instanceof EVCacheNodeImpl) { tags = ((EVCacheNodeImpl)op.getHandlingNode()).getBaseTags(); } else { tags = BasicTagList.of(DataSourceType.COUNTER);
public void receivedStatus(OperationStatus status) { operationDuration .stop(); if (log.isDebugEnabled()) log.debug("Getting Key : " + key + "; Status : " + status.getStatusCode().name() + (log.isTraceEnabled() ? " Node : " + getEVCacheNode(key) : "") + "; Message : " + status.getMessage() + "; Elapsed Time - " + operationDuration.getDuration(TimeUnit.MILLISECONDS)); if (status.getStatusCode().equals(StatusCode.SUCCESS)) { getCounter(GET_OPERATION_STRING + "-SUCCESS").increment(); } else { if (status.getStatusCode().equals(StatusCode.TIMEDOUT)) { final MemcachedNode node = getEVCacheNode(key); if(node instanceof EVCacheNodeImpl) { getCounter(GET_OPERATION_STRING + "-" + status.getStatusCode().name(), ((EVCacheNodeImpl)node).getBaseTags()).increment(); } else { getCounter(GET_OPERATION_STRING + "-"+ status.getStatusCode().name(), BasicTagList.of("HOST", node.getSocketAddress().toString())).increment(); } } else { getCounter(GET_OPERATION_STRING + "-"+ status.getStatusCode().name()).increment(); } } try { if (val != null) { rv.set(val.get(), status); } else { rv.set(null, status); } } catch (Exception e) { log.error(e.getMessage(), e); rv.set(null, status); } }
@Override public void receivedStatus(OperationStatus val) { operationDuration.stop(); if (log.isDebugEnabled()) log.debug("Storing Key : " + key + "; Status : " + val.getStatusCode().name() + (log.isTraceEnabled() ? " Node : " + getEVCacheNode(key) : "") + "; Message : " + val.getMessage() + "; Elapsed Time - " + operationDuration.getDuration(TimeUnit.MILLISECONDS)); if (val.getStatusCode().equals(StatusCode.SUCCESS)) { getCounter(operationSuccessStr).increment(); } else { if (val.getStatusCode().equals(StatusCode.TIMEDOUT)) { final MemcachedNode node = getEVCacheNode(key); if(node instanceof EVCacheNodeImpl) { if (log.isInfoEnabled()) log.info(val.getStatusCode().name() + " Storing Key : " + key + "; Status : " + val.getStatusCode().name() + "; Node : " + node + "; Message : " + val.getMessage() + "; Elapsed Time - " + operationDuration.getDuration(TimeUnit.MILLISECONDS), new Exception()); if(node instanceof EVCacheNodeImpl) { getCounter(operationStr + "-" + val.getStatusCode().name(), ((EVCacheNodeImpl)node).getBaseTags()).increment(); } else { getCounter(operationStr + "-" + val.getStatusCode().name(), BasicTagList.of("HOST", node.getSocketAddress().toString())).increment(); } } } else { getCounter(operationStr + "-" + val.getStatusCode().name()).increment(); } } rv.set(val.isSuccess(), val); }
public void receivedStatus(OperationStatus status) { operationDuration.stop(); if (log.isDebugEnabled()) log.debug("GetAndTouch Key : " + key + "; Status : " + status.getStatusCode().name() + (log.isTraceEnabled() ? " Node : " + getEVCacheNode(key) : "") + "; Message : " + status.getMessage() + "; Elapsed Time - " + operationDuration.getDuration(TimeUnit.MILLISECONDS)); if (status.getStatusCode().equals(StatusCode.SUCCESS)) { getCounter(GET_AND_TOUCH_OPERATION_STRING + "-SUCCESS").increment(); } else { if (status.getStatusCode().equals(StatusCode.TIMEDOUT)) { final MemcachedNode node = getEVCacheNode(key); if(node instanceof EVCacheNodeImpl) { getCounter(GET_AND_TOUCH_OPERATION_STRING + "-" + status.getStatusCode().name(), ((EVCacheNodeImpl)node).getBaseTags()).increment(); } else { getCounter(GET_AND_TOUCH_OPERATION_STRING + "-"+ status.getStatusCode().name(), BasicTagList.of("HOST", node.getSocketAddress().toString())).increment(); } } else { getCounter(GET_AND_TOUCH_OPERATION_STRING + "-"+ status.getStatusCode().name()).increment(); } } rv.set(val, status); }
@Override public void receivedStatus(OperationStatus status) { operationDuration.stop(); rv.set(Boolean.TRUE, status); if (status.getStatusCode().equals(StatusCode.SUCCESS)) { getCounter(DELETE_OPERATION_SUCCESS_STRING).increment(); } else { if (status.getStatusCode().equals(StatusCode.TIMEDOUT)) { final MemcachedNode node = getEVCacheNode(key); if(node instanceof EVCacheNodeImpl) { getCounter("DeleteOperation-"+ status.getStatusCode().name(), ((EVCacheNodeImpl)node).getBaseTags()).increment(); } else { getCounter("DeleteOperation-"+ status.getStatusCode().name(), BasicTagList.of("HOST", node.getSocketAddress().toString())).increment(); } } else { getCounter("DeleteOperation-"+ status.getStatusCode().name()).increment(); } } }
@Override public void receivedStatus(OperationStatus status) { operationDuration.stop(); rv.set(status.isSuccess(), status); if (status.getStatusCode().equals(StatusCode.SUCCESS)) { getCounter(TOUCH_OPERATION_STRING + "-SUCCESS").increment(); } else { if (status.getStatusCode().equals(StatusCode.TIMEDOUT)) { final MemcachedNode node = getEVCacheNode(key); if(node instanceof EVCacheNodeImpl) { getCounter(TOUCH_OPERATION_STRING + "-" + status.getStatusCode().name(), ((EVCacheNodeImpl)node).getBaseTags()).increment(); } else { getCounter(TOUCH_OPERATION_STRING + "-"+ status.getStatusCode().name(), BasicTagList.of("HOST", node.getSocketAddress().toString())).increment(); } } else { getCounter(TOUCH_OPERATION_STRING + "-"+ status.getStatusCode().name()).increment(); } } }