public MemcachedNode createMemcachedNode(SocketAddress sa, SocketChannel c, int bufSize) { boolean doAuth = false; final EVCacheNodeImpl node = new EVCacheNodeImpl(sa, c, bufSize, createReadOperationQueue(), createWriteOperationQueue(), createOperationQueue(), opMaxBlockTime, doAuth, getOperationTimeout(), getAuthWaitTime(), this, appName, id, serverGroup, startTime); node.registerMonitors(); return node; }
private Collection<String> validateReadQueueSize(Collection<String> canonicalKeys) throws EVCacheException { if (evcacheMemcachedClient.getNodeLocator() == null) return canonicalKeys; final Collection<String> retKeys = new ArrayList<>(canonicalKeys.size()); for (String key : canonicalKeys) { final MemcachedNode node = evcacheMemcachedClient.getNodeLocator().getPrimary(key); if (node instanceof EVCacheNodeImpl) { final EVCacheNodeImpl evcNode = (EVCacheNodeImpl) node; if (!evcNode.isAvailable()) { continue; } final int size = evcNode.getReadQueueSize(); final boolean canAddToOpQueue = size < (maxReadQueueSize.get() * 2); // if (log.isDebugEnabled()) log.debug("Bulk Current Read Queue // Size - " + size + " for app " + appName + " & zone " + zone + // " ; node " + node); if (!canAddToOpQueue) { EVCacheMetricsFactory.getCounter(appName + "-READ_QUEUE_FULL", evcNode.getBaseTags()).increment(); if (log.isDebugEnabled()) log.debug("Read Queue Full on Bulk Operation for app : " + appName + "; zone : " + zone + "; Current Size : " + size + "; Max Size : " + maxReadQueueSize.get() * 2); } else { retKeys.add(key); } } } return retKeys; }
@Override public String getSocketChannelLocalAddress() { try { if(getChannel() != null) { return getChannel().getLocalAddress().toString(); } } catch (IOException e) { log.error("Exception", e); } return "NULL"; }
public void reconnectNode(EVCacheNodeImpl evcNode ) { final long upTime = System.currentTimeMillis() - evcNode.getCreateTime(); if (log.isDebugEnabled()) log.debug("Reconnecting node : " + evcNode + "; UpTime : " + upTime); if(upTime > 30000) { //not more than once every 30 seconds : TODO make this configurable EVCacheMetricsFactory.getCounter(appName + "-RECONNECT", evcNode.getBaseTags()).increment(); evcNode.setConnectTime(System.currentTimeMillis()); mconn.queueReconnect(evcNode); } } private final String BULK_OPERATION_STRING = "BulkOperation";
public void shutdown() { removeMonitoring(); writeQ.clear(); readQ.clear(); inputQueue.clear(); try { // Cleanup the ByteBuffers only if they are sun.nio.ch.DirectBuffer // If we don't cleanup then we will leak 16K of memory if (getRbuf() instanceof DirectBuffer) { Cleaner cleaner = ((DirectBuffer) getRbuf()).cleaner(); if (cleaner != null) cleaner.clean(); cleaner = ((DirectBuffer) getWbuf()).cleaner(); if (cleaner != null) cleaner.clean(); } } catch (Throwable t) { getLogger().error("Exception cleaning ByteBuffer.", t); } }
private boolean ensureWriteQueueSize(MemcachedNode node, String key) throws EVCacheException { if (node instanceof EVCacheNodeImpl) { final EVCacheNodeImpl evcNode = (EVCacheNodeImpl) node; int i = 0; while (true) { final int size = evcNode.getWriteQueueSize(); final boolean canAddToOpQueue = size < maxWriteQueueSize; if (log.isDebugEnabled()) log.debug("App : " + appName + "; zone : " + zone + "; key : " + key + "; WriteQSize : " + size); if (canAddToOpQueue) break; EVCacheMetricsFactory.getCounter("EVCacheClient-" + appName + "-WRITE_BLOCK", evcNode.getBaseTags()).increment(); try { Thread.sleep(writeBlock.get()); } catch (InterruptedException e) { throw new EVCacheException("Thread was Interrupted", e); } if(i++ > 3) { EVCacheMetricsFactory.getCounter("EVCacheClient-" + appName + "-INACTIVE_NODE", evcNode.getBaseTags()).increment(); if (log.isDebugEnabled()) log.debug("Node : " + evcNode + " for app : " + appName + "; zone : " + zone + " is not active. Will Fail Fast and the write will be dropped for key : " + key); evcNode.shutdown(); return false; } } } return true; }
public EVCacheNodeImpl(SocketAddress sa, SocketChannel c, int bufSize, BlockingQueue<Operation> rq, BlockingQueue<Operation> wq, BlockingQueue<Operation> iq, long opQueueMaxBlockTimeMillis, boolean waitForAuth, long dt, long at, ConnectionFactory fa, String appName, int id, ServerGroup serverGroup, long stTime) { super(sa, c, bufSize, rq, wq, iq, Long.valueOf(opQueueMaxBlockTimeMillis), waitForAuth, dt, at, fa); this.id = id; this._appName = appName; this._serverGroup = serverGroup; setConnectTime(stTime); this.readQ = rq; this.inputQueue = iq; this.sendMetrics = EVCacheConfig.getInstance().getDynamicBooleanProperty("EVCacheNodeImpl." + appName + ".sendMetrics", false); this.tags = BasicTagList.of("ServerGroup", _serverGroup.getName(), "APP", appName, "Id", String.valueOf(id), EVCacheMetricsFactory.OWNER.getKey(), EVCacheMetricsFactory.OWNER.getValue()); this.hostName = ((InetSocketAddress) getSocketAddress()).getHostName(); this.metricPrefix = "EVCacheNode"; this.baseConfig = MonitorConfig.builder(metricPrefix).build(); baseTags = BasicTagList.concat(tags, BasicTagList.of("HOST", hostName)); setupMonitoring(); }
@Override public void receivedStatus(OperationStatus s) { rv.set(new Long(s.isSuccess() ? s.getMessage() : "-1")); final MemcachedNode node = getEVCacheNode(key); if(node instanceof EVCacheNodeImpl) { getCounter("MutateOperation-"+ s.getStatusCode().name(), ((EVCacheNodeImpl)node).getBaseTags()).increment(); } else { getCounter("MutateOperation-"+ s.getStatusCode().name(), BasicTagList.of("HOST", node.getSocketAddress().toString())).increment(); } }
private void updateQueueStats() { for (ServerGroup serverGroup : memcachedInstancesByServerGroup.keySet()) { List<EVCacheClient> clients = memcachedInstancesByServerGroup.get(serverGroup); for(EVCacheClient client : clients) { final int wSize = client.getWriteQueueLength(); EVCacheMetricsFactory.getLongGauge("EVCacheClientPool-WriteQueueSize", client.getTagList()).set(Long.valueOf(wSize)); final int rSize = client.getReadQueueLength(); EVCacheMetricsFactory.getLongGauge("EVCacheClientPool-ReadQueueSize", client.getTagList()).set(Long.valueOf(rSize)); if(refreshConnectionOnReadQueueFull.get()) { final Collection<MemcachedNode> allNodes = client.getNodeLocator().getAll(); for (MemcachedNode node : allNodes) { if (node instanceof EVCacheNodeImpl) { final EVCacheNodeImpl evcNode = ((EVCacheNodeImpl) node); if(evcNode.getReadQueueSize() >= refreshConnectionOnReadQueueFullSize.get().intValue()) { EVCacheMetricsFactory.getCounter("EVCacheClientPool-REFRESH_ON_QUEUE_FULL", evcNode.getBaseTags()).increment(); client.getEVCacheMemcachedClient().reconnectNode(evcNode); } } } } } } }
if (!evcNode.isActive() && !discoveredHostsInServerGroup.contains(evcNode.getSocketAddress())) { if (log.isDebugEnabled()) log.debug("AppName :" + _appName + "; ServerGroup : " + serverGroup + "; Node : " + node + " is not active. Will shutdown the client and init it again.");
public void removeMonitoring() { try { final ObjectName mBeanName = ObjectName.getInstance(getMonitorName()); final MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer(); if (mbeanServer.isRegistered(mBeanName)) { if (log.isDebugEnabled()) log.debug("MBEAN with name " + mBeanName + " has been registered. Will unregister the previous instance and register a new one."); mbeanServer.unregisterMBean(mBeanName); } } catch (Exception e) { if (log.isDebugEnabled()) log.debug("Exception while setting up the monitoring.", e); } }
public int getReadQueueLength() { final Collection<MemcachedNode> allNodes = evcacheMemcachedClient.getNodeLocator().getAll(); int size = 0; for(MemcachedNode node : allNodes) { if(node instanceof EVCacheNodeImpl) { size += ((EVCacheNodeImpl)node).getReadQueueSize(); } } return size; }
@Override public List<Monitor<?>> getMonitors() { if (!sendMetrics.get() && getContinuousTimeout() == 0) return Collections.<Monitor<?>> emptyList(); if(getContinuousTimeout() > 0) { MonitorConfig monitorConfig = EVCacheConfig.getInstance().getMonitorConfig(metricPrefix + "_ContinuousTimeout", DataSourceType.GAUGE, baseTags); final LongGauge cTimeouts = new LongGauge(monitorConfig); cTimeouts.set(Long.valueOf(getContinuousTimeout())); metrics.add(cTimeouts);
TagList tags = null; if(op.getHandlingNode() instanceof EVCacheNodeImpl) { tags = ((EVCacheNodeImpl)op.getHandlingNode()).getBaseTags(); } else { tags = BasicTagList.of(DataSourceType.COUNTER); TagList tags = null; if(op.getHandlingNode() instanceof EVCacheNodeImpl) { tags = ((EVCacheNodeImpl)op.getHandlingNode()).getBaseTags(); } else { tags = BasicTagList.of(DataSourceType.COUNTER);
private void setupMonitoring() { try { final ObjectName mBeanName = ObjectName.getInstance(getMonitorName()); final MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer(); if (mbeanServer.isRegistered(mBeanName)) { if (log.isDebugEnabled()) log.debug("MBEAN with name " + mBeanName + " has been registered. Will unregister the previous instance and register a new one."); mbeanServer.unregisterMBean(mBeanName); } mbeanServer.registerMBean(this, mBeanName); } catch (Exception e) { if (log.isWarnEnabled()) log.warn("Exception while setting up monitoring.", e); } }
private boolean validateNode(String key, boolean _throwException) throws EVCacheException, EVCacheConnectException { final MemcachedNode node = evcacheMemcachedClient.getEVCacheNode(key); // First check if the node is active if (node instanceof EVCacheNodeImpl) { final EVCacheNodeImpl evcNode = (EVCacheNodeImpl) node; if (!evcNode.isAvailable()) { EVCacheMetricsFactory.getCounter("EVCacheClient-" + appName + "-INACTIVE_NODE", evcNode.getBaseTags()).increment(); if (log.isDebugEnabled()) log.debug("Node : " + node + " for app : " + appName + "; zone : " + zone + " is not active. Will Fail Fast so that we can fallback to Other Zone if available."); if (_throwException) throw new EVCacheConnectException("Connection for Node : " + node + " for app : " + appName + "; zone : " + zone + " is not active"); return false; } final int size = evcNode.getReadQueueSize(); final boolean canAddToOpQueue = size < maxReadQueueSize.get(); if (log.isDebugEnabled()) log.debug("Current Read Queue Size - " + size + " for app " + appName + " & zone " + zone + " and node : " + evcNode); if (!canAddToOpQueue) { EVCacheMetricsFactory.getCounter(appName + "-READ_QUEUE_FULL", evcNode.getBaseTags()).increment(); if (log.isDebugEnabled()) log.debug("Read Queue Full for Node : " + node + "; app : " + appName + "; zone : " + zone + "; Current Size : " + size + "; Max Size : " + maxReadQueueSize.get()); if (_throwException) throw new EVCacheReadQueueException("Read Queue Full for Node : " + node + "; app : " + appName + "; zone : " + zone + "; Current Size : " + size + "; Max Size : " + maxReadQueueSize.get()); return false; } } return true; }
TagList tags = null; if(op.getHandlingNode() instanceof EVCacheNodeImpl) { tags = ((EVCacheNodeImpl)op.getHandlingNode()).getBaseTags(); } else { tags = BasicTagList.of(DataSourceType.COUNTER); TagList tags = null; if(op.getHandlingNode() instanceof EVCacheNodeImpl) { tags = ((EVCacheNodeImpl)op.getHandlingNode()).getBaseTags(); } else { tags = BasicTagList.of(DataSourceType.COUNTER);
@Override public String getSocketChannelRemoteAddress() { try { if(getChannel() != null) { return getChannel().getRemoteAddress().toString(); } } catch (IOException e) { log.error("Exception", e); } return "NULL"; }
TagList tags = null; if(op.getHandlingNode() instanceof EVCacheNodeImpl) { tags = ((EVCacheNodeImpl)op.getHandlingNode()).getBaseTags(); } else { tags = BasicTagList.of(DataSourceType.COUNTER); TagList tags = null; if(op.getHandlingNode() instanceof EVCacheNodeImpl) { tags = ((EVCacheNodeImpl)op.getHandlingNode()).getBaseTags(); } else { tags = BasicTagList.of(DataSourceType.COUNTER);
public void receivedStatus(OperationStatus status) { operationDuration .stop(); if (log.isDebugEnabled()) log.debug("Getting Key : " + key + "; Status : " + status.getStatusCode().name() + (log.isTraceEnabled() ? " Node : " + getEVCacheNode(key) : "") + "; Message : " + status.getMessage() + "; Elapsed Time - " + operationDuration.getDuration(TimeUnit.MILLISECONDS)); if (status.getStatusCode().equals(StatusCode.SUCCESS)) { getCounter(GET_OPERATION_STRING + "-SUCCESS").increment(); } else { if (status.getStatusCode().equals(StatusCode.TIMEDOUT)) { final MemcachedNode node = getEVCacheNode(key); if(node instanceof EVCacheNodeImpl) { getCounter(GET_OPERATION_STRING + "-" + status.getStatusCode().name(), ((EVCacheNodeImpl)node).getBaseTags()).increment(); } else { getCounter(GET_OPERATION_STRING + "-"+ status.getStatusCode().name(), BasicTagList.of("HOST", node.getSocketAddress().toString())).increment(); } } else { getCounter(GET_OPERATION_STRING + "-"+ status.getStatusCode().name()).increment(); } } try { if (val != null) { rv.set(val.get(), status); } else { rv.set(null, status); } } catch (Exception e) { log.error(e.getMessage(), e); rv.set(null, status); } }