SaveQueue(Gondola gondola, CoreMember cmember) throws Exception { this.gondola = gondola; this.cmember = cmember; gondola.getConfig().registerForUpdates(this); storage = gondola.getStorage(); stats = gondola.getStats(); numWorkers = gondola.getConfig().getInt("storage.save_queue_workers"); String address = storage.getAddress(cmember.memberId); if (address != null && gondola.getNetwork().isActive(address)) { throw new IllegalStateException(String.format("[%s-%s] Process %s at address %s is currently using storage", gondola.getHostId(), cmember.memberId, gondola.getProcessId(), address)); } storage.setAddress(cmember.memberId, gondola.getNetwork().getAddress()); storage.setPid(cmember.memberId, gondola.getProcessId()); initSavedIndex(); }
public Peer(Gondola gondola, CoreMember cmember, int peerId) { this.gondola = gondola; this.clock = gondola.getClock(); this.storage = gondola.getStorage(); this.cmember = cmember; this.peerId = peerId; entry0 = new LogEntry(storage, 1) { @Override public void release() { // Don't release back into pool } }; entry0.term = 0; entry0.index = 0; entry0.size = 0; }
CommitQueue(Gondola gondola, CoreMember cmember) throws GondolaException { this.gondola = gondola; this.cmember = cmember; this.storage = gondola.getStorage(); this.pool = gondola.getMessagePool(); this.savedIndex = null; //savedIndex; this.matchIndices = null; //matchIndices; gondola.getConfig().registerForUpdates(configListener); stats = gondola.getStats(); }
@GET @Path("/gondolaStatus") public Map getGondolaStatus() throws InterruptedException { RoutingFilter routingFilter = GondolaApplication.getRoutingFilter(); Gondola gondola = routingFilter.getGondola(); ChangeLogProcessor changeLogProcessor = GondolaApplication.getRoutingFilter().getChangeLogProcessor(); Map<Object, Object> map = new LinkedHashMap<>(); map.put("hostId", gondola.getHostId()); map.put("gondolaStatus", getGondolaStatus(gondola, changeLogProcessor)); map.put("routingTable", routingFilter.getRoutingTable()); map.put("bucketTable", getBucketMapStatus(routingFilter.getBucketManager())); map.put("lock", getLockManagerStatus(routingFilter.getLockManager())); map.put("shardManager", getShardManagerStatus()); map.put("numActiveRequests", routingFilter.getBucketRequestCounters().values().stream().map(AtomicInteger::get) .reduce((i1, i2) -> i1 + i2).orElse(0)); map.put("config", getConfigInfo(gondola)); map.put("stats", gondola.getStats()); map.put("pid", gondola.getConfig().getAttributesForHost(gondola.getHostId()).get("hostname") + ":" + gondola.getProcessId()); map.put("timers", timerMap()); map.put("meters", GondolaApplication.MyMetricsServletContextListener.METRIC_REGISTRY.getMeters()); return map; }
/** * Throws an exception if the file lock can't be acquired. */ void acquireFileLock() throws Exception { File file = new File(fileLockDir, String.format("gondola-lock-%s-%s", gondola.getHostId(), memberId)); FileChannel channel = new RandomAccessFile(file, "rw").getChannel(); if (channel.tryLock() == null) { throw new IllegalStateException(String.format("Another process has the lock on %s", file)); } }
public NastyStorage(Gondola gondola, String hostId) throws Exception { String storageClassName = gondola.getConfig().get(gondola.getConfig().get("storage_nasty.impl") + ".class"); storage = (Storage) Class.forName(storageClassName).getConstructor(Gondola.class, String.class) .newInstance(gondola, hostId); tracing = gondola.getConfig().getBoolean("storage_nasty.tracing"); }
public CoreMember(Gondola gondola, Cluster cluster, int memberId, List<Integer> peerIds, boolean isPrimary) throws Exception { this.gondola = gondola; this.cluster = cluster; this.memberId = memberId; this.isPrimary = isPrimary; gondola.getConfig().registerForUpdates(this); acquireFileLock(); clock = gondola.getClock(); pool = gondola.getMessagePool(); storage = gondola.getStorage(); incomingQueue = new ArrayBlockingQueue<>(incomingQueueSize); saveQueue = new SaveQueue(gondola, this); commitQueue = new CommitQueue(gondola, this); for (int id : peerIds) { Peer peer = new Peer(gondola, this, id); peers.add(peer); peerMap.put(peer.peerId, peer); } // Initialize some convenience variables for use when calculating the commit index majority = (peers.size() + 1) / 2 + 1; matchIndices = new int[peers.size()]; reset(); }
Cluster(Gondola gondola, String clusterId) throws Exception { this.gondola = gondola; this.clusterId = clusterId; config = gondola.getConfig(); stats = gondola.getStats(); List<Config.ConfigMember> configMembers = config.getMembersInCluster(clusterId); List<Integer> peerIds = configMembers.stream() .filter(cm -> !cm.hostId.equals(gondola.getHostId())) .map(cm -> cm.memberId).collect(Collectors.toList()); // First create the local member, because it's needed when creating the remote members. for (int i = 0; i < configMembers.size(); i++) { Config.ConfigMember cm = configMembers.get(i); if (gondola.getHostId().equals(cm.hostId)) { // Local member boolean isPrimary = i == 0; cmember = new CoreMember(gondola, this, cm.memberId, peerIds, isPrimary); localMember = new Member(gondola, cmember, null); members.add(localMember); break; } } if (cmember == null) { throw new IllegalStateException(String.format("Host id %s not found in %d", config.getIdentifier())); } // Create list of peers for (Peer p : cmember.peers) { members.add(new Member(gondola, cmember, p)); } }
Stats stats = gondola.getStats(); logger.info( String.format("[%s-%d] %s%s %spid=%s wait=%dms cmdQ=%d waitQ=%d in=%d" + "|%.1f/s out=%.1f/s lat=%.3fms/%.3fms", gondola.getHostId(), memberId, role, masterId >= 0 ? "-SLAVE" : "", enabled ? "" : " (disabled)", gondola.getProcessId(), waitMs, commandQueue.size(), waitQueue.size(), incomingQueue.size(), stats.incomingMessagesRps, stats.sentMessagesRps, CoreCmd.commitLatency.get(), latency.get())); logger.info(String.format("[%s-%d] - leader=%d cterm=%d ci=%d latest=(%d,%d) votedFor=%d msgPool=%d/%d", gondola.getHostId(), memberId, leaderId, currentTerm, commitIndex, sentRid.term, sentRid.index, votedFor, pool.size(), pool.createdCount)); logger.info(String.format("[%s-%d] - storage %.1f/s ti=(%d,%d) saveQ=%d gap=%d done=%d", gondola.getHostId(), memberId, stats.savedCommandsRps, saveQueue.lastTerm, saveQueue.savedIndex, saveQueue.workQueue.size(), saveQueue.maxGap, saveQueue.saved.size())); for (CoreCmd c : waitQueue) { logger.info("[{}-{}] commit waiter: term={} index={}", gondola.getHostId(), memberId, c.term, c.index); gondola.getHostId(), memberId, c.index);
public void run() { MessageHandler handler = new MyMessageHandler(); MessagePool pool = gondola.getMessagePool(); Message message = pool.checkout(); Message nextMessage = pool.checkout(); if (excess < 0) { logger.warn("[{}-{}] recv({}): end-of-file", gondola.getHostId(), cmember.memberId, peerId); in = channel.getInputStream(in, true); excess = 0; if (networkTracing) { logger.info("[{}-{}] recv({}): read {} bytes", gondola.getHostId(), cmember.memberId, peerId, message.size + excess); gondola.getStats().incomingMessage(message.size + excess); gondola.getHostId(), cmember.memberId, peerId, e.getMessage()); if ("Socket closed".equals(e.getMessage())
long endTs = timeout < 0 ? Long.MAX_VALUE : gondola.getClock().now() + timeout; lock.lock(); try { while (status == Command.STATUS_NONE && gondola.getClock().now() < endTs) { if (timeout < 0) { updateCond.await(); timeout, index, bufLen)); case Command.STATUS_NOT_LEADER: throw new NotLeaderException(leaderId == -1 ? null : gondola.getConfig().getAddressForMember(leaderId)); case Command.STATUS_ERROR: throw new IllegalStateException("Error committing index " + index + ": " + errorMessage); if (commandTracing) { logger.info("[{}-{}] committed(term={} index={} size={}) status={}", gondola.getHostId(), cmember.memberId, term, index, size, status);
gondola = new Gondola(config, hostId); gondola.start(); cluster = gondola.getCluster(clusterId); } catch (Exception e) { logger.error("Could not initialize gondola", e);
gondola.getHostId(), cmember.memberId, peerId, socketInactivityTimeout); out = channel.getOutputStream(out, true); lastReceivedTs = now; Message message = outQueue.take(); if (message.tracingInfo != null) { logger.info("[{}-{}] send({}): {}", gondola.getHostId(), cmember.memberId, peerId, message.tracingInfo); if (networkTracing) { logger.info("[{}-{}] send({}): sent {} bytes", gondola.getHostId(), cmember.memberId, peerId, message.size); gondola.getStats().sentMessage(message.size); lastSentTs = clock.now(); } catch (InterruptedException e) { } catch (Exception e) { String m = String.format("[%s-%d] Failed to send to %d: %s", gondola.getHostId(), cmember.memberId, peerId, e.getMessage()); if ("Socket closed".equals(e.getMessage())
if (!gondola.getProcessId().equals(pid)) { logger.warn("[{}-{}] SaveQueue: another process pid={} may be updating the same tables. Current pid={}", gondola.getHostId(), cmember.memberId, pid, gondola.getProcessId()); gondola.getHostId(), cmember.memberId, g, maxGap, maxGap); } else { maxGap = g; gondola.getHostId(), cmember.memberId, newLastTerm, lastIndex, maxGap); gondola.getHostId(), cmember.memberId, i, lastIndex, savedIndex); deleteFrom(i + 1, lastIndex); assert i > start;
public RoutingService(Gondola gondola, String shardId) { this.gondola = gondola; shard = gondola.getShard(shardId); hostId = gondola.getHostId(); this.shardId = shardId; memberId = shard.getLocalMember().getMemberId(); commitTimer = GondolaApplication.MyMetricsServletContextListener.METRIC_REGISTRY.timer("LogWriter"); }
socket.setTcpNoDelay(true); logger.info("[{}-{}] Socket created to {} ({})", gondola.getHostId(), memberId, peerId, inetSocketAddress); = new SocketNetwork.Hello(gondola.getHostId(), socket.getInputStream(), socket.getOutputStream()); if (callFrom) { hello.callFrom(memberId, peerId); lastError = e.getMessage(); String m = String.format("[%s-%d] Failed to create socket to %d (%s): %s", gondola.getHostId(), memberId, peerId, inetSocketAddress, e.getMessage()); if (e instanceof ConnectException || e instanceof SocketTimeoutException gondola.getHostId(), memberId, peerId, inetSocketAddress, createSocketRetryPeriod); gondola.getClock().awaitCondition(lock, retryCond, createSocketRetryPeriod); } catch (InterruptedException e1) { return;
MessagePool pool = gondola.getMessagePool(); Rid rid = new Rid(); Rid savedRid = new Rid(); logger.info( "[{}-{}] Backfilling {} at index {} paused to allow storage (index={}) to catch up", gondola.getHostId(), cmember.memberId, peerId, nextIndex, savedRid.index); backfillAhead = backfillAheadLimit; } else { throw new IllegalStateException( String.format("[%s-%d] Could not retrieve index=%d to backfill %d. savedIndex=%d", gondola.getHostId(), cmember.memberId, startIndex - 1, peerId, savedRid.index)); throw new IllegalStateException( String.format("[%s-%d] Could not retrieve index=%d to backfill %d. savedIndex=%d", gondola.getHostId(), cmember.memberId, startIndex, peerId, savedRid.index)); backfilling = false; logger.info("[{}-{}] Backfilling {} to {} is done", gondola.getHostId(), cmember.memberId, peerId, backfillToIndex - 1);
public CoreCmd(Gondola gondola, Cluster cluster, CoreMember cmember) { this.gondola = gondola; this.cluster = cluster; this.cmember = cmember; createdCount.incrementAndGet(); stats = gondola.getStats(); buffer = new byte[maxCommandSize]; reset(); }
Map<String, RoutingService> services = new HashMap<>(); for (Shard shard : gondola.getShardsOnHost()) { RoutingService service = serviceClass.getConstructor(Gondola.class, String.class).newInstance(gondola, shard.getShardId()); RoutingFilter routingFilter = new RoutingFilter(gondola, routingHelper, proxyClientProvider, services, changeLogProcessor); gondola.start(); routingFilter.start();