/** * Write log. * * @param bytes the bytes * @throws GondolaException thrown if the local member is not the leader * @throws InterruptedException the interrupted exception */ public void writeLog(byte[] bytes) throws GondolaException, InterruptedException { Command command = shard.checkoutCommand(); Timer.Context time = commitTimer.time(); command.commit(bytes, 0, bytes.length); time.stop(); } /**
/** * Equivalent to getCommittedCommand(index, -1); * * @param index must be > 0. * @return the non-null Command at index. */ public Command getCommittedCommand(int index) throws InterruptedException, GondolaException, TimeoutException { return getCommittedCommand(index, -1); }
private Map<Object, Object> getGondolaStatus(Gondola gondola, ChangeLogProcessor changeLogProcessor) throws InterruptedException { Map<Object, Object> shards = new LinkedHashMap<>(); for (Shard shard : gondola.getShardsOnHost()) { String shardId = shard.getShardId(); Map<Object, Object> shardMap = new LinkedHashMap<>(); shardMap.put("commitIndex", shard.getCommitIndex()); shardMap.put("savedIndex", shard.getLastSavedIndex()); shardMap.put("appliedIndex", changeLogProcessor.getAppliedIndex(shardId)); shardMap.put("slaveStatus", getSlaveStatus(shard)); shardMap.put("role", shard.getLocalRole()); shardMap.put("enabled", shard.getLocalMember().isEnabled()); shards.put(shardId, shardMap); } return shards; }
serviceClass.getConstructor(Gondola.class, String.class).newInstance(gondola, shard.getShardId()); services.put(shard.getShardId(), service);
private Member.SlaveStatus getSlaveStatus(Shard shard) { try { return shard.getLocalMember().getSlaveStatus(); } catch (InterruptedException e) { throw new RuntimeException(e); } } }
private boolean waitLogApproach(String shardId, long timeoutMs, int logPosDiff) throws ShardManagerException, InterruptedException { Shard shard = gondola.getShard(shardId); try { return Utils.pollingWithTimeout(() -> { if (shard.getCommitIndex() != 0 && shard.getCommitIndex() - filter.getChangeLogProcessor().getAppliedIndex(shardId) <= logPosDiff) { return true; } Member.SlaveStatus slaveStatus = shard.getLocalMember().getSlaveStatus(); if (!slaveOperational(slaveStatus)) { throw new ShardManagerException(MASTER_IS_GONE); } trace("[{}] {} Log status={}, ci={}, si={}, ai={} targetDiff={}", gondola.getHostId(), shardId, slaveOperational(slaveStatus) ? "RUNNING" : "DOWN", shard.getCommitIndex(), getSavedIndex(shard), filter.getChangeLogProcessor().getAppliedIndex(shardId), logPosDiff); return false; }, timeoutMs / POLLING_TIMES, timeoutMs); } catch (ExecutionException e) { throw new ShardManagerException(e); } }
private boolean setSlave(String shardId, int memberId, long timeoutMs) throws InterruptedException, ShardManagerException { try { Member localMember = gondola.getShard(shardId).getLocalMember(); Member.SlaveStatus slaveStatus = localMember.getSlaveStatus(); if (slaveStatus != null && slaveStatus.masterId == memberId && slaveStatus.running) { return true; } localMember.setSlave(memberId); return Utils.pollingWithTimeout(() -> { Member.SlaveStatus status = gondola.getShard(shardId).getLocalMember().getSlaveStatus(); if (slaveOperational(status)) { trace("[{}] Successfully connect to leader node={}", gondola.getHostId(), memberId); return true; } trace("[{}] Slave status={} role={}", gondola.getHostId(), status, gondola.getShard(shardId).getLocalRole()); return false; }, timeoutMs / POLLING_TIMES, timeoutMs); } catch (Exception e) { throw new ShardManagerException(e); } }
/** * Returns this command object back to the pool. */ public void release() { shard.checkinCommand(this); } }
private int getSavedIndex(Shard shard) throws ShardManagerException { try { return shard.getLastSavedIndex(); } catch (Exception e) { throw new ShardManagerException(e); } }
private Member getLeader(String shardId) { return gondola.getShard(shardId).getLeader(); }
@POST @Path("/setLeader") public Map setLeader(@QueryParam("shardId") String shardId) { Map<Object, Object> result = new HashMap<>(); try { GondolaApplication.getRoutingFilter().getGondola().getShard(shardId).forceLeader(5000); result.put("success", true); } catch (Exception e) { result.put("success", false); result.put("reason", e.getMessage()); } return result; }
private void init() throws Exception { // Create implementations String clockClassName = config.get(config.get("clock.impl") + ".class"); clock = (Clock) Class.forName(clockClassName).getConstructor(Gondola.class, String.class) .newInstance(this, hostId); String networkClassName = config.get(config.get("network.impl") + ".class"); network = (Network) Class.forName(networkClassName).getConstructor(Gondola.class, String.class) .newInstance(this, hostId); String storageClassName = config.get(config.get("storage.impl") + ".class"); storage = (Storage) Class.forName(storageClassName).getConstructor(Gondola.class, String.class) .newInstance(this, hostId); // Create the shards running on a host for (String shardId : config.getShardIds(hostId)) { Shard shard = new Shard(this, shardId); shards.add(shard); shardMap.put(shardId, shard); } inited = true; }
private void waitDrainRaftLogs(String shardId) { boolean synced = false; long startTime = System.currentTimeMillis(); long checkTime = startTime; while (!synced) { try { Thread.sleep(500); long now = System.currentTimeMillis(); int commitIndex = gondola.getShard(shardId).getCommitIndex(); int appliedIndex = changeLogProcessor.getAppliedIndex(shardId); int diff = commitIndex - appliedIndex; if (now - checkTime > 10000) { checkTime = now; logger.warn("[{}] Recovery running for {} seconds, {} logs left, ci={}, ai={}", gondola.getHostId(), (now - startTime) / 1000, diff, commitIndex, appliedIndex); } synced = diff <= 0; } catch (Exception e) { logger.warn("[{}] Unknown error. message={}", gondola.getHostId(), e); } } }
appUri = Utils.getAppUri(config, config.getMember(roleChangeEvent.leader.getMemberId()).getHostId()); updateShardRoutingEntries(roleChangeEvent.shard.getShardId(), appUri); if (roleChangeEvent.leader.isLocal()) { CompletableFuture.runAsync(() -> { String shardId = roleChangeEvent.shard.getShardId(); trace("[{}-{}] Become leader on \"{}\", blocking all requests to the shard....", gondola.getHostId(), roleChangeEvent.leader.getMemberId(), shardId); trace("[{}-{}] Raft logs are up-to-date, notify application is ready to serve...", gondola.getHostId(), roleChangeEvent.leader.getMemberId()); services.get(roleChangeEvent.shard.getShardId()).ready(); trace("[{}-{}] Ready for serving, unblocking the requests...", gondola.getHostId(), roleChangeEvent.leader.getMemberId());
/** * Is leader boolean. * * @return the boolean */ public boolean isLeader() { return shard.getLocalMember().isLeader(); }
List<Config.ConfigMember> shardMembers = gondola.getConfig().getMembersInShard(shard.getShardId()); for (Config.ConfigMember m : shardMembers) { if (m.getMemberId() == channel.getRemoteMemberId()) {
while (true) { try { command = shard.getCommittedCommand(appliedIndex + 1, 1000); if (changeLogConsumer != null) { Timer.Context time = timer.time();
@POST @Path("/enable") public Map enable(@QueryParam("shardId") String shardId, @QueryParam("enable") boolean enable) { Map<Object, Object> result = new HashMap<>(); Member localMember = GondolaApplication.getRoutingFilter().getGondola().getShard(shardId).getLocalMember(); try { localMember.enable(enable); result.put("success", true); } catch (Exception e) { result.put("success", false); result.put("reason", e.getMessage()); } return result; }
/** * Returns the command at the specified index. This method blocks until index has been committed. An empty command * can be returned. This is an artifact of the Raft protocol to avoid deadlock when used with a finite thread pool. * Empty commands will be inserted right after a leader election when the new leader discovers that it has * uncommitted commands. The leader inserts an empty command to commit these immediately. * * @param index Must be > 0. * @param timeout Returns after timeout milliseconds, even if the command is not yet available. -1 means there is no * timeout. * @return non-null Command */ public Command getCommittedCommand(int index, int timeout) throws GondolaException, InterruptedException, TimeoutException { if (index <= 0) { throw new IllegalStateException(String.format("Index %d must be > 0", index)); } Command command = checkoutCommand(); cmember.getCommittedLogEntry(command.ccmd, index, timeout); try { cmember.getCommittedLogEntry(command.ccmd, index, timeout); return command; } catch (GondolaException e) { command.release(); throw e; } }
/** * Protected Methods. * * @param shardId the shard id * @return the boolean */ protected boolean isLeaderInShard(String shardId) { return gondola.getShard(shardId).getLocalMember().isLeader(); }