@Override public ServerName getServerWALsBelongTo() { return server.getServerName(); }
public WALSplitterHandler(final Server server, SplitLogWorkerCoordination coordination, SplitLogWorkerCoordination.SplitTaskDetails splitDetails, CancelableProgressable reporter, AtomicInteger inProgressTasks, TaskExecutor splitTaskExecutor) { super(server, EventType.RS_LOG_REPLAY); this.splitTaskDetails = splitDetails; this.coordination = coordination; this.reporter = reporter; this.inProgressTasks = inProgressTasks; this.inProgressTasks.incrementAndGet(); this.serverName = server.getServerName(); this.splitTaskExecutor = splitTaskExecutor; }
@VisibleForTesting List<String> getAllQueues() throws IOException { List<String> allQueues = Collections.emptyList(); try { allQueues = queueStorage.getAllQueues(server.getServerName()); } catch (ReplicationException e) { throw new IOException(e); } return allQueues; }
/** * Delete a complete queue of wals associated with a replication source * @param queueId the id of replication queue to delete */ private void deleteQueue(String queueId) { abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(), queueId)); }
protected AuthenticationTokenSecretManager createSecretManager() { if (!isSecurityEnabled) return null; if (server == null) return null; Configuration conf = server.getConfiguration(); long keyUpdateInterval = conf.getLong("hbase.auth.key.update.interval", 24*60*60*1000); long maxAge = conf.getLong("hbase.auth.token.max.lifetime", 7*24*60*60*1000); return new AuthenticationTokenSecretManager(conf, server.getZooKeeper(), server.getServerName().toString(), keyUpdateInterval, maxAge); }
public HeapMemoryTunerChore() { super(server.getServerName() + "-HeapMemoryTunerChore", server, defaultChorePeriod); Class<? extends HeapMemoryTuner> tunerKlass = server.getConfiguration().getClass( HBASE_RS_HEAP_MEMORY_TUNER_CLASS, DefaultHeapMemoryTuner.class, HeapMemoryTuner.class); heapMemTuner = ReflectionUtils.newInstance(tunerKlass, server.getConfiguration()); tunerContext .setOffheapMemStore(regionServerAccounting.isOffheap()); }
() -> this.queueStorage.addWAL(server.getServerName(), source.getQueueId(), logName));
private long getRecoveredQueueStartPos() { long startPosition = 0; String peerClusterZNode = source.getQueueId(); try { startPosition = this.replicationQueues.getWALPosition(source.getServer().getServerName(), peerClusterZNode, this.queue.peek().getName()); LOG.trace("Recovered queue started with log {} at position {}", this.queue.peek(), startPosition); } catch (ReplicationException e) { terminate("Couldn't get the position of this recovered queue " + peerClusterZNode, e); } return startPosition; }
public ZkCoordinatedStateManager(Server server) { this.watcher = server.getZooKeeper(); splitLogWorkerCoordination = new ZkSplitLogWorkerCoordination(server.getServerName(), watcher); splitLogManagerCoordination = new ZKSplitLogManagerCoordination(server.getConfiguration(), watcher); }
() -> this.queueStorage.addWAL(server.getServerName(), peerId, walPath.getName())); src.enqueueLog(walPath);
/** * Transfer all the queues of the specified to this region server. First it tries to grab a lock * and if it works it will move the old queues and finally will delete the old queues. * <p> * It creates one old source for any type of source of the old rs. */ private void transferQueues(ServerName deadRS) { if (server.getServerName().equals(deadRS)) { // it's just us, give up return; } NodeFailoverWorker transfer = new NodeFailoverWorker(deadRS); try { this.executor.execute(transfer); } catch (RejectedExecutionException ex) { CompatibilitySingletonFactory.getInstance(MetricsReplicationSourceFactory.class) .getGlobalSource().incrFailedRecoveryQueue(); LOG.info("Cancelling the transfer of " + deadRS + " because of " + ex.getMessage()); } }
for (String wal : wals) { interruptOrAbortWhenFail( () -> this.queueStorage.removeWAL(server.getServerName(), queueId, wal));
for (NavigableSet<String> walsByGroup : wals.values()) { for (String wal : walsByGroup) { queueStorage.removeWAL(server.getServerName(), peerId, wal); oldSource.terminate(terminateMessage); oldSource.getSourceMetrics().clear(); queueStorage.removeQueue(server.getServerName(), queueId); walsByIdRecoveredQueues.remove(queueId); iter.remove();
/** * This method will log the current position to storage. And also clean old logs from the * replication queue. * @param source the replication source * @param entryBatch the wal entry batch we just shipped */ public void logPositionAndCleanOldLogs(ReplicationSourceInterface source, WALEntryBatch entryBatch) { String fileName = entryBatch.getLastWalPath().getName(); interruptOrAbortWhenFail(() -> this.queueStorage.setWALPosition(server.getServerName(), source.getQueueId(), fileName, entryBatch.getLastWalPosition(), entryBatch.getLastSeqIds())); cleanOldLogs(fileName, entryBatch.isEndOfFile(), source); }
@Test public void testRecoveredReplicationSourceShipperGetPosition() throws Exception { String walGroupId = "fake-wal-group-id"; ServerName serverName = ServerName.valueOf("www.example.com", 12006, 1524679704418L); ServerName deadServer = ServerName.valueOf("www.deadServer.com", 12006, 1524679704419L); PriorityBlockingQueue<Path> queue = new PriorityBlockingQueue<>(); queue.put(new Path("/www/html/test")); RecoveredReplicationSource source = Mockito.mock(RecoveredReplicationSource.class); Server server = Mockito.mock(Server.class); Mockito.when(server.getServerName()).thenReturn(serverName); Mockito.when(source.getServer()).thenReturn(server); Mockito.when(source.getServerWALsBelongTo()).thenReturn(deadServer); ReplicationQueueStorage storage = Mockito.mock(ReplicationQueueStorage.class); Mockito.when(storage.getWALPosition(Mockito.eq(serverName), Mockito.any(), Mockito.any())) .thenReturn(1001L); Mockito.when(storage.getWALPosition(Mockito.eq(deadServer), Mockito.any(), Mockito.any())) .thenReturn(-1L); conf.setInt("replication.source.maxretriesmultiplier", -1); RecoveredReplicationSourceShipper shipper = new RecoveredReplicationSourceShipper(conf, walGroupId, queue, source, storage); Assert.assertEquals(1001L, shipper.getStartPosition()); conf.unset("replication.source.maxretriesmultiplier"); } }
@Override public void run() { try { logZnodesMap = new HashMap<>(); List<String> queues = rq.getAllQueues(deadRS); for (String queue : queues) { Pair<String, SortedSet<String>> pair = rq.claimQueue(deadRS, queue, server.getServerName()); if (pair != null) { logZnodesMap.put(pair.getFirst(), pair.getSecond()); } } server.abort("Done with testing", null); } catch (Exception e) { LOG.error("Got exception while running NodeFailoverWorker", e); } finally { latch.countDown(); } }
files.add("log2"); for (String file : files) { queueStorage.addWAL(server.getServerName(), "1", file); ServerName serverName = server.getServerName(); List<String> unclaimed = queueStorage.getAllQueues(serverName); queueStorage.claimQueue(serverName, unclaimed.get(0), s1.getServerName()); queueStorage.removeReplicatorIfQueueIsEmpty(serverName); serverName = s1.getServerName(); unclaimed = queueStorage.getAllQueues(serverName); queueStorage.claimQueue(serverName, unclaimed.get(0), s2.getServerName()); queueStorage.removeReplicatorIfQueueIsEmpty(serverName); serverName = s2.getServerName(); unclaimed = queueStorage.getAllQueues(serverName); String queue3 = queueStorage.claimQueue(serverName, unclaimed.get(0), s3.getServerName()).getFirst(); queueStorage.removeReplicatorIfQueueIsEmpty(serverName); List<ServerName> result = replicationQueueInfo.getDeadRegionServers(); assertTrue(result.contains(server.getServerName())); assertTrue(result.contains(s1.getServerName())); assertTrue(result.contains(s2.getServerName()));
@Test public void testCleanupUnknownPeerZNode() throws Exception { Server server = new DummyServer("hostname2.example.org"); ReplicationQueueStorage rq = ReplicationStorageFactory .getReplicationQueueStorage(server.getZooKeeper(), server.getConfiguration()); // populate some znodes in the peer znode // add log to an unknown peer String group = "testgroup"; rq.addWAL(server.getServerName(), "2", group + ".log1"); rq.addWAL(server.getServerName(), "2", group + ".log2"); NodeFailoverWorker w1 = manager.new NodeFailoverWorker(server.getServerName()); w1.run(); // The log of the unknown peer should be removed from zk for (String peer : manager.getAllQueues()) { assertTrue(peer.startsWith("1")); } }
files.add("log2"); for (String file : files) { rq.addWAL(server.getServerName(), "1", file); DummyNodeFailoverWorker w1 = new DummyNodeFailoverWorker(server.getServerName(), s1); DummyNodeFailoverWorker w2 = new DummyNodeFailoverWorker(server.getServerName(), s2); DummyNodeFailoverWorker w3 = new DummyNodeFailoverWorker(server.getServerName(), s3);
files.add(file2); for (String file : files) { rq.addWAL(server.getServerName(), "1", file); rp1.init(); NodeFailoverWorker w1 = manager.new NodeFailoverWorker(server.getServerName()); w1.run(); assertEquals(1, manager.getWalsByIdRecoveredQueues().size()); String id = "1-" + server.getServerName().getServerName(); assertEquals(files, manager.getWalsByIdRecoveredQueues().get(id).get(group)); ReplicationSourceInterface source = mock(ReplicationSourceInterface.class);