private boolean hasCriticalFailure() { return !isHealthy() && Exceptions.contains( causeOfPanic, CRITICAL_EXCEPTIONS ); }
private Lifecycle lifecycleToTriggerCheckPointOnShutdown() { // Write new checkpoint in the log only if the kernel is healthy. // We cannot throw here since we need to shutdown without exceptions, // so let's make the checkpointing part of the life, so LifeSupport can handle exceptions properly return LifecycleAdapter.onShutdown( () -> { if ( databaseHealth.isHealthy() ) { // Flushing of neo stores happens as part of the checkpoint transactionLogModule.checkPointing().forceCheckPoint( new SimpleTriggerInfo( "database shutdown" ) ); } } ); }
@Test public void databaseWithCriticalErrorsCanNotBeHealed() { AssertableLogProvider logProvider = new AssertableLogProvider(); DatabaseHealth databaseHealth = new DatabaseHealth( mock( DatabasePanicEventGenerator.class ), logProvider.getLog( DatabaseHealth.class ) ); assertTrue( databaseHealth.isHealthy() ); IOException criticalException = new IOException( "Space exception.", new OutOfMemoryError( "Out of memory." ) ); databaseHealth.panic( criticalException ); assertFalse( databaseHealth.isHealthy() ); assertFalse( databaseHealth.healed() ); logProvider.assertNoMessagesContaining( "Database health set to OK" ); logProvider.assertContainsLogCallContaining( "Database encountered a critical error and can't be healed. Restart required." ); } }
@Test public void flushOfThePageCacheOnShutdownDoesNotHappenIfTheDbIsUnhealthy() throws Throwable { DatabaseHealth health = mock( DatabaseHealth.class ); when( health.isHealthy() ).thenReturn( false ); PageCache pageCache = spy( pageCacheRule.getPageCache( fs.get() ) ); Dependencies dependencies = new Dependencies(); dependencies.satisfyDependency( health ); NeoStoreDataSource ds = dsRule.getDataSource( dir.databaseLayout(), fs.get(), pageCache, dependencies ); ds.start(); verify( pageCache, never() ).flushAndForce(); ds.stop(); ds.shutdown(); verify( pageCache, never() ).flushAndForce( IOLimiter.UNLIMITED ); }
@Test public void healDatabaseWithoutCriticalErrors() { AssertableLogProvider logProvider = new AssertableLogProvider(); DatabaseHealth databaseHealth = new DatabaseHealth( mock( DatabasePanicEventGenerator.class ), logProvider.getLog( DatabaseHealth.class ) ); assertTrue( databaseHealth.isHealthy() ); databaseHealth.panic( new IOException( "Space exception." ) ); assertFalse( databaseHealth.isHealthy() ); assertTrue( databaseHealth.healed() ); logProvider.assertContainsLogCallContaining( "Database health set to OK" ); logProvider.assertNoMessagesContaining( "Database encountered a critical error and can't be healed. Restart required." ); }
@Test public void shouldAlwaysShutdownLifeEvenWhenCheckPointingFails() throws Exception { // Given FileSystemAbstraction fs = this.fs.get(); PageCache pageCache = pageCacheRule.getPageCache( fs ); DatabaseHealth databaseHealth = mock( DatabaseHealth.class ); when( databaseHealth.isHealthy() ).thenReturn( true ); IOException ex = new IOException( "boom!" ); doThrow( ex ).when( databaseHealth ) .assertHealthy( IOException.class ); // <- this is a trick to simulate a failure during checkpointing Dependencies dependencies = new Dependencies(); dependencies.satisfyDependencies( databaseHealth ); NeoStoreDataSource dataSource = dsRule.getDataSource( dir.databaseLayout(), fs, pageCache, dependencies ); dataSource.start(); try { // When dataSource.stop(); fail( "it should have thrown" ); } catch ( LifecycleException e ) { // Then assertEquals( ex, e.getCause() ); } }
private boolean hasCriticalFailure() { return !isHealthy() && Exceptions.contains( causeOfPanic, CRITICAL_EXCEPTIONS ); }
public void run() { if ( !dbHealthSupplier.get().isHealthy() ) { catchUpFuture.completeExceptionally( dbHealthSupplier.get().cause() ); } else if ( iAmAVotingMember() && caughtUpWithLeader() ) { catchUpFuture.complete( true ); } }
@Override public void run() { if ( !dbHealthSupplier.get().isHealthy() ) { catchUpFuture.completeExceptionally( dbHealthSupplier.get().cause() ); } else if ( iAmAVotingMember() && caughtUpWithLeader() ) { catchUpFuture.complete( Boolean.TRUE ); } else { currentCatchupDelayInMs += SECONDS.toMillis( 1 ); long longerDelay = currentCatchupDelayInMs < maxCatchupLag ? currentCatchupDelayInMs : maxCatchupLag; jobScheduler.schedule( new JobScheduler.Group( MembershipWaiter.class.toString() ), this, longerDelay, MILLISECONDS ); } }
private Lifecycle lifecycleToTriggerCheckPointOnShutdown() { // Write new checkpoint in the log only if the kernel is healthy. // We cannot throw here since we need to shutdown without exceptions, // so let's make the checkpointing part of the life, so LifeSupport can handle exceptions properly return LifecycleAdapter.onShutdown( () -> { if ( databaseHealth.isHealthy() ) { // Flushing of neo stores happens as part of the checkpoint transactionLogModule.checkPointing().forceCheckPoint( new SimpleTriggerInfo( "database shutdown" ) ); } } ); }