private void matchAgainstExpectedTransactionIdIfAny( long transactionId, TransactionToApply tx ) { long expectedTransactionId = tx.transactionId(); if ( expectedTransactionId != TRANSACTION_ID_NOT_SPECIFIED ) { if ( transactionId != expectedTransactionId ) { IllegalStateException ex = new IllegalStateException( "Received " + tx.transactionRepresentation() + " with txId:" + expectedTransactionId + " to be applied, but appending it ended up generating an unexpected txId:" + transactionId ); databaseHealth.panic( ex ); throw ex; } } }
@Override public KernelTransaction beginTransaction( Transaction.Type type, LoginContext loginContext, long timeout ) throws TransactionFailureException { health.assertHealthy( TransactionFailureException.class ); KernelTransaction transaction = transactions.newInstance( type, loginContext, timeout ); transactionMonitor.transactionStarted(); return transaction; }
@Test public void healDatabaseWithoutCriticalErrors() { AssertableLogProvider logProvider = new AssertableLogProvider(); DatabaseHealth databaseHealth = new DatabaseHealth( mock( DatabasePanicEventGenerator.class ), logProvider.getLog( DatabaseHealth.class ) ); assertTrue( databaseHealth.isHealthy() ); databaseHealth.panic( new IOException( "Space exception." ) ); assertFalse( databaseHealth.isHealthy() ); assertTrue( databaseHealth.healed() ); logProvider.assertContainsLogCallContaining( "Database health set to OK" ); logProvider.assertNoMessagesContaining( "Database encountered a critical error and can't be healed. Restart required." ); }
@Test public void shouldGenerateDatabasePanicEvents() { // GIVEN DatabasePanicEventGenerator generator = mock( DatabasePanicEventGenerator.class ); DatabaseHealth databaseHealth = new DatabaseHealth( generator, NullLogProvider.getInstance().getLog( DatabaseHealth.class ) ); databaseHealth.healed(); // WHEN Exception cause = new Exception( "My own fault" ); databaseHealth.panic( cause ); databaseHealth.panic( cause ); // THEN verify( generator, times( 1 ) ).generateEvent( TX_MANAGER_NOT_OK, cause ); }
@Test public void shouldAlwaysShutdownLifeEvenWhenCheckPointingFails() throws Exception { // Given FileSystemAbstraction fs = this.fs.get(); PageCache pageCache = pageCacheRule.getPageCache( fs ); DatabaseHealth databaseHealth = mock( DatabaseHealth.class ); when( databaseHealth.isHealthy() ).thenReturn( true ); IOException ex = new IOException( "boom!" ); doThrow( ex ).when( databaseHealth ) .assertHealthy( IOException.class ); // <- this is a trick to simulate a failure during checkpointing Dependencies dependencies = new Dependencies(); dependencies.satisfyDependencies( databaseHealth ); NeoStoreDataSource dataSource = dsRule.getDataSource( dir.databaseLayout(), fs, pageCache, dependencies ); dataSource.start(); try { // When dataSource.stop(); fail( "it should have thrown" ); } catch ( LifecycleException e ) { // Then assertEquals( ex, e.getCause() ); } }
@Test public void databaseHealthShouldBeHealedOnStart() throws Throwable { NeoStoreDataSource theDataSource = null; try { DatabaseHealth databaseHealth = new DatabaseHealth( mock( DatabasePanicEventGenerator.class ), NullLogProvider.getInstance().getLog( DatabaseHealth.class ) ); Dependencies dependencies = new Dependencies(); dependencies.satisfyDependency( databaseHealth ); theDataSource = dsRule.getDataSource( dir.databaseLayout(), fs.get(), pageCacheRule.getPageCache( fs.get() ), dependencies ); databaseHealth.panic( new Throwable() ); theDataSource.start(); databaseHealth.assertHealthy( Throwable.class ); } finally { if ( theDataSource != null ) { theDataSource.stop(); theDataSource.shutdown(); } } }
private boolean hasCriticalFailure() { return !isHealthy() && Exceptions.contains( causeOfPanic, CRITICAL_EXCEPTIONS ); }
private static BatchingTransactionAppender createBatchingTransactionAppender( TransactionIdStore transactionIdStore, TransactionMetadataCache transactionMetadataCache, LogFiles logFiles ) { Log log = NullLog.getInstance(); KernelEventHandlers kernelEventHandlers = new KernelEventHandlers( log ); DatabasePanicEventGenerator panicEventGenerator = new DatabasePanicEventGenerator( kernelEventHandlers ); DatabaseHealth databaseHealth = new DatabaseHealth( panicEventGenerator, log ); LogRotationImpl logRotation = new LogRotationImpl( NOOP_LOGROTATION_MONITOR, logFiles, databaseHealth ); return new BatchingTransactionAppender( logFiles, logRotation, transactionMetadataCache, transactionIdStore, IdOrderingQueue.BYPASS, databaseHealth ); }
databaseHealth.healed();
@Test public void shouldLogDatabasePanicEvent() { // GIVEN AssertableLogProvider logProvider = new AssertableLogProvider(); DatabaseHealth databaseHealth = new DatabaseHealth( mock( DatabasePanicEventGenerator.class ), logProvider.getLog( DatabaseHealth.class ) ); databaseHealth.healed(); // WHEN String message = "Listen everybody... panic!"; Exception exception = new Exception( message ); databaseHealth.panic( exception ); // THEN logProvider.assertAtLeastOnce( inLog( DatabaseHealth.class ).error( is( "Database panic: The database has encountered a critical error, " + "and needs to be restarted. Please see database logs for more details." ), sameInstance( exception ) ) ); }
private Lifecycle lifecycleToTriggerCheckPointOnShutdown() { // Write new checkpoint in the log only if the kernel is healthy. // We cannot throw here since we need to shutdown without exceptions, // so let's make the checkpointing part of the life, so LifeSupport can handle exceptions properly return LifecycleAdapter.onShutdown( () -> { if ( databaseHealth.isHealthy() ) { // Flushing of neo stores happens as part of the checkpoint transactionLogModule.checkPointing().forceCheckPoint( new SimpleTriggerInfo( "database shutdown" ) ); } } ); }
@Override public void run() { if ( !dbHealthSupplier.get().isHealthy() ) { catchUpFuture.completeExceptionally( dbHealthSupplier.get().cause() ); } else if ( iAmAVotingMember() && caughtUpWithLeader() ) { catchUpFuture.complete( Boolean.TRUE ); } else { currentCatchupDelayInMs += SECONDS.toMillis( 1 ); long longerDelay = currentCatchupDelayInMs < maxCatchupLag ? currentCatchupDelayInMs : maxCatchupLag; jobScheduler.schedule( new JobScheduler.Group( MembershipWaiter.class.toString() ), this, longerDelay, MILLISECONDS ); } }
private static DatabaseHealth getDatabaseHealth() { DatabasePanicEventGenerator databasePanicEventGenerator = new DatabasePanicEventGenerator( new KernelEventHandlers( NullLog.getInstance() ) ); return new DatabaseHealth( databasePanicEventGenerator, NullLog.getInstance() ); }
databaseHealth.healed();
@Override public void checkPoint( LogPosition logPosition, LogCheckPointEvent logCheckPointEvent ) throws IOException { try { // Synchronized with logFile to get absolute control over concurrent rotations happening synchronized ( logFile ) { transactionLogWriter.checkPoint( logPosition ); } } catch ( Throwable cause ) { databaseHealth.panic( cause ); throw cause; } forceAfterAppend( logCheckPointEvent ); }
@Test public void databaseWithCriticalErrorsCanNotBeHealed() { AssertableLogProvider logProvider = new AssertableLogProvider(); DatabaseHealth databaseHealth = new DatabaseHealth( mock( DatabasePanicEventGenerator.class ), logProvider.getLog( DatabaseHealth.class ) ); assertTrue( databaseHealth.isHealthy() ); IOException criticalException = new IOException( "Space exception.", new OutOfMemoryError( "Out of memory." ) ); databaseHealth.panic( criticalException ); assertFalse( databaseHealth.isHealthy() ); assertFalse( databaseHealth.healed() ); logProvider.assertNoMessagesContaining( "Database health set to OK" ); logProvider.assertContainsLogCallContaining( "Database encountered a critical error and can't be healed. Restart required." ); } }
private void doRotate() throws IOException { long currentVersion = logFiles.getHighestLogVersion(); /* * In order to rotate the current log file safely we need to assert that the kernel is still * at full health. In case of a panic this rotation will be aborted, which is the safest alternative. */ databaseHealth.assertHealthy( IOException.class ); monitor.startedRotating( currentVersion ); logFile.rotate(); monitor.finishedRotating( currentVersion ); } }
@Test public void flushOfThePageCacheOnShutdownDoesNotHappenIfTheDbIsUnhealthy() throws Throwable { DatabaseHealth health = mock( DatabaseHealth.class ); when( health.isHealthy() ).thenReturn( false ); PageCache pageCache = spy( pageCacheRule.getPageCache( fs.get() ) ); Dependencies dependencies = new Dependencies(); dependencies.satisfyDependency( health ); NeoStoreDataSource ds = dsRule.getDataSource( dir.databaseLayout(), fs.get(), pageCache, dependencies ); ds.start(); verify( pageCache, never() ).flushAndForce(); ds.stop(); ds.shutdown(); verify( pageCache, never() ).flushAndForce( IOLimiter.UNLIMITED ); }