private void matchAgainstExpectedTransactionIdIfAny( long transactionId, TransactionToApply tx ) { long expectedTransactionId = tx.transactionId(); if ( expectedTransactionId != TRANSACTION_ID_NOT_SPECIFIED ) { if ( transactionId != expectedTransactionId ) { IllegalStateException ex = new IllegalStateException( "Received " + tx.transactionRepresentation() + " with txId:" + expectedTransactionId + " to be applied, but appending it ended up generating an unexpected txId:" + transactionId ); databaseHealth.panic( ex ); throw ex; } } }
@Override public void checkPoint( LogPosition logPosition, LogCheckPointEvent logCheckPointEvent ) throws IOException { try { // Synchronized with logFile to get absolute control over concurrent rotations happening synchronized ( logFile ) { transactionLogWriter.checkPoint( logPosition ); } } catch ( Throwable cause ) { databaseHealth.panic( cause ); throw cause; } forceAfterAppend( logCheckPointEvent ); }
private void forceLog( LogForceEvents logForceEvents ) throws IOException { ThreadLink links = threadLinkHead.getAndSet( ThreadLink.END ); try ( LogForceEvent logForceEvent = logForceEvents.beginLogForce() ) { force(); } catch ( final Throwable panic ) { databaseHealth.panic( panic ); throw panic; } finally { unparkAll( links ); } }
health.panic( combinedFailure ); throw combinedFailure;
@Test public void shouldGenerateDatabasePanicEvents() { // GIVEN DatabasePanicEventGenerator generator = mock( DatabasePanicEventGenerator.class ); DatabaseHealth databaseHealth = new DatabaseHealth( generator, NullLogProvider.getInstance().getLog( DatabaseHealth.class ) ); databaseHealth.healed(); // WHEN Exception cause = new Exception( "My own fault" ); databaseHealth.panic( cause ); databaseHealth.panic( cause ); // THEN verify( generator, times( 1 ) ).generateEvent( TX_MANAGER_NOT_OK, cause ); }
@Test public void databasePanicIsRaisedWhenTxApplicationFails() throws Throwable { RecordStorageEngine engine = buildRecordStorageEngine(); Exception applicationError = executeFailingTransaction( engine ); ArgumentCaptor<Exception> captor = ArgumentCaptor.forClass( Exception.class ); verify( databaseHealth ).panic( captor.capture() ); Throwable exception = captor.getValue(); if ( exception instanceof KernelException ) { assertThat( ((KernelException) exception).status(), is( Status.General.UnknownError ) ); exception = exception.getCause(); } assertThat( exception, is( applicationError ) ); }
@Test public void shouldKernelPanicIfTransactionIdsMismatch() throws Throwable { // Given BatchingTransactionAppender appender = life.add( createTransactionAppender() ); when( transactionIdStore.nextCommittingTransactionId() ).thenReturn( 42L ); TransactionToApply batch = new TransactionToApply( mock( TransactionRepresentation.class ), 43L ); // When try { appender.append( batch, LogAppendEvent.NULL ); fail( "should have thrown " ); } catch ( IllegalStateException ex ) { // Then verify( databaseHealth, times( 1 ) ).panic( ex ); } }
verify( transactionIdStore, times( 1 ) ).nextCommittingTransactionId(); verify( transactionIdStore, never() ).transactionClosed( eq( txId ), anyLong(), anyLong() ); verify( databaseHealth ).panic( failure );
verify( transactionIdStore, times( 1 ) ).nextCommittingTransactionId(); verify( transactionIdStore, never() ).transactionClosed( eq( txId ), anyLong(), anyLong() ); verify( databaseHealth ).panic( failure );
@Test public void shouldKernelPanicIfNotAbleToWriteACheckPoint() throws Throwable { // Given IOException ioex = new IOException( "boom!" ); FlushablePositionAwareChannel channel = mock( FlushablePositionAwareChannel.class, RETURNS_MOCKS ); when( channel.put( anyByte() ) ).thenReturn( channel ); when( channel.putLong( anyLong() ) ).thenThrow( ioex ); when( channel.put( anyByte() ) ).thenThrow( ioex ); when( logFile.getWriter() ).thenReturn( channel ); BatchingTransactionAppender appender = life.add( createTransactionAppender() ); // When try { appender.checkPoint( new LogPosition( 0L, 0L ), LogCheckPointEvent.NULL ); fail( "should have thrown " ); } catch ( IOException ex ) { assertEquals( ioex, ex ); } // Then verify( databaseHealth, times( 1 ) ).panic( ioex ); }
@Override public void apply( CommandsToApply batch, TransactionApplicationMode mode ) throws Exception { // Have these command appliers as separate try-with-resource to have better control over // point between closing this and the locks above try ( IndexActivator indexActivator = new IndexActivator( indexingService ); LockGroup locks = new LockGroup(); BatchTransactionApplier batchApplier = applier( mode, indexActivator ) ) { while ( batch != null ) { try ( TransactionApplier txApplier = batchApplier.startTx( batch, locks ) ) { batch.accept( txApplier ); } batch = batch.next(); } } catch ( Throwable cause ) { TransactionApplyKernelException kernelException = new TransactionApplyKernelException( cause, "Failed to apply transaction: %s", batch ); databaseHealth.panic( kernelException ); throw kernelException; } }
@Test public void shouldCausePanicAfterSomeFailures() throws Throwable { // GIVEN RuntimeException[] failures = new RuntimeException[] { new RuntimeException( "First" ), new RuntimeException( "Second" ), new RuntimeException( "Third" ) }; when( checkPointer.checkPointIfNeeded( any( TriggerInfo.class ) ) ).thenThrow( failures ); CheckPointScheduler scheduler = new CheckPointScheduler( checkPointer, ioLimiter, jobScheduler, 1, health ); scheduler.start(); // WHEN for ( int i = 0; i < CheckPointScheduler.MAX_CONSECUTIVE_FAILURES_TOLERANCE - 1; i++ ) { jobScheduler.runJob(); verifyZeroInteractions( health ); } try { jobScheduler.runJob(); fail( "Should have failed" ); } catch ( UnderlyingStorageException e ) { // THEN assertEquals( Iterators.asSet( failures ), Iterators.asSet( e.getSuppressed() ) ); verify( health ).panic( e ); } }
private void dirtyDatabase() throws IOException { db = startDatabase(); DatabaseHealth databaseHealth = databaseHealth( db ); index( db ); someData( db ); checkpoint( db ); someData( db ); databaseHealth.panic( new Throwable( "Trigger recovery on next startup" ) ); db.shutdown(); db = null; }
@Test public void shouldLogDatabasePanicEvent() { // GIVEN AssertableLogProvider logProvider = new AssertableLogProvider(); DatabaseHealth databaseHealth = new DatabaseHealth( mock( DatabasePanicEventGenerator.class ), logProvider.getLog( DatabaseHealth.class ) ); databaseHealth.healed(); // WHEN String message = "Listen everybody... panic!"; Exception exception = new Exception( message ); databaseHealth.panic( exception ); // THEN logProvider.assertAtLeastOnce( inLog( DatabaseHealth.class ).error( is( "Database panic: The database has encountered a critical error, " + "and needs to be restarted. Please see database logs for more details." ), sameInstance( exception ) ) ); }
@Test public void databaseWithCriticalErrorsCanNotBeHealed() { AssertableLogProvider logProvider = new AssertableLogProvider(); DatabaseHealth databaseHealth = new DatabaseHealth( mock( DatabasePanicEventGenerator.class ), logProvider.getLog( DatabaseHealth.class ) ); assertTrue( databaseHealth.isHealthy() ); IOException criticalException = new IOException( "Space exception.", new OutOfMemoryError( "Out of memory." ) ); databaseHealth.panic( criticalException ); assertFalse( databaseHealth.isHealthy() ); assertFalse( databaseHealth.healed() ); logProvider.assertNoMessagesContaining( "Database health set to OK" ); logProvider.assertContainsLogCallContaining( "Database encountered a critical error and can't be healed. Restart required." ); } }
@Test public void healDatabaseWithoutCriticalErrors() { AssertableLogProvider logProvider = new AssertableLogProvider(); DatabaseHealth databaseHealth = new DatabaseHealth( mock( DatabasePanicEventGenerator.class ), logProvider.getLog( DatabaseHealth.class ) ); assertTrue( databaseHealth.isHealthy() ); databaseHealth.panic( new IOException( "Space exception." ) ); assertFalse( databaseHealth.isHealthy() ); assertTrue( databaseHealth.healed() ); logProvider.assertContainsLogCallContaining( "Database health set to OK" ); logProvider.assertNoMessagesContaining( "Database encountered a critical error and can't be healed. Restart required." ); }
@Test public void panicOnExceptionDuringCommandsApply() { IllegalStateException failure = new IllegalStateException( "Too many open files" ); RecordStorageEngine engine = storageEngineRule.getWith( fsRule.get(), pageCacheRule.getPageCache( fsRule.get() ), testDirectory.databaseLayout() ) .databaseHealth( databaseHealth ) .transactionApplierTransformer( facade -> transactionApplierFacadeTransformer( facade, failure ) ) .build(); CommandsToApply commandsToApply = mock( CommandsToApply.class ); try { engine.apply( commandsToApply, TransactionApplicationMode.INTERNAL ); fail( "Exception expected" ); } catch ( Exception exception ) { assertSame( failure, Exceptions.rootCause( exception ) ); } verify( databaseHealth ).panic( any( Throwable.class ) ); }
@Test public void databaseHealthShouldBeHealedOnStart() throws Throwable { NeoStoreDataSource theDataSource = null; try { DatabaseHealth databaseHealth = new DatabaseHealth( mock( DatabasePanicEventGenerator.class ), NullLogProvider.getInstance().getLog( DatabaseHealth.class ) ); Dependencies dependencies = new Dependencies(); dependencies.satisfyDependency( databaseHealth ); theDataSource = dsRule.getDataSource( dir.databaseLayout(), fs.get(), pageCacheRule.getPageCache( fs.get() ), dependencies ); databaseHealth.panic( new Throwable() ); theDataSource.start(); databaseHealth.assertHealthy( Throwable.class ); } finally { if ( theDataSource != null ) { theDataSource.stop(); theDataSource.shutdown(); } } }
databaseHealth.panic( panic ); throw panic;
healthOf( db ).panic( txFailure.getCause() ); // panic the db again to force recovery on the next startup