private PhysicalLogVersionedStoreChannel tryOpenStoreChannel( LogPosition currentPosition ) { try { return logFiles.openForVersion( currentPosition.getLogVersion() ); } catch ( IOException e ) { return null; } }
@Override public void publishAsClosed() { transactionIdStore.transactionClosed( transactionId, logPosition.getLogVersion(), logPosition.getByteOffset() ); }
@Override public boolean next() throws IOException { while ( currentLogTransactionCursor == null || !currentLogTransactionCursor.next() ) { // We've run out of transactions in this log version, back up to a previous one currentVersion--; if ( currentVersion < backToPosition.getLogVersion() ) { return false; } closeCurrent(); LogPosition position = currentVersion > backToPosition.getLogVersion() ? start( currentVersion ) : backToPosition; currentLogTransactionCursor = cursorFactory.apply( position ); } return true; }
void writeLastTxLogPosition( DatabaseLayout migrationStructure, LogPosition lastTxLogPosition ) throws IOException { writeTxLogCounters( fileSystem, lastTxLogPositionFile( migrationStructure ), lastTxLogPosition.getLogVersion(), lastTxLogPosition.getByteOffset() ); }
public void writeCheckPointEntry( LogPosition logPosition ) throws IOException { writeLogEntryHeader( CHECK_POINT, channel ); channel.putLong( logPosition.getLogVersion() ). putLong( logPosition.getByteOffset() ); } }
@Override public ReadableLogChannel getReader( LogPosition position, LogVersionBridge logVersionBridge ) throws IOException { PhysicalLogVersionedStoreChannel logChannel = logFiles.openForVersion( position.getLogVersion() ); logChannel.position( position.getByteOffset() ); return new ReadAheadLogChannel( logChannel, logVersionBridge ); }
private void mockTxIdStore() { long[] triggerCommittedTransaction = {transactionId, logPosition.getLogVersion(), logPosition.getByteOffset()}; when( txIdStore.getLastClosedTransaction() ).thenReturn( triggerCommittedTransaction ); when( txIdStore.getLastClosedTransactionId() ).thenReturn( initialTransactionId, transactionId, transactionId ); }
/** * Truncate all transaction logs after provided position. Log version specified in a position will be * truncated to provided byte offset, any subsequent log files will be deleted. Backup copy of removed data will * be stored in separate archive. * @param positionAfterLastRecoveredTransaction position after last recovered transaction * @throws IOException */ public void truncate( LogPosition positionAfterLastRecoveredTransaction ) throws IOException { long recoveredTransactionLogVersion = positionAfterLastRecoveredTransaction.getLogVersion(); long recoveredTransactionOffset = positionAfterLastRecoveredTransaction.getByteOffset(); if ( isRecoveredLogCorrupted( recoveredTransactionLogVersion, recoveredTransactionOffset ) || haveMoreRecentLogFiles( recoveredTransactionLogVersion ) ) { backupCorruptedContent( recoveredTransactionLogVersion, recoveredTransactionOffset ); truncateLogFiles( recoveredTransactionLogVersion, recoveredTransactionOffset ); } }
@Test public void shouldForceCheckPointAlways() throws Throwable { // Given CheckPointerImpl checkPointing = checkPointer(); when( threshold.isCheckPointingNeeded( anyLong(), eq( INFO ) ) ).thenReturn( false ); mockTxIdStore(); checkPointing.start(); // When long txId = checkPointing.forceCheckPoint( INFO ); // Then assertEquals( transactionId, txId ); verify( storageEngine, times( 1 ) ).flushAndForce( limiter ); verify( health, times( 2 ) ).assertHealthy( IOException.class ); verify( appender, times( 1 ) ).checkPoint( eq( logPosition ), any( LogCheckPointEvent.class ) ); verify( threshold, times( 1 ) ).initialize( initialTransactionId ); verify( threshold, times( 1 ) ).checkPointHappened( transactionId ); verify( threshold, never() ).isCheckPointingNeeded( transactionId, INFO ); verify( logPruning, times( 1 ) ).pruneLogs( logPosition.getLogVersion() ); verifyZeroInteractions( tracer ); verifyNoMoreInteractions( storageEngine, health, appender, threshold, tracer ); }
@Test public void shouldFlushIfItIsNeeded() throws Throwable { // Given CheckPointerImpl checkPointing = checkPointer(); when( threshold.isCheckPointingNeeded( anyLong(), eq( INFO ) ) ).thenReturn( true, false ); mockTxIdStore(); checkPointing.start(); // When long txId = checkPointing.checkPointIfNeeded( INFO ); // Then assertEquals( transactionId, txId ); verify( storageEngine, times( 1 ) ).flushAndForce( limiter ); verify( health, times( 2 ) ).assertHealthy( IOException.class ); verify( appender, times( 1 ) ).checkPoint( eq( logPosition ), any( LogCheckPointEvent.class ) ); verify( threshold, times( 1 ) ).initialize( initialTransactionId ); verify( threshold, times( 1 ) ).checkPointHappened( transactionId ); verify( threshold, times( 1 ) ).isCheckPointingNeeded( transactionId, INFO ); verify( logPruning, times( 1 ) ).pruneLogs( logPosition.getLogVersion() ); verify( tracer, times( 1 ) ).beginCheckPoint(); verifyNoMoreInteractions( storageEngine, health, appender, threshold, tracer ); }
@Test public void shouldCheckPointAlwaysWhenThereIsNoRunningCheckPoint() throws Throwable { // Given CheckPointerImpl checkPointing = checkPointer(); when( threshold.isCheckPointingNeeded( anyLong(), eq( INFO ) ) ).thenReturn( false ); mockTxIdStore(); checkPointing.start(); // When long txId = checkPointing.tryCheckPoint( INFO ); // Then assertEquals( transactionId, txId ); verify( storageEngine, times( 1 ) ).flushAndForce( limiter ); verify( health, times( 2 ) ).assertHealthy( IOException.class ); verify( appender, times( 1 ) ).checkPoint( eq( logPosition ), any( LogCheckPointEvent.class ) ); verify( threshold, times( 1 ) ).initialize( initialTransactionId ); verify( threshold, times( 1 ) ).checkPointHappened( transactionId ); verify( threshold, never() ).isCheckPointingNeeded( transactionId, INFO ); verify( logPruning, times( 1 ) ).pruneLogs( logPosition.getLogVersion() ); verifyZeroInteractions( tracer ); verifyNoMoreInteractions( storageEngine, health, appender, threshold, tracer ); }
private ThrowingFunction<LogPosition,TransactionCursor,IOException> log( int... transactionCounts ) throws IOException { long baseOffset = LogPosition.start( 0 ).getByteOffset(); @SuppressWarnings( "unchecked" ) ThrowingFunction<LogPosition,TransactionCursor,IOException> result = mock( ThrowingFunction.class ); AtomicLong txId = new AtomicLong( 0 ); CommittedTransactionRepresentation[][] logs = new CommittedTransactionRepresentation[transactionCounts.length][]; for ( int logVersion = 0; logVersion < transactionCounts.length; logVersion++ ) { logs[logVersion] = transactions( transactionCounts[logVersion], txId ); } when( result.apply( any( LogPosition.class ) ) ).thenAnswer( invocation -> { LogPosition position = invocation.getArgument( 0 ); if ( position == null ) { // A mockito issue when calling the "when" methods, I believe return null; } // For simplicity the offset means, in this test, the array offset CommittedTransactionRepresentation[] transactions = logs[toIntExact( position.getLogVersion() )]; CommittedTransactionRepresentation[] subset = copyOfRange( transactions, toIntExact( position.getByteOffset() - baseOffset ), transactions.length ); ArrayUtil.reverse( subset ); return given( subset ); } ); return result; }
@Override public void transactionsRecovered( CommittedTransactionRepresentation lastRecoveredTransaction, LogPosition positionAfterLastRecoveredTransaction ) { long recoveredTransactionLogVersion = positionAfterLastRecoveredTransaction.getLogVersion(); long recoveredTransactionOffset = positionAfterLastRecoveredTransaction.getByteOffset(); if ( lastRecoveredTransaction != null ) { LogEntryCommit commitEntry = lastRecoveredTransaction.getCommitEntry(); transactionIdStore.setLastCommittedAndClosedTransactionId( commitEntry.getTxId(), LogEntryStart.checksum( lastRecoveredTransaction.getStartEntry() ), commitEntry.getTimeWritten(), recoveredTransactionOffset, recoveredTransactionLogVersion ); } logVersionRepository.setCurrentLogVersion( recoveredTransactionLogVersion ); }
logPruning.pruneLogs( logPosition.getLogVersion() ); lastCheckPointedTx = lastClosedTransactionId; return lastClosedTransactionId;
@Test public void shouldReadACheckPointLogEntry() throws IOException { // given LogEntryVersion version = LogEntryVersion.CURRENT; final LogPosition logPosition = new LogPosition( 42, 43 ); final CheckPoint checkPoint = new CheckPoint( version, logPosition ); final InMemoryClosableChannel channel = new InMemoryClosableChannel(); channel.put( version.byteCode() ); channel.put( LogEntryByteCodes.CHECK_POINT ); channel.putLong( logPosition.getLogVersion() ); channel.putLong( logPosition.getByteOffset() ); // when final LogEntry logEntry = logEntryReader.readLogEntry( channel ); // then assertEquals( checkPoint, logEntry ); }
private void extractTransactionalInformationFromLogs( String path, File customLogLocation, DatabaseLayout databaseLayout, File storeDir ) throws IOException { LogService logService = new SimpleLogService( NullLogProvider.getInstance(), NullLogProvider.getInstance() ); File neoStore = databaseLayout.metadataStore(); GraphDatabaseService database = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( storeDir ) .setConfig( logical_logs_location, path ).newGraphDatabase(); for ( int i = 0; i < 10; i++ ) { try ( Transaction transaction = database.beginTx() ) { Node node = database.createNode(); transaction.success(); } } database.shutdown(); MetaDataStore.setRecord( pageCache, neoStore, MetaDataStore.Position.LAST_CLOSED_TRANSACTION_LOG_VERSION, MetaDataRecordFormat.FIELD_NOT_PRESENT ); Config config = Config.defaults( logical_logs_location, path ); StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService, jobScheduler ); LogPosition logPosition = migrator.extractTransactionLogPosition( neoStore, databaseLayout, 100 ); File[] logFiles = customLogLocation.listFiles(); assertNotNull( logFiles ); assertEquals( 0, logPosition.getLogVersion() ); assertEquals( logFiles[0].length(), logPosition.getByteOffset() ); }
public static void removeCheckPointFromTxLog( FileSystemAbstraction fileSystem, File databaseDirectory ) throws IOException { LogFiles logFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( databaseDirectory, fileSystem ).build(); LogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader = new VersionAwareLogEntryReader<>(); LogTailScanner tailScanner = new LogTailScanner( logFiles, logEntryReader, new Monitors() ); LogTailScanner.LogTailInformation logTailInformation = tailScanner.getTailInformation(); if ( logTailInformation.commitsAfterLastCheckpoint() ) { // done already return; } // let's assume there is at least a checkpoint assertNotNull( logTailInformation.lastCheckPoint ); LogPosition logPosition = logTailInformation.lastCheckPoint.getLogPosition(); File logFile = logFiles.getLogFileForVersion( logPosition.getLogVersion() ); fileSystem.truncate( logFile, logPosition.getByteOffset() ); } }
@Test public void shouldParseCheckPointEntry() throws IOException { // given final CheckPoint checkPoint = new CheckPoint( new LogPosition( 43, 44 ) ); final InMemoryClosableChannel channel = new InMemoryClosableChannel(); channel.putLong( checkPoint.getLogPosition().getLogVersion() ); channel.putLong( checkPoint.getLogPosition().getByteOffset() ); channel.getCurrentPosition( marker ); // when final LogEntryParser parser = version.entryParser( LogEntryByteCodes.CHECK_POINT ); final LogEntry logEntry = parser.parse( version, channel, marker, commandReader ); // then assertEquals( checkPoint, logEntry ); assertFalse( parser.skip() ); }
private void appendCheckpoint( LogEntryVersion logVersion ) throws IOException { PageCache pageCache = pageCacheRule.getPageCache( fs ); VersionAwareLogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader = new VersionAwareLogEntryReader<>(); LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDirectory.databaseLayout(), fs, pageCache ).withLogEntryReader( logEntryReader ).build(); LogTailScanner tailScanner = new LogTailScanner( logFiles, logEntryReader, new Monitors() ); LogTailScanner.LogTailInformation tailInformation = tailScanner.getTailInformation(); try ( Lifespan lifespan = new Lifespan( logFiles ) ) { FlushablePositionAwareChannel channel = logFiles.getLogFile().getWriter(); LogPosition logPosition = tailInformation.lastCheckPoint.getLogPosition(); // Fake record channel.put( logVersion.byteCode() ) .put( CHECK_POINT ) .putLong( logPosition.getLogVersion() ) .putLong( logPosition.getByteOffset() ); channel.prepareForFlush().flush(); } } }
@Test public void skipLogFileWithoutHeader() throws IOException { FileSystemAbstraction fs = fileSystemRule.get(); LogFiles logFiles = LogFilesBuilder.builder( directory.databaseLayout(), fs ) .withTransactionIdStore( transactionIdStore ) .withLogVersionRepository( logVersionRepository ).build(); life.add( logFiles ); life.start(); // simulate new file without header presence logVersionRepository.incrementAndGetVersion(); fs.create( logFiles.getLogFileForVersion( logVersionRepository.getCurrentLogVersion() ) ).close(); transactionIdStore.transactionCommitted( 5L, 5L, 5L ); PhysicalLogicalTransactionStore.LogVersionLocator versionLocator = new PhysicalLogicalTransactionStore.LogVersionLocator( 4L ); logFiles.accept( versionLocator ); LogPosition logPosition = versionLocator.getLogPosition(); assertEquals( 1, logPosition.getLogVersion() ); }