private LogTailScanner getTailScanner( LogFiles logFiles, LogEntryReader<ReadableClosablePositionAwareChannel> reader ) { return new LogTailScanner( logFiles, reader, monitors, false ); }
public static void check( LogTailScanner tailScanner, Config config ) throws UpgradeNotAllowedByConfigurationException { if ( !config.get( GraphDatabaseSettings.allow_upgrade ) ) { // The user doesn't want us to upgrade the store. LogEntryVersion latestLogEntryVersion = tailScanner.getTailInformation().latestLogEntryVersion; if ( latestLogEntryVersion != null && LogEntryVersion.moreRecentVersionExists( latestLogEntryVersion ) ) { String message = "The version you're upgrading to is using a new transaction log format. This is a " + "non-reversible upgrade and you wont be able to downgrade after starting"; throw new UpgradeNotAllowedByConfigurationException( message ); } } } }
protected LogTailInformation checkpointTailInformation( long highestLogVersion, LogEntryStart latestStartEntry, long oldestVersionFound, LogEntryVersion latestLogEntryVersion, CheckPoint latestCheckPoint, boolean corruptedTransactionLogs ) throws IOException { LogPosition checkPointLogPosition = latestCheckPoint.getLogPosition(); ExtractedTransactionRecord transactionRecord = extractFirstTxIdAfterPosition( checkPointLogPosition, highestLogVersion ); long firstTxIdAfterPosition = transactionRecord.getId(); boolean startRecordAfterCheckpoint = (firstTxIdAfterPosition != NO_TRANSACTION_ID) || ((latestStartEntry != null) && (latestStartEntry.getStartPosition().compareTo( latestCheckPoint.getLogPosition() ) >= 0)); boolean corruptedLogs = transactionRecord.isFailure() || corruptedTransactionLogs; return new LogTailInformation( latestCheckPoint, corruptedLogs || startRecordAfterCheckpoint, firstTxIdAfterPosition, oldestVersionFound, highestLogVersion, latestLogEntryVersion ); }
public static void removeCheckPointFromTxLog( FileSystemAbstraction fileSystem, File databaseDirectory ) throws IOException { LogFiles logFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( databaseDirectory, fileSystem ).build(); LogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader = new VersionAwareLogEntryReader<>(); LogTailScanner tailScanner = new LogTailScanner( logFiles, logEntryReader, new Monitors() ); LogTailScanner.LogTailInformation logTailInformation = tailScanner.getTailInformation(); if ( logTailInformation.commitsAfterLastCheckpoint() ) { // done already return; } // let's assume there is at least a checkpoint assertNotNull( logTailInformation.lastCheckPoint ); LogPosition logPosition = logTailInformation.lastCheckPoint.getLogPosition(); File logFile = logFiles.getLogFileForVersion( logPosition.getLogVersion() ); fileSystem.truncate( logFile, logPosition.getByteOffset() ); } }
if ( hasUnreadableBytes( channel, maxEntryReadPosition ) ) return checkpointTailInformation( highestLogVersion, latestStartEntry, oldestVersionFound, latestLogEntryVersion, latestCheckPoint, corruptedTransactionLogs );
@Test public void bigFileLatestCheckpointFindsStartAfter() throws Throwable { long firstTxAfterCheckpoint = Integer.MAX_VALUE + 4L; LogTailScanner tailScanner = new FirstTxIdConfigurableTailScanner( firstTxAfterCheckpoint, logFiles, reader, monitors ); LogEntryStart startEntry = new LogEntryStart( 1, 2, 3L, 4L, new byte[]{5, 6}, new LogPosition( endLogVersion, Integer.MAX_VALUE + 17L ) ); CheckPoint checkPoint = new CheckPoint( new LogPosition( endLogVersion, 16L ) ); LogTailInformation logTailInformation = tailScanner.checkpointTailInformation( endLogVersion, startEntry, endLogVersion, latestLogEntryVersion, checkPoint, false); assertLatestCheckPoint( true, true, firstTxAfterCheckpoint, endLogVersion, logTailInformation ); }
/** * Collects information about the tail of the transaction log, i.e. last checkpoint, last entry etc. * Since this is an expensive task we do it once and reuse the result. This method is thus lazy and the first one * calling it will take the hit. * <p> * This is only intended to be used during startup. If you need to track the state of the tail, that can be done more * efficiently at runtime, and this method should then only be used to restore said state. * * @return snapshot of the state of the transaction logs tail at startup. * @throws UnderlyingStorageException if any errors occurs while parsing the transaction logs */ public LogTailInformation getTailInformation() throws UnderlyingStorageException { if ( logTailInformation == null ) { try { logTailInformation = findLogTail(); } catch ( IOException e ) { throw new UnderlyingStorageException( "Error encountered while parsing transaction logs", e ); } } return logTailInformation; }
private void appendCheckpoint( LogEntryVersion logVersion ) throws IOException { PageCache pageCache = pageCacheRule.getPageCache( fs ); VersionAwareLogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader = new VersionAwareLogEntryReader<>(); LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDirectory.databaseLayout(), fs, pageCache ).withLogEntryReader( logEntryReader ).build(); LogTailScanner tailScanner = new LogTailScanner( logFiles, logEntryReader, new Monitors() ); LogTailScanner.LogTailInformation tailInformation = tailScanner.getTailInformation(); try ( Lifespan lifespan = new Lifespan( logFiles ) ) { FlushablePositionAwareChannel channel = logFiles.getLogFile().getWriter(); LogPosition logPosition = tailInformation.lastCheckPoint.getLogPosition(); // Fake record channel.put( logVersion.byteCode() ) .put( CHECK_POINT ) .putLong( logPosition.getLogVersion() ) .putLong( logPosition.getByteOffset() ); channel.prepareForFlush().flush(); } } }
if ( hasUnreadableBytes( channel, maxEntryReadPosition ) ) return checkpointTailInformation( highestLogVersion, latestStartEntry, oldestVersionFound, latestLogEntryVersion, latestCheckPoint, corruptedTransactionLogs );
/** * Collects information about the tail of the transaction log, i.e. last checkpoint, last entry etc. * Since this is an expensive task we do it once and reuse the result. This method is thus lazy and the first one * calling it will take the hit. * <p> * This is only intended to be used during startup. If you need to track the state of the tail, that can be done more * efficiently at runtime, and this method should then only be used to restore said state. * * @return snapshot of the state of the transaction logs tail at startup. * @throws UnderlyingStorageException if any errors occurs while parsing the transaction logs */ public LogTailInformation getTailInformation() throws UnderlyingStorageException { if ( logTailInformation == null ) { try { logTailInformation = findLogTail(); } catch ( IOException e ) { throw new UnderlyingStorageException( "Error encountered while parsing transaction logs", e ); } } return logTailInformation; }
private Result checkCleanShutDownByCheckPoint() { // check version try { if ( !tailScanner.getTailInformation().commitsAfterLastCheckpoint() ) { return new Result( Result.Outcome.ok, null, null ); } } catch ( Throwable throwable ) { // ignore exception and return db not cleanly shutdown } return new Result( Result.Outcome.storeNotCleanlyShutDown, null, null ); }
private LogTailScanner getTailScanner( File databaseDirectory ) throws IOException { LogFiles logFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( databaseDirectory, fs ).build(); return new LogTailScanner( logFiles, new VersionAwareLogEntryReader<>(), monitors ); }
protected LogTailInformation checkpointTailInformation( long highestLogVersion, LogEntryStart latestStartEntry, long oldestVersionFound, LogEntryVersion latestLogEntryVersion, CheckPoint latestCheckPoint, boolean corruptedTransactionLogs ) throws IOException { LogPosition checkPointLogPosition = latestCheckPoint.getLogPosition(); ExtractedTransactionRecord transactionRecord = extractFirstTxIdAfterPosition( checkPointLogPosition, highestLogVersion ); long firstTxIdAfterPosition = transactionRecord.getId(); boolean startRecordAfterCheckpoint = (firstTxIdAfterPosition != NO_TRANSACTION_ID) || ((latestStartEntry != null) && (latestStartEntry.getStartPosition().compareTo( latestCheckPoint.getLogPosition() ) >= 0)); boolean corruptedLogs = transactionRecord.isFailure() || corruptedTransactionLogs; return new LogTailInformation( latestCheckPoint, corruptedLogs || startRecordAfterCheckpoint, firstTxIdAfterPosition, oldestVersionFound, highestLogVersion, latestLogEntryVersion ); }
@Test public void noThrowWhenLatestVersionAndUpgradeIsNotAllowed() { when( tailScanner.getTailInformation() ).thenReturn( new OnlyVersionTailInformation( LogEntryVersion.CURRENT ) ); LogVersionUpgradeChecker.check( tailScanner, Config.defaults( GraphDatabaseSettings.allow_upgrade, "false") ); }
public boolean isRecoveryRequiredAt( DatabaseLayout databaseLayout ) throws IOException { // We need config to determine where the logical log files are if ( !NeoStores.isStorePresent( pageCache, databaseLayout ) ) { return false; } LogEntryReader<ReadableClosablePositionAwareChannel> reader = new VersionAwareLogEntryReader<>(); LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( databaseLayout, fs, pageCache ) .withConfig( config ) .withLogEntryReader( reader ).build(); LogTailScanner tailScanner = new LogTailScanner( logFiles, reader, monitors ); return new RecoveryStartInformationProvider( tailScanner, NO_MONITOR ).get().isRecoveryRequired(); }
@Test public void stillAcceptLatestVersionWhenUpgradeIsAllowed() { when( tailScanner.getTailInformation() ).thenReturn( new OnlyVersionTailInformation( LogEntryVersion.CURRENT ) ); LogVersionUpgradeChecker.check( tailScanner, Config.defaults( GraphDatabaseSettings.allow_upgrade, "true") ); }
private UpgradableDatabase getUpgradableDatabase( StoreVersionCheck check ) throws IOException { VersionAwareLogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader = new VersionAwareLogEntryReader<>(); LogFiles logFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( workingDatabaseLayout.databaseDirectory(), fs ).build(); LogTailScanner tailScanner = new LogTailScanner( logFiles, logEntryReader, new Monitors() ); return new UpgradableDatabase( check, Standard.LATEST_RECORD_FORMATS, tailScanner ); }
@Test public void acceptOlderLogsWhenUpgradeIsAllowed() { when( tailScanner.getTailInformation() ).thenReturn( new OnlyVersionTailInformation( LogEntryVersion.V2_3 ) ); LogVersionUpgradeChecker.check( tailScanner, Config.defaults( GraphDatabaseSettings.allow_upgrade, "true") ); }
@Before public void setUp() throws IOException { logVersionRepository = new SimpleLogVersionRepository(); logFiles = LogFilesBuilder .activeFilesBuilder( testDirectory.databaseLayout(), fsRule, pageCacheRule.getPageCache( fsRule ) ) .withLogVersionRepository( logVersionRepository ) .build(); tailScanner = new LogTailScanner( logFiles, reader, monitors ); }
@Test public void noLogFilesFound() { // given no files setupLogFiles(); // when LogTailInformation logTailInformation = tailScanner.getTailInformation(); // then assertLatestCheckPoint( false, false, NO_TRANSACTION_ID, -1, logTailInformation ); }