/** * Performs construction of {@link StoreUpgrader} and all of the necessary participants and performs store * migration if that is required. * @param directoryStructure database to migrate */ public void migrate( DatabaseLayout directoryStructure ) { LogProvider logProvider = logService.getInternalLogProvider(); UpgradableDatabase upgradableDatabase = new UpgradableDatabase( new StoreVersionCheck( pageCache ), format, tailScanner ); StoreUpgrader storeUpgrader = new StoreUpgrader( upgradableDatabase, progressMonitor, config, fs, pageCache, logProvider ); ExplicitIndexMigrator explicitIndexMigrator = new ExplicitIndexMigrator( fs, explicitIndexProvider, logProvider ); StoreMigrator storeMigrator = new StoreMigrator( fs, pageCache, config, logService, jobScheduler ); NativeLabelScanStoreMigrator nativeLabelScanStoreMigrator = new NativeLabelScanStoreMigrator( fs, pageCache, config ); CountsMigrator countsMigrator = new CountsMigrator( fs, pageCache, config ); indexProviderMap.accept( provider -> storeUpgrader.addParticipant( provider.storeMigrationParticipant( fs, pageCache ) ) ); storeUpgrader.addParticipant( explicitIndexMigrator ); storeUpgrader.addParticipant( storeMigrator ); storeUpgrader.addParticipant( nativeLabelScanStoreMigrator ); storeUpgrader.addParticipant( countsMigrator ); storeUpgrader.migrateIfNeeded( directoryStructure ); } }
private void migrateStore( DatabaseLayout dbDirectoryLayout, DatabaseLayout migrationLayout, File migrationStateFile ) { // One or more participants would like to do migration progressMonitor.started( participants.size() ); MigrationStatus migrationStatus = MigrationStatus.readMigrationStatus( fileSystem, migrationStateFile ); String versionToMigrateFrom = null; // We don't need to migrate if we're at the phase where we have migrated successfully // and it's just a matter of moving over the files to the storeDir. if ( MigrationStatus.migrating.isNeededFor( migrationStatus ) ) { versionToMigrateFrom = upgradableDatabase.checkUpgradable( dbDirectoryLayout ).storeVersion(); cleanMigrationDirectory( migrationLayout.databaseDirectory() ); MigrationStatus.migrating.setMigrationStatus( fileSystem, migrationStateFile, versionToMigrateFrom ); migrateToIsolatedDirectory( dbDirectoryLayout, migrationLayout, versionToMigrateFrom ); MigrationStatus.moving.setMigrationStatus( fileSystem, migrationStateFile, versionToMigrateFrom ); } if ( MigrationStatus.moving.isNeededFor( migrationStatus ) ) { versionToMigrateFrom = MigrationStatus.moving.maybeReadInfo( fileSystem, migrationStateFile, versionToMigrateFrom ); moveMigratedFilesToStoreDirectory( participants, migrationLayout, dbDirectoryLayout, versionToMigrateFrom, upgradableDatabase.currentVersion() ); } cleanup( participants, migrationLayout ); progressMonitor.completed(); }
public void migrateIfNeeded( DatabaseLayout layout ) { DatabaseLayout migrationStructure = DatabaseLayout.of( layout.databaseDirectory(), MIGRATION_DIRECTORY ); cleanupLegacyLeftOverDirsIn( layout.databaseDirectory() ); File migrationStateFile = migrationStructure.file( MIGRATION_STATUS_FILE ); // if migration directory exists than we might have failed to move files into the store dir so do it again if ( upgradableDatabase.hasCurrentVersion( layout ) && !fileSystem.fileExists( migrationStateFile ) ) { // No migration needed return; } if ( isUpgradeAllowed() ) { migrateStore( layout, migrationStructure, migrationStateFile ); } else if ( !RecordFormatSelector.isStoreAndConfigFormatsCompatible( config, layout, fileSystem, pageCache, logProvider ) ) { throw new UpgradeNotAllowedByConfigurationException(); } }
private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCache pageCache, MigrationProgressMonitor progressMonitor, SchemaIndexMigrator indexMigrator, StoreMigrator migrator ) { Config allowUpgrade = Config.defaults( GraphDatabaseSettings.allow_upgrade, "true" ); StoreUpgrader upgrader = new StoreUpgrader( upgradableDatabase, progressMonitor, allowUpgrade, fs, pageCache, NullLogProvider.getInstance() ); upgrader.addParticipant( indexMigrator ); upgrader.addParticipant( migrator ); return upgrader; }
@Test public void upgradeShouldNotLeaveLeftoverAndMigrationDirs() throws Exception { // Given fileSystem.deleteFile( databaseLayout.file( INTERNAL_LOG_FILE ) ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); // When newUpgrader( upgradableDatabase, allowMigrateConfig, pageCache ).migrateIfNeeded( databaseLayout ); // Then assertThat( migrationHelperDirs(), is( emptyCollectionOf( File.class ) ) ); }
upgrader.addParticipant( participantThatWillFailWhenMoving( failureMessage ) ); upgrader.migrateIfNeeded( databaseLayout ); fail( "should have thrown" ); upgrader.addParticipant( observingParticipant ); upgrader.migrateIfNeeded( databaseLayout );
private void cleanupLegacyLeftOverDirsIn( File databaseDirectory ) { File[] leftOverDirs = databaseDirectory.listFiles( ( file, name ) -> file.isDirectory() && MIGRATION_LEFTOVERS_PATTERN.matcher( name ).matches() ); if ( leftOverDirs != null ) { for ( File leftOverDir : leftOverDirs ) { deleteSilently( leftOverDir ); } } }
@Test public void notParticipatingParticipantsAreNotPartOfMigration() throws IOException { PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); StoreUpgrader storeUpgrader = newUpgrader( upgradableDatabase, pageCache ); assertThat( storeUpgrader.getParticipants(), hasSize( 3 ) ); }
@Test public void upgradedNeoStoreShouldHaveNewUpgradeTimeAndUpgradeId() throws Exception { // Given fileSystem.deleteFile( databaseLayout.file( INTERNAL_LOG_FILE ) ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); // When newUpgrader( upgradableDatabase, allowMigrateConfig, pageCache ).migrateIfNeeded( databaseLayout ); // Then StoreFactory factory = new StoreFactory( databaseLayout, allowMigrateConfig, new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, NullLogProvider.getInstance(), EmptyVersionContextSupplier.EMPTY ); try ( NeoStores neoStores = factory.openAllNeoStores() ) { assertThat( neoStores.getMetaDataStore().getUpgradeTransaction(), equalTo( neoStores.getMetaDataStore().getLastCommittedTransaction() ) ); assertThat( neoStores.getMetaDataStore().getUpgradeTime(), not( equalTo( MetaDataStore.FIELD_NOT_INITIALIZED ) ) ); long minuteAgo = System.currentTimeMillis() - MINUTES.toMillis( 1 ); assertThat( neoStores.getMetaDataStore().getUpgradeTime(), greaterThan( minuteAgo ) ); } }
private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCache pageCache, Config config, MigrationProgressMonitor progressMonitor ) { NullLogService instance = NullLogService.getInstance(); StoreMigrator defaultMigrator = new StoreMigrator( fileSystem, pageCache, getTuningConfig(), instance, jobScheduler ); CountsMigrator countsMigrator = new CountsMigrator( fileSystem, pageCache, getTuningConfig() ); SchemaIndexMigrator indexMigrator = new SchemaIndexMigrator( fileSystem, IndexProvider.EMPTY ); StoreUpgrader upgrader = new StoreUpgrader( upgradableDatabase, progressMonitor, config, fileSystem, pageCache, NullLogProvider.getInstance() ); upgrader.addParticipant( indexMigrator ); upgrader.addParticipant( AbstractStoreMigrationParticipant.NOT_PARTICIPATING ); upgrader.addParticipant( AbstractStoreMigrationParticipant.NOT_PARTICIPATING ); upgrader.addParticipant( AbstractStoreMigrationParticipant.NOT_PARTICIPATING ); upgrader.addParticipant( AbstractStoreMigrationParticipant.NOT_PARTICIPATING ); upgrader.addParticipant( defaultMigrator ); upgrader.addParticipant( countsMigrator ); return upgrader; }
private void cleanupLegacyLeftOverDirsIn( File databaseDirectory ) { File[] leftOverDirs = databaseDirectory.listFiles( ( file, name ) -> file.isDirectory() && MIGRATION_LEFTOVERS_PATTERN.matcher( name ).matches() ); if ( leftOverDirs != null ) { for ( File leftOverDir : leftOverDirs ) { deleteSilently( leftOverDir ); } } }
/** * Performs construction of {@link StoreUpgrader} and all of the necessary participants and performs store * migration if that is required. * @param directoryStructure database to migrate */ public void migrate( DatabaseLayout directoryStructure ) { LogProvider logProvider = logService.getInternalLogProvider(); UpgradableDatabase upgradableDatabase = new UpgradableDatabase( new StoreVersionCheck( pageCache ), format, tailScanner ); StoreUpgrader storeUpgrader = new StoreUpgrader( upgradableDatabase, progressMonitor, config, fs, pageCache, logProvider ); ExplicitIndexMigrator explicitIndexMigrator = new ExplicitIndexMigrator( fs, explicitIndexProvider, logProvider ); StoreMigrator storeMigrator = new StoreMigrator( fs, pageCache, config, logService, jobScheduler ); NativeLabelScanStoreMigrator nativeLabelScanStoreMigrator = new NativeLabelScanStoreMigrator( fs, pageCache, config ); CountsMigrator countsMigrator = new CountsMigrator( fs, pageCache, config ); indexProviderMap.accept( provider -> storeUpgrader.addParticipant( provider.storeMigrationParticipant( fs, pageCache ) ) ); storeUpgrader.addParticipant( explicitIndexMigrator ); storeUpgrader.addParticipant( storeMigrator ); storeUpgrader.addParticipant( nativeLabelScanStoreMigrator ); storeUpgrader.addParticipant( countsMigrator ); storeUpgrader.migrateIfNeeded( directoryStructure ); } }
@Test public void shouldHaltUpgradeIfUpgradeConfigurationVetoesTheProcess() throws IOException { PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); Config deniedMigrationConfig = Config.defaults( GraphDatabaseSettings.allow_upgrade, "false" ); deniedMigrationConfig.augment( GraphDatabaseSettings.record_format, Standard.LATEST_NAME ); UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); try { newUpgrader( upgradableDatabase, deniedMigrationConfig, pageCache ).migrateIfNeeded( databaseLayout ); fail( "Should throw exception" ); } catch ( UpgradeNotAllowedByConfigurationException e ) { // expected } }
private void migrateStore( DatabaseLayout dbDirectoryLayout, DatabaseLayout migrationLayout, File migrationStateFile ) { // One or more participants would like to do migration progressMonitor.started( participants.size() ); MigrationStatus migrationStatus = MigrationStatus.readMigrationStatus( fileSystem, migrationStateFile ); String versionToMigrateFrom = null; // We don't need to migrate if we're at the phase where we have migrated successfully // and it's just a matter of moving over the files to the storeDir. if ( MigrationStatus.migrating.isNeededFor( migrationStatus ) ) { versionToMigrateFrom = upgradableDatabase.checkUpgradable( dbDirectoryLayout ).storeVersion(); cleanMigrationDirectory( migrationLayout.databaseDirectory() ); MigrationStatus.migrating.setMigrationStatus( fileSystem, migrationStateFile, versionToMigrateFrom ); migrateToIsolatedDirectory( dbDirectoryLayout, migrationLayout, versionToMigrateFrom ); MigrationStatus.moving.setMigrationStatus( fileSystem, migrationStateFile, versionToMigrateFrom ); } if ( MigrationStatus.moving.isNeededFor( migrationStatus ) ) { versionToMigrateFrom = MigrationStatus.moving.maybeReadInfo( fileSystem, migrationStateFile, versionToMigrateFrom ); moveMigratedFilesToStoreDirectory( participants, migrationLayout, dbDirectoryLayout, versionToMigrateFrom, upgradableDatabase.currentVersion() ); } cleanup( participants, migrationLayout ); progressMonitor.completed(); }
public void migrateIfNeeded( DatabaseLayout layout ) { DatabaseLayout migrationStructure = DatabaseLayout.of( layout.databaseDirectory(), MIGRATION_DIRECTORY ); cleanupLegacyLeftOverDirsIn( layout.databaseDirectory() ); File migrationStateFile = migrationStructure.file( MIGRATION_STATUS_FILE ); // if migration directory exists than we might have failed to move files into the store dir so do it again if ( upgradableDatabase.hasCurrentVersion( layout ) && !fileSystem.fileExists( migrationStateFile ) ) { // No migration needed return; } if ( isUpgradeAllowed() ) { migrateStore( layout, migrationStructure, migrationStateFile ); } else if ( !RecordFormatSelector.isStoreAndConfigFormatsCompatible( config, layout, fileSystem, pageCache, logProvider ) ) { throw new UpgradeNotAllowedByConfigurationException(); } }
@Test public void upgradeShouldGiveProgressMonitorProgressMessages() throws Exception { // Given PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); // When AssertableLogProvider logProvider = new AssertableLogProvider(); newUpgrader( upgradableDatabase, pageCache, allowMigrateConfig, new VisibleMigrationProgressMonitor( logProvider.getLog( "test" ) ) ).migrateIfNeeded( databaseLayout ); // Then logProvider.assertContainsLogCallContaining( "Store files" ); logProvider.assertContainsLogCallContaining( "Indexes" ); logProvider.assertContainsLogCallContaining( "Counts store" ); logProvider.assertContainsLogCallContaining( "Successfully finished" ); }
.migrateIfNeeded( workingDatabaseLayout ); fail( "Should throw exception" ); StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ); SchemaIndexMigrator indexMigrator = createIndexMigrator(); newUpgrader( upgradableDatabase, pageCache, progressMonitor, indexMigrator, migrator ).migrateIfNeeded( workingDatabaseLayout );
@Test public void upgraderShouldCleanupLegacyLeftoverAndMigrationDirs() throws Exception { // Given fileSystem.deleteFile( databaseLayout.file( INTERNAL_LOG_FILE ) ); fileSystem.mkdir( databaseLayout.file( StoreUpgrader.MIGRATION_DIRECTORY ) ); fileSystem.mkdir( databaseLayout.file( StoreUpgrader.MIGRATION_LEFT_OVERS_DIRECTORY ) ); fileSystem.mkdir( databaseLayout.file( StoreUpgrader.MIGRATION_LEFT_OVERS_DIRECTORY + "_1" ) ); fileSystem.mkdir( databaseLayout.file( StoreUpgrader.MIGRATION_LEFT_OVERS_DIRECTORY + "_2" ) ); fileSystem.mkdir( databaseLayout.file( StoreUpgrader.MIGRATION_LEFT_OVERS_DIRECTORY + "_42" ) ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); // When UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); StoreUpgrader storeUpgrader = newUpgrader( upgradableDatabase, pageCache ); storeUpgrader.migrateIfNeeded( databaseLayout ); // Then assertThat( migrationHelperDirs(), is( emptyCollectionOf( File.class ) ) ); }
.migrateIfNeeded( workingDatabaseLayout ); fail( "Should throw exception" ); StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ); newUpgrader( upgradableDatabase, pageCache, progressMonitor, createIndexMigrator(), migrator ) .migrateIfNeeded( workingDatabaseLayout );
@Test public void shouldRefuseToUpgradeIfAllOfTheStoresWereNotShutDownCleanly() throws IOException { File comparisonDirectory = directory.directory( "shouldRefuseToUpgradeIfAllOfTheStoresWereNotShutDownCleanly-comparison" ); removeCheckPointFromTxLog( fileSystem, databaseLayout.databaseDirectory() ); fileSystem.deleteRecursively( comparisonDirectory ); fileSystem.copyRecursively( databaseLayout.databaseDirectory(), comparisonDirectory ); PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); try { newUpgrader( upgradableDatabase, pageCache ).migrateIfNeeded( databaseLayout ); fail( "Should throw exception" ); } catch ( StoreUpgrader.UnableToUpgradeException e ) { // expected } verifyFilesHaveSameContent( fileSystem, comparisonDirectory, databaseLayout.databaseDirectory() ); }