@Test public void cleanupMigrationDirectory() throws IOException { when( fs.fileExists( migratedIndexStore ) ).thenReturn( true ); ExplicitIndexProvider indexProviders = getExplicitIndexProvider(); ExplicitIndexMigrator indexMigrator = new TestExplicitIndexMigrator( fs, indexProviders, logProvider, true ); indexMigrator.migrate( storeLayout, migrationLayout, progressMonitor, StandardV2_3.STORE_VERSION, StandardV3_0.STORE_VERSION ); indexMigrator.cleanup( migrationLayout ); verify( fs ).deleteRecursively( migratedIndexStore ); }
private void migrateExplicitIndexes( ProgressReporter progressMonitor ) throws IOException { try { fileSystem.copyRecursively( originalExplicitIndexesRoot, migrationExplicitIndexesRoot ); Path indexRootPath = migrationExplicitIndexesRoot.toPath(); LuceneExplicitIndexUpgrader indexUpgrader = createLuceneExplicitIndexUpgrader( indexRootPath, progressMonitor ); indexUpgrader.upgradeIndexes(); } catch ( ExplicitIndexMigrationException lime ) { log.error( "Migration of explicit indexes failed. Index: " + lime.getFailedIndexName() + " can't be " + "migrated.", lime ); throw new IOException( "Explicit index migration failed.", lime ); } }
@Override public void cleanup( DatabaseLayout migrationLayout ) throws IOException { if ( isIndexMigrationDirectoryExists() ) { fileSystem.deleteRecursively( migrationExplicitIndexesRoot ); } }
@Override public void migrate( DatabaseLayout directoryLayout, DatabaseLayout migrationLayout, ProgressReporter progressMonitor, String versionToMigrateFrom, String versionToMigrateTo ) throws IOException { IndexImplementation indexImplementation = explicitIndexProvider.getProviderByName( LUCENE_EXPLICIT_INDEX_PROVIDER_NAME ); if ( indexImplementation != null ) { RecordFormats from = RecordFormatSelector.selectForVersion( versionToMigrateFrom ); RecordFormats to = RecordFormatSelector.selectForVersion( versionToMigrateTo ); if ( !from.hasCompatibleCapabilities( to, CapabilityType.INDEX ) ) { originalExplicitIndexesRoot = indexImplementation.getIndexImplementationDirectory( directoryLayout ); migrationExplicitIndexesRoot = indexImplementation.getIndexImplementationDirectory( migrationLayout ); if ( isNotEmptyDirectory( originalExplicitIndexesRoot ) ) { migrateExplicitIndexes( progressMonitor ); explicitIndexMigrated = true; } } } else { log.debug( "Lucene index provider not found, nothing to migrate." ); } }
@Test public void transferMigratedIndexesToStoreDirectory() throws IOException { ExplicitIndexProvider indexProviders = getExplicitIndexProvider(); ExplicitIndexMigrator indexMigrator = new TestExplicitIndexMigrator( fs, indexProviders, logProvider, true ); indexMigrator.migrate( storeLayout, migrationLayout, progressMonitor, StandardV2_3.STORE_VERSION, StandardV3_0.STORE_VERSION ); reset( fs ); indexMigrator.moveMigratedFiles( migrationLayout, storeLayout, "any", "any" ); verify( fs ).deleteRecursively( originalIndexStore ); verify( fs ).moveToDirectory( migratedIndexStore, storeLayout.databaseDirectory() ); }
@Test public void skipEmptyIndexStorageMigration() throws IOException { when( fs.listFiles( originalIndexStore ) ).thenReturn( null ); ExplicitIndexProvider indexProviders = getExplicitIndexProvider(); ExplicitIndexMigrator indexMigrator = new TestExplicitIndexMigrator( fs, indexProviders, logProvider, true ); indexMigrator.migrate( storeLayout, migrationLayout, progressMonitor, StandardV2_3.STORE_VERSION, StandardV3_0.STORE_VERSION ); verify( fs, never() ).deleteRecursively( originalIndexStore ); verify( fs, never() ).moveToDirectory( migratedIndexStore, storeLayout.databaseDirectory() ); }
LuceneExplicitIndexUpgrader createLuceneExplicitIndexUpgrader( Path indexRootPath, ProgressReporter progressMonitor ) { return new LuceneExplicitIndexUpgrader( indexRootPath, progressMonitor( progressMonitor ) ); }
/** * Performs construction of {@link StoreUpgrader} and all of the necessary participants and performs store * migration if that is required. * @param directoryStructure database to migrate */ public void migrate( DatabaseLayout directoryStructure ) { LogProvider logProvider = logService.getInternalLogProvider(); UpgradableDatabase upgradableDatabase = new UpgradableDatabase( new StoreVersionCheck( pageCache ), format, tailScanner ); StoreUpgrader storeUpgrader = new StoreUpgrader( upgradableDatabase, progressMonitor, config, fs, pageCache, logProvider ); ExplicitIndexMigrator explicitIndexMigrator = new ExplicitIndexMigrator( fs, explicitIndexProvider, logProvider ); StoreMigrator storeMigrator = new StoreMigrator( fs, pageCache, config, logService, jobScheduler ); NativeLabelScanStoreMigrator nativeLabelScanStoreMigrator = new NativeLabelScanStoreMigrator( fs, pageCache, config ); CountsMigrator countsMigrator = new CountsMigrator( fs, pageCache, config ); indexProviderMap.accept( provider -> storeUpgrader.addParticipant( provider.storeMigrationParticipant( fs, pageCache ) ) ); storeUpgrader.addParticipant( explicitIndexMigrator ); storeUpgrader.addParticipant( storeMigrator ); storeUpgrader.addParticipant( nativeLabelScanStoreMigrator ); storeUpgrader.addParticipant( countsMigrator ); storeUpgrader.migrateIfNeeded( directoryStructure ); } }
@Override public void migrate( DatabaseLayout directoryLayout, DatabaseLayout migrationLayout, ProgressReporter progressMonitor, String versionToMigrateFrom, String versionToMigrateTo ) throws IOException { IndexImplementation indexImplementation = explicitIndexProvider.getProviderByName( LUCENE_EXPLICIT_INDEX_PROVIDER_NAME ); if ( indexImplementation != null ) { RecordFormats from = RecordFormatSelector.selectForVersion( versionToMigrateFrom ); RecordFormats to = RecordFormatSelector.selectForVersion( versionToMigrateTo ); if ( !from.hasCompatibleCapabilities( to, CapabilityType.INDEX ) ) { originalExplicitIndexesRoot = indexImplementation.getIndexImplementationDirectory( directoryLayout ); migrationExplicitIndexesRoot = indexImplementation.getIndexImplementationDirectory( migrationLayout ); if ( isNotEmptyDirectory( originalExplicitIndexesRoot ) ) { migrateExplicitIndexes( progressMonitor ); explicitIndexMigrated = true; } } } else { log.debug( "Lucene index provider not found, nothing to migrate." ); } }
@Test public void transferOriginalDataToMigrationDirectory() throws IOException { ExplicitIndexProvider indexProviders = getExplicitIndexProvider(); ExplicitIndexMigrator indexMigrator = new TestExplicitIndexMigrator( fs, indexProviders, logProvider, true ); indexMigrator.migrate( storeLayout, migrationLayout, progressMonitor, StandardV2_3.STORE_VERSION, StandardV3_0.STORE_VERSION ); verify( fs ).copyRecursively( originalIndexStore, migratedIndexStore ); }
LuceneExplicitIndexUpgrader createLuceneExplicitIndexUpgrader( Path indexRootPath, ProgressReporter progressMonitor ) { return new LuceneExplicitIndexUpgrader( indexRootPath, progressMonitor( progressMonitor ) ); }
/** * Performs construction of {@link StoreUpgrader} and all of the necessary participants and performs store * migration if that is required. * @param directoryStructure database to migrate */ public void migrate( DatabaseLayout directoryStructure ) { LogProvider logProvider = logService.getInternalLogProvider(); UpgradableDatabase upgradableDatabase = new UpgradableDatabase( new StoreVersionCheck( pageCache ), format, tailScanner ); StoreUpgrader storeUpgrader = new StoreUpgrader( upgradableDatabase, progressMonitor, config, fs, pageCache, logProvider ); ExplicitIndexMigrator explicitIndexMigrator = new ExplicitIndexMigrator( fs, explicitIndexProvider, logProvider ); StoreMigrator storeMigrator = new StoreMigrator( fs, pageCache, config, logService, jobScheduler ); NativeLabelScanStoreMigrator nativeLabelScanStoreMigrator = new NativeLabelScanStoreMigrator( fs, pageCache, config ); CountsMigrator countsMigrator = new CountsMigrator( fs, pageCache, config ); indexProviderMap.accept( provider -> storeUpgrader.addParticipant( provider.storeMigrationParticipant( fs, pageCache ) ) ); storeUpgrader.addParticipant( explicitIndexMigrator ); storeUpgrader.addParticipant( storeMigrator ); storeUpgrader.addParticipant( nativeLabelScanStoreMigrator ); storeUpgrader.addParticipant( countsMigrator ); storeUpgrader.migrateIfNeeded( directoryStructure ); } }
@Test public void logErrorWithIndexNameOnIndexMigrationException() { Log log = mock( Log.class ); when( logProvider.getLog( TestExplicitIndexMigrator.class ) ).thenReturn( log ); ExplicitIndexProvider indexProviders = getExplicitIndexProvider(); try { ExplicitIndexMigrator indexMigrator = new TestExplicitIndexMigrator( fs, indexProviders, logProvider, false ); indexMigrator.migrate( storeLayout, migrationLayout, progressMonitor, StandardV2_3.STORE_VERSION, StandardV3_0.STORE_VERSION ); fail( "Index migration should fail" ); } catch ( IOException e ) { // ignored } verify( log ).error( eq( "Migration of explicit indexes failed. Index: testIndex can't be migrated." ), any( Throwable.class ) ); }
@Override public void cleanup( DatabaseLayout migrationLayout ) throws IOException { if ( isIndexMigrationDirectoryExists() ) { fileSystem.deleteRecursively( migrationExplicitIndexesRoot ); } }
private void migrateExplicitIndexes( ProgressReporter progressMonitor ) throws IOException { try { fileSystem.copyRecursively( originalExplicitIndexesRoot, migrationExplicitIndexesRoot ); Path indexRootPath = migrationExplicitIndexesRoot.toPath(); LuceneExplicitIndexUpgrader indexUpgrader = createLuceneExplicitIndexUpgrader( indexRootPath, progressMonitor ); indexUpgrader.upgradeIndexes(); } catch ( ExplicitIndexMigrationException lime ) { log.error( "Migration of explicit indexes failed. Index: " + lime.getFailedIndexName() + " can't be " + "migrated.", lime ); throw new IOException( "Explicit index migration failed.", lime ); } }