@Override public void moveMigratedFiles( DatabaseLayout migrationLayout, DatabaseLayout directoryLayout, String versionToUpgradeFrom, String versionToMigrateTo ) throws IOException { if ( nativeLabelScanStoreMigrated ) { File nativeLabelIndex = migrationLayout.labelScanStore(); moveNativeIndexFile( directoryLayout, nativeLabelIndex ); deleteLuceneLabelIndex( getLuceneStoreDirectory( directoryLayout ) ); } }
@Override public void migrate( DatabaseLayout directoryLayout, DatabaseLayout migrationLayout, ProgressReporter progressReporter, String versionToMigrateFrom, String versionToMigrateTo ) throws IOException { if ( isNativeLabelScanStoreMigrationRequired( directoryLayout ) ) { StoreFactory storeFactory = getStoreFactory( directoryLayout, versionToMigrateFrom ); try ( NeoStores neoStores = storeFactory.openAllNeoStores(); Lifespan lifespan = new Lifespan() ) { neoStores.verifyStoreOk(); // Remove any existing file to ensure we always do migration deleteNativeIndexFile( migrationLayout ); progressReporter.start( neoStores.getNodeStore().getNumberOfIdsInUse() ); NativeLabelScanStore nativeLabelScanStore = getNativeLabelScanStore( migrationLayout, progressReporter, neoStores ); lifespan.add( nativeLabelScanStore ); } nativeLabelScanStoreMigrated = true; } }
@Test public void reportProgressOnNativeIndexPopulation() throws IOException { prepare34DatabaseWithNodes(); indexMigrator.migrate( databaseLayout, migrationLayout, progressReporter, StandardV3_4.STORE_VERSION, StandardV3_4.STORE_VERSION ); indexMigrator.moveMigratedFiles( migrationLayout, databaseLayout, StandardV2_3.STORE_VERSION, StandardV3_2.STORE_VERSION ); verify( progressReporter ).start( 10 ); verify( progressReporter, times( 10 ) ).progress( 1 ); }
@Before public void setUp() throws Exception { databaseLayout = testDirectory.databaseLayout(); storeDir = databaseLayout.databaseDirectory(); nativeLabelIndex = databaseLayout.labelScanStore(); migrationLayout = testDirectory.databaseLayout( "migrationDir" ); luceneLabelScanStore = testDirectory.databaseDir().toPath().resolve( Paths.get( "schema", "label", "lucene" ) ).toFile(); fileSystem = fileSystemRule.get(); pageCache = pageCacheRule.getPageCache( fileSystemRule ); indexMigrator = new NativeLabelScanStoreMigrator( fileSystem, pageCache, Config.defaults() ); fileSystem.mkdirs( luceneLabelScanStore ); }
@Test( expected = InvalidIdGeneratorException.class ) public void failMigrationWhenNodeIdFileIsBroken() throws Exception { prepareEmpty23Database(); File nodeIdFile = databaseLayout.idNodeStore(); writeFile( nodeIdFile, new byte[]{1, 2, 3} ); indexMigrator.migrate( databaseLayout, migrationLayout, progressReporter, StandardV3_2.STORE_VERSION, StandardV3_2.STORE_VERSION ); }
/** * Performs construction of {@link StoreUpgrader} and all of the necessary participants and performs store * migration if that is required. * @param directoryStructure database to migrate */ public void migrate( DatabaseLayout directoryStructure ) { LogProvider logProvider = logService.getInternalLogProvider(); UpgradableDatabase upgradableDatabase = new UpgradableDatabase( new StoreVersionCheck( pageCache ), format, tailScanner ); StoreUpgrader storeUpgrader = new StoreUpgrader( upgradableDatabase, progressMonitor, config, fs, pageCache, logProvider ); ExplicitIndexMigrator explicitIndexMigrator = new ExplicitIndexMigrator( fs, explicitIndexProvider, logProvider ); StoreMigrator storeMigrator = new StoreMigrator( fs, pageCache, config, logService, jobScheduler ); NativeLabelScanStoreMigrator nativeLabelScanStoreMigrator = new NativeLabelScanStoreMigrator( fs, pageCache, config ); CountsMigrator countsMigrator = new CountsMigrator( fs, pageCache, config ); indexProviderMap.accept( provider -> storeUpgrader.addParticipant( provider.storeMigrationParticipant( fs, pageCache ) ) ); storeUpgrader.addParticipant( explicitIndexMigrator ); storeUpgrader.addParticipant( storeMigrator ); storeUpgrader.addParticipant( nativeLabelScanStoreMigrator ); storeUpgrader.addParticipant( countsMigrator ); storeUpgrader.migrateIfNeeded( directoryStructure ); } }
@Test public void clearMigrationDirFromAnyLabelScanStoreBeforeMigrating() throws Exception { // given prepareEmpty23Database(); initializeNativeLabelScanStoreWithContent( migrationLayout ); File toBeDeleted = migrationLayout.labelScanStore(); assertTrue( fileSystem.fileExists( toBeDeleted ) ); // when indexMigrator.migrate( databaseLayout, migrationLayout, progressReporter, StandardV3_2.STORE_VERSION, StandardV3_2.STORE_VERSION ); // then assertNoContentInNativeLabelScanStore( migrationLayout ); }
@Override public void migrate( DatabaseLayout directoryLayout, DatabaseLayout migrationLayout, ProgressReporter progressReporter, String versionToMigrateFrom, String versionToMigrateTo ) throws IOException { if ( isNativeLabelScanStoreMigrationRequired( directoryLayout ) ) { StoreFactory storeFactory = getStoreFactory( directoryLayout, versionToMigrateFrom ); try ( NeoStores neoStores = storeFactory.openAllNeoStores(); Lifespan lifespan = new Lifespan() ) { neoStores.verifyStoreOk(); // Remove any existing file to ensure we always do migration deleteNativeIndexFile( migrationLayout ); progressReporter.start( neoStores.getNodeStore().getNumberOfIdsInUse() ); NativeLabelScanStore nativeLabelScanStore = getNativeLabelScanStore( migrationLayout, progressReporter, neoStores ); lifespan.add( nativeLabelScanStore ); } nativeLabelScanStoreMigrated = true; } }
@Test public void luceneLabelIndexRemovedAfterSuccessfulMigration() throws IOException { prepareEmpty23Database(); indexMigrator.migrate( databaseLayout, migrationLayout, progressReporter, StandardV2_3.STORE_VERSION, StandardV3_2.STORE_VERSION ); indexMigrator.moveMigratedFiles( migrationLayout, databaseLayout, StandardV2_3.STORE_VERSION, StandardV3_2.STORE_VERSION ); assertFalse( fileSystem.fileExists( luceneLabelScanStore ) ); }
@Override public void moveMigratedFiles( DatabaseLayout migrationLayout, DatabaseLayout directoryLayout, String versionToUpgradeFrom, String versionToMigrateTo ) throws IOException { if ( nativeLabelScanStoreMigrated ) { File nativeLabelIndex = migrationLayout.labelScanStore(); moveNativeIndexFile( directoryLayout, nativeLabelIndex ); deleteLuceneLabelIndex( getLuceneStoreDirectory( directoryLayout ) ); } }
/** * Performs construction of {@link StoreUpgrader} and all of the necessary participants and performs store * migration if that is required. * @param directoryStructure database to migrate */ public void migrate( DatabaseLayout directoryStructure ) { LogProvider logProvider = logService.getInternalLogProvider(); UpgradableDatabase upgradableDatabase = new UpgradableDatabase( new StoreVersionCheck( pageCache ), format, tailScanner ); StoreUpgrader storeUpgrader = new StoreUpgrader( upgradableDatabase, progressMonitor, config, fs, pageCache, logProvider ); ExplicitIndexMigrator explicitIndexMigrator = new ExplicitIndexMigrator( fs, explicitIndexProvider, logProvider ); StoreMigrator storeMigrator = new StoreMigrator( fs, pageCache, config, logService, jobScheduler ); NativeLabelScanStoreMigrator nativeLabelScanStoreMigrator = new NativeLabelScanStoreMigrator( fs, pageCache, config ); CountsMigrator countsMigrator = new CountsMigrator( fs, pageCache, config ); indexProviderMap.accept( provider -> storeUpgrader.addParticipant( provider.storeMigrationParticipant( fs, pageCache ) ) ); storeUpgrader.addParticipant( explicitIndexMigrator ); storeUpgrader.addParticipant( storeMigrator ); storeUpgrader.addParticipant( nativeLabelScanStoreMigrator ); storeUpgrader.addParticipant( countsMigrator ); storeUpgrader.migrateIfNeeded( directoryStructure ); } }
@Test public void moveCreatedNativeLabelIndexBackToStoreDirectory() throws IOException { prepareEmpty23Database(); indexMigrator.migrate( databaseLayout, migrationLayout, progressReporter, StandardV2_3.STORE_VERSION, StandardV3_2.STORE_VERSION ); File migrationNativeIndex = migrationLayout.labelScanStore(); ByteBuffer migratedFileContent = writeFile( migrationNativeIndex, new byte[]{5, 4, 3, 2, 1} ); indexMigrator.moveMigratedFiles( migrationLayout, databaseLayout, StandardV2_3.STORE_VERSION, StandardV3_2.STORE_VERSION ); ByteBuffer movedNativeIndex = readFileContent( nativeLabelIndex, 5 ); assertEquals( migratedFileContent, movedNativeIndex ); }
@Test public void skipMigrationIfNativeIndexExist() throws Exception { ByteBuffer sourceBuffer = writeFile( nativeLabelIndex, new byte[]{1, 2, 3} ); indexMigrator.migrate( databaseLayout, migrationLayout, progressReporter, StandardV3_2.STORE_VERSION, StandardV3_2.STORE_VERSION ); indexMigrator.moveMigratedFiles( migrationLayout, databaseLayout, StandardV3_2.STORE_VERSION, StandardV3_2.STORE_VERSION ); ByteBuffer resultBuffer = readFileContent( nativeLabelIndex, 3 ); assertEquals( sourceBuffer, resultBuffer ); assertTrue( fileSystem.fileExists( luceneLabelScanStore ) ); }
@Test public void populateNativeLabelScanIndexDuringMigration() throws IOException { prepare34DatabaseWithNodes(); indexMigrator.migrate( databaseLayout, migrationLayout, progressReporter, StandardV3_4.STORE_VERSION, StandardV3_4.STORE_VERSION ); indexMigrator.moveMigratedFiles( migrationLayout, databaseLayout, StandardV2_3.STORE_VERSION, StandardV3_2.STORE_VERSION ); try ( Lifespan lifespan = new Lifespan() ) { NativeLabelScanStore labelScanStore = getNativeLabelScanStore( databaseLayout, true ); lifespan.add( labelScanStore ); for ( int labelId = 0; labelId < 10; labelId++ ) { try ( LabelScanReader labelScanReader = labelScanStore.newReader() ) { int nodeCount = PrimitiveLongCollections.count( labelScanReader.nodesWithLabel( labelId ) ); assertEquals( format( "Expected to see only one node for label %d but was %d.", labelId, nodeCount ), 1, nodeCount ); } } } }