@Test public void progressNeverReportMoreThenHundredPercent() { AssertableLogProvider logProvider = new AssertableLogProvider(); Log log = logProvider.getLog( getClass() ); VisibleMigrationProgressMonitor monitor = new VisibleMigrationProgressMonitor( log ); monitor.started( 1 ); monitorSection( monitor, "First", 100, 1, 10, 99, 170 ); monitor.completed(); verifySectionReportedCorrectly( logProvider ); }
private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCache pageCache, Config config ) { SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); return newUpgrader( upgradableDatabase, pageCache, config, progressMonitor ); }
private void upgradeStore( RecordFormats format, LogTailScanner tailScanner ) { VisibleMigrationProgressMonitor progressMonitor = new VisibleMigrationProgressMonitor( logService.getUserLog( StoreMigrator.class ) ); new DatabaseMigrator( progressMonitor, fs, config, logService, indexProviderMap, explicitIndexProvider, pageCache, format, tailScanner, scheduler ).migrate( databaseLayout ); }
@Test public void shouldComputeTheLastTxInfoCorrectly() throws Exception { // given DatabaseLayout databaseLayout = directory.databaseLayout(); File prepare = directory.directory( "prepare" ); MigrationTestUtils.prepareSampleLegacyDatabase( version, fs, databaseLayout.databaseDirectory(), prepare ); // and a state of the migration saying that it has done the actual migration LogService logService = NullLogService.getInstance(); PageCache pageCache = pageCacheRule.getPageCache( fs ); LogTailScanner tailScanner = getTailScanner( databaseLayout.databaseDirectory() ); UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache, tailScanner ); String versionToMigrateFrom = upgradableDatabase.checkUpgradable( databaseLayout ).storeVersion(); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ); DatabaseLayout migrationLayout = directory.databaseLayout( StoreUpgrader.MIGRATION_DIRECTORY ); // when migrator.migrate( databaseLayout, migrationLayout, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() ); // then assertTrue( txIdComparator.apply( migrator.readLastTxInformation( migrationLayout ) ) ); }
/** * Builds the counts store. Requires that {@link #importNodes()} and {@link #importRelationships()} has run. */ public void buildCountsStore() { // Count nodes per label and labels per node try ( CountsAccessor.Updater countsUpdater = neoStore.getCountsStore().reset( neoStore.getLastCommittedTransactionId() ) ) { MigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); nodeLabelsCache = new NodeLabelsCache( numberArrayFactory, neoStore.getLabelRepository().getHighId() ); MemoryUsageStatsProvider memoryUsageStats = new MemoryUsageStatsProvider( neoStore, nodeLabelsCache ); executeStage( new NodeCountsAndLabelIndexBuildStage( config, nodeLabelsCache, neoStore.getNodeStore(), neoStore.getLabelRepository().getHighId(), countsUpdater, progressMonitor.startSection( "Nodes" ), neoStore.getLabelScanStore(), memoryUsageStats ) ); // Count label-[type]->label executeStage( new RelationshipCountsStage( config, nodeLabelsCache, neoStore.getRelationshipStore(), neoStore.getLabelRepository().getHighId(), neoStore.getRelationshipTypeRepository().getHighId(), countsUpdater, numberArrayFactory, progressMonitor.startSection( "Relationships" ) ) ); } }
private void migrateStore( DatabaseLayout dbDirectoryLayout, DatabaseLayout migrationLayout, File migrationStateFile ) { // One or more participants would like to do migration progressMonitor.started( participants.size() ); MigrationStatus migrationStatus = MigrationStatus.readMigrationStatus( fileSystem, migrationStateFile ); String versionToMigrateFrom = null; // We don't need to migrate if we're at the phase where we have migrated successfully // and it's just a matter of moving over the files to the storeDir. if ( MigrationStatus.migrating.isNeededFor( migrationStatus ) ) { versionToMigrateFrom = upgradableDatabase.checkUpgradable( dbDirectoryLayout ).storeVersion(); cleanMigrationDirectory( migrationLayout.databaseDirectory() ); MigrationStatus.migrating.setMigrationStatus( fileSystem, migrationStateFile, versionToMigrateFrom ); migrateToIsolatedDirectory( dbDirectoryLayout, migrationLayout, versionToMigrateFrom ); MigrationStatus.moving.setMigrationStatus( fileSystem, migrationStateFile, versionToMigrateFrom ); } if ( MigrationStatus.moving.isNeededFor( migrationStatus ) ) { versionToMigrateFrom = MigrationStatus.moving.maybeReadInfo( fileSystem, migrationStateFile, versionToMigrateFrom ); moveMigratedFilesToStoreDirectory( participants, migrationLayout, dbDirectoryLayout, versionToMigrateFrom, upgradableDatabase.currentVersion() ); } cleanup( participants, migrationLayout ); progressMonitor.completed(); }
private void migrateToIsolatedDirectory( DatabaseLayout directoryLayout, DatabaseLayout migrationLayout, String versionToMigrateFrom ) { try { for ( StoreMigrationParticipant participant : participants ) { ProgressReporter progressReporter = progressMonitor.startSection( participant.getName() ); participant.migrate( directoryLayout, migrationLayout, progressReporter, versionToMigrateFrom, upgradableDatabase.currentVersion() ); progressReporter.completed(); } } catch ( IOException | UncheckedIOException e ) { throw new UnableToUpgradeException( "Failure doing migration", e ); } }
private void monitorSection( VisibleMigrationProgressMonitor monitor, String name, int max, int... steps ) { ProgressReporter progressReporter = monitor.startSection( name ); progressReporter.start( max ); for ( int step : steps ) { progressReporter.progress( step ); } progressReporter.completed(); } }
@Test public void shouldComputeTheLastTxLogPositionCorrectly() throws Throwable { // GIVEN a legacy database DatabaseLayout databaseLayout = directory.databaseLayout(); File prepare = directory.directory( "prepare" ); MigrationTestUtils.prepareSampleLegacyDatabase( version, fs, databaseLayout.databaseDirectory(), prepare ); // and a state of the migration saying that it has done the actual migration LogService logService = NullLogService.getInstance(); PageCache pageCache = pageCacheRule.getPageCache( fs ); LogTailScanner tailScanner = getTailScanner( databaseLayout.databaseDirectory() ); UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache, tailScanner ); String versionToMigrateFrom = upgradableDatabase.checkUpgradable( databaseLayout ).storeVersion(); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ); DatabaseLayout migrationLayout = directory.databaseLayout( StoreUpgrader.MIGRATION_DIRECTORY ); // WHEN migrating migrator.migrate( databaseLayout, migrationLayout, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() ); // THEN it should compute the correct last tx log position assertEquals( expectedLogPosition, migrator.readLastTxLogPosition( migrationLayout ) ); }
@Test public void shouldReportAllPercentageSteps() { // GIVEN AssertableLogProvider logProvider = new AssertableLogProvider(); Log log = logProvider.getLog( getClass() ); VisibleMigrationProgressMonitor monitor = new VisibleMigrationProgressMonitor( log ); monitor.started( 1 ); // WHEN monitorSection( monitor, "First", 100, 40, 25, 23 /*these are too far*/ , 10, 50 ); monitor.completed(); // THEN verifySectionReportedCorrectly( logProvider ); }
/** * Builds the counts store. Requires that {@link #importNodes()} and {@link #importRelationships()} has run. */ public void buildCountsStore() { // Count nodes per label and labels per node try ( CountsAccessor.Updater countsUpdater = neoStore.getCountsStore().reset( neoStore.getLastCommittedTransactionId() ) ) { MigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); nodeLabelsCache = new NodeLabelsCache( numberArrayFactory, neoStore.getLabelRepository().getHighId() ); MemoryUsageStatsProvider memoryUsageStats = new MemoryUsageStatsProvider( neoStore, nodeLabelsCache ); executeStage( new NodeCountsAndLabelIndexBuildStage( config, nodeLabelsCache, neoStore.getNodeStore(), neoStore.getLabelRepository().getHighId(), countsUpdater, progressMonitor.startSection( "Nodes" ), neoStore.getLabelScanStore(), memoryUsageStats ) ); // Count label-[type]->label executeStage( new RelationshipCountsStage( config, nodeLabelsCache, neoStore.getRelationshipStore(), neoStore.getLabelRepository().getHighId(), neoStore.getRelationshipTypeRepository().getHighId(), countsUpdater, numberArrayFactory, progressMonitor.startSection( "Relationships" ) ) ); } }
StoreVersionCheck check = new StoreVersionCheck( pageCache ); UpgradableDatabase upgradableDatabase = getUpgradableDatabase( check ); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); LogService logService = NullLogService.getInstance(); StoreMigrator failingStoreMigrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ) progressMonitor = new SilentMigrationProgressMonitor(); StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ); SchemaIndexMigrator indexMigrator = createIndexMigrator();
@Test public void upgradeShouldGiveProgressMonitorProgressMessages() throws Exception { // Given PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache ); // When AssertableLogProvider logProvider = new AssertableLogProvider(); newUpgrader( upgradableDatabase, pageCache, allowMigrateConfig, new VisibleMigrationProgressMonitor( logProvider.getLog( "test" ) ) ).migrateIfNeeded( databaseLayout ); // Then logProvider.assertContainsLogCallContaining( "Store files" ); logProvider.assertContainsLogCallContaining( "Indexes" ); logProvider.assertContainsLogCallContaining( "Counts store" ); logProvider.assertContainsLogCallContaining( "Successfully finished" ); }
private void migrateStore( DatabaseLayout dbDirectoryLayout, DatabaseLayout migrationLayout, File migrationStateFile ) { // One or more participants would like to do migration progressMonitor.started( participants.size() ); MigrationStatus migrationStatus = MigrationStatus.readMigrationStatus( fileSystem, migrationStateFile ); String versionToMigrateFrom = null; // We don't need to migrate if we're at the phase where we have migrated successfully // and it's just a matter of moving over the files to the storeDir. if ( MigrationStatus.migrating.isNeededFor( migrationStatus ) ) { versionToMigrateFrom = upgradableDatabase.checkUpgradable( dbDirectoryLayout ).storeVersion(); cleanMigrationDirectory( migrationLayout.databaseDirectory() ); MigrationStatus.migrating.setMigrationStatus( fileSystem, migrationStateFile, versionToMigrateFrom ); migrateToIsolatedDirectory( dbDirectoryLayout, migrationLayout, versionToMigrateFrom ); MigrationStatus.moving.setMigrationStatus( fileSystem, migrationStateFile, versionToMigrateFrom ); } if ( MigrationStatus.moving.isNeededFor( migrationStatus ) ) { versionToMigrateFrom = MigrationStatus.moving.maybeReadInfo( fileSystem, migrationStateFile, versionToMigrateFrom ); moveMigratedFilesToStoreDirectory( participants, migrationLayout, dbDirectoryLayout, versionToMigrateFrom, upgradableDatabase.currentVersion() ); } cleanup( participants, migrationLayout ); progressMonitor.completed(); }
private void migrateToIsolatedDirectory( DatabaseLayout directoryLayout, DatabaseLayout migrationLayout, String versionToMigrateFrom ) { try { for ( StoreMigrationParticipant participant : participants ) { ProgressReporter progressReporter = progressMonitor.startSection( participant.getName() ); participant.migrate( directoryLayout, migrationLayout, progressReporter, versionToMigrateFrom, upgradableDatabase.currentVersion() ); progressReporter.completed(); } } catch ( IOException | UncheckedIOException e ) { throw new UnableToUpgradeException( "Failure doing migration", e ); } }
SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ); DatabaseLayout migrationLayout = directory.databaseLayout( StoreUpgrader.MIGRATION_DIRECTORY ); migrator.migrate( databaseLayout, migrationLayout, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() ); progressMonitor = new SilentMigrationProgressMonitor(); CountsMigrator countsMigrator = new CountsMigrator( fs, pageCache, CONFIG ); countsMigrator.migrate( databaseLayout, migrationLayout, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() ); migrator.moveMigratedFiles( migrationLayout, databaseLayout, versionToMigrateFrom,
StoreVersionCheck check = new StoreVersionCheck( pageCache ); UpgradableDatabase upgradableDatabase = getUpgradableDatabase( check ); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); LogService logService = NullLogService.getInstance(); StoreMigrator failingStoreMigrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ) progressMonitor = new SilentMigrationProgressMonitor(); StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ); newUpgrader( upgradableDatabase, pageCache, progressMonitor, createIndexMigrator(), migrator )
private void upgradeStore( RecordFormats format, LogTailScanner tailScanner ) { VisibleMigrationProgressMonitor progressMonitor = new VisibleMigrationProgressMonitor( logService.getUserLog( StoreMigrator.class ) ); new DatabaseMigrator( progressMonitor, fs, config, logService, indexProviderMap, explicitIndexProvider, pageCache, format, tailScanner, scheduler ).migrate( databaseLayout ); }
SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ); CountsMigrator countsMigrator = new CountsMigrator( fs, pageCache, CONFIG ); migrator.migrate( databaseLayout, migrationLayout, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() ); countsMigrator .migrate( databaseLayout, migrationLayout, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() ); migrator.moveMigratedFiles( migrationLayout, databaseLayout, versionToMigrateFrom,
SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ); CountsMigrator countsMigrator = new CountsMigrator( fs, pageCache, CONFIG ); DatabaseLayout migrationLayout = directory.databaseLayout( StoreUpgrader.MIGRATION_DIRECTORY ); migrator.migrate( databaseLayout, migrationLayout, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() ); countsMigrator .migrate( databaseLayout, migrationLayout, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() );