private void createDirtyIndex( PageCache pageCache ) throws IOException { NativeLabelScanStore nativeLabelScanStore = null; try { nativeLabelScanStore = new NativeLabelScanStore( pageCache, testDirectory.databaseLayout(), fileSystemRule.get(), THROWING_STREAM, false, new Monitors(), immediate() ); nativeLabelScanStore.init(); nativeLabelScanStore.start(); } catch ( IllegalArgumentException e ) { if ( nativeLabelScanStore != null ) { nativeLabelScanStore.shutdown(); } } }
NativeLabelScanStore( PageCache pageCache, DatabaseLayout directoryStructure, FileSystemAbstraction fs, FullStoreChangeStream fullStoreChangeStream, boolean readOnly, Monitors monitors, RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, int pageSize ) { this.pageCache = pageCache; this.fs = fs; this.pageSize = pageSize; this.fullStoreChangeStream = fullStoreChangeStream; this.directoryStructure = directoryStructure; this.storeFile = getLabelScanStoreFile( directoryStructure ); this.readOnly = readOnly; this.monitors = monitors; this.monitor = monitors.newMonitor( Monitor.class ); this.recoveryCleanupWorkCollector = recoveryCleanupWorkCollector; this.fileSystem = fs; }
@Override public void run() { if ( labelScanStore instanceof NativeLabelScanStore ) { if ( ((NativeLabelScanStore)labelScanStore).isDirty() ) { reporter.report( new LabelScanIndex( labelScanStore.getLabelScanStoreFile() ), ConsistencyReport.LabelScanConsistencyReport.class, RecordType.LABEL_SCAN_DOCUMENT ).dirtyIndex(); } } } }
private NativeLabelScanStore buildLabelIndex() throws IOException { NativeLabelScanStore labelIndex = new NativeLabelScanStore( pageCache, databaseLayout, fileSystem, new FullLabelStream( storeIndexStoreView ), false, monitors, RecoveryCleanupWorkCollector.immediate() ); if ( labelsTouched ) { labelIndex.drop(); } // Rebuild will happen as part of this call if it was dropped life.add( labelIndex ); return labelIndex; }
private LabelScanStore startLabelScanStore( PageCache pageCache, IndexStoreView indexStoreView, Monitors monitors ) { NativeLabelScanStore labelScanStore = new NativeLabelScanStore( pageCache, directory.databaseLayout(), fileSystem, new FullLabelStream( indexStoreView ), false, monitors, RecoveryCleanupWorkCollector.immediate() ); try { labelScanStore.init(); labelScanStore.start(); } catch ( IOException e ) { throw new UncheckedIOException( e ); } return labelScanStore; }
private LabelScanStore getLabelScanStore( FileSystemAbstraction fileSystemAbstraction, DatabaseLayout databaseLayout, FullStoreChangeStream fullStoreChangeStream, boolean readOnly, Monitors monitors ) { PageCache pageCache = pageCacheRule.getPageCache( fileSystemAbstraction ); return new NativeLabelScanStore( pageCache, databaseLayout, fileSystemAbstraction, fullStoreChangeStream, readOnly, monitors, RecoveryCleanupWorkCollector.immediate() ); }
private void initializeNativeLabelScanStoreWithContent( DatabaseLayout databaseLayout ) throws IOException { try ( Lifespan lifespan = new Lifespan() ) { NativeLabelScanStore nativeLabelScanStore = getNativeLabelScanStore( databaseLayout, false ); lifespan.add( nativeLabelScanStore ); try ( LabelScanWriter labelScanWriter = nativeLabelScanStore.newWriter() ) { labelScanWriter.write( NodeLabelUpdate.labelChanges( 1, new long[0], new long[]{1} ) ); } nativeLabelScanStore.force( IOLimiter.UNLIMITED ); } }
private void verifyReads( long[] expected ) { try ( LabelScanReader reader = store.newReader() ) { for ( int i = 0; i < LABEL_COUNT; i++ ) { long[] actualNodes = asArray( reader.nodesWithLabel( i ) ); long[] expectedNodes = nodesWithLabel( expected, i ); assertArrayEquals( expectedNodes, actualNodes ); } } }
private void randomModifications( long[] expected, int count ) throws IOException { BitSet editedNodes = new BitSet(); try ( LabelScanWriter writer = store.newWriter() ) { for ( int i = 0; i < count; i++ ) { int nodeId = random.nextInt( NODE_COUNT ); if ( editedNodes.get( nodeId ) ) { i--; continue; } int changeSize = random.nextInt( 3 ) + 1; long labels = expected[nodeId]; long[] labelsBefore = getLabels( labels ); for ( int j = 0; j < changeSize; j++ ) { labels = flipRandom( labels, LABEL_COUNT, random.random() ); } long[] labelsAfter = getLabels( labels ); editedNodes.set( nodeId ); NodeLabelUpdate labelChanges = labelChanges( nodeId, labelsBefore, labelsAfter ); writer.write( labelChanges ); expected[nodeId] = labels; } } }
private NativeLabelScanStore getNativeLabelScanStore( DatabaseLayout migrationDirectoryStructure, ProgressReporter progressReporter, NeoStores neoStores ) { NeoStoreIndexStoreView neoStoreIndexStoreView = new NeoStoreIndexStoreView( NO_LOCK_SERVICE, neoStores ); return new NativeLabelScanStore( pageCache, migrationDirectoryStructure, fileSystem, new MonitoredFullLabelStream( neoStoreIndexStoreView, progressReporter ), false, new Monitors(), RecoveryCleanupWorkCollector.immediate() ); }
private NativeLabelScanStore buildLabelIndex() throws IOException { NativeLabelScanStore labelIndex = new NativeLabelScanStore( pageCache, databaseLayout, fileSystem, new FullLabelStream( storeIndexStoreView ), false, monitors, RecoveryCleanupWorkCollector.immediate() ); if ( labelsTouched ) { labelIndex.drop(); } // Rebuild will happen as part of this call if it was dropped life.add( labelIndex ); return labelIndex; }
private void assertNoContentInNativeLabelScanStore( DatabaseLayout databaseLayout ) { try ( Lifespan lifespan = new Lifespan() ) { NativeLabelScanStore nativeLabelScanStore = getNativeLabelScanStore( databaseLayout, true ); lifespan.add( nativeLabelScanStore ); try ( LabelScanReader labelScanReader = nativeLabelScanStore.newReader() ) { int count = PrimitiveLongCollections.count( labelScanReader.nodesWithLabel( 1 ) ); assertEquals( 0, count ); } } }
@Test public void labelScanStoreIsDirtyWhenIndexIsNotClean() throws IOException { PageCache pageCache = pageCacheRule.getPageCache( fileSystemRule.get() ); createDirtyIndex( pageCache ); Monitors monitors = new Monitors(); RecordingMonitor monitor = new RecordingMonitor(); monitors.addMonitorListener( monitor ); NativeLabelScanStore nativeLabelScanStore = new NativeLabelScanStore( pageCache, testDirectory.databaseLayout(), fileSystemRule.get(), EMPTY, true, monitors, ignore() ); nativeLabelScanStore.init(); nativeLabelScanStore.start(); assertTrue( nativeLabelScanStore.isDirty() ); nativeLabelScanStore.shutdown(); }
private void instantiateKernelExtensions() { life = new LifeSupport(); life.start(); labelScanStore = new NativeLabelScanStore( pageCache, databaseLayout, fileSystem, FullStoreChangeStream.EMPTY, false, new Monitors(), RecoveryCleanupWorkCollector.immediate() ); life.add( labelScanStore ); }
private void deleteNativeIndexFile( DatabaseLayout directoryStructure ) throws IOException { Optional<FileHandle> indexFile = fileSystem.streamFilesRecursive( NativeLabelScanStore.getLabelScanStoreFile( directoryStructure ) ).findFirst(); if ( indexFile.isPresent() ) { try { indexFile.get().delete(); } catch ( NoSuchFileException e ) { // Already deleted, ignore } } }
@Test public void populateNativeLabelScanIndexDuringMigration() throws IOException { prepare34DatabaseWithNodes(); indexMigrator.migrate( databaseLayout, migrationLayout, progressReporter, StandardV3_4.STORE_VERSION, StandardV3_4.STORE_VERSION ); indexMigrator.moveMigratedFiles( migrationLayout, databaseLayout, StandardV2_3.STORE_VERSION, StandardV3_2.STORE_VERSION ); try ( Lifespan lifespan = new Lifespan() ) { NativeLabelScanStore labelScanStore = getNativeLabelScanStore( databaseLayout, true ); lifespan.add( labelScanStore ); for ( int labelId = 0; labelId < 10; labelId++ ) { try ( LabelScanReader labelScanReader = labelScanStore.newReader() ) { int nodeCount = PrimitiveLongCollections.count( labelScanReader.nodesWithLabel( labelId ) ); assertEquals( format( "Expected to see only one node for label %d but was %d.", labelId, nodeCount ), 1, nodeCount ); } } } }
@Test public void doNotRebuildIfOpenedInReadOnlyModeAndIndexIsNotClean() throws IOException { PageCache pageCache = pageCacheRule.getPageCache( fileSystemRule.get() ); createDirtyIndex( pageCache ); Monitors monitors = new Monitors(); RecordingMonitor monitor = new RecordingMonitor(); monitors.addMonitorListener( monitor ); NativeLabelScanStore nativeLabelScanStore = new NativeLabelScanStore( pageCache, testDirectory.databaseLayout(), fileSystemRule.get(), EMPTY, true, monitors, ignore() ); nativeLabelScanStore.init(); nativeLabelScanStore.start(); assertTrue( monitor.notValid ); assertFalse( monitor.rebuilt ); assertFalse( monitor.rebuilding ); nativeLabelScanStore.shutdown(); }
private NativeLabelScanStore getNativeLabelScanStore( DatabaseLayout databaseLayout, boolean readOnly ) { return new NativeLabelScanStore( pageCache, databaseLayout, fileSystem, FullStoreChangeStream.EMPTY, readOnly, new Monitors(), RecoveryCleanupWorkCollector.ignore() ); }
/** * Called when expecting a clean {@code storeDir} folder and where a new store will be created. * This happens on an initial attempt to import. * * @throws IOException on I/O error. * @throws IllegalStateException if {@code storeDir} already contains a database. */ public void createNew() throws IOException { assertDatabaseIsEmptyOrNonExistent(); // There may have been a previous import which was killed before it even started, where the label scan store could // be in a semi-initialized state. Better to be on the safe side and deleted it. We get her after determining that // the db is either completely empty or non-existent anyway, so deleting this file is OK. fileSystem.deleteFile( getLabelScanStoreFile( databaseLayout ) ); instantiateStores(); neoStores.getMetaDataStore().setLastCommittedAndClosedTransactionId( initialIds.lastCommittedTransactionId(), initialIds.lastCommittedTransactionChecksum(), BASE_TX_COMMIT_TIMESTAMP, initialIds.lastCommittedTransactionLogByteOffset(), initialIds.lastCommittedTransactionLogVersion() ); neoStores.startCountStore(); }
@Test public void mustBeDirtyIfFailedDuringRebuild() throws Exception { // given PageCache pageCache = pageCacheRule.getPageCache( fileSystemRule.get() ); createDirtyIndex( pageCache ); // when RecordingMonitor monitor = new RecordingMonitor(); Monitors monitors = new Monitors(); monitors.addMonitorListener( monitor ); NativeLabelScanStore nativeLabelScanStore = new NativeLabelScanStore( pageCache, testDirectory.databaseLayout(), fileSystemRule.get(), EMPTY, false, monitors, immediate() ); nativeLabelScanStore.init(); nativeLabelScanStore.start(); // then assertTrue( monitor.notValid ); assertTrue( monitor.rebuilding ); assertTrue( monitor.rebuilt ); nativeLabelScanStore.shutdown(); }