private RecordProxy<RelationshipRecord,Void> getRelationshipRecord( long id ) { if ( id < 0 || id >= relationshipStore.getHighId() ) { throw new NotFoundException( "id=" + id ); } return recordAccess.getRelRecords().getOrLoad( id, null ); }
private static ExecutionMonitor migrationBatchImporterMonitor( NeoStores legacyStore, final ProgressReporter progressReporter, Configuration config ) { return new BatchImporterProgressMonitor( legacyStore.getNodeStore().getHighId(), legacyStore.getRelationshipStore().getHighId(), config, progressReporter ); }
private void initializeMisc( BatchingNeoStores neoStores, DataStatistics distribution ) { printStageHeader( "(4/4) Post processing", ESTIMATED_REQUIRED_MEMORY_USAGE, bytes( baselineMemoryRequirement( neoStores ) ) ); long actualNodeCount = distribution.getNodeCount(); // The reason the highId of the relationship store is used, as opposed to actual number of imported relationships // is that the stages underneath operate on id ranges, not knowing which records are actually in use. long relationshipRecordIdCount = neoStores.getRelationshipStore().getHighId(); long groupCount = neoStores.getTemporaryRelationshipGroupStore().getHighId(); initializeProgress( groupCount + // Count groups groupCount + // Write groups groupCount + // Node --> Group actualNodeCount + // Node counts relationshipRecordIdCount, // Relationship counts ImportStage.postProcessing ); }
private boolean databaseExistsAndContainsData() { File metaDataFile = databaseLayout.metadataStore(); try ( PagedFile pagedFile = pageCache.map( metaDataFile, pageCache.pageSize(), StandardOpenOption.READ ) ) { // OK so the db probably exists } catch ( IOException e ) { // It's OK return false; } try ( NeoStores stores = newStoreFactory( databaseLayout ).openNeoStores( StoreType.NODE, StoreType.RELATIONSHIP ) ) { return stores.getNodeStore().getHighId() > 0 || stores.getRelationshipStore().getHighId() > 0; } }
private void initializeLinking( BatchingNeoStores neoStores, NodeRelationshipCache nodeRelationshipCache, DataStatistics distribution ) { printStageHeader( "(3/4) Relationship linking", ESTIMATED_REQUIRED_MEMORY_USAGE, bytes( baselineMemoryRequirement( neoStores ) + defensivelyPadMemoryEstimate( nodeRelationshipCache.memoryEstimation( distribution.getNodeCount() ) ) ) ); // The reason the highId of the relationship store is used, as opposed to actual number of imported relationships // is that the stages underneath operate on id ranges, not knowing which records are actually in use. long relationshipRecordIdCount = neoStores.getRelationshipStore().getHighId(); // The progress counting of linking stages is special anyway, in that it uses the "progress" stats key, // which is based on actual number of relationships, not relationship ids. long actualRelationshipCount = distribution.getRelationshipCount(); initializeProgress( relationshipRecordIdCount + // node degrees actualRelationshipCount * 2 + // start/end forwards, see RelationshipLinkingProgress actualRelationshipCount * 2, // start/end backwards, see RelationshipLinkingProgress ImportStage.linking ); }
public NodeDegreeCountStage( Configuration config, RelationshipStore store, NodeRelationshipCache cache, StatsProvider memoryUsageStatsProvider ) { super( NAME, null, config, RECYCLE_BATCHES ); add( new BatchFeedStep( control(), config, forwards( 0, store.getHighId(), config ), store.getRecordSize() ) ); add( new ReadRecordsStep<>( control(), config, false, store ) ); add( new CalculateDenseNodesStep( control(), config, cache, memoryUsageStatsProvider ) ); } }
public RelationshipLinkbackStage( String topic, Configuration config, BatchingNeoStores stores, NodeRelationshipCache cache, Predicate<RelationshipRecord> readFilter, Predicate<RelationshipRecord> changeFilter, int nodeTypes, StatsProvider... additionalStatsProvider ) { super( NAME, topic, config, ORDER_SEND_DOWNSTREAM | RECYCLE_BATCHES ); RelationshipStore store = stores.getRelationshipStore(); add( new BatchFeedStep( control(), config, backwards( 0, store.getHighId(), config ), store.getRecordSize() ) ); add( new ReadRecordsStep<>( control(), config, true, store, new RecordDataAssembler<>( store::newRecord, readFilter ) ) ); add( new RelationshipLinkbackStep( control(), config, cache, changeFilter, nodeTypes, additionalStatsProvider ) ); add( new UpdateRecordsStep<>( control(), config, store, PrepareIdSequence.of( stores.usesDoubleRelationshipRecordUnits() ) ) ); } }
public RelationshipLinkforwardStage( String topic, Configuration config, BatchingNeoStores stores, NodeRelationshipCache cache, Predicate<RelationshipRecord> readFilter, Predicate<RelationshipRecord> denseChangeFilter, int nodeTypes, StatsProvider... additionalStatsProvider ) { super( NAME, topic, config, ORDER_SEND_DOWNSTREAM | RECYCLE_BATCHES ); RelationshipStore store = stores.getRelationshipStore(); add( new BatchFeedStep( control(), config, forwards( 0, store.getHighId(), config ), store.getRecordSize() ) ); add( new ReadRecordsStep<>( control(), config, true, store, new RecordDataAssembler<>( store::newRecord, readFilter ) ) ); add( new RelationshipLinkforwardStep( control(), config, cache, denseChangeFilter, nodeTypes, additionalStatsProvider ) ); add( new UpdateRecordsStep<>( control(), config, store, PrepareIdSequence.of( stores.usesDoubleRelationshipRecordUnits() ) ) ); } }
@Test public void shouldDeleteIdGeneratorsWhenOpeningExistingStore() throws IOException { // given long expectedHighId; try ( BatchingNeoStores stores = BatchingNeoStores.batchingNeoStoresWithExternalPageCache( storage.fileSystem(), storage.pageCache(), PageCacheTracer.NULL, storage.directory().absolutePath(), LATEST_RECORD_FORMATS, DEFAULT, NullLogService.getInstance(), EMPTY, Config.defaults() ) ) { stores.createNew(); RelationshipStore relationshipStore = stores.getRelationshipStore(); RelationshipRecord record = relationshipStore.newRecord(); long no = NULL_REFERENCE.longValue(); record.initialize( true, no, 1, 2, 0, no, no, no, no, true, true ); record.setId( relationshipStore.nextId() ); expectedHighId = relationshipStore.getHighId(); relationshipStore.updateRecord( record ); // fiddle with the highId relationshipStore.setHighId( record.getId() + 999 ); } // when try ( BatchingNeoStores stores = BatchingNeoStores.batchingNeoStoresWithExternalPageCache( storage.fileSystem(), storage.pageCache(), PageCacheTracer.NULL, storage.directory().absolutePath(), LATEST_RECORD_FORMATS, DEFAULT, NullLogService.getInstance(), EMPTY, Config.defaults() ) ) { stores.pruneAndOpenExistingStore( Predicates.alwaysTrue(), Predicates.alwaysTrue() ); // then assertEquals( expectedHighId, stores.getRelationshipStore().getHighId() ); } }
@Test public void shouldSetCorrectHighIdWhenApplyingExternalTransactions() throws Exception { // WHEN recovering a transaction that creates some data long nodeId = neoStores.getNodeStore().nextId(); long relationshipId = neoStores.getRelationshipStore().nextId(); int type = 1; applyExternalTransaction( 1, new NodeCommand( new NodeRecord( nodeId ), inUse( created( new NodeRecord( nodeId ) ) ) ), new RelationshipCommand( null, inUse( created( with( new RelationshipRecord( relationshipId ), nodeId, nodeId, type ) ) ) ) ); // and when, later on, recovering a transaction deleting some of those applyExternalTransaction( 2, new NodeCommand( inUse( created( new NodeRecord( nodeId ) ) ), new NodeRecord( nodeId ) ), new RelationshipCommand( null, new RelationshipRecord( relationshipId ) ) ); // THEN that should be possible and the high ids should be correct, i.e. highest applied + 1 assertEquals( nodeId + 1, neoStores.getNodeStore().getHighId() ); assertEquals( relationshipId + 1, neoStores.getRelationshipStore().getHighId() ); }
assertEquals( relationship.getSecondaryUnitId() + 1, neoStores.getRelationshipStore().getHighId() ); assertEquals( relationshipGroup.getSecondaryUnitId() + 1, neoStores.getRelationshipGroupStore().getHighId() );
assertEquals( "RelationshipStore", 45 + 1, neoStores.getRelationshipStore().getHighId() ); assertEquals( "RelationshipTypeStore", 5 + 1, neoStores.getRelationshipTypeTokenStore().getHighId() ); assertEquals( "RelationshipType NameStore", 1 + 1,
private RecordProxy<RelationshipRecord,Void> getRelationshipRecord( long id ) { if ( id < 0 || id >= relationshipStore.getHighId() ) { throw new NotFoundException( "id=" + id ); } return recordAccess.getRelRecords().getOrLoad( id, null ); }
private static ExecutionMonitor migrationBatchImporterMonitor( NeoStores legacyStore, final ProgressReporter progressReporter, Configuration config ) { return new BatchImporterProgressMonitor( legacyStore.getNodeStore().getHighId(), legacyStore.getRelationshipStore().getHighId(), config, progressReporter ); }
private boolean databaseExistsAndContainsData() { File metaDataFile = databaseLayout.metadataStore(); try ( PagedFile pagedFile = pageCache.map( metaDataFile, pageCache.pageSize(), StandardOpenOption.READ ) ) { // OK so the db probably exists } catch ( IOException e ) { // It's OK return false; } try ( NeoStores stores = newStoreFactory( databaseLayout ).openNeoStores( StoreType.NODE, StoreType.RELATIONSHIP ) ) { return stores.getNodeStore().getHighId() > 0 || stores.getRelationshipStore().getHighId() > 0; } }
private void initializeMisc( BatchingNeoStores neoStores, DataStatistics distribution ) { printStageHeader( "(4/4) Post processing", ESTIMATED_REQUIRED_MEMORY_USAGE, bytes( baselineMemoryRequirement( neoStores ) ) ); long actualNodeCount = distribution.getNodeCount(); // The reason the highId of the relationship store is used, as opposed to actual number of imported relationships // is that the stages underneath operate on id ranges, not knowing which records are actually in use. long relationshipRecordIdCount = neoStores.getRelationshipStore().getHighId(); long groupCount = neoStores.getTemporaryRelationshipGroupStore().getHighId(); initializeProgress( groupCount + // Count groups groupCount + // Write groups groupCount + // Node --> Group actualNodeCount + // Node counts relationshipRecordIdCount, // Relationship counts ImportStage.postProcessing ); }
public NodeDegreeCountStage( Configuration config, RelationshipStore store, NodeRelationshipCache cache, StatsProvider memoryUsageStatsProvider ) { super( NAME, null, config, RECYCLE_BATCHES ); add( new BatchFeedStep( control(), config, forwards( 0, store.getHighId(), config ), store.getRecordSize() ) ); add( new ReadRecordsStep<>( control(), config, false, store ) ); add( new CalculateDenseNodesStep( control(), config, cache, memoryUsageStatsProvider ) ); } }
private void initializeLinking( BatchingNeoStores neoStores, NodeRelationshipCache nodeRelationshipCache, DataStatistics distribution ) { printStageHeader( "(3/4) Relationship linking", ESTIMATED_REQUIRED_MEMORY_USAGE, bytes( baselineMemoryRequirement( neoStores ) + defensivelyPadMemoryEstimate( nodeRelationshipCache.memoryEstimation( distribution.getNodeCount() ) ) ) ); // The reason the highId of the relationship store is used, as opposed to actual number of imported relationships // is that the stages underneath operate on id ranges, not knowing which records are actually in use. long relationshipRecordIdCount = neoStores.getRelationshipStore().getHighId(); // The progress counting of linking stages is special anyway, in that it uses the "progress" stats key, // which is based on actual number of relationships, not relationship ids. long actualRelationshipCount = distribution.getRelationshipCount(); initializeProgress( relationshipRecordIdCount + // node degrees actualRelationshipCount * 2 + // start/end forwards, see RelationshipLinkingProgress actualRelationshipCount * 2, // start/end backwards, see RelationshipLinkingProgress ImportStage.linking ); }
public RelationshipLinkforwardStage( String topic, Configuration config, BatchingNeoStores stores, NodeRelationshipCache cache, Predicate<RelationshipRecord> readFilter, Predicate<RelationshipRecord> denseChangeFilter, int nodeTypes, StatsProvider... additionalStatsProvider ) { super( NAME, topic, config, ORDER_SEND_DOWNSTREAM | RECYCLE_BATCHES ); RelationshipStore store = stores.getRelationshipStore(); add( new BatchFeedStep( control(), config, forwards( 0, store.getHighId(), config ), store.getRecordSize() ) ); add( new ReadRecordsStep<>( control(), config, true, store, new RecordDataAssembler<>( store::newRecord, readFilter ) ) ); add( new RelationshipLinkforwardStep( control(), config, cache, denseChangeFilter, nodeTypes, additionalStatsProvider ) ); add( new UpdateRecordsStep<>( control(), config, store, PrepareIdSequence.of( stores.usesDoubleRelationshipRecordUnits() ) ) ); } }
public RelationshipLinkbackStage( String topic, Configuration config, BatchingNeoStores stores, NodeRelationshipCache cache, Predicate<RelationshipRecord> readFilter, Predicate<RelationshipRecord> changeFilter, int nodeTypes, StatsProvider... additionalStatsProvider ) { super( NAME, topic, config, ORDER_SEND_DOWNSTREAM | RECYCLE_BATCHES ); RelationshipStore store = stores.getRelationshipStore(); add( new BatchFeedStep( control(), config, backwards( 0, store.getHighId(), config ), store.getRecordSize() ) ); add( new ReadRecordsStep<>( control(), config, true, store, new RecordDataAssembler<>( store::newRecord, readFilter ) ) ); add( new RelationshipLinkbackStep( control(), config, cache, changeFilter, nodeTypes, additionalStatsProvider ) ); add( new UpdateRecordsStep<>( control(), config, store, PrepareIdSequence.of( stores.usesDoubleRelationshipRecordUnits() ) ) ); } }