@Override public void close() { super.close(); typeCounts.close(); monitor.relationshipsImported( relationshipCount ); } }
public WriteGroupsStage( Configuration config, RelationshipGroupCache cache, RecordStore<RelationshipGroupRecord> store ) { super( NAME, null, config, 0 ); add( new ReadGroupsFromCacheStep( control(), config, cache.iterator(), GROUP_ENTRY_SIZE ) ); add( new EncodeGroupsStep( control(), config, store ) ); add( new UpdateRecordsStep<>( control(), config, store, new StorePrepareIdSequence() ) ); } }
public IdMapperPreparationStage( Configuration config, IdMapper idMapper, LongFunction<Object> inputIdLookup, Collector collector, StatsProvider memoryUsageStats ) { super( NAME, null, config, 0 ); add( new IdMapperPreparationStep( control(), config, idMapper, inputIdLookup, collector, memoryUsageStats ) ); } }
private InputIterable nodes( final long randomSeed, final long count, int batchSize, final InputIdGenerator inputIdGenerator, final IdGroupDistribution groups ) { return () -> new GeneratingInputIterator<>( count, batchSize, new RandomsStates( randomSeed ), ( randoms, visitor, id ) -> { Object nodeId = inputIdGenerator.nextNodeId( randoms, id ); Group group = groups.groupOf( id ); visitor.id( nodeId, group ); randomProperties( randoms, uniqueId( group, nodeId ), visitor ); visitor.labels( randoms.selection( TOKENS, 0, TOKENS.length, true ) ); }, 0 ); }
public SparseNodeFirstRelationshipStage( Configuration config, NodeStore nodeStore, NodeRelationshipCache cache ) { super( NAME, null, config, ORDER_SEND_DOWNSTREAM | RECYCLE_BATCHES ); add( new ReadNodeIdsByCacheStep( control(), config, cache, NodeType.NODE_TYPE_SPARSE ) ); add( new ReadRecordsStep<>( control(), config, true, nodeStore ) ); add( new RecordProcessorStep<>( control(), "LINK", config, new SparseNodeFirstRelationshipProcessor( cache ), false ) ); add( new UpdateRecordsStep<>( control(), config, nodeStore, new StorePrepareIdSequence() ) ); } }
@Test public void shouldGoPageWiseBackwards() { // GIVEN RecordIdIterator ids = RecordIdIterator.backwards( 0, 33, withBatchSize( DEFAULT, 10 ) ); // THEN assertIds( ids, array( 30, 31, 32 ), array( 20, 21, 22, 23, 24, 25, 26, 27, 28, 29 ), array( 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ), array( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ) ); }
public NodeCountsAndLabelIndexBuildStage( Configuration config, NodeLabelsCache cache, NodeStore nodeStore, int highLabelId, CountsAccessor.Updater countsUpdater, ProgressReporter progressReporter, LabelScanStore labelIndex, StatsProvider... additionalStatsProviders ) { super( NAME, null, config, ORDER_SEND_DOWNSTREAM | RECYCLE_BATCHES ); add( new BatchFeedStep( control(), config, allIn( nodeStore, config ), nodeStore.getRecordSize() ) ); add( new ReadRecordsStep<>( control(), config, false, nodeStore ) ); add( new LabelIndexWriterStep( control(), config, labelIndex, nodeStore ) ); add( new RecordProcessorStep<>( control(), "COUNT", config, new NodeCountsProcessor( nodeStore, cache, highLabelId, countsUpdater, progressReporter ), true, additionalStatsProviders ) ); } }
public RelationshipGroupStage( String topic, Configuration config, RecordStore<RelationshipGroupRecord> store, NodeRelationshipCache cache ) { super( NAME, topic, config, RECYCLE_BATCHES ); add( new ReadGroupRecordsByCacheStep( control(), config, store, cache ) ); add( new UpdateRecordsStep<>( control(), config, store, new StorePrepareIdSequence() ) ); } }
@Test public void shouldGoForwardsWhenStartingFromNonZero2() { // GIVEN RecordIdIterator ids = RecordIdIterator.forwards( 34, 66, withBatchSize( DEFAULT, 10 ) ); // THEN assertIds( ids, array( 34, 35, 36, 37, 38, 39 ), array( 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 ), array( 50, 51, 52, 53, 54, 55, 56, 57, 58, 59 ), array( 60, 61, 62, 63, 64, 65 ) ); }
public static void importNodes( int numRunners, Input input, BatchingNeoStores stores, IdMapper idMapper, ExecutionMonitor executionMonitor, Monitor monitor ) throws IOException { Supplier<EntityImporter> importers = () -> new NodeImporter( stores, idMapper, monitor ); importData( NODE_IMPORT_NAME, numRunners, input.nodes(), stores, importers, executionMonitor, new MemoryUsageStatsProvider( stores, idMapper ) ); }
public Backwards( long lowIncluded, long highExcluded, Configuration config ) { this.lowIncluded = lowIncluded; this.highExcluded = highExcluded; this.batchSize = config.batchSize(); this.endId = highExcluded; }
@Override protected Object nextBatchOrNull( long ticket, int batchSize ) { count += batchSize; return ids.nextBatch(); }
@Override public int maxNumberOfProcessors() { return Configuration.allAvailableProcessors(); } }
@Override public void close() { monitor.propertiesImported( propertyCount ); } }
public RelationshipLinkStep( StageControl control, Configuration config, NodeRelationshipCache cache, Predicate<RelationshipRecord> filter, int nodeTypes, boolean forwards, StatsProvider... additionalStatsProvider ) { super( control, "LINK", config, additionalStatsProvider ); this.cache = cache; this.filter = filter; this.nodeTypes = nodeTypes; this.forwards = forwards; this.progress = findLinkingProgressStatsProvider(); }
@Test public void shouldGoForwardsWhenStartingFromNonZero() { // GIVEN RecordIdIterator ids = RecordIdIterator.forwards( 1, 12, withBatchSize( DEFAULT, 10 ) ); // THEN assertIds( ids, array( 1, 2, 3, 4, 5, 6, 7, 8, 9 ), array( 10, 11 ) ); }
public ProducerStep( StageControl control, Configuration config ) { super( control, ">", config ); this.batchSize = config.batchSize(); }
@Test public void shouldGoPageWiseBackwardsOnCleanBreak() { // GIVEN RecordIdIterator ids = RecordIdIterator.backwards( 0, 20, withBatchSize( DEFAULT, 10 ) ); // THEN assertIds( ids, array( 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ), array( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ) ); }
@Test public void shouldGoPageWiseBackwardsOnSingleBatch() { // GIVEN RecordIdIterator ids = RecordIdIterator.backwards( 0, 8, withBatchSize( DEFAULT, 10 ) ); // THEN assertIds( ids, array( 0, 1, 2, 3, 4, 5, 6, 7 ) ); }