@Override public void close() throws IOException { // We're done, do some final logging about it long totalTimeMillis = currentTimeMillis() - startTime; DataStatistics state = getState( DataStatistics.class ); String additionalInformation = Objects.toString( state, "Data statistics is not available." ); executionMonitor.done( successful, totalTimeMillis, format( "%n%s%nPeak memory usage: %s", additionalInformation, bytes( peakMemoryUsage ) ) ); log.info( "Import completed successfully, took " + duration( totalTimeMillis ) + ". " + additionalInformation ); closeAll( nodeRelationshipCache, nodeLabelsCache, idMapper ); }
public static String time( long millis ) { return time( millis, DEFAULT_TIME_ZONE ); }
private void initializeRelationshipImport( Estimates estimates, IdMapper idMapper, BatchingNeoStores neoStores ) { long numberOfRelationships = estimates.numberOfRelationships(); printStageHeader( "(2/4) Relationship import", ESTIMATED_NUMBER_OF_RELATIONSHIPS, count( numberOfRelationships ), ESTIMATED_DISK_SPACE_USAGE, bytes( relationshipsDiskUsage( estimates, neoStores ) + estimates.sizeOfRelationshipProperties() ), ESTIMATED_REQUIRED_MEMORY_USAGE, bytes( baselineMemoryRequirement( neoStores ) + totalMemoryUsageOf( idMapper ) ) ); initializeProgress( numberOfRelationships, ImportStage.relationshipImport ); }
public static String duration( long durationMillis ) { return duration( durationMillis, TimeUnit.DAYS, TimeUnit.MILLISECONDS ); }
@Override public String timestamp( long timeWritten, TimeZone timeZone ) { return Format.date( timeWritten, timeZone ) + "/" + timeWritten; } }
@Test void dateTime() throws Exception { // Date long timeWithDate = System.currentTimeMillis(); String dateAsString = Format.date( timeWithDate ); assertEquals( timeWithDate, new SimpleDateFormat( Format.DATE_FORMAT ).parse( dateAsString ).getTime() ); // Time String timeAsString = Format.time( timeWithDate ); assertEquals( timeWithDate, translateToDate( timeWithDate, new SimpleDateFormat( Format.TIME_FORMAT ).parse( timeAsString ).getTime(), Format.DEFAULT_TIME_ZONE ) ); }
@Test void shouldDisplayPlainCount() { // when String format = Format.count( 10 ); // then assertTrue( format.startsWith( "10" ) ); }
@Override public void initialize( DependencyResolver dependencyResolver ) { this.dependencyResolver = dependencyResolver; Estimates estimates = dependencyResolver.resolveDependency( Estimates.class ); BatchingNeoStores neoStores = dependencyResolver.resolveDependency( BatchingNeoStores.class ); IdMapper idMapper = dependencyResolver.resolveDependency( IdMapper.class ); NodeRelationshipCache nodeRelationshipCache = dependencyResolver.resolveDependency( NodeRelationshipCache.class ); pageCacheArrayFactoryMonitor = dependencyResolver.resolveDependency( PageCacheArrayFactoryMonitor.class ); long biggestCacheMemory = estimatedCacheSize( neoStores, nodeRelationshipCache.memoryEstimation( estimates.numberOfNodes() ), idMapper.memoryEstimation( estimates.numberOfNodes() ) ); printStageHeader( "Import starting", ESTIMATED_NUMBER_OF_NODES, count( estimates.numberOfNodes() ), ESTIMATED_NUMBER_OF_NODE_PROPERTIES, count( estimates.numberOfNodeProperties() ), ESTIMATED_NUMBER_OF_RELATIONSHIPS, count( estimates.numberOfRelationships() ), ESTIMATED_NUMBER_OF_RELATIONSHIP_PROPERTIES, count( estimates.numberOfRelationshipProperties() ), ESTIMATED_DISK_SPACE_USAGE, bytes( nodesDiskUsage( estimates, neoStores ) + relationshipsDiskUsage( estimates, neoStores ) + estimates.sizeOfNodeProperties() + estimates.sizeOfRelationshipProperties() ), ESTIMATED_REQUIRED_MEMORY_USAGE, bytes( biggestCacheMemory ) ); System.out.println(); }
private String durationSinceLastReport() { long diff = currentTimeMillis() - lastReportTime; lastReportTime = currentTimeMillis(); return duration( diff ); }
public static String date( long millis ) { return date( millis, DEFAULT_TIME_ZONE ); }
@Test void dateTimeWithTimeZone() throws Exception { String zoneOffset = "+03:00"; TimeZone zone = TimeZone.getTimeZone( "GMT" + zoneOffset ); // Date String asString = Format.date( zone ); assertTrue( asString.endsWith( withoutColon( zoneOffset ) ) ); long timeWithDate = new SimpleDateFormat( Format.DATE_FORMAT ).parse( asString ).getTime(); asString = Format.date( timeWithDate, zone ); assertTrue( asString.endsWith( withoutColon( zoneOffset ) ) ); assertEquals( timeWithDate, new SimpleDateFormat( Format.DATE_FORMAT ).parse( asString ).getTime() ); asString = Format.date( new Date( timeWithDate ), zone ); assertTrue( asString.endsWith( withoutColon( zoneOffset ) ) ); assertEquals( timeWithDate, new SimpleDateFormat( Format.DATE_FORMAT ).parse( asString ).getTime() ); // Time asString = Format.time( timeWithDate, zone ); assertEquals( timeWithDate, translateToDate( timeWithDate, new SimpleDateFormat( Format.TIME_FORMAT ).parse( asString ).getTime(), zone ) ); asString = Format.time( new Date( timeWithDate ), zone ); assertEquals( timeWithDate, translateToDate( timeWithDate, new SimpleDateFormat( Format.TIME_FORMAT ).parse( asString ).getTime(), zone ) ); }
@Override public void insufficientAvailableMemory( long estimatedCacheSize, long optimalMinimalHeapSize, long availableMemory ) { err.printf( "WARNING: %s memory may not be sufficient to complete this import. Suggested memory distribution is:%n" + "heap size: %s%n" + "minimum free and available memory excluding heap size: %s", bytes( availableMemory ), bytes( optimalMinimalHeapSize ), bytes( estimatedCacheSize ) ); } }
private void initializeNodeImport( Estimates estimates, IdMapper idMapper, BatchingNeoStores neoStores ) { long numberOfNodes = estimates.numberOfNodes(); printStageHeader( "(1/4) Node import", ESTIMATED_NUMBER_OF_NODES, count( numberOfNodes ), ESTIMATED_DISK_SPACE_USAGE, bytes( // node store nodesDiskUsage( estimates, neoStores ) + // property store(s) estimates.sizeOfNodeProperties() ), ESTIMATED_REQUIRED_MEMORY_USAGE, bytes( baselineMemoryRequirement( neoStores ) + defensivelyPadMemoryEstimate( idMapper.memoryEstimation( numberOfNodes ) ) ) ); // A difficulty with the goal here is that we don't know how much work there is to be done in id mapper preparation stage. // In addition to nodes themselves and SPLIT,SORT,DETECT there may be RESOLVE,SORT,DEDUPLICATE too, if there are collisions long goal = idMapper.needsPreparation() ? numberOfNodes + weighted( IdMapperPreparationStage.NAME, numberOfNodes * 4 ) : numberOfNodes; initializeProgress( goal, ImportStage.nodeImport ); }
private void printDetails() { printDetailsHeadline(); long totalTime = 0; for ( StageDetails stageDetails : details ) { stageDetails.print( out ); totalTime += stageDetails.totalTimeMillis; } printIndented( out, "Environment information:" ); printIndented( out, " Free physical memory: " + bytes( OsBeanUtil.getFreePhysicalMemory() ) ); printIndented( out, " Max VM memory: " + bytes( Runtime.getRuntime().maxMemory() ) ); printIndented( out, " Free VM memory: " + bytes( Runtime.getRuntime().freeMemory() ) ); printIndented( out, " VM stop-the-world time: " + duration( vmPauseTimeAccumulator.getPauseTime() ) ); printIndented( out, " Duration: " + duration( totalTime ) ); out.println(); }
public static long time( long startTime, String message ) { System.out.println( duration( currentTimeMillis() - startTime ) + ": " + message ); return currentTimeMillis(); } }
public static String time() { return time( DEFAULT_TIME_ZONE ); }
public static String date() { return date( DEFAULT_TIME_ZONE ); }
@Override public void abundantHeapSize( long optimalMinimalHeapSize, long heapSize ) { err.printf( "WARNING: heap size %s is unnecessarily large for completing this import.%n" + "The abundant heap memory will leave less memory for off-heap importer caches. Suggested heap size is %s", bytes( heapSize ), bytes( optimalMinimalHeapSize ) ); }
private void initializeRelationshipImport( Estimates estimates, IdMapper idMapper, BatchingNeoStores neoStores ) { long numberOfRelationships = estimates.numberOfRelationships(); printStageHeader( "(2/4) Relationship import", ESTIMATED_NUMBER_OF_RELATIONSHIPS, count( numberOfRelationships ), ESTIMATED_DISK_SPACE_USAGE, bytes( relationshipsDiskUsage( estimates, neoStores ) + estimates.sizeOfRelationshipProperties() ), ESTIMATED_REQUIRED_MEMORY_USAGE, bytes( baselineMemoryRequirement( neoStores ) + totalMemoryUsageOf( idMapper ) ) ); initializeProgress( numberOfRelationships, ImportStage.relationshipImport ); }