@Override public void insufficientAvailableMemory( long estimatedCacheSize, long optimalMinimalHeapSize, long availableMemory ) { err.printf( "WARNING: %s memory may not be sufficient to complete this import. Suggested memory distribution is:%n" + "heap size: %s%n" + "minimum free and available memory excluding heap size: %s", bytes( availableMemory ), bytes( optimalMinimalHeapSize ), bytes( estimatedCacheSize ) ); } }
@Override public void abundantHeapSize( long optimalMinimalHeapSize, long heapSize ) { err.printf( "WARNING: heap size %s is unnecessarily large for completing this import.%n" + "The abundant heap memory will leave less memory for off-heap importer caches. Suggested heap size is %s", bytes( heapSize ), bytes( optimalMinimalHeapSize ) ); }
private static String memoryStats() { Runtime runtime = Runtime.getRuntime(); return format( "Memory[used:%s, free:%s, total:%s, max:%s]", Format.bytes( runtime.totalMemory() - runtime.freeMemory() ), Format.bytes( runtime.freeMemory() ), Format.bytes( runtime.totalMemory() ), Format.bytes( runtime.maxMemory() ) ); }
@Override public String toString() { long stat = asLong(); return stat == -1 ? "??" : Format.bytes( stat ) + "/s"; } }
private static void logBytes( Logger logger, String message, long value ) { if ( value != OsBeanUtil.VALUE_UNAVAILABLE ) { logger.log( message + bytes( value ) ); } }
@Override public String toString() { return bytes( asLong() ); } }
@Override public void allocationSuccessful( long memory, NumberArrayFactory successfulFactory, Iterable<NumberArrayFactory.AllocationFailure> attemptedAllocationFailures ) { if ( successfulFactory instanceof PageCachedNumberArrayFactory ) { StringBuilder builder = new StringBuilder( format( "Memory allocation of %s ended up in page cache, which may impact performance negatively", bytes( memory ) ) ); attemptedAllocationFailures.forEach( failure -> builder.append( format( "%n%s: %s", failure.getFactory(), failure.getFailure() ) ) ); failedFactoriesDescription.compareAndSet( null, builder.toString() ); } }
private OutOfMemoryError error( long length, int itemSize, OutOfMemoryError error ) { return Exceptions.withMessage( error, format( "%s: Not enough memory available for allocating %s, tried %s", error.getMessage(), bytes( length * itemSize ), Arrays.toString( candidates ) ) ); }
@Override public void close() throws IOException { // We're done, do some final logging about it long totalTimeMillis = currentTimeMillis() - startTime; DataStatistics state = getState( DataStatistics.class ); String additionalInformation = Objects.toString( state, "Data statistics is not available." ); executionMonitor.done( successful, totalTimeMillis, format( "%n%s%nPeak memory usage: %s", additionalInformation, bytes( peakMemoryUsage ) ) ); log.info( "Import completed successfully, took " + duration( totalTimeMillis ) + ". " + additionalInformation ); closeAll( nodeRelationshipCache, nodeLabelsCache, idMapper ); }
static void printOverview( File storeDir, Collection<Option<File[]>> nodesFiles, Collection<Option<File[]>> relationshipsFiles, org.neo4j.unsafe.impl.batchimport.Configuration configuration, PrintStream out ) { out.println( "Neo4j version: " + Version.getNeo4jVersion() ); out.println( "Importing the contents of these files into " + storeDir + ":" ); printInputFiles( "Nodes", nodesFiles, out ); printInputFiles( "Relationships", relationshipsFiles, out ); out.println(); out.println( "Available resources:" ); printIndented( "Total machine memory: " + bytes( OsBeanUtil.getTotalPhysicalMemory() ), out ); printIndented( "Free machine memory: " + bytes( OsBeanUtil.getFreePhysicalMemory() ), out ); printIndented( "Max heap memory : " + bytes( Runtime.getRuntime().maxMemory() ), out ); printIndented( "Processors: " + configuration.maxNumberOfProcessors(), out ); printIndented( "Configured max memory: " + bytes( configuration.maxMemoryUsage() ), out ); printIndented( "High-IO: " + configuration.highIO(), out ); out.println(); }
private void printDetails() { printDetailsHeadline(); long totalTime = 0; for ( StageDetails stageDetails : details ) { stageDetails.print( out ); totalTime += stageDetails.totalTimeMillis; } printIndented( out, "Environment information:" ); printIndented( out, " Free physical memory: " + bytes( OsBeanUtil.getFreePhysicalMemory() ) ); printIndented( out, " Max VM memory: " + bytes( Runtime.getRuntime().maxMemory() ) ); printIndented( out, " Free VM memory: " + bytes( Runtime.getRuntime().freeMemory() ) ); printIndented( out, " VM stop-the-world time: " + duration( vmPauseTimeAccumulator.getPauseTime() ) ); printIndented( out, " Duration: " + duration( totalTime ) ); out.println(); }
void print( PrintStream out ) { printIndented( out, execution.name() ); StringBuilder builder = new StringBuilder(); SpectrumExecutionMonitor.printSpectrum( builder, execution, SpectrumExecutionMonitor.DEFAULT_WIDTH, DetailLevel.NO ); printIndented( out, builder.toString() ); printValue( out, memoryUsage, "Memory usage", Format::bytes ); printValue( out, ioThroughput, "I/O throughput", value -> bytes( value ) + "/s" ); printValue( out, stageVmPauseTime, "VM stop-the-world time", Format::duration ); printValue( out, totalTimeMillis, "Duration", Format::duration ); printValue( out, doneBatches, "Done batches", String::valueOf ); out.println(); }
private void initializeRelationshipImport( Estimates estimates, IdMapper idMapper, BatchingNeoStores neoStores ) { long numberOfRelationships = estimates.numberOfRelationships(); printStageHeader( "(2/4) Relationship import", ESTIMATED_NUMBER_OF_RELATIONSHIPS, count( numberOfRelationships ), ESTIMATED_DISK_SPACE_USAGE, bytes( relationshipsDiskUsage( estimates, neoStores ) + estimates.sizeOfRelationshipProperties() ), ESTIMATED_REQUIRED_MEMORY_USAGE, bytes( baselineMemoryRequirement( neoStores ) + totalMemoryUsageOf( idMapper ) ) ); initializeProgress( numberOfRelationships, ImportStage.relationshipImport ); }
@Test void shouldDisplayBytes() { // when String format = Format.bytes( 123 ); // then assertTrue( format.contains( String.valueOf( 123 ) ) ); assertTrue( format.endsWith( " B" ) ); }
@Override void dump( Logger logger ) { logger.log( getDiskSpace( databaseLayout ) ); logger.log( "Storage files: (filename : modification date - size)" ); MappedFileCounter mappedCounter = new MappedFileCounter( databaseLayout ); long totalSize = logStoreFiles( logger, " ", databaseLayout.databaseDirectory(), mappedCounter ); logger.log( "Storage summary: " ); logger.log( " Total size of store: " + Format.bytes( totalSize ) ); logger.log( " Total size of mapped files: " + Format.bytes( mappedCounter.getSize() ) ); }
@Test void insufficientHeapSize() { // given long optimalHeapSize = gibiBytes( 2 ); long heapSize = gibiBytes( 1 ); // when monitor.insufficientHeapSize( optimalHeapSize, heapSize ); // then String text = errBuffer.toString(); assertTrue( text.contains( "WARNING" ) ); assertTrue( text.contains( "too small" ) ); assertTrue( text.contains( bytes( heapSize ) ) ); assertTrue( text.contains( bytes( optimalHeapSize ) ) ); }
@Test void abundantHeapSize() { // given long optimalHeapSize = gibiBytes( 2 ); long heapSize = gibiBytes( 10 ); // when monitor.abundantHeapSize( optimalHeapSize, heapSize ); // then String text = errBuffer.toString(); assertTrue( text.contains( "WARNING" ) ); assertTrue( text.contains( "unnecessarily large" ) ); assertTrue( text.contains( bytes( heapSize ) ) ); assertTrue( text.contains( bytes( optimalHeapSize ) ) ); }
@Test void insufficientAvailableMemory() { // given long estimatedCacheSize = gibiBytes( 2 ); long optimalHeapSize = gibiBytes( 2 ); long availableMemory = gibiBytes( 1 ); // when monitor.insufficientAvailableMemory( estimatedCacheSize, optimalHeapSize, availableMemory ); // then String text = errBuffer.toString(); assertTrue( text.contains( "WARNING" ) ); assertTrue( text.contains( "may not be sufficient" ) ); assertTrue( text.contains( bytes( estimatedCacheSize ) ) ); assertTrue( text.contains( bytes( optimalHeapSize ) ) ); assertTrue( text.contains( bytes( availableMemory ) ) ); } }