import mermaid from ''; mermaid.initialize({ startOnLoad: true });
How to profile from a notebook with cProfile, memory_profiler
from jyquickhelper import add_notebook_menu
add_notebook_menu()
def big_list1(n):
l = []
for i in range(n):
l.append(i)
return l
def big_list2(n):
return list(range(n))
def big_list(n):
big_list1(n)
big_list2(n)
%prun -q -T profile_example.txt -D profile_example.stat big_list(100000)
*** Profile stats marshalled to file 'profile_example.stat'. *** Profile printout saved to text file 'profile_example.txt'.
with open('profile_example.txt', 'r') as f: content = f.read()
print(content)
100006 function calls in 0.142 seconds Ordered by: internal time ncalls tottime percall cumtime percall filename:lineno(function) 1 0.098 0.098 0.124 0.124 <ipython-input-45-e328f4dea298>:1(big_list1) 100000 0.026 0.000 0.026 0.000 {method 'append' of 'list' objects} 1 0.009 0.009 0.142 0.142 <ipython-input-45-e328f4dea298>:10(big_list) 1 0.009 0.009 0.009 0.009 <ipython-input-45-e328f4dea298>:7(big_list2) 1 0.000 0.000 0.142 0.142 {built-in method exec} 1 0.000 0.000 0.142 0.142 <string>:1(<module>) 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
import pstats
p = pstats.Stats('profile_example.stat')
p.strip_dirs().sort_stats('cumulative').print_stats()
Wed Sep 9 10:55:54 2015 profile_example.stat 100006 function calls in 0.117 seconds Ordered by: cumulative time ncalls tottime percall cumtime percall filename:lineno(function) 1 0.000 0.000 0.117 0.117 {built-in method exec} 1 0.000 0.000 0.117 0.117 <string>:1(<module>) 1 0.007 0.007 0.117 0.117 <ipython-input-51-720be8d7293b>:10(big_list) 1 0.079 0.079 0.101 0.101 <ipython-input-51-720be8d7293b>:1(big_list1) 100000 0.022 0.000 0.022 0.000 {method 'append' of 'list' objects} 1 0.008 0.008 0.008 0.008 <ipython-input-51-720be8d7293b>:7(big_list2) 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
<pstats.Stats at 0x944c2e8>
from memory_profiler import memory_usage
mem_usage = memory_usage(-1, interval=.2, timeout=1)
mem_usage
[97.69921875, 97.69921875, 97.69921875, 97.69921875, 97.69921875]
Thz functions to test must be part of file and cannot be implemented in the notebook. So we save the funtion in script and we import it just after.
%%file script_test.py
def big_list1(n):
l = []
for i in range(n):
l.append(i)
return l
def big_list2(n):
return list(range(n))
def big_list(n):
big_list1(n)
big_list2(n)
Writing script_test.py
from script_test import big_list, big_list1, big_list2
We run the momory profiling:
%load_ext memory_profiler
prof = %mprun -r -f big_list1 -f big_list2 -T profile_example.mem -r big_list(100000)
*** Profile printout saved to text file profile_example.mem.
with open('profile_example.mem', 'r') as f : content = f.read()
print(content)
Filename: C:\xadupre\__home_\_data\GitHub\pymyinstall\_doc\notebooks\script_test.py Line # Mem usage Increment Line Contents ================================================ 2 38.3 MiB 0.0 MiB def big_list1(n): 3 38.3 MiB 0.0 MiB l = [] 4 42.1 MiB 3.8 MiB for i in range(n): 5 42.1 MiB 0.0 MiB l.append(i) 6 42.1 MiB 0.0 MiB return l Filename: C:\xadupre\__home_\_data\GitHub\pymyinstall\_doc\notebooks\script_test.py Line # Mem usage Increment Line Contents ================================================ 8 39.1 MiB 0.0 MiB def big_list2(n): 9 42.9 MiB 3.8 MiB return list(range(n))
%load_ext snakeviz
%system snakeviz --help
['Usage: snakeviz [options] filename', '', 'Options:', ' -h, --help show this help message and exit', ' -H ADDR, --hostname=ADDR', ' hostname to bind to (default: 127.0.0.1', ' -p PORT, --port=PORT port to bind to; if this port is already in use a free', ' port will be selected automatically (default: 8080)', ' -b PATH, --browser=PATH', ' name of webbrowser to launch as described in the', " documentation of Python's webbrowser module:", ' https://docs.python.org/3/library/webbrowser.html', ' -s, --server start SnakeViz in server-only mode--no attempt will be', ' to open a browser']