# -*- coding: utf-8 -*-
"""
Some automation helpers to test notebooks and check they are still working fine.
:githublink:`%|py|6`
"""
import os
from pyquickhelper.loghelper import noLOG
from pyquickhelper.ipythonhelper import execute_notebook_list
import pyensae
[docs]def ls_notebooks(subfolder):
"""
list the notebooks in a particular subfolder
:param subfolder: subfolder (related to this module)
:return: list of files
:githublink:`%|py|18`
"""
this = os.path.abspath(os.path.dirname(__file__))
docnote = os.path.join(
this,
"..",
"..",
"..",
"_doc",
"notebooks",
subfolder)
notes = [
os.path.normpath(
os.path.join(
docnote,
_)) for _ in os.listdir(docnote)]
keepnote = []
for note in notes:
ext = os.path.splitext(note)[-1]
if ext != ".ipynb":
continue
keepnote.append(note)
return keepnote
[docs]def get_additional_paths():
"""
returns a list of paths to add before running the notebooks,
paths to pyquickhelper, pyensae, pymmails
:return: list of paths
:githublink:`%|py|49`
"""
import pyquickhelper
import pymyinstall
import pyrsslocal
import mlstatpy
import jyquickhelper
addpath = [os.path.dirname(pyquickhelper.__file__),
os.path.dirname(pyensae.__file__),
os.path.dirname(pyrsslocal.__file__),
os.path.dirname(pymyinstall.__file__),
os.path.dirname(mlstatpy.__file__),
os.path.dirname(jyquickhelper.__file__),
os.path.join(os.path.abspath(os.path.dirname(__file__)), ".."),
]
try:
import ensae_teaching_cs
addpath.append(os.path.dirname(ensae_teaching_cs.__file__))
except ImportError: # pragma: no cover
pass
addpath = [os.path.normpath(os.path.join(_, "..")) for _ in addpath]
return addpath
[docs]def clean_function_notebook(code):
"""
Default cleaning for notebooks cells when unittesting.
:param code: cell content
:return: modified code
:githublink:`%|py|78`
"""
code = code.replace(
'run_cmd("exemple.xlsx"',
'skip_run_cmd("exemple.xlsx"')
skip = ["faire une chose avec la probabilité 0.7",
"# déclenche une exception",
"# pour lancer Excel",
"for k in list_exercice_1 :",
"return ....",
"return [ .... ]",
"def __init__(self, ...) :",
"if random.random() <= 0.7 :",
"dictionnaire_depart.items() [0]",
"iterateur(0,10) [0]",
"# ...... à remplir",
'String.Join(",", a.Select(c=>c.ToString()).ToArray())',
"# elle n'existe pas encore",
"# boucle sur les 24 heures de la journée",
"from ggplot import *",
"geocode = True",
# ggplot calls method show and it opens window blocking the offline
# execution
]
rep = [("# ...", "pass # "),
("%timeit -n1 -r1 ", ""),
("%timeit", "#%timeit"),
("show(p)", "#show(p)"),
("show(tabs)", "#show(tabs)"),
('http://www.gutenberg.org/cache/epub/4647/pg4647.txt',
'http://www.xavierdupre.fr/enseignement/complements/pg4647.txt'),
]
spl = ["# ......",
"# elle n'existe pas encore",
]
for s in skip:
if s in code:
return ""
for s in spl:
if s in code:
code = code.split(s)[0]
for s in rep:
code = code.replace(s[0], s[1])
return code
[docs]def execute_notebooks(folder, notebooks, filter, clean_function=None,
fLOG=noLOG, deepfLOG=noLOG, detailed_log=None):
"""
Executes a list of notebooks.
:param folder: folder
:param notebooks: list of notebooks
:param filter: function which validate the notebooks
:param clean_function: cleaning function to apply to the code before running it
:param fLOG: logging function
:param deepfLOG: logging function used to run the notebook
:param detailed_log: log the output while running the notebook (when the notebook execution fails due to timeout
:return: dictionary tuple (statistics, { notebook_file: (isSuccess, outout) })
The signature of function ``filter`` is::
def filter(i, filename):
return True or False
:githublink:`%|py|147`
"""
def valid_cell(cell):
if "%system" in cell:
return False # pragma: no cover
if "df.plot(...)" in cell:
return False # pragma: no cover
if 'df["difference"] = ...' in cell:
return False # pragma: no cover
if 'print(next(it))' in cell:
return False # pragma: no cover
if "est d'indice 8 et non plus 9" in cell:
return False
return True
addpaths = get_additional_paths()
if filter:
notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)]
if len(notebooks) == 0:
raise ValueError( # pragma: no cover
"Empty list of notebooks.")
if clean_function is None:
clean_function = clean_function_notebook # pragma: no cover
return execute_notebook_list(
folder, notebooks, fLOG=fLOG, valid=valid_cell, additional_path=addpaths,
clean_function=clean_function)