1# -*- coding: utf-8 -*- 

2""" 

3@file 

4@brief Some automation helpers to test notebooks and check they are still working fine. 

5""" 

6import os 

7from pyquickhelper.loghelper import noLOG 

8from pyquickhelper.ipythonhelper import execute_notebook_list 

9import pyensae 

10 

11 

12def ls_notebooks(subfolder): 

13 """ 

14 Lists the notebooks in a particular subfolder. 

15 

16 @param subfolder subfolder (related to this module) 

17 @return list of files 

18 """ 

19 this = os.path.abspath(os.path.dirname(__file__)) 

20 docnote = os.path.join( 

21 this, 

22 "..", 

23 "..", 

24 "..", 

25 "_doc", 

26 "notebooks", 

27 subfolder) 

28 notes = [ 

29 os.path.normpath( 

30 os.path.join( 

31 docnote, 

32 _)) for _ in os.listdir(docnote)] 

33 

34 keepnote = [] 

35 for note in notes: 

36 ext = os.path.splitext(note)[-1] 

37 if ext != ".ipynb": 

38 continue 

39 keepnote.append(note) 

40 return keepnote 

41 

42 

43def get_additional_paths(): 

44 """ 

45 Returns a list of paths to add before running the notebooks, 

46 paths to :epkg:`pyquickhelper`, :epkg:`pyensae`. 

47 

48 @return list of paths 

49 """ 

50 import pyquickhelper 

51 import jyquickhelper 

52 addpath = [os.path.dirname(pyquickhelper.__file__), 

53 os.path.dirname(pyensae.__file__), 

54 os.path.dirname(jyquickhelper.__file__), 

55 os.path.join(os.path.abspath(os.path.dirname(__file__)), ".."), 

56 ] 

57 addpath = [os.path.normpath(os.path.join(_, "..")) for _ in addpath] 

58 return addpath 

59 

60 

61def clean_function_notebook(code): 

62 """ 

63 Cleans cells when unittesting notebooks. 

64 

65 @param code cell content 

66 @return modified code 

67 """ 

68 code = code.replace( 

69 'run_cmd("exemple.xlsx"', 

70 'skip_run_cmd("exemple.xlsx"') 

71 

72 skip = ["faire une chose avec la probabilité 0.7", 

73 "# déclenche une exception", 

74 "# pour lancer Excel", 

75 "for k in list_exercice_1 :", 

76 "return ....", 

77 "return [ .... ]", 

78 "def __init__(self, ...) :", 

79 "if random.random() <= 0.7 :", 

80 "dictionnaire_depart.items() [0]", 

81 "iterateur(0,10) [0]", 

82 "# ...... à remplir", 

83 'String.Join(",", a.Select(c=>c.ToString()).ToArray())', 

84 "# elle n'existe pas encore", 

85 "from ggplot import *", 

86 # ggplot calls method show and it opens window blocking the offline 

87 # execution 

88 ] 

89 rep = [("# ...", "pass # "), 

90 ("%timeit", "#%timeit"), 

91 ] 

92 spl = ["# ......", 

93 "# elle n'existe pas encore", 

94 ] 

95 

96 for s in skip: 

97 if s in code: 

98 return "" 

99 

100 for s in spl: 

101 if s in code: 

102 code = code.split(s)[0] 

103 

104 for s in rep: 

105 code = code.replace(s[0], s[1]) 

106 

107 return code 

108 

109 

110def execute_notebooks(folder, notebooks, filter, # pylint: disable= W0622 

111 clean_function=None, 

112 fLOG=noLOG, 

113 deepfLOG=noLOG, 

114 valid_cell=None): 

115 """ 

116 Executes a list of notebooks. 

117 

118 @param folder folder 

119 @param notebooks list of notebooks 

120 @param filter function which validate the notebooks 

121 @param clean_function cleaning function to apply to the code before running it 

122 @param fLOG logging function 

123 @param deepfLOG logging function used to run the notebook 

124 @param valid_cell to disable the execution of a cell 

125 @return dictionary tuple (statistics, { notebook_file: (isSuccess, outout) }) 

126 

127 The signature of function ``filter`` is:: 

128 

129 def filter( i, filename) : return True or False 

130 """ 

131 

132 def _valid_cell(cell): 

133 if valid_cell is not None: 

134 if not valid_cell(cell): 

135 return False 

136 if "%system" in cell: 

137 return False 

138 if "df.plot(...)" in cell: 

139 return False 

140 if 'df["difference"] = ...' in cell: 

141 return False 

142 if 'print(next(it))' in cell: 

143 return False 

144 return True 

145 

146 addpaths = get_additional_paths() 

147 if filter: 

148 notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)] 

149 if len(notebooks) == 0: 

150 raise ValueError("Empty list of notebooks.") 

151 return execute_notebook_list( 

152 folder, notebooks, fLOG=fLOG, valid=_valid_cell, additional_path=addpaths)