Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2@file
3@brief This extension contains various functionalities to help unittesting.
4"""
5import os
6import stat
7import sys
8import re
9import warnings
10import time
11import importlib
12from contextlib import redirect_stdout, redirect_stderr
13from io import StringIO
16def _get_PyLinterRunV():
17 # Separate function to speed up import.
18 from pylint.lint import Run as PyLinterRun
19 from pylint import __version__ as pylint_version
20 if pylint_version >= '2.0.0':
21 PyLinterRunV = PyLinterRun
22 else:
23 PyLinterRunV = lambda *args, do_exit=False: PyLinterRun( # pylint: disable=E1120, E1123
24 *args, exit=do_exit) # pylint: disable=E1120, E1123
25 return PyLinterRunV
28def get_temp_folder(thisfile, name=None, clean=True, create=True,
29 persistent=False, path_name="tpath"):
30 """
31 Creates and returns a local temporary folder to store files
32 when unit testing.
34 @param thisfile use ``__file__`` or the function which runs the test
35 @param name name of the temporary folder
36 @param clean if True, clean the folder first, it can also a function
37 called to determine whether or not the folder should be
38 cleaned
39 @param create if True, creates it (empty if clean is True)
40 @param persistent if True, create a folder at root level to reduce path length,
41 the function checks the ``MAX_PATH`` variable and
42 shorten the test folder is *max_path* is True on :epkg:`Windows`,
43 on :epkg:`Linux`, it creates a folder three level ahead
44 @param path_name test path used when *max_path* is True
45 @return temporary folder
47 The function extracts the file which runs this test and will name
48 the temporary folder base on the name of the method. *name* must be None.
50 Parameter *clean* can be a function.
51 Signature is ``def clean(folder)``.
52 """
53 if name is None:
54 name = thisfile.__name__
55 if name.startswith("test_"):
56 name = "temp_" + name[5:]
57 elif not name.startswith("temp_"):
58 name = "temp_" + name
59 thisfile = os.path.abspath(thisfile.__func__.__code__.co_filename)
60 final = os.path.split(name)[-1]
62 if not final.startswith("temp_") and not final.startswith("temp2_"):
63 raise NameError("the folder '{0}' must begin with temp_".format(name))
65 local = os.path.join(
66 os.path.normpath(os.path.abspath(os.path.dirname(thisfile))), name)
68 if persistent:
69 if sys.platform.startswith("win"):
70 from ctypes.wintypes import MAX_PATH
71 if MAX_PATH <= 300:
72 local = os.path.join(os.path.abspath("\\" + path_name), name)
73 else:
74 local = os.path.join(
75 local, "..", "..", "..", "..", path_name, name)
76 else:
77 local = os.path.join(local, "..", "..", "..",
78 "..", path_name, name)
79 local = os.path.normpath(local)
81 if name == local:
82 raise NameError(
83 "The folder '{0}' must be relative, not absolute".format(name))
85 if not os.path.exists(local):
86 if create:
87 os.makedirs(local)
88 mode = os.stat(local).st_mode
89 nmode = mode | stat.S_IWRITE
90 if nmode != mode:
91 os.chmod(local, nmode)
92 else:
93 if (callable(clean) and clean(local)) or (not callable(clean) and clean):
94 # delayed import to speed up import time of pycode
95 from ..filehelper.synchelper import remove_folder
96 remove_folder(local)
97 time.sleep(0.1)
98 if create and not os.path.exists(local):
99 os.makedirs(local)
100 mode = os.stat(local).st_mode
101 nmode = mode | stat.S_IWRITE
102 if nmode != mode:
103 os.chmod(local, nmode)
105 return local
108def _extended_refactoring(filename, line):
109 """
110 Private function which does extra checkings
111 when refactoring :epkg:`pyquickhelper`.
113 @param filename filename
114 @param line line
115 @return None or error message
116 """
117 if "from pyquickhelper import fLOG" in line:
118 if "test_code_style" not in filename:
119 return "issue with fLOG"
120 if "from pyquickhelper import noLOG" in line:
121 if "test_code_style" not in filename:
122 return "issue with noLOG"
123 if "from pyquickhelper import run_cmd" in line:
124 if "test_code_style" not in filename:
125 return "issue with run_cmd"
126 if "from pyquickhelper import get_temp_folder" in line:
127 if "test_code_style" not in filename:
128 return "issue with get_temp_folder"
129 return None
132class PEP8Exception(Exception):
133 """
134 Code or style issues.
135 """
136 pass
139def check_pep8(folder, ignore=('E265', 'W504'), skip=None,
140 complexity=-1, stop_after=100, fLOG=None,
141 pylint_ignore=('C0103', 'C1801',
142 'R0201', 'R1705',
143 'W0108', 'W0613',
144 'W0107', 'C0415',
145 'C0209'),
146 recursive=True, neg_pattern=None, extended=None,
147 max_line_length=143, pattern=".*[.]py$",
148 run_lint=True, verbose=False, run_cmd_filter=None):
149 """
150 Checks if :epkg:`PEP8`,
151 the function calls command :epkg:`pycodestyle`
152 on a specific folder.
154 @param folder folder to look into
155 @param ignore list of warnings to skip when raising an exception if
156 :epkg:`PEP8` is not verified, see also
157 `Error Codes <http://pep8.readthedocs.org/en/latest/intro.html#error-codes>`_
158 @param pylint_ignore ignore :epkg:`pylint` issues, see
159 :epkg:`pylint error codes`
160 @param complexity see `check_file <https://pycodestyle.pycqa.org/en/latest/api.html>`_
161 @param stop_after stop after *stop_after* issues
162 @param skip skip a warning if a substring in this list is found
163 @param neg_pattern skip files verifying this regular expressions
164 @param extended list of tuple (name, function), see below
165 @param max_line_length maximum allowed length of a line of code
166 @param recursive look into subfolder
167 @param pattern only file matching this pattern will be checked
168 @param run_lint run :epkg:`pylint`
169 @param verbose :epkg:`pylint` is slow, tells which file is
170 investigated (but it is even slower)
171 @param run_cmd_filter some files makes :epkg:`pylint` crashes (``import yaml``),
172 the test for this is run in a separate process
173 if the function *run_cmd_filter* returns True of the filename,
174 *verbose* is set to True in that case
175 @param fLOG logging function
176 @return output
178 Functions mentioned in *extended* takes two parameters (file name and line)
179 and they returned None or an error message or a tuple (position in the line, error message).
180 When the return is not empty, a warning will be added to the ones
181 printed by :epkg:`pycodestyle`.
182 A few codes to ignore:
184 * *E501*: line too long (?? characters)
185 * *E265*: block comments should have a space after #
186 * *W504*: line break after binary operator, this one is raised
187 after the code is modified by @see fn remove_extra_spaces_and_pep8.
189 The full list is available at :epkg:`PEP8 codes`. In addition,
190 the function adds its own codes:
192 * *ECL1*: line too long for a specific reason.
194 Some errors to disable with :epkg:`pylint`:
196 * *C0103*: variable name is not conform
197 * *C0111*: missing function docstring
198 * *C1801*: do not use `len(SEQUENCE)` to determine if a sequence is empty
199 * *R0201*: method could be a function
200 * *R0205*: Class '?' inherits from object, can be safely removed from bases in python3 (pylint)
201 * *R0901*: too many ancestors
202 * *R0902*: too many instance attributes
203 * *R0911*: too many return statements
204 * *R0912*: too many branches
205 * *R0913*: too many arguments
206 * *R0914*: too many local variables
207 * *R0915*: too many statements
208 * *R1702*: too many nested blocks
209 * *R1705*: unnecessary "else" after "return"
210 * *W0107*: unnecessary pass statements
211 * *W0108*: Lambda may not be necessary
212 * *W0613*: unused argument
214 The full list is available at :epkg:`pylint error codes`.
215 :epkg:`pylint` was added used to check the code.
216 It produces the following list of errors
217 :epkg:`pylint error codes`.
219 If *neg_pattern* is empty, it populates with a default value
220 which skips unnecessary folders:
221 ``".*[/\\\\\\\\]((_venv)|([.]git)|(__pycache__)|(temp_)).*"``.
222 """
223 # delayed import to speed up import time of pycode
224 import pycodestyle
225 from ..filehelper.synchelper import explore_folder_iterfile
226 if fLOG is None:
227 from ..loghelper.flog import noLOG
228 fLOG = noLOG
230 def extended_checkings(fname, content, buf, extended):
231 for i, line in enumerate(content):
232 for name, fu in extended:
233 r = fu(fname, line)
234 if isinstance(r, tuple):
235 c, r = r
236 else:
237 c = 1
238 if r is not None:
239 buf.write("{0}:{1}:{4} F{2} {3}\n".format(
240 fname, i + 1, name, r, c))
242 def fkeep(s):
243 if len(s) == 0:
244 return False
245 if skip is not None:
246 for kip in skip:
247 if kip in s:
248 return False
249 return True
251 if max_line_length is not None:
252 if extended is None:
253 extended = []
254 else:
255 extended = extended.copy()
257 def check_lenght_line(fname, line):
258 if len(line) > max_line_length and not line.lstrip().startswith('#'):
259 if ">`_" in line:
260 return "line too long (link) {0} > {1}".format(len(line), max_line_length)
261 if ":math:`" in line:
262 return "line too long (:math:) {0} > {1}".format(len(line), max_line_length)
263 if "ERROR: " in line:
264 return "line too long (ERROR:) {0} > {1}".format(len(line), max_line_length)
265 return None
267 extended.append(("[ECL1]", check_lenght_line))
269 if ignore is None:
270 ignore = tuple()
271 elif isinstance(ignore, list):
272 ignore = tuple(ignore)
274 if neg_pattern is None:
275 neg_pattern = ".*[/\\\\]((_venv)|([.]git)|(__pycache__)|(temp_)|([.]egg)|(bin)).*"
277 try:
278 regneg_filter = None if neg_pattern is None else re.compile(
279 neg_pattern)
280 except re.error as e:
281 raise ValueError("Unable to compile '{0}'".format(neg_pattern)) from e
283 # pycodestyle
284 fLOG("[check_pep8] code style on '{0}'".format(folder))
285 files_to_check = []
286 skipped = []
287 buf = StringIO()
288 with redirect_stdout(buf):
289 for file in explore_folder_iterfile(folder, pattern=pattern,
290 recursive=recursive):
291 if regneg_filter is not None:
292 if regneg_filter.search(file):
293 skipped.append(file)
294 continue
295 if file.endswith("__init__.py"):
296 ig = ignore + ('F401',)
297 else:
298 ig = ignore
299 if file is None:
300 raise TypeError("file cannot be None")
301 if len(file) == 0:
302 raise TypeError("file cannot be empty")
304 # code style
305 files_to_check.append(file)
306 try:
307 style = pycodestyle.StyleGuide(
308 ignore=ig, complexity=complexity, format='pylint',
309 max_line_length=max_line_length)
310 res = style.check_files([file])
311 except TypeError as e:
312 ext = "This is often due to an instruction from . import... The imported module has no name."
313 raise TypeError("Issue with pycodesyle for module '{0}' ig={1} complexity={2}\n{3}".format(
314 file, ig, complexity, ext)) from e
316 if extended is not None:
317 with open(file, "r", errors="ignore") as f:
318 content = f.readlines()
319 extended_checkings(file, content, buf, extended)
321 if res.total_errors + res.file_errors > 0:
322 res.print_filename = True
323 lines = [_ for _ in buf.getvalue().split("\n") if fkeep(_)]
324 if len(lines) > stop_after:
325 raise PEP8Exception(
326 "{0} lines\n{1}".format(len(lines), "\n".join(lines)))
328 lines = [_ for _ in buf.getvalue().split("\n") if fkeep(_)]
329 if len(lines) > 10:
330 raise PEP8Exception(
331 "{0} lines\n{1}".format(len(lines), "\n".join(lines)))
333 if len(files_to_check) == 0:
334 mes = skipped[0] if skipped else "-no skipped file-"
335 raise FileNotFoundError("No file found in '{0}'\n pattern='{1}'\nskipped='{2}'".format(
336 folder, pattern, mes))
338 # pylint
339 if not run_lint:
340 return "\n".join(lines)
341 fLOG("[check_pep8] pylint with {0} files".format(len(files_to_check)))
342 memout = sys.stdout
344 try:
345 fLOG('', OutputStream=memout)
346 regular_print = False
347 except TypeError:
348 regular_print = True
350 def myprint(s):
351 "local print, chooses the right function"
352 if regular_print:
353 memout.write(s + "\n")
354 else:
355 fLOG(s, OutputStream=memout)
357 neg_pat = ".*temp[0-9]?_.*,doc_.*"
358 if neg_pattern is not None:
359 neg_pat += ',' + neg_pattern
361 if run_cmd_filter is not None:
362 verbose = True
364 PyLinterRunV = _get_PyLinterRunV()
365 sout = StringIO()
366 serr = StringIO()
367 with redirect_stdout(sout):
368 with redirect_stderr(serr):
369 with warnings.catch_warnings():
370 warnings.simplefilter("ignore", DeprecationWarning)
371 opt = ["--ignore-patterns=" + neg_pat, "--persistent=n",
372 '--jobs=1', '--suggestion-mode=n', "--score=n",
373 '--max-args=30', '--max-locals=50', '--max-returns=30',
374 '--max-branches=50', '--max-parents=25',
375 '--max-attributes=50', '--min-public-methods=0',
376 '--max-public-methods=100', '--max-bool-expr=10',
377 '--max-statements=200',
378 '--msg-template={abspath}:{line}: {msg_id}: {msg} (pylint)']
379 if pylint_ignore:
380 opt.append('--disable=' + ','.join(pylint_ignore))
381 if max_line_length:
382 opt.append("--max-line-length=%d" % max_line_length)
383 if verbose:
384 for i, name in enumerate(files_to_check):
385 cop = list(opt)
386 cop.append(name)
387 if run_cmd_filter is None or not run_cmd_filter(name):
388 myprint(
389 "[check_pep8] lint file {0}/{1} - '{2}'\n".format(i + 1, len(files_to_check), name))
390 PyLinterRunV(cop, do_exit=False)
391 else:
392 # delayed import to speed up import time of pycode
393 from ..loghelper import run_cmd
394 # runs from command line
395 myprint(
396 "[check_pep8] cmd-lint file {0}/{1} - '{2}'\n".format(i + 1, len(files_to_check), name))
397 cmd = "{0} -m pylint {1}".format(
398 sys.executable, " ".join('"{0}"'.format(_) for _ in cop))
399 out = run_cmd(cmd, wait=True)[0]
400 lines.extend(_ for _ in out.split(
401 '\n') if _.strip('\r '))
402 else:
403 opt.extend(files_to_check)
404 PyLinterRunV(opt, do_exit=False)
406 pylint_lines = sout.getvalue().split('\n')
407 pylint_lines = [
408 _ for _ in pylint_lines if (
409 '(pylint)' in _ and fkeep(_) and _[0] != ' ' and len(_.split(':')) > 2)]
410 pylint_lines = [_ for _ in pylint_lines if not _.startswith(
411 "except ") and not _.startswith("else:") and not _.startswith(
412 "try:") and "# noqa" not in _]
413 lines.extend(pylint_lines)
414 if len(lines) > 0:
415 raise PEP8Exception(
416 "{0} lines\n{1}".format(len(lines), "\n".join(lines)))
418 return "\n".join(lines)
421def add_missing_development_version(names, root, hide=False):
422 """
423 Looks for development version of a given module and add paths to
424 ``sys.path`` after having checked they are working.
426 @param names name or names of the module to import
427 @param root folder where to look (assuming all modules location
428 at the same place in a flat hierarchy)
429 @param hide hide warnings when importing a module (might be a lot)
430 @return added paths
431 """
432 # delayed import to speed up import time
433 from ..loghelper import sys_path_append
435 if not isinstance(names, list):
436 names = [names]
437 root = os.path.abspath(root)
438 if os.path.isfile(root):
439 root = os.path.dirname(root)
440 if not os.path.exists(root):
441 raise FileNotFoundError(root)
443 spl = os.path.split(root)
444 py27 = False
445 if spl[-1].startswith("ut_"):
446 if "dist_module27" in root:
447 # python 27
448 py27 = True
449 newroot = os.path.join(root, "..", "..", "..", "..")
450 else:
451 newroot = os.path.join(root, "..", "..", "..")
452 else:
453 newroot = root
455 newroot = os.path.normpath(os.path.abspath(newroot))
456 found = os.listdir(newroot)
457 dirs = [os.path.join(newroot, _) for _ in found]
459 paths = []
460 for name in names:
461 exc = None
462 try:
463 if hide:
464 with warnings.catch_warnings(record=True):
465 importlib.import_module(name)
466 else:
467 importlib.import_module(name)
468 continue
469 except ImportError as e:
470 # it requires a path
471 exc = e
473 if name not in found:
474 raise FileNotFoundError("Unable to find a subfolder '{0}' in '{1}' (py27={3})\nFOUND:\n{2}\nexc={4}".format(
475 name, newroot, "\n".join(dirs), py27, exc))
477 if py27:
478 this = os.path.join(newroot, name, "dist_module27", "src")
479 if not os.path.exists(this):
480 this = os.path.join(newroot, name, "dist_module27")
481 else:
482 this = os.path.join(newroot, name, "src")
483 if not os.path.exists(this):
484 this = os.path.join(newroot, name)
486 if not os.path.exists(this):
487 raise FileNotFoundError("unable to find a subfolder '{0}' in '{1}' (*py27={3})\nFOUND:\n{2}".format(
488 this, newroot, "\n".join(dirs), py27))
489 with sys_path_append(this):
490 if hide:
491 with warnings.catch_warnings(record=True):
492 importlib.import_module(name)
493 else:
494 importlib.import_module(name)
495 paths.append(this)
496 return paths