Coverage for src/pyquickhelper/pycode/setup_helper.py : 75%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2@file
3@brief Helper for the setup
4"""
6import os
7import sys
8import re
9import warnings
10import hashlib
11import datetime
14def get_available_setup_commands():
15 """
16 Returns the list of commands :epkg:`pyquickhelper` implements
17 or allows.
18 """
19 commands = ['bdist_egg', 'bdist_msi', 'bdist_wheel', 'bdist_wininst', 'build27',
20 'build_ext', 'build_script', 'build_sphinx', 'clean_pyd', 'clean_space',
21 'copy27', 'copy_dist', 'copy_sphinx', 'history', 'lab', 'local_pypi',
22 'notebook', 'publish', 'publish_doc', 'register', 'run27', 'run_pylint',
23 'sdist', 'setup_hook', 'setupdep', 'test_local_pypi',
24 'unittests', 'unittests_GUI', 'unittests_LONG', 'unittests_SKIP',
25 'upload_docs', 'write_version', 'local_jenkins']
26 return commands
29def get_available_build_commands():
30 """
31 Returns commands which build the package.
32 """
33 return {"sdist", "bdist_wheel", "publish", "publish_doc", "register",
34 "upload_docs", "bdist_wininst", "build_ext"}
37def available_commands_list(argv):
38 """
39 Checks that on command handled by pyquickhelper is part of the arguments.
41 @param argv ``sys.args``
42 @return bool
43 """
44 commands = get_available_setup_commands()
45 for c in commands:
46 if c in argv:
47 return True
48 return False
51def process_standard_options_for_setup(
52 argv, file_or_folder, project_var_name, module_name=None, unittest_modules=None,
53 pattern_copy=None,
54 requirements=None, port=8067, blog_list=None, default_engine_paths=None,
55 extra_ext=None, add_htmlhelp=False, setup_params=None, coverage_options=None,
56 coverage_exclude_lines=None, func_sphinx_begin=None, func_sphinx_end=None,
57 additional_notebook_path=None, additional_local_path=None, copy_add_ext=None,
58 nbformats=("ipynb", "html", "python", "rst", "slides",
59 "pdf", "github"),
60 layout=None, direct_call=False,
61 additional_ut_path=None,
62 skip_function=None, covtoken=None, hook_print=True,
63 stdout=None, stderr=None, use_run_cmd=False, filter_warning=None,
64 file_filter_pep8=None, github_owner=None,
65 existing_history=None, coverage_root='src',
66 fexclude=None, skip_issues=None, fLOG=None):
67 """
68 Processes the standard options the module pyquickhelper is
69 able to process assuming the module which calls this function
70 follows the same design as *pyquickhelper*, it will process the following
71 options:
73 .. runpython::
75 from pyquickhelper.pycode import process_standard_options_for_setup_help
76 process_standard_options_for_setup_help("--help-commands")
78 @param argv = *sys.argv*
79 @param file_or_folder file ``setup.py`` or folder which contains it
80 @param project_var_name display name of the module
81 @param module_name module name, None if equal to *project_var_name*
82 (``import <module_name>``)
83 @param unittest_modules modules added for the unit tests,
84 see @see fn py3to2_convert_tree
85 @param pattern_copy see @see fn py3to2_convert_tree
86 @param requirements dependencies, fetched with a local pipy server from
87 ``http://localhost:port/``
88 @param port port for the local pipy server
89 @param blog_list list of blog to listen for this module
90 (usually stored in ``module.__blog__``)
91 @param default_engine_paths define the default location for python engine,
92 should be dictionary *{ engine: path }*, see below.
93 @param extra_ext extra file extension to process (add a page for each of them,
94 ex ``["doc"]``)
95 @param add_htmlhelp run HTML Help too (only on Windows)
96 @param setup_params parameters send to @see fn call_setup_hook
97 @param coverage_options see @see fn main_wrapper_tests
98 @param coverage_exclude_lines see @see fn main_wrapper_tests
99 @param func_sphinx_begin function called before the documentation generation,
100 it gets the same parameters as this function (all named),
101 use ``**args**``
102 @param func_sphinx_end function called after the documentation generation,
103 it gets the same parameters as this function (all named),
104 use ``**args**``
105 @param additional_notebook_path additional paths to add when launching the notebook
106 @param additional_local_path additional paths to add when running a local command
107 @param copy_add_ext additional file extensions to copy
108 @param nbformats requested formats for the notebooks conversion
109 @param layout list of formats sphinx should generate such as html, latex, pdf, docx,
110 it is a list of tuple (layout, build directory, parameters to override),
111 if None --> ``["html", "pdf"]``
112 @param additional_ut_path additional paths to add when running unit tests
113 @param skip_function function to skip unit tests, see @ee fn main_wrapper_tests
114 @param covtoken token used when publishing coverage report to
115 `codecov <https://codecov.io/>`_,
116 more in @see fn main_wrapper_tests
117 @param fLOG logging function
118 @param hook_print enable, disable print when calling *_setup_hook*
119 @param stdout redirect stdout for unit test if not None
120 @param stderr redirect stderr for unit test if not None
121 @param use_run_cmd to run the sphinx documentation with @see fn run_cmd and
122 not ``os.system``
123 @param filter_warning see @see fn main_wrapper_tests
124 @param file_filter_pep8 function to filter out files for which checking pep8
125 (see @see fn remove_extra_spaces_folder)
126 @param github_owner :epkg:`github` owner of the package
127 @param existing_history existing history, retrieves existing issues stored
128 in that file
129 @param coverage_root see @see fn main_wrapper_tests
130 @param direct_call @see fn generate_help_sphinx
131 @param fexclude function which tells which file not to copy in the folder
132 used to build the documentation
133 @param skip_issues skip a given list of issues when building the history
134 @return True (an option was processed) or False,
135 the file ``setup.py`` should call function ``setup``
137 The command ``build_script`` is used, the flag ``--private`` can be used to
138 avoid producing scripts to publish the module on `Pypi <https://pypi.python.org/pypi>`_.
140 An example for *default_engine_paths*::
142 default_engine_paths = {
143 "windows": {
144 "__PY35__": None,
145 "__PY36_X64__": "c:\\Python365_x64",
146 "__PY37_X64__": "c:\\Python372_x64",
147 "__PY38_X64__": "c:\\Python387_x64",
148 "__PY39_X64__": "c:\\Python391_x64",
149 },
150 }
152 Parameters *coverage_options*, *coverage_exclude_lines*, *copy_add_ext* were added
153 for function @see fn main_wrapper_tests.
154 Parameter *unittest_modules* accepts a list of string and 2-uple.
155 If it is a 2-uple, the first string is used to convert Python 3 code into Python 2
156 (in case the local folder is different from the module name),
157 the second string is used to add local path to the variable ``PYTHON_PATH``.
158 If it is a single string, it means both name strings are equal.
159 Parameters *func_sphinx_begin* and *func_sphinx_end* were added
160 to pre-process or post-process the documentation.
161 Parameter *additional_notebook_path* was added to specify some additional
162 paths when preparing the script *auto_cmd_notebook.bat*.
163 Parameters *layout*, *nbformats* were added for
164 function @see fn generate_help_sphinx.
165 The coverage computation can be disable by specifying
166 ``coverage_options["disable_coverage"] = True``.
167 Parameter *covtoken* was added to post the coverage report to :epkg:`codecov`.
168 Option ``-e`` and ``-g`` were added to
169 filter file by regular expressions (in with *e*, out with *g*).
170 """
171 if module_name is not None and (
172 len(module_name) == 0 or module_name[0] == '_'):
173 raise RuntimeError(
174 "module cannot be empty or start with '_': %r." % module_name)
175 if fLOG is None: # pragma: no cover
176 from ..loghelper.flog import noLOG
177 fLOG = noLOG
178 if skip_function is None: # pragma: no cover
179 from .utils_tests_private import default_skip_function
180 skip_function = default_skip_function
181 if pattern_copy is None:
182 # delayed import
183 from .default_regular_expression import _setup_pattern_copy
184 pattern_copy = _setup_pattern_copy
186 if layout is None:
187 layout = ["html", "pdf"]
189 if "--help" in argv or "--help-commands" in argv:
190 process_standard_options_for_setup_help(argv)
191 return not len(get_available_build_commands() & set(argv))
192 fLOG("[process_standard_options_for_setup]", argv)
193 fLOG("[process_standard_options_for_setup] python version:", sys.version_info)
195 def process_argv_for_unittest(argv):
196 if "-d" in argv:
197 ld = argv.index("-d")
198 if ld >= len(argv) - 1:
199 raise ValueError( # pragma: no cover
200 "Option -d should be follow by a duration in seconds.")
201 d = float(argv[ld + 1])
202 else:
203 d = None
205 if "-f" in argv:
206 lf = argv.index("-f")
207 if lf >= len(argv) - 1:
208 raise ValueError( # pragma: no cover
209 "Option -d should be follow by a duration in seconds.")
210 f = argv[lf + 1]
211 else:
212 f = None
214 if "-e" in argv:
215 le = argv.index("-e")
216 if le >= len(argv) - 1:
217 raise ValueError( # pragma: no cover
218 "Option -e should be follow by a regular expression.")
219 pattern = argv[le + 1]
220 if len(pattern) >= 2 and pattern[0] == pattern[-1] == '"':
221 pattern = pattern[1:-1]
222 e = re.compile(pattern)
223 else:
224 e = None
226 if "-g" in argv:
227 lg = argv.index("-g")
228 if lg >= len(argv) - 1:
229 raise ValueError( # pragma: no cover
230 "Option -g should be follow by a regular expression.")
231 pattern = argv[lg + 1]
232 if len(pattern) >= 2 and pattern[0] == pattern[-1] == '"':
233 pattern = pattern[1:-1]
234 g = re.compile(pattern)
235 else:
236 g = None
238 if f is None and d is None and e is None and g is None:
239 return skip_function # pragma: no cover
241 def ereg(name):
242 return (e is None) or (e.search(name) is not None)
244 def greg(name):
245 return (g is None) or (g.search(name) is None)
247 if f is not None:
248 if d is not None: # pragma: no cover
249 raise NotImplementedError(
250 "Options -f and -d cannot be specified at the same time.")
252 def allow(name, code, duration): # pragma: no cover
253 name = os.path.split(name)[-1]
254 return f not in name and ereg(name) and greg(name)
255 return allow # pragma: no cover
256 else:
257 # d is not None
258 def skip_allowd(name, code, duration):
259 name = os.path.split(name)[-1]
260 cond = (
261 (duration is None or d is None or duration <= d) and
262 ereg(name) and greg(name))
263 return not cond
264 return skip_allowd
266 folder = file_or_folder if os.path.isdir(
267 file_or_folder) else os.path.dirname(file_or_folder)
268 unit_test_folder = os.path.join(folder, "_unittests")
269 fLOG("unittest_modules={0}".format(unittest_modules))
271 if unittest_modules is None:
272 unittest_modules_py3to2 = None
273 unittest_modules_script = None
274 else: # pragma: no cover
275 unittest_modules_py3to2 = []
276 unittest_modules_script = []
277 for mod in unittest_modules:
278 if isinstance(mod, tuple):
279 unittest_modules_py3to2.append(mod[0])
280 unittest_modules_script.append(mod[1])
281 else:
282 unittest_modules_py3to2.append(mod)
283 unittest_modules_script.append(mod)
285 # dump unit test coverage?
287 def dump_coverage_fct(full=True):
288 mn = project_var_name if module_name is None else module_name
289 full_path = _get_dump_default_path(folder, mn, argv)
290 if full_path is None or full:
291 return full_path
292 else:
293 sub = os.path.split(full_path)[0]
294 sub = os.path.split(sub)[0]
295 return sub
297 # starts interpreting the commands
299 if "clean_space" in argv: # pragma: no cover
300 rem = clean_space_for_setup(
301 file_or_folder, file_filter=file_filter_pep8)
302 print("[clean_space] number of impacted files (pep8 + rst):", len(rem))
303 rem = clean_notebooks_for_numbers(file_or_folder)
304 print("[clean_space] number of impacted notebooks:", len(rem))
305 return True
307 if "run_pylint" in argv:
308 verbose = '-v' in argv
309 pos = argv.index('run_pylint')
310 ignores = [_[2:] for _ in argv if _[:2] == '-i']
311 ignores = None if len(ignores) == 0 else tuple(ignores)
312 argv = [_ for _ in argv if _ not in (
313 '-v', '-') and not _.startswith('-i')]
314 pattern = argv[pos + 1] if len(argv) > pos + 1 else ".*[.]py$"
315 neg_pattern = argv[pos + 2] if len(argv) > pos + 2 else None
316 print("[run_pylint] run_pylint for sources pattern='{0}' neg_pattern='{1}'".format(
317 pattern, neg_pattern))
318 src_folder = os.path.join(folder, "src")
319 if not os.path.exists(src_folder):
320 src_folder = folder
321 run_pylint_for_setup(src_folder, fLOG=print, pattern=pattern,
322 verbose=verbose, pylint_ignore=ignores)
323 print("[run_pylint] run_pylint for unittest")
324 run_pylint_for_setup(os.path.join(folder, "_unittests"),
325 fLOG=print, pattern=pattern, verbose=verbose,
326 pylint_ignore=ignores)
327 return True
329 elif 'history' in argv: # pragma: no cover
330 dest = ' '.join(argv).split( # pylint: disable=C0207
331 'history')[-1].strip()
332 if not dest:
333 dest = os.path.join(folder, 'HISTORY.rst')
334 if existing_history is None:
335 hfold = get_folder(file_or_folder)
336 histo = os.path.join(hfold, 'HISTORY.rst')
337 if os.path.exists(histo):
338 existing_history = histo
339 if existing_history is not None:
340 print('[history] existing ', existing_history)
341 print('[history] ', dest)
342 build_history_from_setup(
343 dest, owner=github_owner, module=project_var_name,
344 existing_history=existing_history,
345 skip_issues=skip_issues, fLOG=fLOG)
346 return True
348 elif "write_version" in argv:
349 fLOG("---- JENKINS BEGIN WRITE VERSION ----")
350 write_version_for_setup(file_or_folder, module_name=module_name)
351 fLOG("---- JENKINS BEGIN END VERSION ----")
352 return True
354 elif "clean_pyd" in argv:
355 clean_space_for_setup(file_or_folder)
356 return True
358 elif "build_sphinx" in argv:
359 # delayed import
360 from .call_setup_hook import call_setup_hook
361 try:
362 from nbconvert.nbconvertapp import main as nbconvert_main
363 if nbconvert_main is None: # pragma: no cover
364 raise AttributeError("nbconvert_main is None")
365 except AttributeError as e: # pragma: no cover
366 raise ImportError(
367 "Unable to import nbconvert, cannot generate the documentation") from e
368 if setup_params is None:
369 setup_params = {}
370 out, err = call_setup_hook(
371 folder, project_var_name if module_name is None else module_name,
372 fLOG=fLOG, **setup_params)
373 if len(err) > 0 and err != "no _setup_hook":
374 raise Exception( # pragma: no cover
375 "Unable to run _setup_hook\nOUT:\n{0}\n[setuperror]\n"
376 "{1}".format(out, err))
378 if func_sphinx_begin is not None:
379 func_sphinx_begin(
380 argv=argv, file_or_folder=file_or_folder,
381 project_var_name=project_var_name,
382 module_name=module_name, unittest_modules=unittest_modules,
383 pattern_copy=pattern_copy,
384 requirements=requirements, port=port, blog_list=blog_list,
385 default_engine_paths=default_engine_paths,
386 extra_ext=extra_ext, add_htmlhelp=add_htmlhelp,
387 setup_params=setup_params, coverage_options=coverage_options,
388 coverage_exclude_lines=coverage_exclude_lines,
389 func_sphinx_begin=func_sphinx_begin,
390 func_sphinx_end=func_sphinx_end,
391 additional_notebook_path=additional_notebook_path,
392 nbformats=nbformats, layout=layout,
393 skip_function=skip_function,
394 addition_ut_path=additional_ut_path, fLOG=fLOG)
395 standard_help_for_setup(
396 argv, file_or_folder, project_var_name,
397 module_name=module_name, extra_ext=extra_ext,
398 add_htmlhelp=add_htmlhelp, copy_add_ext=copy_add_ext,
399 nbformats=nbformats, layout=layout,
400 use_run_cmd=use_run_cmd, fLOG=fLOG, direct_call=direct_call,
401 fexclude=fexclude)
403 if func_sphinx_end is not None:
404 func_sphinx_end(
405 argv=argv, file_or_folder=file_or_folder,
406 project_var_name=project_var_name,
407 module_name=module_name, unittest_modules=unittest_modules,
408 pattern_copy=pattern_copy,
409 requirements=requirements, port=port, blog_list=blog_list,
410 default_engine_paths=default_engine_paths,
411 extra_ext=extra_ext, add_htmlhelp=add_htmlhelp,
412 setup_params=setup_params, coverage_options=coverage_options,
413 coverage_exclude_lines=coverage_exclude_lines,
414 func_sphinx_begin=func_sphinx_begin,
415 func_sphinx_end=func_sphinx_end,
416 additional_notebook_path=additional_notebook_path,
417 nbformats=nbformats, layout=layout,
418 skip_function=skip_function,
419 addition_ut_path=additional_ut_path, fLOG=fLOG)
421 return True
423 elif "unittests" in argv:
424 skip_f = process_argv_for_unittest(argv)
425 run_unittests_for_setup(
426 file_or_folder, setup_params=setup_params,
427 coverage_options=coverage_options,
428 coverage_exclude_lines=coverage_exclude_lines,
429 additional_ut_path=additional_ut_path,
430 skip_function=skip_f, covtoken=covtoken,
431 hook_print=hook_print, stdout=stdout, stderr=stderr,
432 filter_warning=filter_warning, dump_coverage=dump_coverage_fct(),
433 add_coverage_folder=dump_coverage_fct(False),
434 coverage_root=coverage_root, fLOG=fLOG)
435 return True
437 elif "setup_hook" in argv:
438 fLOG("---- JENKINS BEGIN SETUPHOOK ----")
439 run_unittests_for_setup(
440 file_or_folder, setup_params=setup_params, only_setup_hook=True,
441 coverage_options=coverage_options, coverage_exclude_lines=coverage_exclude_lines,
442 additional_ut_path=additional_ut_path, skip_function=skip_function,
443 hook_print=hook_print, stdout=stdout, stderr=stderr, dump_coverage=dump_coverage_fct(),
444 fLOG=fLOG)
445 fLOG("---- JENKINS END SETUPHOOK ----")
446 return True
448 elif "unittests_LONG" in argv:
449 def skip_long(name, code, duration):
450 return "test_LONG_" not in name
451 run_unittests_for_setup(
452 file_or_folder, skip_function=skip_long, setup_params=setup_params,
453 coverage_options=coverage_options, coverage_exclude_lines=coverage_exclude_lines,
454 additional_ut_path=additional_ut_path, hook_print=hook_print,
455 stdout=stdout, stderr=stderr, dump_coverage=dump_coverage_fct(),
456 fLOG=fLOG)
457 return True
459 elif "unittests_SKIP" in argv:
460 def skip_skip(name, code, duration):
461 return "test_SKIP_" not in name
462 run_unittests_for_setup(
463 file_or_folder, skip_function=skip_skip, setup_params=setup_params,
464 coverage_options=coverage_options, coverage_exclude_lines=coverage_exclude_lines,
465 additional_ut_path=additional_ut_path, hook_print=hook_print,
466 stdout=stdout, stderr=stderr, dump_coverage=dump_coverage_fct(),
467 fLOG=fLOG)
468 return True
470 elif "unittests_GUI" in argv: # pragma: no cover
471 def skip_skip(name, code, duration):
472 return "test_GUI_" not in name
473 run_unittests_for_setup(
474 file_or_folder, skip_function=skip_skip, setup_params=setup_params,
475 coverage_options=coverage_options, coverage_exclude_lines=coverage_exclude_lines,
476 additional_ut_path=additional_ut_path, hook_print=hook_print,
477 stdout=stdout, stderr=stderr, dump_coverage=dump_coverage_fct(),
478 fLOG=fLOG)
479 return True
481 elif "build_script" in argv:
482 # delayed import
483 from .build_helper import get_extra_script_command, get_script_command, get_build_script
485 # script running setup.py
486 script = get_build_script(
487 project_var_name, requirements=requirements, port=port,
488 default_engine_paths=default_engine_paths)
489 binto = os.path.join(folder, "bin")
490 if not os.path.exists(binto):
491 os.mkdir(binto)
492 with open(os.path.join(folder, "bin", "auto_unittest_setup_help.%s" % get_script_extension()), "w") as f:
493 f.write(script)
495 for c in ("build_script", "clean_space",
496 "write_version", "clean_pyd",
497 "build_sphinx", "unittests",
498 "unittests_LONG", "unittests_SKIP", "unittests_GUI",
499 "unittests -d 5", "setup_hook", "copy27",
500 "local_pypi", 'run_pylint'):
501 sc = get_script_command(
502 c, project_var_name, requirements=requirements, port=port, platform=sys.platform,
503 default_engine_paths=default_engine_paths, additional_local_path=additional_local_path)
504 cn = c.replace(" ", "_")
505 with open(os.path.join(folder, "bin", "auto_setup_%s.%s" % (cn, get_script_extension())), "w") as f:
506 f.write(sc)
508 # script running for a developper
510 for c in {"notebook", "publish", "publish_doc", "local_pypi", "run27",
511 "build27", "setupdep", "copy_dist",
512 "any_setup_command", "build_dist",
513 "copy_sphinx", "lab", "history"}:
514 if "--private" in argv and "publish" in c:
515 # we skip this to avoid producing scripts for publish
516 # functionalities
517 continue
518 sc = get_extra_script_command(c, project_var_name, requirements=requirements,
519 port=port, platform=sys.platform,
520 default_engine_paths=default_engine_paths,
521 unit_test_folder=unit_test_folder,
522 unittest_modules=unittest_modules_script,
523 additional_notebook_path=additional_notebook_path,
524 additional_local_path=additional_local_path)
525 if sc is None:
526 continue
527 if c == "setupdep":
528 folder_setup = os.path.join(folder, "build", "auto_setup")
529 if not os.path.exists(folder_setup):
530 os.makedirs(folder_setup)
531 with open(os.path.join(folder_setup, "auto_setup_dep.py"), "w") as f:
532 f.write(sc)
533 else:
534 with open(os.path.join(folder, "bin", "auto_cmd_%s.%s" % (c, get_script_extension())), "w") as f:
535 f.write(sc)
537 # script for anybody
538 write_module_scripts(
539 folder, platform=sys.platform, blog_list=blog_list, default_engine_paths=default_engine_paths)
541 # pyproj for PTVS
542 if sys.platform.startswith("win"): # pragma: no cover
543 write_pyproj(folder)
545 return True
547 elif "copy27" in argv:
548 # delayed import
549 from .py3to2 import py3to2_convert_tree
550 root = os.path.abspath(os.path.dirname(file_or_folder))
551 root = os.path.normpath(root)
552 dest = os.path.join(root, "dist_module27")
553 py3to2_convert_tree(
554 root, dest, unittest_modules=unittest_modules_py3to2, pattern_copy=pattern_copy)
555 return True
557 elif "local_pypi" in argv: # pragma: no cover
558 # delayed import
559 from ..filehelper import get_url_content_timeout
560 url = "http://localhost:{0}/".format(port)
561 content = get_url_content_timeout(url, timeout=5)
562 if content is None or len(content) == 0:
563 raise Exception("test failed for url: " + url)
564 print(content)
565 return True
567 elif 'local_jenkins' in argv: # pragma: no cover
568 pos = argv.index("local_jenkins")
569 user = argv[pos + 1]
570 password = argv[pos + 2]
571 if len(argv) > pos + 3:
572 location = argv[pos + 3]
573 else:
574 if sys.platform.startswith("win"): # pragma: no cover
575 location = "\\Jenkins"
576 else:
577 location = "somewhere/workspace"
578 if len(argv) > pos + 4:
579 server_url = argv[pos + 4]
580 else:
581 server_url = "http://localhost:8080/"
582 from ..jenkinshelper import JenkinsExt, setup_jenkins_server_yml, default_jenkins_jobs
583 modules = default_jenkins_jobs(
584 github_owner=github_owner, module_name=project_var_name if module_name is None else module_name)
585 key = "Python%d%d" % sys.version_info[:2]
586 engines = {key: os.path.abspath(os.path.dirname(sys.executable))}
587 js = JenkinsExt(server_url, user, password, engines=engines)
588 setup_jenkins_server_yml(js, github=github_owner, modules=modules, fLOG=fLOG, overwrite=True,
589 delete_first=False, location=location)
590 return True
592 else:
593 return False
596def get_script_extension():
597 """
598 Returns the scripts extension based on the system it is running on.
600 @return bat or sh
601 """
602 if sys.platform.startswith("win"): # pragma: no cover
603 return "bat"
604 return "sh"
607def get_folder(file_or_folder):
608 """
609 Returns the folder which contains ``setup.py``.
611 @param file_or_folder file ``setup.py`` or folder which contains it
612 @return folder
613 """
614 file_or_folder = os.path.abspath(file_or_folder)
615 if os.path.isdir(file_or_folder):
616 folder = file_or_folder
617 else:
618 folder = os.path.dirname(file_or_folder)
619 return folder
622def write_version_for_setup(file_or_folder, exc=False, module_name=None):
623 """
624 Extracts the version number,
625 the function writes the files ``version.txt`` in this folder.
627 @param file_or_folder file ``setup.py`` or folder which contains it
628 @param exc raises an exception if cannot look into git folder
629 @param module_name module name
630 @return version number
631 """
632 # delayed import to speed up import of pycode
633 from ..loghelper.pyrepo_helper import SourceRepository
634 src = SourceRepository(commandline=True)
635 ffolder = get_folder(file_or_folder)
636 try:
637 version = src.version(ffolder)
638 except Exception as e: # pragma: no cover
639 if exc:
640 raise e
641 return None
642 if version in ["0", 0, None]:
643 raise Exception( # pragma: no cover
644 "issue with version {0}".format(version))
646 # write version number
647 if version is not None:
648 with open(os.path.join(ffolder, "version.txt"), "w") as f:
649 f.write(str(version) + "\n")
651 modifies_init_file(ffolder, version, module_name=module_name)
652 return version
655def clean_space_for_setup(file_or_folder, file_filter=None):
656 """
657 .. index:: pep8
659 Does some cleaning within the module, apply :epkg:`pep8` rules.
661 @param file_or_folder file ``setup.py`` or folder which contains it
662 @param file_filter file filter (see @see fn remove_extra_spaces_folder)
663 @return impacted files
664 """
665 # delayed import
666 from .code_helper import remove_extra_spaces_folder
667 ffolder = get_folder(file_or_folder)
668 rem = remove_extra_spaces_folder(
669 ffolder, extensions=[".py", ".rst", ".md", ".bat", ".sh"],
670 file_filter=file_filter)
671 return rem
674def clean_notebooks_for_numbers(file_or_folder):
675 """
676 Upgrades notebooks to the latest format and
677 cleans notebooks execution numbers and rearranges the JSON file.
679 @param file_or_folder file ``setup.py`` or folder which contains it
680 @return impacted files
682 .. index:: notebooks
683 """
684 from ..ipythonhelper.notebook_helper import upgrade_notebook, remove_execution_number
685 from ..filehelper import explore_folder_iterfile
686 ffolder = get_folder(file_or_folder)
687 fold2 = os.path.normpath(
688 os.path.join(ffolder, "_doc", "notebooks"))
689 mod = []
690 for nbf in explore_folder_iterfile(fold2, pattern=".*[.]ipynb"):
691 t = upgrade_notebook(nbf)
692 if t:
693 mod.append(nbf) # pragma: no cover
694 # remove numbers
695 s = remove_execution_number(nbf, nbf)
696 if s:
697 mod.append(nbf)
698 return mod
701def standard_help_for_setup(argv, file_or_folder, project_var_name, module_name=None,
702 extra_ext=None, add_htmlhelp=False, copy_add_ext=None,
703 nbformats=("ipynb", "html", "python",
704 "rst", "slides", "pdf"),
705 layout=None, use_run_cmd=False, direct_call=False,
706 fexclude=None, fLOG=None):
707 """
708 Standard function which generates help assuming they follow
709 the same design as *pyquickhelper*.
711 @param argv it should be ``sys.argv``
712 @param file_or_folder file ``setup.py`` or folder which contains it
713 @param project_var_name display name of the module
714 @param module_name module name, None if equal to *project_var_name*
715 (``import <module_name>``)
716 @param extra_ext extra file extension to process (ex ``["doc"]``)
717 @param add_htmlhelp run HTML Help too (only on Windows)
718 @param copy_add_ext additional extension of files to copy
719 @param nbformats notebooks format to generate
720 @param layout layout for the documentation, if None --> ``["html", "pdf"]``
721 @param use_run_cmd use function @see fn run_cmd instead of ``os.system``
722 to build the documentation
723 @param direct_call see @see fn generate_help_sphinx
724 @param fexclude function which tells which file not to copy in the folder
725 used to build the documentation
726 @param fLOG logging function
728 The function outputs some information through function @see fn fLOG.
730 A page will be added for each extra file extension mentioned in *extra_ext* if
731 some of these were found.
732 """
733 if fLOG is None: # pragma: no cover
734 from ..loghelper.flog import noLOG
735 fLOG = noLOG
736 if "--help" in argv: # pragma: no cover
737 from ..helpgen.help_usage import get_help_usage
738 print(get_help_usage())
739 else:
740 from ..helpgen.sphinx_main import generate_help_sphinx
742 if layout is None:
743 layout = ["html", "pdf"] # pragma: no cover
744 if module_name is None:
745 module_name = project_var_name
747 ffolder = get_folder(file_or_folder)
748 source = os.path.join(ffolder, "_doc", "sphinxdoc", "source")
750 if not os.path.exists(source):
751 raise FileNotFoundError( # pragma: no cover
752 "you must get the source from GitHub to build the documentation,\nfolder {0} "
753 "should exist\n(file_or_folder={1})\n(ffolder={2})\n(cwd={3})".format(
754 source, file_or_folder, ffolder, os.getcwd()))
756 if "conf" in sys.modules: # pragma: no cover
757 warnings.warn("module conf was imported, this function expects not to:\n{0}".format(
758 sys.modules["conf"].__file__))
759 del sys.modules["conf"]
761 project_name = os.path.split(
762 os.path.split(os.path.abspath(ffolder))[0])[-1]
764 generate_help_sphinx(project_name, module_name=module_name, layout=layout,
765 extra_ext=extra_ext, nbformats=nbformats, add_htmlhelp=add_htmlhelp,
766 copy_add_ext=copy_add_ext, fLOG=fLOG, root=ffolder,
767 direct_call=direct_call, fexclude=fexclude)
770def run_unittests_for_setup(file_or_folder, skip_function=None, setup_params=None,
771 only_setup_hook=False, coverage_options=None, coverage_exclude_lines=None,
772 additional_ut_path=None, covtoken=None, hook_print=True, stdout=None,
773 stderr=None, filter_warning=None, dump_coverage=None,
774 add_coverage_folder=None, coverage_root='src', fLOG=None):
775 """
776 Runs the unit tests and computes the coverage, stores
777 the results in ``_doc/sphinxdoc/source/coverage``
778 assuming the module follows the same design as *pyquickhelper*.
780 @param file_or_folder file ``setup.py`` or folder which contains it
781 @param skip_function see @see fn main_wrapper_tests
782 @param setup_params see @see fn main_wrapper_tests
783 @param only_setup_hook see @see fn main_wrapper_tests
784 @param coverage_options see @see fn main_wrapper_tests
785 @param coverage_exclude_lines see @see fn main_wrapper_tests
786 @param additional_ut_path see @see fn main_wrapper_tests
787 @param covtoken see @see fn main_wrapper_tests
788 @param hook_print see @see fn main_wrapper_tests
789 @param stdout see @see fn main_wrapper_tests
790 @param stderr see @see fn main_wrapper_tests
791 @param filter_warning see @see fn main_wrapper_tests
792 @param coverage_root see @see fn main_wrapper_tests
793 @param dump_coverage location where to dump the coverage
794 @param add_coverage_folder additional folder where to look for other coverage reports
795 @param fLOG logging function
797 See function @see fn main_wrapper_tests.
798 The coverage computation can be disabled by specifying
799 ``coverage_options["disable_coverage"] = True``.
800 *covtoken* was added to post the coverage report to
801 `codecov <https://codecov.io/>`_.
802 Parameter *dump_coverage*
803 dumps the unit test coverage in another location.
804 """
805 # delayed import
806 from .tkinter_helper import fix_tkinter_issues_virtualenv
807 from .utils_tests import main_wrapper_tests
808 ffolder = get_folder(file_or_folder)
809 funit = os.path.join(ffolder, "_unittests")
810 if not os.path.exists(funit):
811 raise FileNotFoundError( # pragma: no cover
812 "You must get the whole source to run the unittests,"
813 "\nfolder {0} should exist".format(funit))
815 if skip_function is None: # pragma: no cover
816 from .utils_tests_private import default_skip_function
817 skip_function = default_skip_function
818 if fLOG is None: # pragma: no cover
819 from ..loghelper.flog import noLOG
820 fLOG = noLOG
822 fix_tkinter_issues_virtualenv(fLOG=fLOG)
824 cov = True
825 if coverage_options:
826 if "disable_coverage" in coverage_options and coverage_options["disable_coverage"]:
827 cov = False
829 if dump_coverage is not None and not cov:
830 dump_coverage = None
832 logfile = os.path.join(funit, "unittests.out")
833 main_wrapper_tests(
834 logfile, add_coverage=cov, skip_function=skip_function, setup_params=setup_params,
835 only_setup_hook=only_setup_hook, coverage_options=coverage_options,
836 coverage_exclude_lines=coverage_exclude_lines, additional_ut_path=additional_ut_path,
837 covtoken=covtoken, hook_print=hook_print, stdout=stdout, stderr=stderr,
838 filter_warning=filter_warning, dump_coverage=dump_coverage,
839 add_coverage_folder=add_coverage_folder, coverage_root=coverage_root, fLOG=fLOG)
842def copy27_for_setup(file_or_folder): # pragma: no cover
843 """
844 Prepares a copy of the source for :epkg:`Python` 2.7,
845 assuming the module follows the same design as *pyquickhelper*.
847 @param file_or_folder file ``setup.py`` or folder which contains it
848 """
849 # delayed import
850 from .py3to2 import py3to2_convert_tree
851 root = get_folder(file_or_folder)
852 root = os.path.normpath(root)
853 dest = os.path.join(root, "dist_module27")
854 py3to2_convert_tree(root, dest)
857def write_pyproj(file_or_folder, location=None):
858 """
859 Creates a project
860 `pyproj <https://docs.microsoft.com/fr-fr/visualstudio/python/managing-python-projects-in-visual-studio>`_
861 to work with `PTVS <https://pytools.codeplex.com/>`_
862 (Python Tools for Visual Studio)
864 @param file_or_folder file ``setup.py`` or folder which contains it
865 @param location if not None, stores the project into this folder
867 This functionality fails with :epkg:`Python` 2.7 (encoding).
868 """
869 # delayed import
870 from ..filehelper import explore_folder_iterfile
871 from .build_helper import get_pyproj_project
873 avoid = ["dist", "build", "dist_module27",
874 "_doc", "_virtualenv", "_virtualenv27", "_venv"]
876 def filter(name):
877 if os.path.splitext(name)[-1] != ".py":
878 return False
879 if "temp_" in name:
880 return False
881 if "temp2_" in name:
882 return False # pragma: no cover
883 for a in avoid:
884 if name.startswith(a + "\\"):
885 return False # pragma: no cover
886 if name.startswith(a + "/"):
887 return False
888 return True
890 root = get_folder(file_or_folder)
891 root = os.path.normpath(root)
892 name = os.path.split(root)[-1]
893 if location is None:
894 dest = os.path.join(root, "ptvs_project.pyproj") # pragma: no cover
895 else:
896 dest = os.path.join(location, "ptvs_project.pyproj")
897 all_files = [os.path.relpath(_, root)
898 for _ in explore_folder_iterfile(root)]
899 all_files = [_ for _ in all_files if filter(_)]
900 pyproj = get_pyproj_project(name, all_files)
901 with open(dest, "w", encoding="utf8") as f:
902 f.write(pyproj.strip())
905def process_standard_options_for_setup_help(argv):
906 """
907 Prints the added options available through this module.
908 """
909 commands = {
910 "build27": "build the wheel for Python 2.7 (if available), it requires to run copy27 first",
911 "build_script": "produce various scripts to build the module",
912 "build_sphinx": "build the documentation",
913 "build_wheel": "build the wheel",
914 "clean_space": "clean unnecessary spaces in the code, applies :epkg:`pycodestyle` on all files",
915 "clean_pyd": "clean file ``*.pyd``",
916 "copy27": "create a modified copy of the module to run on Python 2.7 (if available), it requires to run copy27 first",
917 "copy_dist": "copy documentation to folder dist",
918 "copy_sphinx": "modify and copy sources to _doc/sphinxdoc/source/<module>",
919 "history": "builds the history of the package in RST",
920 "local_jenkins": "sets up jobs on a local jenkins server",
921 "run27": "run the unit tests for the Python 2.7",
922 "run_pylint": "run pylint on the sources, allowed parameters <pattern> <neg_pattern>",
923 "setup_hook": "call function setup_hook which initializes the module before running unit tests",
924 "unittests": "run the unit tests which do not contain test_LONG, test_SKIP or test_GUI in their file name",
925 "unittests_LONG": "run the unit tests which contain test_LONG their file name",
926 "unittests_SKIP": "run the unit tests which contain test_SKIP their file name",
927 "unittests_GUI": "run the unit tests which contain test_GUI their file name",
928 "write_version": "write a file ``version.txt`` with the version number (assuming sources are host with git)",
929 }
931 if "--help-commands" in argv:
932 print("Commands processed by pyquickhelper:")
933 for k, v in sorted(commands.items()):
934 print(" {0}{1}{2}".format(
935 k, " " * (len("copy27 ") - len(k)), v))
936 print()
937 elif "--help" in argv:
938 docu = 0
939 for k, v in sorted(commands.items()):
940 if k in argv:
941 docu += 1
943 if docu == 0: # pragma: no cover
944 print("pyquickhelper commands:")
945 print()
946 for k in sorted(commands):
947 process_standard_options_for_setup_help(['--help', k])
948 print()
949 else:
950 for k, v in sorted(commands.items()):
951 if k in argv:
952 docu += 1
953 print(" setup.py {0}{1}{2}".format(
954 k, " " * (20 - len(k)), v))
955 if k == "unittests":
956 print(
957 "\n {0} [-d seconds] [-f file] [-e regex] [-g regex]\n\n {1}".format(k, v))
958 print(
959 " -d seconds run all unit tests for which predicted duration is below a given threshold.")
960 print(
961 " -f file run all unit tests in file (do not use the full path)")
962 print(
963 " -e regex run all unit tests files matching the regular expression")
964 print(
965 " -g regex run all unit tests files not matching the regular expression")
966 print()
967 elif k == "local_jenkins": # pragma: no cover
968 print()
969 print(
970 " {0} user password [location] [server]".format(k))
971 print(" default location is somewhere/workspace")
972 print(" default server is http://localhost:8080/")
973 print()
976def write_module_scripts(folder, platform=sys.platform, blog_list=None,
977 default_engine_paths=None, command=None):
978 """
979 Writes a couple of scripts which allow a user to be faster on some tasks
980 or to easily get information about the module.
982 @param folder where to write the script
983 @param platform platform
984 @param blog_list blog list to follow, should be attribute ``__blog__`` of the module
985 @param command None to generate scripts for all commands or a value in *[blog, doc]*.
986 @param default_engine_paths default engines (or python distributions)
987 @return list of written scripts
989 The function produces the following files:
991 * *auto_rss_list.xml*: list of rss stream to follow
992 * *auto_rss_database.db3*: stores blog posts
993 * *auto_rss_server.py*: runs a server which updates the scripts and runs a server. It also open the default browser.
994 * *auto_rss_server.(bat|sh)*: run *auto_run_server.py*, the file on Linux might be missing if there is an equivalent python script
996 .. faqref::
997 :title: How to generate auto_rss_server.py?
999 The following code generates the script *auto_rss_local.py*
1000 which runs a local server to read blog posts included
1001 in the documentation (it uses module
1002 `pyrsslocal <http://www.xavierdupre.fr/app/pyrsslocal/helpsphinx/index.html>`_)::
1004 from pyquickhelper.pycode import write_module_scripts, __blog__
1005 write_module_scripts(".", blog_list=__blog__, command="blog")
1006 """
1007 # delayed import
1008 from .build_helper import get_script_module
1009 default_set = {"blog", "doc"}
1010 if command is not None:
1011 if command not in default_set:
1012 raise ValueError( # pragma: no cover
1013 "command {0} is not available in {1}".format(command, default_set))
1014 commands = {command}
1015 else:
1016 commands = default_set
1018 res = []
1019 for c in commands:
1020 sc = get_script_module(
1021 c, platform=sys.platform, blog_list=blog_list, default_engine_paths=default_engine_paths)
1022 if sc is None:
1023 continue # pragma: no cover
1024 tobin = os.path.join(folder, "bin")
1025 if not os.path.exists(tobin):
1026 os.mkdir(tobin)
1027 for item in sc:
1028 if isinstance(item, tuple):
1029 name = os.path.join(folder, "bin", item[0])
1030 with open(name, "w", encoding="utf8") as f:
1031 f.write(item[1])
1032 res.append(name)
1033 else: # pragma: no cover
1034 name = os.path.join(
1035 folder, "bin", "auto_run_%s.%s" % (c, get_script_extension()))
1036 with open(name, "w") as f:
1037 f.write(item)
1038 res.append(name)
1039 return res
1042def _get_dump_default_path(location, module_name, argv):
1043 """
1044 Proposes a default location to dump results about unit tests execution.
1046 @param location location of the module
1047 @param module_name module name
1048 @param argv argument on the command line
1049 @return location of the dump
1051 The result is None for remote continuous integration.
1052 """
1053 from . import is_travis_or_appveyor
1054 if is_travis_or_appveyor():
1055 return None # pragma: no cover
1056 hash = hash_list(argv)
1057 setup = os.path.join(location, "setup.py")
1058 if not os.path.exists(setup):
1059 raise FileNotFoundError(setup) # pragma: no cover
1060 fold = os.path.join(location, "..", "_coverage_dumps")
1061 if not os.path.exists(fold):
1062 os.mkdir(fold)
1063 dt = datetime.datetime.now().strftime("%Y%m%dT%H%M")
1064 if module_name is None:
1065 raise ValueError("module_name cannot be None") # pragma: no cover
1066 dump = os.path.join(fold, module_name, hash, dt)
1067 if not os.path.exists(dump):
1068 os.makedirs(dump)
1069 return dump
1072def hash_list(argv, size=8):
1073 """
1074 Proposes a hash for the list of arguments.
1076 @param argv list of arguments on the command line.
1077 @param size size of the hash
1078 @return string
1079 """
1080 st = "--".join(map(str, argv))
1081 hash = hashlib.md5()
1082 hash.update(st.encode("utf-8"))
1083 res = hash.hexdigest()
1084 if len(res) > 8:
1085 return res[:8]
1086 return res # pragma: no cover
1089def build_history_from_setup(dest, owner, module, existing_history=None,
1090 skip_issues=None, fLOG=None): # pragma: no cover
1091 """
1092 Builds the history from :epkg:`github` and :epkg:`pypi`.
1094 @param dest history will be written in this file
1095 @param owner owner of the package on :epkg:`github`
1096 @param module module name
1097 @param existing_history existing history, retrieves existing issues stored
1098 in that file
1099 @param skip_issues skip a given list of issues when building the history
1100 @param fLOG logging function
1101 @return history
1102 """
1103 # delayed import
1104 from ..loghelper.history_helper import build_history, compile_history
1105 if owner is None:
1106 raise ValueError( # pragma: no cover
1107 "owner must be specified.")
1108 if "/" in owner:
1109 raise ValueError( # pragma: no cover
1110 "owner %r cannot contain '/'." % owner)
1111 if fLOG is None: # pragma: no cover
1112 from ..loghelper.flog import noLOG
1113 fLOG = noLOG
1114 repo = module
1115 hist = build_history(owner, repo, unpublished=True,
1116 existing_history=existing_history,
1117 skip_issues=skip_issues, fLOG=fLOG)
1118 output = compile_history(hist)
1119 if dest is not None:
1120 with open(dest, "w", encoding="utf-8") as f:
1121 f.write(output)
1122 return output
1125def run_pylint_for_setup(folder, pattern=".*[.]py$", neg_pattern=None,
1126 verbose=False, pylint_ignore=None, fLOG=None):
1127 """
1128 Applies :epkg:`pylint` on subfolder *folder*.
1130 @param folder folder where to look
1131 @param pattern file to checks
1132 @param neg_pattern negative pattern
1133 @param pylint_ignore ignore these :epkg:`pylint` warnings or errors
1134 @param verbose verbose
1135 @param fLOG logging function
1136 """
1137 # delayed import
1138 from .utils_tests_helper import check_pep8
1139 if fLOG is None: # pragma: no cover
1140 from ..loghelper.flog import noLOG
1141 fLOG = noLOG
1142 check_pep8(folder, pattern=pattern, neg_pattern=neg_pattern,
1143 pylint_ignore=pylint_ignore, verbose=verbose, fLOG=fLOG)
1146def modifies_init_file(folder, version, module_name=None):
1147 """
1148 Automatically modifies the init file.
1150 @param folder where to find the init file
1151 @param version commit number
1152 @return modified init file
1153 """
1154 def _update_version(v, nv):
1155 vs = v.split('.')
1156 if len(vs) <= 2:
1157 return '.'.join(list(vs) + [nv])
1158 if len(vs) >= 3:
1159 vs = list(vs)
1160 vs[-1] = nv
1161 return '.'.join(vs)
1162 raise ValueError( # pragma: no cover
1163 "Unable to process '{}' with new version '{}'.".format(v, nv))
1165 filename = None
1166 if os.path.exists(folder):
1167 if os.path.isdir(folder):
1168 src = os.path.join(folder, 'src')
1169 if module_name is None:
1170 setu = os.path.join(folder, 'setup.py')
1171 if not os.path.exists(setu):
1172 raise FileNotFoundError( # pragma: no cover
1173 "Unable to find 'setup.py' in '{}' and module_name is "
1174 "None.".format(folder))
1175 reg = re.compile(
1176 "(project_var_name = ['\\\"]([a-zA-Z][a-zA-Z_0-9]+)['\\\"])")
1177 with open(setu, 'r', encoding='utf-8') as f:
1178 cst = f.read()
1179 find = reg.findall(cst)
1180 if len(find) == 0:
1181 raise FileNotFoundError( # pragma: no cover
1182 "Unable to find 'project_var_name' in 'setup.py' in '{}' "
1183 "and module_name is None.".format(folder))
1184 module_name = find[0][1]
1185 if os.path.exists(src) and module_name is not None:
1186 filename = os.path.join(src, module_name, '__init__.py')
1187 elif os.path.exists(src) and module_name is not None:
1188 filename = os.path.join(src, module_name, '__init__.py')
1189 elif module_name is not None:
1190 filename = os.path.join(folder, module_name, '__init__.py')
1191 else:
1192 raise FileNotFoundError( # pragma: no cover
1193 "Unable to find '__init__.py' in '{}' (module_name is None).".format(folder))
1194 if not os.path.exists(filename):
1195 raise FileNotFoundError( # pragma: no cover
1196 "Unable to find '__init__.py' in '{}' (got '{}').".format(folder, filename))
1197 with open(filename, 'r', encoding='utf-8') as f:
1198 content = f.read()
1199 elif '__version__' in folder:
1200 content = folder
1201 else:
1202 raise ValueError("Unable to process '{}'.".format(folder))
1204 reg = re.compile("(__version__ = ['\\\"]([0-9.]+)['\\\"])")
1205 lines = content.split('\n')
1206 modif = []
1207 rep = []
1208 for line in lines:
1209 if line.startswith("__version__"):
1210 find = reg.findall(line)
1211 if len(find) != 1:
1212 raise ValueError( # pragma: no cover
1213 "Unable to find __version__ in '{}'".format(line))
1214 v = find[0][1]
1215 nv = _update_version(v, str(version))
1216 newline = line.replace(v, nv)
1217 modif.append(newline)
1218 rep.append((line, newline))
1219 else:
1220 modif.append(line)
1221 if len(rep) == 0:
1222 raise ValueError( # pragma: no cover
1223 "Unable to find '__version__' in \n{}".format(content))
1225 content = '\n'.join(modif)
1226 if filename is not None:
1227 with open(filename, 'w', encoding='utf-8') as f:
1228 f.write(content)
1229 return content