"""
Helper for the setup
:githublink:`%|py|5`
"""
import os
import sys
import re
import warnings
import hashlib
import datetime
[docs]def get_available_setup_commands():
"""
Returns the list of commands :epkg:`pyquickhelper` implements
or allows.
:githublink:`%|py|18`
"""
commands = ['bdist_egg', 'bdist_msi', 'bdist_wheel', 'bdist_wininst', 'build27',
'build_ext', 'build_script', 'build_sphinx', 'clean_pyd', 'clean_space',
'copy27', 'copy_dist', 'copy_sphinx', 'history', 'lab', 'local_pypi',
'notebook', 'publish', 'publish_doc', 'register', 'run27', 'run_pylint',
'sdist', 'setup_hook', 'setupdep', 'test_local_pypi',
'unittests', 'unittests_GUI', 'unittests_LONG', 'unittests_SKIP',
'upload_docs', 'write_version', 'local_jenkins']
return commands
[docs]def get_available_build_commands():
"""
Returns commands which build the package.
:githublink:`%|py|32`
"""
return {"sdist", "bdist_wheel", "publish", "publish_doc", "register",
"upload_docs", "bdist_wininst", "build_ext"}
[docs]def available_commands_list(argv):
"""
Checks that on command handled by pyquickhelper is part of the arguments.
:param argv: ``sys.args``
:return: bool
:githublink:`%|py|43`
"""
commands = get_available_setup_commands()
for c in commands:
if c in argv:
return True
return False
[docs]def process_standard_options_for_setup(
argv, file_or_folder, project_var_name, module_name=None, unittest_modules=None,
pattern_copy=None,
requirements=None, port=8067, blog_list=None, default_engine_paths=None,
extra_ext=None, add_htmlhelp=False, setup_params=None, coverage_options=None,
coverage_exclude_lines=None, func_sphinx_begin=None, func_sphinx_end=None,
additional_notebook_path=None, additional_local_path=None, copy_add_ext=None,
nbformats=("ipynb", "html", "python", "rst", "slides",
"pdf", "github"),
layout=None, direct_call=False,
additional_ut_path=None,
skip_function=None, covtoken=None, hook_print=True,
stdout=None, stderr=None, use_run_cmd=False, filter_warning=None,
file_filter_pep8=None, github_owner=None,
existing_history=None, coverage_root='src',
fexclude=None, skip_issues=None, fLOG=None):
"""
Processes the standard options the module pyquickhelper is
able to process assuming the module which calls this function
follows the same design as *pyquickhelper*, it will process the following
options:
.. runpython::
from pyquickhelper.pycode import process_standard_options_for_setup_help
process_standard_options_for_setup_help("--help-commands")
:param argv: = *sys.argv*
:param file_or_folder: file ``setup.py`` or folder which contains it
:param project_var_name: display name of the module
:param module_name: module name, None if equal to *project_var_name* (``import <module_name>``)
:param unittest_modules: modules added for the unit tests, see :func:`py3to2_convert_tree <pyquickhelper.pycode.py3to2.py3to2_convert_tree>`
:param pattern_copy: see :func:`py3to2_convert_tree <pyquickhelper.pycode.py3to2.py3to2_convert_tree>`
:param requirements: dependencies, fetched with a local pipy server from ``http://localhost:port/``
:param port: port for the local pipy server
:param blog_list: list of blog to listen for this module (usually stored in ``module.__blog__``)
:param default_engine_paths: define the default location for python engine, should be dictionary *{ engine: path }*, see below.
:param extra_ext: extra file extension to process (add a page for each of them, ex ``["doc"]``)
:param add_htmlhelp: run HTML Help too (only on Windows)
:param setup_params: parameters send to :func:`call_setup_hook <pyquickhelper.pycode.call_setup_hook.call_setup_hook>`
:param coverage_options: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param coverage_exclude_lines: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param func_sphinx_begin: function called before the documentation generation,
it gets the same parameters as this function (all named),
use ``**args**``
:param func_sphinx_end: function called after the documentation generation,
it gets the same parameters as this function (all named),
use ``**args**``
:param additional_notebook_path: additional paths to add when launching the notebook
:param additional_local_path: additional paths to add when running a local command
:param copy_add_ext: additional file extensions to copy
:param nbformats: requested formats for the notebooks conversion
:param layout: list of formats sphinx should generate such as html, latex, pdf, docx,
it is a list of tuple (layout, build directory, parameters to override),
if None --> ``["html", "pdf"]``
:param additional_ut_path: additional paths to add when running unit tests
:param skip_function: function to skip unit tests, see @ee fn main_wrapper_tests
:param covtoken: token used when publishing coverage report to `codecov <https://codecov.io/>`_,
more in :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param fLOG: logging function
:param hook_print: enable, disable print when calling *_setup_hook*
:param stdout: redirect stdout for unit test if not None
:param stderr: redirect stderr for unit test if not None
:param use_run_cmd: to run the sphinx documentation with :func:`run_cmd <pyquickhelper.loghelper.run_cmd.run_cmd>` and not ``os.system``
:param filter_warning: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param file_filter_pep8: function to filter out files for which checking pep8
(see :func:`remove_extra_spaces_folder <pyquickhelper.pycode.code_helper.remove_extra_spaces_folder>`)
:param github_owner: :epkg:`github` owner of the package
:param existing_history: existing history, retrieves existing issues stored
in that file
:param coverage_root: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param direct_call: :func:`generate_help_sphinx <pyquickhelper.helpgen.sphinx_main.generate_help_sphinx>`
:param fexclude: function which tells which file not to copy in the folder
used to build the documentation
:param skip_issues: skip a given list of issues when building the history
:return: True (an option was processed) or False,
the file ``setup.py`` should call function ``setup``
The command ``build_script`` is used, the flag ``--private`` can be used to
avoid producing scripts to publish the module on `Pypi <https://pypi.python.org/pypi>`_.
An example for *default_engine_paths*::
default_engine_paths = {
"windows": {
"__PY35__": None,
"__PY36_X64__": "c:\\Python365_x64",
"__PY37_X64__": "c:\\Python372_x64",
"__PY38_X64__": "c:\\Python387_x64",
"__PY39_X64__": "c:\\Python391_x64",
},
}
Parameters *coverage_options*, *coverage_exclude_lines*, *copy_add_ext* were added
for function :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`.
Parameter *unittest_modules* accepts a list of string and 2-uple.
If it is a 2-uple, the first string is used to convert Python 3 code into Python 2
(in case the local folder is different from the module name),
the second string is used to add local path to the variable ``PYTHON_PATH``.
If it is a single string, it means both name strings are equal.
Parameters *func_sphinx_begin* and *func_sphinx_end* were added
to pre-process or post-process the documentation.
Parameter *additional_notebook_path* was added to specify some additional
paths when preparing the script *auto_cmd_notebook.bat*.
Parameters *layout*, *nbformats* were added for
function :func:`generate_help_sphinx <pyquickhelper.helpgen.sphinx_main.generate_help_sphinx>`.
The coverage computation can be disable by specifying
``coverage_options["disable_coverage"] = True``.
Parameter *covtoken* was added to post the coverage report to :epkg:`codecov`.
Option ``-e`` and ``-g`` were added to
filter file by regular expressions (in with *e*, out with *g*).
.. versionchanged:: 1.8
Parameters *coverage_root*, *direct_call* were added.
:githublink:`%|py|165`
"""
if fLOG is None: # pragma: no cover
from ..loghelper.flog import noLOG
fLOG = noLOG
if skip_function is None: # pragma: no cover
from .utils_tests_private import default_skip_function
skip_function = default_skip_function
if pattern_copy is None:
# delayed import
from .default_regular_expression import _setup_pattern_copy
pattern_copy = _setup_pattern_copy
if layout is None:
layout = ["html", "pdf"]
if "--help" in argv or "--help-commands" in argv:
process_standard_options_for_setup_help(argv)
return not len(get_available_build_commands() & set(argv))
fLOG("[process_standard_options_for_setup]", argv)
fLOG("[process_standard_options_for_setup] python version:", sys.version_info)
def process_argv_for_unittest(argv):
if "-d" in argv:
ld = argv.index("-d")
if ld >= len(argv) - 1:
raise ValueError(
"Option -d should be follow by a duration in seconds.")
d = float(argv[ld + 1])
else:
d = None
if "-f" in argv:
lf = argv.index("-f")
if lf >= len(argv) - 1:
raise ValueError(
"Option -d should be follow by a duration in seconds.")
f = argv[lf + 1]
else:
f = None
if "-e" in argv:
le = argv.index("-e")
if le >= len(argv) - 1:
raise ValueError( # pragma: no cover
"Option -e should be follow by a regular expression.")
e = re.compile(argv[le + 1])
else:
e = None
if "-g" in argv:
lg = argv.index("-g")
if lg >= len(argv) - 1:
raise ValueError( # pragma: no cover
"Option -g should be follow by a regular expression.")
g = re.compile(argv[lg + 1])
else:
g = None
if f is None and d is None and e is None and g is None:
return skip_function # pragma: no cover
def ereg(name):
return (e is None) or (e.search(name) is not None)
def greg(name):
return (g is None) or (g.search(name) is None)
if f is not None:
if d is not None: # pragma: no cover
raise NotImplementedError(
"Options -f and -d cannot be specified at the same time.")
def allow(name, code, duration): # pragma: no cover
name = os.path.split(name)[-1]
return f not in name and ereg(name) and greg(name)
return allow # pragma: no cover
else:
# d is not None
def skip_allowd(name, code, duration):
name = os.path.split(name)[-1]
cond = (duration is None or d is None or duration <=
d) and ereg(name) and greg(name)
return not cond
return skip_allowd
folder = file_or_folder if os.path.isdir(
file_or_folder) else os.path.dirname(file_or_folder)
unit_test_folder = os.path.join(folder, "_unittests")
fLOG("unittest_modules={0}".format(unittest_modules))
if unittest_modules is None:
unittest_modules_py3to2 = None
unittest_modules_script = None
else: # pragma: no cover
unittest_modules_py3to2 = []
unittest_modules_script = []
for mod in unittest_modules:
if isinstance(mod, tuple):
unittest_modules_py3to2.append(mod[0])
unittest_modules_script.append(mod[1])
else:
unittest_modules_py3to2.append(mod)
unittest_modules_script.append(mod)
# dump unit test coverage?
def dump_coverage_fct(full=True):
mn = project_var_name if module_name is None else module_name
full_path = _get_dump_default_path(folder, mn, argv)
if full_path is None or full:
return full_path
else:
sub = os.path.split(full_path)[0]
sub = os.path.split(sub)[0]
return sub
# starts interpreting the commands
if "clean_space" in argv: # pragma: no cover
rem = clean_space_for_setup(
file_or_folder, file_filter=file_filter_pep8)
print("[clean_space] number of impacted files (pep8 + rst):", len(rem))
rem = clean_notebooks_for_numbers(file_or_folder)
print("[clean_space] number of impacted notebooks:", len(rem))
return True
if "run_pylint" in argv:
verbose = '-v' in argv
pos = argv.index('run_pylint')
ignores = [_[2:] for _ in argv if _[:2] == '-i']
ignores = None if len(ignores) == 0 else tuple(ignores)
argv = [_ for _ in argv if _ not in (
'-v', '-') and not _.startswith('-i')]
pattern = argv[pos + 1] if len(argv) > pos + 1 else ".*[.]py$"
neg_pattern = argv[pos + 2] if len(argv) > pos + 2 else None
print("[run_pylint] run_pylint for sources pattern='{0}' neg_pattern='{1}'".format(
pattern, neg_pattern))
src_folder = os.path.join(folder, "src")
if not os.path.exists(src_folder):
src_folder = folder
run_pylint_for_setup(src_folder, fLOG=print, pattern=pattern,
verbose=verbose, pylint_ignore=ignores)
print("[run_pylint] run_pylint for unittest")
run_pylint_for_setup(os.path.join(folder, "_unittests"),
fLOG=print, pattern=pattern, verbose=verbose,
pylint_ignore=ignores)
return True
elif 'history' in argv: # pragma: no cover
dest = ' '.join(argv).split('history')[-1].strip()
if not dest:
dest = os.path.join(folder, 'HISTORY.rst')
if existing_history is None:
hfold = get_folder(file_or_folder)
histo = os.path.join(hfold, 'HISTORY.rst')
if os.path.exists(histo):
existing_history = histo
if existing_history is not None:
print('[history] existing ', existing_history)
print('[history] ', dest)
build_history_from_setup(dest, owner=github_owner, module=project_var_name,
existing_history=existing_history,
skip_issues=skip_issues, fLOG=fLOG)
return True
elif "write_version" in argv:
fLOG("---- JENKINS BEGIN WRITE VERSION ----")
write_version_for_setup(file_or_folder, module_name=module_name)
fLOG("---- JENKINS BEGIN END VERSION ----")
return True
elif "clean_pyd" in argv:
clean_space_for_setup(file_or_folder)
return True
elif "build_sphinx" in argv:
# delayed import
from .call_setup_hook import call_setup_hook
try:
from nbconvert.nbconvertapp import main as nbconvert_main
if nbconvert_main is None: # pragma: no cover
raise AttributeError("nbconvert_main is None")
except AttributeError as e: # pragma: no cover
raise ImportError(
"Unable to import nbconvert, cannot generate the documentation") from e
if setup_params is None:
setup_params = {}
out, err = call_setup_hook(folder,
project_var_name if module_name is None else module_name,
fLOG=fLOG,
**setup_params)
if len(err) > 0 and err != "no _setup_hook":
raise Exception( # pragma: no cover
"unable to run _setup_hook\nOUT:\n{0}\n[setuperror]\n{1}".format(out, err))
if func_sphinx_begin is not None:
func_sphinx_begin(
argv=argv, file_or_folder=file_or_folder,
project_var_name=project_var_name,
module_name=module_name, unittest_modules=unittest_modules,
pattern_copy=pattern_copy,
requirements=requirements, port=port, blog_list=blog_list,
default_engine_paths=default_engine_paths,
extra_ext=extra_ext, add_htmlhelp=add_htmlhelp,
setup_params=setup_params, coverage_options=coverage_options,
coverage_exclude_lines=coverage_exclude_lines,
func_sphinx_begin=func_sphinx_begin,
func_sphinx_end=func_sphinx_end,
additional_notebook_path=additional_notebook_path,
nbformats=nbformats, layout=layout,
skip_function=skip_function,
addition_ut_path=additional_ut_path, fLOG=fLOG)
standard_help_for_setup(
argv,
file_or_folder, project_var_name,
module_name=module_name, extra_ext=extra_ext,
add_htmlhelp=add_htmlhelp, copy_add_ext=copy_add_ext,
nbformats=nbformats, layout=layout,
use_run_cmd=use_run_cmd, fLOG=fLOG, direct_call=direct_call,
fexclude=fexclude)
if func_sphinx_end is not None:
func_sphinx_end(
argv=argv, file_or_folder=file_or_folder,
project_var_name=project_var_name,
module_name=module_name, unittest_modules=unittest_modules,
pattern_copy=pattern_copy,
requirements=requirements, port=port, blog_list=blog_list,
default_engine_paths=default_engine_paths,
extra_ext=extra_ext, add_htmlhelp=add_htmlhelp,
setup_params=setup_params, coverage_options=coverage_options,
coverage_exclude_lines=coverage_exclude_lines,
func_sphinx_begin=func_sphinx_begin,
func_sphinx_end=func_sphinx_end,
additional_notebook_path=additional_notebook_path,
nbformats=nbformats, layout=layout,
skip_function=skip_function,
addition_ut_path=additional_ut_path, fLOG=fLOG)
return True
elif "unittests" in argv:
skip_f = process_argv_for_unittest(argv)
run_unittests_for_setup(
file_or_folder, setup_params=setup_params,
coverage_options=coverage_options,
coverage_exclude_lines=coverage_exclude_lines,
additional_ut_path=additional_ut_path,
skip_function=skip_f, covtoken=covtoken,
hook_print=hook_print, stdout=stdout, stderr=stderr,
filter_warning=filter_warning, dump_coverage=dump_coverage_fct(),
add_coverage_folder=dump_coverage_fct(False),
coverage_root=coverage_root, fLOG=fLOG)
return True
elif "setup_hook" in argv:
fLOG("---- JENKINS BEGIN SETUPHOOK ----")
run_unittests_for_setup(
file_or_folder, setup_params=setup_params, only_setup_hook=True,
coverage_options=coverage_options, coverage_exclude_lines=coverage_exclude_lines,
additional_ut_path=additional_ut_path, skip_function=skip_function,
hook_print=hook_print, stdout=stdout, stderr=stderr, dump_coverage=dump_coverage_fct(),
fLOG=fLOG)
fLOG("---- JENKINS END SETUPHOOK ----")
return True
elif "unittests_LONG" in argv:
def skip_long(name, code, duration):
return "test_LONG_" not in name
run_unittests_for_setup(
file_or_folder, skip_function=skip_long, setup_params=setup_params,
coverage_options=coverage_options, coverage_exclude_lines=coverage_exclude_lines,
additional_ut_path=additional_ut_path, hook_print=hook_print,
stdout=stdout, stderr=stderr, dump_coverage=dump_coverage_fct(),
fLOG=fLOG)
return True
elif "unittests_SKIP" in argv:
def skip_skip(name, code, duration):
return "test_SKIP_" not in name
run_unittests_for_setup(
file_or_folder, skip_function=skip_skip, setup_params=setup_params,
coverage_options=coverage_options, coverage_exclude_lines=coverage_exclude_lines,
additional_ut_path=additional_ut_path, hook_print=hook_print,
stdout=stdout, stderr=stderr, dump_coverage=dump_coverage_fct(),
fLOG=fLOG)
return True
elif "unittests_GUI" in argv: # pragma: no cover
def skip_skip(name, code, duration):
return "test_GUI_" not in name
run_unittests_for_setup(
file_or_folder, skip_function=skip_skip, setup_params=setup_params,
coverage_options=coverage_options, coverage_exclude_lines=coverage_exclude_lines,
additional_ut_path=additional_ut_path, hook_print=hook_print,
stdout=stdout, stderr=stderr, dump_coverage=dump_coverage_fct(),
fLOG=fLOG)
return True
elif "build_script" in argv:
# delayed import
from .build_helper import get_extra_script_command, get_script_command, get_build_script
# script running setup.py
script = get_build_script(
project_var_name, requirements=requirements, port=port,
default_engine_paths=default_engine_paths)
binto = os.path.join(folder, "bin")
if not os.path.exists(binto):
os.mkdir(binto)
with open(os.path.join(folder, "bin", "auto_unittest_setup_help.%s" % get_script_extension()), "w") as f:
f.write(script)
for c in ("build_script", "clean_space",
"write_version", "clean_pyd",
"build_sphinx", "unittests",
"unittests_LONG", "unittests_SKIP", "unittests_GUI",
"unittests -d 5", "setup_hook", "copy27",
"test_local_pypi", 'run_pylint'):
sc = get_script_command(
c, project_var_name, requirements=requirements, port=port, platform=sys.platform,
default_engine_paths=default_engine_paths, additional_local_path=additional_local_path)
cn = c.replace(" ", "_")
with open(os.path.join(folder, "bin", "auto_setup_%s.%s" % (cn, get_script_extension())), "w") as f:
f.write(sc)
# script running for a developper
for c in {"notebook", "publish", "publish_doc", "local_pypi", "run27",
"build27", "setupdep", "copy_dist",
"any_setup_command", "build_dist",
"copy_sphinx", "lab", "history"}:
if "--private" in argv and "publish" in c:
# we skip this to avoid producing scripts for publish
# functionalities
continue
sc = get_extra_script_command(c, project_var_name, requirements=requirements,
port=port, platform=sys.platform,
default_engine_paths=default_engine_paths,
unit_test_folder=unit_test_folder,
unittest_modules=unittest_modules_script,
additional_notebook_path=additional_notebook_path,
additional_local_path=additional_local_path)
if sc is None:
continue
if c == "setupdep":
folder_setup = os.path.join(folder, "build", "auto_setup")
if not os.path.exists(folder_setup):
os.makedirs(folder_setup)
with open(os.path.join(folder_setup, "auto_setup_dep.py"), "w") as f:
f.write(sc)
else:
with open(os.path.join(folder, "bin", "auto_cmd_%s.%s" % (c, get_script_extension())), "w") as f:
f.write(sc)
# script for anybody
write_module_scripts(
folder, platform=sys.platform, blog_list=blog_list, default_engine_paths=default_engine_paths)
# pyproj for PTVS
if sys.platform.startswith("win"): # pragma: no cover
write_pyproj(folder)
return True
elif "copy27" in argv:
# delayed import
from .py3to2 import py3to2_convert_tree
root = os.path.abspath(os.path.dirname(file_or_folder))
root = os.path.normpath(root)
dest = os.path.join(root, "dist_module27")
py3to2_convert_tree(
root, dest, unittest_modules=unittest_modules_py3to2, pattern_copy=pattern_copy)
return True
elif "test_local_pypi" in argv: # pragma: no cover
# delayed import
from ..filehelper import get_url_content_timeout
url = "http://localhost:{0}/".format(port)
content = get_url_content_timeout(url, timeout=5)
if content is None or len(content) == 0:
raise Exception("test failed for url: " + url)
print(content)
return True
elif 'local_jenkins' in argv: # pragma: no cover
pos = argv.index("local_jenkins")
user = argv[pos + 1]
password = argv[pos + 2]
if len(argv) > pos + 3:
location = argv[pos + 3]
else:
if sys.platform.startswith("win"): # pragma: no cover
location = "\\Jenkins"
else:
location = "somewhere/workspace"
if len(argv) > pos + 4:
server_url = argv[pos + 4]
else:
server_url = "http://localhost:8080/"
from ..jenkinshelper import JenkinsExt, setup_jenkins_server_yml, default_jenkins_jobs
modules = default_jenkins_jobs(
github_owner=github_owner, module_name=project_var_name if module_name is None else module_name)
key = "Python%d%d" % sys.version_info[:2]
engines = {key: os.path.abspath(os.path.dirname(sys.executable))}
js = JenkinsExt(server_url, user, password, engines=engines)
setup_jenkins_server_yml(js, github=github_owner, modules=modules, fLOG=fLOG, overwrite=True,
delete_first=False, location=location)
return True
else:
return False
[docs]def get_script_extension():
"""
Returns the scripts extension based on the system it is running on.
:return: bat or sh
:githublink:`%|py|584`
"""
if sys.platform.startswith("win"): # pragma: no cover
return "bat"
return "sh"
[docs]def get_folder(file_or_folder):
"""
Returns the folder which contains ``setup.py``.
:param file_or_folder: file ``setup.py`` or folder which contains it
:return: folder
:githublink:`%|py|596`
"""
file_or_folder = os.path.abspath(file_or_folder)
if os.path.isdir(file_or_folder):
folder = file_or_folder
else:
folder = os.path.dirname(file_or_folder)
return folder
[docs]def write_version_for_setup(file_or_folder, exc=False, module_name=None):
"""
Extracts the version number,
the function writes the files ``version.txt`` in this folder.
:param file_or_folder: file ``setup.py`` or folder which contains it
:param exc: raises an exception if cannot look into git folder
:param module_name: module name
:return: version number
.. versionchanged:: 1.8
Parameter *exc* was added.
:githublink:`%|py|617`
"""
# delayed import to speed up import of pycode
from ..loghelper.pyrepo_helper import SourceRepository
src = SourceRepository(commandline=True)
ffolder = get_folder(file_or_folder)
try:
version = src.version(ffolder)
except Exception as e: # pragma: no cover
if exc:
raise e
return None
if version in ["0", 0, None]:
raise Exception( # pragma: no cover
"issue with version {0}".format(version))
# write version number
if version is not None:
with open(os.path.join(ffolder, "version.txt"), "w") as f:
f.write(str(version) + "\n")
modifies_init_file(ffolder, version, module_name=module_name)
return version
[docs]def clean_space_for_setup(file_or_folder, file_filter=None):
"""
.. index:: pep8
Does some cleaning within the module, apply :epkg:`pep8` rules.
:param file_or_folder: file ``setup.py`` or folder which contains it
:param file_filter: file filter (see :func:`remove_extra_spaces_folder <pyquickhelper.pycode.code_helper.remove_extra_spaces_folder>`)
:return: impacted files
:githublink:`%|py|650`
"""
# delayed import
from .code_helper import remove_extra_spaces_folder
ffolder = get_folder(file_or_folder)
rem = remove_extra_spaces_folder(
ffolder, extensions=[".py", ".rst", ".md", ".bat", ".sh"],
file_filter=file_filter)
return rem
[docs]def clean_notebooks_for_numbers(file_or_folder):
"""
Upgrades notebooks to the latest format and
cleans notebooks execution numbers and rearranges the JSON file.
:param file_or_folder: file ``setup.py`` or folder which contains it
:return: impacted files
.. index:: notebooks
:githublink:`%|py|669`
"""
from ..ipythonhelper.notebook_helper import upgrade_notebook, remove_execution_number
from ..filehelper import explore_folder_iterfile
ffolder = get_folder(file_or_folder)
fold2 = os.path.normpath(
os.path.join(ffolder, "_doc", "notebooks"))
mod = []
for nbf in explore_folder_iterfile(fold2, pattern=".*[.]ipynb"):
t = upgrade_notebook(nbf)
if t:
mod.append(nbf) # pragma: no cover
# remove numbers
s = remove_execution_number(nbf, nbf)
if s:
mod.append(nbf)
return mod
[docs]def standard_help_for_setup(argv, file_or_folder, project_var_name, module_name=None, extra_ext=None,
add_htmlhelp=False, copy_add_ext=None,
nbformats=("ipynb", "html", "python",
"rst", "slides", "pdf"),
layout=None, use_run_cmd=False, direct_call=False,
fexclude=None, fLOG=None):
"""
Standard function which generates help assuming they follow the same design
as *pyquickhelper*.
:param argv: it should be ``sys.argv``
:param file_or_folder: file ``setup.py`` or folder which contains it
:param project_var_name: display name of the module
:param module_name: module name, None if equal to *project_var_name* (``import <module_name>``)
:param extra_ext: extra file extension to process (ex ``["doc"]``)
:param add_htmlhelp: run HTML Help too (only on Windows)
:param copy_add_ext: additional extension of files to copy
:param nbformats: notebooks format to generate
:param layout: layout for the documentation, if None --> ``["html", "pdf"]``
:param use_run_cmd: use function :func:`run_cmd <pyquickhelper.loghelper.run_cmd.run_cmd>` instead of ``os.system``
to build the documentation
:param direct_call: see :func:`generate_help_sphinx <pyquickhelper.helpgen.sphinx_main.generate_help_sphinx>`
:param fexclude: function which tells which file not to copy in the folder
used to build the documentation
:param fLOG: logging function
The function outputs some information through function :func:`fLOG <pyquickhelper.loghelper.flog.fLOG>`.
A page will be added for each extra file extension mentioned in *extra_ext* if
some of these were found.
:githublink:`%|py|717`
"""
if fLOG is None: # pragma: no cover
from ..loghelper.flog import noLOG
fLOG = noLOG
if "--help" in argv: # pragma: no cover
from ..helpgen.help_usage import get_help_usage
print(get_help_usage())
else:
from ..helpgen.sphinx_main import generate_help_sphinx
if layout is None:
layout = ["html", "pdf"] # pragma: no cover
if module_name is None:
module_name = project_var_name
ffolder = get_folder(file_or_folder)
source = os.path.join(ffolder, "_doc", "sphinxdoc", "source")
if not os.path.exists(source):
raise FileNotFoundError( # pragma: no cover
"you must get the source from GitHub to build the documentation,\nfolder {0} "
"should exist\n(file_or_folder={1})\n(ffolder={2})\n(cwd={3})".format(
source, file_or_folder, ffolder, os.getcwd()))
if "conf" in sys.modules: # pragma: no cover
warnings.warn("module conf was imported, this function expects not to:\n{0}".format(
sys.modules["conf"].__file__))
del sys.modules["conf"]
project_name = os.path.split(
os.path.split(os.path.abspath(ffolder))[0])[-1]
generate_help_sphinx(project_name, module_name=module_name, layout=layout,
extra_ext=extra_ext, nbformats=nbformats, add_htmlhelp=add_htmlhelp,
copy_add_ext=copy_add_ext, fLOG=fLOG, root=ffolder,
direct_call=direct_call, fexclude=fexclude)
[docs]def run_unittests_for_setup(file_or_folder, skip_function=None, setup_params=None,
only_setup_hook=False, coverage_options=None, coverage_exclude_lines=None,
additional_ut_path=None, covtoken=None, hook_print=True, stdout=None,
stderr=None, filter_warning=None, dump_coverage=None,
add_coverage_folder=None, coverage_root='src', fLOG=None):
"""
Runs the unit tests and computes the coverage, stores
the results in ``_doc/sphinxdoc/source/coverage``
assuming the module follows the same design as *pyquickhelper*.
:param file_or_folder: file ``setup.py`` or folder which contains it
:param skip_function: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param setup_params: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param only_setup_hook: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param coverage_options: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param coverage_exclude_lines: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param additional_ut_path: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param covtoken: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param hook_print: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param stdout: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param stderr: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param filter_warning: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param coverage_root: see :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`
:param dump_coverage: location where to dump the coverage
:param add_coverage_folder: additional folder where to look for other coverage reports
:param fLOG: logging function
See function :func:`main_wrapper_tests <pyquickhelper.pycode.utils_tests.main_wrapper_tests>`.
The coverage computation can be disabled by specifying
``coverage_options["disable_coverage"] = True``.
*covtoken* was added to post the coverage report to
`codecov <https://codecov.io/>`_.
Parameter *dump_coverage*
dumps the unit test coverage in another location.
.. versionchanged:: 1.8
Parameter *coverage_root* was added.
:githublink:`%|py|792`
"""
# delayed import
from .tkinter_helper import fix_tkinter_issues_virtualenv
from .utils_tests import main_wrapper_tests
ffolder = get_folder(file_or_folder)
funit = os.path.join(ffolder, "_unittests")
if not os.path.exists(funit):
raise FileNotFoundError( # pragma: no cover
"You must get the whole source to run the unittests,"
"\nfolder {0} should exist".format(funit))
if skip_function is None: # pragma: no cover
from .utils_tests_private import default_skip_function
skip_function = default_skip_function
if fLOG is None: # pragma: no cover
from ..loghelper.flog import noLOG
fLOG = noLOG
fix_tkinter_issues_virtualenv(fLOG=fLOG)
cov = True
if coverage_options:
if "disable_coverage" in coverage_options and coverage_options["disable_coverage"]:
cov = False
if dump_coverage is not None and not cov:
dump_coverage = None
logfile = os.path.join(funit, "unittests.out")
main_wrapper_tests(
logfile, add_coverage=cov, skip_function=skip_function, setup_params=setup_params,
only_setup_hook=only_setup_hook, coverage_options=coverage_options,
coverage_exclude_lines=coverage_exclude_lines, additional_ut_path=additional_ut_path,
covtoken=covtoken, hook_print=hook_print, stdout=stdout, stderr=stderr,
filter_warning=filter_warning, dump_coverage=dump_coverage,
add_coverage_folder=add_coverage_folder, coverage_root=coverage_root, fLOG=fLOG)
[docs]def copy27_for_setup(file_or_folder): # pragma: no cover
"""
Prepares a copy of the source for :epkg:`Python` 2.7,
assuming the module follows the same design as *pyquickhelper*.
:param file_or_folder: file ``setup.py`` or folder which contains it
:githublink:`%|py|836`
"""
# delayed import
from .py3to2 import py3to2_convert_tree
root = get_folder(file_or_folder)
root = os.path.normpath(root)
dest = os.path.join(root, "dist_module27")
py3to2_convert_tree(root, dest)
[docs]def write_pyproj(file_or_folder, location=None):
"""
Creates a project
`pyproj <https://docs.microsoft.com/fr-fr/visualstudio/python/managing-python-projects-in-visual-studio>`_
to work with `PTVS <https://pytools.codeplex.com/>`_
(Python Tools for Visual Studio)
:param file_or_folder: file ``setup.py`` or folder which contains it
:param location: if not None, stores the project into this folder
This functionality fails with :epkg:`Python` 2.7 (encoding).
:githublink:`%|py|856`
"""
# delayed import
from ..filehelper import explore_folder_iterfile
from .build_helper import get_pyproj_project
avoid = ["dist", "build", "dist_module27",
"_doc", "_virtualenv", "_virtualenv27", "_venv"]
def filter(name):
if os.path.splitext(name)[-1] != ".py":
return False
if "temp_" in name:
return False
if "temp2_" in name:
return False # pragma: no cover
for a in avoid:
if name.startswith(a + "\\"):
return False # pragma: no cover
if name.startswith(a + "/"):
return False
return True
root = get_folder(file_or_folder)
root = os.path.normpath(root)
name = os.path.split(root)[-1]
if location is None:
dest = os.path.join(root, "ptvs_project.pyproj") # pragma: no cover
else:
dest = os.path.join(location, "ptvs_project.pyproj")
all_files = [os.path.relpath(_, root)
for _ in explore_folder_iterfile(root)]
all_files = [_ for _ in all_files if filter(_)]
pyproj = get_pyproj_project(name, all_files)
with open(dest, "w", encoding="utf8") as f:
f.write(pyproj.strip())
[docs]def process_standard_options_for_setup_help(argv):
"""
Prints the added options available through this module.
:githublink:`%|py|896`
"""
commands = {
"build27": "build the wheel for Python 2.7 (if available), it requires to run copy27 first",
"build_script": "produce various scripts to build the module",
"build_sphinx": "build the documentation",
"build_wheel": "build the wheel",
"clean_space": "clean unnecessary spaces in the code, applies :epkg:`pycodestyle` on all files",
"clean_pyd": "clean file ``*.pyd``",
"copy27": "create a modified copy of the module to run on Python 2.7 (if available), it requires to run copy27 first",
"copy_dist": "copy documentation to folder dist",
"copy_sphinx": "modify and copy sources to _doc/sphinxdoc/source/<module>",
"history": "builds the history of the package in RST",
"local_jenkins": "sets up jobs on a local jenkins server",
"run27": "run the unit tests for the Python 2.7",
"run_pylint": "run pylint on the sources, allowed parameters <pattern> <neg_pattern>",
"setup_hook": "call function setup_hook which initializes the module before running unit tests",
"unittests": "run the unit tests which do not contain test_LONG, test_SKIP or test_GUI in their file name",
"unittests_LONG": "run the unit tests which contain test_LONG their file name",
"unittests_SKIP": "run the unit tests which contain test_SKIP their file name",
"unittests_GUI": "run the unit tests which contain test_GUI their file name",
"write_version": "write a file ``version.txt`` with the version number (assuming sources are host with git)",
}
if "--help-commands" in argv:
print("Commands processed by pyquickhelper:")
for k, v in sorted(commands.items()):
print(" {0}{1}{2}".format(
k, " " * (len("copy27 ") - len(k)), v))
print()
elif "--help" in argv:
docu = 0
for k, v in sorted(commands.items()):
if k in argv:
docu += 1
if docu == 0: # pragma: no cover
print("pyquickhelper commands:")
print()
for k in sorted(commands):
process_standard_options_for_setup_help(['--help', k])
print()
else:
for k, v in sorted(commands.items()):
if k in argv:
docu += 1
print(" setup.py {0}{1}{2}".format(
k, " " * (20 - len(k)), v))
if k == "unittests":
print(
"\n {0} [-d seconds] [-f file] [-e regex] [-g regex]\n\n {1}".format(k, v))
print(
" -d seconds run all unit tests for which predicted duration is below a given threshold.")
print(
" -f file run all unit tests in file (do not use the full path)")
print(
" -e regex run all unit tests files matching the regular expression")
print(
" -g regex run all unit tests files not matching the regular expression")
print()
elif k == "local_jenkins": # pragma: no cover
print()
print(
" {0} user password [location] [server]".format(k))
print(" default location is somewhere/workspace")
print(" default server is http://localhost:8080/")
print()
[docs]def write_module_scripts(folder, platform=sys.platform, blog_list=None,
default_engine_paths=None, command=None):
"""
Writes a couple of scripts which allow a user to be faster on some tasks
or to easily get information about the module.
:param folder: where to write the script
:param platform: platform
:param blog_list: blog list to follow, should be attribute ``__blog__`` of the module
:param command: None to generate scripts for all commands or a value in *[blog, doc]*.
:param default_engine_paths: default engines (or python distributions)
:return: list of written scripts
The function produces the following files:
* *auto_rss_list.xml*: list of rss stream to follow
* *auto_rss_database.db3*: stores blog posts
* *auto_rss_server.py*: runs a server which updates the scripts and runs a server. It also open the default browser.
* *auto_rss_server.(bat|sh)*: run *auto_run_server.py*, the file on Linux might be missing if there is an equivalent python script
.. faqref::
:title: How to generate auto_rss_server.py?
The following code generates the script *auto_rss_local.py*
which runs a local server to read blog posts included
in the documentation (it uses module
`pyrsslocal <http://www.xavierdupre.fr/app/pyrsslocal/helpsphinx/index.html>`_)::
from pyquickhelper.pycode import write_module_scripts, __blog__
write_module_scripts(".", blog_list=__blog__, command="blog")
:githublink:`%|py|994`
"""
# delayed import
from .build_helper import get_script_module
default_set = {"blog", "doc"}
if command is not None:
if command not in default_set:
raise ValueError(
"command {0} is not available in {1}".format(command, default_set))
commands = {command}
else:
commands = default_set
res = []
for c in commands:
sc = get_script_module(
c, platform=sys.platform, blog_list=blog_list, default_engine_paths=default_engine_paths)
if sc is None:
continue # pragma: no cover
tobin = os.path.join(folder, "bin")
if not os.path.exists(tobin):
os.mkdir(tobin)
for item in sc:
if isinstance(item, tuple):
name = os.path.join(folder, "bin", item[0])
with open(name, "w", encoding="utf8") as f:
f.write(item[1])
res.append(name)
else: # pragma: no cover
name = os.path.join(
folder, "bin", "auto_run_%s.%s" % (c, get_script_extension()))
with open(name, "w") as f:
f.write(item)
res.append(name)
return res
[docs]def _get_dump_default_path(location, module_name, argv):
"""
Proposes a default location to dump results about unit tests execution.
:param location: location of the module
:param module_name: module name
:param argv: argument on the command line
:return: location of the dump
The result is None for remote continuous integration.
:githublink:`%|py|1040`
"""
from . import is_travis_or_appveyor
if is_travis_or_appveyor():
return None # pragma: no cover
hash = hash_list(argv)
setup = os.path.join(location, "setup.py")
if not os.path.exists(setup):
raise FileNotFoundError(setup) # pragma: no cover
fold = os.path.join(location, "..", "_coverage_dumps")
if not os.path.exists(fold):
os.mkdir(fold)
dt = datetime.datetime.now().strftime("%Y%m%dT%H%M")
if module_name is None:
raise ValueError("module_name cannot be None") # pragma: no cover
dump = os.path.join(fold, module_name, hash, dt)
if not os.path.exists(dump):
os.makedirs(dump)
return dump
[docs]def hash_list(argv, size=8):
"""
Proposes a hash for the list of arguments.
:param argv: list of arguments on the command line.
:param size: size of the hash
:return: string
:githublink:`%|py|1067`
"""
st = "--".join(map(str, argv))
hash = hashlib.md5()
hash.update(st.encode("utf-8"))
res = hash.hexdigest()
if len(res) > 8:
return res[:8]
return res # pragma: no cover
[docs]def build_history_from_setup(dest, owner, module, existing_history=None,
skip_issues=None, fLOG=None): # pragma: no cover
"""
Builds the history from :epkg:`github` and :epkg:`pypi`.
:param dest: history will be written in this file
:param owner: owner of the package on :epkg:`github`
:param module: module name
:param existing_history: existing history, retrieves existing issues stored
in that file
:param skip_issues: skip a given list of issues when building the history
:param fLOG: logging function
:return: history
:githublink:`%|py|1090`
"""
# delayed import
from ..loghelper.history_helper import build_history, compile_history
if owner is None:
raise ValueError("owner must be specified")
if fLOG is None: # pragma: no cover
from ..loghelper.flog import noLOG
fLOG = noLOG
repo = module
hist = build_history(owner, repo, unpublished=True,
existing_history=existing_history,
skip_issues=skip_issues, fLOG=fLOG)
output = compile_history(hist)
if dest is not None:
with open(dest, "w", encoding="utf-8") as f:
f.write(output)
return output
[docs]def run_pylint_for_setup(folder, pattern=".*[.]py$", neg_pattern=None,
verbose=False, pylint_ignore=None, fLOG=None):
"""
Applies :epkg:`pylint` on subfolder *folder*.
:param folder: folder where to look
:param pattern: file to checks
:param neg_pattern: negative pattern
@parm pylint_ignore ignore these :epkg:`pylint` warnings or errors
:param verbose: verbose
:param fLOG: logging function
:githublink:`%|py|1120`
"""
# delayed import
from .utils_tests_helper import check_pep8
if fLOG is None: # pragma: no cover
from ..loghelper.flog import noLOG
fLOG = noLOG
check_pep8(folder, pattern=pattern, neg_pattern=neg_pattern,
pylint_ignore=pylint_ignore, verbose=verbose, fLOG=fLOG)
[docs]def modifies_init_file(folder, version, module_name=None):
"""
Automatically modifies the init file.
:param folder: where to find the init file
:param version: commit number
:return: modified init file
:githublink:`%|py|1137`
"""
def _update_version(v, nv):
vs = v.split('.')
if len(vs) <= 2:
return '.'.join(list(vs) + [nv])
if len(vs) >= 3:
vs = list(vs)
vs[-1] = nv
return '.'.join(vs)
raise ValueError( # pragma: no cover
"Unable to process '{}' with new version '{}'.".format(v, nv))
filename = None
if os.path.exists(folder):
if os.path.isdir(folder):
src = os.path.join(folder, 'src')
if module_name is None:
setu = os.path.join(folder, 'setup.py')
if not os.path.exists(setu):
raise FileNotFoundError( # pragma: no cover
"Unable to find 'setup.py' in '{}' and module_name is "
"None.".format(folder))
reg = re.compile(
"(project_var_name = ['\\\"]([a-zA-Z][a-zA-Z_0-9]+)['\\\"])")
with open(setu, 'r', encoding='utf-8') as f:
cst = f.read()
find = reg.findall(cst)
if len(find) == 0:
raise FileNotFoundError( # pragma: no cover
"Unable to find 'project_var_name' in 'setup.py' in '{}' "
"and module_name is None.".format(folder))
module_name = find[0][1]
if os.path.exists(src) and module_name is not None:
filename = os.path.join(src, module_name, '__init__.py')
elif os.path.exists(src) and module_name is not None:
filename = os.path.join(src, module_name, '__init__.py')
elif module_name is not None:
filename = os.path.join(folder, module_name, '__init__.py')
else:
raise FileNotFoundError( # pragma: no cover
"Unable to find '__init__.py' in '{}' (module_name is None).".format(folder))
if not os.path.exists(filename):
raise FileNotFoundError( # pragma: no cover
"Unable to find '__init__.py' in '{}' (got '{}').".format(folder, filename))
with open(filename, 'r', encoding='utf-8') as f:
content = f.read()
elif '__version__' in folder:
content = folder
else:
raise ValueError("Unable to process '{}'.".format(folder))
reg = re.compile("(__version__ = ['\\\"]([0-9.]+)['\\\"])")
lines = content.split('\n')
modif = []
rep = []
for line in lines:
if line.startswith("__version__"):
find = reg.findall(line)
if len(find) != 1:
raise ValueError( # pragma: no cover
"Unable to find __version__ in '{}'".format(line))
v = find[0][1]
nv = _update_version(v, str(version))
newline = line.replace(v, nv)
modif.append(newline)
rep.append((line, newline))
else:
modif.append(line)
if len(rep) == 0:
raise ValueError( # pragma: no cover
"Unable to find '__version__' in \n{}".format(content))
content = '\n'.join(modif)
if filename is not None:
with open(filename, 'w', encoding='utf-8') as f:
f.write(content)
return content