Source code for pyquickhelper.pycode.venv_helper

"""
Helpers for virtualenv


:githublink:`%|py|5`
"""
import os
import sys
from ..loghelper import noLOG, run_cmd


[docs]class VirtualEnvError(Exception): """ Exception raised by the function implemented in this file. :githublink:`%|py|13` """ pass
[docs]def is_virtual_environment(): """ Tells if the script is run from a virtual environment. :return: boolean :githublink:`%|py|22` """ return (getattr(sys, "base_exec_prefix", sys.exec_prefix) != sys.exec_prefix) or hasattr(sys, 'real_prefix')
[docs]class NotImplementedErrorFromVirtualEnvironment(NotImplementedError): """ Defines an exception when a function does not work in a virtual environment. :githublink:`%|py|30` """ pass
[docs]def build_venv_cmd(params, posparams): """ Builds the command line for virtual env. :param params: dictionary of parameters :param posparams: positional arguments :return: string :githublink:`%|py|41` """ import venv v = venv.__file__ if v is None: raise ImportError("module venv should have a version number") exe = sys.executable.replace("w.exe", "").replace(".exe", "") cmd = [exe, "-m", "venv"] for k, v in params.items(): if v is None: cmd.append("--" + k) else: cmd.append("--" + k + "=" + v) cmd.extend(posparams) return " ".join(cmd)
[docs]def create_virtual_env(where, symlinks=False, system_site_packages=False, clear=True, packages=None, fLOG=noLOG, temp_folder=None, platform=None): """ Creates a virtual environment. :param where: location of this virtual environment :param symlinks: attempt to symlink rather than copy :param system_site_packages: Give the virtual environment access to the system site-packages dir :param clear: Delete the environment directory if it already exists. If not specified and the directory exists, an error is raised. :param packages: list of packages to install (it will install module :epkg:`pymyinstall`). :param fLOG: logging function :param temp_folder: temporary folder (to download module if needed), by default ``<where>/download`` :param platform: platform to use :return: stand output .. index:: virtual environment .. faqref:: :title: How to create a virtual environment? The following example creates a virtual environment. Packages can be added by specifying the parameter *package*. :: from pyquickhelper.pycode import create_virtual_env fold = "my_env" if not os.path.exists(fold): os.mkdir(fold) create_virtual_env(fold) The function does not work from a virtual environment. :githublink:`%|py|92` """ if is_virtual_environment(): raise NotImplementedErrorFromVirtualEnvironment() fLOG("create virtual environment at:", where) params = {} if symlinks: params["symlinks"] = None if system_site_packages: params["system-site-packages"] = None if clear: params["clear"] = None cmd = build_venv_cmd(params, [where]) out, err = run_cmd(cmd, wait=True, fLOG=fLOG) if len(err) > 0: raise VirtualEnvError( "unable to create virtual environement at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, where, cmd)) if platform is None: platform = sys.platform if platform.startswith("win"): scripts = os.path.join(where, "Scripts") else: scripts = os.path.join(where, "bin") if not os.path.exists(scripts): files = "\n ".join(os.listdir(where)) raise FileNotFoundError( "unable to find {0}, content:\n {1}".format(scripts, files)) in_scripts = os.listdir(scripts) pips = [_ for _ in in_scripts if _.startswith("pip")] if len(pips) == 0: out += venv_install(where, "pip", fLOG=fLOG, temp_folder=temp_folder, platform=platform) in_scripts = os.listdir(scripts) pips = [_ for _ in in_scripts if _.startswith("pip")] if len(pips) == 0: raise FileNotFoundError( "unable to find pip in {0}, content:\n {1}".format(scripts, in_scripts)) out += venv_install(where, "pymyinstall", fLOG=fLOG, temp_folder=temp_folder, platform=platform) if packages is not None and len(packages) > 0: fLOG("install packages in:", where) packages = [_ for _ in packages if _ not in ("pymyinstall", "pip")] if len(packages) > 0: out += venv_install(where, packages, fLOG=fLOG, temp_folder=temp_folder, platform=platform) return out
[docs]def venv_install(venv, packages, fLOG=noLOG, temp_folder=None, platform=None): """ Installs a package or a list of packages in a virtual environment. :param venv: location of the virtual environment :param packages: a package (str) or a list of packages(list[str]) :param fLOG: logging function :param temp_folder: temporary folder (to download module if needed), by default ``<where>/download`` :param platform: platform (``sys.platform`` by default) :return: standard output The function does not work from a virtual environment. :githublink:`%|py|159` """ if is_virtual_environment(): raise NotImplementedErrorFromVirtualEnvironment() if temp_folder is None: temp_folder = os.path.join(venv, "download") if isinstance(packages, str): packages = [packages] if platform is None: platform = sys.platform if packages == "pip" or packages == ["pip"]: # pylint: disable=R1714 from .get_pip import __file__ as pip_loc # pylint: disable=E0401 ppath = os.path.abspath(pip_loc.replace(".pyc", ".py")) script = ["-u", ppath] return run_venv_script(venv, script, fLOG=fLOG, is_cmd=True, platform=platform) elif packages == "pymyinstall" or packages == ["pymyinstall"]: # pylint: disable=R1714 if platform.startswith("win"): pip = os.path.join(venv, "Scripts", "pip") else: pip = os.path.join(venv, "bin", "pip") local_setup = os.path.abspath(os.path.join(os.path.dirname( __file__), "..", "..", "..", "..", "pymyinstall", "setup.py")) if os.path.exists(local_setup): cwd = os.getcwd() os.chdir(os.path.dirname(local_setup)) script = ["-u", local_setup, "install"] out = run_venv_script(venv, script, fLOG=fLOG, is_cmd=True, skip_err_if="Finished processing dependencies for pymyinstall==", platform=platform) os.chdir(cwd) return out else: cmd = pip + " install pymyinstall" out, err = run_cmd(cmd, wait=True, fLOG=fLOG) if len(err) > 0: raise VirtualEnvError( "unable to install pymyinstall at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, venv, cmd)) return out else: p = os.path.normpath(os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "..")) ls = ','.join("'{0}'".format(_) for _ in packages) script = ["import sys", "sys.path.append('{0}')".format(p.replace("\\", "\\\\")), "import pymyinstall", "ps=[{0}]".format(ls), "t='{0}'".format(temp_folder.replace("\\", "\\\\")), "pymyinstall.packaged.install_all(temp_folder=t,list_module=ps,up_pip=False)"] return run_venv_script(venv, "\n".join(script), fLOG=fLOG, platform=platform)
[docs]def run_venv_script(venv, script, fLOG=noLOG, file=False, is_cmd=False, skip_err_if=None, platform=None, **kwargs): """ Runs a script on a vritual environment (the script should be simple). :param venv: virtual environment :param script: script as a string (not a file) :param fLOG: logging function :param file: is script a file or a string to execute :param is_cmd: if True, script is a command line to run (as a list) for python executable :param skip_err_if: do not pay attention to standard error if this string was found in standard output :param platform: platform (``sys.platform`` by default) :param kwargs: others arguments for function :func:`run_cmd <pyquickhelper.loghelper.run_cmd.run_cmd>`. :return: output The function does not work from a virtual environment. :githublink:`%|py|229` """ if is_virtual_environment(): raise NotImplementedErrorFromVirtualEnvironment() if platform is None: platform = sys.platform if platform.startswith("win"): exe = os.path.join(venv, "Scripts", "python") else: exe = os.path.join(venv, "bin", "python") if is_cmd: cmd = " ".join([exe] + script) out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs) if len(err) > 0 and (skip_err_if is None or skip_err_if not in out): raise VirtualEnvError( "unable to run cmd at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, venv, cmd)) return out else: script = ";".join(script.split("\n")) if file: if not os.path.exists(script): raise FileNotFoundError(script) cmd = " ".join([exe, "-u", '"{0}"'.format(script)]) else: cmd = " ".join([exe, "-u", "-c", '"{0}"'.format(script)]) out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs) if len(err) > 0: raise VirtualEnvError( "unable to run script at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, venv, cmd)) return out
[docs]def run_base_script(script, fLOG=noLOG, file=False, is_cmd=False, skip_err_if=None, argv=None, platform=None, **kwargs): """ Runs a script with the original intepreter even if this function is run from a virtual environment. :param script: script as a string (not a file) :param fLOG: logging function :param file: is script a file or a string to execute :param is_cmd: if True, script is a command line to run (as a list) for python executable :param skip_err_if: do not pay attention to standard error if this string was found in standard output :param argv: list of arguments to add on the command line :param platform: platform (``sys.platform`` by default) :param kwargs: others arguments for function :func:`run_cmd <pyquickhelper.loghelper.run_cmd.run_cmd>`. :return: output The function does not work from a virtual environment. The function does not raise an exception if the standard error contains something like:: ---------------------------------------------------------------------- Ran 1 test in 0.281s OK :githublink:`%|py|286` """ def true_err(err): if "Ran 1 test" in err and "OK" in err: return False else: return True if platform is None: platform = sys.platform if hasattr(sys, 'real_prefix'): exe = sys.base_prefix elif hasattr(sys, "base_exec_prefix"): exe = sys.base_exec_prefix else: exe = sys.exec_prefix if platform.startswith("win"): exe = os.path.join(exe, "python") else: exe = os.path.join(exe, "bin", "python%d.%d" % sys.version_info[:2]) if not os.path.exists(exe): exe = os.path.join(exe, "bin", "python") if is_cmd: cmd = " ".join([exe] + script) if argv is not None: cmd += " " + " ".join(argv) out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs) if len(err) > 0 and (skip_err_if is None or skip_err_if not in out) and true_err(err): p = sys.base_prefix if hasattr(sys, "base_prefix") else sys.prefix raise VirtualEnvError( "unable to run cmd at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, p, cmd)) return out else: script = ";".join(script.split("\n")) if file: if not os.path.exists(script): raise FileNotFoundError(script) cmd = " ".join([exe, "-u", '"{0}"'.format(script)]) else: cmd = " ".join([exe, "-u", "-c", '"{0}"'.format(script)]) if argv is not None: cmd += " " + " ".join(argv) out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs) if len(err) > 0 and true_err(err): p = sys.base_prefix if hasattr(sys, "base_prefix") else sys.prefix raise VirtualEnvError( "unable to run script with {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, p, cmd)) return out
[docs]def check_readme_syntax(readme, folder, version="0.8", fLOG=noLOG): """ Checks the syntax of the file ``readme.rst`` which describes a python project. :param readme: file to check :param folder: location for the virtual environment :param version: version of docutils :param fLOG: logging function :return: output or SyntaxError exception `pipy server <https://pypi.python.org/pypi/>`_ is based on `docutils <https://pypi.python.org/pypi/docutils/>`_ ==0.8. The most simple way to check its syntax is to create a virtual environment, to install docutils==0.8 and to compile the file. This is what this function does. Unfortunately, this functionality does not work yet from a virtual environment. :githublink:`%|py|357` """ if is_virtual_environment(): raise NotImplementedErrorFromVirtualEnvironment() if not os.path.exists(folder): os.makedirs(folder) out = create_virtual_env(folder, fLOG=fLOG, packages=[ "docutils==" + version, "pipdeptree"]) outfile = os.path.join(folder, "conv_readme.html") script = ["from docutils import core", "import io", 'from docutils.readers.standalone import Reader', 'from docutils.parsers.rst import Parser', 'from docutils.parsers.rst.directives.images import Image', 'from docutils.parsers.rst.directives import _directives', 'from docutils.writers.html4css1 import Writer', "from docutils.languages import _languages", "from docutils.languages import en, fr", "_languages['en'] = en", "_languages['fr'] = fr", "_directives['image'] = Image", "with open('{0}', 'r', encoding='utf8') as g: s = g.read()".format( readme.replace("\\", "\\\\")), "settings_overrides = {'output_encoding': 'unicode', 'doctitle_xform': True,", " initial_header_level': 2, 'warning_stream': io.StringIO()}", "parts = core.publish_parts(source=s, parser=Parser(), reader=Reader(), source_path=None,", " destination_path=None, writer=Writer(),", " settings_overrides=settings_overrides)", "with open('{0}', 'w', encoding='utf8') as f: f.write(parts['whole'])".format( outfile.replace("\\", "\\\\")), ] file_script = os.path.join(folder, "test_" + os.path.split(readme)[-1]) with open(file_script, "w") as f: f.write("\n".join(script)) out = run_venv_script(folder, file_script, fLOG=fLOG, file=True) with open(outfile, "r", encoding="utf8") as h: content = h.read() if "System Message" in content: raise SyntaxError( "unable to parse a file with docutils==" + version + "\nCONTENT:\n" + content) return out