Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2@file
3@brief Helpers for virtualenv
4"""
5import os
6import sys
9class VirtualEnvError(Exception):
10 """
11 Exception raised by the function implemented in this file.
12 """
13 pass
16def is_virtual_environment():
17 """
18 Tells if the script is run from a virtual environment.
20 @return boolean
21 """
22 return (getattr(sys, "base_exec_prefix", sys.exec_prefix) != sys.exec_prefix) or hasattr(sys, 'real_prefix')
25class NotImplementedErrorFromVirtualEnvironment(NotImplementedError):
26 """
27 Defines an exception when a function does not work
28 in a virtual environment.
29 """
30 pass
33def build_venv_cmd(params, posparams): # pragma: no cover
34 """
35 Builds the command line for virtual env.
37 @param params dictionary of parameters
38 @param posparams positional arguments
39 @return string
40 """
41 import venv
42 v = venv.__file__
43 if v is None:
44 raise ImportError("module venv should have a version number")
45 exe = sys.executable.replace("w.exe", "").replace(".exe", "")
46 cmd = [exe, "-m", "venv"]
47 for k, v in params.items():
48 if v is None:
49 cmd.append("--" + k)
50 else:
51 cmd.append("--" + k + "=" + v)
52 cmd.extend(posparams)
53 return " ".join(cmd)
56def create_virtual_env(where, symlinks=False,
57 system_site_packages=False,
58 clear=True, packages=None, fLOG=None,
59 temp_folder=None, platform=None): # pragma: no cover
60 """
61 Creates a virtual environment.
63 @param where location of this virtual environment
64 @param symlinks attempt to symlink rather than copy
65 @param system_site_packages Give the virtual environment access to the system site-packages dir
66 @param clear Delete the environment directory if it already exists.
67 If not specified and the directory exists, an error is raised.
68 @param packages list of packages to install (it will install module
69 :epkg:`pymyinstall`).
70 @param fLOG logging function
71 @param temp_folder temporary folder (to download module if needed), by default ``<where>/download``
72 @param platform platform to use
73 @return stand output
75 .. index:: virtual environment
77 .. faqref::
78 :title: How to create a virtual environment?
80 The following example creates a virtual environment.
81 Packages can be added by specifying the parameter *package*.
83 ::
85 from pyquickhelper.pycode import create_virtual_env
86 fold = "my_env"
87 if not os.path.exists(fold):
88 os.mkdir(fold)
89 create_virtual_env(fold)
91 The function does not work from a virtual environment.
92 """
93 from ..loghelper import run_cmd
94 if fLOG is None:
95 from ..loghelper import noLOG
96 fLOG = noLOG
97 if is_virtual_environment():
98 raise NotImplementedErrorFromVirtualEnvironment()
100 fLOG("create virtual environment at:", where)
101 params = {}
102 if symlinks:
103 params["symlinks"] = None
104 if system_site_packages:
105 params["system-site-packages"] = None
106 if clear:
107 params["clear"] = None
108 cmd = build_venv_cmd(params, [where])
109 out, err = run_cmd(cmd, wait=True, fLOG=fLOG)
110 if len(err) > 0:
111 raise VirtualEnvError(
112 "unable to create virtual environement at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, where, cmd))
114 if platform is None:
115 platform = sys.platform
116 if platform.startswith("win"):
117 scripts = os.path.join(where, "Scripts")
118 else:
119 scripts = os.path.join(where, "bin")
121 if not os.path.exists(scripts):
122 files = "\n ".join(os.listdir(where))
123 raise FileNotFoundError(
124 "unable to find {0}, content:\n {1}".format(scripts, files))
126 in_scripts = os.listdir(scripts)
127 pips = [_ for _ in in_scripts if _.startswith("pip")]
128 if len(pips) == 0:
129 out += venv_install(where, "pip", fLOG=fLOG,
130 temp_folder=temp_folder,
131 platform=platform)
132 in_scripts = os.listdir(scripts)
133 pips = [_ for _ in in_scripts if _.startswith("pip")]
134 if len(pips) == 0:
135 raise FileNotFoundError(
136 "unable to find pip in {0}, content:\n {1}".format(scripts, in_scripts))
138 out += venv_install(where, "pymyinstall", fLOG=fLOG,
139 temp_folder=temp_folder, platform=platform)
141 if packages is not None and len(packages) > 0:
142 fLOG("install packages in:", where)
143 packages = [_ for _ in packages if _ not in ("pymyinstall", "pip")]
144 if len(packages) > 0:
145 out += venv_install(where, packages, fLOG=fLOG,
146 temp_folder=temp_folder,
147 platform=platform)
148 return out
151def venv_install(venv, packages, fLOG=None,
152 temp_folder=None, platform=None): # pragma: no cover
153 """
154 Installs a package or a list of packages in a virtual environment.
156 @param venv location of the virtual environment
157 @param packages a package (str) or a list of packages(list[str])
158 @param fLOG logging function
159 @param temp_folder temporary folder (to download module if needed), by default ``<where>/download``
160 @param platform platform (``sys.platform`` by default)
161 @return standard output
163 The function does not work from a virtual environment.
164 """
165 from ..loghelper import run_cmd
166 if fLOG is None:
167 from ..loghelper import noLOG
168 fLOG = noLOG
169 if is_virtual_environment():
170 raise NotImplementedErrorFromVirtualEnvironment()
171 if temp_folder is None:
172 temp_folder = os.path.join(venv, "download")
173 if isinstance(packages, str):
174 packages = [packages]
175 if platform is None:
176 platform = sys.platform
178 if packages == "pip" or packages == ["pip"]: # pylint: disable=R1714
179 from .get_pip import __file__ as pip_loc # pylint: disable=E0401
180 ppath = os.path.abspath(pip_loc.replace(".pyc", ".py"))
181 script = ["-u", ppath]
182 return run_venv_script(venv, script, fLOG=fLOG, is_cmd=True, platform=platform)
183 elif packages == "pymyinstall" or packages == ["pymyinstall"]: # pylint: disable=R1714
184 if platform.startswith("win"):
185 pip = os.path.join(venv, "Scripts", "pip")
186 else:
187 pip = os.path.join(venv, "bin", "pip")
188 local_setup = os.path.abspath(os.path.join(os.path.dirname(
189 __file__), "..", "..", "..", "..", "pymyinstall", "setup.py"))
190 if os.path.exists(local_setup):
191 cwd = os.getcwd()
192 os.chdir(os.path.dirname(local_setup))
193 script = ["-u", local_setup, "install"]
194 out = run_venv_script(venv, script, fLOG=fLOG, is_cmd=True,
195 skip_err_if="Finished processing dependencies for pymyinstall==",
196 platform=platform)
197 os.chdir(cwd)
198 return out
199 else:
200 cmd = pip + " install pymyinstall"
201 out, err = run_cmd(cmd, wait=True, fLOG=fLOG)
202 if len(err) > 0:
203 raise VirtualEnvError(
204 "unable to install pymyinstall at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, venv, cmd))
205 return out
206 else:
207 p = os.path.normpath(os.path.join(
208 os.path.abspath(os.path.dirname(__file__)), "..", ".."))
209 ls = ','.join("'{0}'".format(_) for _ in packages)
210 script = ["import sys",
211 "sys.path.append('{0}')".format(p.replace("\\", "\\\\")),
212 "import pymyinstall",
213 "ps=[{0}]".format(ls),
214 "t='{0}'".format(temp_folder.replace("\\", "\\\\")),
215 "pymyinstall.packaged.install_all(temp_folder=t,list_module=ps,up_pip=False)"]
216 return run_venv_script(venv, "\n".join(script), fLOG=fLOG, platform=platform)
219def run_venv_script(venv, script, fLOG=None,
220 file=False, is_cmd=False,
221 skip_err_if=None, platform=None,
222 **kwargs): # pragma: no cover
223 """
224 Runs a script on a vritual environment (the script should be simple).
226 @param venv virtual environment
227 @param script script as a string (not a file)
228 @param fLOG logging function
229 @param file is script a file or a string to execute
230 @param is_cmd if True, script is a command line to run (as a list) for python executable
231 @param skip_err_if do not pay attention to standard error if this string was found in standard output
232 @param platform platform (``sys.platform`` by default)
233 @param kwargs others arguments for function @see fn run_cmd.
234 @return output
236 The function does not work from a virtual environment.
237 """
238 from ..loghelper import run_cmd
239 if fLOG is None:
240 from ..loghelper import noLOG
241 fLOG = noLOG
243 def filter_err(err):
244 lis = err.split("\n")
245 lines = []
246 for li in lis:
247 if "missing dependencies" in li:
248 continue
249 if "' misses '" in li:
250 continue
251 lines.append(li)
252 return "\n".join(lines).strip(" \r\n\t")
254 if is_virtual_environment():
255 raise NotImplementedErrorFromVirtualEnvironment()
257 if platform is None:
258 platform = sys.platform
260 if platform.startswith("win"):
261 exe = os.path.join(venv, "Scripts", "python")
262 else:
263 exe = os.path.join(venv, "bin", "python")
264 if is_cmd:
265 cmd = " ".join([exe] + script)
266 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs)
267 err = filter_err(err)
268 if len(err) > 0 and (skip_err_if is None or skip_err_if not in out):
269 raise VirtualEnvError(
270 "unable to run cmd at {2}\n--CMD--\n{3}\n--OUT--\n{0}\n[pyqerror]"
271 "\n{1}".format(out, err, venv, cmd))
272 return out
273 else:
274 script = ";".join(script.split("\n"))
275 if file:
276 if not os.path.exists(script):
277 raise FileNotFoundError(script)
278 cmd = " ".join([exe, "-u", '"{0}"'.format(script)])
279 else:
280 cmd = " ".join([exe, "-u", "-c", '"{0}"'.format(script)])
281 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs)
282 err = filter_err(err)
283 if len(err) > 0:
284 raise VirtualEnvError(
285 "Unable to run script at {2}\n--CMD--\n{3}\n--OUT--\n{0}\n"
286 "[pyqerror]\n{1}".format(out, err, venv, cmd))
287 return out
290def run_base_script(script, fLOG=None, file=False, is_cmd=False,
291 skip_err_if=None, argv=None, platform=None, **kwargs):
292 """
293 Runs a script with the original intepreter even if this function
294 is run from a virtual environment.
296 @param script script as a string (not a file)
297 @param fLOG logging function
298 @param file is script a file or a string to execute
299 @param is_cmd if True, script is a command line to run (as a list) for python executable
300 @param skip_err_if do not pay attention to standard error if this string was found in standard output
301 @param argv list of arguments to add on the command line
302 @param platform platform (``sys.platform`` by default)
303 @param kwargs others arguments for function @see fn run_cmd.
304 @return output
306 The function does not work from a virtual environment.
307 The function does not raise an exception if the standard error
308 contains something like::
310 ----------------------------------------------------------------------
311 Ran 1 test in 0.281s
313 OK
314 """
315 from ..loghelper import run_cmd
316 if fLOG is None: # pragma: no cover
317 from ..loghelper import noLOG
318 fLOG = noLOG
320 def true_err(err): # pragma: no cover
321 if "Ran 1 test" in err and "OK" in err:
322 return False
323 return True
325 if platform is None:
326 platform = sys.platform
328 if hasattr(sys, 'real_prefix'): # pragma: no cover
329 exe = sys.base_prefix
330 elif hasattr(sys, "base_exec_prefix"): # pragma: no cover
331 exe = sys.base_exec_prefix
332 else:
333 exe = sys.exec_prefix # pragma: no cover
335 if platform.startswith("win"):
336 exe = os.path.join(exe, "python") # pragma: no cover
337 else:
338 exe = os.path.join(exe, "bin", "python%d.%d" % sys.version_info[:2])
339 if not os.path.exists(exe):
340 exe = os.path.join(exe, "bin", "python") # pragma: no cover
342 if is_cmd: # pragma: no cover
343 cmd = " ".join([exe] + script)
344 if argv is not None:
345 cmd += " " + " ".join(argv)
346 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs)
347 if len(err) > 0 and (skip_err_if is None or skip_err_if not in out) and true_err(err):
348 p = sys.base_prefix if hasattr(sys, "base_prefix") else sys.prefix
349 raise VirtualEnvError(
350 "unable to run cmd at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, p, cmd))
351 return out
352 else:
353 script = ";".join(script.split("\n"))
354 if file:
355 if not os.path.exists(script):
356 raise FileNotFoundError(script) # pragma: no cover
357 cmd = " ".join([exe, "-u", '"{0}"'.format(script)])
358 else:
359 cmd = " ".join(
360 [exe, "-u", "-c", '"{0}"'.format(script)]) # pragma: no cover
361 if argv is not None:
362 cmd += " " + " ".join(argv) # pragma: no cover
363 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs)
364 if len(err) > 0 and true_err(err):
365 p = (sys.base_prefix # pragma: no cover
366 if hasattr(sys, "base_prefix")
367 else sys.prefix)
368 raise VirtualEnvError( # pragma: no cover
369 "unable to run script with {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, p, cmd))
370 return out
373def check_readme_syntax(readme, folder,
374 version="0.8", fLOG=None): # pragma: no cover
375 """
376 Checks the syntax of the file ``readme.rst``
377 which describes a python project.
379 @param readme file to check
380 @param folder location for the virtual environment
381 @param version version of docutils
382 @param fLOG logging function
383 @return output or SyntaxError exception
385 `pipy server <https://pypi.python.org/pypi/>`_ is based on
386 `docutils <https://pypi.python.org/pypi/docutils/>`_ ==0.8.
387 The most simple way to check its syntax is to create a virtual environment,
388 to install docutils==0.8 and to compile the file.
389 This is what this function does.
391 Unfortunately, this functionality does not work yet
392 from a virtual environment.
393 """
394 if fLOG is None:
395 from ..loghelper import noLOG
396 fLOG = noLOG
397 if is_virtual_environment():
398 raise NotImplementedErrorFromVirtualEnvironment()
399 if not os.path.exists(folder):
400 os.makedirs(folder)
402 out = create_virtual_env(folder, fLOG=fLOG, packages=[
403 "docutils==" + version,
404 "pipdeptree"])
405 outfile = os.path.join(folder, "conv_readme.html")
407 script = ["from docutils import core",
408 "import io",
409 'from docutils.readers.standalone import Reader',
410 'from docutils.parsers.rst import Parser',
411 'from docutils.parsers.rst.directives.images import Image',
412 'from docutils.parsers.rst.directives import _directives',
413 'from docutils.writers.html4css1 import Writer',
414 "from docutils.languages import _languages",
415 "from docutils.languages import en, fr",
416 "_languages['en'] = en",
417 "_languages['fr'] = fr",
418 "_directives['image'] = Image",
419 "with open('{0}', 'r', encoding='utf8') as g: s = g.read()".format(
420 readme.replace("\\", "\\\\")),
421 "settings_overrides = {'output_encoding': 'unicode', 'doctitle_xform': True,",
422 " 'initial_header_level': 2, 'warning_stream': io.StringIO()}",
423 "parts = core.publish_parts(source=s, parser=Parser(), reader=Reader(), source_path=None,",
424 " destination_path=None, writer=Writer(),",
425 " settings_overrides=settings_overrides)",
426 "with open('{0}', 'w', encoding='utf8') as f: f.write(parts['whole'])".format(
427 outfile.replace("\\", "\\\\")),
428 ]
430 file_script = os.path.join(folder, "test_" + os.path.split(readme)[-1])
431 with open(file_script, "w") as f:
432 f.write("\n".join(script))
434 out = run_venv_script(folder, file_script, fLOG=fLOG, file=True)
435 with open(outfile, "r", encoding="utf8") as h:
436 content = h.read()
438 if "System Message" in content:
439 raise SyntaxError(
440 "unable to parse a file with docutils==" + version + "\nCONTENT:\n" + content)
442 return out