Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1""" 

2@file 

3@brief Helpers for virtualenv 

4""" 

5import os 

6import sys 

7 

8 

9class VirtualEnvError(Exception): 

10 """ 

11 Exception raised by the function implemented in this file. 

12 """ 

13 pass 

14 

15 

16def is_virtual_environment(): 

17 """ 

18 Tells if the script is run from a virtual environment. 

19 

20 @return boolean 

21 """ 

22 return (getattr(sys, "base_exec_prefix", sys.exec_prefix) != sys.exec_prefix) or hasattr(sys, 'real_prefix') 

23 

24 

25class NotImplementedErrorFromVirtualEnvironment(NotImplementedError): 

26 """ 

27 Defines an exception when a function does not work 

28 in a virtual environment. 

29 """ 

30 pass 

31 

32 

33def build_venv_cmd(params, posparams): # pragma: no cover 

34 """ 

35 Builds the command line for virtual env. 

36 

37 @param params dictionary of parameters 

38 @param posparams positional arguments 

39 @return string 

40 """ 

41 import venv 

42 v = venv.__file__ 

43 if v is None: 

44 raise ImportError("module venv should have a version number") 

45 exe = sys.executable.replace("w.exe", "").replace(".exe", "") 

46 cmd = [exe, "-m", "venv"] 

47 for k, v in params.items(): 

48 if v is None: 

49 cmd.append("--" + k) 

50 else: 

51 cmd.append("--" + k + "=" + v) 

52 cmd.extend(posparams) 

53 return " ".join(cmd) 

54 

55 

56def create_virtual_env(where, symlinks=False, 

57 system_site_packages=False, 

58 clear=True, packages=None, fLOG=None, 

59 temp_folder=None, platform=None): # pragma: no cover 

60 """ 

61 Creates a virtual environment. 

62 

63 @param where location of this virtual environment 

64 @param symlinks attempt to symlink rather than copy 

65 @param system_site_packages Give the virtual environment access to the system site-packages dir 

66 @param clear Delete the environment directory if it already exists. 

67 If not specified and the directory exists, an error is raised. 

68 @param packages list of packages to install (it will install module 

69 :epkg:`pymyinstall`). 

70 @param fLOG logging function 

71 @param temp_folder temporary folder (to download module if needed), by default ``<where>/download`` 

72 @param platform platform to use 

73 @return stand output 

74 

75 .. index:: virtual environment 

76 

77 .. faqref:: 

78 :title: How to create a virtual environment? 

79 

80 The following example creates a virtual environment. 

81 Packages can be added by specifying the parameter *package*. 

82 

83 :: 

84 

85 from pyquickhelper.pycode import create_virtual_env 

86 fold = "my_env" 

87 if not os.path.exists(fold): 

88 os.mkdir(fold) 

89 create_virtual_env(fold) 

90 

91 The function does not work from a virtual environment. 

92 """ 

93 from ..loghelper import run_cmd 

94 if fLOG is None: 

95 from ..loghelper import noLOG 

96 fLOG = noLOG 

97 if is_virtual_environment(): 

98 raise NotImplementedErrorFromVirtualEnvironment() 

99 

100 fLOG("create virtual environment at:", where) 

101 params = {} 

102 if symlinks: 

103 params["symlinks"] = None 

104 if system_site_packages: 

105 params["system-site-packages"] = None 

106 if clear: 

107 params["clear"] = None 

108 cmd = build_venv_cmd(params, [where]) 

109 out, err = run_cmd(cmd, wait=True, fLOG=fLOG) 

110 if len(err) > 0: 

111 raise VirtualEnvError( 

112 "unable to create virtual environement at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, where, cmd)) 

113 

114 if platform is None: 

115 platform = sys.platform 

116 if platform.startswith("win"): 

117 scripts = os.path.join(where, "Scripts") 

118 else: 

119 scripts = os.path.join(where, "bin") 

120 

121 if not os.path.exists(scripts): 

122 files = "\n ".join(os.listdir(where)) 

123 raise FileNotFoundError( 

124 "unable to find {0}, content:\n {1}".format(scripts, files)) 

125 

126 in_scripts = os.listdir(scripts) 

127 pips = [_ for _ in in_scripts if _.startswith("pip")] 

128 if len(pips) == 0: 

129 out += venv_install(where, "pip", fLOG=fLOG, 

130 temp_folder=temp_folder, 

131 platform=platform) 

132 in_scripts = os.listdir(scripts) 

133 pips = [_ for _ in in_scripts if _.startswith("pip")] 

134 if len(pips) == 0: 

135 raise FileNotFoundError( 

136 "unable to find pip in {0}, content:\n {1}".format(scripts, in_scripts)) 

137 

138 out += venv_install(where, "pymyinstall", fLOG=fLOG, 

139 temp_folder=temp_folder, platform=platform) 

140 

141 if packages is not None and len(packages) > 0: 

142 fLOG("install packages in:", where) 

143 packages = [_ for _ in packages if _ not in ("pymyinstall", "pip")] 

144 if len(packages) > 0: 

145 out += venv_install(where, packages, fLOG=fLOG, 

146 temp_folder=temp_folder, 

147 platform=platform) 

148 return out 

149 

150 

151def venv_install(venv, packages, fLOG=None, 

152 temp_folder=None, platform=None): # pragma: no cover 

153 """ 

154 Installs a package or a list of packages in a virtual environment. 

155 

156 @param venv location of the virtual environment 

157 @param packages a package (str) or a list of packages(list[str]) 

158 @param fLOG logging function 

159 @param temp_folder temporary folder (to download module if needed), by default ``<where>/download`` 

160 @param platform platform (``sys.platform`` by default) 

161 @return standard output 

162 

163 The function does not work from a virtual environment. 

164 """ 

165 from ..loghelper import run_cmd 

166 if fLOG is None: 

167 from ..loghelper import noLOG 

168 fLOG = noLOG 

169 if is_virtual_environment(): 

170 raise NotImplementedErrorFromVirtualEnvironment() 

171 if temp_folder is None: 

172 temp_folder = os.path.join(venv, "download") 

173 if isinstance(packages, str): 

174 packages = [packages] 

175 if platform is None: 

176 platform = sys.platform 

177 

178 if packages == "pip" or packages == ["pip"]: # pylint: disable=R1714 

179 from .get_pip import __file__ as pip_loc # pylint: disable=E0401 

180 ppath = os.path.abspath(pip_loc.replace(".pyc", ".py")) 

181 script = ["-u", ppath] 

182 return run_venv_script(venv, script, fLOG=fLOG, is_cmd=True, platform=platform) 

183 elif packages == "pymyinstall" or packages == ["pymyinstall"]: # pylint: disable=R1714 

184 if platform.startswith("win"): 

185 pip = os.path.join(venv, "Scripts", "pip") 

186 else: 

187 pip = os.path.join(venv, "bin", "pip") 

188 local_setup = os.path.abspath(os.path.join(os.path.dirname( 

189 __file__), "..", "..", "..", "..", "pymyinstall", "setup.py")) 

190 if os.path.exists(local_setup): 

191 cwd = os.getcwd() 

192 os.chdir(os.path.dirname(local_setup)) 

193 script = ["-u", local_setup, "install"] 

194 out = run_venv_script(venv, script, fLOG=fLOG, is_cmd=True, 

195 skip_err_if="Finished processing dependencies for pymyinstall==", 

196 platform=platform) 

197 os.chdir(cwd) 

198 return out 

199 else: 

200 cmd = pip + " install pymyinstall" 

201 out, err = run_cmd(cmd, wait=True, fLOG=fLOG) 

202 if len(err) > 0: 

203 raise VirtualEnvError( 

204 "unable to install pymyinstall at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, venv, cmd)) 

205 return out 

206 else: 

207 p = os.path.normpath(os.path.join( 

208 os.path.abspath(os.path.dirname(__file__)), "..", "..")) 

209 ls = ','.join("'{0}'".format(_) for _ in packages) 

210 script = ["import sys", 

211 "sys.path.append('{0}')".format(p.replace("\\", "\\\\")), 

212 "import pymyinstall", 

213 "ps=[{0}]".format(ls), 

214 "t='{0}'".format(temp_folder.replace("\\", "\\\\")), 

215 "pymyinstall.packaged.install_all(temp_folder=t,list_module=ps,up_pip=False)"] 

216 return run_venv_script(venv, "\n".join(script), fLOG=fLOG, platform=platform) 

217 

218 

219def run_venv_script(venv, script, fLOG=None, 

220 file=False, is_cmd=False, 

221 skip_err_if=None, platform=None, 

222 **kwargs): # pragma: no cover 

223 """ 

224 Runs a script on a vritual environment (the script should be simple). 

225 

226 @param venv virtual environment 

227 @param script script as a string (not a file) 

228 @param fLOG logging function 

229 @param file is script a file or a string to execute 

230 @param is_cmd if True, script is a command line to run (as a list) for python executable 

231 @param skip_err_if do not pay attention to standard error if this string was found in standard output 

232 @param platform platform (``sys.platform`` by default) 

233 @param kwargs others arguments for function @see fn run_cmd. 

234 @return output 

235 

236 The function does not work from a virtual environment. 

237 """ 

238 from ..loghelper import run_cmd 

239 if fLOG is None: 

240 from ..loghelper import noLOG 

241 fLOG = noLOG 

242 

243 def filter_err(err): 

244 lis = err.split("\n") 

245 lines = [] 

246 for li in lis: 

247 if "missing dependencies" in li: 

248 continue 

249 if "' misses '" in li: 

250 continue 

251 lines.append(li) 

252 return "\n".join(lines).strip(" \r\n\t") 

253 

254 if is_virtual_environment(): 

255 raise NotImplementedErrorFromVirtualEnvironment() 

256 

257 if platform is None: 

258 platform = sys.platform 

259 

260 if platform.startswith("win"): 

261 exe = os.path.join(venv, "Scripts", "python") 

262 else: 

263 exe = os.path.join(venv, "bin", "python") 

264 if is_cmd: 

265 cmd = " ".join([exe] + script) 

266 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs) 

267 err = filter_err(err) 

268 if len(err) > 0 and (skip_err_if is None or skip_err_if not in out): 

269 raise VirtualEnvError( 

270 "unable to run cmd at {2}\n--CMD--\n{3}\n--OUT--\n{0}\n[pyqerror]" 

271 "\n{1}".format(out, err, venv, cmd)) 

272 return out 

273 else: 

274 script = ";".join(script.split("\n")) 

275 if file: 

276 if not os.path.exists(script): 

277 raise FileNotFoundError(script) 

278 cmd = " ".join([exe, "-u", '"{0}"'.format(script)]) 

279 else: 

280 cmd = " ".join([exe, "-u", "-c", '"{0}"'.format(script)]) 

281 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs) 

282 err = filter_err(err) 

283 if len(err) > 0: 

284 raise VirtualEnvError( 

285 "Unable to run script at {2}\n--CMD--\n{3}\n--OUT--\n{0}\n" 

286 "[pyqerror]\n{1}".format(out, err, venv, cmd)) 

287 return out 

288 

289 

290def run_base_script(script, fLOG=None, file=False, is_cmd=False, 

291 skip_err_if=None, argv=None, platform=None, **kwargs): 

292 """ 

293 Runs a script with the original intepreter even if this function 

294 is run from a virtual environment. 

295 

296 @param script script as a string (not a file) 

297 @param fLOG logging function 

298 @param file is script a file or a string to execute 

299 @param is_cmd if True, script is a command line to run (as a list) for python executable 

300 @param skip_err_if do not pay attention to standard error if this string was found in standard output 

301 @param argv list of arguments to add on the command line 

302 @param platform platform (``sys.platform`` by default) 

303 @param kwargs others arguments for function @see fn run_cmd. 

304 @return output 

305 

306 The function does not work from a virtual environment. 

307 The function does not raise an exception if the standard error 

308 contains something like:: 

309 

310 ---------------------------------------------------------------------- 

311 Ran 1 test in 0.281s 

312 

313 OK 

314 """ 

315 from ..loghelper import run_cmd 

316 if fLOG is None: # pragma: no cover 

317 from ..loghelper import noLOG 

318 fLOG = noLOG 

319 

320 def true_err(err): # pragma: no cover 

321 if "Ran 1 test" in err and "OK" in err: 

322 return False 

323 return True 

324 

325 if platform is None: 

326 platform = sys.platform 

327 

328 if hasattr(sys, 'real_prefix'): # pragma: no cover 

329 exe = sys.base_prefix 

330 elif hasattr(sys, "base_exec_prefix"): # pragma: no cover 

331 exe = sys.base_exec_prefix 

332 else: 

333 exe = sys.exec_prefix 

334 

335 if platform.startswith("win"): 

336 exe = os.path.join(exe, "python") 

337 else: 

338 exe = os.path.join(exe, "bin", "python%d.%d" % sys.version_info[:2]) 

339 if not os.path.exists(exe): 

340 exe = os.path.join(exe, "bin", "python") 

341 

342 if is_cmd: # pragma: no cover 

343 cmd = " ".join([exe] + script) 

344 if argv is not None: 

345 cmd += " " + " ".join(argv) 

346 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs) 

347 if len(err) > 0 and (skip_err_if is None or skip_err_if not in out) and true_err(err): 

348 p = sys.base_prefix if hasattr(sys, "base_prefix") else sys.prefix 

349 raise VirtualEnvError( 

350 "unable to run cmd at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, p, cmd)) 

351 return out 

352 else: 

353 script = ";".join(script.split("\n")) 

354 if file: 

355 if not os.path.exists(script): 

356 raise FileNotFoundError(script) # pragma: no cover 

357 cmd = " ".join([exe, "-u", '"{0}"'.format(script)]) 

358 else: 

359 cmd = " ".join([exe, "-u", "-c", '"{0}"'.format(script)]) 

360 if argv is not None: 

361 cmd += " " + " ".join(argv) 

362 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs) 

363 if len(err) > 0 and true_err(err): 

364 p = sys.base_prefix if hasattr(sys, "base_prefix") else sys.prefix 

365 raise VirtualEnvError( # pragma: no cover 

366 "unable to run script with {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, p, cmd)) 

367 return out 

368 

369 

370def check_readme_syntax(readme, folder, 

371 version="0.8", fLOG=None): # pragma: no cover 

372 """ 

373 Checks the syntax of the file ``readme.rst`` 

374 which describes a python project. 

375 

376 @param readme file to check 

377 @param folder location for the virtual environment 

378 @param version version of docutils 

379 @param fLOG logging function 

380 @return output or SyntaxError exception 

381 

382 `pipy server <https://pypi.python.org/pypi/>`_ is based on 

383 `docutils <https://pypi.python.org/pypi/docutils/>`_ ==0.8. 

384 The most simple way to check its syntax is to create a virtual environment, 

385 to install docutils==0.8 and to compile the file. 

386 This is what this function does. 

387 

388 Unfortunately, this functionality does not work yet 

389 from a virtual environment. 

390 """ 

391 if fLOG is None: 

392 from ..loghelper import noLOG 

393 fLOG = noLOG 

394 if is_virtual_environment(): 

395 raise NotImplementedErrorFromVirtualEnvironment() 

396 if not os.path.exists(folder): 

397 os.makedirs(folder) 

398 

399 out = create_virtual_env(folder, fLOG=fLOG, packages=[ 

400 "docutils==" + version, 

401 "pipdeptree"]) 

402 outfile = os.path.join(folder, "conv_readme.html") 

403 

404 script = ["from docutils import core", 

405 "import io", 

406 'from docutils.readers.standalone import Reader', 

407 'from docutils.parsers.rst import Parser', 

408 'from docutils.parsers.rst.directives.images import Image', 

409 'from docutils.parsers.rst.directives import _directives', 

410 'from docutils.writers.html4css1 import Writer', 

411 "from docutils.languages import _languages", 

412 "from docutils.languages import en, fr", 

413 "_languages['en'] = en", 

414 "_languages['fr'] = fr", 

415 "_directives['image'] = Image", 

416 "with open('{0}', 'r', encoding='utf8') as g: s = g.read()".format( 

417 readme.replace("\\", "\\\\")), 

418 "settings_overrides = {'output_encoding': 'unicode', 'doctitle_xform': True,", 

419 " 'initial_header_level': 2, 'warning_stream': io.StringIO()}", 

420 "parts = core.publish_parts(source=s, parser=Parser(), reader=Reader(), source_path=None,", 

421 " destination_path=None, writer=Writer(),", 

422 " settings_overrides=settings_overrides)", 

423 "with open('{0}', 'w', encoding='utf8') as f: f.write(parts['whole'])".format( 

424 outfile.replace("\\", "\\\\")), 

425 ] 

426 

427 file_script = os.path.join(folder, "test_" + os.path.split(readme)[-1]) 

428 with open(file_script, "w") as f: 

429 f.write("\n".join(script)) 

430 

431 out = run_venv_script(folder, file_script, fLOG=fLOG, file=True) 

432 with open(outfile, "r", encoding="utf8") as h: 

433 content = h.read() 

434 

435 if "System Message" in content: 

436 raise SyntaxError( 

437 "unable to parse a file with docutils==" + version + "\nCONTENT:\n" + content) 

438 

439 return out