Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

""" 

@file 

@brief Helpers for virtualenv 

""" 

import os 

import sys 

from ..loghelper import noLOG, run_cmd 

 

 

class VirtualEnvError(Exception): 

""" 

Exception raised by the function implemented in this file. 

""" 

pass 

 

 

def is_virtual_environment(): 

""" 

Tells if the script is run from a virtual environment. 

 

@return boolean 

""" 

return (getattr(sys, "base_exec_prefix", sys.exec_prefix) != sys.exec_prefix) or hasattr(sys, 'real_prefix') 

 

 

class NotImplementedErrorFromVirtualEnvironment(NotImplementedError): 

""" 

Defines an exception when a function does not work 

in a virtual environment. 

""" 

pass 

 

 

def build_venv_cmd(params, posparams): 

""" 

Builds the command line for virtual env. 

 

@param params dictionary of parameters 

@param posparams positional arguments 

@return string 

""" 

import venv 

v = venv.__file__ 

if v is None: 

raise ImportError("module venv should have a version number") 

exe = sys.executable.replace("w.exe", "").replace(".exe", "") 

cmd = [exe, "-m", "venv"] 

for k, v in params.items(): 

if v is None: 

cmd.append("--" + k) 

else: 

cmd.append("--" + k + "=" + v) 

cmd.extend(posparams) 

return " ".join(cmd) 

 

 

def create_virtual_env(where, symlinks=False, system_site_packages=False, 

clear=True, packages=None, fLOG=noLOG, 

temp_folder=None, platform=None): 

""" 

Creates a virtual environment. 

 

@param where location of this virtual environment 

@param symlinks attempt to symlink rather than copy 

@param system_site_packages Give the virtual environment access to the system site-packages dir 

@param clear Delete the environment directory if it already exists. 

If not specified and the directory exists, an error is raised. 

@param packages list of packages to install (it will install module 

:epkg:`pymyinstall`). 

@param fLOG logging function 

@param temp_folder temporary folder (to download module if needed), by default ``<where>/download`` 

@param platform platform to use 

@return stand output 

 

.. index:: virtual environment 

 

.. faqref:: 

:title: How to create a virtual environment? 

 

The following example creates a virtual environment. 

Packages can be added by specifying the parameter *package*. 

 

:: 

 

from pyquickhelper.pycode import create_virtual_env 

fold = "my_env" 

if not os.path.exists(fold): 

os.mkdir(fold) 

create_virtual_env(fold) 

 

The function does not work from a virtual environment. 

""" 

if is_virtual_environment(): 

raise NotImplementedErrorFromVirtualEnvironment() 

 

fLOG("create virtual environment at:", where) 

params = {} 

if symlinks: 

params["symlinks"] = None 

if system_site_packages: 

params["system-site-packages"] = None 

if clear: 

params["clear"] = None 

cmd = build_venv_cmd(params, [where]) 

out, err = run_cmd(cmd, wait=True, fLOG=fLOG) 

if len(err) > 0: 

raise VirtualEnvError( 

"unable to create virtual environement at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, where, cmd)) 

 

if platform is None: 

platform = sys.platform 

if platform.startswith("win"): 

scripts = os.path.join(where, "Scripts") 

else: 

scripts = os.path.join(where, "bin") 

 

if not os.path.exists(scripts): 

files = "\n ".join(os.listdir(where)) 

raise FileNotFoundError( 

"unable to find {0}, content:\n {1}".format(scripts, files)) 

 

in_scripts = os.listdir(scripts) 

pips = [_ for _ in in_scripts if _.startswith("pip")] 

if len(pips) == 0: 

out += venv_install(where, "pip", fLOG=fLOG, 

temp_folder=temp_folder, 

platform=platform) 

in_scripts = os.listdir(scripts) 

pips = [_ for _ in in_scripts if _.startswith("pip")] 

if len(pips) == 0: 

raise FileNotFoundError( 

"unable to find pip in {0}, content:\n {1}".format(scripts, in_scripts)) 

 

out += venv_install(where, "pymyinstall", fLOG=fLOG, 

temp_folder=temp_folder, platform=platform) 

 

if packages is not None and len(packages) > 0: 

fLOG("install packages in:", where) 

packages = [_ for _ in packages if _ not in ("pymyinstall", "pip")] 

if len(packages) > 0: 

out += venv_install(where, packages, fLOG=fLOG, 

temp_folder=temp_folder, 

platform=platform) 

return out 

 

 

def venv_install(venv, packages, fLOG=noLOG, temp_folder=None, platform=None): 

""" 

Installs a package or a list of packages in a virtual environment. 

 

@param venv location of the virtual environment 

@param packages a package (str) or a list of packages(list[str]) 

@param fLOG logging function 

@param temp_folder temporary folder (to download module if needed), by default ``<where>/download`` 

@param platform platform (``sys.platform`` by default) 

@return standard output 

 

The function does not work from a virtual environment. 

""" 

if is_virtual_environment(): 

raise NotImplementedErrorFromVirtualEnvironment() 

 

if temp_folder is None: 

temp_folder = os.path.join(venv, "download") 

 

if isinstance(packages, str): 

packages = [packages] 

 

if platform is None: 

platform = sys.platform 

 

if packages == "pip" or packages == ["pip"]: # pylint: disable=R1714 

from .get_pip import __file__ as pip_loc # pylint: disable=E0401 

ppath = os.path.abspath(pip_loc.replace(".pyc", ".py")) 

script = ["-u", ppath] 

return run_venv_script(venv, script, fLOG=fLOG, is_cmd=True, platform=platform) 

elif packages == "pymyinstall" or packages == ["pymyinstall"]: # pylint: disable=R1714 

if platform.startswith("win"): 

pip = os.path.join(venv, "Scripts", "pip") 

else: 

pip = os.path.join(venv, "bin", "pip") 

local_setup = os.path.abspath(os.path.join(os.path.dirname( 

__file__), "..", "..", "..", "..", "pymyinstall", "setup.py")) 

if os.path.exists(local_setup): 

cwd = os.getcwd() 

os.chdir(os.path.dirname(local_setup)) 

script = ["-u", local_setup, "install"] 

out = run_venv_script(venv, script, fLOG=fLOG, is_cmd=True, 

skip_err_if="Finished processing dependencies for pymyinstall==", 

platform=platform) 

os.chdir(cwd) 

return out 

else: 

cmd = pip + " install pymyinstall" 

out, err = run_cmd(cmd, wait=True, fLOG=fLOG) 

if len(err) > 0: 

raise VirtualEnvError( 

"unable to install pymyinstall at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, venv, cmd)) 

return out 

else: 

p = os.path.normpath(os.path.join( 

os.path.abspath(os.path.dirname(__file__)), "..", "..")) 

ls = ','.join("'{0}'".format(_) for _ in packages) 

script = ["import sys", 

"sys.path.append('{0}')".format(p.replace("\\", "\\\\")), 

"import pymyinstall", 

"ps=[{0}]".format(ls), 

"t='{0}'".format(temp_folder.replace("\\", "\\\\")), 

"pymyinstall.packaged.install_all(temp_folder=t,list_module=ps,up_pip=False)"] 

return run_venv_script(venv, "\n".join(script), fLOG=fLOG, platform=platform) 

 

 

def run_venv_script(venv, script, fLOG=noLOG, file=False, is_cmd=False, 

skip_err_if=None, platform=None, **kwargs): 

""" 

Runs a script on a vritual environment (the script should be simple). 

 

@param venv virtual environment 

@param script script as a string (not a file) 

@param fLOG logging function 

@param file is script a file or a string to execute 

@param is_cmd if True, script is a command line to run (as a list) for python executable 

@param skip_err_if do not pay attention to standard error if this string was found in standard output 

@param platform platform (``sys.platform`` by default) 

@param kwargs others arguments for function @see fn run_cmd. 

@return output 

 

The function does not work from a virtual environment. 

""" 

if is_virtual_environment(): 

raise NotImplementedErrorFromVirtualEnvironment() 

 

if platform is None: 

platform = sys.platform 

 

if platform.startswith("win"): 

exe = os.path.join(venv, "Scripts", "python") 

else: 

exe = os.path.join(venv, "bin", "python") 

if is_cmd: 

cmd = " ".join([exe] + script) 

out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs) 

if len(err) > 0 and (skip_err_if is None or skip_err_if not in out): 

raise VirtualEnvError( 

"unable to run cmd at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, venv, cmd)) 

return out 

else: 

script = ";".join(script.split("\n")) 

if file: 

if not os.path.exists(script): 

raise FileNotFoundError(script) 

cmd = " ".join([exe, "-u", '"{0}"'.format(script)]) 

else: 

cmd = " ".join([exe, "-u", "-c", '"{0}"'.format(script)]) 

out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs) 

if len(err) > 0: 

raise VirtualEnvError( 

"unable to run script at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, venv, cmd)) 

return out 

 

 

def run_base_script(script, fLOG=noLOG, file=False, is_cmd=False, 

skip_err_if=None, argv=None, platform=None, **kwargs): 

""" 

Runs a script with the original intepreter even if this function 

is run from a virtual environment. 

 

@param script script as a string (not a file) 

@param fLOG logging function 

@param file is script a file or a string to execute 

@param is_cmd if True, script is a command line to run (as a list) for python executable 

@param skip_err_if do not pay attention to standard error if this string was found in standard output 

@param argv list of arguments to add on the command line 

@param platform platform (``sys.platform`` by default) 

@param kwargs others arguments for function @see fn run_cmd. 

@return output 

 

The function does not work from a virtual environment. 

The function does not raise an exception if the standard error 

contains something like:: 

 

---------------------------------------------------------------------- 

Ran 1 test in 0.281s 

 

OK 

""" 

def true_err(err): 

if "Ran 1 test" in err and "OK" in err: 

return False 

else: 

return True 

 

if platform is None: 

platform = sys.platform 

 

if hasattr(sys, 'real_prefix'): 

exe = sys.base_prefix 

elif hasattr(sys, "base_exec_prefix"): 

exe = sys.base_exec_prefix 

else: 

exe = sys.exec_prefix 

 

if platform.startswith("win"): 

exe = os.path.join(exe, "python") 

else: 

exe = os.path.join(exe, "bin", "python%d.%d" % sys.version_info[:2]) 

if not os.path.exists(exe): 

exe = os.path.join(exe, "bin", "python") 

 

if is_cmd: 

cmd = " ".join([exe] + script) 

if argv is not None: 

cmd += " " + " ".join(argv) 

out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs) 

if len(err) > 0 and (skip_err_if is None or skip_err_if not in out) and true_err(err): 

p = sys.base_prefix if hasattr(sys, "base_prefix") else sys.prefix 

raise VirtualEnvError( 

"unable to run cmd at {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, p, cmd)) 

return out 

else: 

script = ";".join(script.split("\n")) 

if file: 

if not os.path.exists(script): 

raise FileNotFoundError(script) 

cmd = " ".join([exe, "-u", '"{0}"'.format(script)]) 

else: 

cmd = " ".join([exe, "-u", "-c", '"{0}"'.format(script)]) 

if argv is not None: 

cmd += " " + " ".join(argv) 

out, err = run_cmd(cmd, wait=True, fLOG=fLOG, **kwargs) 

if len(err) > 0 and true_err(err): 

p = sys.base_prefix if hasattr(sys, "base_prefix") else sys.prefix 

raise VirtualEnvError( 

"unable to run script with {2}\nCMD:\n{3}\nOUT:\n{0}\n[pyqerror]\n{1}".format(out, err, p, cmd)) 

return out 

 

 

def check_readme_syntax(readme, folder, version="0.8", fLOG=noLOG): 

""" 

Checks the syntax of the file ``readme.rst`` 

which describes a python project. 

 

@param readme file to check 

@param folder location for the virtual environment 

@param version version of docutils 

@param fLOG logging function 

@return output or SyntaxError exception 

 

`pipy server <https://pypi.python.org/pypi/>`_ is based on 

`docutils <https://pypi.python.org/pypi/docutils/>`_ ==0.8. 

The most simple way to check its syntax is to create a virtual environment, 

to install docutils==0.8 and to compile the file. 

This is what this function does. 

 

Unfortunately, this functionality does not work yet 

from a virtual environment. 

""" 

if is_virtual_environment(): 

raise NotImplementedErrorFromVirtualEnvironment() 

 

if not os.path.exists(folder): 

os.makedirs(folder) 

 

out = create_virtual_env(folder, fLOG=fLOG, packages=[ 

"docutils==" + version, 

"pipdeptree"]) 

outfile = os.path.join(folder, "conv_readme.html") 

 

script = ["from docutils import core", 

"import io", 

'from docutils.readers.standalone import Reader', 

'from docutils.parsers.rst import Parser', 

'from docutils.parsers.rst.directives.images import Image', 

'from docutils.parsers.rst.directives import _directives', 

'from docutils.writers.html4css1 import Writer', 

"from docutils.languages import _languages", 

"from docutils.languages import en, fr", 

"_languages['en'] = en", 

"_languages['fr'] = fr", 

"_directives['image'] = Image", 

"with open('{0}', 'r', encoding='utf8') as g: s = g.read()".format( 

readme.replace("\\", "\\\\")), 

"settings_overrides = {'output_encoding': 'unicode', 'doctitle_xform': True,", 

" initial_header_level': 2, 'warning_stream': io.StringIO()}", 

"parts = core.publish_parts(source=s, parser=Parser(), reader=Reader(), source_path=None,", 

" destination_path=None, writer=Writer(),", 

" settings_overrides=settings_overrides)", 

"with open('{0}', 'w', encoding='utf8') as f: f.write(parts['whole'])".format( 

outfile.replace("\\", "\\\\")), 

] 

 

file_script = os.path.join(folder, "test_" + os.path.split(readme)[-1]) 

with open(file_script, "w") as f: 

f.write("\n".join(script)) 

 

out = run_venv_script(folder, file_script, fLOG=fLOG, file=True) 

with open(outfile, "r", encoding="utf8") as h: 

content = h.read() 

 

if "System Message" in content: 

raise SyntaxError( 

"unable to parse a file with docutils==" + version + "\nCONTENT:\n" + content) 

 

return out