Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2""" 

3@file 

4@brief Various functions to install `python <http://www.python.org/>`_. 

5""" 

6from __future__ import print_function 

7import sys 

8import os 

9import datetime 

10 

11from ..installhelper.install_cmd_helper import run_cmd, unzip_files 

12from .install_custom import download_page, download_file 

13 

14if sys.version_info[0] == 2: 

15 FileNotFoundError = Exception 

16 

17 

18def unzip7_files(filename_7z, fLOG=print, dest="."): 

19 """ 

20 If `7z <http://www.7-zip.org/>`_ is installed, the function uses it 

21 to uncompress file into *7z* format. The file *filename_7z* must not exist. 

22 

23 .. index:: 7zip, 7z 

24 

25 :param filename_7z: final destination 

26 :param fLOG: logging function 

27 :param dest: destination folder 

28 

29 :return: output of 7z 

30 

31 .. versionadded:: 1.1 

32 """ 

33 if sys.platform.startswith("win"): 

34 exe = r"C:\Program Files\7-Zip\7z.exe" 

35 if not os.path.exists(exe): 

36 raise FileNotFoundError("unable to find: {0}".format(exe)) 

37 else: 

38 exe = "7z" 

39 

40 if not os.path.exists(filename_7z): 

41 raise FileNotFoundError(filename_7z) 

42 

43 cmd = '"{0}"-y -o"{2}" x "{1}"'.format(exe, filename_7z, dest) 

44 out, err = run_cmd(cmd, wait=True) 

45 

46 if err is not None and len(err) > 0: 

47 raise Exception("OUT:\n{0}\nERR-A:\n{1}".format(out, err)) 

48 

49 return out 

50 

51 

52def fix_fcntl_windows(path): 

53 """ 

54 Adds a file `fnctl.py` on :epkg:`Windows` 

55 (only available on :epkg:`Linux`). 

56 

57 @param path path to the python installation 

58 """ 

59 if not sys.platform.startswith("win"): 

60 raise Exception("fcntl should only be added on Windows.") 

61 dest = os.path.join(path, "Lib", "fcntl.py") 

62 if os.path.exists(dest): 

63 # already done 

64 return 

65 module = """ 

66 def fcntl(fd, op, arg=0): 

67 return 0 

68 def ioctl(fd, op, arg=0, mutable_flag=True): 

69 if mutable_flag: 

70 return 0 

71 else: 

72 return "" 

73 def flock(fd, op): 

74 return 

75 def lockf(fd, operation, length=0, start=0, whence=0): 

76 return 

77 """.replace(" ", "") 

78 with open(dest, "w") as f: 

79 f.write(module) 

80 

81 

82def fix_termios_windows(path): 

83 """ 

84 Adds a file `termios.py` on :epkg:`Windows` 

85 (only available on :epkg:`Linux`). 

86 

87 @param path path to the python installation 

88 """ 

89 if not sys.platform.startswith("win"): 

90 raise Exception("fcntl should only be added on Windows.") 

91 dest = os.path.join(path, "Lib", "termios.py") 

92 if os.path.exists(dest): 

93 # already done 

94 return 

95 module = """ 

96 TCSAFLUSH = 1 

97 """.replace(" ", "") 

98 with open(dest, "w") as f: 

99 f.write(module) 

100 

101 

102def fix_resource_windows(path): 

103 """ 

104 Adds a file `resource.py` on :epkg:`Windows` 

105 (only available on :epkg:`Linux`). 

106 

107 @param path path to the python installation 

108 """ 

109 if not sys.platform.startswith("win"): 

110 raise Exception("fcntl should only be added on Windows.") 

111 dest = os.path.join(path, "Lib", "resource.py") 

112 if os.path.exists(dest): 

113 # already done 

114 return 

115 module = """ 

116 """.replace(" ", "") 

117 with open(dest, "w") as f: 

118 f.write(module) 

119 

120 

121def _clean_err1(err): 

122 if err: 

123 lines = [] 

124 for line in err.split("\n"): 

125 if "find: ‘build’: No such file or directory" in line: 

126 continue 

127 if "(ignored)" in line: 

128 continue 

129 if "Task was destroyed but it is pending!" in line: 

130 continue 

131 if "[libinstall] Error 1 (ignored)" in line: 

132 continue 

133 if "task: <Task finished coro=<<async_generator_athrow without __name__>()" in line: 

134 continue 

135 if "stty: 'standard input': Inappropriate ioctl for device" in line: 

136 continue 

137 if "task: <Task pending coro=<<async_generator_athrow without __name__>()>>" in line: 

138 continue 

139 if "unhandled exception during asyncio.run() shutdown" in line: 

140 continue 

141 if "RuntimeError: can't send non-None value to a just-started coroutine" in line: 

142 continue 

143 if " which is not installed." in line: 

144 continue 

145 lines.append(line) 

146 err = "\n".join(lines).strip() if lines else None 

147 errl = err.lower() 

148 if 'error' not in errl and 'exception' not in errl: 

149 lines = [] 

150 for line in err.split("\n"): 

151 if line.startswith(' '): 

152 continue 

153 if 'note: declared here' in line: 

154 continue 

155 if "In file included" in line: 

156 continue 

157 if "warning:" in line: 

158 continue 

159 if "In function " in line: 

160 continue 

161 lines.append(line) 

162 err = "\n".join(lines).strip() if lines else None 

163 return err 

164 

165 

166def _clean_err0(err): 

167 # remove a couple of warnings. 

168 lines = err.split("\n") 

169 lines2 = [ 

170 _ for _ in lines if "UserWarning: Module pymyinstall was already imported" not in _] 

171 if len(lines2) < len(lines): 

172 lines2 = [ 

173 _ for _ in lines2 if "from pip._vendor import pkg_resources" not in _] 

174 return "\n".join(lines2) 

175 

176 

177def install_python(temp_folder=".", fLOG=print, install=True, force_download=False, # pylint: disable=R0914 

178 version=None, modules=None, custom=False, latest=False, 

179 download_folder="download", verbose=False, make_first=False): 

180 """ 

181 Installs :epkg:`python`. 

182 It does not do it a second time if it is already installed. 

183 

184 @param temp_folder where to download the setup 

185 @param fLOG logging function 

186 @param install install (otherwise only download) 

187 @param force_download force the downloading of python 

188 @param version version to download (by default the current version of Python) 

189 @param modules modules to install 

190 @param custom the standalone distribution has issue when installing new packages, 

191 custom is True means switching to a zip of the standard distribution, 

192 see below 

193 @param latest install this version of pymyinstall and not the pypi version 

194 @param download_folder download folder for packages 

195 @param verbose more display 

196 @param make_first run *make* before *make altinstall* 

197 @return temporary file 

198 

199 The version is fixed to the current version of Python and amd64. 

200 The standalone distribution has an issue and raises an error for some 

201 packages such as `smart_open <https://pypi.python.org/pypi/smart_open>`_: 

202 

203 :: 

204 

205 error: [Errno 2] No such file or directory: '<python>\\python36.zip\\lib2to3\\Grammar.txt' 

206 

207 In that case, you should consider using ``custom=True``. 

208 The function work for :epkg:`Linux` too. 

209 List of steps done in linux: 

210 

211 :: 

212 

213 mkdir install_folder 

214 cd install_folder 

215 curl -O https://www.python.org/ftp/python/3.7.2/Python-3.7.2.tgz 

216 tar xzf Python-3.7.2.tgz 

217 mkdir dist372 

218 cd Python-3.7.2/ 

219 # current folder is /home/dupre/temp/temp_py/dist372/ 

220 ./configure --enable-optimizations --with-ensurepip=install --prefix=/home/dupre/temp/temp_py/dist372/inst --exec-prefix=/home/dupre/temp/temp_py/dist372/bin --datadir=/home/dupre/temp/temp_py/dist372/data 

221 """ 

222 cmds = [] 

223 

224 if version is None: 

225 version = "%s.%s.%s" % sys.version_info[:3] 

226 versioni = tuple([int(_) for _ in version.split(".")]) 

227 link = "https://www.python.org/downloads/release/python-%s/" % version.replace( 

228 ".", "") 

229 page = download_page(link) 

230 if page is None: 

231 raise ValueError("page is None for link '{0}'".format(link)) 

232 

233 if sys.platform.startswith("win"): 

234 if versioni[:2] <= (3, 4): 

235 raise NotImplementedError( 

236 "Python <= 3.4 is not supported anymore.") 

237 # The setup for Python 3.5 does not accept multiple versions, 

238 # it was installed on one machine and then compressed into a 7z 

239 # file 

240 if versioni >= (3, 7, 0): 

241 if custom: 

242 if versioni > (3, 7, 0): 

243 raise ValueError( 

244 "Not custom zip available for Python {0}".format(versioni)) 

245 url = "http://www.xavierdupre.fr/enseignement/setup/Python{0}{1}-{0}.{1}.{2}-amd64.zip".format( 

246 *versioni[:3]) 

247 else: 

248 url = "https://www.python.org/ftp/python/{0}.{1}.{2}/python-{0}.{1}.{2}-embed-amd64.zip".format( 

249 *versioni[:3]) 

250 elif versioni >= (3, 6, 0): 

251 if custom: 

252 if versioni > (3, 6, 5): 

253 raise ValueError( 

254 "Not custom zip available for Python {0}".format(versioni)) 

255 url = "http://www.xavierdupre.fr/enseignement/setup/Python{0}{1}-{0}.{1}.{2}-amd64.zip".format( 

256 *versioni[:3]) 

257 else: 

258 url = "https://www.python.org/ftp/python/{0}.{1}.{2}/python-{0}.{1}.{2}-embed-amd64.zip".format( 

259 *versioni[:3]) 

260 elif versioni >= (3, 5, 0): 

261 if custom: 

262 if versioni not in [(3, 5, 3), (3, 5, 2)]: 

263 raise ValueError( 

264 "Not custom zip available for Python {0}".format(versioni)) 

265 url = "http://www.xavierdupre.fr/enseignement/setup/Python35-3.5.3-amd64.zip" 

266 else: 

267 url = "https://www.python.org/ftp/python/3.5.3/python-3.5.3-embed-amd64.zip" 

268 else: 

269 raise Exception( 

270 "Unable to find a proper version for version {0}".format(version)) 

271 else: 

272 url = "https://www.python.org/ftp/python/{0}.{1}.{2}/Python-{0}.{1}.{2}.tgz".format( 

273 *versioni) 

274 

275 full = url.split("/")[-1] 

276 outfile = os.path.join(temp_folder, full) 

277 fLOG("[install_python] download", url) 

278 local = download_file(url, outfile, fLOG=fLOG) 

279 

280 # Install 

281 if install: 

282 # unzip files 

283 if sys.platform.startswith("win"): 

284 unzip_files(local, temp_folder, fLOG=fLOG) 

285 else: 

286 cmd = "tar xzf {0}".format(outfile) 

287 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, 

288 change_path=temp_folder) 

289 cmds.append(cmd) 

290 if err: 

291 raise RuntimeError( 

292 "Issue with running '{0}'\n--OUT--\n{1}\n--ERR--\n{2}\n--IN--\n{3}\n--CMDS--\n{4}".format( 

293 cmd, out, err, temp_folder, "\n".join(cmds))) 

294 pyinstall = os.path.join( 

295 temp_folder, "Python-{0}.{1}.{2}".format(*versioni)) 

296 

297 cmd = "./configure --enable-optimizations --with-ensurepip=install --prefix={0}/inst --exec-prefix={0}/bin --datadir={0}/data" 

298 cmd = cmd.format(temp_folder) 

299 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, 

300 change_path=pyinstall) 

301 cmds.append(cmd) 

302 if err: 

303 lines = [] 

304 for line in err.split("\n"): 

305 if "[libinstall] Error 1 (ignored)" in line: 

306 continue 

307 lines.append(line) 

308 err = "\n".join(lines).strip() if lines else None 

309 if err: 

310 raise RuntimeError( 

311 "Issue with running '{0}'\n--OUT--\n{1}\n--ERR--\n{2}\n--CMDS--\n{3}".format( 

312 cmd, out, err, "\n".join(cmds))) 

313 

314 # See https://stackoverflow.com/questions/44708262/make-install-from-source-python-without-running-tests. 

315 os.environ["EXTRATESTOPTS"] = "--list-tests" 

316 

317 if make_first: 

318 cmd = "make" 

319 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, 

320 change_path=pyinstall) 

321 cmds.append(cmd) 

322 err = _clean_err1(err) 

323 if err: 

324 raise RuntimeError( 

325 "Issue while running '{0}'\n---URL---\n{1}\n---OUT---\n{2}\n" 

326 "---ERR---?1-\n{3}\n---IN---\n{4}\n---CMDS---\n{5}".format( 

327 cmd, url, out, err, pyinstall, "\n".join(cmds))) 

328 

329 cmd = "make altinstall" 

330 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, 

331 change_path=pyinstall) 

332 cmds.append(cmd) 

333 err = _clean_err1(err) 

334 if err: 

335 lines = [] 

336 for line in err.split("\n"): 

337 if "[libinstall] Error 1 (ignored)" in line: 

338 continue 

339 lines.append(line) 

340 err = "\n".join(lines).strip() if lines else None 

341 if err: 

342 raise RuntimeError( 

343 "Issue while running '{0}'\n---URL---\n{1}\n---OUT---\n{2}\n---ERR---?2-\n{3}\n---IN---\n{4}\n---CMDS---\n{5}".format( 

344 cmd, url, out, err, pyinstall, "\n".join(cmds))) 

345 

346 # has pip? 

347 if sys.platform.startswith("win"): 

348 pyexe = os.path.join(temp_folder, "python.exe") 

349 else: 

350 pyexe = os.path.join(temp_folder, "bin", "python") 

351 cmd = "{0} -m pip --help" 

352 cmds.append(cmd) 

353 try: 

354 _, err = run_cmd(cmd, wait=True) 

355 has_pip = not err 

356 except Exception: 

357 has_pip = False 

358 

359 # get-pip 

360 if not has_pip: 

361 get_pip = "https://bootstrap.pypa.io/get-pip.py" 

362 outfile_pip = os.path.join(temp_folder, "get-pip.py") 

363 download_file(get_pip, outfile_pip, fLOG=fLOG) 

364 

365 # following issue https://github.com/pypa/get-pip/issues/7 

366 if sys.platform.startswith("win"): 

367 vers = "%d%d" % versioni[:2] 

368 if vers in ("36", "37"): 

369 pth = os.path.join(temp_folder, "python%s._pth" % vers) 

370 if os.path.exists(pth): 

371 with open(pth, "r") as f: 

372 content = f.read() 

373 content = content.replace( 

374 "#import site", "import site") 

375 with open(pth, "w") as f: 

376 f.write(content) 

377 

378 # run get-pip.py 

379 if sys.platform.startswith("win"): 

380 pyexe = os.path.join(temp_folder, "python.exe") 

381 else: 

382 versioni3 = versioni[:3] 

383 pyexe = os.path.join( 

384 temp_folder, "Python-{}.{}.{}".format(*versioni3), "python") 

385 if not os.path.exists(pyexe): 

386 raise FileNotFoundError(pyexe) 

387 

388 # Patches for windows. 

389 if install and sys.platform.startswith("win"): 

390 if not custom: 

391 cmd = '"{0}" -u "{1}"'.format(pyexe, outfile_pip) 

392 out, err = run_cmd(cmd, wait=True, fLOG=fLOG) 

393 cmds.append(cmd) 

394 if len(err) > 0: 

395 skip = ['Consider adding this directory to PATH', 

396 'which is not on PATH.'] 

397 lines = err.split('\n') 

398 errs = [] 

399 for line in lines: 

400 zoo = True 

401 for sk in skip: 

402 if sk in line: 

403 zoo = False 

404 break 

405 if zoo: 

406 errs.append(line) 

407 err = "\n".join(errs).strip(' \n\r') 

408 if len(err) > 0: 

409 raise Exception( 

410 "Something went wrong:\nCMD\n{0}\nOUT\n{1}\nERR-B\n{2}\n---CMDS--\n{3}".format( 

411 cmd, out, err, "\n".join(cmds))) 

412 else: 

413 from ..win_installer.win_patch import win_patch_paths 

414 fLOG("[install_python] Patch scripts .exe") 

415 patched = win_patch_paths(temp_folder, pyexe, fLOG=fLOG) 

416 for pat in patched: 

417 fLOG(" - ", pat) 

418 

419 # fix fcntl 

420 fix_fcntl_windows(temp_folder) 

421 fix_termios_windows(temp_folder) 

422 fix_resource_windows(temp_folder) 

423 

424 # modules 

425 if install and modules is not None: 

426 if isinstance(modules, list): 

427 raise NotImplementedError( 

428 "Not implemented for a list of modules.") 

429 

430 # cmd = '"{0}" -u -c "import pip;pip.main([\'install\', 

431 # \'https://github.com/sdpython/pymyinstall/archive/master.zip\'])"'.format(pyexe) 

432 if latest: 

433 folder = os.path.normpath(os.path.join(os.path.abspath( 

434 os.path.dirname(__file__)), "..", "..", "..")) 

435 setup = os.path.join(folder, "setup.py") 

436 if not os.path.exists(setup): 

437 raise FileNotFoundError(setup) 

438 sep = "\\" if sys.platform.startswith("win") else "/" 

439 cmd = '"{0}" -u "{1}{2}setup.py" install'.format( 

440 pyexe, folder, sep) 

441 change_path = folder 

442 else: 

443 cmd = '"{0}" -u -c "import pip._internal;pip._internal.main([\'install\', \'pymyinstall\'])"'.format( 

444 pyexe) 

445 change_path = None 

446 fLOG("[install_python] " + cmd) 

447 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, change_path=change_path) 

448 cmds.append(cmd) 

449 err_keep = err 

450 err = [_ for _ in err.split("\n") if not _.startswith("pymyinstall.") and not _.startswith( 

451 "zip_safe flag not set; analyzing archive contents...") and not _.startswith("error removing build")] 

452 err = "\n".join(_ for _ in err if _) 

453 

454 exp = ".zip/lib2to3/Grammar.txt" 

455 if len(err) > 0 and exp not in out.replace("\\", "/").replace("//", "/"): 

456 raise Exception( 

457 "Something went wrong:\nCMD\n{0}\nOUT\n{1}\nERR-C\n{2}".format(cmd, out, err_keep)) 

458 fLOG(out) 

459 

460 dirpyexe = os.path.dirname(pyexe) 

461 fLOG( 

462 "[install_python] add python to PATH='{0}'".format(dirpyexe)) 

463 path = os.environ['PATH'] 

464 path = ";".join([dirpyexe, path]) 

465 os.environ['PATH'] = path 

466 

467 fLOG("[install_python] install modules") 

468 pattern = '"{0}" -u -c "import sys;from pymyinstall.packaged import install_all;install_all(fLOG=print, temp_folder=\'{2}\',' + \ 

469 'verbose=True, source=\'2\', list_module=\'{1}\')"' 

470 cmd = pattern.format( 

471 pyexe, modules, download_folder.replace("\\", "/")) 

472 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, 

473 communicate=False, catch_exit=True) 

474 cmds.append(cmd) 

475 fLOG("[install_python] end installed modules.") 

476 if len(err) > 0: 

477 # We try a second time to make sure a second pass does not help. 

478 fLOG("[install_python2] install modules") 

479 out_, err_ = run_cmd( 

480 cmd, wait=True, fLOG=fLOG, communicate=False, catch_exit=False) 

481 err__ = _clean_err0(err_) 

482 if len(err__) > 0: 

483 mes = "[install_python2] end installed modules. Something went wrong:\n" 

484 raise Exception( 

485 mes + "ERR-D-CMD\n{0}\nOUT\n{1}\nOUT2\n{3}\nERR-D\n{2}\nERR2-D\n{4}\nERR2-Dc\n{5}\n**CMD**\n{0}\n--CMDS--\n{6}".format( 

486 cmd, out, err, out_, err_, err__, "\n".join(cmds))) 

487 out += ("\n-------------" * 5) + "\n" + out_ 

488 fLOG("[install_python2] end installed modules.") 

489 fLOG(out) 

490 

491 return local 

492 

493 

494def folder_older_than(folder, delay=datetime.timedelta(30)): 

495 """ 

496 Tells if a folder is older than a given timespan. 

497 

498 @param folder folder name 

499 @param delay delay 

500 @return boolean 

501 """ 

502 folder = os.path.abspath(folder) 

503 if not os.path.exists(folder): 

504 return False 

505 cre = os.stat(folder).st_ctime 

506 dt = datetime.datetime.fromtimestamp(cre) 

507 now = datetime.datetime.now() 

508 delta = now - dt 

509 return delta > delay