Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1""" 

2@file 

3@brief Various function to install various python module from various location. 

4""" 

5from __future__ import print_function 

6 

7import sys 

8import platform 

9import os 

10import zipfile 

11import datetime 

12from .module_install_exceptions import UpdatePipError 

13from .run_cmd import run_cmd_private, run_cmd_old 

14 

15if sys.version_info[0] == 2: 

16 FileNotFoundError = Exception 

17 

18 

19def python_version(): 

20 """ 

21 Retrieves the platform and version of this :epkg:`python`. 

22 

23 @return tuple, example: ("win32","32bit") or ("win32","64bit") 

24 """ 

25 return sys.platform, platform.architecture()[0] 

26 

27 

28def unzip_files(zipf, whereTo, fLOG=print): 

29 """ 

30 Unzip files from a :epkg:`zip` archive. 

31 

32 @param zipf archive 

33 @param whereTo destination folder 

34 @param fLOG logging function 

35 @return list of unzipped files 

36 """ 

37 files = [] 

38 with zipfile.ZipFile(zipf, "r") as file: 

39 for info in file.infolist(): 

40 if not os.path.exists(info.filename): 

41 data = file.read(info.filename) 

42 tos = os.path.join(whereTo, info.filename) 

43 if not os.path.exists(tos): 

44 finalfolder = os.path.split(tos)[0] 

45 if not os.path.exists(finalfolder): 

46 fLOG(" creating folder ", finalfolder) 

47 os.makedirs(finalfolder) 

48 if not info.filename.endswith("/"): 

49 u = open(tos, "wb") 

50 u.write(data) 

51 u.close() 

52 files.append(tos) 

53 if fLOG: 

54 fLOG(" unzipped ", info.filename, " to ", tos) 

55 elif not tos.endswith("/"): 

56 files.append(tos) 

57 elif not info.filename.endswith("/"): 

58 files.append(info.filename) 

59 return files 

60 

61 

62def add_shortcut_to_desktop_for_module(name): 

63 """ 

64 Adds a shortcut on a module which includes a script. 

65 

66 @param name name of the module 

67 @return shortcut was added or not 

68 """ 

69 if name == "spyder": 

70 from .link_shortcuts import add_shortcut_to_desktop, suffix 

71 from .module_install import ModuleInstall 

72 md = ModuleInstall("spyder", "exe", script="spyder.bat") 

73 sc = md.Script 

74 if os.path.exists(sc): 

75 ver = suffix() 

76 r = add_shortcut_to_desktop(sc, name + "." + ver, name + "." + ver) 

77 return os.path.exists(r) 

78 else: 

79 return False 

80 else: 

81 raise NotImplementedError( 

82 "nothing implemented for module: {0}".format(name)) 

83 

84 

85def get_pip_program(exe=None): 

86 """ 

87 Gets :epkg:`pip` executable and 

88 fixes an issue with :epkg:`Pandoc`. 

89 

90 @param exe path to python executable 

91 @return pip executable 

92 

93 .. faqref:: 

94 :title: How can I check the dependencies? 

95 

96 The module `pipdeptree <https://github.com/naiquevin/pipdeptree>`_ gives 

97 you something like:: 

98 

99 d3py==0.2.3 

100 - ipython [installed: 3.1.0] 

101 - networkx [installed: 1.9.1] 

102 - decorator [required: >=3.4.0, installed: 3.4.2] 

103 - numpy [installed: 1.9.2] 

104 - pandas [installed: 0.16.0] 

105 - pytz [required: >=2011k, installed: 2015.4] 

106 - python-dateutil [required: >=2, installed: 2.4.2] 

107 - six [required: >=1.5, installed: 1.9.0] 

108 - numpy [required: >=1.7.0, installed: 1.9.2] 

109 autopep8==1.1.1 

110 - pep8 [required: >=1.5.7, installed: 1.5.7] 

111 sphinxjp.themes.basicstrap==0.4.2 

112 - setuptools 

113 - Sphinx [installed: 1.3.1] 

114 - alabaster [required: >=0.7, installed: 0.7.4] 

115 - six [required: >=1.4, installed: 1.9.0] 

116 - colorama [installed: 0.3.3] 

117 - Pygments [required: >=2.0, installed: 2.0.2] 

118 - Babel [required: >=1.3, installed: 1.3] 

119 - pytz [required: >=0a, installed: 2015.4] 

120 - snowballstemmer [required: >=1.1, installed: 1.2.0] 

121 - docutils [required: >=0.11, installed: 0.12] 

122 - sphinx-rtd-theme [required: >=0.1, installed: 0.1.8] 

123 - Sphinx [required: >=1.3, installed: 1.3.1] 

124 - alabaster [required: >=0.7, installed: 0.7.4] 

125 - six [required: >=1.4, installed: 1.9.0] 

126 - colorama [installed: 0.3.3] 

127 - Pygments [required: >=2.0, installed: 2.0.2] 

128 - Babel [required: >=1.3, installed: 1.3] 

129 - pytz [required: >=0a, installed: 2015.4] 

130 - snowballstemmer [required: >=1.1, installed: 1.2.0] 

131 - docutils [required: >=0.11, installed: 0.12] 

132 - Jinja2 [required: >=2.3, installed: 2.7.3] 

133 - MarkupSafe [installed: 0.23] 

134 - Jinja2 [required: >=2.3, installed: 2.7.3] 

135 - MarkupSafe [installed: 0.23] 

136 

137 ... 

138 """ 

139 tried = [] 

140 if exe is None: 

141 exe = os.path.dirname(sys.executable) 

142 major, minor = sys.version_info[0:2] 

143 if sys.platform.startswith("win"): 

144 if not exe.lower().endswith("scripts"): 

145 pi = os.path.join(exe, "Scripts", "pip.exe") 

146 tried.append(pi) 

147 if not os.path.exists(pi): 

148 pi = os.path.join(exe, "Scripts", "pip%d.exe" % major) 

149 tried.append(pi) 

150 if not os.path.exists(pi): 

151 # Anaconda is different 

152 pi = os.path.join(exe, "Scripts", "pip.exe") 

153 tried.append(pi) 

154 if not os.path.exists(pi): 

155 pi = os.path.join(exe, "Scripts", "pip%d.exe" % major) 

156 tried.append(pi) 

157 if not os.path.exists(pi): 

158 pi = os.path.join( 

159 exe, "Scripts", "pip%d.%d.exe" % (major, minor)) 

160 tried.append(pi) 

161 raise FileNotFoundError( 

162 "tried (1):\n" + "\n".join(tried) + "\n---- try ---\npython -m pip install -U pip --force") 

163 else: 

164 pi = os.path.join(exe, "pip.exe") 

165 tried.append(pi) 

166 if not os.path.exists(pi): 

167 # Anaconda is different 

168 pi = os.path.join(exe, "pip.exe") 

169 tried.append(pi) 

170 if not os.path.exists(pi): 

171 pi = os.path.join(exe, "pip%d.exe" % major) 

172 tried.append(pi) 

173 if not os.path.exists(pi): 

174 pi = os.path.join(exe, "pip%d.%d.exe" % (major, minor)) 

175 tried.append(pi) 

176 if not os.path.exists(pi): 

177 raise FileNotFoundError( 

178 "tried (2):\n" + "\n".join(tried) + "\n---- try ---\npython -m pip install -U pip --force") 

179 else: 

180 if sys.version_info[0] == 2: 

181 if exe is None: 

182 return "pip" 

183 else: 

184 pi = os.path.join(exe, "pip") 

185 else: 

186 major = sys.version_info[0] 

187 minor = sys.version_info[1] 

188 if exe is None: 

189 return "pip%d.%d" % (major, minor) 

190 else: 

191 # this does not work because on Linux, the binary is installed on the local path 

192 # pip3.4 are not in the same place 

193 # pi = os.path.join(exe, "pip%d.%d" % (major, minor)) 

194 import pip 

195 exe = os.path.normpath(os.path.join(os.path.dirname( 

196 pip.__file__), "..", "..", "..", "..", "bin")) 

197 pi = os.path.join(exe, "pip%d.%d" % (major, minor)) 

198 if not os.path.exists(pi): 

199 pi = os.path.join(exe, "pip") 

200 if not os.path.exists(pi): 

201 raise FileNotFoundError( 

202 "unable to find pip: {0}\n__file__={1}\nexe={2}".format(pi, pip.__file__, exe)) 

203 return pi 

204 

205 

206def get_python_program(): 

207 """ 

208 Returns the executable for :epkg:`python`. 

209 

210 .. versionadded:: 1.1 

211 """ 

212 pip = get_pip_program() 

213 dirname = os.path.dirname(pip) 

214 exe = os.path.join( 

215 dirname, "python.exe" if sys.platform.startswith("win") else "python") 

216 if os.path.exists(exe): 

217 return exe 

218 exe = os.path.normpath(os.path.join( 

219 dirname, "..", "python.exe" if sys.platform.startswith("win") else "python")) 

220 if os.path.exists(exe): 

221 return exe 

222 raise FileNotFoundError(exe) 

223 

224 

225def get_conda_program(exe=None): 

226 """ 

227 Gets :epkg:`conda` executable and 

228 fixes an issue with :epkg:`Pandoc`. 

229 

230 @param exe path to python executable 

231 @return conda executable 

232 """ 

233 tried = [] 

234 if exe is None: 

235 exe = os.path.dirname(sys.executable) 

236 if sys.platform.startswith("win"): 

237 if not exe.lower().endswith("scripts"): 

238 pi = os.path.join(exe, "Scripts", "conda.exe") 

239 tried.append(pi) 

240 if not os.path.exists(pi): 

241 # Anaconda is different 

242 pi = os.path.join(exe, "Scripts", "conda.exe") 

243 tried.append(pi) 

244 if not os.path.exists(pi): 

245 raise FileNotFoundError( 

246 "tried (1):\n" + "\n".join(tried)) 

247 else: 

248 pi = os.path.join(exe, "conda.exe") 

249 tried.append(pi) 

250 if not os.path.exists(pi): 

251 # Anaconda is different 

252 pi = os.path.join(exe, "conda.exe") 

253 tried.append(pi) 

254 if not os.path.exists(pi): 

255 raise FileNotFoundError( 

256 "tried (2):\n" + "\n".join(tried)) 

257 else: 

258 if exe is None: 

259 return "conda" 

260 else: 

261 pi = os.path.join(exe, "conda") 

262 

263 return pi 

264 

265 

266def get_file_modification_date(filename): 

267 """ 

268 Gets the date modification for a filename. 

269 

270 @param filename filename 

271 @return datetime 

272 """ 

273 t = os.path.getmtime(filename) 

274 return datetime.datetime.fromtimestamp(t) 

275 

276 

277def update_pip(python_path=None, fLOG=print): 

278 """ 

279 Updates :epkg:`pip` for a specific distribution. 

280 

281 @param python_path python path (or sys.executable if None) 

282 @param fLOG logging function 

283 @return output 

284 

285 The command ``python -m pip install -U pip`` or 

286 ``pip install --upgrade pip`` might fail on Windows due to very long paths 

287 (see `Upgrading pip fails on Windows when install path is too long <https://github.com/pypa/pip/issues/3055>`_). 

288 If that happens, 

289 assuming the module *pymyinstall* was installed with pip, we can now remove 

290 *pip* and use *get_pip.py* instead. This part requires *pyquickhelper*. 

291 

292 We try the url `bootstrap.pypa.io/get-pip.py <https://bootstrap.pypa.io/get-pip.py>`_ first 

293 then a local copy. 

294 """ 

295 if python_path is None: 

296 python_path = sys.executable 

297 else: 

298 python_path = os.path.join(python_path, "python") 

299 cmd = python_path + " -m pip install -U pip" 

300 out, err = run_cmd(cmd, wait=True, fLOG=fLOG) 

301 if err and len(err) > 0: 

302 if ("FileNotFoundError" in err or "No module named pip.__main__" in err) \ 

303 and sys.platform.startswith("win"): 

304 from pyquickhelper.filehelper import remove_folder 

305 # we try to remove pip and to install it again 

306 # it might be due to long path on Windows 

307 pack = os.path.join(os.path.dirname( 

308 python_path), "Lib", "site-packages") 

309 if not os.path.exists(pack): 

310 raise FileNotFoundError(pack) 

311 fpip = os.path.join(pack, "pip") 

312 if os.path.exists(fpip): 

313 # remove the folder 

314 fLOG(" remove folder", fpip) 

315 remove_folder(fpip) 

316 

317 pip_ = [_ for _ in os.listdir(pack) if _.startswith("pip-")] 

318 if len(pip_) > 0: 

319 for _ in pip_: 

320 fp = os.path.join(pack, _) 

321 fLOG(" remove folder", fp) 

322 remove_folder(fpip) 

323 

324 url = "https://bootstrap.pypa.io/get-pip.py" 

325 cmd = python_path + " " + url 

326 out, err = run_cmd(cmd, wait=True) 

327 if err and len(err) > 0: 

328 get_pip = os.path.abspath(os.path.join( 

329 os.path.dirname(__file__), "get_pip.py")) 

330 if not os.path.exists(get_pip): 

331 raise FileNotFoundError(get_pip) 

332 cmd = python_path + " " + get_pip 

333 out, err = run_cmd(cmd, wait=True) 

334 if err and len(err) > 0: 

335 raise UpdatePipError( 

336 "unable to update pip with get_pip.\nCMD:\n{0}\nOUT:\n{1}\nERR-E:\n{2}".format(cmd, out, err)) 

337 else: 

338 lines = err.split("\n") 

339 keep = [] 

340 for line in lines: 

341 if not line.startswith(" ") and "RuntimeWarning: Config variable" not in line and \ 

342 not(" which is incompatible." in line and " has requirement " in line) and \ 

343 not(" requires " in line and " which is not installed." in line) and \ 

344 not("Cache entry deserialization failed, entry ignored" in line) and \ 

345 len(line.strip()) > 3: 

346 keep.append(line) 

347 if len(keep) > 0 and "Requirement already up-to-date" not in out: 

348 for _ in keep: 

349 print("++", _) 

350 raise UpdatePipError( 

351 "Unable to update pip.\nCMD:\n{0}\nOUT:\n{1}\nERR-F:\n{2}".format(cmd, out, err)) 

352 return out 

353 

354 

355def has_pip(): 

356 """ 

357 Tells if :epkg:`pip` is installed. 

358 

359 @return boolean 

360 """ 

361 try: 

362 import pip 

363 return pip is not None 

364 except ImportError: 

365 return False 

366 

367 

368def is_conda_distribution(): 

369 """ 

370 Tells if it is a :epkg:`conda` distribution or not, 

371 check the presence of ``Continuum Analytics`` 

372 or ``|Anaconda`` in ``sys.version``. 

373 

374 @return boolean 

375 

376 .. versionadded:: 1.1 

377 """ 

378 return "Continuum Analytics" in sys.version or "|Anaconda" in sys.version 

379 

380 

381def run_cmd(cmd, sin="", shell=sys.platform.startswith("win"), wait=False, log_error=True, 

382 stop_running_if=None, encerror="ignore", 

383 encoding="utf8", change_path=None, communicate=True, 

384 preprocess=True, timeout=None, catch_exit=False, fLOG=None, 

385 tell_if_no_output=None, old_behavior=False): 

386 """ 

387 Runs a command line and waits for the results, 

388 @see fn run_cmd_private. 

389 """ 

390 if old_behavior or not sys.platform.startswith("win"): 

391 return run_cmd_old(cmd=cmd, sin=sin, shell=shell, wait=wait, log_error=log_error, 

392 secure=None, stop_waiting_if=stop_running_if, do_not_log=False, 

393 encerror=encerror, encoding=encoding, cwd=change_path, fLOG=fLOG) 

394 else: 

395 return run_cmd_private(cmd=cmd, sin=sin, shell=shell, wait=wait, log_error=log_error, 

396 stop_running_if=stop_running_if, encerror=encerror, 

397 encoding=encoding, change_path=change_path, communicate=communicate, 

398 preprocess=preprocess, timeout=timeout, catch_exit=catch_exit, fLOG=fLOG, 

399 tell_if_no_output=tell_if_no_output)