Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1""" 

2@file 

3@brief Install or update all packages. 

4 

5.. faqref:: 

6 :title: Skipped modules 

7 

8 Some modules are still marked as a dependency by others 

9 even if they are not needed or cannot be installed. 

10 

11 * `enum-compat <https://pypi.python.org/pypi/enum-compat>`_: 

12 not needed after Python 3.5 

13 * `prettytable <https://pypi.python.org/pypi/PrettyTable>`_: 

14 should be repladed by `PTable <https://pypi.python.org/pypi/PTable>`_. 

15""" 

16from __future__ import print_function 

17import sys 

18import os 

19import warnings 

20from textwrap import wrap 

21from ..installhelper import ModuleInstall, has_pip, update_pip, is_installed, get_module_dependencies 

22from ..installhelper.module_install_exceptions import MissingVersionOnPyPiException, MissingPackageOnPyPiException, MissingReferenceException 

23from ..installhelper.module_dependencies import missing_dependencies 

24from .packaged_exception import ModuleNotFoundError 

25from .packaged_config import all_set 

26 

27 

28def _build_reverse_index(): 

29 """ 

30 builds a reverse index of the module, 

31 """ 

32 res = {} 

33 mods = all_set() 

34 for m in mods: 

35 res[m.name] = m 

36 if m.mname is not None and m.mname != m.name: 

37 res[m.mname] = m 

38 if m.name.lower() not in (m.name, m.mname): 

39 res[m.name.lower()] = m 

40 return res 

41 

42 

43_reverse_module_index = _build_reverse_index() 

44 

45 

46def find_module_install(name, must_exist=False): 

47 """ 

48 Checks if there are specific instructions 

49 to run before installing module *name*, 

50 on Windows, some modules requires compilation, 

51 if not uses default option with *pip*. 

52 

53 @param name module name, the name can include a specific version number with '==' 

54 @param must_exist if True, raise an exception if not found 

55 @return @see cl ModuleInstall 

56 """ 

57 if isinstance(name, ModuleInstall): 

58 return name 

59 

60 if name in {"pip", "python"}: 

61 return None 

62 

63 if '=' in name: 

64 spl = name.split('==') 

65 if len(spl) != 2: 

66 raise ValueError( 

67 "[find_module_install] unable to interpret " + name) 

68 name = spl[0] 

69 version = spl[1] 

70 else: 

71 version = None 

72 

73 if name in _reverse_module_index: 

74 mod = _reverse_module_index[name] 

75 else: 

76 names = [name[0].upper() + name[1:], 

77 name.replace("-python", ""), 

78 name + "-python"] 

79 if name == "pytables": 

80 names.append("tables") 

81 elif name == "pyopengl-accelerate": 

82 names.append("PyOpenGL_accelerate") 

83 elif name == "pywin32": 

84 names.append("pywin32") # pypiwin32 

85 for n in names: 

86 if n in _reverse_module_index: 

87 return _reverse_module_index[n] 

88 if must_exist: 

89 raise MissingReferenceException( 

90 "Unable to find reference for module '{0}'\nCheck '{0}'".format(name)) 

91 mod = ModuleInstall(name, "pip") 

92 

93 if version is not None: 

94 mod.version = version 

95 return mod 

96 

97 

98def reorder_module_list(list_module): 

99 """ 

100 reorder a list of modules to install, a module at position *p* 

101 should not depend not module at position *p+1* 

102 

103 @param list_module list of module (@see cl ModuleInstall) 

104 @return list of modules 

105 

106 The function uses modules stored in :mod:`pyminstall.packaged.packaged_config`, 

107 it does not go to pypi. If a module was not registered, it will be placed 

108 at the end in the order it was given to this function. 

109 """ 

110 inset = {m.name: m for m in list_module} 

111 res = [] 

112 for mod in all_set(): 

113 if mod.name in inset and inset[mod.name] is not None: 

114 res.append(mod.copy(version=inset[mod.name].version)) 

115 inset[mod.name] = None 

116 for mod in list_module: 

117 if inset[mod.name] is not None: 

118 res.append(mod) 

119 return res 

120 

121 

122class FileShouldNotBeFound(Exception): 

123 """ 

124 Raised by function @see fn check_sys_path. 

125 """ 

126 pass 

127 

128 

129def check_sys_path(): 

130 """ 

131 An issue is happening during unit test as pymyinstall 

132 could be imported from two locations. We check this is 

133 not the case. 

134 """ 

135 exp = "pymyinstall_ut_skip_pyquickhelper_%d%d_std" % sys.version_info[:2] 

136 exp = os.path.join(exp, "src") 

137 for path in sys.path: 

138 if exp in path: 

139 raise FileShouldNotBeFound( 

140 "'{0}' was found in sys.path:\n{1}\n----\ncwd='{2}'\nexe='{3}'".format(path, "\n".join(sys.path), os.getcwd(), sys.executable)) 

141 

142 

143def update_all(temp_folder=".", fLOG=print, verbose=True, 

144 list_module=None, reorder=True, 

145 skip_module=None, schedule_only=False, 

146 source=None, download_only=False): 

147 """ 

148 Updates modules in *list_module* 

149 if None, this list will be returned by @see fn ensae_fullset, 

150 the function starts by updating :epkg:`pip`. 

151 

152 @param temp_folder temporary folder 

153 @param verbose more display 

154 @param list_module None or of list of str or @see cl ModuleInstall or a name set 

155 @param fLOG logging function 

156 @param reorder reorder the modules to update first modules with less dependencies (as much as as possible) 

157 @param skip_module module to skip (list of str) 

158 @param schedule_only if True, the function returns the list of modules scheduled to be installed 

159 @param source overwrite the wheels location, see @see me get_exewheel_url_link2 

160 @param download_only only downloads the module, no installation 

161 

162 *list_module* can be a set of modules of a name set. 

163 Name sets can be accesses by function @see fn get_package_set. 

164 See the function to get the list of possible name sets. 

165 

166 .. versionchanged:: 1.1 

167 Catch an exception while updating modules and walk through the end of the list. 

168 The function should be run a second time to make sure an exception remains. 

169 It can be due to python keeping in memory an updated module. 

170 Parameters *source*, *download_only* were added. 

171 """ 

172 check_sys_path() 

173 if not os.path.exists(temp_folder): 

174 os.makedirs(temp_folder) 

175 if not has_pip(): 

176 fLOG("install pip") 

177 from .get_pip import main # pylint: disable=E0401 

178 main() 

179 

180 if skip_module is None: 

181 skip_module = [] 

182 

183 if list_module is None: 

184 from ..packaged import ensae_fullset 

185 list_module = ensae_fullset() 

186 elif isinstance(list_module, str # unicode# 

187 ): 

188 from .packaged_config import get_package_set 

189 f = get_package_set(list_module) 

190 list_module = f() 

191 else: 

192 list_module = [find_module_install(mod) if isinstance( 

193 mod, str) else mod for mod in list_module] 

194 

195 list_module = [_ for _ in list_module if _.name not in skip_module] 

196 check_sys_path() 

197 

198 if reorder: 

199 list_module = reorder_module_list(list_module) 

200 

201 if verbose: 

202 fLOG("update pip if needed") 

203 

204 if "pip" not in skip_module: 

205 update_pip() 

206 

207 modules = list_module 

208 again = [] 

209 errors = [] 

210 schedule = [] 

211 for mod in modules: 

212 check_sys_path() 

213 is_installed = mod.is_installed_version() 

214 if not is_installed: 

215 continue 

216 if verbose: 

217 fLOG("[update-check] ##", mod.name, "## [begin]") 

218 

219 try: 

220 has_update = mod.has_update() 

221 except (MissingVersionOnPyPiException, MissingPackageOnPyPiException) as e: 

222 # this happens for custom made version such as xgboost 

223 fLOG(" - unable to check updates", e) 

224 has_update = False 

225 if not has_update: 

226 if verbose: 

227 fLOG("[update-check] ##", mod.name, "## [end no update]") 

228 continue 

229 

230 ver = mod.get_pypi_version() 

231 inst = mod.get_installed_version() 

232 

233 schedule.append(mod) 

234 if not schedule_only: 

235 m = " - updating module {0} --- {1} --> {2} (kind={3})" \ 

236 .format(mod.name, inst, ver, mod.kind) 

237 fLOG(m) 

238 try: 

239 if download_only: 

240 b = mod.download(temp_folder=temp_folder, source=source) 

241 else: 

242 b = mod.update(temp_folder=temp_folder, 

243 log=verbose, source=source) 

244 except (SystemExit, Exception) as e: 

245 b = False 

246 m = " - failed to update module {0} --- {1} --> {2} (kind={3}) due to {4} ({5})" \ 

247 .format(mod.name, inst, ver, mod.kind, str(e), type(e)) 

248 fLOG(m) 

249 errors.append((mod, e)) 

250 if b: 

251 again.append(m) 

252 

253 if verbose: 

254 fLOG("[update-check] ##", mod.name, "## [end]") 

255 

256 if schedule_only: 

257 return schedule 

258 

259 if verbose: 

260 fLOG("") 

261 fLOG("updated modules") 

262 for m in again: 

263 fLOG(" ", m) 

264 if len(errors) > 0: 

265 fLOG("failed modules") 

266 for m in errors: 

267 fLOG(" ", m[0], m[1]) 

268 return None 

269 

270 

271def install_all(temp_folder=".", fLOG=print, verbose=True, 

272 list_module=None, reorder=True, skip_module=None, 

273 up_pip=True, skip_missing=False, deps=False, 

274 schedule_only=False, deep_deps=False, 

275 _memory=None, source=None, download_only=False, 

276 force=False): 

277 """ 

278 Installs modules in *list_module* 

279 if None, this list will be returned by @see fn ensae_fullset, 

280 the function starts by updating pip. 

281 

282 @param temp_folder temporary folder 

283 @param verbose more display 

284 @param list_module None or of list of str or @see cl ModuleInstall or a name set 

285 @param fLOG logging function 

286 @param reorder reorder the modules to update first modules with less dependencies (as much as as possible) 

287 @param skip_module module to skip (list of str) 

288 @param up_pip upgrade pip (pip must not be in *skip_module*) 

289 @param skip_missing skip the checking of the missing dependencies 

290 @param deps install the dependencies of the installed modules 

291 @param schedule_only if True, the function returns the list of modules scheduled to be installed 

292 @param deep_deps check dependencies for dependencies 

293 @param _memory stores installed packages, avoid going into an infinite loop 

294 @param source overwrite the location of the wheels, see @see me get_exewheel_url_link2 

295 @param download_only only downloads the module, no installation 

296 @param force force the installation or the download 

297 @return output streams 

298 

299 *list_module* can be a set of modules of a name set. 

300 Name sets can be accesses by function @see fn get_package_set. 

301 See the function to get the list of possible name sets. 

302 

303 About *name set*, the function can install a set of modules but these set of modules have dependencies: 

304 

305 * @see fn extended_set, @see fn sphinx_theme_set requires @see fn small_set 

306 * @see fn ml_set, @see fn ensae_set, @see fn teachings_set, @see fn iot_set, @see fn scraping_set 

307 requires @see fn small_set and @see fn extended_set 

308 

309 If you want to install a specific module with its dependencies, I suggest 

310 to use option *deep_deps*. 

311 

312 .. versionadded:: 1.1 

313 """ 

314 check_sys_path() 

315 fLOG("[install_all] begin, exe:", sys.executable) 

316 if _memory is None: 

317 _memory = {} 

318 deps = deps or deep_deps 

319 if not os.path.exists(temp_folder): 

320 os.makedirs(temp_folder) 

321 if not has_pip(): 

322 fLOG("install pip") 

323 from ..installhelper.get_pip import main 

324 main() 

325 

326 if skip_module is None: 

327 skip_module = [] 

328 

329 if list_module is None: 

330 list_module = all_set() 

331 elif isinstance(list_module, str # unicode# 

332 ): 

333 from .packaged_config import get_package_set 

334 f = get_package_set(list_module) 

335 list_module = f() 

336 else: 

337 list_module = [find_module_install(mod) if isinstance( 

338 mod, str) else mod for mod in list_module] 

339 

340 list_module = [_ for _ in list_module if _.name not in skip_module] 

341 check_sys_path() 

342 

343 if reorder: 

344 list_module = reorder_module_list(list_module) 

345 

346 if up_pip and "pip" not in skip_module: 

347 if verbose: 

348 fLOG("update pip if needed") 

349 update_pip() 

350 

351 # More information. 

352 sorted_names = ", ".join(sorted(_.name.lower() for _ in list_module)) 

353 ordered_names = ", ".join(_.name for _ in list_module) 

354 fLOG('[install_all] sorted modules ----------------------------') 

355 fLOG("\n".join(wrap(sorted_names, width=100, 

356 initial_indent=' ', subsequent_indent=' '))) 

357 fLOG('[install_all] modules installation order ----------------') 

358 fLOG("\n".join(wrap(ordered_names, width=100, 

359 initial_indent=' ', subsequent_indent=' '))) 

360 fLOG('[install_all] begin ----------------------') 

361 

362 modules = list_module 

363 again = [] 

364 errors = [] 

365 installed = [] 

366 schedule = [] 

367 out_streams_module = {} 

368 for mod in modules: 

369 check_sys_path() 

370 if mod.name in _memory: 

371 # already done 

372 continue 

373 if verbose: 

374 fLOG("[install-check] ##", mod.name, "## [begin]") 

375 if force or not mod.is_installed_version(): 

376 schedule.append(mod) 

377 if not schedule_only: 

378 ver = mod.version 

379 m = " - installing module '{0}' --- --> '{1}' (kind='{2}')" \ 

380 .format(mod.name, ver, mod.kind) 

381 fLOG(m) 

382 if deps: 

383 fLOG("[install-check-dep] ##", mod.name, "## [begin]") 

384 install_module_deps(mod.name, temp_folder=temp_folder, 

385 fLOG=fLOG, verbose=verbose, deps=deps, deep_deps=deep_deps, 

386 _memory=_memory, source=source, download_only=download_only, 

387 force=force) 

388 fLOG("[install-check-dep] ##", mod.name, "## [end]") 

389 else: 

390 out_streams = [] 

391 try: 

392 if download_only: 

393 b = mod.download( 

394 temp_folder=temp_folder, source=source) 

395 else: 

396 b = mod.install(temp_folder=temp_folder, 

397 log=verbose, source=source, 

398 out_streams=out_streams) 

399 except (SystemExit, Exception) as e: 

400 b = False 

401 m = " - failed to update module '{0}' --- '{1}' --> '{2}' (kind='{3}') due to '{4}' ('{5}')" \ 

402 .format(mod.name, '', ver, mod.kind, str(e), type(e)) 

403 fLOG(m) 

404 errors.append((mod, e)) 

405 if b: 

406 again.append(m) 

407 installed.append(mod) 

408 if verbose: 

409 ne = 0 

410 for lll in out_streams: 

411 if lll[-1]: 

412 ne += 1 

413 if ne > 0: 

414 fLOG("###################") 

415 fLOG("##", mod.name, "##") 

416 for jj in out_streams: 

417 for la, va in zip(("CMD", "OUT", "ERR-1"), jj): 

418 fLOG(la, va) 

419 fLOG("###################.") 

420 out_streams_module[mod.name] = out_streams 

421 if verbose: 

422 fLOG("[install-check] ##", mod.name, "## [end]") 

423 _memory[mod.name] = True 

424 

425 if schedule_only: 

426 return schedule 

427 

428 if verbose: 

429 fLOG("") 

430 fLOG("[install_all]" + 

431 (" downloaded modules" if download_only else "installed modules")) 

432 for m in again: 

433 fLOG(" ", m) 

434 fLOG("[install_all] end: ", len(errors), "errors") 

435 if len(errors) > 0: 

436 fLOG("[install_all] failed modules") 

437 for m in errors: 

438 fLOG(" ", m[0], m[1]) 

439 fLOG("[install_all] end failed modules") 

440 

441 if not skip_missing: 

442 fLOG("[install_all] check dependencies") 

443 miss = missing_dependencies() 

444 if len(miss) > 0: 

445 mes = "\n".join("Module '{0}' misses '{1}'".format(k, ", ".join(v)) 

446 for k, v in sorted(miss.items())) 

447 warnings.warn("[install_all] missing dependencies\n" + mes) 

448 fLOG("[install_all] end check dependencies") 

449 

450 for k, v in sorted(out_streams_module.items()): 

451 fLOG("[%s] #########################" % k) 

452 if isinstance(v, list): 

453 for _ in v: 

454 for la, va in zip(("CMD", "OUT", "ERR-2"), _): 

455 if va is not None and len(va) > 0: 

456 fLOG(la, va) 

457 else: 

458 for la, va in zip(("CMD", "OUT", "ERR-3"), v): 

459 if va is not None and len(va) > 0: 

460 fLOG(la, va) 

461 fLOG(".") 

462 return out_streams_module 

463 

464 

465def install_module_deps(name, temp_folder=".", fLOG=print, verbose=True, deps=True, 

466 deep_deps=False, _memory=None, source=None, 

467 download_only=False, force=False): 

468 """ 

469 Installs a module with its dependencies, 

470 if a module is already installed, it installs 

471 the missing dependencies. 

472 

473 @param module module name 

474 @param temp_folder where to download 

475 @param fLOG logging function 

476 @param verbose more display 

477 @param deps add dependencies 

478 @param deep_deps check dependencies for dependencies 

479 @param _memory stores installed packages, avoid going into an infinite loop 

480 @param source overwrite the wheels location, see @see me get_exewheel_url_link2 

481 @param download_only only downloads the module, no installation 

482 @param force force the download or the update 

483 @return list of installed modules 

484 

485 The function does not handle properly contraints on versions. 

486 It checks in the registered list of modules if *name* is present. 

487 If it is the case, it uses it, otherwise, it uses *pip*. 

488 

489 .. versionadded:: 1.1 

490 """ 

491 check_sys_path() 

492 if _memory is None: 

493 _memory = {} 

494 deps = deps or deep_deps 

495 installed = [] 

496 first_name = name 

497 memory2 = {} 

498 if not is_installed(name): 

499 install_all(temp_folder=temp_folder, fLOG=fLOG, verbose=verbose, 

500 list_module=[name], reorder=True, up_pip=False, 

501 skip_missing=True, _memory=memory2, deps=False, 

502 deep_deps=False, source=source, download_only=download_only, 

503 force=force) 

504 installed.append(name) 

505 

506 stack = [name] 

507 while len(stack) > 0: 

508 check_sys_path() 

509 name = stack[-1] 

510 del stack[-1] 

511 if name in _memory: 

512 # already done, avoid loops on 

513 continue 

514 if name == first_name: 

515 _memory.update(memory2) 

516 

517 res = get_module_dependencies(name, refresh_cache=True, use_pip=True) 

518 if res is None: 

519 raise ImportError("unable to check dependencies of module {0}\ninstalled:\n{1}".format( 

520 name, "\n".join(installed))) 

521 list_m = [k for k, v in res.items()] 

522 fLOG("[deps] [{}] requires".format(name), list_m) 

523 inst = [k for k in list_m if deep_deps or not is_installed(k)] 

524 fLOG("[deps] installation of ", inst) 

525 install_all(temp_folder=temp_folder, fLOG=fLOG, verbose=verbose, 

526 list_module=inst, reorder=True, up_pip=False, 

527 skip_missing=True, deep_deps=deep_deps, 

528 _memory=_memory, source=source, download_only=download_only) 

529 stack.extend(inst) 

530 installed.extend(inst) 

531 for i in inst: 

532 _memory[i] = True 

533 

534 return list(sorted(set(installed))) 

535 

536 

537def install_module(module_name, temp_folder=".", fLOG=print, verbose=True, 

538 reorder=True, skip_module=None, 

539 up_pip=True, skip_missing=False, deps=False, 

540 schedule_only=False, deep_deps=False, 

541 _memory=None, source=None, download_only=False, 

542 force=False): 

543 """ 

544 Installs a module. 

545 

546 @param module_name module to install 

547 @param temp_folder temporary folder 

548 @param verbose more display 

549 @param fLOG logging function 

550 @param reorder reorder the modules to update first modules with less dependencies (as much as as possible) 

551 @param skip_module module to skip (list of str) 

552 @param up_pip upgrade pip (pip must not be in *skip_module*) 

553 @param skip_missing skip the checking of the missing dependencies 

554 @param deps install the dependencies of the installed modules 

555 @param schedule_only if True, the function returns the list of modules scheduled to be installed 

556 @param deep_deps check dependencies for dependencies 

557 @param _memory stores installed packages, avoid going into an infinite loop 

558 @param source overwrite the wheels location, see @see me get_exewheel_url_link2 

559 @param download_only only downloads the module, no installation 

560 @param force force the download or the install 

561 

562 .. versionadded:: 1.1 

563 """ 

564 if not isinstance(module_name, list): 

565 module_name = [module_name] 

566 install_all(list_module=module_name, temp_folder=temp_folder, fLOG=fLOG, 

567 verbose=verbose, reorder=reorder, skip_module=skip_module, 

568 up_pip=up_pip, skip_missing=skip_missing, deps=deps, 

569 schedule_only=schedule_only, deep_deps=deep_deps, 

570 _memory=_memory, source=source, download_only=download_only, 

571 force=force) 

572 

573 

574def update_module(module_name, temp_folder=".", fLOG=print, verbose=True, 

575 reorder=True, skip_module=None, schedule_only=False, 

576 source=None): 

577 """ 

578 Updates modules in *list_module* 

579 if None, this list will be returned by @see fn ensae_fullset, 

580 the function starts by updating pip. 

581 

582 @param module_name module to update 

583 @param temp_folder temporary folder 

584 @param verbose more display 

585 @param fLOG logging function 

586 @param reorder reorder the modules to update first modules with less dependencies (as much as as possible) 

587 @param skip_module module to skip (list of str) 

588 @param schedule_only if True, the function returns the list of modules scheduled to be installed 

589 @param source overwrite the wheels location, see @see me get_exewheel_url_link2 

590 """ 

591 if not isinstance(module_name, list): 

592 module_name = [module_name] 

593 update_all(list_module=module_name, temp_folder=temp_folder, fLOG=fLOG, verbose=verbose, 

594 reorder=reorder, skip_module=skip_module, schedule_only=schedule_only, 

595 source=source) 

596 

597 

598def download_module(module_name, 

599 temp_folder=".", force=False, 

600 unzipFile=True, file_save=None, deps=False, 

601 source=None, fLOG=print): 

602 """ 

603 Downloads the module without installation. 

604 

605 @param temp_folder destination 

606 @param force force the installation even if already installed 

607 @param unzipFile if it can be unzipped, it will be (for github, mostly) 

608 @param file_save for debug purposes, do not change it unless you know what you are doing 

609 @param deps download the dependencies too (only available for pip) 

610 @param source overwrite the wheels location, see @see me get_exewheel_url_link2 

611 @return downloaded files 

612 """ 

613 if not isinstance(module_name, list): 

614 module_name = [module_name] 

615 

616 res = [] 

617 for mod in module_name: 

618 if not isinstance(mod, ModuleInstall): 

619 mod = find_module_install(mod, True) 

620 if mod is None: 

621 raise ModuleNotFoundError( 

622 "unable to find an objec for module: " + ", ".join(module_name)) 

623 k = mod.fLOG 

624 mod.fLOG = fLOG 

625 f = mod.download(temp_folder=temp_folder, force=force, 

626 unzipFile=unzipFile, file_save=file_save, deps=deps, source=source) 

627 mod.fLOG = k 

628 res.append(f) 

629 return res