Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1""" 

2@file 

3@brief Install or update all packages. 

4 

5.. faqref:: 

6 :title: Skipped modules 

7 

8 Some modules are still marked as a dependency by others 

9 even if they are not needed or cannot be installed. 

10 

11 * `enum-compat <https://pypi.python.org/pypi/enum-compat>`_: 

12 not needed after Python 3.5 

13 * `prettytable <https://pypi.python.org/pypi/PrettyTable>`_: 

14 should be repladed by `PTable <https://pypi.python.org/pypi/PTable>`_. 

15""" 

16from __future__ import print_function 

17import sys 

18import os 

19import warnings 

20import time 

21from textwrap import wrap 

22from ..installhelper import ModuleInstall, has_pip, update_pip, is_installed, get_module_dependencies 

23from ..installhelper.module_install_exceptions import MissingVersionOnPyPiException, MissingPackageOnPyPiException, MissingReferenceException 

24from ..installhelper.module_dependencies import missing_dependencies 

25from .packaged_exception import ModuleNotFoundError 

26from .packaged_config import small_set 

27 

28 

29def _build_reverse_index(): 

30 """ 

31 builds a reverse index of the module, 

32 """ 

33 res = {} 

34 mods = small_set() 

35 for m in mods: 

36 res[m.name] = m 

37 if m.mname is not None and m.mname != m.name: 

38 res[m.mname] = m 

39 if m.name.lower() not in (m.name, m.mname): 

40 res[m.name.lower()] = m 

41 return res 

42 

43 

44_reverse_module_index = _build_reverse_index() 

45 

46 

47def find_module_install(name, must_exist=False): 

48 """ 

49 Checks if there are specific instructions 

50 to run before installing module *name*, 

51 on Windows, some modules requires compilation, 

52 if not uses default option with *pip*. 

53 

54 @param name module name, the name can include a specific version number with '==' 

55 @param must_exist if True, raise an exception if not found 

56 @return @see cl ModuleInstall 

57 """ 

58 if isinstance(name, ModuleInstall): 

59 return name 

60 

61 if name in {"pip", "python"}: 

62 return None 

63 

64 if '=' in name: 

65 spl = name.split('==') 

66 if len(spl) != 2: 

67 raise ValueError( 

68 "[find_module_install] unable to interpret " + name) 

69 name = spl[0] 

70 version = spl[1] 

71 else: 

72 version = None 

73 

74 if name in _reverse_module_index: 

75 mod = _reverse_module_index[name] 

76 else: 

77 names = [name[0].upper() + name[1:], 

78 name.replace("-python", ""), 

79 name + "-python"] 

80 if name == "pytables": 

81 names.append("tables") 

82 elif name == "pyopengl-accelerate": 

83 names.append("PyOpenGL_accelerate") 

84 elif name == "pywin32": 

85 names.append("pywin32") # pypiwin32 

86 for n in names: 

87 if n in _reverse_module_index: 

88 return _reverse_module_index[n] 

89 if must_exist: 

90 raise MissingReferenceException( 

91 "Unable to find reference for module '{0}'\nCheck '{0}'".format(name)) 

92 mod = ModuleInstall(name, "pip") 

93 

94 if version is not None: 

95 mod.version = version 

96 return mod 

97 

98 

99def reorder_module_list(list_module): 

100 """ 

101 reorder a list of modules to install, a module at position *p* 

102 should not depend not module at position *p+1* 

103 

104 @param list_module list of module (@see cl ModuleInstall) 

105 @return list of modules 

106 

107 The function uses modules stored in :mod:`pyminstall.packaged.packaged_config`, 

108 it does not go to pypi. If a module was not registered, it will be placed 

109 at the end in the order it was given to this function. 

110 """ 

111 inset = {m.name: m for m in list_module} 

112 res = [] 

113 for mod in small_set(): 

114 if mod.name in inset and inset[mod.name] is not None: 

115 res.append(mod.copy(version=inset[mod.name].version)) 

116 inset[mod.name] = None 

117 for mod in list_module: 

118 if inset[mod.name] is not None: 

119 res.append(mod) 

120 return res 

121 

122 

123class FileShouldNotBeFound(Exception): 

124 """ 

125 Raised by function @see fn check_sys_path. 

126 """ 

127 pass 

128 

129 

130def check_sys_path(): 

131 """ 

132 An issue is happening during unit test as pymyinstall 

133 could be imported from two locations. We check this is 

134 not the case. 

135 """ 

136 exp = "pymyinstall_ut_skip_pyquickhelper_%d%d_std" % sys.version_info[:2] 

137 exp = os.path.join(exp, "src") 

138 for path in sys.path: 

139 if exp in path: 

140 raise FileShouldNotBeFound( 

141 "'{0}' was found in sys.path:\n{1}\n----\ncwd='{2}'\nexe='{3}'".format(path, "\n".join(sys.path), os.getcwd(), sys.executable)) 

142 

143 

144def update_all(temp_folder=".", fLOG=print, verbose=True, 

145 list_module=None, reorder=True, 

146 skip_module=None, schedule_only=False, 

147 source=None, download_only=False): 

148 """ 

149 Updates modules in *list_module* 

150 if None, this list will be returned by @see fn ensae_fullset, 

151 the function starts by updating :epkg:`pip`. 

152 

153 @param temp_folder temporary folder 

154 @param verbose more display 

155 @param list_module None or of list of str or @see cl ModuleInstall or a name set 

156 @param fLOG logging function 

157 @param reorder reorder the modules to update first modules with less dependencies (as much as as possible) 

158 @param skip_module module to skip (list of str) 

159 @param schedule_only if True, the function returns the list of modules scheduled to be installed 

160 @param source overwrite the wheels location, see @see me get_exewheel_url_link2 

161 @param download_only only downloads the module, no installation 

162 

163 *list_module* can be a set of modules of a name set. 

164 Name sets can be accesses by function @see fn get_package_set. 

165 See the function to get the list of possible name sets. 

166 

167 .. versionchanged:: 1.1 

168 Catch an exception while updating modules and walk through the end of the list. 

169 The function should be run a second time to make sure an exception remains. 

170 It can be due to python keeping in memory an updated module. 

171 Parameters *source*, *download_only* were added. 

172 """ 

173 check_sys_path() 

174 if not os.path.exists(temp_folder): 

175 os.makedirs(temp_folder) 

176 if not has_pip(): 

177 fLOG("install pip") 

178 from .get_pip import main # pylint: disable=E0401 

179 main() 

180 

181 if skip_module is None: 

182 skip_module = [] 

183 

184 if list_module is None: 

185 from ..packaged import small_set 

186 list_module = small_set() 

187 elif isinstance(list_module, str): 

188 from .packaged_config import get_package_set 

189 f = get_package_set(list_module) 

190 list_module = f() 

191 else: 

192 list_module = [find_module_install(mod) if isinstance( 

193 mod, str) else mod for mod in list_module] 

194 

195 list_module = [_ for _ in list_module if _.name not in skip_module] 

196 check_sys_path() 

197 

198 if reorder: 

199 list_module = reorder_module_list(list_module) 

200 

201 if verbose: 

202 fLOG("update pip if needed") 

203 

204 if "pip" not in skip_module: 

205 update_pip() 

206 

207 modules = list_module 

208 again = [] 

209 errors = [] 

210 schedule = [] 

211 for mod in modules: 

212 check_sys_path() 

213 is_installed = mod.is_installed_version() 

214 if not is_installed: 

215 continue 

216 if verbose: 

217 fLOG("[wait] 0.5 seconds") 

218 time.sleep(0.5) 

219 if verbose: 

220 fLOG("[update-check] ##", mod.name, "## [begin]") 

221 

222 try: 

223 has_update = mod.has_update() 

224 except (MissingVersionOnPyPiException, MissingPackageOnPyPiException) as e: 

225 # this happens for custom made version such as xgboost 

226 fLOG(" - unable to check updates", e) 

227 has_update = False 

228 if not has_update: 

229 if verbose: 

230 fLOG("[update-check] ##", mod.name, "## [end no update]") 

231 continue 

232 

233 ver = mod.get_pypi_version() 

234 inst = mod.get_installed_version() 

235 

236 schedule.append(mod) 

237 if not schedule_only: 

238 m = " - updating module {0} --- {1} --> {2} (kind={3})" \ 

239 .format(mod.name, inst, ver, mod.kind) 

240 fLOG(m) 

241 try: 

242 if download_only: 

243 b = mod.download(temp_folder=temp_folder, source=source) 

244 else: 

245 b = mod.update(temp_folder=temp_folder, 

246 log=verbose, source=source) 

247 except (SystemExit, Exception) as e: 

248 b = False 

249 m = " - failed to update module {0} --- {1} --> {2} (kind={3}) due to {4} ({5})" \ 

250 .format(mod.name, inst, ver, mod.kind, str(e), type(e)) 

251 fLOG(m) 

252 errors.append((mod, e)) 

253 if b: 

254 again.append(m) 

255 

256 if verbose: 

257 fLOG("[update-check] ##", mod.name, "## [end]") 

258 

259 if schedule_only: 

260 return schedule 

261 

262 if verbose: 

263 fLOG("") 

264 fLOG("updated modules") 

265 for m in again: 

266 fLOG(" ", m) 

267 if len(errors) > 0: 

268 fLOG("failed modules") 

269 for m in errors: 

270 fLOG(" ", m[0], m[1]) 

271 return None 

272 

273 

274def install_all(temp_folder=".", fLOG=print, verbose=True, 

275 list_module=None, reorder=True, skip_module=None, 

276 up_pip=True, skip_missing=False, deps=False, 

277 schedule_only=False, deep_deps=False, 

278 _memory=None, source=None, download_only=False, 

279 force=False): 

280 """ 

281 Installs modules in *list_module* 

282 if None, this list will be returned by @see fn ensae_fullset, 

283 the function starts by updating pip. 

284 

285 @param temp_folder temporary folder 

286 @param verbose more display 

287 @param list_module None or of list of str or @see cl ModuleInstall or a name set 

288 @param fLOG logging function 

289 @param reorder reorder the modules to update first modules with less dependencies (as much as as possible) 

290 @param skip_module module to skip (list of str) 

291 @param up_pip upgrade pip (pip must not be in *skip_module*) 

292 @param skip_missing skip the checking of the missing dependencies 

293 @param deps install the dependencies of the installed modules 

294 @param schedule_only if True, the function returns the list of modules scheduled to be installed 

295 @param deep_deps check dependencies for dependencies 

296 @param _memory stores installed packages, avoid going into an infinite loop 

297 @param source overwrite the location of the wheels, see @see me get_exewheel_url_link2 

298 @param download_only only downloads the module, no installation 

299 @param force force the installation or the download 

300 @return output streams 

301 

302 *list_module* can be a set of modules of a name set. 

303 Name sets can be accesses by function @see fn get_package_set. 

304 See the function to get the list of possible name sets. 

305 

306 About *name set*, the function can install a set of modules but these set of modules have dependencies: 

307 

308 * @see fn extended_set, @see fn sphinx_theme_set requires @see fn small_set 

309 * @see fn ml_set, @see fn ensae_set, @see fn teachings_set, @see fn iot_set, @see fn scraping_set 

310 requires @see fn small_set and @see fn extended_set 

311 

312 If you want to install a specific module with its dependencies, I suggest 

313 to use option *deep_deps*. 

314 

315 .. versionadded:: 1.1 

316 """ 

317 check_sys_path() 

318 fLOG("[install_all] begin, exe:", sys.executable) 

319 if _memory is None: 

320 _memory = {} 

321 deps = deps or deep_deps 

322 if not os.path.exists(temp_folder): 

323 os.makedirs(temp_folder) 

324 if not has_pip(): 

325 fLOG("install pip") 

326 from ..installhelper.get_pip import main 

327 main() 

328 

329 if skip_module is None: 

330 skip_module = [] 

331 

332 if list_module is None: 

333 list_module = small_set() 

334 elif isinstance(list_module, str # unicode# 

335 ): 

336 from .packaged_config import get_package_set 

337 f = get_package_set(list_module) 

338 list_module = f() 

339 else: 

340 list_module = [find_module_install(mod) if isinstance( 

341 mod, str) else mod for mod in list_module] 

342 

343 list_module = [_ for _ in list_module if _.name not in skip_module] 

344 check_sys_path() 

345 

346 if reorder: 

347 list_module = reorder_module_list(list_module) 

348 

349 if up_pip and "pip" not in skip_module: 

350 if verbose: 

351 fLOG("update pip if needed") 

352 update_pip() 

353 

354 # More information. 

355 sorted_names = ", ".join(sorted(_.name.lower() for _ in list_module)) 

356 ordered_names = ", ".join(_.name for _ in list_module) 

357 fLOG('[install_all] sorted modules ----------------------------') 

358 fLOG("\n".join(wrap(sorted_names, width=100, 

359 initial_indent=' ', subsequent_indent=' '))) 

360 fLOG('[install_all] modules installation order ----------------') 

361 fLOG("\n".join(wrap(ordered_names, width=100, 

362 initial_indent=' ', subsequent_indent=' '))) 

363 fLOG('[install_all] begin ----------------------') 

364 

365 modules = list_module 

366 again = [] 

367 errors = [] 

368 installed = [] 

369 schedule = [] 

370 out_streams_module = {} 

371 for mod in modules: 

372 check_sys_path() 

373 if mod.name in _memory: 

374 # already done 

375 continue 

376 if verbose: 

377 fLOG("[install-check] ##", mod.name, "## [begin]") 

378 if force or not mod.is_installed_version(): 

379 schedule.append(mod) 

380 if not schedule_only: 

381 ver = mod.version 

382 m = " - installing module '{0}' --- --> '{1}' (kind='{2}')" \ 

383 .format(mod.name, ver, mod.kind) 

384 fLOG(m) 

385 if deps: 

386 fLOG("[install-check-dep] ##", mod.name, "## [begin]") 

387 install_module_deps(mod.name, temp_folder=temp_folder, 

388 fLOG=fLOG, verbose=verbose, deps=deps, deep_deps=deep_deps, 

389 _memory=_memory, source=source, download_only=download_only, 

390 force=force) 

391 fLOG("[install-check-dep] ##", mod.name, "## [end]") 

392 else: 

393 out_streams = [] 

394 try: 

395 if download_only: 

396 b = mod.download( 

397 temp_folder=temp_folder, source=source) 

398 else: 

399 b = mod.install(temp_folder=temp_folder, 

400 log=verbose, source=source, 

401 out_streams=out_streams) 

402 except (SystemExit, Exception) as e: 

403 b = False 

404 m = " - failed to update module '{0}' --- '{1}' --> '{2}' (kind='{3}') due to '{4}' ('{5}')" \ 

405 .format(mod.name, '', ver, mod.kind, str(e), type(e)) 

406 fLOG(m) 

407 errors.append((mod, e)) 

408 if b: 

409 again.append(m) 

410 installed.append(mod) 

411 if verbose: 

412 ne = 0 

413 for lll in out_streams: 

414 if lll[-1]: 

415 ne += 1 

416 if ne > 0: 

417 fLOG("###################") 

418 fLOG("##", mod.name, "##") 

419 for jj in out_streams: 

420 for la, va in zip(("CMD", "OUT", "ERR-1"), jj): 

421 fLOG(la, va) 

422 fLOG("###################.") 

423 out_streams_module[mod.name] = out_streams 

424 if verbose: 

425 fLOG("[install-check] ##", mod.name, "## [end]") 

426 _memory[mod.name] = True 

427 

428 if schedule_only: 

429 return schedule 

430 

431 if verbose: 

432 fLOG("") 

433 fLOG("[install_all]" + 

434 (" downloaded modules" if download_only else "installed modules")) 

435 for m in again: 

436 fLOG(" ", m) 

437 fLOG("[install_all] end: ", len(errors), "errors") 

438 if len(errors) > 0: 

439 fLOG("[install_all] failed modules") 

440 for m in errors: 

441 fLOG(" ", m[0], m[1]) 

442 fLOG("[install_all] end failed modules") 

443 

444 if not skip_missing: 

445 fLOG("[install_all] check dependencies") 

446 miss = missing_dependencies() 

447 if len(miss) > 0: 

448 mes = "\n".join("Module '{0}' misses '{1}'".format(k, ", ".join(v)) 

449 for k, v in sorted(miss.items())) 

450 warnings.warn("[install_all] missing dependencies\n" + mes) 

451 fLOG("[install_all] end check dependencies") 

452 

453 for k, v in sorted(out_streams_module.items()): 

454 fLOG("[%s] #########################" % k) 

455 if isinstance(v, list): 

456 for _ in v: 

457 for la, va in zip(("CMD", "OUT", "ERR-2"), _): 

458 if va is not None and len(va) > 0: 

459 fLOG(la, va) 

460 else: 

461 for la, va in zip(("CMD", "OUT", "ERR-3"), v): 

462 if va is not None and len(va) > 0: 

463 fLOG(la, va) 

464 fLOG(".") 

465 return out_streams_module 

466 

467 

468def install_module_deps(name, temp_folder=".", fLOG=print, verbose=True, deps=True, 

469 deep_deps=False, _memory=None, source=None, 

470 download_only=False, force=False): 

471 """ 

472 Installs a module with its dependencies, 

473 if a module is already installed, it installs 

474 the missing dependencies. 

475 

476 @param module module name 

477 @param temp_folder where to download 

478 @param fLOG logging function 

479 @param verbose more display 

480 @param deps add dependencies 

481 @param deep_deps check dependencies for dependencies 

482 @param _memory stores installed packages, avoid going into an infinite loop 

483 @param source overwrite the wheels location, see @see me get_exewheel_url_link2 

484 @param download_only only downloads the module, no installation 

485 @param force force the download or the update 

486 @return list of installed modules 

487 

488 The function does not handle properly contraints on versions. 

489 It checks in the registered list of modules if *name* is present. 

490 If it is the case, it uses it, otherwise, it uses *pip*. 

491 

492 .. versionadded:: 1.1 

493 """ 

494 check_sys_path() 

495 if _memory is None: 

496 _memory = {} 

497 deps = deps or deep_deps 

498 installed = [] 

499 first_name = name 

500 memory2 = {} 

501 if not is_installed(name): 

502 install_all(temp_folder=temp_folder, fLOG=fLOG, verbose=verbose, 

503 list_module=[name], reorder=True, up_pip=False, 

504 skip_missing=True, _memory=memory2, deps=False, 

505 deep_deps=False, source=source, download_only=download_only, 

506 force=force) 

507 installed.append(name) 

508 

509 stack = [name] 

510 while len(stack) > 0: 

511 check_sys_path() 

512 name = stack[-1] 

513 del stack[-1] 

514 if name in _memory: 

515 # already done, avoid loops on 

516 continue 

517 if name == first_name: 

518 _memory.update(memory2) 

519 

520 res = get_module_dependencies(name, refresh_cache=True, use_pip=True) 

521 if res is None: 

522 raise ImportError("unable to check dependencies of module {0}\ninstalled:\n{1}".format( 

523 name, "\n".join(installed))) 

524 list_m = [k for k, v in res.items()] 

525 fLOG("[deps] [{}] requires".format(name), list_m) 

526 inst = [k for k in list_m if deep_deps or not is_installed(k)] 

527 fLOG("[deps] installation of ", inst) 

528 install_all(temp_folder=temp_folder, fLOG=fLOG, verbose=verbose, 

529 list_module=inst, reorder=True, up_pip=False, 

530 skip_missing=True, deep_deps=deep_deps, 

531 _memory=_memory, source=source, download_only=download_only) 

532 stack.extend(inst) 

533 installed.extend(inst) 

534 for i in inst: 

535 _memory[i] = True 

536 

537 return list(sorted(set(installed))) 

538 

539 

540def install_module(module_name, temp_folder=".", fLOG=print, verbose=True, 

541 reorder=True, skip_module=None, 

542 up_pip=True, skip_missing=False, deps=False, 

543 schedule_only=False, deep_deps=False, 

544 _memory=None, source=None, download_only=False, 

545 force=False): 

546 """ 

547 Installs a module. 

548 

549 @param module_name module to install 

550 @param temp_folder temporary folder 

551 @param verbose more display 

552 @param fLOG logging function 

553 @param reorder reorder the modules to update first modules with less dependencies (as much as as possible) 

554 @param skip_module module to skip (list of str) 

555 @param up_pip upgrade pip (pip must not be in *skip_module*) 

556 @param skip_missing skip the checking of the missing dependencies 

557 @param deps install the dependencies of the installed modules 

558 @param schedule_only if True, the function returns the list of modules scheduled to be installed 

559 @param deep_deps check dependencies for dependencies 

560 @param _memory stores installed packages, avoid going into an infinite loop 

561 @param source overwrite the wheels location, see @see me get_exewheel_url_link2 

562 @param download_only only downloads the module, no installation 

563 @param force force the download or the install 

564 

565 .. versionadded:: 1.1 

566 """ 

567 if not isinstance(module_name, list): 

568 module_name = [module_name] 

569 install_all(list_module=module_name, temp_folder=temp_folder, fLOG=fLOG, 

570 verbose=verbose, reorder=reorder, skip_module=skip_module, 

571 up_pip=up_pip, skip_missing=skip_missing, deps=deps, 

572 schedule_only=schedule_only, deep_deps=deep_deps, 

573 _memory=_memory, source=source, download_only=download_only, 

574 force=force) 

575 

576 

577def update_module(module_name, temp_folder=".", fLOG=print, verbose=True, 

578 reorder=True, skip_module=None, schedule_only=False, 

579 source=None): 

580 """ 

581 Updates modules in *list_module* 

582 if None, this list will be returned by @see fn ensae_fullset, 

583 the function starts by updating pip. 

584 

585 @param module_name module to update 

586 @param temp_folder temporary folder 

587 @param verbose more display 

588 @param fLOG logging function 

589 @param reorder reorder the modules to update first modules with less dependencies (as much as as possible) 

590 @param skip_module module to skip (list of str) 

591 @param schedule_only if True, the function returns the list of modules scheduled to be installed 

592 @param source overwrite the wheels location, see @see me get_exewheel_url_link2 

593 """ 

594 if not isinstance(module_name, list): 

595 module_name = [module_name] 

596 update_all(list_module=module_name, temp_folder=temp_folder, fLOG=fLOG, verbose=verbose, 

597 reorder=reorder, skip_module=skip_module, schedule_only=schedule_only, 

598 source=source) 

599 

600 

601def download_module(module_name, 

602 temp_folder=".", force=False, 

603 unzipFile=True, file_save=None, deps=False, 

604 source=None, fLOG=print): 

605 """ 

606 Downloads the module without installation. 

607 

608 @param temp_folder destination 

609 @param force force the installation even if already installed 

610 @param unzipFile if it can be unzipped, it will be (for github, mostly) 

611 @param file_save for debug purposes, do not change it unless you know what you are doing 

612 @param deps download the dependencies too (only available for pip) 

613 @param source overwrite the wheels location, see @see me get_exewheel_url_link2 

614 @return downloaded files 

615 """ 

616 if not isinstance(module_name, list): 

617 module_name = [module_name] 

618 

619 res = [] 

620 for mod in module_name: 

621 if not isinstance(mod, ModuleInstall): 

622 mod = find_module_install(mod, True) 

623 if mod is None: 

624 raise ModuleNotFoundError( 

625 "unable to find an objec for module: " + ", ".join(module_name)) 

626 k = mod.fLOG 

627 mod.fLOG = fLOG 

628 f = mod.download(temp_folder=temp_folder, force=force, 

629 unzipFile=unzipFile, file_save=file_save, deps=deps, source=source) 

630 mod.fLOG = k 

631 res.append(f) 

632 return res