Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2@file
3@brief Install or update all packages.
5.. faqref::
6 :title: Skipped modules
8 Some modules are still marked as a dependency by others
9 even if they are not needed or cannot be installed.
11 * `enum-compat <https://pypi.python.org/pypi/enum-compat>`_:
12 not needed after Python 3.5
13 * `prettytable <https://pypi.python.org/pypi/PrettyTable>`_:
14 should be repladed by `PTable <https://pypi.python.org/pypi/PTable>`_.
15"""
16from __future__ import print_function
17import sys
18import os
19import warnings
20import time
21from textwrap import wrap
22from ..installhelper import ModuleInstall, has_pip, update_pip, is_installed, get_module_dependencies
23from ..installhelper.module_install_exceptions import MissingVersionOnPyPiException, MissingPackageOnPyPiException, MissingReferenceException
24from ..installhelper.module_dependencies import missing_dependencies
25from .packaged_exception import ModuleNotFoundError
26from .packaged_config import small_set
29def _build_reverse_index():
30 """
31 builds a reverse index of the module,
32 """
33 res = {}
34 mods = small_set()
35 for m in mods:
36 res[m.name] = m
37 if m.mname is not None and m.mname != m.name:
38 res[m.mname] = m
39 if m.name.lower() not in (m.name, m.mname):
40 res[m.name.lower()] = m
41 return res
44_reverse_module_index = _build_reverse_index()
47def find_module_install(name, must_exist=False):
48 """
49 Checks if there are specific instructions
50 to run before installing module *name*,
51 on Windows, some modules requires compilation,
52 if not uses default option with *pip*.
54 @param name module name, the name can include a specific version number with '=='
55 @param must_exist if True, raise an exception if not found
56 @return @see cl ModuleInstall
57 """
58 if isinstance(name, ModuleInstall):
59 return name
61 if name in {"pip", "python"}:
62 return None
64 if '=' in name:
65 spl = name.split('==')
66 if len(spl) != 2:
67 raise ValueError(
68 "[find_module_install] unable to interpret " + name)
69 name = spl[0]
70 version = spl[1]
71 else:
72 version = None
74 if name in _reverse_module_index:
75 mod = _reverse_module_index[name]
76 else:
77 names = [name[0].upper() + name[1:],
78 name.replace("-python", ""),
79 name + "-python"]
80 if name == "pytables":
81 names.append("tables")
82 elif name == "pyopengl-accelerate":
83 names.append("PyOpenGL_accelerate")
84 elif name == "pywin32":
85 names.append("pywin32") # pypiwin32
86 for n in names:
87 if n in _reverse_module_index:
88 return _reverse_module_index[n]
89 if must_exist:
90 raise MissingReferenceException(
91 "Unable to find reference for module '{0}'\nCheck '{0}'".format(name))
92 mod = ModuleInstall(name, "pip")
94 if version is not None:
95 mod.version = version
96 return mod
99def reorder_module_list(list_module):
100 """
101 reorder a list of modules to install, a module at position *p*
102 should not depend not module at position *p+1*
104 @param list_module list of module (@see cl ModuleInstall)
105 @return list of modules
107 The function uses modules stored in :mod:`pyminstall.packaged.packaged_config`,
108 it does not go to pypi. If a module was not registered, it will be placed
109 at the end in the order it was given to this function.
110 """
111 inset = {m.name: m for m in list_module}
112 res = []
113 for mod in small_set():
114 if mod.name in inset and inset[mod.name] is not None:
115 res.append(mod.copy(version=inset[mod.name].version))
116 inset[mod.name] = None
117 for mod in list_module:
118 if inset[mod.name] is not None:
119 res.append(mod)
120 return res
123class FileShouldNotBeFound(Exception):
124 """
125 Raised by function @see fn check_sys_path.
126 """
127 pass
130def check_sys_path():
131 """
132 An issue is happening during unit test as pymyinstall
133 could be imported from two locations. We check this is
134 not the case.
135 """
136 exp = "pymyinstall_ut_skip_pyquickhelper_%d%d_std" % sys.version_info[:2]
137 exp = os.path.join(exp, "src")
138 for path in sys.path:
139 if exp in path:
140 raise FileShouldNotBeFound(
141 "'{0}' was found in sys.path:\n{1}\n----\ncwd='{2}'\nexe='{3}'".format(path, "\n".join(sys.path), os.getcwd(), sys.executable))
144def update_all(temp_folder=".", fLOG=print, verbose=True,
145 list_module=None, reorder=True,
146 skip_module=None, schedule_only=False,
147 source=None, download_only=False):
148 """
149 Updates modules in *list_module*
150 if None, this list will be returned by @see fn ensae_fullset,
151 the function starts by updating :epkg:`pip`.
153 @param temp_folder temporary folder
154 @param verbose more display
155 @param list_module None or of list of str or @see cl ModuleInstall or a name set
156 @param fLOG logging function
157 @param reorder reorder the modules to update first modules with less dependencies (as much as as possible)
158 @param skip_module module to skip (list of str)
159 @param schedule_only if True, the function returns the list of modules scheduled to be installed
160 @param source overwrite the wheels location, see @see me get_exewheel_url_link2
161 @param download_only only downloads the module, no installation
163 *list_module* can be a set of modules of a name set.
164 Name sets can be accesses by function @see fn get_package_set.
165 See the function to get the list of possible name sets.
167 .. versionchanged:: 1.1
168 Catch an exception while updating modules and walk through the end of the list.
169 The function should be run a second time to make sure an exception remains.
170 It can be due to python keeping in memory an updated module.
171 Parameters *source*, *download_only* were added.
172 """
173 check_sys_path()
174 if not os.path.exists(temp_folder):
175 os.makedirs(temp_folder)
176 if not has_pip():
177 fLOG("install pip")
178 from .get_pip import main # pylint: disable=E0401
179 main()
181 if skip_module is None:
182 skip_module = []
184 if list_module is None:
185 from ..packaged import small_set
186 list_module = small_set()
187 elif isinstance(list_module, str):
188 from .packaged_config import get_package_set
189 f = get_package_set(list_module)
190 list_module = f()
191 else:
192 list_module = [find_module_install(mod) if isinstance(
193 mod, str) else mod for mod in list_module]
195 list_module = [_ for _ in list_module if _.name not in skip_module]
196 check_sys_path()
198 if reorder:
199 list_module = reorder_module_list(list_module)
201 if verbose:
202 fLOG("update pip if needed")
204 if "pip" not in skip_module:
205 update_pip()
207 modules = list_module
208 again = []
209 errors = []
210 schedule = []
211 for mod in modules:
212 check_sys_path()
213 is_installed = mod.is_installed_version()
214 if not is_installed:
215 continue
216 if verbose:
217 fLOG("[wait] 0.5 seconds")
218 time.sleep(0.5)
219 if verbose:
220 fLOG("[update-check] ##", mod.name, "## [begin]")
222 try:
223 has_update = mod.has_update()
224 except (MissingVersionOnPyPiException, MissingPackageOnPyPiException) as e:
225 # this happens for custom made version such as xgboost
226 fLOG(" - unable to check updates", e)
227 has_update = False
228 if not has_update:
229 if verbose:
230 fLOG("[update-check] ##", mod.name, "## [end no update]")
231 continue
233 ver = mod.get_pypi_version()
234 inst = mod.get_installed_version()
236 schedule.append(mod)
237 if not schedule_only:
238 m = " - updating module {0} --- {1} --> {2} (kind={3})" \
239 .format(mod.name, inst, ver, mod.kind)
240 fLOG(m)
241 try:
242 if download_only:
243 b = mod.download(temp_folder=temp_folder, source=source)
244 else:
245 b = mod.update(temp_folder=temp_folder,
246 log=verbose, source=source)
247 except (SystemExit, Exception) as e:
248 b = False
249 m = " - failed to update module {0} --- {1} --> {2} (kind={3}) due to {4} ({5})" \
250 .format(mod.name, inst, ver, mod.kind, str(e), type(e))
251 fLOG(m)
252 errors.append((mod, e))
253 if b:
254 again.append(m)
256 if verbose:
257 fLOG("[update-check] ##", mod.name, "## [end]")
259 if schedule_only:
260 return schedule
262 if verbose:
263 fLOG("")
264 fLOG("updated modules")
265 for m in again:
266 fLOG(" ", m)
267 if len(errors) > 0:
268 fLOG("failed modules")
269 for m in errors:
270 fLOG(" ", m[0], m[1])
271 return None
274def install_all(temp_folder=".", fLOG=print, verbose=True,
275 list_module=None, reorder=True, skip_module=None,
276 up_pip=True, skip_missing=False, deps=False,
277 schedule_only=False, deep_deps=False,
278 _memory=None, source=None, download_only=False,
279 force=False):
280 """
281 Installs modules in *list_module*
282 if None, this list will be returned by @see fn ensae_fullset,
283 the function starts by updating pip.
285 @param temp_folder temporary folder
286 @param verbose more display
287 @param list_module None or of list of str or @see cl ModuleInstall or a name set
288 @param fLOG logging function
289 @param reorder reorder the modules to update first modules with less dependencies (as much as as possible)
290 @param skip_module module to skip (list of str)
291 @param up_pip upgrade pip (pip must not be in *skip_module*)
292 @param skip_missing skip the checking of the missing dependencies
293 @param deps install the dependencies of the installed modules
294 @param schedule_only if True, the function returns the list of modules scheduled to be installed
295 @param deep_deps check dependencies for dependencies
296 @param _memory stores installed packages, avoid going into an infinite loop
297 @param source overwrite the location of the wheels, see @see me get_exewheel_url_link2
298 @param download_only only downloads the module, no installation
299 @param force force the installation or the download
300 @return output streams
302 *list_module* can be a set of modules of a name set.
303 Name sets can be accesses by function @see fn get_package_set.
304 See the function to get the list of possible name sets.
306 About *name set*, the function can install a set of modules but these set of modules have dependencies:
308 * @see fn extended_set, @see fn sphinx_theme_set requires @see fn small_set
309 * @see fn ml_set, @see fn ensae_set, @see fn teachings_set, @see fn iot_set, @see fn scraping_set
310 requires @see fn small_set and @see fn extended_set
312 If you want to install a specific module with its dependencies, I suggest
313 to use option *deep_deps*.
315 .. versionadded:: 1.1
316 """
317 check_sys_path()
318 fLOG("[install_all] begin, exe:", sys.executable)
319 if _memory is None:
320 _memory = {}
321 deps = deps or deep_deps
322 if not os.path.exists(temp_folder):
323 os.makedirs(temp_folder)
324 if not has_pip():
325 fLOG("install pip")
326 from ..installhelper.get_pip import main
327 main()
329 if skip_module is None:
330 skip_module = []
332 if list_module is None:
333 list_module = small_set()
334 elif isinstance(list_module, str # unicode#
335 ):
336 from .packaged_config import get_package_set
337 f = get_package_set(list_module)
338 list_module = f()
339 else:
340 list_module = [find_module_install(mod) if isinstance(
341 mod, str) else mod for mod in list_module]
343 list_module = [_ for _ in list_module if _.name not in skip_module]
344 check_sys_path()
346 if reorder:
347 list_module = reorder_module_list(list_module)
349 if up_pip and "pip" not in skip_module:
350 if verbose:
351 fLOG("update pip if needed")
352 update_pip()
354 # More information.
355 sorted_names = ", ".join(sorted(_.name.lower() for _ in list_module))
356 ordered_names = ", ".join(_.name for _ in list_module)
357 fLOG('[install_all] sorted modules ----------------------------')
358 fLOG("\n".join(wrap(sorted_names, width=100,
359 initial_indent=' ', subsequent_indent=' ')))
360 fLOG('[install_all] modules installation order ----------------')
361 fLOG("\n".join(wrap(ordered_names, width=100,
362 initial_indent=' ', subsequent_indent=' ')))
363 fLOG('[install_all] begin ----------------------')
365 modules = list_module
366 again = []
367 errors = []
368 installed = []
369 schedule = []
370 out_streams_module = {}
371 for mod in modules:
372 check_sys_path()
373 if mod.name in _memory:
374 # already done
375 continue
376 if verbose:
377 fLOG("[install-check] ##", mod.name, "## [begin]")
378 if force or not mod.is_installed_version():
379 schedule.append(mod)
380 if not schedule_only:
381 ver = mod.version
382 m = " - installing module '{0}' --- --> '{1}' (kind='{2}')" \
383 .format(mod.name, ver, mod.kind)
384 fLOG(m)
385 if deps:
386 fLOG("[install-check-dep] ##", mod.name, "## [begin]")
387 install_module_deps(mod.name, temp_folder=temp_folder,
388 fLOG=fLOG, verbose=verbose, deps=deps, deep_deps=deep_deps,
389 _memory=_memory, source=source, download_only=download_only,
390 force=force)
391 fLOG("[install-check-dep] ##", mod.name, "## [end]")
392 else:
393 out_streams = []
394 try:
395 if download_only:
396 b = mod.download(
397 temp_folder=temp_folder, source=source)
398 else:
399 b = mod.install(temp_folder=temp_folder,
400 log=verbose, source=source,
401 out_streams=out_streams)
402 except (SystemExit, Exception) as e:
403 b = False
404 m = " - failed to update module '{0}' --- '{1}' --> '{2}' (kind='{3}') due to '{4}' ('{5}')" \
405 .format(mod.name, '', ver, mod.kind, str(e), type(e))
406 fLOG(m)
407 errors.append((mod, e))
408 if b:
409 again.append(m)
410 installed.append(mod)
411 if verbose:
412 ne = 0
413 for lll in out_streams:
414 if lll[-1]:
415 ne += 1
416 if ne > 0:
417 fLOG("###################")
418 fLOG("##", mod.name, "##")
419 for jj in out_streams:
420 for la, va in zip(("CMD", "OUT", "ERR-1"), jj):
421 fLOG(la, va)
422 fLOG("###################.")
423 out_streams_module[mod.name] = out_streams
424 if verbose:
425 fLOG("[install-check] ##", mod.name, "## [end]")
426 _memory[mod.name] = True
428 if schedule_only:
429 return schedule
431 if verbose:
432 fLOG("")
433 fLOG("[install_all]" +
434 (" downloaded modules" if download_only else "installed modules"))
435 for m in again:
436 fLOG(" ", m)
437 fLOG("[install_all] end: ", len(errors), "errors")
438 if len(errors) > 0:
439 fLOG("[install_all] failed modules")
440 for m in errors:
441 fLOG(" ", m[0], m[1])
442 fLOG("[install_all] end failed modules")
444 if not skip_missing:
445 fLOG("[install_all] check dependencies")
446 miss = missing_dependencies()
447 if len(miss) > 0:
448 mes = "\n".join("Module '{0}' misses '{1}'".format(k, ", ".join(v))
449 for k, v in sorted(miss.items()))
450 warnings.warn("[install_all] missing dependencies\n" + mes)
451 fLOG("[install_all] end check dependencies")
453 for k, v in sorted(out_streams_module.items()):
454 fLOG("[%s] #########################" % k)
455 if isinstance(v, list):
456 for _ in v:
457 for la, va in zip(("CMD", "OUT", "ERR-2"), _):
458 if va is not None and len(va) > 0:
459 fLOG(la, va)
460 else:
461 for la, va in zip(("CMD", "OUT", "ERR-3"), v):
462 if va is not None and len(va) > 0:
463 fLOG(la, va)
464 fLOG(".")
465 return out_streams_module
468def install_module_deps(name, temp_folder=".", fLOG=print, verbose=True, deps=True,
469 deep_deps=False, _memory=None, source=None,
470 download_only=False, force=False):
471 """
472 Installs a module with its dependencies,
473 if a module is already installed, it installs
474 the missing dependencies.
476 @param module module name
477 @param temp_folder where to download
478 @param fLOG logging function
479 @param verbose more display
480 @param deps add dependencies
481 @param deep_deps check dependencies for dependencies
482 @param _memory stores installed packages, avoid going into an infinite loop
483 @param source overwrite the wheels location, see @see me get_exewheel_url_link2
484 @param download_only only downloads the module, no installation
485 @param force force the download or the update
486 @return list of installed modules
488 The function does not handle properly contraints on versions.
489 It checks in the registered list of modules if *name* is present.
490 If it is the case, it uses it, otherwise, it uses *pip*.
492 .. versionadded:: 1.1
493 """
494 check_sys_path()
495 if _memory is None:
496 _memory = {}
497 deps = deps or deep_deps
498 installed = []
499 first_name = name
500 memory2 = {}
501 if not is_installed(name):
502 install_all(temp_folder=temp_folder, fLOG=fLOG, verbose=verbose,
503 list_module=[name], reorder=True, up_pip=False,
504 skip_missing=True, _memory=memory2, deps=False,
505 deep_deps=False, source=source, download_only=download_only,
506 force=force)
507 installed.append(name)
509 stack = [name]
510 while len(stack) > 0:
511 check_sys_path()
512 name = stack[-1]
513 del stack[-1]
514 if name in _memory:
515 # already done, avoid loops on
516 continue
517 if name == first_name:
518 _memory.update(memory2)
520 res = get_module_dependencies(name, refresh_cache=True, use_pip=True)
521 if res is None:
522 raise ImportError("unable to check dependencies of module {0}\ninstalled:\n{1}".format(
523 name, "\n".join(installed)))
524 list_m = [k for k, v in res.items()]
525 fLOG("[deps] [{}] requires".format(name), list_m)
526 inst = [k for k in list_m if deep_deps or not is_installed(k)]
527 fLOG("[deps] installation of ", inst)
528 install_all(temp_folder=temp_folder, fLOG=fLOG, verbose=verbose,
529 list_module=inst, reorder=True, up_pip=False,
530 skip_missing=True, deep_deps=deep_deps,
531 _memory=_memory, source=source, download_only=download_only)
532 stack.extend(inst)
533 installed.extend(inst)
534 for i in inst:
535 _memory[i] = True
537 return list(sorted(set(installed)))
540def install_module(module_name, temp_folder=".", fLOG=print, verbose=True,
541 reorder=True, skip_module=None,
542 up_pip=True, skip_missing=False, deps=False,
543 schedule_only=False, deep_deps=False,
544 _memory=None, source=None, download_only=False,
545 force=False):
546 """
547 Installs a module.
549 @param module_name module to install
550 @param temp_folder temporary folder
551 @param verbose more display
552 @param fLOG logging function
553 @param reorder reorder the modules to update first modules with less dependencies (as much as as possible)
554 @param skip_module module to skip (list of str)
555 @param up_pip upgrade pip (pip must not be in *skip_module*)
556 @param skip_missing skip the checking of the missing dependencies
557 @param deps install the dependencies of the installed modules
558 @param schedule_only if True, the function returns the list of modules scheduled to be installed
559 @param deep_deps check dependencies for dependencies
560 @param _memory stores installed packages, avoid going into an infinite loop
561 @param source overwrite the wheels location, see @see me get_exewheel_url_link2
562 @param download_only only downloads the module, no installation
563 @param force force the download or the install
565 .. versionadded:: 1.1
566 """
567 if not isinstance(module_name, list):
568 module_name = [module_name]
569 install_all(list_module=module_name, temp_folder=temp_folder, fLOG=fLOG,
570 verbose=verbose, reorder=reorder, skip_module=skip_module,
571 up_pip=up_pip, skip_missing=skip_missing, deps=deps,
572 schedule_only=schedule_only, deep_deps=deep_deps,
573 _memory=_memory, source=source, download_only=download_only,
574 force=force)
577def update_module(module_name, temp_folder=".", fLOG=print, verbose=True,
578 reorder=True, skip_module=None, schedule_only=False,
579 source=None):
580 """
581 Updates modules in *list_module*
582 if None, this list will be returned by @see fn ensae_fullset,
583 the function starts by updating pip.
585 @param module_name module to update
586 @param temp_folder temporary folder
587 @param verbose more display
588 @param fLOG logging function
589 @param reorder reorder the modules to update first modules with less dependencies (as much as as possible)
590 @param skip_module module to skip (list of str)
591 @param schedule_only if True, the function returns the list of modules scheduled to be installed
592 @param source overwrite the wheels location, see @see me get_exewheel_url_link2
593 """
594 if not isinstance(module_name, list):
595 module_name = [module_name]
596 update_all(list_module=module_name, temp_folder=temp_folder, fLOG=fLOG, verbose=verbose,
597 reorder=reorder, skip_module=skip_module, schedule_only=schedule_only,
598 source=source)
601def download_module(module_name,
602 temp_folder=".", force=False,
603 unzipFile=True, file_save=None, deps=False,
604 source=None, fLOG=print):
605 """
606 Downloads the module without installation.
608 @param temp_folder destination
609 @param force force the installation even if already installed
610 @param unzipFile if it can be unzipped, it will be (for github, mostly)
611 @param file_save for debug purposes, do not change it unless you know what you are doing
612 @param deps download the dependencies too (only available for pip)
613 @param source overwrite the wheels location, see @see me get_exewheel_url_link2
614 @return downloaded files
615 """
616 if not isinstance(module_name, list):
617 module_name = [module_name]
619 res = []
620 for mod in module_name:
621 if not isinstance(mod, ModuleInstall):
622 mod = find_module_install(mod, True)
623 if mod is None:
624 raise ModuleNotFoundError(
625 "unable to find an objec for module: " + ", ".join(module_name))
626 k = mod.fLOG
627 mod.fLOG = fLOG
628 f = mod.download(temp_folder=temp_folder, force=force,
629 unzipFile=unzipFile, file_save=file_save, deps=deps, source=source)
630 mod.fLOG = k
631 res.append(f)
632 return res