Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2@file
3@brief Functions to get module version, license, dependencies
4"""
5import sys
6import re
7import warnings
8import functools
9import time
10import xmlrpc.client as xmlrpc_client
11from .install_cmd_helper import run_cmd, get_pip_program
12from .module_install_exceptions import MissingPackageOnPyPiException, AnnoyingPackageException
13from .module_install_exceptions import ConfigurationError, MissingVersionOnPyPiException, WrongVersionError, MissingVersionWheelException
14from .install_cmd_regex import regex_wheel_versions
15from .pip_helper import get_installed_distributions
18annoying_modules = {"pygame", "liblinear", "mlpy", "VideoCapture",
19 "libsvm", "opencv_python", "scikits.cuda",
20 "NLopt"}
23_get_module_version_manual_memoize = {}
26def get_module_version(module, use_cmd=False):
27 """
28 Returns a dictionary ``{ module: version }``.
30 @param module unused, None
31 @param use_cmd use command line
32 @return dictionary
33 """
34 if module is not None:
35 modl = module.lower()
36 res = get_module_version(None, use_cmd=use_cmd)
37 return res.get(modl, None)
39 global _get_module_version_manual_memoize # pylint: disable=W0602
40 if len(_get_module_version_manual_memoize) > 0:
41 return _get_module_version_manual_memoize
43 res = {}
45 if use_cmd:
46 prog = get_pip_program()
47 cmd = prog + " list"
48 out, err = run_cmd(cmd, wait=True, fLOG=None)
50 if err is not None and len(err) > 0:
51 if len(err.split("\n")) > 3 or \
52 "You should consider upgrading via the 'pip install --upgrade pip' command." not in err:
53 raise Exception("unable to run, #lines {0}\nCMD:\n{3}\nERR-J:\n{1}\nOUT:\n{2}".format(
54 len(err.split("\n")), err, out, cmd))
55 lines = out.split("\n")
57 for line in lines:
58 if "(" in line:
59 spl = line.split()
60 if len(spl) == 2:
61 a = spl[0]
62 b = spl[1].strip(" \n\r")
63 res[a] = b.strip("()")
64 al = a.lower()
65 if al != a:
66 res[al] = res[a]
67 else:
68 # local_only must be False to get all modules
69 # not only the ones installed in the virtual environment
70 dist = get_installed_distributions(local_only=False)
71 if len(dist) == 0:
72 raise ConfigurationError("no installed module, unexpected (pip should be there): " +
73 "sys.executable={0}, sys.prefix={1}, sys.base_prefix={2}".format(
74 sys.executable, sys.prefix, sys.base_prefix))
75 for mod in dist:
76 al = mod.key.lower()
77 a = mod.key
78 try:
79 v = mod.version
80 except ValueError:
81 v = "UNKNOWN"
82 res[a] = v
83 if a != al:
84 res[al] = v
86 _get_module_version_manual_memoize.update(res)
87 return res
90def is_installed(name):
91 """
92 Tells if a module is installed or not.
94 @param name module name
95 @return boolean
96 """
97 return get_module_version(name) is not None
100_get_module_metadata_manual_memoize = {}
103def get_module_metadata(module, use_cmd=False, refresh_cache=False):
104 """
105 Returns a dictionary ``{ module: metadata }``.
107 @param module unused, None
108 @param refresh_cache refresh the cache before getting metadata
109 @return dictionary
110 """
111 if module is not None:
112 modl = module.lower()
113 res = get_module_metadata(
114 None, use_cmd=use_cmd, refresh_cache=refresh_cache)
115 return res.get(modl, None)
117 global _get_module_metadata_manual_memoize # pylint: disable=W0602
118 if not refresh_cache and len(_get_module_metadata_manual_memoize) > 0:
119 return _get_module_metadata_manual_memoize
121 # local_only must be False to get all modules
122 # not only the ones installed in the virtual environment
123 dist = get_installed_distributions(local_only=False, use_cmd=use_cmd)
124 if len(dist) == 0:
125 raise ConfigurationError("no installed module, unexpected (pip should be there): " +
126 "sys.executable={0}, sys.prefix={1}, sys.base_prefix={2}".format(
127 sys.executable, sys.prefix, sys.base_prefix))
128 res = {}
129 for mod in dist:
130 d = {}
131 lines = mod._get_metadata(mod.PKG_INFO)
132 for line in lines:
133 if sys.version_info[0] == 2:
134 typstr = str # unicode#
135 if not isinstance(line, typstr):
136 line = typstr(line, encoding="utf8", errors="ignore")
137 try:
138 spl = line.split(":")
139 except UnicodeDecodeError:
140 warnings.warn("UnicodeDecodeError with: " + line)
141 continue
142 key = spl[0].strip()
143 value = ":".join(spl[1:]).strip()
144 if key not in d:
145 d[key] = value
146 else:
147 if not isinstance(d[key], list):
148 d[key] = [d[key]]
149 d[key].append(value)
151 a = mod.key
152 res[a] = d
153 al = mod.key.lower()
154 if a != al:
155 res[al] = d
157 _get_module_metadata_manual_memoize.update(res)
158 return res
161def _get_pypi_version_memoize_op(f):
162 memo = {}
164 def helper(module_name, full_list=False, url="https://pypi.python.org/pypi"):
165 key = module_name, full_list, url
166 if key not in memo:
167 memo[key] = f(module_name, full_list, url)
168 return memo[key]
169 return helper
172_get_pypi_version_memoize = {}
175def get_pypi_version(module_name, full_list=False, url="https://pypi.python.org/pypi", skip_betas=True):
176 """
177 Returns the version of a package on :epkg:`pypi`,
178 we skip alpha, beta or dev version.
180 @param module_name module name
181 @param url pypi server
182 @param full_list results as a list or return the last stable version
183 @param skip_betas skip the intermediate functions
184 @return version (str or list)
186 See also `installing_python_packages_programatically.py <https://gist.github.com/rwilcox/755524>`_,
187 `pkgtools.pypi: PyPI interface <http://pkgtools.readthedocs.org/en/latest/pypi.html>`_.
189 It the function fails, check the status of
190 `Python Infrastructure <https://status.python.org/>`_.
191 It can return errors::
193 ProtocolError: ProtocolError for pypi.python.org/pypi: 503 No healthy backends
194 """
196 global _get_pypi_version_memoize # pylint: disable=W0602
197 key = module_name, full_list, url
198 if key in _get_pypi_version_memoize:
199 available = _get_pypi_version_memoize[key]
200 if full_list:
201 return available
202 elif available is not None and len(available) > 0:
203 return available[0]
204 return None
206 def pypi_package_releases(module_name, b):
207 nbtry = 0
208 while nbtry < 40:
209 try:
210 available = pypi.package_releases(module_name, True)
211 return available
212 except Exception as e:
213 if ("HTTPTooManyRequests" in str(type(e)) or
214 "HTTPTooManyRequests" in str(e)):
215 nbtry += 1
216 warnings.warn(str(e))
217 time.sleep(90)
218 continue
219 if ("TimeoutError" in str(type(e)) or
220 "TimeoutError" in str(e)):
221 nbtry += 1
222 warnings.warn(str(e))
223 time.sleep(20)
224 continue
225 raise e
226 return None
228 def _inside_loop_(pypi, module_name, tried):
230 available = pypi_package_releases(module_name, True)
232 if available is None or len(available) == 0:
233 ntry = module_name.capitalize()
234 if ntry not in tried:
235 tried.append(ntry)
236 available = pypi_package_releases(tried[-1], True)
238 if available is None or len(available) == 0:
239 ntry = module_name.replace("-", "_")
240 if ntry not in tried:
241 tried.append(ntry)
242 available = pypi_package_releases(tried[-1], True)
244 if available is None or len(available) == 0:
245 ntry = module_name.replace("_", "-")
246 if ntry not in tried:
247 tried.append(ntry)
248 available = pypi_package_releases(tried[-1], True)
250 if available is None or len(available) == 0:
251 ntry = module_name.lower()
252 if ntry not in tried:
253 tried.append(ntry)
254 available = pypi_package_releases(tried[-1], True)
256 if available is None or len(available) == 0:
257 ml = module_name.lower()
258 if ml == "markupsafe":
259 tried.append("MarkupSafe")
260 available = pypi_package_releases(tried[-1], True)
261 elif ml == "flask-sqlalchemy":
262 tried.append("Flask-SQLAlchemy")
263 available = pypi_package_releases(tried[-1], True)
264 elif ml == "apscheduler":
265 tried.append("APScheduler")
266 available = pypi_package_releases(tried[-1], True)
267 elif ml == "datashape":
268 tried.append("DataShape")
269 available = pypi_package_releases(tried[-1], True)
270 elif ml == "pycontracts":
271 tried.append("PyContracts")
272 available = pypi_package_releases(tried[-1], True)
273 elif ml == "pybrain":
274 tried.append("PyBrain")
275 available = pypi_package_releases(tried[-1], True)
276 elif ml == "pyexecjs":
277 tried.append("PyExecJS")
278 available = pypi_package_releases(tried[-1], True)
279 elif ml == "dataspyre":
280 tried.append("DataSpyre")
281 available = pypi_package_releases(tried[-1], True)
282 elif ml == "heapdict":
283 tried.append("HeapDict")
284 available = pypi_package_releases(tried[-1], True)
285 elif ml == "pyreact":
286 tried.append("PyReact")
287 available = pypi_package_releases(tried[-1], True)
288 elif ml == "qtpy":
289 tried.append("QtPy")
290 available = pypi_package_releases(tried[-1], True)
291 elif ml == "pythonqwt":
292 tried.append("PythonQwt")
293 available = pypi_package_releases(tried[-1], True)
294 elif ml == "onedrive-sdk-python":
295 tried.append("onedrivesdk")
296 available = pypi_package_releases(tried[-1], True)
297 elif ml.startswith("orange3-"):
298 s = ml.split("-")[1]
299 ntry = "Orange3-" + s[0].upper() + s[1:]
300 tried.append(ntry)
301 available = pypi_package_releases(tried[-1], True)
302 elif module_name in annoying_modules:
303 raise AnnoyingPackageException(module_name)
305 # this raises a warning about an opened connection
306 # see documentation of the function
307 # del pypi
309 return available
311 tried = [module_name]
313 if sys.version_info[:2] <= (3, 4):
314 # the client does not an implemented of __exit__ for version <= 3.4
315 pypi = xmlrpc_client.ServerProxy(url)
316 available = _inside_loop_(pypi, module_name, tried)
317 else:
318 with xmlrpc_client.ServerProxy(url) as pypi:
319 available = _inside_loop_(pypi, module_name, tried)
321 if available is None or len(available) == 0:
322 raise MissingPackageOnPyPiException("tried:\n" + "\n".join(tried))
324 def filter_betas(a):
325 spl = a.split(".")
326 if len(spl) in (2, 3):
327 last = spl[-1]
328 if not skip_betas or ("a" not in last and "b" not in last and "dev" not in last):
329 return True
330 else:
331 # we don't really know here, so we assume it is not
332 return True
333 return False
335 if available:
336 available2 = list(filter(filter_betas, available))
337 if available2:
338 _get_pypi_version_memoize[key] = available2
339 if full_list:
340 return available2
341 return available2[0]
343 raise MissingVersionOnPyPiException(
344 "{0}\nversion:\n{1}".format(module_name, "\n".join(available)))
347def numeric_version(vers):
348 """
349 convert a string into a tuple with numbers wherever possible
351 @param vers string
352 @return tuple
353 """
354 if isinstance(vers, tuple):
355 return vers
356 if isinstance(vers, list):
357 raise Exception("unexpected value:" + str(vers))
358 spl = str(vers).split(".")
359 r = []
360 for _ in spl:
361 try:
362 i = int(_)
363 r.append(i)
364 except ValueError:
365 r.append(_)
366 return tuple(r)
369def compare_version(num, vers):
370 """
371 Compares two versions.
373 @param num first version
374 @param vers second version
375 @return -1, 0, 1
376 """
377 if num is None:
378 if vers is None:
379 return 0
380 return 1
381 if vers is None:
382 return -1
384 if not isinstance(vers, tuple):
385 vers = numeric_version(vers)
386 if not isinstance(num, tuple):
387 num = numeric_version(num)
389 if len(num) == len(vers):
390 for a, b in zip(num, vers):
391 if isinstance(a, int) and isinstance(b, int):
392 if a < b:
393 return -1
394 if a > b:
395 return 1
396 else:
397 a = str(a)
398 b = str(b)
399 if a < b:
400 return -1
401 if a > b:
402 return 1
403 return 0
405 if len(num) < len(vers):
406 num = num + (0,) * (len(vers) - len(num))
407 return compare_version(num, vers)
408 vers = vers + (0,) * (len(num) - len(vers))
409 return compare_version(num, vers)
412def version_consensus(v1, v2):
413 """
414 *v1* and *v2* are two versions of the same module, which one to keep?
416 @param v1 version 1
417 @param v2 version 2
418 @return consensus
420 * ``v1=None``, ``v2='(>=1.5)'`` --> ``v='>=1.5'``
422 To improve:
424 * ``v1='<=1.6'``, ``v2='(>=1.5)'`` --> ``v='==1.6'``
425 """
426 reg = re.compile("([><=]*)([^><=]+)")
428 def process_version(v):
429 if isinstance(v, str # unicode#
430 ):
431 v = v.strip('()')
432 find = reg.search(v)
433 if not find:
434 raise WrongVersionError(v)
435 sign = find.groups()[0]
436 number = numeric_version(find.groups()[1])
437 else:
438 try:
439 sign, number = v
440 except ValueError as e:
441 raise ValueError("weird format: " + str(v) +
442 " - " + str(type(v))) from e
443 return sign, number
445 if v1 is None:
446 return v2
447 if v2 is None:
448 return v1
450 s1, n1 = process_version(v1)
451 s2, n2 = process_version(v2)
453 if s1 not in ('<=', '==', '<', '>', '>='):
454 raise ValueError("wrong sign {0} for v1={1}".format(s1, v1))
455 if s2 not in ('<=', '==', '<', '>', '>='):
456 raise ValueError("wrong sign {0} for v1={1}".format(s2, v2))
458 if s1 == "==":
459 if s2 == "==":
460 if compare_version(n1, n2) != 0:
461 raise WrongVersionError(
462 "incompatible version: {0}{1} and {2}{3}".format(s1, n1, s2, n2))
463 else:
464 res = s1, n1
466 elif s1 == "<=":
467 if s2 == "<=":
468 res = s1, min(n1, n2)
469 elif s2 == "==":
470 if compare_version(n1, n2) < 0:
471 raise WrongVersionError(
472 "incompatible version: {0}{1} and {2}{3}".format(s1, n1, s2, n2))
473 res = s2, n2
474 elif s2 == '<':
475 if compare_version(n1, n2) == -1:
476 res = s1, n1
477 else:
478 res = s2, n2
479 elif s2 in ('>', '>='):
480 if compare_version(n1, n2) <= 0:
481 raise WrongVersionError(
482 "incompatible version: {0}{1} and {2}{3}".format(s1, n1, s2, n2))
483 res = s1, n1
485 elif s1 == "<":
486 if s2 == "<":
487 res = s1, min(n1, n2)
488 elif s2 == "==":
489 if compare_version(n1, n2) <= 0:
490 raise WrongVersionError(
491 "incompatible version: {0}{1} and {2}{3}".format(s1, n1, s2, n2))
492 res = s2, n2
493 elif s2 == '<=':
494 if compare_version(n1, n2) <= 0:
495 res = s1, n1
496 else:
497 res = s2, n2
498 elif s2 in ('>', '>='):
499 if compare_version(n1, n2) <= 0:
500 raise WrongVersionError(
501 "incompatible version: {0}{1} and {2}{3}".format(s1, n1, s2, n2))
502 res = s1, n1
504 elif s1 == ">=":
505 if s2 == ">=":
506 res = s1, max(n1, n2)
507 elif s2 == "==":
508 if compare_version(n1, n2) == -1:
509 raise WrongVersionError(
510 "incompatible version: {0}{1} and {2}{3}".format(s1, n1, s2, n2))
511 res = s2, n2
512 elif s2 == '>':
513 if compare_version(n1, n2) <= 0:
514 res = s2, n2
515 else:
516 res = s1, n1
517 elif s2 in ('<', '<='):
518 if compare_version(n1, n2) >= 0:
519 raise WrongVersionError(
520 "incompatible version: {0}{1} and {2}{3}".format(s1, n1, s2, n2))
521 res = s2, n2
523 elif s1 == ">":
524 if s2 == ">":
525 res = s1, max(n1, n2)
526 elif s2 == "==":
527 if compare_version(n1, n2) >= 0:
528 raise WrongVersionError(
529 "incompatible version: {0}{1} and {2}{3}".format(s1, n1, s2, n2))
530 res = s2, n2
531 elif s2 == '>=':
532 if compare_version(n1, n2) == -1:
533 res = s2, n2
534 else:
535 res = s1, n1
536 elif s2 in ('<', '<='):
537 if compare_version(n1, n2) == 1:
538 raise WrongVersionError(
539 "incompatible version: {0}{1} and {2}{3}".format(s1, n1, s2, n2))
540 res = s2, n2
541 else:
542 res = None, None
544 if res[0] is None:
545 raise WrongVersionError(
546 "incompatible version and wrong format: {0}{1} and {2}{3}".format(s1, n1, s2, n2))
548 return '{0}{1}'.format(res[0], '.'.join(str(_) for _ in res[1]))
551_get_module_dependencies_deps = None
554def get_module_dependencies(module, use_cmd=False, deep=False, collapse=True, use_pip=None, refresh_cache=False):
555 """
556 Returns the dependencies for a module.
558 @param module unused, None
559 @param use_cmd use command line
560 @param deep dig into dependencies of dependencies
561 @param collapse only one row per module
562 @param use_pip use pip to discover dependencies or not (parse metadata)
563 @param refresh_cache refresh the cache (see below)
564 @return list of tuple (module, version, required by as a str)
565 or dictionary { module: (version, required by as a list) } if *collapse* is True
567 The function which uses *use_pip=True* is not fully tested, it does not
568 return contraints (== 2.4). The function caches the results to avoid doing it again
569 during a second execution unless *refresh_cache* is True.
570 This function is not tested on Python 2.7.
571 """
572 if use_pip is None:
573 use_pip = not sys.platform.startswith("win")
575 def evaluate_condition(cond, full):
576 # example python_version=="3.3" or python_version=="2.7" and extra ==
577 # \'test\'
578 python_version = ".".join(str(_) for _ in sys.version_info[:3])
579 extra = ""
580 sys_platform = sys.platform
581 try:
582 return eval(cond)
583 except Exception:
584 if "python_version" not in cond and "extra" not in cond:
585 # probably something like cycler (>=0.10)
586 # we don't check that
587 return True
588 else:
589 mes = "Unable to evaluate condition '{0}' from '{1}', extra='{2}', python_version='{3}', sys_platform='{4}'.".format(
590 cond, full, extra, python_version, sys_platform)
591 raise Exception(mes)
593 if use_pip:
594 global _get_module_dependencies_deps
595 if _get_module_dependencies_deps is None or refresh_cache:
596 temp = get_installed_distributions(
597 local_only=False, skip=[], use_cmd=use_cmd)
598 _get_module_dependencies_deps = dict(
599 (p.key, (p, p.requires())) for p in temp)
600 if module not in _get_module_dependencies_deps:
601 raise ValueError("module {0} was not installed".format(module))
602 res = []
603 req = _get_module_dependencies_deps[module][1]
604 if isinstance(req, list):
605 for r in req:
606 res.append((r.key, None, module))
607 else:
608 res.append((req.key, None, module))
609 else:
610 meta = get_module_metadata(
611 module, use_cmd, refresh_cache=refresh_cache)
612 if meta is None:
613 raise ImportError(
614 "unable to get metadata for module '{0}' - refresh_cache={1}".format(module, refresh_cache))
615 deps = [v for k, v in meta.items()
616 if "Requires" in k and "Requires-Python" not in k]
617 res = []
618 for d in deps:
619 if not isinstance(d, list):
620 dl = [d]
621 else:
622 dl = d
623 for v in dl:
624 spl = v.split()
625 if len(spl) > 1:
626 spl = [spl[0], " ".join(spl[1:])]
627 if len(spl) == 1:
628 key = (v, None, module)
629 else:
630 conds = spl[1].split(";")
631 ok = [evaluate_condition(cond, v) for cond in conds]
632 if not all(ok):
633 continue
634 key = (spl[0].strip(";"), spl[1], module)
635 if key not in res:
636 res.append(key)
638 # specific filters
639 def validate_module(name):
640 if name == "enum34" and sys.version_info[:2] > (3, ):
641 raise NameError(
642 "Unexpected dependency '{0}' for module '{1}'.".format(name, module))
643 if name == "configparser":
644 raise NameError(
645 "Unexpected dependency '{0}' for module '{1}'.".format(name, module))
646 return True
648 res = [key for key in res if validate_module(key[0])]
650 if deep:
651 done = {module: None}
652 mod = 1
653 while mod > 0:
654 mod = 0
655 for r in res:
656 if r[0] not in done:
657 if r[0].lower() < 'a' or r[0].lower() > 'z' or r[0].endswith(";"):
658 raise NameError(
659 "A module has an unexpected name '{0}', r={1} when looking for dependencies of '{2}'.".format(r[0], r, module))
660 temp = get_module_dependencies(
661 r[0], use_cmd=use_cmd, deep=deep, collapse=False, use_pip=use_pip, refresh_cache=refresh_cache)
662 for key in temp:
663 if key not in res:
664 res.append(key)
665 mod += 1
666 done[r[0]] = None
668 if collapse:
669 final = {}
670 for name, version, required in res:
671 if name not in final:
672 final[name] = (version, [required])
673 else:
674 ex = final[name][1]
675 if required not in ex:
676 ex.append(required)
677 try:
678 v = version_consensus(final[name][0], version)
679 except WrongVersionError as e:
680 raise WrongVersionError("unable to reconcile versions:\n{0}\n{1}".format(
681 ex, str((name, version, required)))) from e
682 final[name] = (v, ex)
683 final = {k: (v[0], list(sorted(v[1]))) for k, v in final.items()}
684 return final
686 return [(name, version.strip('()') if version is not None else version, required)
687 for name, version, required in res]
690def choose_most_recent(list_name):
691 """
692 Chooses the most recent version for a list of module names.
694 @param list_name list of names
695 @return most recent version or None if the input list is empty
697 In the following case, we would choose the first option::
699 numpy-1.10.0+mkl-cp34-none-win_amd64.whl
700 numpy-1.9.1.0+mkl-cp34-none-win_amd64.whl
701 """
702 def find_wheel(tu):
703 for t in tu:
704 if ".whl" in t:
705 return t
706 raise ValueError("unable to find a wheel in {0}".format(tu))
707 if len(list_name) == 0:
708 return None
709 if isinstance(list_name[0], tuple):
710 list_name = [(find_wheel(_), _) for _ in list_name]
711 else:
712 list_name = [(_, _) for _ in list_name]
714 versions = [re.compile(_) for _ in regex_wheel_versions]
716 def search_regex(_):
717 resv = None
718 for version in versions:
719 try:
720 resv = version.search(_[0])
721 except TypeError as e:
722 raise TypeError("Unable to parse '{0}'".format(_)) from e
723 if resv is not None:
724 return resv
725 raise MissingVersionWheelException(
726 "Unable to get version number for module '{}':\nREGEX\n{}".format(_, "\n".join(regex_wheel_versions)))
728 list_name = [(search_regex(_).groups()[0], _[0], _[1])
729 for _ in list_name]
731 def cmp(el1, el2):
732 return compare_version(el1[0], el2[0])
734 list_name = list(sorted(list_name, key=functools.cmp_to_key(cmp)))
735 return list_name[-1][-1]
738def get_wheel_version(whlname):
739 """
740 extract the version from a wheel file,
741 return ``2.6.0`` for ``rpy2-2.6.0-cp34-none-win_amd64.whl``
743 @param whlname file name
744 @return string
745 """
746 find = []
747 for reg in regex_wheel_versions:
748 if len(find) == 0:
749 exp = re.compile(reg)
750 find = exp.findall(whlname)
751 else:
752 break
753 if len(find) == 0:
754 mes = "Unable to extract version for '{0}'\nREGEX\n{1}"
755 raise ValueError(mes.format(whlname, "\n".join(regex_wheel_versions)))
756 if len(find) > 1:
757 mes = "Too many options for '{0}'\nOPTIONS\n{1}\nREGEX\n{2}"
758 raise ValueError(mes.format(
759 whlname, find, "\n".join(regex_wheel_versions)))
760 if isinstance(find[0], tuple):
761 return find[0][0]
762 else:
763 return find[0]