"""
Various function to install various python module from various location.
:githublink:`%|py|5`
"""
from __future__ import print_function
import sys
import re
import os
import time
import importlib
import datetime
import warnings
from pip import __version__ as pip_version
from .install_cmd_helper import python_version, run_cmd, unzip_files, get_pip_program
from .install_cmd_helper import get_python_program, get_file_modification_date, get_conda_program, is_conda_distribution
from .module_install_exceptions import MissingPackageOnPyPiException, MissingInstalledPackageException, InstallError
from .module_install_exceptions import DownloadError, MissingVersionWheelException, WrongWheelException, MissingWheelException
from .module_install_version import get_pypi_version, get_module_version, annoying_modules, get_module_metadata
from .module_install_version import numeric_version, compare_version, choose_most_recent, get_wheel_version
from .module_install_page_wheel import get_page_wheel, read_page_wheel, save_page_wheel, enumerate_links_module, extract_all_links
from .missing_license import missing_module_licenses
from .internet_settings import default_user_agent
from .install_cmd_regex import regex_wheel_versions
if sys.version_info[0] == 2:
from urlparse import urlsplit
import urllib2 as urllib_request
import urllib2 as urllib_error
import xmlrpclib as xmlrpc_client
from codecs import open
FileNotFoundError = Exception
PermissionError = Exception
else:
from urllib.parse import urlsplit
import urllib.request as urllib_request
import urllib.error as urllib_error
import importlib.util
import xmlrpc.client as xmlrpc_client
[docs]def _filter_pip_out(out):
lines = out.split('\n')
res = []
for line in lines:
if "WARNING:" in line:
continue
if "Consider adding this directory to PATH" in line:
continue
res.append(line)
return "\n".join(res).strip(' \n\r\t')
[docs]class ModuleInstall:
"""
defines the necessary information for a module
.. exref::
:title: Installation from GitHub
::
ModuleInstall("pyquickhelper", "github",
"sdpython").install(temp_folder="temp")
:githublink:`%|py|65`
"""
allowedKind = ["pip", "github", "exe", "exe2", "wheel", "wheel2"]
exeLocation = "https://www.lfd.uci.edu/~gohlke/pythonlibs/"
exeLocationXd_Default = "http://www.xavierdupre.fr/enseignement/setup/"
gitexe = r"C:\Program Files (x86)\Git"
github_pattern_zip = "https://github.com/{1}/{0}/archive/{2}.zip"
github_pattern_git = "https://github.com/{1}/{0}.git{2}"
[docs] @staticmethod
def is_annoying(module_name):
"""
some modules are not available on pipy
:githublink:`%|py|78`
"""
return module_name in annoying_modules
[docs] def __init__(self, name, kind="pip", gitrepo=None, mname=None, fLOG=print,
version=None, script=None, index_url=None, deps=None,
purpose=None, usage=None, web=None, source=None, custom=None,
branch="master", pip_options=None, overwrite=None, post=None,
skip_import=False, pipgit=False):
"""
:param name: name
:param kind: kind of installation (*pip*, *github*, *wheel*)
:param gitrepo: github repository (example: sdpython)
:param mname: sometimes, the module name is different from its official name
:param version: to install a specific version (None for the latest)
:param fLOG: logging function
:param script: some extensions are not a module but an application (such as ``spyder``),
the class will check this script is available
:param deps: overwrite deps parameters when installing the module
:param index_url: to get the package from a custom pypi server
:param purpose: purpose of the module
:param usage: main usage for the module
:param web: website for the module, if None, default to pipy
:param source: to overwrite parameter *source* of methods
:meth:`download <pymyinstall.installhelper.module_install.ModuleInstall.download>`, :meth:`install` or :meth:`update`.
:param custom: custom instructions to install, usually
``['build', 'install']`` to run
``setup.py build`` and ``setup.py install``
:param branch: only necessary for install process with github
:param pip_options: additional options for pip (list)
:param overwrite: overwrite the location of the wheel
:param post: instructions post installation (look for this parameter
in the code to see what is supported)
:param pipgit: install the module with ``pip + git`` instead of
getting the full archive
:param skip_import: added to indicate the module cannot be imported
.. versionchanged:: 1.1
Parameters *source*, *custom*, *branch*, *pip_options*, *overwrite*, *post* were added.
Parameter *skip_import* was introduced to skip the checking of the installation.
Parameter *pipgit* was added.
:githublink:`%|py|118`
"""
if kind != "pip" and version is not None:
raise NotImplementedError(
"version can be only specified if kind=='pip'")
self.name = name
self.kind = kind
self.gitrepo = gitrepo
self.version = version
self.mname = mname
self.script = script
self.index_url = index_url
self.deps = deps
self.purpose = purpose
self.usage = usage
self.existing_version = None
self.source = source
self.custom = custom
self.branch = branch
self.pip_options = pip_options
self.overwrite = overwrite
self.post_installation = post
self.pipgit = pipgit
if self.mname == self.name:
raise ValueError(
"Do not specify mname if it is equal to name '{0}'.".format(self.name))
if self.pipgit and self.gitrepo is None:
raise ValueError("If pipgit=True, gitrepo must be specified.")
if self.pipgit and kind != 'github':
raise ValueError(
"If pipgit=True, kind must be 'github' not '{0}'.".format(kind))
if self.version is not None and self.gitrepo is not None:
raise ValueError("version must be None if gitrepo is not.")
if self.mname is not None and self.mname.startswith("-"):
self.mname = self.mname[1:]
self.skip_import = True
else:
self.skip_import = skip_import
self.web = web if web is not None else (
"https://pypi.python.org/pypi/" + self.name)
if self.kind not in ModuleInstall.allowedKind:
raise Exception(
"unable to interpret kind {0}, it should be in {1}".format(
kind, ",".join(
ModuleInstall.allowedKind)))
if self.kind == "github" and self.gitrepo is None:
raise Exception("gitrepo cannot be empty")
self.fLOG = fLOG
[docs] def copy(self, version=None):
"""
copy the module, if version is not None, change the version number
:param version: version number or None for unchanged
:return: :class:`ModuleInstall <pymyinstall.installhelper.module_install.ModuleInstall>`
.. versionadded:: 1.0
:githublink:`%|py|179`
"""
mod = ModuleInstall(**self.as_dict())
if version is not None:
mod.version = version
return mod
[docs] def as_dict(self, rst_link=False):
"""
returns the members in a dictionary
:param rst_link: if True, add rst_link, license, classifier
:return: dictionary
:githublink:`%|py|191`
"""
r = dict(name=self.name, kind=self.kind, gitrepo=self.gitrepo, version=self.version,
mname=self.mname, script=self.script, deps=self.deps, index_url=self.index_url,
purpose=self.purpose, usage=self.usage, web=self.web,
post=None if self.post_installation is None else self.post_installation.copy(),
skip_import=self.skip_import, pipgit=self.pipgit)
if rst_link:
r["rst_link"] = "`{0} <{1}>`_".format(self.name, self.web)
r["license"] = self.get_installed_license()
r["installed"] = self.get_installed_version()
r["classifier"] = self.get_installed_classifier()
return r
[docs] def __cmp__(self, o):
"""
to sort modules
:param o: other module
:return: -1, 0, 1
:githublink:`%|py|210`
"""
def compare(v1, v2):
if v1 is None:
if v2 is None:
return 0
else:
return 1
else:
if v2 is None:
return -1
else:
if v1 < v2:
return -1
elif v1 > v2:
return 1
else:
return 0
r = compare(self.usage, o.usage)
if r != 0:
return r
r = compare(self.name.lower(), o.name.lower())
return r
[docs] def __lt__(self, o):
"""
overload operator ``<``
:param o: other module
:return: boolean
:githublink:`%|py|240`
"""
return self.__cmp__(o) < 0
[docs] @staticmethod
def clear_cache():
"""
clear the local cache to get wheel link
:githublink:`%|py|247`
"""
if os.path.exists(ModuleInstall._page_cache_html):
os.remove(ModuleInstall._page_cache_html)
if os.path.exists(ModuleInstall._page_cache_html2):
os.remove(ModuleInstall._page_cache_html2)
@property
def Purpose(self):
"""
returns the comment
:githublink:`%|py|257`
"""
return "-" if self.purpose is None else self.purpose
@property
def Script(self):
"""
returns the script to run if the extension is an application and not a module
:githublink:`%|py|264`
"""
exe = os.path.split(sys.executable)[0]
if sys.platform.startswith("win"):
sc = os.path.join(exe, "Scripts", self.script)
else:
sc = os.path.join(exe, "Scripts", os.path.splitext(self.script)[0])
return sc
[docs] def __str__(self):
"""
usual
:githublink:`%|py|275`
"""
if self.script is None:
return "{0}:{1}:import {2}:v{3}".format(
self.name, self.kind, self.ImportName, self.version)
else:
return "{0}:{1}:{2}:v{3}".format(self.name, self.kind, self.Script, self.version)
@property
def ImportName(self):
"""
return the import name
:githublink:`%|py|286`
"""
if self.mname is not None:
return self.mname
if self.name.startswith("python-"):
return self.name[len("python-"):]
else:
return self.name
[docs] def is_installed_local(self):
"""
checks if a module is installed
:githublink:`%|py|297`
"""
if self.script is None:
if self.skip_import:
try:
return self.is_installed_local_cmd()
except InstallError:
return False
else:
try:
if "." in self.ImportName:
raise ImportError(self.ImportName)
if sys.version_info[0] == 2:
r = importlib.import_module(self.ImportName)
return r
else:
r = importlib.util.find_spec(self.ImportName)
return r is not None
return r is not None
except ImportError:
txt = "import {0} # {1}".format(
self.ImportName, self.name)
try:
exec(txt)
return True
except Exception:
return False
else:
return os.path.exists(self.Script)
[docs] def is_installed_local_cmd(self):
"""
Test the module by running a command line.
Does some others verifications for a specific modules such as scipy.
.. versionadded:: 1.1
:githublink:`%|py|332`
"""
exe = get_python_program()
if self.skip_import:
cmd = exe + \
' -u -c "import pip # pip.main(["show", "{0}"])'.format(
self.name)
out, err = run_cmd(cmd, fLOG=self.fLOG)
if err or out is None:
raise InstallError("Cannot find {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-K1:\n{3}".format(
self.name, cmd, out, err))
if ("Name: " + self.name) not in out:
raise InstallError("Cannot find {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-K3:\n{3}".format(
self.name, cmd, out, err))
return True
else:
cmd = exe + \
' -u -c "import {0} # {1}"'.format(self.ImportName, self.name)
out, err = run_cmd(cmd, fLOG=self.fLOG)
if err:
raise InstallError("cannot import module {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-K2:\n{3}".format(
self.ImportName, cmd, out, err))
if self.name == "scipy":
cmd = exe + ' -u -c "import scipy.sparse"'
out, err = run_cmd(cmd, fLOG=self.fLOG)
if err:
if sys.platform.startswith("win") and sys.version_info[:2] >= (3, 5) and "DLL" in err:
mes = ("scipy.sparse is failing, you should check that Visual Studio 2015 is " +
"installed\n{0}\nCMD:\n{1}\nOUT:\n{2}\nERR-M:\n{3}")
raise InstallError(mes.format(
self.ImportName, cmd, out, err))
raise InstallError("scipy.sparse is failing\n{0}\nCMD:\n{1}\nOUT:\n{2}\nERR-L:\n{3}".format(
self.ImportName, cmd, out, err))
return True
_page_cache_html2 = os.path.join(
os.path.abspath(os.path.split(__file__)[0]), "page2.html")
[docs] def get_exewheel_url_link2(self, file_save=None, wheel=False, source=None):
"""
for windows, get the url of the setup using a webpage
:param file_save: for debug purposes
:param wheel: returns the wheel file or the exe file
:param source: source of the wheels (ex: ``2`` or ``http://...``)
:return: url, exe name
.. versionchanged:: 1.1
Parameter *source* was added.
:githublink:`%|py|380`
"""
if source is None or source == "2":
source = ModuleInstall.exeLocationXd_Default
source_page = source.rstrip("/") + "/index_modules_list.html"
if "cached_page2" not in self.__dict__:
page = ModuleInstall._page_cache_html2
exi = os.path.exists(page)
if exi:
dt = get_file_modification_date(page)
now = datetime.datetime.now()
df = now - dt
if df > datetime.timedelta(1):
exi = False
if exi:
text = read_page_wheel(page)
self.cached_page2 = text
else:
text = get_page_wheel(source_page)
save_page_wheel(page, text)
self.cached_page2 = text
page = self.cached_page2
reg = re.compile('href=\\"(.*?)\\"')
alls = reg.findall(page)
if len(alls) == 0:
keep = []
for line in page.split("\n"):
lline = line.lower()
if self.name in lline or (self.mname and self.mname in lline):
keep.append(line)
raise Exception(
"module " +
self.name + "\nexample:\n" + "\n".join(keep))
version = python_version()
plat = version[0] if version[0] == "win32" else version[1]
if version[1] == '64bit' and version[0] == 'win32':
plat = "amd64"
cp = "-cp%d%d-" % sys.version_info[:2]
py = "-py%d%d-" % sys.version_info[:2]
pyn = "-py%d-" % sys.version_info[0]
links = [_ for _ in alls if "/" +
self.name in _ and (pyn in _ or py in _ or cp in _) and (plat in _ or "-any" in _)]
if len(links) == 0 and "-" in self.name:
name_ = self.name.replace("-", "_")
links = [_ for _ in alls if "/" +
name_ in _ and (pyn in _ or py in _ or cp in _) and (plat in _ or "-any" in _)]
if len(links) == 0:
# exception for guidata
links = [_ for _ in alls if self.name in _ and "-py2.py3-" in _]
if len(links) == 0:
if file_save is not None:
with open(file_save, "w", encoding="utf8") as f:
f.write(page)
short_list = [_ for _ in alls if self.name in _]
raise MissingWheelException("unable to find a single link for " +
self.name + "\n" + "\n".join(short_list))
# Last filter. Removes modules with a longer name.
pref1 = self.name.lower() + "-"
pref2 = self.name.lower().replace("-", "_") + "-"
pref3 = self.name.lower().replace("_", "-") + "-"
def filter_cond(name):
name = name.split("/")[-1].lower()
return name.startswith(pref1) or name.startswith(pref2) or name.startswith(pref3)
links_ = [_ for _ in links if filter_cond(_)]
if len(links_) == 0:
prefs = "\npref1={0}\npref2={1}\npref3={2}".format(
pref1, pref2, pref3)
raise MissingWheelException("unable to find a single link for " +
self.name + "\n" + "\n".join(links) + prefs)
links = links_
links = [(lu.split("/")[-1], lu) for lu in links]
links0 = links
if self.name == "numpy":
links = [lu for lu in links if "unoptimized" not in lu[
0].lower() and "vanilla" not in lu[0].lower()]
if len(links) == 0:
raise Exception("unable to find a single link for " +
self.name +
"\nEX:\n" +
"\n".join(str(_) for _ in links0))
link = choose_most_recent(links)
self.existing_version = self.extract_version(link[0])
url, whl = link[1], link[0]
if not whl.endswith(".whl"):
whl += ".whl"
return url, whl
_page_cache_html = os.path.join(
os.path.abspath(os.path.split(__file__)[0]), "page.html")
[docs] def get_exewheel_url_link(self, file_save=None, wheel=False):
"""
for windows, get the url of the setup using a webpage
:param file_save: for debug purposes
:param wheel: returns the wheel file or the exe file
:return: url, exe name
:githublink:`%|py|488`
"""
if "cached_page" not in self.__dict__:
page = ModuleInstall._page_cache_html
exi = os.path.exists(page)
if exi:
dt = get_file_modification_date(page)
now = datetime.datetime.now()
df = now - dt
if df > datetime.timedelta(1):
exi = False
if exi:
text = read_page_wheel(page)
self.cached_page = text
else:
text = get_page_wheel(ModuleInstall.exeLocation)
save_page_wheel(page, text)
self.cached_page = text
page = self.cached_page
alls = extract_all_links(page)
if len(alls) == 0:
keep = []
for line in page.split("\n"):
lline = line.lower()
if self.name in lline or (self.mname and self.mname in lline):
keep.append(line)
raise Exception(
"module " +
self.name + "\nexample:\n" + "\n".join(keep))
version = python_version()
plat = version[0] if version[0] == "win32" else version[1]
if version[1] == '64bit' and version[0] == 'win32':
plat = "amd64"
links = list(enumerate_links_module(
self.name, alls, sys.version_info, plat))
if len(links) == 0:
if file_save is not None:
with open(file_save, "w", encoding="utf8") as f:
f.write(page)
raise MissingWheelException(
"Unable to find a single link for " + self.name)
nbnone = [lu for lu in links if lu[2] is None]
if len(nbnone) * 2 > len(links):
raise WrongWheelException("Unable to find any version in\n{0}".format(
"\n".join(str(_) for _ in links)))
links0 = links
if self.name == "numpy":
links = [lu for lu in links if "unoptimized" not in lu[
0].lower() and "vanilla" not in lu[0].lower()]
if len(links) == 0:
raise Exception("unable to find a single link for " +
self.name +
"\nEX:\n" +
"\n".join(str(_) for _ in links0))
link = choose_most_recent(links)
self.existing_version = self.extract_version(link[0])
if link[2] is None:
raise WrongWheelException("Unable to find a proper link in {0}\n{1}".format(
link, "\n".join(str(_) for _ in links)))
url, whl = ModuleInstall.exeLocation + link[2], link[0]
if not whl.endswith(".whl"):
whl += ".whl"
return url, whl
[docs] def unzipfiles(self, zipf, whereTo):
"""
unzip files from a zip archive
:param zipf: archive
:param whereTo: destination folder
:return: list of unzipped files
:githublink:`%|py|565`
"""
return unzip_files(zipf, whereTo, self.fLOG)
[docs] def download(self, temp_folder=".", force=False, unzipFile=True,
file_save=None, deps=False, source=None):
"""
Downloads the module without installation.
:param temp_folder: destination
:param force: force the installation even if already installed
:param unzipFile: if it can be unzipped, it will be (for github, mostly)
:param file_save: for debug purposes, do not change it unless you know what you are doing
:param deps: download the dependencies too (only available for pip)
:param source: overwrite source of the download, only for wheel packages,
see :meth:`get_exewheel_url_link2 <pymyinstall.installhelper.module_install.ModuleInstall.get_exewheel_url_link2>`
:return: downloaded files
*deps* is overwritten by *self.deps* if not None
If *source* is None, it is overwritten by *self.source*.
:githublink:`%|py|603`
"""
disable_options = {}
if source is None:
source = self.source
kind = self.kind
deps = deps if self.deps is None else self.deps
if kind == "pip" or self.pipgit:
# see https://pip.pypa.io/en/latest/reference/pip_install.html
# we use pip install <package> --download=temp_folder
pp = get_pip_program()
if self.pipgit:
br = "@" + \
self.branch if self.branch not in (None, "master") else ""
link = ModuleInstall.github_pattern_git.format(
self.name, self.gitrepo, br)
cmd = pp + ' download git+' + link
else:
cmd = pp + ' download {0}'.format(self.name)
if self.version is not None:
cmd += "=={0}".format(self.version)
if " " in temp_folder:
raise FileNotFoundError(
"no space allowed in folders: [" + temp_folder + "]")
if deps:
cmd += ' --dest={0}'.format(temp_folder)
else:
cmd += ' --dest={0} --no-deps'.format(temp_folder)
if self.index_url is not None:
slash = '' if self.index_url.endswith('/') else '/'
cmd += ' --no-cache-dir --index={0}{1}simple/'.format(
self.index_url, slash)
parsed_uri = urlsplit(self.index_url)
cmd += " --trusted-host " + parsed_uri.hostname
if self.pip_options is not None:
diff = [_ for _ in self.pip_options if _ not in disable_options]
cmd += " " + " ".join(diff)
out, err = run_cmd(
cmd, wait=True, fLOG=self.fLOG)
if "Successfully downloaded" not in out:
raise DownloadError(
"unable to download with pip " +
str(self) + "\nCMD:\n" +
cmd + "\nOUT:\n" +
out + "\nERR-N:\n" +
err)
lines = out.split("\n")
for line in lines:
if line.strip().startswith("Saved "):
return line.split("Saved")[-1].strip()
elif line.strip().startswith("File was already downloaded"):
return line.split("File was already downloaded")[-1].strip()
raise FileNotFoundError(
"unable to find downloaded file " +
str(self) + "\nCMD:\n" +
cmd + "\nOUT:\n" +
out + "\nERR-O:\n" +
err)
if kind in ("wheel", "wheel2"):
if source is not None:
kind = "wheel2"
ver = python_version()
if ver[0] != "win32":
# nothing to download, you should use pip
return None
else:
if hasattr(self, "overwrite") and self.overwrite is not None:
over = self.overwrite.format(*sys.version_info[0:2])
url, whl = over, over.split("/")[-1]
self.existing_version = "{0}{1}".format(
*sys.version_info[0:2])
elif kind == "wheel":
url, whl = self.get_exewheel_url_link(
file_save=file_save, wheel=True)
else:
url, whl = self.get_exewheel_url_link2(
file_save=file_save, wheel=True, source=source)
whlname = os.path.join(temp_folder, whl)
exi = os.path.exists(whlname)
if force or not exi:
self.fLOG("[pymy] downloading", whl)
# self.fLOG("url", url)
if self.existing_version is None:
self.existing_version = self.extract_version(whl)
req = urllib_request.Request(
url, headers={
'User-agent': default_user_agent})
try:
u = urllib_request.urlopen(req)
text = u.read()
u.close()
except urllib_error.HTTPError as e:
raise DownloadError("unable to download {} from {}".format(
os.path.split(whlname)[-1], url)) from e
if not os.path.exists(temp_folder):
os.makedirs(temp_folder)
if len(text) <= 1000:
raise WrongWheelException("Size of downloaded wheel is too small: {0}\nurl={1}\nagent={2}".format(
len(text), url, default_user_agent))
self.fLOG("[pymy] writing", whl)
with open(whlname, "wb") as f:
f.write(text)
return whlname
elif kind == "github" and not self.pipgit:
outfile = os.path.join(temp_folder, self.name + ".zip")
if force or not os.path.exists(outfile):
zipurl = ModuleInstall.github_pattern_zip.format(
self.name, self.gitrepo, self.branch)
self.fLOG("[pymy] downloading", zipurl)
try:
req = urllib_request.Request(
zipurl, headers={
'User-agent': default_user_agent})
u = urllib_request.urlopen(req)
text = u.read()
u.close()
except urllib_error.HTTPError as e:
raise Exception(
"unable to get archive from: " +
zipurl) from e
if not os.path.exists(temp_folder):
os.makedirs(temp_folder)
u = open(outfile, "wb")
u.write(text)
u.close()
if unzipFile:
self.fLOG("[pymy] unzipping ", outfile)
files = self.unzipfiles(outfile, temp_folder)
return files
else:
return outfile
if kind in ("exe", "exe2"):
if source is not None:
kind = "exe2"
ver = python_version()
if ver[0] != "win32":
raise Exception(
"this option is not available on other systems than Windows, version={0}".format(ver))
url, exe = self.get_exewheel_url_link(
file_save=file_save) if kind == "exe" else self.get_exewheel_url_link2(
file_save=file_save, source=source)
self.fLOG("[pymy] downloading", exe)
req = urllib_request.Request(
url, headers={
'User-agent': default_user_agent})
u = urllib_request.urlopen(req)
text = u.read()
u.close()
if not os.path.exists(temp_folder):
os.makedirs(temp_folder)
exename = os.path.join(temp_folder, exe)
self.fLOG("[pymy] writing", exe)
with open(exename, "wb") as f:
f.write(text)
return exename
raise ImportError(
"unknown kind: {0} for module {1}".format(
kind,
self.name))
[docs] def get_pypi_version(self, url='https://pypi.python.org/pypi'):
"""
returns the version of a package on pypi
:param url: pipy server
:return: version
See also `installing_python_packages_programatically.py <https://gist.github.com/rwilcox/755524>`_,
`pkgtools.pypi: PyPI interface <http://pkgtools.readthedocs.org/en/latest/pypi.html>`_.
:githublink:`%|py|790`
"""
if url == 'https://pypi.python.org/pypi':
# we use a function which caches the result
return get_pypi_version(self.name)
else:
pypi = xmlrpc_client.ServerProxy(url)
available = pypi.package_releases(self.name)
if available is None or len(available) == 0:
available = pypi.package_releases(self.name.capitalize())
if (available is None or len(available) == 0) and self.mname is not None:
available = pypi.package_releases(self.mname)
if available is None:
raise MissingPackageOnPyPiException(
"; ".join([self.name, self.name.capitalize(), self.mname]))
return available[0]
[docs] def get_pypi_numeric_version(self):
"""
returns the version of a package in pypi
:return: tuple
:githublink:`%|py|813`
"""
vers = self.get_pypi_version()
if vers is None:
return None
if isinstance(vers, list):
v = self.get_pypi_version()
raise TypeError("unexpected type: {0} -- {1}".format(vers, v))
return numeric_version(vers)
[docs] def get_installed_version(self):
"""
return the version of the installed package
:return: version
:githublink:`%|py|827`
"""
vers = get_module_version(None)
if self.name in vers:
return vers[self.name]
cap = self.name.capitalize()
if cap in vers:
return vers[cap]
cap = self.name.lower()
if cap in vers:
return vers[cap]
cap = self.name.replace("-", "_")
if cap in vers:
return vers[cap]
cap = self.name.replace("_", "-")
if cap in vers:
return vers[cap]
cap = self.name.lower().replace("_", "-")
if cap in vers:
return vers[cap]
if self.mname is not None:
if self.mname in vers:
return vers[self.mname]
cap = self.mname.lower()
if cap in vers:
return vers[cap]
return None
[docs] def get_installed_license(self):
"""
return the license of the installed package
:return: string
:githublink:`%|py|871`
"""
meta = self.get_installed_metadata()
if meta is None or len(meta) == 0:
res = None
else:
res = None
for k, v in meta.items():
if k.lower() == "license":
res = v
break
adm = {None, "", "UNKNOWN"}
if res is not None:
if isinstance(res, list):
res = [_ for _ in res if _ and _ not in adm]
if len(res) > 0:
res = res[0]
else:
res = None
if res in adm:
res = missing_module_licenses.get(self.name, None)
return res
[docs] def get_installed_classifier(self):
"""
return the classifier of the installed package
:return: string
:githublink:`%|py|898`
"""
meta = self.get_installed_metadata()
if meta is None:
return None
for k, v in meta.items():
if k.lower() == "classifier":
return v
return None
[docs] def is_installed_version(self):
"""
tells if a module is installed
:return: boolean
:githublink:`%|py|912`
"""
return self.get_installed_version() is not None
[docs] def get_installed_numeric_version(self):
"""
returns the version as number (not string)
:return: tuple
:githublink:`%|py|920`
"""
vers = self.get_installed_version()
if vers is None:
return None
return numeric_version(vers)
[docs] def has_update(self):
"""
tells if the package has a newer version on pipy
:return: boolean
:githublink:`%|py|931`
"""
if ModuleInstall.is_annoying(self.name):
return False
vers = self.get_installed_numeric_version()
if self.version is None:
pypi = self.get_pypi_numeric_version()
return compare_version(pypi, vers) > 0
else:
num = numeric_version(self.version)
return compare_version(num, vers) > 0
[docs] def _check_installation(self):
"""
some modules uninstall and install modules with another version number,
we try to track that
:githublink:`%|py|946`
"""
try:
import numpy
if compare_version(numpy.__version__, "1.10") < 0:
raise InstallError(
"numpy does not have a goof version number, it should be >= 1.10 not {0}".format(numpy.__version__))
except ImportError:
# not installed
pass
return True
[docs] def install(self, force_kind=None, force=False, temp_folder=".",
log=False, options=None, deps=False, source=None,
custom=None, post=None, out_streams=None):
"""
Installs the package.
:param force_kind: overwrite self.kind
:param force: force the installation even if already installed
:param temp_folder: folder where to download the setup
:param log: display logs or not
:param options: other options to add to the command line (see below) in a list
:param deps: install the dependencies too (only available for pip)
:param source: overwrite the source of the wheels,
see :meth:`get_exewheel_url_link2 <pymyinstall.installhelper.module_install.ModuleInstall.get_exewheel_url_link2>`
:param custom: overwrite parameters in ``self.custom``
:param post: instructions post installation (see the cnostructor for more help)
:param out_streams: if it is a list, the function will add standard outputs
:return: boolean
The options mentioned in parameter ``options``
are described here: `pip install <http://www.pip-installer.org/en/latest/usage.html>`_
or `setup.py options <http://docs.python.org/3.4/install/>`_ if you
installing a module from github.
.. versionchanged:: 1.0
*deps* is overwritten by *self.deps* if not None
.. versionchanged:: 1.1
On Anaconda (based on function :func:`is_conda_distribution <pymyinstall.installhelper.install_cmd_helper.is_conda_distribution>`), we try *conda* first
before switching to the regular way if it did not work.
Exception were changed from ``Exception`` to ``InstallError``.
Parameter *source* was added, if None, it is overwritten by *self.source*.
Parameter *custom* was added, it works the same as *source*.
Parameter *post* was added.
Parameter *out_streas* added.
:githublink:`%|py|992`
"""
if source is None:
source = self.source
if post is None:
post = self.post_installation
if not force and force_kind is None and is_conda_distribution():
try:
return self.install(force_kind="conda", force=True, temp_folder=temp_folder,
log=log, options=options, deps=deps)
except InstallError as e:
warnings.warn(str(e))
# we try the regular way now
if not force and self.is_installed_version():
return True
deps = deps if self.deps is None else self.deps
if options is None:
options = self.pip_options
if options is None:
options = []
kind = force_kind if force_kind is not None else self.kind
add = (" with " + kind) if kind != self.kind else ""
self.fLOG("[pymy] installation of " + str(self) + add)
ret = None
custom = custom or self.custom
if kind == "pip" or self.pipgit:
if custom is not None:
raise NotImplementedError(
"custom must be None not '{0}' when kind is '{1}'".format(custom, kind))
pp = get_pip_program()
if self.pipgit:
br = "@" + \
self.branch if self.branch not in (None, "master") else ""
link = ModuleInstall.github_pattern_git.format(
self.name, self.gitrepo, br)
cmd = pp + ' install git+' + link
else:
cmd = pp + " install {0}".format(self.name)
if self.version is not None:
cmd += "=={0}".format(self.version)
if len(options) > 0:
cmd += " " + " ".join(options)
if not deps:
cmd += ' --no-deps'
if self.index_url is not None:
slash = '' if self.index_url.endswith('/') else '/'
cmd += ' --no-cache-dir --index={0}{1}simple/'.format(
self.index_url, slash)
else:
cmd += " --cache-dir={0}".format(temp_folder)
if self.name == "kivy-garden":
memo = sys.argv
sys.argv = []
out, err = run_cmd(
cmd, wait=True, fLOG=self.fLOG)
out = _filter_pip_out(out)
if out_streams is not None:
out_streams.append((cmd, out, err))
if self.name == "kivy-garden":
sys.argv = memo
success2 = "Requirement already up-to-date: " + self.name
uptodate = success2.replace("-", "_") in out.replace("-", "_")
if "No distributions matching the version" in out:
mes = "(1) unable to install with pip {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-P:\n{3}".format(
str(self), cmd, out, err)
raise InstallError(mes)
if "Testing of typecheck-decorator passed without failure." in out:
ret = True
elif "Successfully installed" not in out and not uptodate:
if "error: Unable to find vcvarsall.bat" in out:
url = "http://www.xavierdupre.fr/blog/2013-07-07_nojs.html"
mes = "(2) unable to install with pip {0}\nread:\n{1}\nCMD:\n{2}\nOUT:\n{3}\nERR-P:\n{4}".format(
str(self), url, cmd, out, err)
raise InstallError(mes)
if "Requirement already satisfied" not in out and not uptodate:
mes = "(3) unable to install with pip {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-Q:\n{3}".format(
str(self), cmd, out, err)
raise InstallError(mes)
else:
ret = not uptodate
elif kind == "conda":
if custom is not None:
raise NotImplementedError(
"custom must be None not '{0}' when kind is '{1}'".format(custom, kind))
if "--upgrade" in options:
options = [_ for _ in options if _ != "--upgrade"]
command = "update"
else:
command = "install"
pp = get_conda_program()
cmd = pp + " {1} {0}".format(self.name, command)
if self.version is not None:
cmd += "=={0}".format(self.version)
if len(options) > 0:
cmd += " " + " ".join(options)
cmd += " --no-pin --yes --quiet"
if not deps:
cmd += ' --no-deps'
out, err = run_cmd(
cmd, wait=True, fLOG=self.fLOG)
if out_streams is not None:
out_streams.append((cmd, out, err))
if "No distributions matching the version" in out or \
"No packages found in current linux" in out:
mes = "(4) unable to install with conda {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-R:\n{3}".format(
str(self), cmd, out, err)
raise InstallError(mes)
if "Testing of typecheck-decorator passed without failure." in out:
ret = True
elif "Successfully installed" not in out:
if "error: Unable to find vcvarsall.bat" in out:
url = "http://www.xavierdupre.fr/blog/2013-07-07_nojs.html"
mes = "(5) unable to install with conda {0}\nread:\n{1}\nCMD:\n{2}\nOUT:\n{3}\nERR-S:\n{4}".format(
str(self), url, cmd, out, err)
raise InstallError(mes)
if "Requirement already satisfied" not in out:
mes = "(6) unable to install with conda {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-T:\n{3}".format(
str(self), cmd, out, err)
raise InstallError(mes)
else:
ret = True
elif kind in ("wheel", "wheel2"):
if custom is not None:
raise NotImplementedError(
"custom must be None not '{0}' when kind is '{1}'".format(custom, kind))
ver = python_version()
if ver[0] != "win32":
ret = self.install("pip")
whlname = self.name
ret = True
else:
whlname = self.download(
temp_folder=temp_folder,
force=force,
unzipFile=True,
source=source)
vers = self.get_installed_numeric_version()
ret = True
if vers is not None:
whlvers = numeric_version(get_wheel_version(whlname))
if compare_version(vers, whlvers) >= 0:
self.fLOG("[pymy] skipping, no newer version {0} <= {1}: whl= {2}".format(
whlvers, vers, whlname))
ret = False
if ret:
self.fLOG("[pymy] installing", os.path.split(whlname)[-1])
pip = get_pip_program()
cmd = pip + " install {0}".format(whlname)
if self.version is not None:
cmd += "=={0}".format(self.version)
if len(options) > 0:
opts = list(options)
if len(opts):
cmd += " " + " ".join(opts)
if not deps:
cmd += ' --no-deps'
out, err = run_cmd(
cmd, wait=True, fLOG=self.fLOG)
if out_streams is not None:
out_streams.append((cmd, out, err))
if "No distributions matching the version" in out:
mes = "(7) unable to install with wheel {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-U:\n{3}".format(
str(self), cmd, out, err)
raise InstallError(mes)
if "Testing of typecheck-decorator passed without failure." in out:
ret = True
elif "Successfully installed" not in out:
if "error: Unable to find vcvarsall.bat" in out:
url = "http://www.xavierdupre.fr/blog/2013-07-07_nojs.html"
mes = "(8) unable to install with wheel {0}\nread:\n{1}\nCMD:\n{2}\nOUT:\n{3}\nERR-V:\n{4}".format(
str(self), url, cmd, out, err)
raise InstallError(mes)
if "Requirement already satisfied" not in out and "Requirement already up-to-date" not in out:
mes = "(9) unable to install with wheel {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-W:\n{3}".format(
str(self), cmd, out, err)
raise InstallError(mes)
else:
ret = True
elif kind == "github" and not self.pipgit:
# the following code requires admin rights
# if python_version()[0].startswith("win") and kind == "git" and not os.path.exists(ModuleInstall.gitexe) :
# raise FileNotFoundError("you need to install github first: see http://windows.github.com/")
# if python_version()[0].startswith("win"):
# os.chdir(os.path.join(ModuleInstall.gitexe,"bin"))
# cmd = pip + " install -e
# git://github.com/{0}/{1}-python.git#egg={1}".format(self.gitrepo,
# self.name)
files = self.download(temp_folder=temp_folder,
force=force, unzipFile=True)
setu = [_ for _ in files if _.endswith("setup.py")]
if len(setu) == 0:
raise Exception(
"unable to find setup.py for module " +
self.name)
if len(setu) > 1:
setu = [(len(_), _) for _ in setu]
setu.sort()
if setu[0][0] == setu[1][0]:
raise InstallError(
"(10) more than one setup.py for module " +
self.name +
"\n" +
"\n".join(
str(_) for _ in setu))
self.fLOG(
"[pymy] warning: more than one setup: " + str(setu))
setu = [setu[0][1]]
setu = os.path.abspath(setu[0])
self.fLOG("[pymy] install ", setu[0])
cwd = os.getcwd()
os.chdir(os.path.split(setu)[0])
if custom is None:
custom = ["install"]
cmds = []
for command in custom:
cmd1 = "{0} setup.py {1}".format(
sys.executable.replace("pythonw.exe", "python.exe"),
command)
cmds.append(cmd1)
def enumerate_filtered_option(options):
for o in options:
if o not in ('--no-deps', '--upgrade'):
yield o
filter_options = list(enumerate_filtered_option(options))
if len(filter_options) > 0:
cmds[-1] += " " + " ".join(filter_options)
if deps:
# it will not work
# cmd += ' --no-deps'
pass
outs = ""
errs = ""
for cmd in cmds:
out, err = run_cmd(
cmd, wait=True, fLOG=self.fLOG)
if out_streams is not None:
out_streams.append((cmd, out, err))
if len(outs) > 0:
outs += "\n"
if len(errs) > 0:
errs += "\n"
outs += out
errs += err
os.chdir(cwd)
out, err = outs, errs
if "Successfully installed" not in out and "install C" not in out:
if "Finished processing dependencies" not in out:
raise InstallError(
"(11) unable to install with github " +
str(self) +
"\nOUT:\n" +
out +
"\nERR-X:\n" +
err)
self.fLOG(
"warning: ``Successfully installed`` or ``install C`` not found")
if "Permission denied" in out:
raise PermissionError(" ".join(["(12) unable to install with github", str(self), "\n--CMD--:\n",
"\n".join(cmds), "\n--OUT--\n", out, "\n--ERR-Y--\n", err]))
ret = True
elif kind == "exe":
if custom is not None:
raise NotImplementedError(
"custom must be None not '{0}' when kind is '{1}'".format(custom, kind))
ver = python_version()
if ver[0] != "win32":
ret = self.install("pip")
else:
exename = self.download(
temp_folder=temp_folder,
force=force,
unzipFile=True,
source=source)
self.fLOG("[pymy] executing", os.path.split(exename)[-1])
out, err = run_cmd(
exename + " /s /qn /SILENT", wait=True, fLOG=self.fLOG)
if out_streams is not None:
out_streams.append((cmd, out, err))
ret = len(err) == 0
elif kind == "exe2":
if custom is not None:
raise NotImplementedError(
"custom must be None not '{0}' when kind is '{1}'".format(custom, kind))
ver = python_version()
if ver[0] != "win32":
ret = self.install("pip")
else:
exename = self.download(
temp_folder=temp_folder,
force=force,
unzipFile=True,
source=source)
self.fLOG("[pymy] executing", os.path.split(exename)[-1])
out, err = run_cmd(
exename + " /s /qn", wait=True, fLOG=self.fLOG)
if out_streams is not None:
out_streams.append((cmd, out, err))
ret = len(err) == 0
else:
raise ImportError(
"unknown kind: {0} for module {1}".format(
kind,
self.name))
if ret is not None and ret:
self._check_installation()
# at this stage, there is a bug, for some executable, the execution
# takes longer than expected
# if not self.is_installed_version() :
# raise Exception("unable to install module: {0}, str:{1}".format(self.name, self))
if ret is not None and ret and self.script is not None:
if sys.platform.startswith("win"):
# here, we have to wait until the script is installed
ti = 0
while not self.is_installed_local():
time.sleep(0.5)
ti += 0.5
if ti > 60:
self.fLOG(
"wait for the end of execution of ",
self.name)
ti = 0
# we add set path=%path%;PYTHONPATH
with open(self.Script, "r") as f:
full = f.read()
exe = os.path.split(sys.executable)[0]
cmd = "set path=%path%;{0}".format(exe)
if cmd not in full:
self.fLOG("[pymy] add {0} to {1}".format(cmd, self.Script))
with open(self.Script, "w") as f:
if full.startswith("@echo off"):
f.write(
full.replace(
"@echo off",
"@echo off\n{0}".format(cmd)))
else:
f.write(cmd)
f.write("\n")
f.write(full)
if ret is not None and post is not None:
self.fLOG("[pymy] _ run_post_installation [begin]", post)
ret = self.run_post_installation(post)
self.fLOG("[pymy] _ run_post_installation [end]")
if ret is not None and ret:
# we check the module was properly installed
if not self.is_installed_local_cmd():
raise InstallError(
"** unable to install module {0}, unable to import it".format(self.name))
return ret
[docs] def run_post_installation(self, post):
"""
Run instructions post installation
:param post: dictionary, instructions post installation
:return: boolean
Example:
::
post = dict(
cmd_python="Scripts\\\\pywin32_postinstall.py -install")
.. versionadded:: 1.1
:githublink:`%|py|1389`
"""
if not isinstance(post, dict):
raise TypeError("expecting a dictionary")
for k, v in post.items():
if k == "cmd_python":
# processed just above
continue
if k == "pre_cmd":
if v == "module_install_preprocess":
self.fLOG("[pymy] _ module_install_preprocess [begin]")
self.module_install_preprocess(post)
self.fLOG("[pymy] _ module_install_preprocess [end]")
else:
raise ValueError(
"Unable to interpret value '{0}'.".format(v))
else:
raise KeyError("Unable to interpret command '{0}'".format(k))
for k, v in post.items():
if k == "cmd_python":
exe = os.path.abspath(sys.executable)
cmd = '"{0}" {1}'.format(exe, v.format(os.path.dirname(exe)))
out, err = run_cmd(cmd, wait=True, fLOG=self.fLOG)
if err is not None and len(err) > 0:
raise InstallError(
"Post installation script failed.\nCMD\n{0}\nOUT\n{1}\nERR-Z\n{2}".format(cmd, out, err))
self.fLOG("[pymy] OUT:\n{0}".format(out))
elif k == "pre_cmd":
# processed just above
continue
else:
raise KeyError("Unable to interpret command '{0}'".format(k))
return True
[docs] def update(self, force_kind=None, force=False, temp_folder=".",
log=False, options=None, deps=False, source=None):
"""
Updates the package if necessary, we use
``pip install <module_name> --upgrade --no-deps``,
:param force_kind: overwrite self.kind
:param force: force the installation even if not need to update
:param temp_folder: folder where to download the setup
:param log: display logs or not
:param options: others options to add to the command line (see below) an a list
:param deps: download the dependencies too (only available for pip)
:param source: overwrite the source of the wheel, see :meth:`get_exewheel_url_link2 <pymyinstall.installhelper.module_install.ModuleInstall.get_exewheel_url_link2>`
:return: boolean
The options mentioned in parameter ``options``
are described here: `pip install <http://www.pip-installer.org/en/latest/usage.html>`_
or `setup.py options <http://docs.python.org/3.4/install/>`_ if you
installing a module from github.
.. versionchanged:: 1.1
Parameter *source* was added, if None, it is overwritten by *self.source*.
:githublink:`%|py|1444`
"""
if source is None:
source = self.source
if ModuleInstall.is_annoying(self.name):
return False
if not self.is_installed_version():
raise MissingInstalledPackageException(self.name)
if not force and not self.has_update():
return True
self.fLOG("[pymy] update of ", self)
options = [] if options is None else list(options)
for opt in ["--upgrade", "--no-deps"]:
if opt not in options:
if not deps or opt == "--no-deps":
options.append(opt)
res = self.install(force_kind=force_kind, force=True,
temp_folder=temp_folder, log=log, options=options, source=source)
return res
[docs] def module_install_preprocess(self, post):
"""
Run some preprocessing for specific modules
:param post: dictionary
:githublink:`%|py|1473`
"""
if self.name == "pywin32":
self.fLOG("[pymy] _ module_install_preprocess", self.name)
if "cmd_python" not in post:
raise KeyError("Key 'cmd_python' is not in post.")
cmd_python = post["cmd_python"]
name = cmd_python.split()[0].format(
os.path.dirname(sys.executable))
self.fLOG("[pymy] _ opening", name)
if not os.path.exists(name):
raise FileNotFoundError(name)
with open(name, "r") as f:
content = f.read()
repby = "os.path.exists(dest)"
if repby not in content:
self.fLOG("[pymy] _ Preprocess '{0}'".format(name))
rep = "def CopyTo(desc, src, dest):"
content = content.replace(
rep, "{0}\n if {1}: return".format(rep, repby))
with open(name, "w") as f:
f.write(content)
self.fLOG("[pymy] _ Done.")
else:
self.fLOG(
"Skip preprocess '{0}' because '{1}' was found.".format(name, repby))
else:
raise ValueError(
"No preprocessing for module '{0}'.".format(self.name))
