Source code for pymyinstall.installhelper.module_install

"""
Various function to install various python module from various location.


:githublink:`%|py|5`
"""
from __future__ import print_function
import sys
import re
import os
import time
import importlib
import datetime
import warnings
from pip import __version__ as pip_version
from .install_cmd_helper import python_version, run_cmd, unzip_files, get_pip_program
from .install_cmd_helper import get_python_program, get_file_modification_date, get_conda_program, is_conda_distribution
from .module_install_exceptions import MissingPackageOnPyPiException, MissingInstalledPackageException, InstallError
from .module_install_exceptions import DownloadError, MissingVersionWheelException, WrongWheelException, MissingWheelException
from .module_install_version import get_pypi_version, get_module_version, annoying_modules, get_module_metadata
from .module_install_version import numeric_version, compare_version, choose_most_recent, get_wheel_version
from .module_install_page_wheel import get_page_wheel, read_page_wheel, save_page_wheel, enumerate_links_module, extract_all_links
from .missing_license import missing_module_licenses
from .internet_settings import default_user_agent
from .install_cmd_regex import regex_wheel_versions

if sys.version_info[0] == 2:
    from urlparse import urlsplit
    import urllib2 as urllib_request
    import urllib2 as urllib_error
    import xmlrpclib as xmlrpc_client
    from codecs import open
    FileNotFoundError = Exception
    PermissionError = Exception
else:
    from urllib.parse import urlsplit
    import urllib.request as urllib_request
    import urllib.error as urllib_error
    import importlib.util
    import xmlrpc.client as xmlrpc_client


[docs]class ModuleInstall: """ defines the necessary information for a module .. exref:: :title: Installation from GitHub :: ModuleInstall("pyquickhelper", "github", "sdpython").install(temp_folder="temp") :githublink:`%|py|53` """ allowedKind = ["pip", "github", "exe", "exe2", "wheel", "wheel2"] exeLocation = "https://www.lfd.uci.edu/~gohlke/pythonlibs/" exeLocationXd_Default = "http://www.xavierdupre.fr/enseignement/setup/" gitexe = r"C:\Program Files (x86)\Git" github_pattern_zip = "https://github.com/{1}/{0}/archive/{2}.zip" github_pattern_git = "https://github.com/{1}/{0}.git{2}"
[docs] @staticmethod def is_annoying(module_name): """ some modules are not available on pipy :githublink:`%|py|66` """ return module_name in annoying_modules
[docs] def __init__(self, name, kind="pip", gitrepo=None, mname=None, fLOG=print, version=None, script=None, index_url=None, deps=None, purpose=None, usage=None, web=None, source=None, custom=None, branch="master", pip_options=None, overwrite=None, post=None, skip_import=False, pipgit=False): """ :param name: name :param kind: kind of installation (*pip*, *github*, *wheel*) :param gitrepo: github repository (example: sdpython) :param mname: sometimes, the module name is different from its official name :param version: to install a specific version (None for the latest) :param fLOG: logging function :param script: some extensions are not a module but an application (such as ``spyder``), the class will check this script is available :param deps: overwrite deps parameters when installing the module :param index_url: to get the package from a custom pypi server :param purpose: purpose of the module :param usage: main usage for the module :param web: website for the module, if None, default to pipy :param source: to overwrite parameter *source* of methods :meth:`download <pymyinstall.installhelper.module_install.ModuleInstall.download>`, :meth:`install` or :meth:`update`. :param custom: custom instructions to install, usually ``['build', 'install']`` to run ``setup.py build`` and ``setup.py install`` :param branch: only necessary for install process with github :param pip_options: additional options for pip (list) :param overwrite: overwrite the location of the wheel :param post: instructions post installation (look for this parameter in the code to see what is supported) :param pipgit: install the module with ``pip + git`` instead of getting the full archive :param skip_import: added to indicate the module cannot be imported .. versionchanged:: 1.1 Parameters *source*, *custom*, *branch*, *pip_options*, *overwrite*, *post* were added. Parameter *skip_import* was introduced to skip the checking of the installation. Parameter *pipgit* was added. :githublink:`%|py|106` """ if kind != "pip" and version is not None: raise NotImplementedError( "version can be only specified if kind=='pip'") self.name = name self.kind = kind self.gitrepo = gitrepo self.version = version self.mname = mname self.script = script self.index_url = index_url self.deps = deps self.purpose = purpose self.usage = usage self.existing_version = None self.source = source self.custom = custom self.branch = branch self.pip_options = pip_options self.overwrite = overwrite self.post_installation = post self.pipgit = pipgit if self.mname == self.name: raise ValueError( "Do not specify mname if it is equal to name '{0}'.".format(self.name)) if self.pipgit and self.gitrepo is None: raise ValueError("If pipgit=True, gitrepo must be specified.") if self.pipgit and kind != 'github': raise ValueError( "If pipgit=True, kind must be 'github' not '{0}'.".format(kind)) if self.version is not None and self.gitrepo is not None: raise ValueError("version must be None if gitrepo is not.") if self.mname is not None and self.mname.startswith("-"): self.mname = self.mname[1:] self.skip_import = True else: self.skip_import = skip_import self.web = web if web is not None else ( "https://pypi.python.org/pypi/" + self.name) if self.kind not in ModuleInstall.allowedKind: raise Exception( "unable to interpret kind {0}, it should be in {1}".format( kind, ",".join( ModuleInstall.allowedKind))) if self.kind == "github" and self.gitrepo is None: raise Exception("gitrepo cannot be empty") self.fLOG = fLOG
[docs] def copy(self, version=None): """ copy the module, if version is not None, change the version number :param version: version number or None for unchanged :return: :class:`ModuleInstall <pymyinstall.installhelper.module_install.ModuleInstall>` .. versionadded:: 1.0 :githublink:`%|py|167` """ mod = ModuleInstall(**self.as_dict()) if version is not None: mod.version = version return mod
[docs] def as_dict(self, rst_link=False): """ returns the members in a dictionary :param rst_link: if True, add rst_link, license, classifier :return: dictionary :githublink:`%|py|179` """ r = dict(name=self.name, kind=self.kind, gitrepo=self.gitrepo, version=self.version, mname=self.mname, script=self.script, deps=self.deps, index_url=self.index_url, purpose=self.purpose, usage=self.usage, web=self.web, post=None if self.post_installation is None else self.post_installation.copy(), skip_import=self.skip_import, pipgit=self.pipgit) if rst_link: r["rst_link"] = "`{0} <{1}>`_".format(self.name, self.web) r["license"] = self.get_installed_license() r["installed"] = self.get_installed_version() r["classifier"] = self.get_installed_classifier() return r
[docs] def __cmp__(self, o): """ to sort modules :param o: other module :return: -1, 0, 1 :githublink:`%|py|198` """ def compare(v1, v2): if v1 is None: if v2 is None: return 0 else: return 1 else: if v2 is None: return -1 else: if v1 < v2: return -1 elif v1 > v2: return 1 else: return 0 r = compare(self.usage, o.usage) if r != 0: return r r = compare(self.name.lower(), o.name.lower()) return r
[docs] def __lt__(self, o): """ overload operator ``<`` :param o: other module :return: boolean :githublink:`%|py|228` """ return self.__cmp__(o) < 0
[docs] @staticmethod def clear_cache(): """ clear the local cache to get wheel link :githublink:`%|py|235` """ if os.path.exists(ModuleInstall._page_cache_html): os.remove(ModuleInstall._page_cache_html) if os.path.exists(ModuleInstall._page_cache_html2): os.remove(ModuleInstall._page_cache_html2)
@property def Purpose(self): """ returns the comment :githublink:`%|py|245` """ return "-" if self.purpose is None else self.purpose @property def Script(self): """ returns the script to run if the extension is an application and not a module :githublink:`%|py|252` """ exe = os.path.split(sys.executable)[0] if sys.platform.startswith("win"): sc = os.path.join(exe, "Scripts", self.script) else: sc = os.path.join(exe, "Scripts", os.path.splitext(self.script)[0]) return sc
[docs] def __str__(self): """ usual :githublink:`%|py|263` """ if self.script is None: return "{0}:{1}:import {2}:v{3}".format( self.name, self.kind, self.ImportName, self.version) else: return "{0}:{1}:{2}:v{3}".format(self.name, self.kind, self.Script, self.version)
@property def ImportName(self): """ return the import name :githublink:`%|py|274` """ if self.mname is not None: return self.mname if self.name.startswith("python-"): return self.name[len("python-"):] else: return self.name
[docs] def is_installed_local(self): """ checks if a module is installed :githublink:`%|py|285` """ if self.script is None: if self.skip_import: try: return self.is_installed_local_cmd() except InstallError: return False else: try: if "." in self.ImportName: raise ImportError(self.ImportName) if sys.version_info[0] == 2: r = importlib.import_module(self.ImportName) return r else: r = importlib.util.find_spec(self.ImportName) return r is not None return r is not None except ImportError: txt = "import {0} # {1}".format( self.ImportName, self.name) try: exec(txt) return True except Exception: return False else: return os.path.exists(self.Script)
[docs] def is_installed_local_cmd(self): """ Test the module by running a command line. Does some others verifications for a specific modules such as scipy. .. versionadded:: 1.1 :githublink:`%|py|320` """ exe = get_python_program() if self.skip_import: cmd = exe + \ ' -u -c "import pip # pip.main(["show", "{0}"])'.format( self.name) out, err = run_cmd(cmd, fLOG=self.fLOG) if err or out is None: raise InstallError("Cannot find {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-K1:\n{3}".format( self.name, cmd, out, err)) if ("Name: " + self.name) not in out: raise InstallError("Cannot find {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-K3:\n{3}".format( self.name, cmd, out, err)) return True else: cmd = exe + \ ' -u -c "import {0} # {1}"'.format(self.ImportName, self.name) out, err = run_cmd(cmd, fLOG=self.fLOG) if err: raise InstallError("cannot import module {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-K2:\n{3}".format( self.ImportName, cmd, out, err)) if self.name == "scipy": cmd = exe + ' -u -c "import scipy.sparse"' out, err = run_cmd(cmd, fLOG=self.fLOG) if err: if sys.platform.startswith("win") and sys.version_info[:2] >= (3, 5) and "DLL" in err: mes = ("scipy.sparse is failing, you should check that Visual Studio 2015 is " + "installed\n{0}\nCMD:\n{1}\nOUT:\n{2}\nERR-M:\n{3}") raise InstallError(mes.format( self.ImportName, cmd, out, err)) raise InstallError("scipy.sparse is failing\n{0}\nCMD:\n{1}\nOUT:\n{2}\nERR-L:\n{3}".format( self.ImportName, cmd, out, err)) return True
_page_cache_html2 = os.path.join( os.path.abspath(os.path.split(__file__)[0]), "page2.html")
[docs] def get_exewheel_url_link2(self, file_save=None, wheel=False, source=None): """ for windows, get the url of the setup using a webpage :param file_save: for debug purposes :param wheel: returns the wheel file or the exe file :param source: source of the wheels (ex: ``2`` or ``http://...``) :return: url, exe name .. versionchanged:: 1.1 Parameter *source* was added. :githublink:`%|py|368` """ if source is None or source == "2": source = ModuleInstall.exeLocationXd_Default source_page = source.rstrip("/") + "/index_modules_list.html" if "cached_page2" not in self.__dict__: page = ModuleInstall._page_cache_html2 exi = os.path.exists(page) if exi: dt = get_file_modification_date(page) now = datetime.datetime.now() df = now - dt if df > datetime.timedelta(1): exi = False if exi: text = read_page_wheel(page) self.cached_page2 = text else: text = get_page_wheel(source_page) save_page_wheel(page, text) self.cached_page2 = text page = self.cached_page2 reg = re.compile('href=\\"(.*?)\\"') alls = reg.findall(page) if len(alls) == 0: keep = [] for line in page.split("\n"): lline = line.lower() if self.name in lline or (self.mname and self.mname in lline): keep.append(line) raise Exception( "module " + self.name + "\nexample:\n" + "\n".join(keep)) version = python_version() plat = version[0] if version[0] == "win32" else version[1] if version[1] == '64bit' and version[0] == 'win32': plat = "amd64" cp = "-cp%d%d-" % sys.version_info[:2] py = "-py%d%d-" % sys.version_info[:2] pyn = "-py%d-" % sys.version_info[0] links = [_ for _ in alls if "/" + self.name in _ and (pyn in _ or py in _ or cp in _) and (plat in _ or "-any" in _)] if len(links) == 0 and "-" in self.name: name_ = self.name.replace("-", "_") links = [_ for _ in alls if "/" + name_ in _ and (pyn in _ or py in _ or cp in _) and (plat in _ or "-any" in _)] if len(links) == 0: # exception for guidata links = [_ for _ in alls if self.name in _ and "-py2.py3-" in _] if len(links) == 0: if file_save is not None: with open(file_save, "w", encoding="utf8") as f: f.write(page) short_list = [_ for _ in alls if self.name in _] raise MissingWheelException("unable to find a single link for " + self.name + "\n" + "\n".join(short_list)) # Last filter. Removes modules with a longer name. pref1 = self.name.lower() + "-" pref2 = self.name.lower().replace("-", "_") + "-" pref3 = self.name.lower().replace("_", "-") + "-" def filter_cond(name): name = name.split("/")[-1].lower() return name.startswith(pref1) or name.startswith(pref2) or name.startswith(pref3) links_ = [_ for _ in links if filter_cond(_)] if len(links_) == 0: prefs = "\npref1={0}\npref2={1}\npref3={2}".format( pref1, pref2, pref3) raise MissingWheelException("unable to find a single link for " + self.name + "\n" + "\n".join(links) + prefs) links = links_ links = [(l.split("/")[-1], l) for l in links] links0 = links if self.name == "numpy": links = [l for l in links if "unoptimized" not in l[ 0].lower() and "vanilla" not in l[0].lower()] if len(links) == 0: raise Exception("unable to find a single link for " + self.name + "\nEX:\n" + "\n".join(str(_) for _ in links0)) link = choose_most_recent(links) self.existing_version = self.extract_version(link[0]) url, whl = link[1], link[0] if not whl.endswith(".whl"): whl += ".whl" return url, whl
_page_cache_html = os.path.join( os.path.abspath(os.path.split(__file__)[0]), "page.html")
[docs] def unzipfiles(self, zipf, whereTo): """ unzip files from a zip archive :param zipf: archive :param whereTo: destination folder :return: list of unzipped files :githublink:`%|py|553` """ return unzip_files(zipf, whereTo, self.fLOG)
[docs] def extract_version(self, name): """ extract the version from a filename :param name: filename :return: verions (str) :githublink:`%|py|562` """ res = None for i, regve in enumerate(regex_wheel_versions): reg = re.compile(regve) res = reg.search(name) if res is not None: if i == 6: return ".".join(res.groups()) else: return res.groups()[0] raise MissingVersionWheelException( "Unable to extract version number from '{0}'\nREGEX\n{1}".format(name, "\n".join(regex_wheel_versions)))
[docs] def download(self, temp_folder=".", force=False, unzipFile=True, file_save=None, deps=False, source=None): """ Downloads the module without installation. :param temp_folder: destination :param force: force the installation even if already installed :param unzipFile: if it can be unzipped, it will be (for github, mostly) :param file_save: for debug purposes, do not change it unless you know what you are doing :param deps: download the dependencies too (only available for pip) :param source: overwrite source of the download, only for wheel packages, see :meth:`get_exewheel_url_link2 <pymyinstall.installhelper.module_install.ModuleInstall.get_exewheel_url_link2>` :return: downloaded files *deps* is overwritten by *self.deps* if not None If *source* is None, it is overwritten by *self.source*. :githublink:`%|py|591` """ disable_options = {} if source is None: source = self.source kind = self.kind deps = deps if self.deps is None else self.deps if kind == "pip" or self.pipgit: # see https://pip.pypa.io/en/latest/reference/pip_install.html # we use pip install <package> --download=temp_folder pp = get_pip_program() if self.pipgit: br = "@" + \ self.branch if self.branch not in (None, "master") else "" link = ModuleInstall.github_pattern_git.format( self.name, self.gitrepo, br) cmd = pp + ' download git+' + link else: cmd = pp + ' download {0}'.format(self.name) if self.version is not None: cmd += "=={0}".format(self.version) if " " in temp_folder: raise FileNotFoundError( "no space allowed in folders: [" + temp_folder + "]") if deps: cmd += ' --dest={0}'.format(temp_folder) else: cmd += ' --dest={0} --no-deps'.format(temp_folder) if self.index_url is not None: slash = '' if self.index_url.endswith('/') else '/' cmd += ' --no-cache-dir --index={0}{1}simple/'.format( self.index_url, slash) parsed_uri = urlsplit(self.index_url) cmd += " --trusted-host " + parsed_uri.hostname if self.pip_options is not None: diff = [_ for _ in self.pip_options if _ not in disable_options] cmd += " " + " ".join(diff) out, err = run_cmd( cmd, wait=True, fLOG=self.fLOG) if "Successfully downloaded" not in out: raise DownloadError( "unable to download with pip " + str(self) + "\nCMD:\n" + cmd + "\nOUT:\n" + out + "\nERR-N:\n" + err) lines = out.split("\n") for line in lines: if line.strip().startswith("Saved "): return line.split("Saved")[-1].strip() elif line.strip().startswith("File was already downloaded"): return line.split("File was already downloaded")[-1].strip() raise FileNotFoundError( "unable to find downloaded file " + str(self) + "\nCMD:\n" + cmd + "\nOUT:\n" + out + "\nERR-O:\n" + err) if kind in ("wheel", "wheel2"): if source is not None: kind = "wheel2" ver = python_version() if ver[0] != "win32": # nothing to download, you should use pip return None else: if hasattr(self, "overwrite") and self.overwrite is not None: over = self.overwrite.format(*sys.version_info[0:2]) url, whl = over, over.split("/")[-1] self.existing_version = "{0}{1}".format( *sys.version_info[0:2]) elif kind == "wheel": url, whl = self.get_exewheel_url_link( file_save=file_save, wheel=True) else: url, whl = self.get_exewheel_url_link2( file_save=file_save, wheel=True, source=source) whlname = os.path.join(temp_folder, whl) exi = os.path.exists(whlname) if force or not exi: self.fLOG("[pymy] downloading", whl) # self.fLOG("url", url) if self.existing_version is None: self.existing_version = self.extract_version(whl) req = urllib_request.Request( url, headers={ 'User-agent': default_user_agent}) try: u = urllib_request.urlopen(req) text = u.read() u.close() except urllib_error.HTTPError as e: raise DownloadError("unable to download {} from {}".format( os.path.split(whlname)[-1], url)) from e if not os.path.exists(temp_folder): os.makedirs(temp_folder) if len(text) <= 1000: raise WrongWheelException("Size of downloaded wheel is too small: {0}\nurl={1}\nagent={2}".format( len(text), url, default_user_agent)) self.fLOG("[pymy] writing", whl) with open(whlname, "wb") as f: f.write(text) return whlname elif kind == "github" and not self.pipgit: outfile = os.path.join(temp_folder, self.name + ".zip") if force or not os.path.exists(outfile): zipurl = ModuleInstall.github_pattern_zip.format( self.name, self.gitrepo, self.branch) self.fLOG("[pymy] downloading", zipurl) try: req = urllib_request.Request( zipurl, headers={ 'User-agent': default_user_agent}) u = urllib_request.urlopen(req) text = u.read() u.close() except urllib_error.HTTPError as e: raise Exception( "unable to get archive from: " + zipurl) from e if not os.path.exists(temp_folder): os.makedirs(temp_folder) u = open(outfile, "wb") u.write(text) u.close() if unzipFile: self.fLOG("[pymy] unzipping ", outfile) files = self.unzipfiles(outfile, temp_folder) return files else: return outfile if kind in ("exe", "exe2"): if source is not None: kind = "exe2" ver = python_version() if ver[0] != "win32": raise Exception( "this option is not available on other systems than Windows, version={0}".format(ver)) url, exe = self.get_exewheel_url_link( file_save=file_save) if kind == "exe" else self.get_exewheel_url_link2( file_save=file_save, source=source) self.fLOG("[pymy] downloading", exe) req = urllib_request.Request( url, headers={ 'User-agent': default_user_agent}) u = urllib_request.urlopen(req) text = u.read() u.close() if not os.path.exists(temp_folder): os.makedirs(temp_folder) exename = os.path.join(temp_folder, exe) self.fLOG("[pymy] writing", exe) with open(exename, "wb") as f: f.write(text) return exename raise ImportError( "unknown kind: {0} for module {1}".format( kind, self.name))
[docs] def get_pypi_version(self, url='https://pypi.python.org/pypi'): """ returns the version of a package on pypi :param url: pipy server :return: version See also `installing_python_packages_programatically.py <https://gist.github.com/rwilcox/755524>`_, `pkgtools.pypi: PyPI interface <http://pkgtools.readthedocs.org/en/latest/pypi.html>`_. :githublink:`%|py|778` """ if url == 'https://pypi.python.org/pypi': # we use a function which caches the result return get_pypi_version(self.name) else: pypi = xmlrpc_client.ServerProxy(url) available = pypi.package_releases(self.name) if available is None or len(available) == 0: available = pypi.package_releases(self.name.capitalize()) if (available is None or len(available) == 0) and self.mname is not None: available = pypi.package_releases(self.mname) if available is None: raise MissingPackageOnPyPiException( "; ".join([self.name, self.name.capitalize(), self.mname])) return available[0]
[docs] def get_pypi_numeric_version(self): """ returns the version of a package in pypi :return: tuple :githublink:`%|py|801` """ vers = self.get_pypi_version() if vers is None: return None if isinstance(vers, list): v = self.get_pypi_version() raise TypeError("unexpected type: {0} -- {1}".format(vers, v)) return numeric_version(vers)
[docs] def get_installed_version(self): """ return the version of the installed package :return: version :githublink:`%|py|815` """ vers = get_module_version(None) if self.name in vers: return vers[self.name] cap = self.name.capitalize() if cap in vers: return vers[cap] cap = self.name.lower() if cap in vers: return vers[cap] cap = self.name.replace("-", "_") if cap in vers: return vers[cap] cap = self.name.replace("_", "-") if cap in vers: return vers[cap] cap = self.name.lower().replace("_", "-") if cap in vers: return vers[cap] if self.mname is not None: if self.mname in vers: return vers[self.mname] cap = self.mname.lower() if cap in vers: return vers[cap] return None
[docs] def get_installed_metadata(self): """ return the metadata of the installed package :return: dictionary :githublink:`%|py|847` """ r = get_module_metadata(self.name) if r is None: return get_module_metadata(self.mname) else: return r
[docs] def get_installed_license(self): """ return the license of the installed package :return: string :githublink:`%|py|859` """ meta = self.get_installed_metadata() if meta is None or len(meta) == 0: res = None else: res = None for k, v in meta.items(): if k.lower() == "license": res = v break adm = {None, "", "UNKNOWN"} if res is not None: if isinstance(res, list): res = [_ for _ in res if _ and _ not in adm] if len(res) > 0: res = res[0] else: res = None if res in adm: res = missing_module_licenses.get(self.name, None) return res
[docs] def get_installed_classifier(self): """ return the classifier of the installed package :return: string :githublink:`%|py|886` """ meta = self.get_installed_metadata() if meta is None: return None for k, v in meta.items(): if k.lower() == "classifier": return v return None
[docs] def is_installed_version(self): """ tells if a module is installed :return: boolean :githublink:`%|py|900` """ return self.get_installed_version() is not None
[docs] def get_installed_numeric_version(self): """ returns the version as number (not string) :return: tuple :githublink:`%|py|908` """ vers = self.get_installed_version() if vers is None: return None return numeric_version(vers)
[docs] def has_update(self): """ tells if the package has a newer version on pipy :return: boolean :githublink:`%|py|919` """ if ModuleInstall.is_annoying(self.name): return False vers = self.get_installed_numeric_version() if self.version is None: pypi = self.get_pypi_numeric_version() return compare_version(pypi, vers) > 0 else: num = numeric_version(self.version) return compare_version(num, vers) > 0
[docs] def _check_installation(self): """ some modules uninstall and install modules with another version number, we try to track that :githublink:`%|py|934` """ try: import numpy if compare_version(numpy.__version__, "1.10") < 0: raise InstallError( "numpy does not have a goof version number, it should be >= 1.10 not {0}".format(numpy.__version__)) except ImportError: # not installed pass return True
[docs] def install(self, force_kind=None, force=False, temp_folder=".", log=False, options=None, deps=False, source=None, custom=None, post=None, out_streams=None): """ Installs the package. :param force_kind: overwrite self.kind :param force: force the installation even if already installed :param temp_folder: folder where to download the setup :param log: display logs or not :param options: other options to add to the command line (see below) in a list :param deps: install the dependencies too (only available for pip) :param source: overwrite the source of the wheels, see :meth:`get_exewheel_url_link2 <pymyinstall.installhelper.module_install.ModuleInstall.get_exewheel_url_link2>` :param custom: overwrite parameters in ``self.custom`` :param post: instructions post installation (see the cnostructor for more help) :param out_streams: if it is a list, the function will add standard outputs :return: boolean The options mentioned in parameter ``options`` are described here: `pip install <http://www.pip-installer.org/en/latest/usage.html>`_ or `setup.py options <http://docs.python.org/3.4/install/>`_ if you installing a module from github. .. versionchanged:: 1.0 *deps* is overwritten by *self.deps* if not None .. versionchanged:: 1.1 On Anaconda (based on function :func:`is_conda_distribution <pymyinstall.installhelper.install_cmd_helper.is_conda_distribution>`), we try *conda* first before switching to the regular way if it did not work. Exception were changed from ``Exception`` to ``InstallError``. Parameter *source* was added, if None, it is overwritten by *self.source*. Parameter *custom* was added, it works the same as *source*. Parameter *post* was added. Parameter *out_streas* added. :githublink:`%|py|980` """ if source is None: source = self.source if post is None: post = self.post_installation if not force and force_kind is None and is_conda_distribution(): try: return self.install(force_kind="conda", force=True, temp_folder=temp_folder, log=log, options=options, deps=deps) except InstallError as e: warnings.warn(str(e)) # we try the regular way now if not force and self.is_installed_version(): return True deps = deps if self.deps is None else self.deps if options is None: options = self.pip_options if options is None: options = [] kind = force_kind if force_kind is not None else self.kind add = (" with " + kind) if kind != self.kind else "" self.fLOG("[pymy] installation of " + str(self) + add) ret = None custom = custom or self.custom if kind == "pip" or self.pipgit: if custom is not None: raise NotImplementedError( "custom must be None not '{0}' when kind is '{1}'".format(custom, kind)) pp = get_pip_program() if self.pipgit: br = "@" + \ self.branch if self.branch not in (None, "master") else "" link = ModuleInstall.github_pattern_git.format( self.name, self.gitrepo, br) cmd = pp + ' install git+' + link else: cmd = pp + " install {0}".format(self.name) if self.version is not None: cmd += "=={0}".format(self.version) if len(options) > 0: cmd += " " + " ".join(options) if not deps: cmd += ' --no-deps' if self.index_url is not None: slash = '' if self.index_url.endswith('/') else '/' cmd += ' --no-cache-dir --index={0}{1}simple/'.format( self.index_url, slash) else: cmd += " --cache-dir={0}".format(temp_folder) if self.name == "kivy-garden": memo = sys.argv sys.argv = [] out, err = run_cmd( cmd, wait=True, fLOG=self.fLOG) if out_streams is not None: out_streams.append((cmd, out, err)) if self.name == "kivy-garden": sys.argv = memo success2 = "Requirement already up-to-date: " + self.name uptodate = success2.replace("-", "_") in out.replace("-", "_") if "No distributions matching the version" in out: mes = "(1) unable to install with pip {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-P:\n{3}".format( str(self), cmd, out, err) raise InstallError(mes) if "Testing of typecheck-decorator passed without failure." in out: ret = True elif "Successfully installed" not in out and not uptodate: if "error: Unable to find vcvarsall.bat" in out: url = "http://www.xavierdupre.fr/blog/2013-07-07_nojs.html" mes = "(2) unable to install with pip {0}\nread:\n{1}\nCMD:\n{2}\nOUT:\n{3}\nERR-P:\n{4}".format( str(self), url, cmd, out, err) raise InstallError(mes) if "Requirement already satisfied" not in out and not uptodate: mes = "(3) unable to install with pip {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-Q:\n{3}".format( str(self), cmd, out, err) raise InstallError(mes) else: ret = not uptodate elif kind == "conda": if custom is not None: raise NotImplementedError( "custom must be None not '{0}' when kind is '{1}'".format(custom, kind)) if "--upgrade" in options: options = [_ for _ in options if _ != "--upgrade"] command = "update" else: command = "install" pp = get_conda_program() cmd = pp + " {1} {0}".format(self.name, command) if self.version is not None: cmd += "=={0}".format(self.version) if len(options) > 0: cmd += " " + " ".join(options) cmd += " --no-pin --yes --quiet" if not deps: cmd += ' --no-deps' out, err = run_cmd( cmd, wait=True, fLOG=self.fLOG) if out_streams is not None: out_streams.append((cmd, out, err)) if "No distributions matching the version" in out or \ "No packages found in current linux" in out: mes = "(4) unable to install with conda {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-R:\n{3}".format( str(self), cmd, out, err) raise InstallError(mes) if "Testing of typecheck-decorator passed without failure." in out: ret = True elif "Successfully installed" not in out: if "error: Unable to find vcvarsall.bat" in out: url = "http://www.xavierdupre.fr/blog/2013-07-07_nojs.html" mes = "(5) unable to install with conda {0}\nread:\n{1}\nCMD:\n{2}\nOUT:\n{3}\nERR-S:\n{4}".format( str(self), url, cmd, out, err) raise InstallError(mes) if "Requirement already satisfied" not in out: mes = "(6) unable to install with conda {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-T:\n{3}".format( str(self), cmd, out, err) raise InstallError(mes) else: ret = True elif kind in ("wheel", "wheel2"): if custom is not None: raise NotImplementedError( "custom must be None not '{0}' when kind is '{1}'".format(custom, kind)) ver = python_version() if ver[0] != "win32": ret = self.install("pip") whlname = self.name ret = True else: whlname = self.download( temp_folder=temp_folder, force=force, unzipFile=True, source=source) vers = self.get_installed_numeric_version() ret = True if vers is not None: whlvers = numeric_version(get_wheel_version(whlname)) if compare_version(vers, whlvers) >= 0: self.fLOG("[pymy] skipping, no newer version {0} <= {1}: whl= {2}".format( whlvers, vers, whlname)) ret = False if ret: self.fLOG("[pymy] installing", os.path.split(whlname)[-1]) pip = get_pip_program() cmd = pip + " install {0}".format(whlname) if self.version is not None: cmd += "=={0}".format(self.version) if len(options) > 0: opts = [_ for _ in options] if len(opts): cmd += " " + " ".join(opts) if not deps: cmd += ' --no-deps' out, err = run_cmd( cmd, wait=True, fLOG=self.fLOG) if out_streams is not None: out_streams.append((cmd, out, err)) if "No distributions matching the version" in out: mes = "(7) unable to install with wheel {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-U:\n{3}".format( str(self), cmd, out, err) raise InstallError(mes) if "Testing of typecheck-decorator passed without failure." in out: ret = True elif "Successfully installed" not in out: if "error: Unable to find vcvarsall.bat" in out: url = "http://www.xavierdupre.fr/blog/2013-07-07_nojs.html" mes = "(8) unable to install with wheel {0}\nread:\n{1}\nCMD:\n{2}\nOUT:\n{3}\nERR-V:\n{4}".format( str(self), url, cmd, out, err) raise InstallError(mes) if "Requirement already satisfied" not in out and "Requirement already up-to-date" not in out: mes = "(9) unable to install with wheel {0}\nCMD:\n{1}\nOUT:\n{2}\nERR-W:\n{3}".format( str(self), cmd, out, err) raise InstallError(mes) else: ret = True elif kind == "github" and not self.pipgit: # the following code requires admin rights # if python_version()[0].startswith("win") and kind == "git" and not os.path.exists(ModuleInstall.gitexe) : # raise FileNotFoundError("you need to install github first: see http://windows.github.com/") # if python_version()[0].startswith("win"): # os.chdir(os.path.join(ModuleInstall.gitexe,"bin")) # cmd = pip + " install -e # git://github.com/{0}/{1}-python.git#egg={1}".format(self.gitrepo, # self.name) files = self.download(temp_folder=temp_folder, force=force, unzipFile=True) setu = [_ for _ in files if _.endswith("setup.py")] if len(setu) == 0: raise Exception( "unable to find setup.py for module " + self.name) if len(setu) > 1: setu = [(len(_), _) for _ in setu] setu.sort() if setu[0][0] == setu[1][0]: raise InstallError( "(10) more than one setup.py for module " + self.name + "\n" + "\n".join( str(_) for _ in setu)) self.fLOG( "[pymy] warning: more than one setup: " + str(setu)) setu = [setu[0][1]] setu = os.path.abspath(setu[0]) self.fLOG("[pymy] install ", setu[0]) cwd = os.getcwd() os.chdir(os.path.split(setu)[0]) if custom is None: custom = ["install"] cmds = [] for command in custom: cmd1 = "{0} setup.py {1}".format( sys.executable.replace("pythonw.exe", "python.exe"), command) cmds.append(cmd1) def enumerate_filtered_option(options): for o in options: if o not in ('--no-deps', '--upgrade'): yield o filter_options = list(enumerate_filtered_option(options)) if len(filter_options) > 0: cmds[-1] += " " + " ".join(filter_options) if deps: # it will not work # cmd += ' --no-deps' pass outs = "" errs = "" for cmd in cmds: out, err = run_cmd( cmd, wait=True, fLOG=self.fLOG) if out_streams is not None: out_streams.append((cmd, out, err)) if len(outs) > 0: outs += "\n" if len(errs) > 0: errs += "\n" outs += out errs += err os.chdir(cwd) out, err = outs, errs if "Successfully installed" not in out and "install C" not in out: if "Finished processing dependencies" not in out: raise InstallError( "(11) unable to install with github " + str(self) + "\nOUT:\n" + out + "\nERR-X:\n" + err) self.fLOG( "warning: ``Successfully installed`` or ``install C`` not found") if "Permission denied" in out: raise PermissionError(" ".join(["(12) unable to install with github", str(self), "\n--CMD--:\n", "\n".join(cmds), "\n--OUT--\n", out, "\n--ERR-Y--\n", err])) ret = True elif kind == "exe": if custom is not None: raise NotImplementedError( "custom must be None not '{0}' when kind is '{1}'".format(custom, kind)) ver = python_version() if ver[0] != "win32": ret = self.install("pip") else: exename = self.download( temp_folder=temp_folder, force=force, unzipFile=True, source=source) self.fLOG("[pymy] executing", os.path.split(exename)[-1]) out, err = run_cmd( exename + " /s /qn /SILENT", wait=True, fLOG=self.fLOG) if out_streams is not None: out_streams.append((cmd, out, err)) ret = len(err) == 0 elif kind == "exe2": if custom is not None: raise NotImplementedError( "custom must be None not '{0}' when kind is '{1}'".format(custom, kind)) ver = python_version() if ver[0] != "win32": ret = self.install("pip") else: exename = self.download( temp_folder=temp_folder, force=force, unzipFile=True, source=source) self.fLOG("[pymy] executing", os.path.split(exename)[-1]) out, err = run_cmd( exename + " /s /qn", wait=True, fLOG=self.fLOG) if out_streams is not None: out_streams.append((cmd, out, err)) ret = len(err) == 0 else: raise ImportError( "unknown kind: {0} for module {1}".format( kind, self.name)) if ret is not None and ret: self._check_installation() # at this stage, there is a bug, for some executable, the execution # takes longer than expected # if not self.is_installed_version() : # raise Exception("unable to install module: {0}, str:{1}".format(self.name, self)) if ret is not None and ret and self.script is not None: if sys.platform.startswith("win"): # here, we have to wait until the script is installed ti = 0 while not self.is_installed_local(): time.sleep(0.5) ti += 0.5 if ti > 60: self.fLOG( "wait for the end of execution of ", self.name) ti = 0 # we add set path=%path%;PYTHONPATH with open(self.Script, "r") as f: full = f.read() exe = os.path.split(sys.executable)[0] cmd = "set path=%path%;{0}".format(exe) if cmd not in full: self.fLOG("[pymy] add {0} to {1}".format(cmd, self.Script)) with open(self.Script, "w") as f: if full.startswith("@echo off"): f.write( full.replace( "@echo off", "@echo off\n{0}".format(cmd))) else: f.write(cmd) f.write("\n") f.write(full) if ret is not None and post is not None: self.fLOG("[pymy] _ run_post_installation [begin]", post) ret = self.run_post_installation(post) self.fLOG("[pymy] _ run_post_installation [end]") if ret is not None and ret: # we check the module was properly installed if not self.is_installed_local_cmd(): raise InstallError( "** unable to install module {0}, unable to import it".format(self.name)) return ret
[docs] def run_post_installation(self, post): """ Run instructions post installation :param post: dictionary, instructions post installation :return: boolean Example: :: post = dict( cmd_python="Scripts\\\\pywin32_postinstall.py -install") .. versionadded:: 1.1 :githublink:`%|py|1376` """ if not isinstance(post, dict): raise TypeError("expecting a dictionary") for k, v in post.items(): if k == "cmd_python": # processed just above continue elif k == "pre_cmd": if v == "module_install_preprocess": self.fLOG("[pymy] _ module_install_preprocess [begin]") self.module_install_preprocess(post) self.fLOG("[pymy] _ module_install_preprocess [end]") else: raise ValueError( "Unable to interpret value '{0}'.".format(v)) else: raise KeyError("Unable to interpret command '{0}'".format(k)) for k, v in post.items(): if k == "cmd_python": exe = os.path.abspath(sys.executable) cmd = '"{0}" {1}'.format(exe, v.format(os.path.dirname(exe))) out, err = run_cmd(cmd, wait=True, fLOG=self.fLOG) if err is not None and len(err) > 0: raise InstallError( "Post installation script failed.\nCMD\n{0}\nOUT\n{1}\nERR-Z\n{2}".format(cmd, out, err)) self.fLOG("[pymy] OUT:\n{0}".format(out)) elif k == "pre_cmd": # processed just above continue else: raise KeyError("Unable to interpret command '{0}'".format(k)) return True
[docs] def update(self, force_kind=None, force=False, temp_folder=".", log=False, options=None, deps=False, source=None): """ Updates the package if necessary, we use ``pip install <module_name> --upgrade --no-deps``, :param force_kind: overwrite self.kind :param force: force the installation even if not need to update :param temp_folder: folder where to download the setup :param log: display logs or not :param options: others options to add to the command line (see below) an a list :param deps: download the dependencies too (only available for pip) :param source: overwrite the source of the wheel, see :meth:`get_exewheel_url_link2 <pymyinstall.installhelper.module_install.ModuleInstall.get_exewheel_url_link2>` :return: boolean The options mentioned in parameter ``options`` are described here: `pip install <http://www.pip-installer.org/en/latest/usage.html>`_ or `setup.py options <http://docs.python.org/3.4/install/>`_ if you installing a module from github. .. versionchanged:: 1.1 Parameter *source* was added, if None, it is overwritten by *self.source*. :githublink:`%|py|1431` """ if source is None: source = self.source if ModuleInstall.is_annoying(self.name): return False if not self.is_installed_version(): raise MissingInstalledPackageException(self.name) if not force and not self.has_update(): return True self.fLOG("[pymy] update of ", self) options = [] if options is None else list(options) for opt in ["--upgrade", "--no-deps"]: if opt not in options: if not deps or opt == "--no-deps": options.append(opt) res = self.install(force_kind=force_kind, force=True, temp_folder=temp_folder, log=log, options=options, source=source) return res
[docs] def module_install_preprocess(self, post): """ Run some preprocessing for specific modules :param post: dictionary :githublink:`%|py|1460` """ if self.name == "pywin32": self.fLOG("[pymy] _ module_install_preprocess", self.name) if "cmd_python" not in post: raise KeyError("Key 'cmd_python' is not in post.") cmd_python = post["cmd_python"] name = cmd_python.split()[0].format( os.path.dirname(sys.executable)) self.fLOG("[pymy] _ opening", name) if not os.path.exists(name): raise FileNotFoundError(name) with open(name, "r") as f: content = f.read() repby = "os.path.exists(dest)" if repby not in content: self.fLOG("[pymy] _ Preprocess '{0}'".format(name)) rep = "def CopyTo(desc, src, dest):" content = content.replace( rep, "{0}\n if {1}: return".format(rep, repby)) with open(name, "w") as f: f.write(content) self.fLOG("[pymy] _ Done.") else: self.fLOG( "Skip preprocess '{0}' because '{1}' was found.".format(name, repby)) else: raise ValueError( "No preprocessing for module '{0}'.".format(self.name))