Coverage for src/ensae_teaching_cs/automation/jenkins_helper.py: 50%
88 statements
« prev ^ index » next coverage.py v7.1.0, created at 2023-04-28 06:23 +0200
« prev ^ index » next coverage.py v7.1.0, created at 2023-04-28 06:23 +0200
1"""
2@file
3@brief Set up a jenkins server with all the necessary job
4"""
5import os
6import re
7import sys
8from pyquickhelper.loghelper import noLOG
9from pyquickhelper.jenkinshelper import setup_jenkins_server_yml
10from pyquickhelper.jenkinshelper.yaml_helper import load_yaml
11from .teaching_modules import get_teaching_modules
14def get_interpreter(platform=None):
15 """
16 Returns the default interpreter.
18 @param platform platform
19 """
20 if platform == sys.platform:
21 return os.path.dirname(sys.executable)
22 elif platform.startswith("win"):
23 return "c:\\Python%d%d%d_x64" % sys.version_info[:3]
24 elif platform.startswith("darwin"):
25 return "/usr/local/bin"
26 return "/usr/local/bin"
29def engines_default(prefix="c:\\", prefix_python="c:\\", prefix_conda="c:\\", platform=None):
30 """
31 Returns a dictionary with default values for a :epkg:`Jenkins` server.
33 @param prefix prefix for Jenkins location
34 @param prefix_python prefix for Python distribution
35 @param prefix_conda prefix for Anaconda or Miniconda distribution
36 @param platform platform to use
37 @return dictionary
39 .. warning::
41 Virtual environment with conda must be created on the same disk
42 as the original interpreter. The other scenario is not supported.
43 """
44 if platform is None:
45 platform = sys.platform
46 if platform.startswith("win"):
47 res = dict(anaconda2=os.path.join(prefix_conda, "Anaconda2"),
48 anaconda3=os.path.join(prefix_conda, "Anaconda3"),
49 default=get_interpreter(platform=platform),
50 py38=get_interpreter(platform=platform),
51 py37=get_interpreter(platform=platform),
52 py36=os.path.join(prefix_python, "Python36_x64"),
53 py27=os.path.join(prefix_python, "Python27_x64"),
54 Python38pyq=os.path.join(
55 prefix, "jenkins", "venv", "py38", "pyq", "Scripts"),
56 Python37pyq=os.path.join(
57 prefix, "jenkins", "venv", "py37", "pyq", "Scripts"),
58 Python36pyq=os.path.join(prefix, "jenkins", "venv", "py36", "pyq", "Scripts"))
59 res["Python27"] = res["py27"]
60 res["Python36"] = res["py36"]
61 res["Python37"] = res["py37"]
62 res["Python38"] = res["py38"]
63 res["Anaconda2"] = res["anaconda2"]
64 res["Anaconda3"] = res["anaconda3"]
65 return res
67 res = {}
68 for k in [-1, 0, 1]:
69 vers = (sys.version_info[0], sys.version_info[1] + k)
70 key = "Python%d%d" % vers
71 res[key] = get_interpreter(platform=platform)
72 res["py%d%d" % vers] = res[key]
73 return res
76def default_jenkins_jobs(filter=None, neg_filter=None, root=None, platform=None):
77 """
78 Default list of :epkg:`Jenkins` jobs.
80 @param filter keep a subset of jobs (regular expression)
81 @param neg_filter remove a subset of jobs (regular expression)
82 @param root where to find yml project
83 @param platform platform or None for the current one
84 @return list
86 It produces a subset of the following list of jobs:
88 .. runpython::
90 from ensae_teaching_cs.automation.jenkins_helper import default_jenkins_jobs
91 modules = default_jenkins_jobs()
92 text = [str(m) for m in modules]
93 print("\\n".join(text))
94 """
95 if platform is None:
96 platform = sys.platform
97 plat = "win" if platform.startswith("win") else "lin"
98 yml = []
99 pattern = "https://raw.githubusercontent.com/sdpython/%s/%s/.local.jenkins.{0}.yml".format(
100 plat)
101 modules = ["_automation"] + get_teaching_modules(branch=True)
102 for c in modules:
103 if ':' in c:
104 c, branch = c.split(':')
105 else:
106 branch = 'master'
107 yml.append(pattern % (c, branch))
109 if filter is not None or neg_filter is not None:
110 reg = re.compile(filter if filter else ".*")
111 neg_reg = re.compile(neg_filter if neg_filter else "^$")
112 res = default_jenkins_jobs(platform=platform)
113 new_res = []
114 for row in res:
115 if isinstance(row, str):
116 if reg.search(row) and not neg_reg.search(row):
117 new_res.append(row)
118 elif isinstance(row, tuple):
119 if reg.search(row[0]) and not neg_reg.search(row[0]):
120 new_res.append(row)
121 elif isinstance(row, list):
122 # list
123 sub = []
124 for item in row:
125 if isinstance(item, str):
126 if reg.search(item) and not neg_reg.search(item):
127 sub.append(item)
128 elif isinstance(item, tuple):
129 if reg.search(item[0]) and not neg_reg.search(item[0]):
130 sub.append(item)
131 else:
132 raise TypeError(f"{item} - {type(item)}")
133 if len(sub) > 0:
134 new_res.append(sub)
135 else:
136 raise TypeError(f"{row} - {type(row)}")
137 return new_res
138 else:
139 context = {'Python37': 'python3.7',
140 'Python38': 'python3.8', 'Python39': 'python3.9'}
141 yml_data = load_yaml(pattern %
142 ('_automation', 'master'), context=context)
143 pyth = yml_data[0]['python']
144 res = []
145 for pyt in pyth:
146 v = pyt['VERSION'] + 0.01
147 vers = (int(v), int((v - int(v)) * 10))
148 res.extend(["standalone [local_pypi] [py%d%d]" % vers,
149 ("pymyinstall [update_modules] [py%d%d]" % vers,
150 "H H(0-1) * * 5")])
151 res.extend(('yml', c, 'H H(0-1) * * %d' % (i % 7))
152 for i, c in enumerate(yml))
153 return res
156def setup_jenkins_server(js, github="sdpython", modules=None,
157 overwrite=False, location=None, prefix="",
158 delete_first=False, disable_schedule=False,
159 fLOG=noLOG):
160 """
161 Sets up many jobs on :epkg:`Jenkins`.
163 @param js (JenkinsExt) jenkins server (specially if you need credentials)
164 @param github github account if it does not start with *http://*,
165 the link to git repository of the project otherwise
166 @param modules modules for which to generate the :epkg:`Jenkins` job
167 (see @see fn default_jenkins_jobs which provides the default value
168 if *modules* is None)
169 @param overwrite do not create the job if it already exists
170 @param location None for default or a local folder
171 @param prefix add a prefix to the name
172 @param delete_first remove all jobs first
173 @param disable_schedule disable schedule for all jobs
174 @param fLOG logging function
175 @return list of created jobs
177 *modules* is a list defined as follows:
179 * each element can be a string or a tuple (string, schedule time) or a list
180 * if it is a list, it contains a list of elements defined as previously
181 * the job at position i is not scheduled, it will start after the last
182 job at position i-1 whether or not it fails
183 """
184 platform = js.platform
185 if modules is None:
186 modules = default_jenkins_jobs(platform=platform)
187 r = setup_jenkins_server_yml(js, github=github, modules=modules, get_jenkins_script=None,
188 overwrite=overwrite, location=location, prefix=prefix,
189 delete_first=delete_first, disable_schedule=disable_schedule)
190 return r