Coverage for src/pyquickhelper/jenkinshelper/jenkins_server.py : 93%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2@file
3@brief Extends Jenkins Server from :epkg:`python-jenkins`.
4"""
6import os
7import sys
8import socket
9import hashlib
10import re
11from xml.sax.saxutils import escape
12import requests
13import jenkins
14from ..loghelper.flog import noLOG
15from ..pycode.windows_scripts import windows_jenkins, windows_jenkins_any
16from ..pycode.windows_scripts import windows_jenkins_27_conda, windows_jenkins_27_def
17from ..pycode.linux_scripts import linux_jenkins, linux_jenkins_any
18from ..pycode.build_helper import private_script_replacements
19from .jenkins_exceptions import JenkinsExtException, JenkinsJobException
20from .jenkins_server_template import _config_job, _trigger_up, _trigger_time, _git_repo, _task_batch_win, _task_batch_lin
21from .jenkins_server_template import _trigger_startup, _publishers, _file_creation, _wipe_repo, _artifacts, _cleanup_repo
22from .yaml_helper import enumerate_processed_yml
23from .jenkins_helper import jenkins_final_postprocessing, get_platform
25_timeout_default = 1200
27_default_engine_paths = {
28 "windows": {
29 "__PY36__": "__PY36__",
30 "__PY37__": "__PY37__",
31 "__PY38__": "__PY38__",
32 "__PY39__": "__PY39__",
33 "__PY36_X64__": "__PY36_X64__",
34 "__PY37_X64__": "__PY37_X64__",
35 "__PY38_X64__": "__PY38_X64__",
36 "__PY39_X64__": "__PY39_X64__",
37 },
38}
41def _modified_windows_jenkins(requirements_local, requirements_pypi, module="__MODULE__",
42 port="__PORT__", platform=None):
43 return private_script_replacements(
44 linux_jenkins, module,
45 (requirements_local, requirements_pypi),
46 port, raise_exception=False,
47 default_engine_paths=_default_engine_paths,
48 platform=get_platform(platform))
51def _modified_linux_jenkins(requirements_local, requirements_pypi, module="__MODULE__",
52 port="__PORT__", platform=None):
53 return private_script_replacements(
54 windows_jenkins, module,
55 (requirements_local, requirements_pypi),
56 port, raise_exception=False,
57 default_engine_paths=_default_engine_paths,
58 platform=get_platform(platform))
61def _modified_windows_jenkins_27(requirements_local, requirements_pypi, module="__MODULE__",
62 port="__PORT__", anaconda=True, platform=None):
63 return private_script_replacements(
64 windows_jenkins_27_conda if anaconda else windows_jenkins_27_def,
65 module, (requirements_local, requirements_pypi),
66 port, raise_exception=False,
67 default_engine_paths=_default_engine_paths,
68 platform=get_platform(platform))
71def _modified_windows_jenkins_any(requirements_local, requirements_pypi, module="__MODULE__",
72 port="__PORT__", platform=None):
73 res = private_script_replacements(
74 windows_jenkins_any, module,
75 (requirements_local, requirements_pypi),
76 port, raise_exception=False,
77 default_engine_paths=_default_engine_paths,
78 platform=get_platform(platform))
79 return res.replace("virtual_env_suffix=%2", "virtual_env_suffix=___SUFFIX__")
82def _modified_linux_jenkins_any(requirements_local, requirements_pypi, module="__MODULE__",
83 port="__PORT__", platform=None):
84 res = private_script_replacements(
85 linux_jenkins_any, module,
86 (requirements_local, requirements_pypi),
87 port, raise_exception=False,
88 default_engine_paths=_default_engine_paths,
89 platform=get_platform(platform))
90 return res.replace("virtual_env_suffix=%2", "virtual_env_suffix=___SUFFIX__")
93class JenkinsExt(jenkins.Jenkins):
95 """
96 Extensions for the :epkg:`Jenkins` server
97 based on module :epkg:`python-jenkins`.
99 .. index:: Jenkins, Jenkins extensions
101 Some useful :epkg:`Jenkins` extensions:
103 * `Credentials Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Credentials+Plugin>`_
104 * `Extra Column Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Extra+Columns+Plugin>`_
105 * `Git Client Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Git+Client+Plugin>`_
106 * `GitHub Client Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Github+Plugin>`_
107 * `GitLab Client Plugin <https://wiki.jenkins-ci.org/display/JENKINS/GitLab+Plugin>`_
108 * `Matrix Project Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Matrix+Project+Plugin>`_
109 * `Build Pipeline Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Build+Pipeline+Plugin>`_
111 The whole class can define many different engines.
112 A job can send a mail at the end of the job execution.
113 """
115 _config_job = _config_job # pylint: disable=W0127
116 _trigger_up = _trigger_up # pylint: disable=W0127
117 _trigger_time = _trigger_time # pylint: disable=W0127
118 _trigger_startup = _trigger_startup # pylint: disable=W0127
119 _git_repo = _git_repo # pylint: disable=W0127
120 _task_batch_win = _task_batch_win # pylint: disable=W0127
121 _task_batch_lin = _task_batch_lin # pylint: disable=W0127
122 _publishers = _publishers # pylint: disable=W0127
123 _wipe_repo = _wipe_repo # pylint: disable=W0127
124 _artifacts = _artifacts # pylint: disable=W0127
125 _cleanup_repo = _cleanup_repo # pylint: disable=W0127
127 def __init__(self, url, username=None, password=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
128 mock=False, engines=None, platform=None, pypi_port=8067, fLOG=noLOG,
129 mails=None):
130 """
131 @param url url of the server
132 @param username username
133 @param password password
134 @param timeout timeout
135 @param mock True by default, if False, avoid talking to the server
136 @param engines list of Python engines *{name: path to python.exe}*
137 @param platform platform of the Jenkins server
138 @param pypi_port pypi port used for the documentation server
139 @param mails (str) list of mails to contact in case of a mistaje
140 @param fLOG logging function
142 If *platform* is None, it is replace by the value returned
143 by @see fn get_platform.
144 """
145 if platform is None:
146 platform = get_platform(platform)
147 jenkins.Jenkins.__init__(
148 self, url, username, password, timeout=timeout)
149 self._mock = mock
150 self.platform = platform
151 self.pypi_port = pypi_port
152 self.mails = mails
153 self.fLOG = fLOG
154 if engines is None:
155 engines = {"default": os.path.dirname(sys.executable)}
156 self.engines = engines
157 for k, v in self.engines.items():
158 if v.endswith(".exe"):
159 raise FileNotFoundError( # pragma: no cover
160 "{}:{} is not a folder".format(k, v))
161 if " " in v:
162 raise JenkinsJobException( # pragma: no cover
163 "No space allowed in engine path: " + v)
165 @property
166 def Engines(self):
167 """
168 @return the available engines
169 """
170 return self.engines
172 def jenkins_open(self, req, add_crumb=True, resolve_auth=True): # pragma: no cover
173 '''
174 Overloads the same method from module :epkg:`python-jenkins`
175 to replace string by bytes.
177 @param req see :epkg:`Jenkins API`
178 @param add_crumb see :epkg:`Jenkins API`
179 @param resolve_auth see :epkg:`Jenkins API`
180 '''
181 if self._mock:
182 raise JenkinsExtException("mocking server, cannot be open")
184 response = self.jenkins_request(
185 req=req, add_crumb=add_crumb, resolve_auth=resolve_auth)
186 if response is None:
187 raise jenkins.EmptyResponseException(
188 "Error communicating with server[%s]: "
189 "empty response" % self.server)
190 return response.content
192 def delete_job(self, name): # pragma: no cover
193 '''
194 Deletes :epkg:`Jenkins` job permanently.
196 :param name: name of :epkg:`Jenkins` job, ``str``
197 '''
198 if self._mock:
199 return
200 r = self._get_job_folder(name)
201 if r is None:
202 raise JenkinsExtException('delete[%s] failed (no job)' % (name))
204 folder_url, short_name = self._get_job_folder(name)
205 if folder_url is None:
206 raise ValueError("folder_url is None for job '{0}'".format(name))
207 self.jenkins_open(requests.Request(
208 'POST', self._build_url(jenkins.DELETE_JOB, locals())
209 ))
210 if self.job_exists(name) or self.job_exists(short_name):
211 raise jenkins.JenkinsException('delete[%s] failed' % (name))
213 def get_jobs(self, folder_depth=0, folder_depth_per_request=10, view_name=None):
214 """
215 Gets the list of all jobs recursively to the given folder depth,
216 see `get_all_jobs
217 <https://python-jenkins.readthedocs.org/en/latest/api.html
218 #jenkins.Jenkins.get_all_jobs>`_.
220 @return list of jobs, ``[ { str: str} ]``
221 """
222 return jenkins.Jenkins.get_jobs(self, folder_depth=folder_depth,
223 folder_depth_per_request=folder_depth_per_request,
224 view_name=view_name)
226 def delete_all_jobs(self): # pragma: no cover
227 """
228 Deletes all jobs permanently.
230 @return list of deleted jobs
231 """
232 jobs = self.get_jobs()
233 res = []
234 for k in jobs:
235 self.fLOG("[jenkins] remove job", k["name"])
236 self.delete_job(k["name"])
237 res.append(k["name"])
238 return res
240 def get_jenkins_job_name(self, job):
241 """
242 Infers a name for the jenkins job.
244 @param job str
245 @return name
246 """
247 if "<--" in job:
248 job = job.split("<--")[0]
249 if job.startswith("custom "):
250 return job.replace(" ", "_").replace("[", "").replace("]", "").strip("_")
251 def_prefix = ["doc", "setup", "setup_big"]
252 def_prefix.extend(self.engines.keys())
253 for prefix in def_prefix:
254 p = "[%s]" % prefix
255 if p in job:
256 job = p + " " + job.replace(" " + p, "")
257 return job.replace(" ", "_").replace("[", "").replace("]", "").strip("_")
259 def get_engine_from_job(self, job, return_key=False):
260 """
261 Extracts the engine from the job definition,
262 it should be like ``[engine]``.
264 @param job job string
265 @param return_key return the engine name too
266 @return engine or tuple(engine, name)
268 If their is no engine definition, the system
269 uses the default one (key=*default*) if it was defined.
270 Otherwise, it raises an exception.
271 """
272 res = None
273 spl = job.split()
274 for s in spl:
275 t = s.strip(" []")
276 if t in self.engines:
277 res = self.engines[t]
278 key = t
279 break
280 if res is None and "default" in self.engines:
281 res = self.engines["default"]
282 key = "default"
283 if res is None:
284 raise JenkinsJobException( # pragma: no cover
285 "Unable to find engine in job '{}', available: {}".format(
286 job, ", ".join(self.engines.keys())))
287 if "[27]" in job and "python34" in res.lower(): # pragma: no cover
288 mes = "\n".join(" {0}={1}".format(k, v)
289 for k, v in sorted(self.engines.items()))
290 raise ValueError(
291 "Python mismatch in version:\nJOB = {0}\nRES = {1}\nENGINES\n{2}"
292 "".format(job, res, mes))
293 return (res, key) if return_key else res
295 def get_cmd_standalone(self, job):
296 """
297 Custom command for :epkg:`Jenkins` (such as updating conda)
299 @param job module and options
300 @return script
301 """
302 spl = job.split()
303 if spl[0] != "standalone":
304 raise JenkinsExtException( # pragma: no cover
305 "the job should start by standalone: " + job)
307 if self.platform.startswith("win"):
308 # windows
309 if "[conda_update]" in spl:
310 cmd = "__ENGINE__\\Scripts\\conda update -y --all"
311 elif "[local_pypi]" in spl:
312 cmd = "if not exist ..\\..\\local_pypi mkdir ..\\local_pypi"
313 cmd += "\nif not exist ..\\..\\local_pypi\\local_pypi_server mkdir ..\\..\\local_pypi\\local_pypi_server"
314 cmd += "\necho __ENGINE__\\..\\Scripts\\pypi-server.exe -v -u -p __PORT__ --disable-fallback "
315 cmd += "..\\..\\local_pypi\\local_pypi_server > ..\\..\\local_pypi\\local_pypi_server\\start_local_pypi.bat"
316 cmd = cmd.replace("__PORT__", str(self.pypi_port))
317 elif "[update]" in spl:
318 cmd = "__ENGINE__\\python -u -c \"from pymyinstall.packaged import update_all;"
319 cmd += "update_all(temp_folder='build/update_modules', "
320 cmd += "verbose=True, source='2')\""
321 elif "[install]" in spl:
322 cmd = "__ENGINE__\\python -u -c \"from pymyinstall.packaged import install_all;install_all"
323 cmd += "(temp_folder='build/update_modules', "
324 cmd += "verbose=True, source='2')\""
325 else:
326 raise JenkinsExtException(
327 "cannot interpret job: " + job) # pragma: no cover
329 engine = self.get_engine_from_job(job)
330 cmd = cmd.replace("__ENGINE__", engine)
331 return cmd
332 else: # pragma: no cover
333 if "[conda_update]" in spl:
334 cmd = "__ENGINE__/bin/conda update -y --all"
335 elif "[local_pypi]" in spl:
336 cmd = 'if [-f ../local_pypi ]; then mkdir "../local_pypi"; fi'
337 cmd += '\nif [-f ../local_pypi/local_pypi_server]; then mkdir "../local_pypi/local_pypi_server"; fi'
338 cmd += "\necho pypi-server -v -u -p __PORT__ --disable-fallback "
339 cmd += "../local_pypi/local_pypi_server > ../local_pypi/local_pypi_server/start_local_pypi.sh"
340 cmd = cmd.replace("__PORT__", str(self.pypi_port))
341 elif "[update]" in spl:
342 cmd = "__ENGINE__/python -u -c \"from pymyinstall.packaged import update_all;"
343 cmd += "update_all(temp_folder='build/update_modules', "
344 cmd += "verbose=True, source='2')\""
345 elif "[install]" in spl:
346 cmd = "__ENGINE__/python -u -c \"from pymyinstall.packaged import install_all;install_all"
347 cmd += "(temp_folder='build/update_modules', "
348 cmd += "verbose=True, source='2')\""
349 else:
350 raise JenkinsExtException("cannot interpret job: " + job)
352 engine = self.get_engine_from_job(job)
353 cmd = cmd.replace("__ENGINE__", engine)
354 return cmd
356 @staticmethod
357 def get_cmd_custom(job):
358 """
359 Custom script for :epkg:`Jenkins`.
361 @param job module and options
362 @return script
363 """
364 spl = job.split()
365 if spl[0] != "custom":
366 raise JenkinsExtException(
367 "the job should start by custom: " + job)
368 # we expect __SCRIPTOPTIONS__ to be replaced by a script later on
369 return "__SCRIPTOPTIONS__"
371 @staticmethod
372 def hash_string(s, le=4):
373 """
374 Hashes a string.
376 @param s string
377 @param le cut the string to the first *l* character
378 @return hashed string
379 """
380 m = hashlib.md5()
381 m.update(s.encode("ascii"))
382 r = m.hexdigest().upper()
383 if len(r) < le:
384 return r # pragma: no cover
385 m = le // 2
386 return r[:m] + r[len(r) - le + m:]
388 def extract_requirements(self, job):
389 """
390 Extracts the requirements for a job.
392 @param job job name
393 @return 3-tuple job, local requirements, pipy requirements
395 Example::
397 "pyensae <-- pyquickhelper <---- qgrid"
399 The function returns::
401 (pyensae, ["pyquickhelper"], ["qgrid"])
402 """
403 if "<--" in job:
404 spl = job.split("<--")
405 job = spl[0]
406 rl, rp = None, None
407 for o in spl[1:]:
408 if o.startswith("--"):
409 rp = [_.strip("- ") for _ in o.split(",")]
410 else:
411 rl = [_.strip() for _ in o.split(",")]
412 return job, rl, rp
413 return job, None, None
415 def get_jenkins_script(self, job):
416 """
417 Builds the :epkg:`Jenkins` script for a module and its options.
419 @param job module and options
420 @return script
422 Method @see me setup_jenkins_server describes which tags this method can interpret.
423 The method allow command such as ``[custom...]``, they will be
424 run in a virtual environment as ``setup.py custom...``.
425 Parameter *job* can be ``empty``, in that case, this function returns an empty string.
426 Requirements local and from pipy can be specified by added in the job name:
428 * ``<-- module1, module2`` for local requirements
429 """
430 job_verbose = job
432 def replacements(cmd, engine, python, suffix, module_name):
433 res = cmd.replace("__ENGINE__", engine) \
434 .replace("__PYTHON__", python) \
435 .replace("__SUFFIX__", suffix + "_" + job_hash) \
436 .replace("__PORT__", str(self.pypi_port)) \
437 .replace("__MODULE__", module_name) # suffix for the virtual environment and module name
438 if "[27]" in job:
439 res = res.replace("__PYTHON27__", python)
440 if "__DEFAULTPYTHON__" in res:
441 if "default" not in self.engines:
442 raise JenkinsExtException( # pragma: no cover
443 "a default engine (Python 3.4) must be defined for script using Python 27, job={}".format(job))
444 res = res.replace("__DEFAULTPYTHON__",
445 os.path.join(self.engines["default"], "python"))
447 # patch for pyquickhelper
448 if "PACTHPQ" in res:
449 if hasattr(self, "PACTHPQ"):
450 if not hasattr(self, "pyquickhelper"):
451 raise RuntimeError( # pragma: no cover
452 "this should not happen:\n{0}\n---\n{1}".format(job_verbose, res))
453 if "pyquickhelper" in module_name:
454 repb = "@echo ~~SET set PYTHONPATH=src\nset PYTHONPATH=src"
455 else:
456 repb = "@echo ~~SET set PYTHONPATH={0}\nset PYTHONPATH={0}".format(self.pyquickhelper.replace(
457 "\\\\", "\\"))
458 repe = "@echo ~~SET set PYTHONPATH=\nset PYTHONPATH="
459 else:
460 repb = ""
461 repe = ""
462 res = res.replace("__PACTHPQb__", repb).replace(
463 "__PACTHPQe__", repe)
465 if "__" in res:
466 raise JenkinsJobException( # pragma: no cover
467 "unable to interpret command line: {}\nCMD: {}\nRES:\n{}".format(job_verbose, cmd, res))
469 # patch to avoid installing pyquickhelper when testing
470 # pyquickhelper
471 if module_name == "pyquickhelper":
472 lines = res.split("\n")
473 for i, line in enumerate(lines):
474 if "/simple/ pyquickhelper" in line and "--find-links http://localhost" in line:
475 lines[i] = "" # pragma: no cover
476 res = "\n".join(lines)
478 return res
480 # job hash
481 job_hash = JenkinsExt.hash_string(job)
483 # extact requirements
484 job, requirements_local, requirements_pypi = self.extract_requirements(
485 job)
486 spl = job.split()
487 module_name = spl[0]
489 if self.platform.startswith("win"):
490 # windows
491 engine, namee = self.get_engine_from_job(job, True)
492 python = os.path.join(engine, "python.exe")
494 if len(spl) == 1:
495 script = _modified_windows_jenkins(
496 requirements_local, requirements_pypi, platform=self.platform)
497 if not isinstance(script, list):
498 script = [script]
499 return [replacements(s, engine, python, namee + "_" + job_hash, module_name) for s in script]
501 if len(spl) == 0:
502 raise ValueError("job is empty") # pragma: no cover
504 if spl[0] == "standalone":
505 # conda update
506 return self.get_cmd_standalone(job)
508 if spl[0] == "custom":
509 # custom script
510 return JenkinsExt.get_cmd_custom(job)
512 if spl[0] == "empty":
513 return "" # pragma: no cover
515 if len(spl) in [2, 3, 4, 5]:
516 # step 1: define the script
518 if "[test_local_pypi]" in spl: # pragma: no cover
519 cmd = """__PYTHON__ -u setup.py test_local_pypi"""
520 cmd = "auto_setup_test_local_pypi.bat __PYTHON__"
521 elif "[update_modules]" in spl:
522 cmd = """__PYTHON__ -u -c "import sys;sys.path.append('src');from pymyinstall.packaged import update_all;""" + \
523 """update_all(temp_folder='build/update_modules', verbose=True, source='2')" """
524 elif "[UT]" in spl:
525 parameters = [_ for _ in spl if _.startswith(
526 "{") and _.endswith("}")]
527 if len(parameters) != 1:
528 raise ValueError( # pragma: no cover
529 "Unable to extract parameters for the unittests:"
530 "\n{0}".format(" ".join(spl)))
531 p = parameters[0].replace("_", " ").strip("{}")
532 cmd = _modified_windows_jenkins_any(requirements_local, requirements_pypi, platform=self.platform).replace(
533 "__COMMAND__", "unittests " + p)
534 elif "[LONG]" in spl:
535 cmd = _modified_windows_jenkins_any(requirements_local, requirements_pypi, platform=self.platform).replace(
536 "__COMMAND__", "unittests_LONG")
537 elif "[SKIP]" in spl:
538 cmd = _modified_windows_jenkins_any( # pragma: no cover
539 requirements_local, requirements_pypi, platform=self.platform).replace(
540 "__COMMAND__", "unittests_SKIP")
541 elif "[GUI]" in spl:
542 cmd = _modified_windows_jenkins_any( # pragma: no cover
543 requirements_local, requirements_pypi, platform=self.platform).replace(
544 "__COMMAND__", "unittests_GUI")
545 elif "[27]" in spl:
546 cmd = _modified_windows_jenkins_27(
547 requirements_local, requirements_pypi, anaconda=" [anaconda" in job, platform=self.platform)
548 if not isinstance(cmd, list):
549 cmd = [cmd] # pragma: no cover
550 else:
551 cmd = list(cmd)
552 if spl[0] == "pyquickhelper":
553 # exception for this job, we don't want to import pyquickhelper
554 # c:/jenkins/pymy/anaconda2_pyquickhelper_27/../virtual/pyquickhelper_conda27vir/Scripts/pip
555 # install --no-cache-dir --index
556 # http://localhost:8067/simple/ pyquickhelper
557 for i in range(0, len(cmd)):
558 lines = cmd[i].split("\n")
559 lines = [
560 (_ if "simple/ pyquickhelper" not in _ else "rem do not import pyquickhelper") for _ in lines]
561 cmd[i] = "\n".join(lines)
562 elif "[doc]" in spl:
563 # documentation
564 cmd = _modified_windows_jenkins_any(requirements_local, requirements_pypi, platform=self.platform).replace(
565 "__COMMAND__", "build_sphinx")
566 else:
567 cmd = _modified_windows_jenkins(
568 requirements_local, requirements_pypi, platform=self.platform)
569 for pl in spl[1:]:
570 if pl.startswith("[custom_") and pl.endswith("]"):
571 cus = pl.strip("[]")
572 cmd = _modified_windows_jenkins_any(requirements_local, requirements_pypi,
573 platform=self.platform).replace("__COMMAND__", cus)
575 # step 2: replacement (python __PYTHON__, virtual environnement
576 # __SUFFIX__)
578 cmds = cmd if isinstance(cmd, list) else [cmd]
579 res = []
580 for cmd in cmds:
581 cmdn = replacements(cmd, engine, python,
582 namee + "_" + job_hash, module_name)
583 if "run27" in cmdn and (
584 "Python34" in cmdn or "Python35" in cmdn or
585 "Python36" in cmdn or "Python37" in cmdn or
586 "Python38" in cmdn or "Python39" in cmdn):
587 raise ValueError( # pragma: no cover
588 "Python version mismatch\nENGINE\n{2}\n----BEFORE"
589 "\n{0}\n-----\nAFTER\n-----\n{1}".format(cmd, cmdn, engine))
590 res.append(cmdn)
592 return res
593 else:
594 raise ValueError("unable to interpret: " +
595 job) # pragma: no cover
596 else: # pragma: no cover
597 # linux
598 engine, namee = self.get_engine_from_job(job, True)
599 if engine is None:
600 python = "python%d.%d" % sys.version_info[:2]
601 elif namee.startswith('py'):
602 vers = (int(namee[2:3]), int(namee[3:]))
603 python = "python%d.%d" % vers
604 else:
605 raise ValueError(
606 "Unable to handle engine ='{}', namee='{}'.".format(engine, namee))
608 if len(spl) == 1:
609 script = _modified_linux_jenkins(
610 requirements_local, requirements_pypi, platform=self.platform)
611 if not isinstance(script, list):
612 script = [script]
613 return [replacements(s, engine, python, namee + "_" + job_hash, module_name) for s in script]
615 elif len(spl) == 0:
616 raise ValueError("job is empty")
618 elif spl[0] == "standalone":
619 # conda update
620 return self.get_cmd_standalone(job)
622 elif spl[0] == "empty":
623 return ""
625 elif len(spl) in [2, 3, 4, 5]:
626 # step 1: define the script
628 if "[test_local_pypi]" in spl:
629 cmd = """__PYTHON__ -u setup.py test_local_pypi"""
630 cmd = "auto_setup_test_local_pypi.bat __PYTHON__"
631 elif "[update_modules]" in spl:
632 cmd = """__PYTHON__ -u -c "import sys;sys.path.append('src');from pymyinstall.packaged import update_all;""" + \
633 """update_all(temp_folder='build/update_modules', verbose=True, source='2')" """
635 else:
636 cmd = _modified_linux_jenkins(
637 requirements_local, requirements_pypi, platform=self.platform)
638 for pl in spl[1:]:
639 if pl.startswith("[custom_") and pl.endswith("]"):
640 cus = pl.strip("[]")
641 cmd = _modified_linux_jenkins_any(requirements_local, requirements_pypi,
642 platform=self.platform).replace("__COMMAND__", cus)
644 # step 2: replacement (python __PYTHON__, virtual environnement
645 # __SUFFIX__)
647 cmds = cmd if isinstance(cmd, list) else [cmd]
648 res = []
649 for cmd in cmds:
650 cmdn = replacements(cmd, engine, python,
651 namee + "_" + job_hash, module_name)
652 if "run27" in cmdn and (
653 "Python34" in cmdn or "Python35" in cmdn or
654 "Python36" in cmdn or "Python37" in cmdn or
655 "Python38" in cmdn or "Python39" in cmdn):
656 raise ValueError(
657 "Python version mismatch\nENGINE\n{2}\n----BEFORE\n{0}\n-----\nAFTER\n-----\n{1}".format(cmd, cmdn, engine))
658 res.append(cmdn)
660 return res
662 # other possibilities
663 raise NotImplementedError("On Linux, unable to interpret: " + job)
665 def adjust_scheduler(self, scheduler, adjust_scheduler=True):
666 """
667 Adjusts the scheduler to avoid having two jobs starting at the same time,
668 jobs are delayed by an hour, two hours, three hours...
670 @param scheduler existing scheduler
671 @param adjust_scheduler True to change it
672 @return new scheduler (only hours are changed)
674 The function uses member ``_scheduled_jobs``.
675 It creates it if it does not exist.
676 """
677 if not adjust_scheduler:
678 return scheduler # pragma: no cover
679 if scheduler is None:
680 raise ValueError("scheduler is None") # pragma: no cover
681 if not hasattr(self, "_scheduled_jobs"):
682 self._scheduled_jobs = {}
683 if scheduler not in self._scheduled_jobs:
684 self._scheduled_jobs[scheduler] = 1
685 return scheduler
686 else:
687 if "H(" in scheduler:
688 cp = re.compile("H[(]([0-9]+-[0-9]+)[)]")
689 f = cp.findall(scheduler)
690 if len(f) != 1:
691 raise ValueError( # pragma: no cover
692 "Unable to find hours in the scheduler '{0}', expects 'H(a-b)'".format(scheduler))
693 a, b = f[0].split('-')
694 a0 = a
695 a = int(a)
696 b = int(b)
697 new_value = scheduler
698 rep = 'H(%s)' % f[0]
699 iter = 0
700 while iter < 100 and (new_value in self._scheduled_jobs or (a0 == a)):
701 a += 1
702 b += 1
703 if a >= 24 or b > 24:
704 a = 0 # pragma: no cover
705 b = 1 + iter // 24 # pragma: no cover
706 r = 'H(%d-%d)' % (a, b)
707 new_value = scheduler.replace(rep, r)
708 iter += 1
709 scheduler = new_value
710 self._scheduled_jobs[
711 scheduler] = self._scheduled_jobs.get(scheduler, 0) + 1
712 return scheduler
714 def create_job_template(self, name, git_repo, credentials="", upstreams=None, script=None,
715 location=None, keep=10, scheduler=None, py27=False, description=None,
716 default_engine_paths=None, success_only=False, update=False,
717 timeout=_timeout_default, additional_requirements=None,
718 return_job=False, adjust_scheduler=True, clean_repo=True, **kwargs):
719 """
720 Adds a job to the :epkg:`Jenkins` server.
722 @param name name
723 @param credentials credentials
724 @param git_repo git repository
725 @param upstreams the build must run after... (even if failures),
726 must be None in that case
727 @param script script to execute or list of scripts
728 @param keep number of builds to keep
729 @param location location of the build
730 @param scheduler add a schedule time (upstreams must be None in that case)
731 @param py27 python 2.7 (True) or Python 3 (False)
732 @param description add a description to the job
733 @param default_engine_paths define the default location for python engine,
734 should be dictionary ``{ engine: path }``, see below.
735 @param success_only only triggers the job if the previous one was successful
736 @param update update the job instead of creating it
737 @param additional_requirements requirements for this module built by this Jenkins server,
738 otherthise, we assume they are available
739 on the installed distribution
740 @param timeout specify a timeout
741 @param kwargs additional parameters
742 @param adjust_scheduler adjust the scheduler of a job so that it is delayed if this spot
743 is already taken
744 @param return_job return job instead of submitting the job
745 @param clean_repo clean the repository before building (default is yes)
747 The job can be modified on Jenkins. To add a time trigger::
749 H H(13-14) * * *
751 Same trigger but once every week and not every day (Sunday for example)::
753 H H(13-14) * * 0
755 Parameter *success_only* prevents a job from running if the previous one failed.
756 Options *success_only* must be specified.
757 Parameter *update* updates a job instead of creating it.
758 """
759 if 'platform' in kwargs:
760 raise NameError( # pragma: no cover
761 "Parameter 'platform' should be set up in the constructor.")
762 if script is None:
763 if self.platform.startswith("win"):
764 if default_engine_paths is None and "default" in self.engines:
765 ver = "__PY%d%d__" % sys.version_info[:2]
766 pat = os.path.join(self.engines["default"], "python")
767 default_engine_paths = dict(
768 windows={ver: pat, "__PYTHON__": pat})
770 script = private_script_replacements(
771 windows_jenkins, "____", additional_requirements, "____",
772 raise_exception=False, platform=self.platform,
773 default_engine_paths=default_engine_paths)
775 hash = JenkinsExt.hash_string(script)
776 script = script.replace("__SUFFIX__", hash)
777 else:
778 raise JenkinsExtException(
779 "no default script for linux") # pragma: no cover
781 if upstreams is not None and len(upstreams) > 0 and scheduler is not None:
782 raise JenkinsExtException(
783 "upstreams and scheduler cannot be not null at the same time: {0}".format(name))
785 # overwrite parameters with job_options
786 job_options = kwargs.get('job_options', None)
787 if job_options is not None:
788 job_options = job_options.copy()
789 if "scheduler" in job_options:
790 scheduler = job_options["scheduler"]
791 del job_options["scheduler"]
792 if "git_repo" in job_options:
793 git_repo = job_options["git_repo"]
794 del job_options["git_repo"]
795 if "credentials" in job_options:
796 credentials = job_options["credentials"]
797 del job_options["credentials"]
799 if upstreams is not None and len(upstreams) > 0:
800 trigger = JenkinsExt._trigger_up \
801 .replace("__UP__", ",".join(upstreams)) \
802 .replace("__FAILURE__", "SUCCESS" if success_only else "FAILURE") \
803 .replace("__ORDINAL__", "0" if success_only else "2") \
804 .replace("__COLOR__", "BLUE" if success_only else "RED")
805 elif scheduler is not None:
806 if scheduler.lower() == "startup":
807 trigger = JenkinsExt._trigger_startup
808 elif scheduler.lower() == "NONE":
809 trigger = "" # pragma: no cover
810 else:
811 new_scheduler = self.adjust_scheduler(
812 scheduler, adjust_scheduler)
813 trigger = JenkinsExt._trigger_time.replace(
814 "__SCHEDULER__", new_scheduler)
815 if description is not None:
816 description = description.replace(scheduler, new_scheduler)
817 scheduler = new_scheduler
818 else:
819 trigger = ""
821 if not isinstance(script, list):
822 script = [script]
824 underscore = re.compile("(__[A-Z_]+__)")
826 # we modify the scripts
827 script_mod = []
828 for scr in script:
829 search = underscore.search(scr)
830 if search:
831 raise ValueError( # pragma: no cover
832 "script still contains __\ndefault_engine_paths: {}\n"
833 "found: {}\nscr:\n{}\nSCRIPT:\n{}\n".format(
834 default_engine_paths, search.groups()[0],
835 scr, str(script)))
836 script_mod.append(scr)
838 # wrappers
839 bwrappers = []
841 # repo
842 if clean_repo:
843 wipe = JenkinsExt._wipe_repo
844 bwrappers.append(JenkinsExt._cleanup_repo)
845 else:
846 wipe = ""
847 if git_repo is None:
848 git_repo_xml = ""
849 else:
850 if not isinstance(git_repo, str):
851 raise TypeError( # pragma: no cover
852 "git_repo must be str not '{0}'".format(git_repo))
853 git_repo_xml = JenkinsExt._git_repo \
854 .replace("__GITREPO__", git_repo) \
855 .replace("__WIPE__", wipe) \
856 .replace("__CRED__", "<credentialsId>%s</credentialsId>" % credentials)
858 # additional scripts
859 before = []
860 if job_options is not None:
861 if 'scripts' in job_options:
862 lscripts = job_options['scripts']
863 for scr in lscripts:
864 au = _file_creation.replace("__FILENAME__", scr["name"]) \
865 .replace("__CONTENT__", scr["content"])
866 if "__" in au:
867 raise Exception(
868 "Unable to fully replace expected string in:\n{0}".format(au))
869 before.append(au)
870 del job_options['scripts']
871 if len(job_options) > 0:
872 keys = ", ".join(
873 ["credentials", "git_repo", "scheduler", "scripts"])
874 raise ValueError( # pragma: no cover
875 "Unable to process options\n{0}\nYou can specify the "
876 "following options:\n{1}".format(job_options, keys))
878 # scripts
879 # tasks is XML, we need to encode s into XML format
880 if self.platform.startswith("win"):
881 scr = JenkinsExt._task_batch_win
882 else:
883 scr = JenkinsExt._task_batch_lin
884 tasks = before + [scr.replace("__SCRIPT__", escape(s))
885 for s in script_mod]
887 # location
888 if location is not None and "<--" in location:
889 raise Exception( # pragma: no cover
890 "this should not happen")
891 location = "" if location is None else "<customWorkspace>%s</customWorkspace>" % location
893 # emailing
894 publishers = []
895 mails = kwargs.get("mails", None)
896 if mails is not None:
897 publishers.append( # pragma: no cover
898 JenkinsExt._publishers.replace("__MAIL__", mails))
899 publishers.append(JenkinsExt._artifacts.replace(
900 "__PATTERN__", "dist/*.whl,dist/*.zip"))
902 # replacements
903 conf = JenkinsExt._config_job
904 rep = dict(__KEEP__=str(keep),
905 __TASKS__="\n".join(tasks),
906 __TRIGGER__=trigger,
907 __LOCATION__=location,
908 __DESCRIPTION__="" if description is None else description,
909 __GITREPOXML__=git_repo_xml,
910 __TIMEOUT__=str(timeout),
911 __PUBLISHERS__="\n".join(publishers),
912 __BUILDWRAPPERS__="\n".join(bwrappers))
914 for k, v in rep.items():
915 conf = conf.replace(k, v)
917 # final processing
918 conf = jenkins_final_postprocessing(conf, py27)
920 if self._mock or return_job:
921 return conf
922 if update:
923 return self.reconfig_job(name, conf) # pragma: no cover
924 return self.create_job(name, conf) # pragma: no cover
926 def process_options(self, script, options):
927 """
928 Postprocesses a script inserted in a job definition.
930 @param script script to execute (in a list)
931 @param options dictionary with options
932 @return new script
933 """
934 if not isinstance(script, list):
935 script = [script]
936 for k, v in options.items():
937 if k == "pre":
938 script.insert(0, v)
939 elif k == "post":
940 script.append(v)
941 elif k == "pre_set":
942 script = [v + "\n" + _ for _ in script] # pragma: no cover
943 elif k == "post_set":
944 script = [_ + "\n" + v for _ in script] # pragma: no cover
945 elif k == "script":
946 script = [_.replace("__SCRIPTOPTIONS__", v) for _ in script]
947 else:
948 raise JenkinsJobException( # pragma: no cover
949 "unable to interpret options: " + str(options))
950 return script
952 def setup_jenkins_server(self, github, modules, get_jenkins_script=None, overwrite=False,
953 location=None, prefix="", credentials="", update=True, yml_engine="jinja2",
954 add_environ=True, disable_schedule=False, adjust_scheduler=True):
955 """
956 Sets up many jobs in :epkg:`Jenkins`.
958 @param github github account if it does not start with *http://*,
959 the link to git repository of the project otherwise,
960 we assume all jobs in *modules* are located on the same
961 account otherwise the function will have to called twice with
962 different parameters
963 @param modules modules for which to generate the
964 @param get_jenkins_script see @see me get_jenkins_script (default value if this parameter is None)
965 @param overwrite do not create the job if it already exists
966 @param location None for default or a local folder
967 @param prefix add a prefix to the name
968 @param credentials credentials to use for the job (string or dictionary)
969 @param update update job instead of deleting it if the job already exists
970 @param yml_engine templating engine used to process yaml config files
971 @param add_environ use of local environment variables to interpret the job
972 @param adjust_scheduler adjust the scheduler of a job so that it is delayed if this spot is already taken
973 @param disable_schedule disable scheduling for all jobs
974 @return list of created jobs
976 If *credentials* are a dictionary, the function looks up
977 into it by using the git repository as a key. If it does not find
978 it, it looks for default key. If there is not found,
979 the function assumes, there is not credentials for this git repository.
981 The function *get_jenkins_script* is called with the following parameters:
983 * job
985 The extension
986 `Extra Columns Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Extra+Columns+Plugin>`_
987 is very useful to add extra columns to a view (the description, the output of the
988 last execution). Here is a list of useful extensions:
990 * `Build Graph View Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Build+Graph+View+Plugin>`_
991 * `Build Pipeline Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Build+Pipeline+Plugin>`_
992 * `Credentials Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Credentials+Plugin>`_
993 * `Extra Columns Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Extra+Columns+Plugin>`_
994 * `Git Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Git+Plugin>`_
995 * `GitHub Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Github+Plugin>`_
996 * `GitLab Plugin <https://wiki.jenkins-ci.org/display/JENKINS/GitLab+Plugin>`_
997 * :epkg:`Python`
998 * `Python Wrapper Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Python+Wrapper+Plugin>`_
999 * `Build timeout plugin <https://wiki.jenkins-ci.org/display/JENKINS/Build-timeout+Plugin>`_
1001 Tag description:
1003 * ``[engine]``: to use this specific engine (Python path)
1004 * ``[27]``: run with python 2.7
1005 * ``[LONG]``: run longer unit tests (files start by ``test_LONG_``)
1006 * ``[SKIP]``: run skipped unit tests (files start by ``test_SKIP_``)
1007 * ``[GUI]``: run skipped unit tests (files start by ``test_GUI_``)
1008 * ``[custom.+]``: run ``setup.py <custom.+>`` in a virtual environment
1009 * ``[UT] {-d_10}``: run ``setup.py unittests -d 10`` in a virtual environment, ``-d 10`` is one of the possible parameters
1011 Others tags:
1013 * ``[conda_update]``: update conda distribution
1014 * ``[update]``: update distribution
1015 * ``[install]``: update distribution
1016 * ``[local_pypi]``: write a script to run a local pypi server on port 8067 (default option)
1017 * ``pymyinstall [update_modules]``: run a script to update all modules
1018 (might have to be ran a couple of times before being successful)
1020 *modules* is a list defined as follows:
1022 * each element can be a string or a tuple (string, schedule time) or a list
1023 * if it is a list, it contains a list of elements defined as previously
1024 * if the job at position i is not scheduled, it will start after the last
1025 job at position i-1 whether or not it fails
1026 * the job can be defined as a tuple of 3 elements, the last one contains options
1028 The available options are:
1030 * pre: defines a string to insert at the beginning of a job
1031 * post: defines a string to insert at the end of a job
1032 * script: defines a full script if the job to execute is ``custom``
1034 Example ::
1036 modules=[ # update anaconda
1037 ("standalone [conda_update] [anaconda3]",
1038 "H H(0-1) * * 0"),
1039 "standalone [conda_update] [anaconda2] [27]",
1040 "standalone [local_pypi]",
1041 #"standalone [install]",
1042 #"standalone [update]",
1043 #"standalone [install] [py34]",
1044 #"standalone [update] [py34]",
1045 #"standalone [install] [winpython]",
1046 #"standalone [update] [winpython]",
1047 # pyquickhelper and others,
1048 ("pyquickhelper", "H H(2-3) * * 0"),
1049 ("pysqllike <-- pyquickhelper", None, dict(success_only=True)),
1050 ["python3_module_template <-- pyquickhelper",
1051 "pyquickhelper [27] [anaconda2]"],
1052 ["pyquickhelper [winpython]",
1053 "python3_module_template [27] [anaconda2] <-- pyquickhelper", ],
1054 ["pymyinstall <-- pyquickhelper", "pyensae <-- pyquickhelper"],
1055 ["pymmails <-- pyquickhelper", "pyrsslocal <-- pyquickhelper, pyensae"],
1056 ["pymyinstall [27] [anaconda2] <-- pyquickhelper", "pymyinstall [LONG] <-- pyquickhelper"],
1057 # update, do not move, it depends on pyquickhelper
1058 ("pyquickhelper [anaconda3]", "H H(2-3) * * 1"),
1059 ["pyquickhelper [winpython]", "pysqllike [anaconda3]",
1060 "pysqllike [winpython] <-- pyquickhelper",
1061 "python3_module_template [anaconda3] <-- pyquickhelper",
1062 "python3_module_template [winpython] <-- pyquickhelper",
1063 "pymmails [anaconda3] <-- pyquickhelper",
1064 "pymmails [winpython] <-- pyquickhelper",
1065 "pymyinstall [anaconda3] <-- pyquickhelper",
1066 "pymyinstall [winpython] <-- pyquickhelper"],
1067 ["pyensae [anaconda3] <-- pyquickhelper",
1068 "pyensae [winpython] <-- pyquickhelper",
1069 "pyrsslocal [anaconda3] <-- pyquickhelper, pyensae",
1070 "pyrsslocal [winpython] <-- pyquickhelper"],
1071 ("pymyinstall [update_modules]",
1072 "H H(0-1) * * 5"),
1073 "pymyinstall [update_modules] [winpython]",
1074 "pymyinstall [update_modules] [py34]",
1075 "pymyinstall [update_modules] [anaconda2]",
1076 "pymyinstall [update_modules] [anaconda3]",
1077 # py35
1078 ("pyquickhelper [py34]", "H H(2-3) * * 2"),
1079 ["pysqllike [py34]",
1080 "pymmails [py34] <-- pyquickhelper",
1081 "python3_module_template [py34] <-- pyquickhelper",
1082 "pymyinstall [py34] <-- pyquickhelper"],
1083 "pyensae [py34] <-- pyquickhelper",
1084 "pyrsslocal [py34] <-- pyquickhelper, pyensae",
1085 ],
1087 Example::
1089 from ensae_teaching_cs.automation.jenkins_helper import setup_jenkins_server
1090 from pyquickhelper.jenkinshelper import JenkinsExt
1092 engines = dict(Anaconda2=r"C:\\Anaconda2",
1093 Anaconda3=r"C:\\Anaconda3",
1094 py35=r"c:\\Python35_x64",
1095 py36=r"c:\\Python36_x64",
1096 default=r"c:\\Python36_x64",
1097 custom=r"c:\\CustomPython")
1099 js = JenkinsExt('http://machine:8080/', "user", "password", engines=engines)
1101 if True:
1102 js.setup_jenkins_server(github="sdpython", overwrite = True,
1103 location = r"c:\\jenkins\\pymy")
1106 Another example::
1108 import sys
1109 sys.path.append(r"C:\\<path>\\ensae_teaching_cs\\src")
1110 sys.path.append(r"C:\\<path>\\pyquickhelper\\src")
1111 sys.path.append(r"C:\\<path>\\pyensae\\src")
1112 sys.path.append(r"C:\\<path>\\pyrsslocal\\src")
1113 from ensae_teaching_cs.automation.jenkins_helper import setup_jenkins_server, JenkinsExt
1114 js = JenkinsExt("http://<machine>:8080/", <user>, <password>)
1115 js.setup_jenkins_server(location=r"c:\\jenkins\\pymy", overwrite=True, engines=engines)
1117 Parameter *credentials* can be a dictionary where the key is
1118 the git repository. Parameter *dependencies* and *no_dep*
1119 were removed. Dependencies are now specified
1120 in the job name using ``<--`` and they exclusively rely
1121 on pipy (local or remote). Add options for module
1122 *Build Timeout Plugin*.
1123 """
1124 # we do a patch for pyquickhelper
1125 all_jobs = []
1126 for jobs in modules:
1127 jobs = jobs if isinstance(jobs, list) else [jobs]
1128 for job in jobs:
1129 if isinstance(job, tuple):
1130 job = job[0]
1131 job = job.split("<--")[0]
1132 name = self.get_jenkins_job_name(job)
1133 all_jobs.append(name)
1134 all_jobs = set(all_jobs)
1135 if "pyquickhelper" in all_jobs:
1136 self.PACTHPQ = True
1137 self.pyquickhelper = os.path.join(
1138 location, "_pyquickhelper", "src")
1140 # rest of the function
1141 if get_jenkins_script is None:
1142 get_jenkins_script = JenkinsExt.get_jenkins_script
1144 if github is not None and "https://" not in github:
1145 github = "https://github.com/" + github + "/"
1147 deps = []
1148 created = []
1149 locations = []
1150 indexes = dict(order=0, dozen="A")
1151 counts = {}
1152 for jobs in modules:
1153 if isinstance(jobs, tuple):
1154 if len(jobs) == 0:
1155 raise ValueError(
1156 "Empty jobs in the list.") # pragma: no cover
1157 if jobs[0] == "yml" and len(jobs) != 3:
1158 raise ValueError( # pragma: no cover
1159 "If it is a yml jobs, the tuple should contain 3 elements: ('yml', filename, schedule or None or dictionary).\n" +
1160 "Not: {0}".format(jobs))
1162 cre, ds, locs = self._setup_jenkins_server_modules_loop(
1163 jobs=jobs, counts=counts,
1164 get_jenkins_script=get_jenkins_script,
1165 location=location, adjust_scheduler=adjust_scheduler,
1166 add_environ=add_environ, yml_engine=yml_engine,
1167 overwrite=overwrite, prefix=prefix,
1168 credentials=credentials, github=github,
1169 disable_schedule=disable_schedule, jenkins_server=self,
1170 update=update, indexes=indexes, deps=deps)
1171 created.extend(cre)
1172 locations.extend(locs)
1173 deps.extend(ds)
1174 return created
1176 def _setup_jenkins_server_modules_loop(self, jobs, counts, get_jenkins_script, location, adjust_scheduler,
1177 add_environ, yml_engine, overwrite, prefix, credentials, github,
1178 disable_schedule, jenkins_server, update, indexes, deps):
1179 if not isinstance(jobs, list):
1180 jobs = [jobs]
1181 indexes["unit"] = 0
1182 new_dep = []
1183 created = []
1184 locations = []
1185 for i, job in enumerate(jobs):
1186 indexes["unit"] += 1
1187 cre, dep, loc = self._setup_jenkins_server_job_iteration(
1188 job, counts=counts,
1189 get_jenkins_script=get_jenkins_script,
1190 location=location, adjust_scheduler=adjust_scheduler,
1191 add_environ=add_environ, yml_engine=yml_engine,
1192 overwrite=overwrite, prefix=prefix,
1193 credentials=credentials, github=github,
1194 disable_schedule=disable_schedule,
1195 jenkins_server=jenkins_server,
1196 update=update, indexes=indexes,
1197 deps=deps, i=i)
1198 created.extend(cre)
1199 new_dep.extend(dep)
1200 locations.extend(loc)
1201 if len(new_dep) > 20000:
1202 raise JenkinsExtException( # pragma: no cover
1203 "unreasonable number of dependencies: {0}".format(len(new_dep)))
1204 return created, new_dep, locations
1206 def _setup_jenkins_server_job_iteration(self, job, get_jenkins_script, location, adjust_scheduler,
1207 add_environ, yml_engine, overwrite, prefix, credentials, github,
1208 disable_schedule, jenkins_server, update, indexes, deps, i, counts):
1209 order = indexes["order"]
1210 dozen = indexes["dozen"]
1211 unit = indexes["unit"]
1212 new_dep = []
1213 created = []
1214 locations = []
1216 if isinstance(job, tuple):
1217 if len(job) < 2:
1218 raise JenkinsJobException( # pragma: no cover
1219 "the tuple must contain at least two elements:\nJOB:"
1220 "\n" + str(job))
1222 if job[0] == "yml":
1223 is_yml = True
1224 job = job[1:]
1225 else:
1226 is_yml = False
1228 # we extract options if any
1229 if len(job) == 3:
1230 options = job[2]
1231 if not isinstance(options, dict):
1232 raise JenkinsJobException( # pragma: no cover
1233 "The last element of the tuple must be a dictionary:\nJOB:\n" + str(options))
1234 else:
1235 options = {}
1237 # job and scheduler
1238 job, scheduler_options = job[:2]
1239 if isinstance(scheduler_options, dict):
1240 scheduler = scheduler_options.get('scheduler', None)
1241 else:
1242 scheduler = scheduler_options
1243 scheduler_options = None
1244 if scheduler is not None:
1245 order = 1
1246 if counts.get(dozen, 0) > 0:
1247 dozen = chr(ord(dozen) + 1)
1248 else:
1249 if i == 0:
1250 order += 1
1251 else:
1252 scheduler = None
1253 if i == 0:
1254 order += 1
1255 options = {}
1256 is_yml = False
1258 # all schedule are disabled if disable_schedule is True
1259 if disable_schedule:
1260 scheduler = None
1261 counts[dozen] = counts.get(dozen, 0) + 1
1263 # success_only
1264 if "success_only" in options:
1265 success_only = options["success_only"]
1266 del options["success_only"]
1267 else:
1268 success_only = False
1270 # timeout
1271 if "timeout" in options:
1272 timeout = options["timeout"]
1273 del options["timeout"]
1274 else:
1275 timeout = _timeout_default
1277 # script
1278 if not is_yml:
1279 script = get_jenkins_script(self, job)
1281 # we process the repository
1282 if "repo" in options:
1283 gitrepo = options["repo"]
1284 options = options.copy()
1285 del options["repo"]
1286 else:
1287 gitrepo = github
1289 # add a description to the job
1290 description = ["%s%02d%02d" % (dozen, order, unit)]
1291 if scheduler is not None:
1292 description.append(scheduler)
1293 try:
1294 description = " - ".join(description)
1295 except TypeError as e: # pragma: no cover
1296 raise TypeError("Issue with {}.".format(description)) from e
1298 # credentials
1299 if isinstance(credentials, dict): # pragma: no cover
1300 cred = credentials.get(gitrepo, None)
1301 if cred is None:
1302 cred = credentials.get("default", "")
1303 else:
1304 cred = credentials
1306 if not is_yml:
1307 mod = job.split()[0]
1308 name = self.get_jenkins_job_name(job)
1309 jname = prefix + name
1311 try:
1312 j = jenkins_server.get_job_config(
1313 jname) if not jenkins_server._mock else None
1314 except jenkins.NotFoundException: # pragma: no cover
1315 j = None
1316 except jenkins.JenkinsException as e: # pragma: no cover
1317 raise JenkinsExtException(
1318 "unable to retrieve job config for job={0}, name={1}".format(job, jname)) from e
1320 if overwrite or j is None:
1322 update_job = False
1323 if j is not None: # pragma: no cover
1324 if update:
1325 update_job = True
1326 else:
1327 self.fLOG("[jenkins] delete job", jname)
1328 jenkins_server.delete_job(jname)
1330 # we post process the script
1331 script = self.process_options(script, options)
1333 # if there is a script
1334 if script is not None and len(script) > 0:
1335 new_dep.append(name)
1336 upstreams = [] if (
1337 scheduler is not None) else deps[-1:]
1338 self.fLOG("[jenkins] create job", jname, " - ", job,
1339 " : ", scheduler, " / ", upstreams)
1341 # set up location
1342 if location is None:
1343 loc = None # pragma: no cover
1344 else:
1345 if "_" in jname:
1346 loc = os.path.join(location, name, jname)
1347 else:
1348 loc = os.path.join(location, name, "_" + jname)
1350 if mod in ("standalone", "custom"):
1351 gpar = None
1352 elif gitrepo is None:
1353 raise JenkinsJobException( # pragma: no cover
1354 "gitrepo cannot must not be None if standalone or "
1355 "custom is not defined,\njob=" + str(job))
1356 elif gitrepo.endswith(".git"):
1357 gpar = gitrepo
1358 else:
1359 gpar = gitrepo + "%s/" % mod
1361 # create the template
1362 r = jenkins_server.create_job_template(jname, git_repo=gpar, upstreams=upstreams, script=script,
1363 location=loc, scheduler=scheduler, py27="[27]" in job,
1364 description=description, credentials=cred, success_only=success_only,
1365 update=update_job, timeout=timeout, adjust_scheduler=adjust_scheduler,
1366 mails=self.mails)
1368 # check some inconsistencies
1369 if "[27]" in job and "Anaconda3" in script:
1370 raise JenkinsExtException( # pragma: no cover
1371 "incoherence for job {0}, script:\n{1}".format(job, script))
1373 locations.append((job, loc))
1374 created.append((job, name, loc, job, r))
1375 else: # pragma: no cover
1376 # skip the job
1377 loc = None if location is None else os.path.join(
1378 location, jname)
1379 locations.append((job, loc))
1380 self.fLOG("[jenkins] skipping",
1381 job, "location", loc)
1382 elif j is not None:
1383 new_dep.append(name)
1385 else:
1386 # yml file
1387 if location is not None:
1388 options["root_path"] = location
1389 for k, v in self.engines.items():
1390 if k not in options:
1391 options[k] = v
1392 jobdef = job[0] if isinstance(job, tuple) else job
1394 done = {}
1395 for aj, name, var in enumerate_processed_yml(
1396 jobdef, context=options, engine=yml_engine,
1397 add_environ=add_environ, server=self, git_repo=gitrepo,
1398 scheduler=scheduler, description=description, credentials=cred,
1399 success_only=success_only, timeout=timeout, platform=self.platform,
1400 adjust_scheduler=adjust_scheduler, overwrite=overwrite,
1401 build_location=location, mails=self.mails,
1402 job_options=scheduler_options):
1403 if name in done:
1404 s = "A name '{0}' was already used for a job, from:\n{1}\nPROCESS:\n{2}" # pragma: no cover
1405 raise ValueError( # pragma: no cover
1406 s.format(name, jobdef, "\n".join(sorted(set(done.keys())))))
1407 done[name] = (aj, name, var)
1408 loc = None if location is None else os.path.join(
1409 location, name)
1410 self.fLOG("[jenkins] adding i={2}: '{0}' var='{1}'".format(
1411 name, var, len(created)))
1412 created.append((job, name, loc, job, aj))
1414 indexes["order"] = order
1415 indexes["dozen"] = dozen
1416 indexes["unit"] = unit
1417 return created, new_dep, locations