Coverage for pyquickhelper/pycode/build_helper.py: 84%
139 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
1"""
2@file
3@brief Produces a build file for a module following *pyquickhelper* design.
4"""
6import sys
7import os
8import uuid
9import re
10from .windows_scripts import windows_error, windows_prefix, windows_setup, windows_notebook
11from .windows_scripts import windows_publish, windows_publish_doc, windows_pypi, setup_script_dependency_py
12from .windows_scripts import windows_prefix_27, windows_unittest27, copy_dist_to_local_pypi
13from .windows_scripts import windows_any_setup_command, windows_blogpost, windows_docserver, windows_build_setup, windows_build
14from .windows_scripts import pyproj_template, copy_sphinx_to_dist
15from ..filehelper.file_info import is_file_string
17#: nickname for no folder
18_default_nofolder = "__NOFOLDERSHOULDNOTEXIST%d%d__" % sys.version_info[:2]
21def choose_path(*paths):
22 """
23 Returns the first path which exists in the list.
25 @param paths list of paths
26 @return a path
27 """
28 found = None
29 for path in paths:
30 if "{" in path:
31 if found is None:
32 root = os.path.dirname(path)
33 if not root:
34 root = '.'
35 founds = [os.path.join(root, _) for _ in os.listdir(root)]
36 founds.sort(reverse=True)
37 reg = re.compile(path.replace("\\", "\\\\"))
38 found = [(_, reg.search(_)) for _ in founds]
39 found = [_ for _ in found if _[1]]
40 if len(found) > 0: # pragma: no cover
41 full = found[0][0]
42 return full
43 elif os.path.exists(path):
44 return path # pragma: no cover
45 if paths[-1] != _default_nofolder:
46 raise FileNotFoundError( # pragma: no cover
47 "No path exist in: " + ", ".join(paths))
48 return _default_nofolder
51#: default values, to be replaced in the build script
52#: ``'c:\\python39x64'`` --> appveyor
53#: ``'c:\\python39_x64'`` --> custom installation
55default_values = {
56 "windows": {
57 "__PY36_X64__": choose_path("c:\\Python36[0-9]{1}_x64", "c:\\Python36_x64", "c:\\Python36-x64", _default_nofolder),
58 "__PY37_X64__": choose_path("c:\\Python37[0-9]{1}_x64", "c:\\Python37_x64", "c:\\Python37-x64", _default_nofolder),
59 "__PY38_X64__": choose_path("c:\\Python38[0-9]{1}_x64", "c:\\Python38_x64", "c:\\Python38-x64", _default_nofolder),
60 "__PY39_X64__": choose_path("c:\\Python39[0-9]{1}_x64", "c:\\Python39_x64", "c:\\Python39-x64", _default_nofolder),
61 "__PY310_X64__": choose_path("c:\\Python310[0-9]{1}_x64", "c:\\Python310_x64", "c:\\Python310-x64", _default_nofolder),
62 },
63}
66def private_path_choice(path):
67 """
68 Custom logic to reference other currently developped modules.
69 """
70 s = path
71 current = '%current%' if sys.platform.startswith('win') else '~'
72 if "/" in s or "\\" in s:
73 return s # pragma: no cover
74 if 'ROOT' in s:
75 return os.path.join(current, "..", s.replace('ROOT', ''))
76 if 'BLIB' in s:
77 return os.path.join(current, "..", s.replace('BLIB', ''), "build", "lib")
78 if 'NSRC' in s:
79 return os.path.join(current, "..", s.replace("NSRC", '')) # pragma: no cover
80 return os.path.join(current, "..", s, "src")
83def private_replacement_(script, paths, key="__ADDITIONAL_LOCAL_PATH__"):
84 """
85 Less copy/paste.
86 """
87 unique_paths = []
88 for p in paths:
89 if p not in unique_paths:
90 unique_paths.append(p)
91 rows = [private_path_choice(_) for _ in unique_paths]
92 sep = ";" if sys.platform.startswith("win") else ":"
93 rep = sep + sep.join(rows)
94 script = script.replace(key, rep)
95 return script
98def private_script_replacements(script, module, requirements, port, raise_exception=True,
99 platform=sys.platform, default_engine_paths=None,
100 additional_local_path=None):
101 """
102 Runs last replacements.
104 @param script script or list of scripts
105 @param module module name
106 @param requirements requirements - (list or 2-uple of lists)
107 @param port port
108 @param raise_exception raise an exception if there is an error, otherwise, return None
109 @param platform platform
110 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below.
111 @param additional_local_path additional local path to add to PYTHONPATH
112 @return modified script
114 An example for *default_engine_paths*::
116 default_engine_paths = {
117 "windows": {
118 "__PY35__": None,
119 "__PY36_X64__": "c:\\Python365_x64",
120 "__PY37_X64__": "c:\\Python372_x64",
121 "__PY38_X64__": "c:\\Python387_x64",
122 "__PY39_X64__": "c:\\Python391_x64",
123 "__PY310_X64__": "c:\\Python3101_x64",
124 },
125 }
127 Parameter *requirements* can a list of requirements,
128 we assume these requirements are available from a local PyPi server.
129 There can be extra requirements obtained from PiPy. In that case,
130 those can be specified as a tuple *(requirements_local, requirements_pipy)*.
132 The function replaces ``rem _PATH_VIRTUAL_ENV_``
133 with an instruction to copy these DLLs.
134 Parameter *requirements* can be a list or a tuple.
135 """
136 global default_values
137 if default_engine_paths is None:
138 default_engine_paths = default_values
140 if isinstance(script, list):
141 return [private_script_replacements(s, module, requirements,
142 port, raise_exception, platform,
143 default_engine_paths=default_engine_paths) for s in script]
145 if platform.startswith("win"):
146 plat = "windows"
147 global _default_nofolder
148 def_values = default_engine_paths
150 values = [v for v in def_values[
151 plat].values() if v is not None and v != _default_nofolder]
152 if raise_exception and len(values) != len(set(values)):
153 raise FileNotFoundError( # pragma: no cover
154 "One path is wrong among:\n %s" % (
155 "\n".join("{0}={1}".format(k, v)
156 for k, v in def_values[plat].items())))
158 if module is not None:
159 script = script.replace("__MODULE__", module)
161 for k, v in def_values[plat].items():
162 script = script.replace(k, v)
164 # requirements
165 if requirements is not None:
166 if isinstance(requirements, list):
167 requirements_pipy = []
168 requirements_local = requirements
169 else:
170 requirements_local, requirements_pipy = requirements
172 if requirements_pipy is None:
173 requirements_pipy = []
174 if requirements_local is None:
175 requirements_local = []
177 cj = "%jenkinspythonpip%" if "jenkinspythonpip" in script else "%pythonpip%"
178 patternr = "install {0}"
179 patternl = "install --no-cache-dir --index http://localhost:{0}/simple/ {1}"
180 rows = []
181 for r in requirements_pipy:
182 r = cj + " " + patternr.format(r)
183 rows.append(r)
184 for r in requirements_local:
185 r = cj + " " + patternl.format(port, r)
186 rows.append(r)
188 reqs = "\n".join(rows)
189 else:
190 reqs = ""
192 script = script.replace("__REQUIREMENTS__", reqs) \
193 .replace("__PORT__", str(port)) \
194 .replace("__USERNAME__", os.environ.get("USERNAME", os.environ.get("USER", "UNKNOWN-USER")))
196 if "__ADDITIONAL_LOCAL_PATH__" in script:
197 paths = []
198 if additional_local_path is not None and len(additional_local_path) > 0:
199 paths.extend(additional_local_path)
200 if len(paths) > 0:
201 script = private_replacement_(
202 script, paths, key="__ADDITIONAL_LOCAL_PATH__")
203 else:
204 script = script.replace("__ADDITIONAL_LOCAL_PATH__", "")
206 if "rem _PATH_VIRTUAL_ENV_" in script:
207 script = script.replace(
208 "rem _PATH_VIRTUAL_ENV_", "rem nothing to do here")
210 return script
212 else:
213 if raise_exception:
214 raise NotImplementedError(
215 f"not implemented yet for this platform {sys.platform}")
216 return None
219def get_build_script(module, requirements=None, port=8067, default_engine_paths=None,
220 additional_local_path=None):
221 """
222 Builds the build script which builds the setup, run the unit tests
223 and the documentation.
225 @param module module name
226 @param requirements list of dependencies (not in your python distribution)
227 @param port port for the local pypi_server which gives the dependencies
228 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below.
229 @param additional_local_path additional paths to add to PYTHONPATH
230 @return scripts
231 """
232 if requirements is None:
233 requirements = []
234 return private_script_replacements(windows_build, module, requirements, port,
235 default_engine_paths=default_engine_paths,
236 additional_local_path=additional_local_path)
239def get_script_command(command, module, requirements, port=8067, platform=sys.platform,
240 default_engine_paths=None,
241 additional_local_path=None): # pragma: no cover
242 """
243 Produces a script which runs a command available through the setup.
245 @param command command to run
246 @param module module name
247 @param requirements list of dependencies (not in your python distribution)
248 @param port port for the local pypi_server which gives the dependencies
249 @param platform platform (only Windows)
250 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below.
251 @param additional_local_path additional local path to add before running command ``setup.py <command>``
252 @return scripts
254 The available list of commands is given by function @see fn process_standard_options_for_setup.
255 """
256 if not platform.startswith("win"):
257 raise NotImplementedError( # pragma: no cover
258 "not yet available on linux")
259 global windows_error, windows_prefix, windows_setup
260 rows = [windows_prefix]
262 if additional_local_path is not None and len(additional_local_path):
263 addp = "set PYTHONPATH=%PYTHONPATH%;" + \
264 ";".join(private_path_choice(_) for _ in additional_local_path)
265 else:
266 addp = ""
267 rows.append(windows_setup.replace(
268 "rem set PYTHONPATH=additional_path", addp) + " " + command)
269 rows.append(windows_error)
270 sc = "\n".join(rows)
271 res = private_script_replacements(
272 sc, module, requirements, port, default_engine_paths=default_engine_paths,
273 additional_local_path=additional_local_path)
274 if sys.platform.startswith("win"):
275 if command == "copy27":
276 res = """
277 if exist dist_module27 (
278 rmdir /Q /S dist_module27
279 if %errorlevel% neq 0 exit /b %errorlevel%
280 )
281 """.replace(" ", "") + res
282 elif command == "clean_space":
283 # Run the test which test pep8 and convert the convert the
284 # notebooks.
285 res += """
286 if not exist _unittests\\ut_module\\test_code_style.py goto end:
287 %pythonexe% -u _unittests\\ut_module\\test_code_style.py -v
288 if %errorlevel% neq 0 exit /b %errorlevel%
290 if not exist _unittests\\ut_module\\test_convert_notebooks.py goto end:
291 %pythonexe% -u _unittests\\ut_module\\test_convert_notebooks.py
292 if %errorlevel% neq 0 exit /b %errorlevel%
293 )
294 """.replace(" ", "") + res
295 return res
298def get_extra_script_command(command, module, requirements, port=8067, blog_list=None, platform=sys.platform,
299 default_engine_paths=None, unit_test_folder=None, unittest_modules=None,
300 additional_notebook_path=None,
301 additional_local_path=None): # pragma: no cover
302 """
303 Produces a script which runs the notebook, a documentation server, which
304 publishes...
306 @param command command to run (*notebook*, *publish*, *publish_doc*, *local_pypi*, *setupdep*,
307 *run27*, *build27*, *copy_dist*, *any_setup_command*, *lab*)
308 @param module module name
309 @param requirements list of dependencies (not in your python distribution)
310 @param port port for the local pypi_server which gives the dependencies
311 @param blog_list list of blog to listen for this module (usually stored in
312 ``module.__blog__``)
313 @param platform platform (only Windows)
314 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below.
315 @param unit_test_folder unit test folders, used for command ``run27``
316 @param additional_notebook_path additional paths to add when running the script launching the notebooks
317 @param additional_local_path additional paths to add when running a local command
318 @param unittest_modules list of modules to be used during unit tests
319 @return scripts
321 The available list of commands is given by function @see fn process_standard_options_for_setup.
322 """
323 if not platform.startswith("win"):
324 raise NotImplementedError("linux not yet available")
326 script = None
327 if command == "notebook":
328 script = windows_notebook
329 elif command == "lab":
330 script = windows_notebook.replace("jupyter-notebook", "jupyter-lab")
331 elif command == "publish":
332 script = "\n".join([windows_prefix, windows_publish])
333 elif command == "publish_doc":
334 script = "\n".join([windows_prefix, windows_publish_doc])
335 elif command == "local_pypi":
336 script = "\n".join([windows_prefix, windows_pypi])
337 elif command == "run27":
338 script = "\n".join(
339 [windows_prefix_27, windows_unittest27, windows_error])
340 if unit_test_folder is None:
341 raise FileNotFoundError(
342 "the unit test folder must be specified and cannot be None")
343 if not os.path.exists(unit_test_folder):
344 raise FileNotFoundError(
345 "the unit test folder must exist: " + unit_test_folder)
346 ut_ = [("%pythonexe27%\\..\\Scripts\\nosetests.exe -w " + _)
347 for _ in os.listdir(unit_test_folder) if _.startswith("ut_")]
348 stut = "\nif %errorlevel% neq 0 exit /b %errorlevel%\n".join(ut_)
349 script = script.replace("__LOOP_UNITTEST_FOLDERS__", stut)
350 elif command == "build27":
351 script = "\n".join([windows_prefix_27, "cd dist_module27", "rmdir /S /Q dist",
352 windows_setup.replace(
353 "exe%", "exe27%") + " bdist_wheel",
354 windows_error, "cd ..", "copy dist_module27\\dist\\*.whl dist"])
355 elif command == "copy_dist":
356 script = copy_dist_to_local_pypi
357 elif command == "copy_sphinx":
358 script = copy_sphinx_to_dist
359 elif command == "setupdep":
360 script = setup_script_dependency_py
361 elif command == "any_setup_command":
362 script = windows_any_setup_command
363 elif command == "build_dist":
364 script = windows_build_setup
365 elif command == "history":
366 script = "\n".join(
367 [windows_prefix, '\n%pythonexe% %current%setup.py history\n'])
368 else:
369 raise RuntimeError("unable to interpret command: " + command)
371 # additional paths
372 if "__ADDITIONAL_LOCAL_PATH__" in script:
373 paths = []
374 if command in ("notebook", "lab") and additional_notebook_path is not None and len(additional_notebook_path) > 0:
375 paths.extend(additional_notebook_path)
376 if unittest_modules is not None and len(unittest_modules) > 0:
377 paths.extend(unittest_modules)
378 if additional_local_path is not None and len(additional_local_path) > 0:
379 paths.extend(additional_local_path)
380 if len(paths) > 0:
381 script = private_replacement_(
382 script, paths, key="__ADDITIONAL_LOCAL_PATH__")
383 else:
384 script = script.replace("__ADDITIONAL_LOCAL_PATH__", "")
386 script = script.replace("__ADDITIONAL_NOTEBOOK_PATH__", "")
388 # common post-processing
389 if script is None:
390 raise RuntimeError("unexpected command: " + command)
391 return private_script_replacements(script, module, requirements, port, default_engine_paths=default_engine_paths)
394def get_script_module(command, platform=sys.platform, blog_list=None,
395 default_engine_paths=None):
396 """
397 Produces a script which runs the notebook, a documentation server, which
398 publishes and other scripts.
400 @param command command to run (*blog*)
401 @param platform platform (only Windows)
402 @param blog_list list of blog to listen for this module (usually stored in ``module.__blog__``)
403 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below.
404 @return scripts
406 The available list of commands is given by function @see fn process_standard_options_for_setup.
407 """
408 prefix_setup = ""
409 filename = os.path.abspath(__file__)
410 if "site-packages" not in filename:
411 folder = os.path.normpath(
412 os.path.join(os.path.dirname(filename), "..", ".."))
413 prefix_setup = """
414 import sys
415 import os
416 sys.path.append(r"{0}")
417 sys.path.append(r"{1}")
418 sys.path.append(r"{2}")
419 """.replace(" ", "").format(folder,
420 folder.replace(
421 "pyquickhelper", "pyensae"),
422 folder.replace(
423 "pyquickhelper", "pyrsslocal")
424 )
426 script = None
427 if command == "blog":
428 if blog_list is None:
429 return None
430 else:
431 list_xml = blog_list.strip("\n\r\t ")
432 if '<?xml version="1.0" encoding="UTF-8"?>' not in list_xml and is_file_string(list_xml) and os.path.exists(list_xml):
433 with open(list_xml, "r", encoding="utf8") as f:
434 list_xml = f.read()
435 if "<body>" not in list_xml:
436 raise ValueError( # pragma: no cover
437 f"Wrong XML format:\n{list_xml}")
438 script = [("auto_rss_list.xml", list_xml)]
439 script.append(("auto_rss_server.py", prefix_setup + """
440 from pyquickhelper.pycode.blog_helper import rss_update_run_server
441 rss_update_run_server("auto_rss_database.db3", "auto_rss_list.xml")
442 """.replace(" ", "")))
443 if platform.startswith("win"):
444 script.append("\n".join([windows_prefix, windows_blogpost]))
445 elif command == "doc":
446 script = []
447 script.append(("auto_doc_server.py", prefix_setup + """
448 # address http://localhost:8079/
449 from pyquickhelper import fLOG
450 from pyquickhelper.server import run_doc_server, get_jenkins_mappings
451 fLOG(OutputPrint=True)
452 fLOG("running documentation server")
453 thisfile = os.path.dirname(__file__)
454 mappings = get_jenkins_mappings(os.path.join(thisfile, ".."))
455 fLOG("goto", "http://localhost:8079/")
456 for k,v in sorted(mappings.items()):
457 fLOG(k,"-->",v)
458 run_doc_server(None, mappings=mappings)
459 """.replace(" ", "")))
460 if platform.startswith("win"):
461 script.append("\n".join([windows_prefix, "rem http://localhost:8079/",
462 windows_docserver]))
463 else:
464 raise RuntimeError( # pragma: no cover
465 f"Unable to interpret command: {command!r}")
467 # common post-processing
468 for i, item in enumerate(script):
469 if isinstance(item, tuple):
470 ext = os.path.splitext(item[0])
471 if ext == ".py":
472 s = private_script_replacements(
473 item[1], None, None, None, default_engine_paths=default_engine_paths)
474 script[i] = (item[0], s)
475 else:
476 script[i] = private_script_replacements(
477 item, None, None, None, default_engine_paths=default_engine_paths)
478 return script
481def get_pyproj_project(name, file_list):
482 """
483 returns a string which corresponds to a pyproj project
485 @param name project name
486 @param file_list file_list
487 @return string
488 """
489 guid = uuid.uuid3(uuid.NAMESPACE_DNS, name)
490 folders = list(_ for _ in sorted(set(os.path.dirname(f)
491 for f in file_list)) if len(_) > 0)
492 sfold = "\n".join(f' <Folder Include="{_}" />' for _ in folders)
493 sfiles = "\n".join(f' <Compile Include="{_}" />' for _ in file_list)
495 script = pyproj_template.replace("__GUID__", str(guid)) \
496 .replace("__NAME__", name) \
497 .replace("__INCLUDEFILES__", sfiles) \
498 .replace("__INCLUDEFOLDERS__", sfold)
499 return script