Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2@file
3@brief Produces a build file for a module following *pyquickhelper* design.
4"""
6import sys
7import os
8import uuid
9import re
10from .windows_scripts import windows_error, windows_prefix, windows_setup, windows_notebook
11from .windows_scripts import windows_publish, windows_publish_doc, windows_pypi, setup_script_dependency_py
12from .windows_scripts import windows_prefix_27, windows_unittest27, copy_dist_to_local_pypi
13from .windows_scripts import windows_any_setup_command, windows_blogpost, windows_docserver, windows_build_setup, windows_build
14from .windows_scripts import pyproj_template, copy_sphinx_to_dist
15from ..filehelper.file_info import is_file_string
17#: nickname for no folder
18_default_nofolder = "__NOFOLDERSHOULDNOTEXIST%d%d__" % sys.version_info[:2]
21def choose_path(*paths):
22 """
23 Returns the first path which exists in the list.
25 @param paths list of paths
26 @return a path
27 """
28 found = None
29 for path in paths:
30 if "{" in path:
31 if found is None:
32 root = os.path.dirname(path)
33 if not root:
34 root = '.'
35 founds = [os.path.join(root, _) for _ in os.listdir(root)]
36 founds.sort(reverse=True)
37 reg = re.compile(path.replace("\\", "\\\\"))
38 found = [(_, reg.search(_)) for _ in founds]
39 found = [_ for _ in found if _[1]]
40 if len(found) > 0: # pragma: no cover
41 full = found[0][0]
42 return full
43 elif os.path.exists(path):
44 return path # pragma: no cover
45 if paths[-1] != _default_nofolder:
46 raise FileNotFoundError( # pragma: no cover
47 "No path exist in: " + ", ".join(paths))
48 return _default_nofolder
51#: default values, to be replaced in the build script
52#: ``'c:\\python39x64'`` --> appveyor
53#: ``'c:\\python39_x64'`` --> custom installation
55default_values = {
56 "windows": {
57 "__PY36_X64__": choose_path("c:\\Python36[0-9]{1}_x64", "c:\\Python36_x64", "c:\\Python36-x64", _default_nofolder),
58 "__PY37_X64__": choose_path("c:\\Python37[0-9]{1}_x64", "c:\\Python37_x64", "c:\\Python37-x64", _default_nofolder),
59 "__PY38_X64__": choose_path("c:\\Python38[0-9]{1}_x64", "c:\\Python38_x64", "c:\\Python38-x64", _default_nofolder),
60 "__PY39_X64__": choose_path("c:\\Python39[0-9]{1}_x64", "c:\\Python38_x64", "c:\\Python39-x64", _default_nofolder),
61 },
62}
65def private_path_choice(path):
66 """
67 Custom logic to reference other currently developped modules.
68 """
69 s = path
70 current = '%current%' if sys.platform.startswith('win') else '~'
71 if "/" in s or "\\" in s:
72 return s # pragma: no cover
73 if 'ROOT' in s:
74 return os.path.join(current, "..", s.replace('ROOT', ''))
75 if 'BLIB' in s:
76 return os.path.join(current, "..", s.replace('BLIB', ''), "build", "lib")
77 if 'NSRC' in s:
78 return os.path.join(current, "..", s.replace("NSRC", '')) # pragma: no cover
79 return os.path.join(current, "..", s, "src")
82def private_replacement_(script, paths, key="__ADDITIONAL_LOCAL_PATH__"):
83 """
84 Less copy/paste.
85 """
86 unique_paths = []
87 for p in paths:
88 if p not in unique_paths:
89 unique_paths.append(p)
90 rows = [private_path_choice(_) for _ in unique_paths]
91 sep = ";" if sys.platform.startswith("win") else ":"
92 rep = sep + sep.join(rows)
93 script = script.replace(key, rep)
94 return script
97def private_script_replacements(script, module, requirements, port, raise_exception=True,
98 platform=sys.platform, default_engine_paths=None,
99 additional_local_path=None):
100 """
101 Runs last replacements.
103 @param script script or list of scripts
104 @param module module name
105 @param requirements requirements - (list or 2-uple of lists)
106 @param port port
107 @param raise_exception raise an exception if there is an error, otherwise, return None
108 @param platform platform
109 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below.
110 @param additional_local_path additional local path to add to PYTHONPATH
111 @return modified script
113 An example for *default_engine_paths*::
115 default_engine_paths = {
116 "windows": {
117 "__PY35__": None,
118 "__PY36_X64__": "c:\\Python365_x64",
119 "__PY37_X64__": "c:\\Python372_x64",
120 "__PY38_X64__": "c:\\Python387_x64",
121 "__PY39_X64__": "c:\\Python391_x64",
122 },
123 }
125 Parameter *requirements* can a list of requirements,
126 we assume these requirements are available from a local PyPi server.
127 There can be extra requirements obtained from PiPy. In that case,
128 those can be specified as a tuple *(requirements_local, requirements_pipy)*.
130 The function replaces ``rem _PATH_VIRTUAL_ENV_``
131 with an instruction to copy these DLLs.
132 Parameter *requirements* can be a list or a tuple.
133 """
134 global default_values
135 if default_engine_paths is None:
136 default_engine_paths = default_values
138 if isinstance(script, list):
139 return [private_script_replacements(s, module, requirements,
140 port, raise_exception, platform,
141 default_engine_paths=default_engine_paths) for s in script]
143 if platform.startswith("win"):
144 plat = "windows"
145 global _default_nofolder
146 def_values = default_engine_paths
148 values = [v for v in def_values[
149 plat].values() if v is not None and v != _default_nofolder]
150 if raise_exception and len(values) != len(set(values)):
151 raise FileNotFoundError( # pragma: no cover
152 "One path is wrong among:\n %s" % (
153 "\n".join("{0}={1}".format(k, v)
154 for k, v in def_values[plat].items())))
156 if module is not None:
157 script = script.replace("__MODULE__", module)
159 for k, v in def_values[plat].items():
160 script = script.replace(k, v)
162 # requirements
163 if requirements is not None:
164 if isinstance(requirements, list):
165 requirements_pipy = []
166 requirements_local = requirements
167 else:
168 requirements_local, requirements_pipy = requirements
170 if requirements_pipy is None:
171 requirements_pipy = []
172 if requirements_local is None:
173 requirements_local = []
175 cj = "%jenkinspythonpip%" if "jenkinspythonpip" in script else "%pythonpip%"
176 patternr = "install {0}"
177 patternl = "install --no-cache-dir --index http://localhost:{0}/simple/ {1}"
178 rows = []
179 for r in requirements_pipy:
180 r = cj + " " + patternr.format(r)
181 rows.append(r)
182 for r in requirements_local:
183 r = cj + " " + patternl.format(port, r)
184 rows.append(r)
186 reqs = "\n".join(rows)
187 else:
188 reqs = ""
190 script = script.replace("__REQUIREMENTS__", reqs) \
191 .replace("__PORT__", str(port)) \
192 .replace("__USERNAME__", os.environ.get("USERNAME", os.environ.get("USER", "UNKNOWN-USER")))
194 if "__ADDITIONAL_LOCAL_PATH__" in script:
195 paths = []
196 if additional_local_path is not None and len(additional_local_path) > 0:
197 paths.extend(additional_local_path)
198 if len(paths) > 0:
199 script = private_replacement_(
200 script, paths, key="__ADDITIONAL_LOCAL_PATH__")
201 else:
202 script = script.replace("__ADDITIONAL_LOCAL_PATH__", "")
204 if "rem _PATH_VIRTUAL_ENV_" in script:
205 script = script.replace(
206 "rem _PATH_VIRTUAL_ENV_", "rem nothing to do here")
208 return script
210 else:
211 if raise_exception:
212 raise NotImplementedError(
213 "not implemented yet for this platform %s" % sys.platform)
214 return None
217def get_build_script(module, requirements=None, port=8067, default_engine_paths=None,
218 additional_local_path=None):
219 """
220 Builds the build script which builds the setup, run the unit tests
221 and the documentation.
223 @param module module name
224 @param requirements list of dependencies (not in your python distribution)
225 @param port port for the local pypi_server which gives the dependencies
226 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below.
227 @param additional_local_path additional paths to add to PYTHONPATH
228 @return scripts
229 """
230 if requirements is None:
231 requirements = []
232 return private_script_replacements(windows_build, module, requirements, port,
233 default_engine_paths=default_engine_paths,
234 additional_local_path=additional_local_path)
237def get_script_command(command, module, requirements, port=8067, platform=sys.platform,
238 default_engine_paths=None,
239 additional_local_path=None): # pragma: no cover
240 """
241 Produces a script which runs a command available through the setup.
243 @param command command to run
244 @param module module name
245 @param requirements list of dependencies (not in your python distribution)
246 @param port port for the local pypi_server which gives the dependencies
247 @param platform platform (only Windows)
248 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below.
249 @param additional_local_path additional local path to add before running command ``setup.py <command>``
250 @return scripts
252 The available list of commands is given by function @see fn process_standard_options_for_setup.
253 """
254 if not platform.startswith("win"):
255 raise NotImplementedError( # pragma: no cover
256 "not yet available on linux")
257 global windows_error, windows_prefix, windows_setup
258 rows = [windows_prefix]
260 if additional_local_path is not None and len(additional_local_path):
261 addp = "set PYTHONPATH=%PYTHONPATH%;" + \
262 ";".join(private_path_choice(_) for _ in additional_local_path)
263 else:
264 addp = ""
265 rows.append(windows_setup.replace(
266 "rem set PYTHONPATH=additional_path", addp) + " " + command)
267 rows.append(windows_error)
268 sc = "\n".join(rows)
269 res = private_script_replacements(
270 sc, module, requirements, port, default_engine_paths=default_engine_paths,
271 additional_local_path=additional_local_path)
272 if sys.platform.startswith("win"):
273 if command == "copy27":
274 res = """
275 if exist dist_module27 (
276 rmdir /Q /S dist_module27
277 if %errorlevel% neq 0 exit /b %errorlevel%
278 )
279 """.replace(" ", "") + res
280 elif command == "clean_space":
281 # Run the test which test pep8 and convert the convert the
282 # notebooks.
283 res += """
284 if not exist _unittests\\ut_module\\test_code_style.py goto end:
285 %pythonexe% -u _unittests\\ut_module\\test_code_style.py -v
286 if %errorlevel% neq 0 exit /b %errorlevel%
288 if not exist _unittests\\ut_module\\test_convert_notebooks.py goto end:
289 %pythonexe% -u _unittests\\ut_module\\test_convert_notebooks.py
290 if %errorlevel% neq 0 exit /b %errorlevel%
291 )
292 """.replace(" ", "") + res
293 return res
296def get_extra_script_command(command, module, requirements, port=8067, blog_list=None, platform=sys.platform,
297 default_engine_paths=None, unit_test_folder=None, unittest_modules=None,
298 additional_notebook_path=None,
299 additional_local_path=None): # pragma: no cover
300 """
301 Produces a script which runs the notebook, a documentation server, which
302 publishes...
304 @param command command to run (*notebook*, *publish*, *publish_doc*, *local_pypi*, *setupdep*,
305 *run27*, *build27*, *copy_dist*, *any_setup_command*, *lab*)
306 @param module module name
307 @param requirements list of dependencies (not in your python distribution)
308 @param port port for the local pypi_server which gives the dependencies
309 @param blog_list list of blog to listen for this module (usually stored in
310 ``module.__blog__``)
311 @param platform platform (only Windows)
312 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below.
313 @param unit_test_folder unit test folders, used for command ``run27``
314 @param additional_notebook_path additional paths to add when running the script launching the notebooks
315 @param additional_local_path additional paths to add when running a local command
316 @param unittest_modules list of modules to be used during unit tests
317 @return scripts
319 The available list of commands is given by function @see fn process_standard_options_for_setup.
320 """
321 if not platform.startswith("win"):
322 raise NotImplementedError("linux not yet available")
324 script = None
325 if command == "notebook":
326 script = windows_notebook
327 elif command == "lab":
328 script = windows_notebook.replace("jupyter-notebook", "jupyter-lab")
329 elif command == "publish":
330 script = "\n".join([windows_prefix, windows_publish])
331 elif command == "publish_doc":
332 script = "\n".join([windows_prefix, windows_publish_doc])
333 elif command == "local_pypi":
334 script = "\n".join([windows_prefix, windows_pypi])
335 elif command == "run27":
336 script = "\n".join(
337 [windows_prefix_27, windows_unittest27, windows_error])
338 if unit_test_folder is None:
339 raise FileNotFoundError(
340 "the unit test folder must be specified and cannot be None")
341 if not os.path.exists(unit_test_folder):
342 raise FileNotFoundError(
343 "the unit test folder must exist: " + unit_test_folder)
344 ut_ = [("%pythonexe27%\\..\\Scripts\\nosetests.exe -w " + _)
345 for _ in os.listdir(unit_test_folder) if _.startswith("ut_")]
346 stut = "\nif %errorlevel% neq 0 exit /b %errorlevel%\n".join(ut_)
347 script = script.replace("__LOOP_UNITTEST_FOLDERS__", stut)
348 elif command == "build27":
349 script = "\n".join([windows_prefix_27, "cd dist_module27", "rmdir /S /Q dist",
350 windows_setup.replace(
351 "exe%", "exe27%") + " bdist_wheel",
352 windows_error, "cd ..", "copy dist_module27\\dist\\*.whl dist"])
353 elif command == "copy_dist":
354 script = copy_dist_to_local_pypi
355 elif command == "copy_sphinx":
356 script = copy_sphinx_to_dist
357 elif command == "setupdep":
358 script = setup_script_dependency_py
359 elif command == "any_setup_command":
360 script = windows_any_setup_command
361 elif command == "build_dist":
362 script = windows_build_setup
363 elif command == "history":
364 script = "\n".join(
365 [windows_prefix, '\n%pythonexe% %current%setup.py history\n'])
366 else:
367 raise Exception("unable to interpret command: " + command)
369 # additional paths
370 if "__ADDITIONAL_LOCAL_PATH__" in script:
371 paths = []
372 if command in ("notebook", "lab") and additional_notebook_path is not None and len(additional_notebook_path) > 0:
373 paths.extend(additional_notebook_path)
374 if unittest_modules is not None and len(unittest_modules) > 0:
375 paths.extend(unittest_modules)
376 if additional_local_path is not None and len(additional_local_path) > 0:
377 paths.extend(additional_local_path)
378 if len(paths) > 0:
379 script = private_replacement_(
380 script, paths, key="__ADDITIONAL_LOCAL_PATH__")
381 else:
382 script = script.replace("__ADDITIONAL_LOCAL_PATH__", "")
384 script = script.replace("__ADDITIONAL_NOTEBOOK_PATH__", "")
386 # common post-processing
387 if script is None:
388 raise Exception("unexpected command: " + command)
389 return private_script_replacements(script, module, requirements, port, default_engine_paths=default_engine_paths)
392def get_script_module(command, platform=sys.platform, blog_list=None,
393 default_engine_paths=None):
394 """
395 Produces a script which runs the notebook, a documentation server, which
396 publishes and other scripts.
398 @param command command to run (*blog*)
399 @param platform platform (only Windows)
400 @param blog_list list of blog to listen for this module (usually stored in ``module.__blog__``)
401 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below.
402 @return scripts
404 The available list of commands is given by function @see fn process_standard_options_for_setup.
405 """
406 prefix_setup = ""
407 filename = os.path.abspath(__file__)
408 if "site-packages" not in filename:
409 folder = os.path.normpath(
410 os.path.join(os.path.dirname(filename), "..", ".."))
411 prefix_setup = """
412 import sys
413 import os
414 sys.path.append(r"{0}")
415 sys.path.append(r"{1}")
416 sys.path.append(r"{2}")
417 """.replace(" ", "").format(folder,
418 folder.replace(
419 "pyquickhelper", "pyensae"),
420 folder.replace(
421 "pyquickhelper", "pyrsslocal")
422 )
424 script = None
425 if command == "blog":
426 if blog_list is None:
427 return None
428 else:
429 list_xml = blog_list.strip("\n\r\t ")
430 if '<?xml version="1.0" encoding="UTF-8"?>' not in list_xml and is_file_string(list_xml) and os.path.exists(list_xml):
431 with open(list_xml, "r", encoding="utf8") as f:
432 list_xml = f.read()
433 if "<body>" not in list_xml:
434 raise ValueError( # pragma: no cover
435 "Wrong XML format:\n{0}".format(list_xml))
436 script = [("auto_rss_list.xml", list_xml)]
437 script.append(("auto_rss_server.py", prefix_setup + """
438 from pyquickhelper.pycode.blog_helper import rss_update_run_server
439 rss_update_run_server("auto_rss_database.db3", "auto_rss_list.xml")
440 """.replace(" ", "")))
441 if platform.startswith("win"):
442 script.append("\n".join([windows_prefix, windows_blogpost]))
443 elif command == "doc":
444 script = []
445 script.append(("auto_doc_server.py", prefix_setup + """
446 # address http://localhost:8079/
447 from pyquickhelper import fLOG
448 from pyquickhelper.server import run_doc_server, get_jenkins_mappings
449 fLOG(OutputPrint=True)
450 fLOG("running documentation server")
451 thisfile = os.path.dirname(__file__)
452 mappings = get_jenkins_mappings(os.path.join(thisfile, ".."))
453 fLOG("goto", "http://localhost:8079/")
454 for k,v in sorted(mappings.items()):
455 fLOG(k,"-->",v)
456 run_doc_server(None, mappings=mappings)
457 """.replace(" ", "")))
458 if platform.startswith("win"):
459 script.append("\n".join([windows_prefix, "rem http://localhost:8079/",
460 windows_docserver]))
461 else:
462 raise RuntimeError( # pragma: no cover
463 "Unable to interpret command: %r" % command)
465 # common post-processing
466 for i, item in enumerate(script):
467 if isinstance(item, tuple):
468 ext = os.path.splitext(item[0])
469 if ext == ".py":
470 s = private_script_replacements(
471 item[1], None, None, None, default_engine_paths=default_engine_paths)
472 script[i] = (item[0], s)
473 else:
474 script[i] = private_script_replacements(
475 item, None, None, None, default_engine_paths=default_engine_paths)
476 return script
479def get_pyproj_project(name, file_list):
480 """
481 returns a string which corresponds to a pyproj project
483 @param name project name
484 @param file_list file_list
485 @return string
486 """
487 guid = uuid.uuid3(uuid.NAMESPACE_DNS, name)
488 folders = list(_ for _ in sorted(set(os.path.dirname(f)
489 for f in file_list)) if len(_) > 0)
490 sfold = "\n".join(' <Folder Include="%s\" />' % _ for _ in folders)
491 sfiles = "\n".join(' <Compile Include="%s\" />' % _ for _ in file_list)
493 script = pyproj_template.replace("__GUID__", str(guid)) \
494 .replace("__NAME__", name) \
495 .replace("__INCLUDEFILES__", sfiles) \
496 .replace("__INCLUDEFOLDERS__", sfold)
497 return script