Coverage for pyquickhelper/helpgen/utils_sphinx_doc.py: 89%
851 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
1"""
2@file
3@brief various helpers to produce a Sphinx documentation
5"""
6import os
7import re
8import sys
9import shutil
10import importlib
11from ..loghelper.flog import fLOG, noLOG
12from ..filehelper.synchelper import remove_folder, synchronize_folder, explore_folder
13from ._my_doxypy import process_string
14from .utils_sphinx_doc_helpers import add_file_rst_template, process_var_tag, import_module
15from .utils_sphinx_doc_helpers import get_module_objects, add_file_rst_template_cor, add_file_rst_template_title
16from .utils_sphinx_doc_helpers import IndexInformation, RstFileHelp, HelpGenException, process_look_for_tag, make_label_index
17from ..pandashelper.tblformat import df2rst
20def validate_file_for_help(filename, fexclude=lambda f: False):
21 """
22 Accepts or rejects a file to be copied in the help folder.
24 @param filename filename
25 @param fexclude function to exclude some files
26 @return boolean
27 """
28 if fexclude is not None and fexclude(filename):
29 return False # pragma: no cover
31 if filename.endswith(".pyd") or filename.endswith(".so"):
32 return True
34 if "rpy2" in filename: # pragma: no cover
35 with open(filename, "r") as ff:
36 content = ff.read()
37 if "from pandas.core." in content:
38 return False
40 return True
43def replace_relative_import_fct(fullname, content=None):
44 """
45 Takes a :epkg:`python` file and replaces all relative
46 imports it was able to find by an import which can be
47 processed by :epkg:`Python` if the file were the main file.
49 @param fullname name of the file
50 @param content a preprocessed content of the file of
51 the content if it is None
52 @return content of the file without relative imports
54 Does not change imports in comments.
55 """
56 if content is None:
57 with open(fullname, "r", encoding="utf8") as f:
58 content = f.read()
60 fullpath = os.path.dirname(fullname)
61 fullsplit = fullpath.replace('\\', '/').split('/')
62 root = None
63 for i in range(len(fullsplit), 1, -1):
64 path = "/".join(fullsplit[:i])
65 init = os.path.join(path, '__init__.py')
66 src = os.path.join(path, 'src')
67 cond = init or (not init and src)
68 if not cond:
69 root = i + 1
70 break
71 if i < len(fullsplit) and fullsplit[i] in ('src', 'site-packages'):
72 root = i + 1
73 break
74 if root is None:
75 raise FileNotFoundError( # pragma: no cover
76 f"Unable to package root for '{fullname}'.")
78 lines = content.split("\n")
79 name = "([a-zA-Z_][a-zA-Z_0-9]*)"
80 namedot = "([a-zA-Z_][a-zA-Z_0-9.]*)"
81 names = name + "(, " + name + ")*"
82 end = "( .*)?$"
83 regi = re.compile(f"^( *)from ([.]{{1,3}}){namedot} import {names}{end}")
85 for i in range(0, len(lines)):
86 line = lines[i]
87 find = regi.search(line)
89 if find:
90 space, dot, rel, name0, names, _, end = find.groups()
91 idot = len(dot)
92 level = len(fullsplit) - root - idot + 1
93 if level > 0:
94 if end is None:
95 end = ""
96 if names is None:
97 names = ""
98 packname = ".".join(fullsplit[root:root + level])
99 if rel:
100 packname += '.' + rel
101 line = f"{space}from {packname} import {name0}{names}{end}"
102 lines[i] = line
103 else:
104 raise ValueError( # pragma: no cover
105 "Unable to replace relative import in '{0}', "
106 "root='{1}'\n{2}|{3}|{4}|{5}| level={6}".format(
107 line, fullsplit[root], dot, rel, name0, names, level))
109 return "\n".join(lines)
112def _private_process_one_file(
113 fullname, to, silent, fmod, replace_relative_import, use_sys):
114 """
115 Copies one file from the source to the documentation folder.
116 It processes some comments in doxygen format (@ param, @ return).
117 It replaces relatives imports by a regular import.
119 @param fullname name of the file
120 @param to location (folder)
121 @param silent no logs if True
122 @param fmod modification functions
123 @param replace_relative_import replace relative import
124 @param use_sys @see fn remove_undesired_part_for_documentation
125 @return extension, number of lines, number of lines in documentation
126 """
127 ext = os.path.splitext(fullname)[-1]
129 if ext in {".jpeg", ".jpg", ".pyd", ".png", ".dat", ".dll", ".o",
130 ".so", ".exe", ".enc", ".txt", ".gif", ".csv", '.pyx',
131 '*.mp3', '*.mp4', '.tmpl'}:
132 if ext in (".pyd", ".so"):
133 # If the file is being executed, the copy might keep the properties of
134 # the original (only Windows).
135 with open(fullname, "rb") as f:
136 bin = f.read()
137 with open(to, "wb") as f:
138 f.write(bin)
139 else:
140 shutil.copy(fullname, to)
141 return os.path.splitext(fullname)[-1], 0, 0
142 else:
143 try:
144 with open(fullname, "r", encoding="utf8") as g:
145 content = g.read()
146 except UnicodeDecodeError: # pragma: no cover
147 try:
148 with open(fullname, "r") as g:
149 content = g.read()
150 except UnicodeDecodeError as e:
151 raise UnicodeDecodeError(e.encoding, e.object, e.start, e.end,
152 f"Unable to read '{fullname}' due to '{e.reason}'") from e
154 lines = [_.strip(" \t\n\r") for _ in content.split("\n")]
155 lines = [_ for _ in lines if len(_) > 0]
156 nblines = len(lines)
158 keepc = content
159 try:
160 counts, content = migrating_doxygen_doc(content, fullname, silent)
161 except SyntaxError as e: # pragma: no cover
162 if not silent:
163 raise e
164 content = keepc
165 counts = dict(docrows=0)
167 content = fmod(content, fullname)
168 content = remove_undesired_part_for_documentation(
169 content, fullname, use_sys)
170 fold = os.path.split(to)[0]
171 if not os.path.exists(fold):
172 os.makedirs(fold)
173 if replace_relative_import:
174 content = replace_relative_import_fct(fullname, content)
175 with open(to, "w", encoding="utf8") as g:
176 g.write(content)
178 return os.path.splitext(fullname)[-1], nblines, counts["docrows"]
181def remove_undesired_part_for_documentation(content, filename, use_sys):
182 """
183 Some files contains blocs inserted between the two lines:
185 * ``# -- HELP BEGIN EXCLUDE --``
186 * ``# -- HELP END EXCLUDE --``
188 Those lines will be commented out.
190 @param content file content
191 @param filename for error message
192 @param use_sys string or None, enables, disables a section based on variables added to sys module
193 @return modified file content
195 If the parameter *use_sys* is false, the section of code
196 will be commented out. If true, the section can be enabled.
197 It relies on the following code::
199 import sys
200 if hasattr(sys, "<use_sys>") and sys.<use_sys>:
202 # section to enable or disables
204 The string ``<use_sys>`` will be replaced by the value of
205 parameter *use_sys*.
206 """
207 marker_in = "# -- HELP BEGIN EXCLUDE --"
208 marker_out = "# -- HELP END EXCLUDE --"
210 lines = content.split("\n")
211 res = []
212 inside = False
213 has_sys = False
214 flask_trick = False
215 for line in lines:
216 if line.startswith("import sys"):
217 has_sys = True
218 if line.startswith(marker_in):
219 if inside:
220 raise HelpGenException( # pragma: no cover
221 "issues with undesired blocs in file " + filename + " with: " + marker_in + "|" + marker_out)
222 inside = True
223 if use_sys: # pragma: no cover
224 if not has_sys:
225 res.append("import sys")
226 res.append(
227 "if hasattr(sys, '{0}') and sys.{0}:".format(use_sys))
228 res.append(line)
229 elif line.startswith(marker_out):
230 if use_sys and flask_trick: # pragma: no cover
231 res.append(" pass")
232 if not inside:
233 raise HelpGenException( # pragma: no cover
234 "issues with undesired blocs in file " + filename + " with: " + marker_in + "|" + marker_out)
235 inside = False
236 flask_trick = False
237 res.append(line)
238 else:
239 if inside:
240 if use_sys: # pragma: no cover
241 # specific trick for Flask
242 if line.startswith("@app."):
243 line = "# " + line
244 flask_trick = True
245 res.append(" " + line)
246 else:
247 res.append("### " + line)
248 else:
249 res.append(line)
250 return "\n".join(res)
253def copy_source_files(input, output, fmod=lambda v, filename: v,
254 silent=False, filter=None, remove=True,
255 softfile=lambda f: False,
256 fexclude=lambda f: False,
257 addfilter=None, replace_relative_import=False,
258 copy_add_ext=None, use_sys=None, fLOG=fLOG):
259 """
260 Copies all sources files (input) into a folder (output),
261 apply on each of them a modification.
263 :param input: input folder
264 :param output: output folder (it will be cleaned each time)
265 :param fmod: modifies the content of each file,
266 this function takes a string and returns a string
267 :param silent: if True, do not stop when facing an issue with :epkg:`doxygen` documentation
268 :param filter: if None, process only file related to python code, otherwise,
269 use this filter to select file (regular expression). If this parameter
270 is None or is empty, the default value is something like:
271 ``"(.+[.]py$)|(.+[.]pyd$)|(.+[.]cpp$)|(.+[.]h$)|(.+[.]dll$))"``.
272 :param remove: if True, remove every files in the output folder first
273 :param softfile: softfile is a function (f : filename --> True or False), when it is True,
274 the documentation is lighter (no special members)
275 :param fexclude: function to exclude some files from the help
276 :param addfilter: additional filter, it should look like: ``"(.+[.]pyx$)|(.+[.]pyh$)"``
277 :param replace_relative_import: replace relative import
278 :param copy_add_ext: additional extension file to copy
279 :param use_sys: see :func:`remove_undesired_part_for_documentation
280 <pyquickhelper.helpgen.utils_sphinx_doc.remove_undesired_part_for_documentation>`
281 :param fLOG: logging function
282 :return: list of copied files
283 """
284 if not os.path.exists(output):
285 os.makedirs(output)
287 if remove:
288 remove_folder(output, False, raise_exception=False)
290 def_ext = ['py', 'pyd', 'cpp', 'h', 'dll', 'so', 'yml', 'o', 'def', 'gif',
291 'exe', 'data', 'config', 'css', 'js', 'png', 'map', 'sass',
292 'csv', 'tpl', 'jpg', 'jpeg', 'hpp', 'cc', 'tmpl']
293 deffilter = "|".join(f"(.+[.]{_}$)" for _ in def_ext)
294 if copy_add_ext is not None:
295 res = [f"(.+[.]{e}$)" for e in copy_add_ext]
296 deffilter += "|" + "|".join(res)
298 fLOG(f"[copy_source_files] copy filter '{deffilter}'")
300 if addfilter is not None and len(addfilter) > 0:
301 if filter is None or len(filter) == 0:
302 filter = "|".join([deffilter, addfilter])
303 else:
304 filter = "|".join([filter, addfilter])
306 if filter is None:
307 actions = synchronize_folder(input, output, filter=deffilter,
308 avoid_copy=True, fLOG=fLOG)
309 else:
310 actions = synchronize_folder(input, output, filter=filter,
311 avoid_copy=True, fLOG=fLOG)
313 if len(actions) == 0:
314 raise FileNotFoundError("empty folder: " + input) # pragma: no cover
316 ractions = []
317 for a, file, dest in actions:
318 if a != ">+":
319 continue
320 if not validate_file_for_help(file.fullname, fexclude):
321 continue
322 if file.name.endswith("setup.py"):
323 continue
324 if "setup.py" in file.name:
325 raise FileNotFoundError( # pragma: no cover
326 "are you sure (setup.py)?, file: " + file.fullname)
328 to = os.path.join(dest, file.name)
329 dd = os.path.split(to)[0]
330 if not os.path.exists(dd):
331 fLOG("[copy_source_files] create ", dd,
332 f"softfile={softfile} fexclude={fexclude}")
333 os.makedirs(dd)
334 fLOG("[copy_source_files] copy ", file.fullname, " to ", to)
336 rext, rline, rdocline = _private_process_one_file(
337 file.fullname, to, silent, fmod, replace_relative_import, use_sys)
338 ractions.append((a, file, dest, rext, rline, rdocline))
340 return ractions
343def apply_modification_template(rootm, store_obj, template, fullname, rootrep,
344 softfile, indexes, additional_sys_path, fLOG=noLOG):
345 """
346 See @see fn add_file_rst.
348 @param rootm root of the module
349 @param store_obj keep track of all objects extracted from the module
350 @param template rst template to produce
351 @param fullname full name of the file
352 @param rootrep file name in the documentation contains some folders which are not desired in the documentation
353 @param softfile a function (f : filename --> True or False), when it is True,
354 the documentation is lighter (no special members)
355 @param indexes dictionary with the label and some information (IndexInformation)
356 @param additional_sys_path additional path to include to sys.path before importing a module
357 (will be removed afterwards)
358 @param fLOG logging function
359 @return content of a .rst file
361 .. faqref::
362 :title: Why doesn't the documentation show compiled submodules?
364 The instruction ``.. automodule:: <name>`` only shows objects *obj*
365 which verify ``obj.__module__ == name``. This is always the case
366 for modules written in Python but not necessarily for module
367 compiled from C language. When the module is declared,
368 the following structure contains the module name in second position.
369 This name must not be the submodule shortname but the name
370 the module has is the package. The C file
371 *pyquickhelper/helpgen/compiled.c*
372 implements submodule
373 ``pyquickhelper.helpgen.compiled``, this value must replace
374 ``<fullname>`` in the structure below, not simply *compiled*.
376 ::
378 static struct PyModuleDef moduledef = {
379 PyModuleDef_HEAD_INIT,
380 "<fullname>",
381 "Helper for parallelization with threads with C++.",
382 sizeof(struct module_state),
383 fonctions,
384 NULL,
385 threader_module_traverse,
386 threader_module_clear,
387 NULL
388 };
390 .. warning::
391 This function still needs some improvments
392 for C++ modules on MacOSX.
393 """
394 from pandas import DataFrame
396 keepf = fullname
397 filename = os.path.split(fullname)[-1]
398 filenoext = os.path.splitext(filename)[0]
399 fullname = fullname.strip(".").replace(
400 "\\", "/").replace("/", ".").strip(".")
401 if rootrep[0] in fullname:
402 pos = fullname.index(rootrep[0])
403 fullname = rootrep[1] + fullname[pos + len(rootrep[0]):]
404 fullnamenoext = fullname[:-3] if fullname.endswith(".py") else fullname
405 if fullnamenoext.endswith(".pyd"):
406 fullnamenoext = '.'.join(fullnamenoext.split('.')[:-2])
407 elif fullnamenoext.endswith('-linux-gnu.so'):
408 fullnamenoext = '.'.join(fullnamenoext.split('.')[:-2])
409 pythonname = None
411 not_expected = os.environ.get(
412 "USERNAME", os.environ.get("USER", "````````````"))
413 if not_expected not in ('jenkins', 'vsts', 'runner') and not_expected in fullnamenoext:
414 mes = ("The title is probably wrong (5): {0}\nnoext='{1}'\npython='{2}'\nrootm='{3}'\nrootrep='{4}'"
415 "\nfullname='{5}'\nkeepf='{6}'\nnot_expected='{7}'") # pragma: no cover
416 raise HelpGenException(mes.format( # pragma: no cover
417 fullnamenoext, filenoext, pythonname, rootm, rootrep, fullname, keepf, not_expected))
419 mo, prefix = import_module(
420 rootm, keepf, fLOG, additional_sys_path=additional_sys_path)
421 doc = ""
422 shortdoc = ""
424 additional = {}
425 tspecials = {}
427 if mo is not None:
428 if isinstance(mo, str): # pragma: no cover
429 # it is an error
430 spl = mo.split("\n")
431 mo = "\n".join([" " + _ for _ in spl])
432 mo = "::\n\n" + mo + "\n\n"
433 doc = mo
434 shortdoc = "Error"
435 pythonname = fullnamenoext
436 else:
437 pythonname = mo.__name__
438 if mo.__doc__ is not None:
439 doc = mo.__doc__
440 doc = private_migrating_doxygen_doc(
441 doc.split("\n"), 0, fullname)
442 doct = doc
443 doc = []
445 for d in doct:
446 if len(doc) != 0 or len(d) > 0:
447 doc.append(d)
448 while len(doc) > 0 and len(doc[-1]) == 0:
449 doc.pop()
451 shortdoc = doc[0] if len(doc) > 0 else ""
452 if len(doc) > 1:
453 shortdoc += "..."
455 doc = "\n".join(doc)
456 doc = "module ``" + mo.__name__ + "``\n\n" + doc
457 if ":githublink:" not in doc:
458 doc += "\n\n:githublink:`GitHub|py|*`"
459 else:
460 doc = ""
461 shortdoc = "empty"
463 # it produces the table for the function, classes, and
464 objs = get_module_objects(mo)
466 prefix = ".".join(fullnamenoext.split(".")[:-1])
467 for ob in objs:
469 if ob.type in ["method"] and ob.name.startswith("_"):
470 tspecials[ob.name] = ob
472 ob.add_prefix(prefix)
473 if ob.key in store_obj:
474 if isinstance(store_obj[ob.key], list):
475 store_obj[ob.key].append(ob)
476 else:
477 store_obj[ob.key] = [store_obj[ob.key], ob]
478 else:
479 store_obj[ob.key] = ob
481 for k, v in add_file_rst_template_cor.items():
482 values = [[o.rst_link(None, class_in_bracket=False), o.truncdoc]
483 for o in objs if o.type == k]
484 if len(values) > 0:
485 tbl = DataFrame(
486 columns=[k, "truncated documentation"], data=values)
487 for row in tbl.values:
488 if ":meth:`_" in row[0]:
489 row[0] = row[0].replace(":meth:`_", ":py:meth:`_")
491 if len(tbl) > 0:
492 maxi = max(len(_) for _ in tbl[k])
493 s = 0 if tbl.iloc[0, 1] is None else len(
494 tbl.iloc[0, 1])
495 t = "" if tbl.iloc[0, 1] is None else tbl.iloc[0, 1]
496 tbl.iloc[0, 1] = t + (" " * (3 * maxi - s))
497 sph = df2rst(tbl)
498 titl = "\n\n" + add_file_rst_template_title[k] + "\n"
499 titl += "+" * len(add_file_rst_template_title[k])
500 titl += "\n\n"
501 additional[v] = titl + sph
502 else:
503 additional[v] = ""
504 else:
505 additional[v] = ""
507 del mo
509 else:
510 doc = "[sphinxerror]-C unable to import."
512 if indexes is None:
513 indexes = {}
514 label = IndexInformation.get_label(indexes, "f-" + filenoext)
515 indexes[label] = IndexInformation(
516 "module", label, filenoext, doc, None, keepf)
517 fLOG("[apply_modification_template] adding into index ", indexes[label])
519 try:
520 with open(keepf, "r") as ft:
521 content = ft.read()
522 except UnicodeDecodeError:
523 try:
524 with open(keepf, "r", encoding="latin-1") as ft:
525 content = ft.read()
526 except UnicodeDecodeError: # pragma: no cover
527 with open(keepf, "r", encoding="utf8") as ft:
528 content = ft.read()
530 plat = "Windows" if "This example only runs on Windows." in content else "any"
532 # dealing with special members (does not work)
533 # text_specials = "".join([" :special-members: %s\n" % k for k in tspecials ])
534 text_specials = ""
536 if fullnamenoext.endswith(".__init__"):
537 fullnamenoext = fullnamenoext[: -len(".__init__")]
538 if filenoext.endswith(".__init__"):
539 filenoext = filenoext[: -len(".__init__")]
541 not_expected = os.environ.get(
542 "USERNAME", os.environ.get("USER", "````````````"))
543 if not_expected not in ('jenkins', 'vsts', 'runner') and not_expected in fullnamenoext:
544 mes = ("The title is probably wrong (3): {0}\nnoext={1}\npython={2}\nrootm={3}\nrootrep={4}"
545 "\nfullname={5}\nkeepf={6}\nnot_expected='{7}'") # pragma: no cover
546 raise HelpGenException(mes.format( # pragma: no cover
547 fullnamenoext, filenoext, pythonname, rootm, rootrep, fullname, keepf, not_expected))
549 ttitle = f"module ``{fullnamenoext}``"
550 rep = {
551 "__FULLNAME_UNDERLINED__": ttitle + "\n" + ("=" * len(ttitle)) + "\n",
552 "__FILENAMENOEXT__": filenoext,
553 "__FULLNAMENOEXT__": pythonname,
554 "__DOCUMENTATION__": doc.split("\n.. ")[0],
555 "__DOCUMENTATIONLINE__":
556 shortdoc.split(".. todoext::", maxsplit=1)[0],
557 "__PLATFORM__": plat,
558 "__ADDEDMEMBERS__": text_specials}
560 for k, v in additional.items():
561 rep[k] = v
563 res = template
564 for a, b in rep.items():
565 res = res.replace(a, b)
567 has_class = any(
568 filter(lambda _: _.startswith("class "), content.split("\n")))
569 if not has_class:
570 spl = res.split("\n")
571 spl = [_ for _ in spl if not _.startswith(".. inheritance-diagram::")]
572 res = "\n".join(spl)
574 if softfile(fullname):
575 res = res.replace(":special-members:", "")
577 return res
580def add_file_rst(rootm, store_obj, actions, template=add_file_rst_template,
581 rootrep=("_doc.sphinxdoc.source.pyquickhelper.", ""),
582 fmod=lambda v, filename: v, softfile=lambda f: False,
583 mapped_function=None, indexes=None,
584 additional_sys_path=None, fLOG=noLOG):
585 """
586 Creates a :epkg:`rst` file for every source file.
588 @param rootm root of the module (for relative import)
589 @param store_obj to keep table of all objects
590 @param actions output from @see fn copy_source_files
591 @param template :epkg:`rst` template to produce
592 @param rootrep file name in the documentation contains some folders
593 which are not desired in the documentation
594 @param fmod applies modification to the instanciated template
595 @param softfile softfile is a function (f : filename --> True or False), when it is True,
596 the documentation is lighter (no special members)
597 @param mapped_function list of 2-tuple (pattern, function). Every file matching the pattern
598 will be copied to the documentation folder, its content will be sent
599 to the function and will produce a file <filename>.rst. Example:
600 ``[ (".*[.]sql$", filecontent_to_rst) ]``
601 The function takes two parameters: full_filename, content_filename. It returns
602 a string (the rst file) or a tuple (rst file, short description).
603 By default (if function is None), the function ``filecontent_to_rst`` will be called
604 except for .rst file for which nothing is done.
605 @param indexes to index some information { dictionary label:IndexInformation (...) },
606 the function populates it
607 @param additional_sys_path additional path to include to sys.path before importing a module
608 (will be removed afterwards)
609 @param fLOG logging function
610 @return list of written files stored in RstFileHelp
611 """
612 if indexes is None:
613 indexes = {}
614 if mapped_function is None:
615 mapped_function = []
617 if additional_sys_path is None:
618 additional_sys_path = []
620 memo = {}
621 app = []
622 for action in actions:
623 _, file, dest = action[:3]
624 if not isinstance(file, str):
625 file = file.name
627 to = os.path.join(dest, file)
628 rst = os.path.splitext(to)[0]
629 rst += ".rst"
630 ext = os.path.splitext(to)[-1]
632 if sys.platform == "win32":
633 cpxx = ".cp%d%d-" % sys.version_info[:2]
634 elif sys.version_info[:2] <= (3, 7):
635 cpxx = ".cpython-%d%dm-" % sys.version_info[:2]
636 else:
637 cpxx = ".cpython-%d%d-" % sys.version_info[:2]
639 if file.endswith(".py") or (
640 cpxx in file and (
641 file.endswith(".pyd") or file.endswith("linux-gnu.so")
642 )):
643 if os.stat(to).st_size > 0:
644 content = apply_modification_template(
645 rootm, store_obj, template, to, rootrep, softfile, indexes,
646 additional_sys_path=additional_sys_path, fLOG=fLOG)
647 content = fmod(content, file)
649 # tweaks for example and studies
650 zzz = to.replace("\\", "/")
651 name = os.path.split(file)[-1]
652 noex = os.path.splitext(name)[0]
654 # todo: specific case: should be removed and added back in a
655 # proper way
656 if "examples/" in zzz or "studies/" in zzz:
657 content += "\n.. _%s_literal:\n\nCode\n----\n\n.. literalinclude:: %s\n\n" % (
658 noex, name)
660 with open(rst, "w", encoding="utf8") as g:
661 g.write(content)
662 app.append(RstFileHelp(to, rst, ""))
664 for vv in indexes.values():
665 if vv.fullname == to:
666 vv.set_rst_file(rst)
667 break
669 else:
670 for pat, func in mapped_function:
671 if func is None and ext == ".rst":
672 continue
673 if pat not in memo:
674 memo[pat] = re.compile(pat)
675 exp = memo[pat]
676 if exp.search(file):
677 if isinstance(func, bool) and not func:
678 # we copy but we do nothing with it
679 pass
680 else:
681 with open(to, "r", encoding="utf8") as g:
682 content = g.read()
683 if func is None:
684 func = filecontent_to_rst
685 content = func(to, content)
687 if isinstance(content, tuple) and len(content) == 2:
688 content, doc = content
689 else:
690 doc = ""
692 with open(rst, "w", encoding="utf8") as g:
693 g.write(content)
694 app.append(RstFileHelp(to, rst, ""))
696 filenoext, ext = os.path.splitext(
697 os.path.split(to)[-1])
698 ext = ext.strip(".")
699 label = IndexInformation.get_label(
700 indexes, "ext-" + filenoext)
701 indexes[label] = IndexInformation(
702 "ext-" + ext, label, filenoext, doc, rst, to)
703 fLOG("[add_file_rst] add ext into index ", indexes[label])
705 return app
708def produces_indexes(store_obj, indexes, fexclude_index, titles=None,
709 correspondances=None, fLOG=fLOG):
710 """
711 Produces a file for each category of object found in the module.
713 @param store_obj list of collected object, it is a dictionary
714 key : ModuleMemberDoc or key : [ list of ModuleMemberDoc ]
715 @param indexes list of things to index, dictionary { label : IndexInformation }
716 @param fexclude_index to exclude files from the indices
717 @param titles each type is mapped to a title to add to the :epkg:`rst` file
718 @param correspondances each type is mapped to a label to add to the :epkg:`rst` file
719 @param fLOG logging function
720 @return dictionary: { type : rst content of the index }
722 Default values if *titles* of *correspondances* is None:
724 ::
726 title = {"method": "Methods",
727 "staticmethod": "Static Methods",
728 "property": "Properties",
729 "function": "Functions",
730 "class": "Classes",
731 "module": "Modules"}
733 correspondances = {"method": "l-methods",
734 "function": "l-functions",
735 "staticmethod": "l-staticmethods",
736 "property": "l-properties",
737 "class": "l-classes",
738 "module": "l-modules"}
739 """
740 from pandas import DataFrame
742 if titles is None:
743 titles = {"method": "Methods",
744 "staticmethod": "Static Methods",
745 "property": "Properties",
746 "function": "Functions",
747 "class": "Classes",
748 "module": "Modules"}
750 if correspondances is None:
751 correspondances = {"method": "l-methods",
752 "function": "l-functions",
753 "staticmethod": "l-staticmethods",
754 "property": "l-properties",
755 "class": "l-classes",
756 "module": "l-modules"}
758 # we process store_obj
759 types = {}
760 for k, v in store_obj.items():
761 if not isinstance(v, list):
762 v = [v]
763 for _ in v:
764 if fexclude_index(_):
765 continue
766 types[_.type] = types.get(_.type, 0) + 1
768 fLOG(f"[produces_indexes] store_obj: extraction of types: {types}")
769 res = {}
770 for k in types:
771 fLOG(f"[produces_indexes] type: [{k}] - rst")
772 values = []
773 for t, so in store_obj.items():
774 if not isinstance(so, list):
775 so = [so]
777 for o in so:
778 if fexclude_index(o):
779 continue
780 if o.type != k:
781 continue
782 oclname = o.classname.__name__ if o.classname is not None else ""
783 rlink = o.rst_link(class_in_bracket=False)
784 fLOG(f"[produces_indexes] + '{o.name}': {rlink}")
785 values.append([o.name, rlink, oclname, o.truncdoc])
787 values.sort()
788 for row in values:
789 if ":meth:`_" in row[1]:
790 row[1] = row[1].replace(":meth:`_", ":py:meth:`_")
792 # we filter private method or functions
793 values = [
794 row for row in values if ":meth:`__" in row or ":meth:`_" not in row]
795 values = [
796 row for row in values if ":func:`__" in row or ":func:`_" not in row]
798 columns = ["_", k, "class parent", "truncated documentation"]
799 tbl = DataFrame(columns=columns, data=values)
800 if len(tbl.columns) >= 2:
801 tbl = tbl.iloc[:, 1:].copy()
803 if len(tbl) > 0:
804 maxi = max(len(_) for _ in tbl[k])
805 s = 0 if tbl.iloc[0, 1] is None else len(tbl.iloc[0, 1])
806 t = "" if tbl.iloc[0, 1] is None else tbl.iloc[0, 1]
807 tbl.iloc[0, 1] = t + (" " * (3 * maxi - s))
808 sph = df2rst(tbl)
809 res[k] = sph
810 fLOG(f"[produces_indexes] type: [{k}] - shape: {tbl.shape}")
812 # we process indexes
814 fLOG("[produces_indexes] indexes")
815 types = {}
816 for k, v in indexes.items():
817 if fexclude_index(v):
818 continue
819 types[v.type] = types.get(v.type, 0) + 1
821 fLOG(f"[produces_indexes] extraction of types: {types}")
823 for k in types:
824 if k in res:
825 raise HelpGenException( # pragma: no cover
826 f"you should not index anything related to classes, functions or method (conflict: {k})")
827 values = []
828 for t, o in indexes.items():
829 if fexclude_index(o):
830 continue
831 if o.type != k:
832 continue
833 values.append([o.name,
834 o.rst_link(),
835 o.truncdoc])
836 values.sort()
838 tbl = DataFrame(
839 columns=["_", k, "truncated documentation"], data=values)
840 if len(tbl.columns) >= 2:
841 tbl = tbl[tbl.columns[1:]]
843 if len(tbl) > 0:
844 maxi = max(len(_) for _ in tbl[k])
845 tbl.iloc[0, 1] = tbl.iloc[0, 1] + \
846 (" " * (3 * maxi - len(tbl.iloc[0, 1])))
847 sph = df2rst(tbl)
848 res[k] = sph
850 # end
852 keys = list(res.keys())
853 for k in keys:
854 fLOG(f"[produces_indexes] index name '{k}'")
855 label = correspondances.get(k, k)
856 title = titles.get(k, k)
857 under = "=" * len(title)
859 content = "\n".join([".. contents::", " :local:",
860 " :depth: 1", "", "", "Summary", "+++++++"])
862 not_expected = os.environ.get(
863 "USERNAME", os.environ.get("USER", "````````````"))
864 if not_expected != "jenkins" and not_expected in title:
865 raise HelpGenException( # pragma: no cover
866 f"The title is probably wrong (2), found '{not_expected}' in '{title}'")
868 res[k] = f"\n.. _{label}:\n\n{title}\n{under}\n\n{content}\n\n{res[k]}"
870 return res
873def filecontent_to_rst(filename, content):
874 """
875 Produces a *.rst* file which contains the file.
876 It adds a title and a label based on the
877 filename (no folder included).
879 @param filename filename
880 @param content content
881 @return new content
882 """
883 file = os.path.split(filename)[-1]
884 full = file + "\n" + ("=" * len(file)) + "\n"
886 not_expected = os.environ.get(
887 "USERNAME", os.environ.get("USER", "````````````"))
888 if not_expected != "jenkins" and not_expected in file:
889 raise HelpGenException( # pragma: no cover
890 f"The title is probably wrong (1): '{not_expected}' found in '{file}'")
892 rows = ["", f".. _f-{file}:", "", "", full, "",
893 # "fullpath: ``%s``" % filename,
894 "", ""]
895 if ".. RSTFORMAT." in content:
896 rows.append(f".. include:: {file} ")
897 else:
898 rows.append(f".. literalinclude:: {file} ")
899 rows.append("")
901 nospl = content.replace("\n", "_!_!:!_")
903 reg = re.compile("(.. beginshortsummary[.](.*?).. endshortsummary[.])")
904 cont = reg.search(nospl)
905 if cont:
906 g = cont.groups()[1].replace("_!_!:!_", "\n")
907 return "\n".join(rows), g.strip("\n\r ")
909 if "@brief" in content:
910 spl = content.split("\n")
911 begin = None
912 end = None
913 for i, r in enumerate(spl):
914 if "@brief" in r:
915 begin = i
916 if end is None and begin is not None and len(
917 r.strip(" \n\r\t")) == 0:
918 end = i
920 if begin is not None and end is not None:
921 summary = "\n".join(spl[begin:end]).replace(
922 "@brief", "").strip("\n\t\r ")
923 else:
924 summary = "no documentation" # pragma: no cover
926 # looking for C++/java/C# comments
927 spl = content.split("\n")
928 begin = None
929 end = None
930 for i, r in enumerate(spl):
931 if "/**" in r:
932 begin = i
933 if end is None and begin is not None and "*/" in r:
934 end = i
936 content = "\n".join(rows)
937 if begin is not None and end is not None: # pragma: no cover
938 filerows = private_migrating_doxygen_doc(
939 spl[begin + 1:end - 1], 1, filename)
940 rstr = "\n".join(filerows)
941 rstr = re.sub(
942 ":param +([a-zA-Z_][[a-zA-Z_0-9]*) *:", r"* **\1**:", rstr)
943 content = content.replace(
944 ".. literalinclude::", f"\n{rstr}\n\n.. literalinclude::")
946 return content, summary
948 return "\n".join(rows), "no documentation"
951def prepare_file_for_sphinx_help_generation(store_obj, input, output,
952 subfolders, fmod_copy=lambda v, filename: v,
953 template=add_file_rst_template,
954 rootrep=(
955 "_doc.sphinxdoc.source.project_name.", ""),
956 fmod_res=lambda v, filename: v, silent=False,
957 optional_dirs=None, softfile=lambda f: False,
958 fexclude=lambda f: False, mapped_function=None,
959 fexclude_index=lambda f: False, issues=None,
960 additional_sys_path=None, replace_relative_import=False,
961 module_name=None, copy_add_ext=None, use_sys=None,
962 auto_rst_generation=True, fLOG=fLOG):
963 """
964 Prepares all files for :epkg:`Sphinx` generation.
966 @param store_obj to keep track of all objects, it should be a dictionary
967 @param input input folder
968 @param output output folder (it will be cleaned each time)
969 @param subfolders list of subfolders to copy from input to output, two cases:
970 * a string input/<sub> --> output/<sub>
971 * a tuple input/<sub[0]> --> output/<sub[1]>
972 @param fmod_copy modifies the content of each file,
973 this function takes a string and the filename and returns a string
974 ``f(content, filename) --> string``
975 @param template rst template to produce
976 @param rootrep file name in the documentation contains some folders which are not desired in the documentation
977 @param fmod_res applies modification to the instanciated template
978 @param silent if True, do not stop when facing an issue with doxygen migration
979 @param optional_dirs list of tuple with a list of folders (source, copy, filter) to
980 copy for the documentation, example:
981 ``( <folder_help>, "coverage", ".*" )``
982 @param softfile softfile is a function (f : filename --> True or False), when it is True,
983 the documentation is lighter (no special members)
984 @param fexclude function to exclude some files from the help
985 @param fexclude_index function to exclude some files from the indices
987 @param mapped_function list of 2-tuple (pattern, function). Every file matching the pattern
988 will be copied to the documentation folder, its content will be sent
989 to the function and will produce a file <filename>.rst. Example:
990 ``[ (".*[.]sql$", filecontent_to_rst) ]``
991 The function takes two parameters: full_filename, content_filename. It returns
992 a string (the rst file) or a tuple (rst file, short description).
993 By default (if function is None), the function ``filecontent_to_rst`` will be called.
995 @param issues if not None (a list), the function will store some issues here.
997 @param additional_sys_path additional paths to includes to sys.path when import a module (will be removed afterwards)
998 @param replace_relative_import replace relative import
999 @param module_name module name (cannot be None)
1000 @param copy_add_ext additional file extension to copy
1001 @param use_sys @see fn remove_undesired_part_for_documentation
1002 @param auto_rst_generation add a file *.rst* for each source file
1003 @param fLOG logging function
1005 @return list of written files stored in @see cl RstFileHelp
1007 Example:
1009 ::
1011 prepare_file_for_sphinx_help_generation (
1012 {},
1013 ".",
1014 "_doc/sphinxdoc/source/",
1015 subfolders = [
1016 ("src/" + project_var_name,
1017 project_var_name),
1018 ],
1019 silent = True,
1020 rootrep = ("_doc.sphinxdoc.source.%s." %
1021 (project_var_name,), ""),
1022 optional_dirs = optional_dirs,
1023 mapped_function = [ (".*[.]tohelp$", None) ] )
1025 It produces a file with the number of lines and files per extension.
1026 """
1027 if optional_dirs is None:
1028 optional_dirs = []
1030 if mapped_function is None:
1031 mapped_function = []
1033 if additional_sys_path is None:
1034 additional_sys_path = []
1036 if module_name is None:
1037 raise ValueError( # pragma: no cover
1038 "module_name cannot be None")
1040 fLOG(f"[prepare_file_for_sphinx_help_generation] output='{output}'")
1041 rootm = os.path.abspath(output)
1042 fLOG(f"[prepare_file_for_sphinx_help_generation] input='{input}'")
1044 actions = []
1045 rsts = []
1046 indexes = {}
1048 for sub in subfolders:
1049 if isinstance(sub, str):
1050 src = (input + "/" + sub).replace("//", "/")
1051 dst = (output + "/" + sub).replace("//", "/")
1052 else:
1053 src = (input + "/" + sub[0]).replace("//", "/")
1054 dst = (output + "/" + sub[1]).replace("//", "/")
1055 if os.path.split(src)[-1][0] == '_':
1056 raise RuntimeError( # pragma: no cover
1057 f"Subfolder {src!r} cannot start with '_'.")
1058 if os.path.split(dst)[-1][0] == '_':
1059 raise RuntimeError( # pragma: no cover
1060 f"Destination {dst!r} cannot start with '_'.")
1062 if os.path.isfile(src):
1063 fLOG(" [p] ", src)
1064 _private_process_one_file(
1065 src, dst, silent, fmod_copy, replace_relative_import, use_sys)
1067 temp = os.path.split(dst)
1068 actions_t = [(">", temp[1], temp[0], 0, 0)]
1069 if auto_rst_generation:
1070 rstadd = add_file_rst(rootm, store_obj, actions_t,
1071 template, rootrep, fmod_res,
1072 softfile=softfile,
1073 mapped_function=mapped_function,
1074 indexes=indexes,
1075 additional_sys_path=additional_sys_path,
1076 fLOG=fLOG)
1077 rsts += rstadd
1078 else:
1079 fLOG(
1080 f"[prepare_file_for_sphinx_help_generation] processing '{src}'")
1082 actions_t = copy_source_files(src, dst, fmod_copy, silent=silent,
1083 softfile=softfile, fexclude=fexclude,
1084 addfilter="|".join(
1085 [f'({_[0]})' for _ in mapped_function]),
1086 replace_relative_import=replace_relative_import,
1087 copy_add_ext=copy_add_ext,
1088 use_sys=use_sys, fLOG=fLOG)
1090 # without those two lines, importing the module might crash later
1091 importlib.invalidate_caches()
1092 importlib.util.find_spec(module_name)
1094 if auto_rst_generation:
1095 rsts += add_file_rst(rootm, store_obj, actions_t, template,
1096 rootrep, fmod_res, softfile=softfile,
1097 mapped_function=mapped_function,
1098 indexes=indexes,
1099 additional_sys_path=additional_sys_path,
1100 fLOG=fLOG)
1102 actions += actions_t
1104 # everything is cleaned from the build folder, so, it is no use
1105 for tu in optional_dirs:
1106 if len(tu) == 2:
1107 fold, dest, filt = tu + (".*", )
1108 else:
1109 fold, dest, filt = tu
1110 if filt is None:
1111 filt = ".*"
1112 if not os.path.exists(dest):
1113 fLOG("creating folder (sphinx) ", dest)
1114 os.makedirs(dest)
1116 copy_source_files(fold, dest, silent=silent, filter=filt,
1117 softfile=softfile, fexclude=fexclude,
1118 addfilter="|".join([f'({_[0]})'
1119 for _ in mapped_function]),
1120 replace_relative_import=replace_relative_import,
1121 copy_add_ext=copy_add_ext, fLOG=fLOG)
1123 # processing all store_obj to compute some indices
1124 fLOG("[prepare_file_for_sphinx_help_generation] processing all store_obj to compute some indices")
1125 fLOG("[prepare_file_for_sphinx_help_generation] extracted ",
1126 len(store_obj), " objects")
1127 res = produces_indexes(store_obj, indexes, fexclude_index, fLOG=fLOG)
1129 fLOG("[prepare_file_for_sphinx_help_generation] generating ",
1130 len(res), " indexes for ", ", ".join(list(res.keys())))
1131 allfiles = []
1132 for k, vv in res.items():
1133 out = os.path.join(output, "index_" + k + ".rst")
1134 allfiles.append("index_" + k)
1135 fLOG(" generates index", out)
1136 if k == "module":
1137 toc = ["\n\n.. toctree::"]
1138 toc.append(" :maxdepth: 1\n")
1139 for _ in rsts:
1140 if _.file is not None and len(_.file) > 0:
1141 na = os.path.splitext(_.rst)[0].replace(
1142 "\\", "/").split("/")
1143 if "source" in na:
1144 na = na[na.index("source") + 1:]
1145 na = "/".join(na)
1146 toc.append(" " + na)
1147 vv += "\n".join(toc)
1148 with open(out, "w", encoding="utf8") as fh:
1149 fh.write(vv)
1150 rsts.append(RstFileHelp(None, out, None))
1152 # generates a table with the number of lines per extension
1153 rows = []
1154 for act in actions:
1155 if "__init__.py" not in act[1].get_fullname() or act[-1] > 0:
1156 v = 1
1157 rows.append(act[-3:] + (v,))
1158 name = os.path.split(act[1].get_fullname())[-1]
1159 if name.startswith("auto_"):
1160 rows.append(("auto_*" + act[-3], act[-2], act[-1], v))
1161 elif "__init__.py" in name:
1162 rows.append(("__init__.py", act[-2], act[-1], v))
1163 elif "__init__.py" in act[1].get_fullname():
1164 v = 1
1165 rows.append(("empty __init__.py", act[-2], act[-1], v))
1167 # use DataFrame to produce a RST table
1168 from pandas import DataFrame
1169 df = DataFrame(
1170 data=rows, columns=["extension/kind", "nb lines", "nb doc lines", "nb files"])
1171 try:
1172 # for pandas >= 0.17
1173 df = df.groupby(
1174 "extension/kind", as_index=False).sum().sort_values("extension/kind")
1175 except AttributeError: # pragma: no cover
1176 # for pandas < 0.17
1177 df = df.groupby(
1178 "extension/kind", as_index=False).sum().sort("extension/kind")
1180 # reports
1181 fLOG("[prepare_file_for_sphinx_help_generation] writing ", "all_report.rst")
1182 all_report = os.path.join(output, "all_report.rst")
1183 with open(all_report, "w") as falli:
1184 falli.write("\n:orphan:\n\n")
1185 falli.write(".. _l-statcode:\n")
1186 falli.write("\n")
1187 falli.write("Statistics on code\n")
1188 falli.write("==================\n")
1189 falli.write("\n\n")
1190 sph = df2rst(df, list_table=True)
1191 falli.write(sph)
1192 falli.write("\n")
1193 rsts.append(RstFileHelp(None, all_report, None))
1195 # all indexes
1196 fLOG("[prepare_file_for_sphinx_help_generation] writing ", "all_indexes.rst")
1197 all_index = os.path.join(output, "all_indexes.rst")
1198 with open(all_index, "w") as falli:
1199 falli.write("\n:orphan:\n\n")
1200 falli.write("\n")
1201 falli.write("All indexes\n")
1202 falli.write("===========\n")
1203 falli.write("\n\n")
1204 falli.write(".. toctree::\n")
1205 falli.write(" :maxdepth: 2\n")
1206 falli.write("\n")
1207 for k in sorted(allfiles):
1208 falli.write(f" {k}\n")
1209 falli.write("\n")
1210 rsts.append(RstFileHelp(None, all_index, None))
1212 # last function to process images
1213 fLOG("looking for images", output)
1215 images = os.path.join(output, "images")
1216 fLOG("+looking for images into ", images, " for folder ", output)
1217 if os.path.exists(images):
1218 process_copy_images(output, images)
1220 # fixes indexed objects with incomplete names
1221 # :class:`name` --> :class:`name <...>`
1222 fLOG("+looking for incomplete references", output)
1223 fix_incomplete_references(output, store_obj, issues=issues, fLOG=fLOG)
1224 # for t,so in store_obj.items() :
1226 # look for FAQ and example
1227 fLOG("[prepare_file_for_sphinx_help_generation] FAQ + examples")
1228 app = []
1229 for tag, title in [("FAQ", "FAQ"),
1230 ("example", "Examples"),
1231 ("NB", "Magic commands"), ]:
1232 onefiles = process_look_for_tag(tag, title, rsts)
1233 for page, onefile in onefiles:
1234 saveas = os.path.join(output, "all_%s%s.rst" %
1235 (tag,
1236 page.replace(":", "").replace("/", "").replace(" ", "")))
1237 with open(saveas, "w", encoding="utf8") as fh:
1238 fh.write(onefile)
1239 app.append(RstFileHelp(saveas, onefile, ""))
1240 rsts += app
1242 fLOG("[prepare_file_for_sphinx_help_generation] END", output)
1243 return actions, rsts
1246def process_copy_images(folder_source, folder_images):
1247 """
1248 Looks into every file .rst or .py for images (.. image:: imagename),
1249 if this image was found in directory folder_images, then the image is copied
1250 closes to the file.
1252 @param folder_source folder where to look for sources
1253 @param folder_images folder where to look for images
1254 @return list of copied images
1255 """
1256 _, files = explore_folder(folder_source, "[.]((rst)|(py))$")
1257 reg = re.compile(".. image::(.*)")
1258 cop = []
1259 for fn in files:
1260 try:
1261 with open(fn, "r", encoding="utf8") as f:
1262 content = f.read()
1263 except Exception as e: # pragma: no cover
1264 try:
1265 with open(fn, "r") as f:
1266 content = f.read()
1267 except Exception:
1268 raise RuntimeError(f"Issue with file '{fn}'") from e
1270 lines = content.split("\n")
1271 for line in lines:
1272 img = reg.search(line)
1273 if img:
1274 name = img.groups()[0].strip()
1275 fin = os.path.split(name)[-1]
1276 path = os.path.join(folder_images, fin)
1277 if os.path.exists(path):
1278 dest = os.path.join(os.path.split(fn)[0], fin)
1279 shutil.copy(path, dest)
1280 fLOG("+copy img ", fin, " to ", dest)
1281 cop.append(dest)
1282 else:
1283 fLOG("-unable to find image ", name)
1284 return cop
1287def fix_incomplete_references(folder_source, store_obj, issues=None, fLOG=fLOG):
1288 """
1289 Looks into every file .rst or .py for incomplete reference. Example::
1291 :class:`name` --> :class:`name <...>`.
1294 @param folder_source folder where to look for sources
1295 @param store_obj container for indexed objects
1296 @param issues if not None (a list), it will add issues (function, message)
1297 @param fLOG logging function
1298 @return list of fixed references
1299 """
1300 cor = {"func": ["function"],
1301 "meth": ["method", "property", "staticmethod"]
1302 }
1304 _, files = explore_folder(folder_source, "[.](py)$")
1305 reg = re.compile(
1306 "(:(py:)?((class)|(meth)|(func)):`([a-zA-Z_][a-zA-Z0-9_]*?)`)")
1307 cop = []
1308 for fn in files:
1309 try:
1310 with open(fn, "r", encoding="utf8") as f:
1311 content = f.read()
1312 encoding = "utf8"
1313 except Exception: # pragma: no cover
1314 with open(fn, "r") as f:
1315 content = f.read()
1316 encoding = None
1318 mainname = os.path.splitext(os.path.split(fn)[-1])[0]
1320 modif = False
1321 lines = content.split("\n")
1322 rline = []
1323 for line in lines:
1324 ref = reg.search(line)
1325 if ref:
1326 all = ref.groups()[0]
1327 # pre = ref.groups()[1]
1328 typ = ref.groups()[2]
1329 nam = ref.groups()[-1]
1331 key = None
1332 obj = None
1333 for cand in cor.get(typ, [typ]):
1334 k = f"{cand};{nam}"
1335 if k in store_obj:
1336 if isinstance(store_obj[k], list):
1337 se = [
1338 _s for _s in store_obj[k] if mainname in _s.rst_link()]
1339 if len(se) == 1:
1340 obj = se[0]
1341 break
1342 else:
1343 key = k
1344 obj = store_obj[k]
1345 break
1347 if key in store_obj:
1348 modif = True
1349 lnk = obj.rst_link(class_in_bracket=False)
1350 fLOG(" i,ref, found ", all, " --> ", lnk)
1351 line = line.replace(all, lnk)
1352 else:
1353 fLOG(
1354 " w,unable to replace key ", key, ": ", all, "in file", fn)
1355 if issues is not None:
1356 issues.append(("fix_incomplete_references",
1357 "Unable to replace key '%s', link '%s' in file "
1358 "'%s'." % (key, all, fn)))
1360 rline.append(line)
1362 if modif:
1363 if encoding == "utf8":
1364 with open(fn, "w", encoding="utf8") as f:
1365 f.write("\n".join(rline))
1366 else:
1367 with open(fn, "w") as f:
1368 f.write("\n".join(rline))
1369 return cop
1372def migrating_doxygen_doc(content, filename, silent=False, log=False, debug=False):
1373 """
1374 Migrates the doxygen documentation to rst format.
1376 @param content file content
1377 @param filename filename (to display useful error messages)
1378 @param silent if silent, do not raise an exception
1379 @param log if True, write some information in the logs (not only exceptions)
1380 @param debug display more information on the output if True
1381 @return statistics, new content file
1383 Function ``private_migrating_doxygen_doc`` enumerates the list of conversion
1384 which will be done.
1385 """
1386 if log:
1387 fLOG("migrating_doxygen_doc: ", filename)
1389 rows = []
1390 counts = {"docrows": 0}
1392 def print_in_rows(v, file=None):
1393 rows.append(v)
1395 def local_private_migrating_doxygen_doc(r, index_first_line, filename):
1396 counts["docrows"] += len(r)
1397 return _private_migrating_doxygen_doc(r, index_first_line,
1398 filename, debug=debug, silent=silent)
1400 process_string(content, print_in_rows, local_private_migrating_doxygen_doc,
1401 filename, 0, debug=debug)
1402 return counts, "\n".join(rows)
1404# -- HELP BEGIN EXCLUDE --
1407def private_migrating_doxygen_doc(rows, index_first_line, filename,
1408 debug=False, silent=False):
1409 """
1410 Processes a block help (from doxygen to rst).
1412 @param rows list of text lines
1413 @param index_first_line index of the first line (to display useful message error)
1414 @param filename filename (to display useful message error)
1415 @param silent if True, do not display anything
1416 @param debug display more information if True
1417 @return another list of text lines
1419 @warning This function uses regular expression to process the documentation,
1420 it does not import the module (as Sphinx does). It might misunderstand some code.
1422 @todo Try to import the module and if it possible, uses that information to help
1423 the parsing.
1425 The following line displays error message you can click on using SciTe
1427 ::
1429 raise SyntaxError(" File \"%s\", line %d, in ???\n unable to process: %s " %(
1430 filename, index_first_line+i+1, row))
1432 __sphinx__skip__
1434 The previous string tells the function to stop processing the help.
1436 Doxygen conversions::
1438 @param <param_name> description
1439 :param <param_name>: description
1441 @var <param_name> produces a table with the attributes
1443 @return description
1444 :return: description
1446 @rtype description
1447 :rtype: description
1449 @code
1450 code:: + indentation
1452 @endcode
1453 nothing
1455 @file
1456 nothing
1458 @brief
1459 nothing
1461 @ingroup ...
1462 nothing
1464 @defgroup ....
1465 nothing
1467 @image html ...
1469 @see,@ref label forbidden
1470 should be <op> <fn> <label>, example: @ref cl label
1471 <op> must be in [fn, cl, at, me, te, md]
1473 :class:`label`
1474 :func:`label`
1475 :attr:`label`
1476 :meth:`label`
1477 :mod:`label`
1479 @warning description (until next empty line)
1480 .. warning::
1481 description
1483 @todo
1484 .. todo:: a todo box
1486 ------------- not done yet
1488 @img image name
1489 .. image:: test.png
1490 :width: 200pt
1492 .. raw:: html
1493 html indente
1495 """
1496 return _private_migrating_doxygen_doc(rows, index_first_line, filename,
1497 debug=debug, silent=silent)
1499# -- HELP END EXCLUDE --
1502def _private_migrating_doxygen_doc(rows, index_first_line, filename,
1503 debug=False, silent=False):
1504 if debug: # pragma: no cover
1505 fLOG("------------------ P0")
1506 fLOG("\n".join(rows))
1507 fLOG("------------------ P")
1509 debugrows = rows
1510 rows = [_.replace("\t", " ") for _ in rows]
1511 pars = re.compile("([@]param( +)([a-zA-Z0-9_]+)) ")
1512 refe = re.compile(
1513 "([@]((see)|(ref)) +((fn)|(cl)|(at)|(me)|(te)|(md)) +([a-zA-Z0-9_]+))($|[^a-zA-Z0-9_])")
1514 exce = re.compile("([@]exception( +)([a-zA-Z0-9_]+)) ")
1515 exem = re.compile("([@]example[(](.*?___)?(.*?)[)])")
1516 faq_ = re.compile("([@]FAQ[(](.*?___)?(.*?)[)])")
1517 nb_ = re.compile("([@]NB[(](.*?___)?(.*?)[)])")
1519 # min indent
1520 if len(rows) > 1:
1521 space_rows = [(r.lstrip(), r) for r in rows[1:] if len(r.strip()) > 0]
1522 else:
1523 space_rows = []
1524 if len(space_rows) > 0:
1525 min_indent = min(len(r[1]) - len(r[0]) for r in space_rows)
1526 else:
1527 min_indent = 0
1529 # We fix the first rows which might be different from the others.
1530 if len(rows) > 1:
1531 r = rows[0]
1532 r = (r.lstrip(), r)
1533 delta = len(r[1]) - len(r[0])
1534 if delta != min_indent:
1535 rows = rows.copy()
1536 rows[0] = " " * min_indent + rows[0].lstrip()
1538 # processing doxygen documentation
1539 indent = False
1540 openi = False
1541 beginends = {}
1543 typstr = str
1545 whole = "\n".join(rows)
1546 if "@var" in whole:
1547 whole = process_var_tag(whole, True)
1548 rows = whole.split("\n")
1550 for i in range(len(rows)):
1551 row = rows[i]
1553 if debug:
1554 fLOG( # pragma: no cover
1555 f"-- indent={indent} openi={openi} row={row}")
1557 if "__sphinx__skip__" in row:
1558 if not silent:
1559 fLOG(" File \"%s\", line %s, skipping" %
1560 (filename, index_first_line + i + 1))
1561 break
1563 strow = row.strip(" ")
1565 if "@endFAQ" in strow or "@endexample" in strow or "@endNB" in strow:
1566 if "@endFAQ" in strow:
1567 beginends["FAQ"] = beginends.get("FAQ", 0) - 1
1568 sp = " " * row.index("@endFAQ")
1569 rows[i] = "\n" + sp + ".. endFAQ.\n"
1570 if "@endexample" in strow:
1571 beginends["example"] = beginends.get("example", 0) - 1
1572 sp = " " * row.index("@endexample")
1573 rows[i] = "\n" + sp + ".. endexample.\n"
1574 if "@endNB" in strow: # pragma: no cover
1575 beginends["NB"] = beginends.get("NB", 0) - 1
1576 sp = " " * row.index("@endNB")
1577 rows[i] = "\n" + sp + ".. endNB.\n"
1578 continue
1580 if indent:
1581 if (not openi and len(strow) == 0) or "@endcode" in strow:
1582 indent = False
1583 rows[i] = ""
1584 openi = False
1585 if "@endcode" in strow:
1586 beginends["code"] = beginends.get("code", 0) - 1
1587 else:
1588 rows[i] = " " + rows[i]
1590 else:
1592 if strow.startswith("@warning"):
1593 pos = rows[i].find("@warning")
1594 sp = " " * pos
1595 rows[i] = rows[i].replace("@warning", f"\n{sp}.. warning:: ")
1596 indent = True
1598 elif strow.startswith("@todo"):
1599 pos = rows[i].find("@todo")
1600 sp = " " * pos
1601 rows[i] = rows[i].replace("@todo", f"\n{sp}.. todo:: ")
1602 indent = True
1604 elif strow.startswith("@ingroup"):
1605 rows[i] = ""
1607 elif strow.startswith("@defgroup"):
1608 rows[i] = ""
1610 elif strow.startswith("@image"):
1611 pos = rows[i].find("@image")
1612 sp = " " * pos
1613 spl = strow.split()
1614 img = spl[-1]
1615 if img.startswith("http://"):
1616 rows[i] = f"\n{sp}.. fancybox:: " + img + "\n\n"
1617 else:
1619 if img.startswith("images") or img.startswith("~"):
1620 # we assume it is a relative path to the source
1621 img = img.strip("~")
1622 spl_path = filename.replace("\\", "/").split("/")
1623 pos = spl_path.index("src")
1624 dots = [".."] * (len(spl_path) - pos - 2)
1625 ref = "/".join(dots) + "/"
1626 else:
1627 ref = ""
1629 sp = " " * row.index("@image")
1630 rows[i] = f"\n{sp}.. image:: {ref}{img}\n{sp} :align: center\n"
1632 elif strow.startswith("@code"):
1633 pos = rows[i].find("@code")
1634 sp = " " * pos
1635 prev = i - 1
1636 while prev > 0 and len(rows[prev].strip(" \n\r\t")) == 0:
1637 prev -= 1
1638 rows[i] = ""
1639 if rows[prev].strip("\n").endswith("."):
1640 rows[prev] += f"\n\n{sp}::\n"
1641 else:
1642 rows[prev] += (":" if rows[prev].endswith(":") else "::")
1643 indent = True
1644 openi = True
1645 beginends["code"] = beginends.get("code", 0) + 1
1647 # basic tags
1648 row = rows[i]
1650 # tag param
1651 look = pars.search(row)
1652 lexxce = exce.search(row)
1653 example = exem.search(row)
1654 faq = faq_.search(row)
1655 nbreg = nb_.search(row)
1657 if look:
1658 rep = look.groups()[0]
1659 sp = look.groups()[1]
1660 name = look.groups()[2]
1661 to = f":param{sp}{name}:"
1662 rows[i] = row.replace(rep, to)
1664 # it requires an empty line before if the previous line does
1665 # not start by :
1666 if i > 0 and not rows[
1667 i - 1].strip().startswith(":") and len(rows[i - 1].strip()) > 0:
1668 rows[i] = "\n" + rows[i]
1670 elif lexxce:
1671 rep = lexxce.groups()[0]
1672 sp = lexxce.groups()[1]
1673 name = lexxce.groups()[2]
1674 to = f":raises{sp}{name}:"
1675 rows[i] = row.replace(rep, to)
1677 # it requires an empty line before if the previous line does
1678 # not start by :
1679 if i > 0 and not rows[
1680 i - 1].strip().startswith(":") and len(rows[i - 1].strip()) > 0:
1681 rows[i] = "\n" + rows[i]
1683 elif example: # pragma: no cover
1684 sp = " " * row.index("@example")
1685 rep = example.groups()[0]
1686 exa = example.groups()[2].replace("[|", "(").replace("|]", ")")
1687 pag = example.groups()[1]
1688 if pag is None:
1689 pag = ""
1690 fil = os.path.splitext(os.path.split(filename)[-1])[0]
1691 fil = re.sub(r'([^a-zA-Z0-9_])', "", fil)
1692 ref = fil + "-l%d" % (i + index_first_line)
1693 ref2 = make_label_index(exa, typstr(example.groups()))
1694 to = "\n\n%s.. _le-%s:\n\n%s.. _le-%s:\n\n%s**Example: %s** \n\n%s.. example(%s%s;;le-%s).\n" % (
1695 sp, ref, sp, ref2, sp, exa, sp, pag, exa, ref)
1696 rows[i] = row.replace(rep, to)
1698 # it requires an empty line before if the previous line does
1699 # not start by :
1700 if i > 0 and not rows[
1701 i - 1].strip().startswith(":") and len(rows[i - 1].strip()) > 0:
1702 rows[i] = "\n" + rows[i]
1703 beginends["example"] = beginends.get("example", 0) + 1
1705 elif faq:
1706 sp = " " * row.index("@FAQ")
1707 rep = faq.groups()[0]
1708 exa = faq.groups()[2].replace("[|", "(").replace("|]", ")")
1709 pag = faq.groups()[1]
1710 if pag is None:
1711 pag = ""
1712 fil = os.path.splitext(os.path.split(filename)[-1])[0]
1713 fil = re.sub(r'([^a-zA-Z0-9_])', "", fil)
1714 ref = fil + "-l%d" % (i + index_first_line)
1715 ref2 = make_label_index(exa, typstr(faq.groups()))
1716 to = "\n\n%s.. _le-%s:\n\n%s.. _le-%s:\n\n%s**FAQ: %s** \n\n%s.. FAQ(%s%s;;le-%s).\n" % (
1717 sp, ref, sp, ref2, sp, exa, sp, pag, exa, ref)
1718 rows[i] = row.replace(rep, to)
1720 # it requires an empty line before if the previous line does
1721 # not start by :
1722 if i > 0 and not rows[
1723 i - 1].strip().startswith(":") and len(rows[i - 1].strip()) > 0:
1724 rows[i] = "\n" + rows[i]
1725 beginends["FAQ"] = beginends.get("FAQ", 0) + 1
1727 elif nbreg: # pragma: no cover
1728 sp = " " * row.index("@NB")
1729 rep = nbreg.groups()[0]
1730 exa = nbreg.groups()[2].replace("[|", "(").replace("|]", ")")
1731 pag = nbreg.groups()[1]
1732 if pag is None:
1733 pag = ""
1734 fil = os.path.splitext(os.path.split(filename)[-1])[0]
1735 fil = re.sub(r'([^a-zA-Z0-9_])', "", fil)
1736 ref = fil + "-l%d" % (i + index_first_line)
1737 ref2 = make_label_index(exa, typstr(nbreg.groups()))
1738 to = "\n\n%s.. _le-%s:\n\n%s.. _le-%s:\n\n%s**NB: %s** \n\n%s.. NB(%s%s;;le-%s).\n" % (
1739 sp, ref, sp, ref2, sp, exa, sp, pag, exa, ref)
1740 rows[i] = row.replace(rep, to)
1742 # it requires an empty line before if the previous line does
1743 # not start by :
1744 if i > 0 and not rows[
1745 i - 1].strip().startswith(":") and len(rows[i - 1].strip()) > 0:
1746 rows[i] = "\n" + rows[i]
1747 beginends["NB"] = beginends.get("NB", 0) + 1
1749 elif "@return" in row:
1750 rows[i] = row.replace("@return", ":return:")
1751 # it requires an empty line before if the previous line does
1752 # not start by :
1753 if i > 0 and not rows[
1754 i - 1].strip().startswith(":") and len(rows[i - 1].strip()) > 0:
1755 rows[i] = "\n" + rows[i]
1757 elif "@rtype" in row:
1758 rows[i] = row.replace("@rtype", ":rtype:")
1759 # it requires an empty line before if the previous line does
1760 # not start by :
1761 if i > 0 and not rows[
1762 i - 1].strip().startswith(":") and len(rows[i - 1].strip()) > 0:
1763 rows[i] = "\n" + rows[i]
1765 elif "@brief" in row:
1766 rows[i] = row.replace("@brief", "").strip()
1767 elif "@file" in row:
1768 rows[i] = row.replace("@file", "").strip()
1770 # loop on references
1771 refl = refe.search(rows[i])
1772 while refl:
1773 see = "see" in refl.groups()[1]
1774 see = "" # " " if see else ""
1775 ty = refl.groups()[4]
1776 name = refl.groups()[-2]
1777 if len(name) == 0:
1778 raise SyntaxError( # pragma: no cover
1779 "name should be empty: " + typstr(refl.groups()))
1780 rep = refl.groups()[0]
1781 ty = {"cl": "class", "me": "meth", "at": "attr",
1782 "fn": "func", "te": "term", "md": "mod"}[ty]
1783 to = f"{see}:{ty}:`{name}`"
1784 rows[i] = rows[i].replace(rep, to)
1785 refl = refe.search(rows[i])
1787 if not debug:
1788 for i, row in enumerate(rows):
1789 if "__sphinx__skip__" in row:
1790 break
1791 if "@param" in row or "@return" in row or "@see" in row or "@warning" in row \
1792 or "@todo" in row or "@code" in row or "@endcode" in row or "@brief" in row or "@file" in row \
1793 or "@rtype" in row or "@exception" in row \
1794 or "@example" in row or "@NB" in row or "@endNB" in row or "@endexample" in row:
1795 if not silent: # pragma: no cover
1796 fLOG("#########################")
1797 _private_migrating_doxygen_doc(
1798 debugrows, index_first_line, filename, debug=True)
1799 fLOG("#########################")
1800 mes = " File \"%s\", line %d, in ???\n unable to process: %s \nwhole blocks:\n%s" % (
1801 filename, index_first_line + i + 1, row, "\n".join(rows))
1802 fLOG("[sphinxerror]-D ", mes)
1803 else: # pragma: no cover
1804 mes = " File \"%s\", line %d, in ???\n unable to process: %s \nwhole blocks:\n%s" % (
1805 filename, index_first_line + i + 1, row, "\n".join(rows))
1806 raise SyntaxError(mes) # pragma: no cover
1808 for k, v in beginends.items():
1809 if v != 0: # pragma: no cover
1810 mes = " File \"%s\", line %d, in ???\n unbalanced tag %s: %s \nwhole blocks:\n%s" % (
1811 filename, index_first_line + i + 1, k, row, "\n".join(rows))
1812 fLOG("[sphinxerror]-E ", mes)
1813 raise SyntaxError(mes)
1815 # add githublink
1816 link = [_ for _ in rows if ":githublink:" in _]
1817 if len(link) == 0:
1818 rows.append("")
1819 rows.append(f"{' ' * min_indent}:githublink:`%|py|{index_first_line}`")
1821 # clean rows
1822 clean_rows = []
1823 for row in rows:
1824 if row.strip():
1825 clean_rows.append(row)
1826 elif len(clean_rows) > 0:
1827 clean_rows.append('')
1828 return clean_rows
1831def doc_checking():
1832 """
1833 Example of a doc string.
1834 """
1835 pass
1838class useless_class_UnicodeStringIOThreadSafe (str):
1840 """avoid conversion problem between str and char,
1841 class protected again Thread issue"""
1843 def __init__(self):
1844 """
1845 creates a lock
1846 """
1847 str.__init__(self)
1848 import threading
1849 self.lock = threading.Lock()