Coverage for src/pyquickhelper/helpgen/utils_sphinx_doc.py : 95%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2@file
3@brief various helpers to produce a Sphinx documentation
5"""
6import os
7import re
8import sys
9import shutil
10import importlib
11from ..loghelper.flog import fLOG, noLOG
12from ..filehelper.synchelper import remove_folder, synchronize_folder, explore_folder
13from ._my_doxypy import process_string
14from .utils_sphinx_doc_helpers import add_file_rst_template, process_var_tag, import_module
15from .utils_sphinx_doc_helpers import get_module_objects, add_file_rst_template_cor, add_file_rst_template_title
16from .utils_sphinx_doc_helpers import IndexInformation, RstFileHelp, HelpGenException, process_look_for_tag, make_label_index
17from ..pandashelper.tblformat import df2rst
20def validate_file_for_help(filename, fexclude=lambda f: False):
21 """
22 Accepts or rejects a file to be copied in the help folder.
24 @param filename filename
25 @param fexclude function to exclude some files
26 @return boolean
27 """
28 if fexclude is not None and fexclude(filename):
29 return False # pragma: no cover
31 if filename.endswith(".pyd") or filename.endswith(".so"):
32 return True
34 if "rpy2" in filename: # pragma: no cover
35 with open(filename, "r") as ff:
36 content = ff.read()
37 if "from pandas.core." in content:
38 return False
40 return True
43def replace_relative_import_fct(fullname, content=None):
44 """
45 Takes a :epkg:`python` file and replaces all relative
46 imports it was able to find by an import which can be
47 processed by :epkg:`Python` if the file were the main file.
49 @param fullname name of the file
50 @param content a preprocessed content of the file of
51 the content if it is None
52 @return content of the file without relative imports
54 Does not change imports in comments.
55 """
56 if content is None:
57 with open(fullname, "r", encoding="utf8") as f:
58 content = f.read()
60 fullpath = os.path.dirname(fullname)
61 fullsplit = fullpath.replace('\\', '/').split('/')
62 root = None
63 for i in range(len(fullsplit), 1, -1):
64 path = "/".join(fullsplit[:i])
65 init = os.path.join(path, '__init__.py')
66 src = os.path.join(path, 'src')
67 cond = init or (not init and src)
68 if not cond:
69 root = i + 1
70 break
71 if i < len(fullsplit) and fullsplit[i] in ('src', 'site-packages'):
72 root = i + 1
73 break
74 if root is None:
75 raise FileNotFoundError( # pragma: no cover
76 "Unable to package root for '{}'.".format(fullname))
78 lines = content.split("\n")
79 name = "([a-zA-Z_][a-zA-Z_0-9]*)"
80 namedot = "([a-zA-Z_][a-zA-Z_0-9.]*)"
81 names = name + "(, " + name + ")*"
82 end = "( .*)?$"
83 regi = re.compile("{0}{1}{2}{3}{4}".format("^( *)from ([.]{1,3})",
84 namedot, " import ", names, end))
86 for i in range(0, len(lines)):
87 line = lines[i]
88 find = regi.search(line)
90 if find:
91 space, dot, rel, name0, names, _, end = find.groups()
92 idot = len(dot)
93 level = len(fullsplit) - root - idot + 1
94 if level > 0:
95 if end is None:
96 end = ""
97 if names is None:
98 names = ""
99 packname = ".".join(fullsplit[root:root + level])
100 if rel:
101 packname += '.' + rel
102 line = "{space}from {packname} import {name0}{names}{end}".format(
103 space=space, packname=packname, name0=name0, names=names, end=end)
104 lines[i] = line
105 else:
106 raise ValueError( # pragma: no cover
107 "Unable to replace relative import in '{0}', "
108 "root='{1}'\n{2}|{3}|{4}|{5}| level={6}".format(
109 line, fullsplit[root], dot, rel, name0, names, level))
111 return "\n".join(lines)
114def _private_process_one_file(
115 fullname, to, silent, fmod, replace_relative_import, use_sys):
116 """
117 Copies one file from the source to the documentation folder.
118 It processes some comments in doxygen format (@ param, @ return).
119 It replaces relatives imports by a regular import.
121 @param fullname name of the file
122 @param to location (folder)
123 @param silent no logs if True
124 @param fmod modification functions
125 @param replace_relative_import replace relative import
126 @param use_sys @see fn remove_undesired_part_for_documentation
127 @return extension, number of lines, number of lines in documentation
128 """
129 ext = os.path.splitext(fullname)[-1]
131 if ext in {".jpeg", ".jpg", ".pyd", ".png", ".dat", ".dll", ".o",
132 ".so", ".exe", ".enc", ".txt", ".gif", ".csv", '.pyx',
133 '*.mp3', '*.mp4', '.tmpl'}:
134 if ext in (".pyd", ".so"):
135 # If the file is being executed, the copy might keep the properties of
136 # the original (only Windows).
137 with open(fullname, "rb") as f:
138 bin = f.read()
139 with open(to, "wb") as f:
140 f.write(bin)
141 else:
142 shutil.copy(fullname, to)
143 return os.path.splitext(fullname)[-1], 0, 0
144 else:
145 try:
146 with open(fullname, "r", encoding="utf8") as g:
147 content = g.read()
148 except UnicodeDecodeError: # pragma: no cover
149 try:
150 with open(fullname, "r") as g:
151 content = g.read()
152 except UnicodeDecodeError as e:
153 raise UnicodeDecodeError(e.encoding, e.object, e.start, e.end,
154 "Unable to read '{0}' due to '{1}'".format(fullname, e.reason)) from e
156 lines = [_.strip(" \t\n\r") for _ in content.split("\n")]
157 lines = [_ for _ in lines if len(_) > 0]
158 nblines = len(lines)
160 keepc = content
161 try:
162 counts, content = migrating_doxygen_doc(content, fullname, silent)
163 except SyntaxError as e: # pragma: no cover
164 if not silent:
165 raise e
166 content = keepc
167 counts = dict(docrows=0)
169 content = fmod(content, fullname)
170 content = remove_undesired_part_for_documentation(
171 content, fullname, use_sys)
172 fold = os.path.split(to)[0]
173 if not os.path.exists(fold):
174 os.makedirs(fold)
175 if replace_relative_import:
176 content = replace_relative_import_fct(fullname, content)
177 with open(to, "w", encoding="utf8") as g:
178 g.write(content)
180 return os.path.splitext(fullname)[-1], nblines, counts["docrows"]
183def remove_undesired_part_for_documentation(content, filename, use_sys):
184 """
185 Some files contains blocs inserted between the two lines:
187 * ``# -- HELP BEGIN EXCLUDE --``
188 * ``# -- HELP END EXCLUDE --``
190 Those lines will be commented out.
192 @param content file content
193 @param filename for error message
194 @param use_sys string or None, enables, disables a section based on variables added to sys module
195 @return modified file content
197 If the parameter *use_sys* is false, the section of code
198 will be commented out. If true, the section can be enabled.
199 It relies on the following code::
201 import sys
202 if hasattr(sys, "<use_sys>") and sys.<use_sys>:
204 # section to enable or disables
206 The string ``<use_sys>`` will be replaced by the value of
207 parameter *use_sys*.
208 """
209 marker_in = "# -- HELP BEGIN EXCLUDE --"
210 marker_out = "# -- HELP END EXCLUDE --"
212 lines = content.split("\n")
213 res = []
214 inside = False
215 has_sys = False
216 flask_trick = False
217 for line in lines:
218 if line.startswith("import sys"):
219 has_sys = True
220 if line.startswith(marker_in):
221 if inside:
222 raise HelpGenException( # pragma: no cover
223 "issues with undesired blocs in file " + filename + " with: " + marker_in + "|" + marker_out)
224 inside = True
225 if use_sys: # pragma: no cover
226 if not has_sys:
227 res.append("import sys")
228 res.append(
229 "if hasattr(sys, '{0}') and sys.{0}:".format(use_sys))
230 res.append(line)
231 elif line.startswith(marker_out):
232 if use_sys and flask_trick: # pragma: no cover
233 res.append(" pass")
234 if not inside:
235 raise HelpGenException( # pragma: no cover
236 "issues with undesired blocs in file " + filename + " with: " + marker_in + "|" + marker_out)
237 inside = False
238 flask_trick = False
239 res.append(line)
240 else:
241 if inside:
242 if use_sys: # pragma: no cover
243 # specific trick for Flask
244 if line.startswith("@app."):
245 line = "# " + line
246 flask_trick = True
247 res.append(" " + line)
248 else:
249 res.append("### " + line)
250 else:
251 res.append(line)
252 return "\n".join(res)
255def copy_source_files(input, output, fmod=lambda v, filename: v,
256 silent=False, filter=None, remove=True,
257 softfile=lambda f: False,
258 fexclude=lambda f: False,
259 addfilter=None, replace_relative_import=False,
260 copy_add_ext=None, use_sys=None, fLOG=fLOG):
261 """
262 Copies all sources files (input) into a folder (output),
263 apply on each of them a modification.
265 :param input: input folder
266 :param output: output folder (it will be cleaned each time)
267 :param fmod: modifies the content of each file,
268 this function takes a string and returns a string
269 :param silent: if True, do not stop when facing an issue with :epkg:`doxygen` documentation
270 :param filter: if None, process only file related to python code, otherwise,
271 use this filter to select file (regular expression). If this parameter
272 is None or is empty, the default value is something like:
273 ``"(.+[.]py$)|(.+[.]pyd$)|(.+[.]cpp$)|(.+[.]h$)|(.+[.]dll$))"``.
274 :param remove: if True, remove every files in the output folder first
275 :param softfile: softfile is a function (f : filename --> True or False), when it is True,
276 the documentation is lighter (no special members)
277 :param fexclude: function to exclude some files from the help
278 :param addfilter: additional filter, it should look like: ``"(.+[.]pyx$)|(.+[.]pyh$)"``
279 :param replace_relative_import: replace relative import
280 :param copy_add_ext: additional extension file to copy
281 :param use_sys: see :func:`remove_undesired_part_for_documentation
282 <pyquickhelper.helpgen.utils_sphinx_doc.remove_undesired_part_for_documentation>`
283 :param fLOG: logging function
284 :return: list of copied files
285 """
286 if not os.path.exists(output):
287 os.makedirs(output)
289 if remove:
290 remove_folder(output, False, raise_exception=False)
292 def_ext = ['py', 'pyd', 'cpp', 'h', 'dll', 'so', 'yml', 'o', 'def', 'gif',
293 'exe', 'data', 'config', 'css', 'js', 'png', 'map', 'sass',
294 'csv', 'tpl', 'jpg', 'jpeg', 'hpp', 'cc', 'tmpl']
295 deffilter = "|".join("(.+[.]{0}$)".format(_) for _ in def_ext)
296 if copy_add_ext is not None:
297 res = ["(.+[.]%s$)" % e for e in copy_add_ext]
298 deffilter += "|" + "|".join(res)
300 fLOG("[copy_source_files] copy filter '{0}'".format(deffilter))
302 if addfilter is not None and len(addfilter) > 0:
303 if filter is None or len(filter) == 0:
304 filter = "|".join([deffilter, addfilter])
305 else:
306 filter = "|".join([filter, addfilter])
308 if filter is None:
309 actions = synchronize_folder(input, output, filter=deffilter,
310 avoid_copy=True, fLOG=fLOG)
311 else:
312 actions = synchronize_folder(input, output, filter=filter,
313 avoid_copy=True, fLOG=fLOG)
315 if len(actions) == 0:
316 raise FileNotFoundError("empty folder: " + input) # pragma: no cover
318 ractions = []
319 for a, file, dest in actions:
320 if a != ">+":
321 continue
322 if not validate_file_for_help(file.fullname, fexclude):
323 continue
324 if file.name.endswith("setup.py"):
325 continue
326 if "setup.py" in file.name:
327 raise FileNotFoundError( # pragma: no cover
328 "are you sure (setup.py)?, file: " + file.fullname)
330 to = os.path.join(dest, file.name)
331 dd = os.path.split(to)[0]
332 if not os.path.exists(dd):
333 fLOG("[copy_source_files] create ", dd,
334 "softfile={0} fexclude={1}".format(softfile, fexclude))
335 os.makedirs(dd)
336 fLOG("[copy_source_files] copy ", file.fullname, " to ", to)
338 rext, rline, rdocline = _private_process_one_file(
339 file.fullname, to, silent, fmod, replace_relative_import, use_sys)
340 ractions.append((a, file, dest, rext, rline, rdocline))
342 return ractions
345def apply_modification_template(rootm, store_obj, template, fullname, rootrep,
346 softfile, indexes, additional_sys_path, fLOG=noLOG):
347 """
348 See @see fn add_file_rst.
350 @param rootm root of the module
351 @param store_obj keep track of all objects extracted from the module
352 @param template rst template to produce
353 @param fullname full name of the file
354 @param rootrep file name in the documentation contains some folders which are not desired in the documentation
355 @param softfile a function (f : filename --> True or False), when it is True,
356 the documentation is lighter (no special members)
357 @param indexes dictionary with the label and some information (IndexInformation)
358 @param additional_sys_path additional path to include to sys.path before importing a module
359 (will be removed afterwards)
360 @param fLOG logging function
361 @return content of a .rst file
363 .. faqref::
364 :title: Why doesn't the documentation show compiled submodules?
366 The instruction ``.. automodule:: <name>`` only shows objects *obj*
367 which verify ``obj.__module__ == name``. This is always the case
368 for modules written in Python but not necessarily for module
369 compiled from C language. When the module is declared,
370 the following structure contains the module name in second position.
371 This name must not be the submodule shortname but the name
372 the module has is the package. The C file
373 *pyquickhelper/helpgen/compiled.c*
374 implements submodule
375 ``pyquickhelper.helpgen.compiled``, this value must replace
376 ``<fullname>`` in the structure below, not simply *compiled*.
378 ::
380 static struct PyModuleDef moduledef = {
381 PyModuleDef_HEAD_INIT,
382 "<fullname>",
383 "Helper for parallelization with threads with C++.",
384 sizeof(struct module_state),
385 fonctions,
386 NULL,
387 threader_module_traverse,
388 threader_module_clear,
389 NULL
390 };
392 .. warning::
393 This function still needs some improvments
394 for C++ modules on MacOSX.
395 """
396 from pandas import DataFrame
398 keepf = fullname
399 filename = os.path.split(fullname)[-1]
400 filenoext = os.path.splitext(filename)[0]
401 fullname = fullname.strip(".").replace(
402 "\\", "/").replace("/", ".").strip(".")
403 if rootrep[0] in fullname:
404 pos = fullname.index(rootrep[0])
405 fullname = rootrep[1] + fullname[pos + len(rootrep[0]):]
406 fullnamenoext = fullname[:-3] if fullname.endswith(".py") else fullname
407 if fullnamenoext.endswith(".pyd"):
408 fullnamenoext = '.'.join(fullnamenoext.split('.')[:-2])
409 elif fullnamenoext.endswith('-linux-gnu.so'):
410 fullnamenoext = '.'.join(fullnamenoext.split('.')[:-2])
411 pythonname = None
413 not_expected = os.environ.get(
414 "USERNAME", os.environ.get("USER", "````````````"))
415 if not_expected not in ('jenkins', 'vsts', 'runner') and not_expected in fullnamenoext:
416 mes = ("The title is probably wrong (5): {0}\nnoext='{1}'\npython='{2}'\nrootm='{3}'\nrootrep='{4}'"
417 "\nfullname='{5}'\nkeepf='{6}'\nnot_expected='{7}'") # pragma: no cover
418 raise HelpGenException(mes.format( # pragma: no cover
419 fullnamenoext, filenoext, pythonname, rootm, rootrep, fullname, keepf, not_expected))
421 mo, prefix = import_module(
422 rootm, keepf, fLOG, additional_sys_path=additional_sys_path)
423 doc = ""
424 shortdoc = ""
426 additional = {}
427 tspecials = {}
429 if mo is not None:
430 if isinstance(mo, str): # pragma: no cover
431 # it is an error
432 spl = mo.split("\n")
433 mo = "\n".join([" " + _ for _ in spl])
434 mo = "::\n\n" + mo + "\n\n"
435 doc = mo
436 shortdoc = "Error"
437 pythonname = fullnamenoext
438 else:
439 pythonname = mo.__name__
440 if mo.__doc__ is not None:
441 doc = mo.__doc__
442 doc = private_migrating_doxygen_doc(
443 doc.split("\n"), 0, fullname)
444 doct = doc
445 doc = []
447 for d in doct:
448 if len(doc) != 0 or len(d) > 0:
449 doc.append(d)
450 while len(doc) > 0 and len(doc[-1]) == 0:
451 doc.pop()
453 shortdoc = doc[0] if len(doc) > 0 else ""
454 if len(doc) > 1:
455 shortdoc += "..."
457 doc = "\n".join(doc)
458 doc = "module ``" + mo.__name__ + "``\n\n" + doc
459 if ":githublink:" not in doc:
460 doc += "\n\n:githublink:`GitHub|py|*`"
461 else:
462 doc = ""
463 shortdoc = "empty"
465 # it produces the table for the function, classes, and
466 objs = get_module_objects(mo)
468 prefix = ".".join(fullnamenoext.split(".")[:-1])
469 for ob in objs:
471 if ob.type in ["method"] and ob.name.startswith("_"):
472 tspecials[ob.name] = ob
474 ob.add_prefix(prefix)
475 if ob.key in store_obj:
476 if isinstance(store_obj[ob.key], list):
477 store_obj[ob.key].append(ob)
478 else:
479 store_obj[ob.key] = [store_obj[ob.key], ob]
480 else:
481 store_obj[ob.key] = ob
483 for k, v in add_file_rst_template_cor.items():
484 values = [[o.rst_link(None, class_in_bracket=False), o.truncdoc]
485 for o in objs if o.type == k]
486 if len(values) > 0:
487 tbl = DataFrame(
488 columns=[k, "truncated documentation"], data=values)
489 for row in tbl.values:
490 if ":meth:`_" in row[0]:
491 row[0] = row[0].replace(":meth:`_", ":py:meth:`_")
493 if len(tbl) > 0:
494 maxi = max([len(_) for _ in tbl[k]])
495 s = 0 if tbl.iloc[0, 1] is None else len(
496 tbl.iloc[0, 1])
497 t = "" if tbl.iloc[0, 1] is None else tbl.iloc[0, 1]
498 tbl.iloc[0, 1] = t + (" " * (3 * maxi - s))
499 sph = df2rst(tbl)
500 titl = "\n\n" + add_file_rst_template_title[k] + "\n"
501 titl += "+" * len(add_file_rst_template_title[k])
502 titl += "\n\n"
503 additional[v] = titl + sph
504 else:
505 additional[v] = ""
506 else:
507 additional[v] = ""
509 del mo
511 else:
512 doc = "[sphinxerror]-C unable to import."
514 if indexes is None:
515 indexes = {}
516 label = IndexInformation.get_label(indexes, "f-" + filenoext)
517 indexes[label] = IndexInformation(
518 "module", label, filenoext, doc, None, keepf)
519 fLOG("[apply_modification_template] adding into index ", indexes[label])
521 try:
522 with open(keepf, "r") as ft:
523 content = ft.read()
524 except UnicodeDecodeError:
525 try:
526 with open(keepf, "r", encoding="latin-1") as ft:
527 content = ft.read()
528 except UnicodeDecodeError: # pragma: no cover
529 with open(keepf, "r", encoding="utf8") as ft:
530 content = ft.read()
532 plat = "Windows" if "This example only runs on Windows." in content else "any"
534 # dealing with special members (does not work)
535 # text_specials = "".join([" :special-members: %s\n" % k for k in tspecials ])
536 text_specials = ""
538 if fullnamenoext.endswith(".__init__"):
539 fullnamenoext = fullnamenoext[: -len(".__init__")]
540 if filenoext.endswith(".__init__"):
541 filenoext = filenoext[: -len(".__init__")]
543 not_expected = os.environ.get(
544 "USERNAME", os.environ.get("USER", "````````````"))
545 if not_expected not in ('jenkins', 'vsts', 'runner') and not_expected in fullnamenoext:
546 mes = ("The title is probably wrong (3): {0}\nnoext={1}\npython={2}\nrootm={3}\nrootrep={4}"
547 "\nfullname={5}\nkeepf={6}\nnot_expected='{7}'") # pragma: no cover
548 raise HelpGenException(mes.format( # pragma: no cover
549 fullnamenoext, filenoext, pythonname, rootm, rootrep, fullname, keepf, not_expected))
551 ttitle = "module ``{0}``".format(fullnamenoext)
552 rep = {
553 "__FULLNAME_UNDERLINED__": ttitle + "\n" + ("=" * len(ttitle)) + "\n",
554 "__FILENAMENOEXT__": filenoext,
555 "__FULLNAMENOEXT__": pythonname,
556 "__DOCUMENTATION__": doc.split("\n.. ")[0],
557 "__DOCUMENTATIONLINE__":
558 shortdoc.split(".. todoext::", maxsplit=1)[0],
559 "__PLATFORM__": plat,
560 "__ADDEDMEMBERS__": text_specials}
562 for k, v in additional.items():
563 rep[k] = v
565 res = template
566 for a, b in rep.items():
567 res = res.replace(a, b)
569 has_class = any(
570 filter(lambda _: _.startswith("class "), content.split("\n")))
571 if not has_class:
572 spl = res.split("\n")
573 spl = [_ for _ in spl if not _.startswith(".. inheritance-diagram::")]
574 res = "\n".join(spl)
576 if softfile(fullname):
577 res = res.replace(":special-members:", "")
579 return res
582def add_file_rst(rootm, store_obj, actions, template=add_file_rst_template,
583 rootrep=("_doc.sphinxdoc.source.pyquickhelper.", ""),
584 fmod=lambda v, filename: v, softfile=lambda f: False,
585 mapped_function=None, indexes=None,
586 additional_sys_path=None, fLOG=noLOG):
587 """
588 Creates a :epkg:`rst` file for every source file.
590 @param rootm root of the module (for relative import)
591 @param store_obj to keep table of all objects
592 @param actions output from @see fn copy_source_files
593 @param template :epkg:`rst` template to produce
594 @param rootrep file name in the documentation contains some folders
595 which are not desired in the documentation
596 @param fmod applies modification to the instanciated template
597 @param softfile softfile is a function (f : filename --> True or False), when it is True,
598 the documentation is lighter (no special members)
599 @param mapped_function list of 2-tuple (pattern, function). Every file matching the pattern
600 will be copied to the documentation folder, its content will be sent
601 to the function and will produce a file <filename>.rst. Example:
602 ``[ (".*[.]sql$", filecontent_to_rst) ]``
603 The function takes two parameters: full_filename, content_filename. It returns
604 a string (the rst file) or a tuple (rst file, short description).
605 By default (if function is None), the function ``filecontent_to_rst`` will be called
606 except for .rst file for which nothing is done.
607 @param indexes to index some information { dictionary label:IndexInformation (...) },
608 the function populates it
609 @param additional_sys_path additional path to include to sys.path before importing a module
610 (will be removed afterwards)
611 @param fLOG logging function
612 @return list of written files stored in RstFileHelp
613 """
614 if indexes is None:
615 indexes = {}
616 if mapped_function is None:
617 mapped_function = []
619 if additional_sys_path is None:
620 additional_sys_path = []
622 memo = {}
623 app = []
624 for action in actions:
625 _, file, dest = action[:3]
626 if not isinstance(file, str):
627 file = file.name
629 to = os.path.join(dest, file)
630 rst = os.path.splitext(to)[0]
631 rst += ".rst"
632 ext = os.path.splitext(to)[-1]
634 if sys.platform == "win32":
635 cpxx = ".cp%d%d-" % sys.version_info[:2]
636 elif sys.version_info[:2] <= (3, 7):
637 cpxx = ".cpython-%d%dm-" % sys.version_info[:2]
638 else:
639 cpxx = ".cpython-%d%d-" % sys.version_info[:2]
641 if file.endswith(".py") or (
642 cpxx in file and (
643 file.endswith(".pyd") or file.endswith("linux-gnu.so")
644 )):
645 if os.stat(to).st_size > 0:
646 content = apply_modification_template(
647 rootm, store_obj, template, to, rootrep, softfile, indexes,
648 additional_sys_path=additional_sys_path, fLOG=fLOG)
649 content = fmod(content, file)
651 # tweaks for example and studies
652 zzz = to.replace("\\", "/")
653 name = os.path.split(file)[-1]
654 noex = os.path.splitext(name)[0]
656 # todo: specific case: should be removed and added back in a
657 # proper way
658 if "examples/" in zzz or "studies/" in zzz:
659 content += "\n.. _%s_literal:\n\nCode\n----\n\n.. literalinclude:: %s\n\n" % (
660 noex, name)
662 with open(rst, "w", encoding="utf8") as g:
663 g.write(content)
664 app.append(RstFileHelp(to, rst, ""))
666 for vv in indexes.values():
667 if vv.fullname == to:
668 vv.set_rst_file(rst)
669 break
671 else:
672 for pat, func in mapped_function:
673 if func is None and ext == ".rst":
674 continue
675 if pat not in memo:
676 memo[pat] = re.compile(pat)
677 exp = memo[pat]
678 if exp.search(file):
679 if isinstance(func, bool) and not func:
680 # we copy but we do nothing with it
681 pass
682 else:
683 with open(to, "r", encoding="utf8") as g:
684 content = g.read()
685 if func is None:
686 func = filecontent_to_rst
687 content = func(to, content)
689 if isinstance(content, tuple) and len(content) == 2:
690 content, doc = content
691 else:
692 doc = ""
694 with open(rst, "w", encoding="utf8") as g:
695 g.write(content)
696 app.append(RstFileHelp(to, rst, ""))
698 filenoext, ext = os.path.splitext(
699 os.path.split(to)[-1])
700 ext = ext.strip(".")
701 label = IndexInformation.get_label(
702 indexes, "ext-" + filenoext)
703 indexes[label] = IndexInformation(
704 "ext-" + ext, label, filenoext, doc, rst, to)
705 fLOG("[add_file_rst] add ext into index ", indexes[label])
707 return app
710def produces_indexes(store_obj, indexes, fexclude_index, titles=None,
711 correspondances=None, fLOG=fLOG):
712 """
713 Produces a file for each category of object found in the module.
715 @param store_obj list of collected object, it is a dictionary
716 key : ModuleMemberDoc or key : [ list of ModuleMemberDoc ]
717 @param indexes list of things to index, dictionary { label : IndexInformation }
718 @param fexclude_index to exclude files from the indices
719 @param titles each type is mapped to a title to add to the :epkg:`rst` file
720 @param correspondances each type is mapped to a label to add to the :epkg:`rst` file
721 @param fLOG logging function
722 @return dictionary: { type : rst content of the index }
724 Default values if *titles* of *correspondances* is None:
726 ::
728 title = {"method": "Methods",
729 "staticmethod": "Static Methods",
730 "property": "Properties",
731 "function": "Functions",
732 "class": "Classes",
733 "module": "Modules"}
735 correspondances = {"method": "l-methods",
736 "function": "l-functions",
737 "staticmethod": "l-staticmethods",
738 "property": "l-properties",
739 "class": "l-classes",
740 "module": "l-modules"}
741 """
742 from pandas import DataFrame
744 if titles is None:
745 titles = {"method": "Methods",
746 "staticmethod": "Static Methods",
747 "property": "Properties",
748 "function": "Functions",
749 "class": "Classes",
750 "module": "Modules"}
752 if correspondances is None:
753 correspondances = {"method": "l-methods",
754 "function": "l-functions",
755 "staticmethod": "l-staticmethods",
756 "property": "l-properties",
757 "class": "l-classes",
758 "module": "l-modules"}
760 # we process store_obj
761 types = {}
762 for k, v in store_obj.items():
763 if not isinstance(v, list):
764 v = [v]
765 for _ in v:
766 if fexclude_index(_):
767 continue
768 types[_.type] = types.get(_.type, 0) + 1
770 fLOG("[produces_indexes] store_obj: extraction of types: {}".format(types))
771 res = {}
772 for k in types:
773 fLOG("[produces_indexes] type: [{}] - rst".format(k))
774 values = []
775 for t, so in store_obj.items():
776 if not isinstance(so, list):
777 so = [so]
779 for o in so:
780 if fexclude_index(o):
781 continue
782 if o.type != k:
783 continue
784 oclname = o.classname.__name__ if o.classname is not None else ""
785 rlink = o.rst_link(class_in_bracket=False)
786 fLOG("[produces_indexes] + '{}': {}".format(o.name, rlink))
787 values.append([o.name, rlink, oclname, o.truncdoc])
789 values.sort()
790 for row in values:
791 if ":meth:`_" in row[1]:
792 row[1] = row[1].replace(":meth:`_", ":py:meth:`_")
794 # we filter private method or functions
795 values = [
796 row for row in values if ":meth:`__" in row or ":meth:`_" not in row]
797 values = [
798 row for row in values if ":func:`__" in row or ":func:`_" not in row]
800 columns = ["_", k, "class parent", "truncated documentation"]
801 tbl = DataFrame(columns=columns, data=values)
802 if len(tbl.columns) >= 2:
803 tbl = tbl.iloc[:, 1:].copy()
805 if len(tbl) > 0:
806 maxi = max([len(_) for _ in tbl[k]])
807 s = 0 if tbl.iloc[0, 1] is None else len(tbl.iloc[0, 1])
808 t = "" if tbl.iloc[0, 1] is None else tbl.iloc[0, 1]
809 tbl.iloc[0, 1] = t + (" " * (3 * maxi - s))
810 sph = df2rst(tbl)
811 res[k] = sph
812 fLOG("[produces_indexes] type: [{}] - shape: {}".format(k, tbl.shape))
814 # we process indexes
816 fLOG("[produces_indexes] indexes")
817 types = {}
818 for k, v in indexes.items():
819 if fexclude_index(v):
820 continue
821 types[v.type] = types.get(v.type, 0) + 1
823 fLOG("[produces_indexes] extraction of types: {}".format(types))
825 for k in types:
826 if k in res:
827 raise HelpGenException( # pragma: no cover
828 "you should not index anything related to classes, functions or method (conflict: %s)" % k)
829 values = []
830 for t, o in indexes.items():
831 if fexclude_index(o):
832 continue
833 if o.type != k:
834 continue
835 values.append([o.name,
836 o.rst_link(),
837 o.truncdoc])
838 values.sort()
840 tbl = DataFrame(
841 columns=["_", k, "truncated documentation"], data=values)
842 if len(tbl.columns) >= 2:
843 tbl = tbl[tbl.columns[1:]]
845 if len(tbl) > 0:
846 maxi = max([len(_) for _ in tbl[k]])
847 tbl.iloc[0, 1] = tbl.iloc[0, 1] + \
848 (" " * (3 * maxi - len(tbl.iloc[0, 1])))
849 sph = df2rst(tbl)
850 res[k] = sph
852 # end
854 for k in res: # pylint: disable=C0206
855 fLOG("[produces_indexes] index name '{}'".format(k))
856 label = correspondances.get(k, k)
857 title = titles.get(k, k)
858 under = "=" * len(title)
860 content = "\n".join([".. contents::", " :local:",
861 " :depth: 1", "", "", "Summary", "+++++++"])
863 not_expected = os.environ.get(
864 "USERNAME", os.environ.get("USER", "````````````"))
865 if not_expected != "jenkins" and not_expected in title:
866 raise HelpGenException( # pragma: no cover
867 "The title is probably wrong (2), found '{0}' in '{1}'".format(not_expected, title))
869 res[k] = "\n.. _%s:\n\n%s\n%s\n\n%s\n\n%s" % (
870 label, title, under, content, res[k])
872 return res
875def filecontent_to_rst(filename, content):
876 """
877 Produces a *.rst* file which contains the file.
878 It adds a title and a label based on the
879 filename (no folder included).
881 @param filename filename
882 @param content content
883 @return new content
884 """
885 file = os.path.split(filename)[-1]
886 full = file + "\n" + ("=" * len(file)) + "\n"
888 not_expected = os.environ.get(
889 "USERNAME", os.environ.get("USER", "````````````"))
890 if not_expected != "jenkins" and not_expected in file:
891 raise HelpGenException( # pragma: no cover
892 "The title is probably wrong (1): '{0}' found in '{1}'".format(not_expected, file))
894 rows = ["", ".. _f-%s:" % file, "", "", full, "",
895 # "fullpath: ``%s``" % filename,
896 "", ""]
897 if ".. RSTFORMAT." in content:
898 rows.append(".. include:: %s " % file)
899 else:
900 rows.append(".. literalinclude:: %s " % file)
901 rows.append("")
903 nospl = content.replace("\n", "_!_!:!_")
905 reg = re.compile("(.. beginshortsummary[.](.*?).. endshortsummary[.])")
906 cont = reg.search(nospl)
907 if cont:
908 g = cont.groups()[1].replace("_!_!:!_", "\n")
909 return "\n".join(rows), g.strip("\n\r ")
911 if "@brief" in content:
912 spl = content.split("\n")
913 begin = None
914 end = None
915 for i, r in enumerate(spl):
916 if "@brief" in r:
917 begin = i
918 if end is None and begin is not None and len(
919 r.strip(" \n\r\t")) == 0:
920 end = i
922 if begin is not None and end is not None:
923 summary = "\n".join(spl[begin:end]).replace(
924 "@brief", "").strip("\n\t\r ")
925 else:
926 summary = "no documentation" # pragma: no cover
928 # looking for C++/java/C# comments
929 spl = content.split("\n")
930 begin = None
931 end = None
932 for i, r in enumerate(spl):
933 if "/**" in spl[i]:
934 begin = i
935 if end is None and begin is not None and "*/" in spl[i]:
936 end = i
938 content = "\n".join(rows)
939 if begin is not None and end is not None: # pragma: no cover
940 filerows = private_migrating_doxygen_doc(
941 spl[begin + 1:end - 1], 1, filename)
942 rstr = "\n".join(filerows)
943 rstr = re.sub(
944 ":param +([a-zA-Z_][[a-zA-Z_0-9]*) *:", r"* **\1**:", rstr)
945 content = content.replace(
946 ".. literalinclude::", "\n%s\n\n.. literalinclude::" % rstr)
948 return content, summary
950 return "\n".join(rows), "no documentation"
953def prepare_file_for_sphinx_help_generation(store_obj, input, output,
954 subfolders, fmod_copy=lambda v, filename: v,
955 template=add_file_rst_template,
956 rootrep=(
957 "_doc.sphinxdoc.source.project_name.", ""),
958 fmod_res=lambda v, filename: v, silent=False,
959 optional_dirs=None, softfile=lambda f: False,
960 fexclude=lambda f: False, mapped_function=None,
961 fexclude_index=lambda f: False, issues=None,
962 additional_sys_path=None, replace_relative_import=False,
963 module_name=None, copy_add_ext=None, use_sys=None,
964 auto_rst_generation=True, fLOG=fLOG):
965 """
966 Prepares all files for :epkg:`Sphinx` generation.
968 @param store_obj to keep track of all objects, it should be a dictionary
969 @param input input folder
970 @param output output folder (it will be cleaned each time)
971 @param subfolders list of subfolders to copy from input to output, two cases:
972 * a string input/<sub> --> output/<sub>
973 * a tuple input/<sub[0]> --> output/<sub[1]>
974 @param fmod_copy modifies the content of each file,
975 this function takes a string and the filename and returns a string
976 ``f(content, filename) --> string``
977 @param template rst template to produce
978 @param rootrep file name in the documentation contains some folders which are not desired in the documentation
979 @param fmod_res applies modification to the instanciated template
980 @param silent if True, do not stop when facing an issue with doxygen migration
981 @param optional_dirs list of tuple with a list of folders (source, copy, filter) to
982 copy for the documentation, example:
983 ``( <folder_help>, "coverage", ".*" )``
984 @param softfile softfile is a function (f : filename --> True or False), when it is True,
985 the documentation is lighter (no special members)
986 @param fexclude function to exclude some files from the help
987 @param fexclude_index function to exclude some files from the indices
989 @param mapped_function list of 2-tuple (pattern, function). Every file matching the pattern
990 will be copied to the documentation folder, its content will be sent
991 to the function and will produce a file <filename>.rst. Example:
992 ``[ (".*[.]sql$", filecontent_to_rst) ]``
993 The function takes two parameters: full_filename, content_filename. It returns
994 a string (the rst file) or a tuple (rst file, short description).
995 By default (if function is None), the function ``filecontent_to_rst`` will be called.
997 @param issues if not None (a list), the function will store some issues here.
999 @param additional_sys_path additional paths to includes to sys.path when import a module (will be removed afterwards)
1000 @param replace_relative_import replace relative import
1001 @param module_name module name (cannot be None)
1002 @param copy_add_ext additional file extension to copy
1003 @param use_sys @see fn remove_undesired_part_for_documentation
1004 @param auto_rst_generation add a file *.rst* for each source file
1005 @param fLOG logging function
1007 @return list of written files stored in @see cl RstFileHelp
1009 Example:
1011 ::
1013 prepare_file_for_sphinx_help_generation (
1014 {},
1015 ".",
1016 "_doc/sphinxdoc/source/",
1017 subfolders = [
1018 ("src/" + project_var_name,
1019 project_var_name),
1020 ],
1021 silent = True,
1022 rootrep = ("_doc.sphinxdoc.source.%s." %
1023 (project_var_name,), ""),
1024 optional_dirs = optional_dirs,
1025 mapped_function = [ (".*[.]tohelp$", None) ] )
1027 It produces a file with the number of lines and files per extension.
1028 """
1029 if optional_dirs is None:
1030 optional_dirs = []
1032 if mapped_function is None:
1033 mapped_function = []
1035 if additional_sys_path is None:
1036 additional_sys_path = []
1038 if module_name is None:
1039 raise ValueError( # pragma: no cover
1040 "module_name cannot be None")
1042 fLOG("[prepare_file_for_sphinx_help_generation] output='{}'".format(output))
1043 rootm = os.path.abspath(output)
1044 fLOG("[prepare_file_for_sphinx_help_generation] input='{}'".format(input))
1046 actions = []
1047 rsts = []
1048 indexes = {}
1050 for sub in subfolders:
1051 if isinstance(sub, str):
1052 src = (input + "/" + sub).replace("//", "/")
1053 dst = (output + "/" + sub).replace("//", "/")
1054 else:
1055 src = (input + "/" + sub[0]).replace("//", "/")
1056 dst = (output + "/" + sub[1]).replace("//", "/")
1057 if os.path.split(src)[-1][0] == '_':
1058 raise RuntimeError(
1059 "Subfolder %r cannot start with '_'." % src)
1060 if os.path.split(dst)[-1][0] == '_':
1061 raise RuntimeError( # pragma: no cover
1062 "Destination %r cannot start with '_'." % dst)
1064 if os.path.isfile(src):
1065 fLOG(" [p] ", src)
1066 _private_process_one_file(
1067 src, dst, silent, fmod_copy, replace_relative_import, use_sys)
1069 temp = os.path.split(dst)
1070 actions_t = [(">", temp[1], temp[0], 0, 0)]
1071 if auto_rst_generation:
1072 rstadd = add_file_rst(rootm, store_obj, actions_t,
1073 template, rootrep, fmod_res,
1074 softfile=softfile,
1075 mapped_function=mapped_function,
1076 indexes=indexes,
1077 additional_sys_path=additional_sys_path,
1078 fLOG=fLOG)
1079 rsts += rstadd
1080 else:
1081 fLOG("[prepare_file_for_sphinx_help_generation] processing '{}'".format(src))
1083 actions_t = copy_source_files(src, dst, fmod_copy, silent=silent,
1084 softfile=softfile, fexclude=fexclude,
1085 addfilter="|".join(
1086 ['(%s)' % _[0] for _ in mapped_function]),
1087 replace_relative_import=replace_relative_import,
1088 copy_add_ext=copy_add_ext,
1089 use_sys=use_sys, fLOG=fLOG)
1091 # without those two lines, importing the module might crash later
1092 importlib.invalidate_caches()
1093 importlib.util.find_spec(module_name)
1095 if auto_rst_generation:
1096 rsts += add_file_rst(rootm, store_obj, actions_t, template,
1097 rootrep, fmod_res, softfile=softfile,
1098 mapped_function=mapped_function,
1099 indexes=indexes,
1100 additional_sys_path=additional_sys_path,
1101 fLOG=fLOG)
1103 actions += actions_t
1105 # everything is cleaned from the build folder, so, it is no use
1106 for tu in optional_dirs:
1107 if len(tu) == 2:
1108 fold, dest, filt = tu + (".*", )
1109 else:
1110 fold, dest, filt = tu
1111 if filt is None:
1112 filt = ".*"
1113 if not os.path.exists(dest):
1114 fLOG("creating folder (sphinx) ", dest)
1115 os.makedirs(dest)
1117 copy_source_files(fold, dest, silent=silent, filter=filt,
1118 softfile=softfile, fexclude=fexclude,
1119 addfilter="|".join(['(%s)' % _[0]
1120 for _ in mapped_function]),
1121 replace_relative_import=replace_relative_import,
1122 copy_add_ext=copy_add_ext, fLOG=fLOG)
1124 # processing all store_obj to compute some indices
1125 fLOG("[prepare_file_for_sphinx_help_generation] processing all store_obj to compute some indices")
1126 fLOG("[prepare_file_for_sphinx_help_generation] extracted ",
1127 len(store_obj), " objects")
1128 res = produces_indexes(store_obj, indexes, fexclude_index, fLOG=fLOG)
1130 fLOG("[prepare_file_for_sphinx_help_generation] generating ",
1131 len(res), " indexes for ", ", ".join(list(res.keys())))
1132 allfiles = []
1133 for k, vv in res.items():
1134 out = os.path.join(output, "index_" + k + ".rst")
1135 allfiles.append("index_" + k)
1136 fLOG(" generates index", out)
1137 if k == "module":
1138 toc = ["\n\n.. toctree::"]
1139 toc.append(" :maxdepth: 1\n")
1140 for _ in rsts:
1141 if _.file is not None and len(_.file) > 0:
1142 na = os.path.splitext(_.rst)[0].replace(
1143 "\\", "/").split("/")
1144 if "source" in na:
1145 na = na[na.index("source") + 1:]
1146 na = "/".join(na)
1147 toc.append(" " + na)
1148 vv += "\n".join(toc)
1149 with open(out, "w", encoding="utf8") as fh:
1150 fh.write(vv)
1151 rsts.append(RstFileHelp(None, out, None))
1153 # generates a table with the number of lines per extension
1154 rows = []
1155 for act in actions:
1156 if "__init__.py" not in act[1].get_fullname() or act[-1] > 0:
1157 v = 1
1158 rows.append(act[-3:] + (v,))
1159 name = os.path.split(act[1].get_fullname())[-1]
1160 if name.startswith("auto_"):
1161 rows.append(("auto_*" + act[-3], act[-2], act[-1], v))
1162 elif "__init__.py" in name:
1163 rows.append(("__init__.py", act[-2], act[-1], v))
1164 elif "__init__.py" in act[1].get_fullname():
1165 v = 1
1166 rows.append(("empty __init__.py", act[-2], act[-1], v))
1168 # use DataFrame to produce a RST table
1169 from pandas import DataFrame
1170 df = DataFrame(
1171 data=rows, columns=["extension/kind", "nb lines", "nb doc lines", "nb files"])
1172 try:
1173 # for pandas >= 0.17
1174 df = df.groupby(
1175 "extension/kind", as_index=False).sum().sort_values("extension/kind")
1176 except AttributeError: # pragma: no cover
1177 # for pandas < 0.17
1178 df = df.groupby(
1179 "extension/kind", as_index=False).sum().sort("extension/kind")
1181 # reports
1182 fLOG("[prepare_file_for_sphinx_help_generation] writing ", "all_report.rst")
1183 all_report = os.path.join(output, "all_report.rst")
1184 with open(all_report, "w") as falli:
1185 falli.write("\n:orphan:\n\n")
1186 falli.write(".. _l-statcode:\n")
1187 falli.write("\n")
1188 falli.write("Statistics on code\n")
1189 falli.write("==================\n")
1190 falli.write("\n\n")
1191 sph = df2rst(df, list_table=True)
1192 falli.write(sph)
1193 falli.write("\n")
1194 rsts.append(RstFileHelp(None, all_report, None))
1196 # all indexes
1197 fLOG("[prepare_file_for_sphinx_help_generation] writing ", "all_indexes.rst")
1198 all_index = os.path.join(output, "all_indexes.rst")
1199 with open(all_index, "w") as falli:
1200 falli.write("\n:orphan:\n\n")
1201 falli.write("\n")
1202 falli.write("All indexes\n")
1203 falli.write("===========\n")
1204 falli.write("\n\n")
1205 falli.write(".. toctree::\n")
1206 falli.write(" :maxdepth: 2\n")
1207 falli.write("\n")
1208 for k in sorted(allfiles):
1209 falli.write(" %s\n" % k)
1210 falli.write("\n")
1211 rsts.append(RstFileHelp(None, all_index, None))
1213 # last function to process images
1214 fLOG("looking for images", output)
1216 images = os.path.join(output, "images")
1217 fLOG("+looking for images into ", images, " for folder ", output)
1218 if os.path.exists(images):
1219 process_copy_images(output, images)
1221 # fixes indexed objects with incomplete names
1222 # :class:`name` --> :class:`name <...>`
1223 fLOG("+looking for incomplete references", output)
1224 fix_incomplete_references(output, store_obj, issues=issues, fLOG=fLOG)
1225 # for t,so in store_obj.items() :
1227 # look for FAQ and example
1228 fLOG("[prepare_file_for_sphinx_help_generation] FAQ + examples")
1229 app = []
1230 for tag, title in [("FAQ", "FAQ"),
1231 ("example", "Examples"),
1232 ("NB", "Magic commands"), ]:
1233 onefiles = process_look_for_tag(tag, title, rsts)
1234 for page, onefile in onefiles:
1235 saveas = os.path.join(output, "all_%s%s.rst" %
1236 (tag,
1237 page.replace(":", "").replace("/", "").replace(" ", "")))
1238 with open(saveas, "w", encoding="utf8") as fh:
1239 fh.write(onefile)
1240 app.append(RstFileHelp(saveas, onefile, ""))
1241 rsts += app
1243 fLOG("[prepare_file_for_sphinx_help_generation] END", output)
1244 return actions, rsts
1247def process_copy_images(folder_source, folder_images):
1248 """
1249 Looks into every file .rst or .py for images (.. image:: imagename),
1250 if this image was found in directory folder_images, then the image is copied
1251 closes to the file.
1253 @param folder_source folder where to look for sources
1254 @param folder_images folder where to look for images
1255 @return list of copied images
1256 """
1257 _, files = explore_folder(folder_source, "[.]((rst)|(py))$")
1258 reg = re.compile(".. image::(.*)")
1259 cop = []
1260 for fn in files:
1261 try:
1262 with open(fn, "r", encoding="utf8") as f:
1263 content = f.read()
1264 except Exception as e: # pragma: no cover
1265 try:
1266 with open(fn, "r") as f:
1267 content = f.read()
1268 except Exception:
1269 raise Exception("Issue with file '{0}'".format(fn)) from e
1271 lines = content.split("\n")
1272 for line in lines:
1273 img = reg.search(line)
1274 if img:
1275 name = img.groups()[0].strip()
1276 fin = os.path.split(name)[-1]
1277 path = os.path.join(folder_images, fin)
1278 if os.path.exists(path):
1279 dest = os.path.join(os.path.split(fn)[0], fin)
1280 shutil.copy(path, dest)
1281 fLOG("+copy img ", fin, " to ", dest)
1282 cop.append(dest)
1283 else:
1284 fLOG("-unable to find image ", name)
1285 return cop
1288def fix_incomplete_references(folder_source, store_obj, issues=None, fLOG=fLOG):
1289 """
1290 Looks into every file .rst or .py for incomplete reference. Example::
1292 :class:`name` --> :class:`name <...>`.
1295 @param folder_source folder where to look for sources
1296 @param store_obj container for indexed objects
1297 @param issues if not None (a list), it will add issues (function, message)
1298 @param fLOG logging function
1299 @return list of fixed references
1300 """
1301 cor = {"func": ["function"],
1302 "meth": ["method", "property", "staticmethod"]
1303 }
1305 _, files = explore_folder(folder_source, "[.](py)$")
1306 reg = re.compile(
1307 "(:(py:)?((class)|(meth)|(func)):`([a-zA-Z_][a-zA-Z0-9_]*?)`)")
1308 cop = []
1309 for fn in files:
1310 try:
1311 with open(fn, "r", encoding="utf8") as f:
1312 content = f.read()
1313 encoding = "utf8"
1314 except Exception: # pragma: no cover
1315 with open(fn, "r") as f:
1316 content = f.read()
1317 encoding = None
1319 mainname = os.path.splitext(os.path.split(fn)[-1])[0]
1321 modif = False
1322 lines = content.split("\n")
1323 rline = []
1324 for line in lines:
1325 ref = reg.search(line)
1326 if ref:
1327 all = ref.groups()[0]
1328 # pre = ref.groups()[1]
1329 typ = ref.groups()[2]
1330 nam = ref.groups()[-1]
1332 key = None
1333 obj = None
1334 for cand in cor.get(typ, [typ]):
1335 k = "%s;%s" % (cand, nam)
1336 if k in store_obj:
1337 if isinstance(store_obj[k], list):
1338 se = [
1339 _s for _s in store_obj[k] if mainname in _s.rst_link()]
1340 if len(se) == 1:
1341 obj = se[0]
1342 break
1343 else:
1344 key = k
1345 obj = store_obj[k]
1346 break
1348 if key in store_obj:
1349 modif = True
1350 lnk = obj.rst_link(class_in_bracket=False)
1351 fLOG(" i,ref, found ", all, " --> ", lnk)
1352 line = line.replace(all, lnk)
1353 else:
1354 fLOG(
1355 " w,unable to replace key ", key, ": ", all, "in file", fn)
1356 if issues is not None:
1357 issues.append(("fix_incomplete_references",
1358 "Unable to replace key '%s', link '%s' in file "
1359 "'%s'." % (key, all, fn)))
1361 rline.append(line)
1363 if modif:
1364 if encoding == "utf8":
1365 with open(fn, "w", encoding="utf8") as f:
1366 f.write("\n".join(rline))
1367 else:
1368 with open(fn, "w") as f:
1369 f.write("\n".join(rline))
1370 return cop
1373def migrating_doxygen_doc(content, filename, silent=False, log=False, debug=False):
1374 """
1375 Migrates the doxygen documentation to rst format.
1377 @param content file content
1378 @param filename filename (to display useful error messages)
1379 @param silent if silent, do not raise an exception
1380 @param log if True, write some information in the logs (not only exceptions)
1381 @param debug display more information on the output if True
1382 @return statistics, new content file
1384 Function ``private_migrating_doxygen_doc`` enumerates the list of conversion
1385 which will be done.
1386 """
1387 if log:
1388 fLOG("migrating_doxygen_doc: ", filename)
1390 rows = []
1391 counts = {"docrows": 0}
1393 def print_in_rows(v, file=None):
1394 rows.append(v)
1396 def local_private_migrating_doxygen_doc(r, index_first_line, filename):
1397 counts["docrows"] += len(r)
1398 return _private_migrating_doxygen_doc(r, index_first_line,
1399 filename, debug=debug, silent=silent)
1401 process_string(content, print_in_rows, local_private_migrating_doxygen_doc,
1402 filename, 0, debug=debug)
1403 return counts, "\n".join(rows)
1405# -- HELP BEGIN EXCLUDE --
1408def private_migrating_doxygen_doc(rows, index_first_line, filename,
1409 debug=False, silent=False):
1410 """
1411 Processes a block help (from doxygen to rst).
1413 @param rows list of text lines
1414 @param index_first_line index of the first line (to display useful message error)
1415 @param filename filename (to display useful message error)
1416 @param silent if True, do not display anything
1417 @param debug display more information if True
1418 @return another list of text lines
1420 @warning This function uses regular expression to process the documentation,
1421 it does not import the module (as Sphinx does). It might misunderstand some code.
1423 @todo Try to import the module and if it possible, uses that information to help
1424 the parsing.
1426 The following line displays error message you can click on using SciTe
1428 ::
1430 raise SyntaxError(" File \"%s\", line %d, in ???\n unable to process: %s " %(
1431 filename, index_first_line+i+1, row))
1433 __sphinx__skip__
1435 The previous string tells the function to stop processing the help.
1437 Doxygen conversions::
1439 @param <param_name> description
1440 :param <param_name>: description
1442 @var <param_name> produces a table with the attributes
1444 @return description
1445 :return: description
1447 @rtype description
1448 :rtype: description
1450 @code
1451 code:: + indentation
1453 @endcode
1454 nothing
1456 @file
1457 nothing
1459 @brief
1460 nothing
1462 @ingroup ...
1463 nothing
1465 @defgroup ....
1466 nothing
1468 @image html ...
1470 @see,@ref label forbidden
1471 should be <op> <fn> <label>, example: @ref cl label
1472 <op> must be in [fn, cl, at, me, te, md]
1474 :class:`label`
1475 :func:`label`
1476 :attr:`label`
1477 :meth:`label`
1478 :mod:`label`
1480 @warning description (until next empty line)
1481 .. warning::
1482 description
1484 @todo
1485 .. todo:: a todo box
1487 ------------- not done yet
1489 @img image name
1490 .. image:: test.png
1491 :width: 200pt
1493 .. raw:: html
1494 html indente
1496 """
1497 return _private_migrating_doxygen_doc(rows, index_first_line, filename,
1498 debug=debug, silent=silent)
1500# -- HELP END EXCLUDE --
1503def _private_migrating_doxygen_doc(rows, index_first_line, filename,
1504 debug=False, silent=False):
1505 if debug: # pragma: no cover
1506 fLOG("------------------ P0")
1507 fLOG("\n".join(rows))
1508 fLOG("------------------ P")
1510 debugrows = rows
1511 rows = [_.replace("\t", " ") for _ in rows]
1512 pars = re.compile("([@]param( +)([a-zA-Z0-9_]+)) ")
1513 refe = re.compile(
1514 "([@]((see)|(ref)) +((fn)|(cl)|(at)|(me)|(te)|(md)) +([a-zA-Z0-9_]+))($|[^a-zA-Z0-9_])")
1515 exce = re.compile("([@]exception( +)([a-zA-Z0-9_]+)) ")
1516 exem = re.compile("([@]example[(](.*?___)?(.*?)[)])")
1517 faq_ = re.compile("([@]FAQ[(](.*?___)?(.*?)[)])")
1518 nb_ = re.compile("([@]NB[(](.*?___)?(.*?)[)])")
1520 # min indent
1521 if len(rows) > 1:
1522 space_rows = [(r.lstrip(), r) for r in rows[1:] if len(r.strip()) > 0]
1523 else:
1524 space_rows = []
1525 if len(space_rows) > 0:
1526 min_indent = min(len(r[1]) - len(r[0]) for r in space_rows)
1527 else:
1528 min_indent = 0
1530 # We fix the first rows which might be different from the others.
1531 if len(rows) > 1:
1532 r = rows[0]
1533 r = (r.lstrip(), r)
1534 delta = len(r[1]) - len(r[0])
1535 if delta != min_indent:
1536 rows = rows.copy()
1537 rows[0] = " " * min_indent + rows[0].lstrip()
1539 # processing doxygen documentation
1540 indent = False
1541 openi = False
1542 beginends = {}
1544 typstr = str
1546 whole = "\n".join(rows)
1547 if "@var" in whole:
1548 whole = process_var_tag(whole, True)
1549 rows = whole.split("\n")
1551 for i in range(len(rows)):
1552 row = rows[i]
1554 if debug:
1555 fLOG( # pragma: no cover
1556 "-- indent=%s openi=%s row=%s" % (indent, openi, row))
1558 if "__sphinx__skip__" in row:
1559 if not silent:
1560 fLOG(" File \"%s\", line %s, skipping" %
1561 (filename, index_first_line + i + 1))
1562 break
1564 strow = row.strip(" ")
1566 if "@endFAQ" in strow or "@endexample" in strow or "@endNB" in strow:
1567 if "@endFAQ" in strow:
1568 beginends["FAQ"] = beginends.get("FAQ", 0) - 1
1569 sp = " " * row.index("@endFAQ")
1570 rows[i] = "\n" + sp + ".. endFAQ.\n"
1571 if "@endexample" in strow:
1572 beginends["example"] = beginends.get("example", 0) - 1
1573 sp = " " * row.index("@endexample")
1574 rows[i] = "\n" + sp + ".. endexample.\n"
1575 if "@endNB" in strow: # pragma: no cover
1576 beginends["NB"] = beginends.get("NB", 0) - 1
1577 sp = " " * row.index("@endNB")
1578 rows[i] = "\n" + sp + ".. endNB.\n"
1579 continue
1581 if indent:
1582 if (not openi and len(strow) == 0) or "@endcode" in strow:
1583 indent = False
1584 rows[i] = ""
1585 openi = False
1586 if "@endcode" in strow:
1587 beginends["code"] = beginends.get("code", 0) - 1
1588 else:
1589 rows[i] = " " + rows[i]
1591 else:
1593 if strow.startswith("@warning"):
1594 pos = rows[i].find("@warning")
1595 sp = " " * pos
1596 rows[i] = rows[i].replace("@warning", "\n%s.. warning:: " % sp)
1597 indent = True
1599 elif strow.startswith("@todo"):
1600 pos = rows[i].find("@todo")
1601 sp = " " * pos
1602 rows[i] = rows[i].replace("@todo", "\n%s.. todo:: " % sp)
1603 indent = True
1605 elif strow.startswith("@ingroup"):
1606 rows[i] = ""
1608 elif strow.startswith("@defgroup"):
1609 rows[i] = ""
1611 elif strow.startswith("@image"):
1612 pos = rows[i].find("@image")
1613 sp = " " * pos
1614 spl = strow.split()
1615 img = spl[-1]
1616 if img.startswith("http://"):
1617 rows[i] = "\n%s.. fancybox:: " % sp + img + "\n\n"
1618 else:
1620 if img.startswith("images") or img.startswith("~"):
1621 # we assume it is a relative path to the source
1622 img = img.strip("~")
1623 spl_path = filename.replace("\\", "/").split("/")
1624 pos = spl_path.index("src")
1625 dots = [".."] * (len(spl_path) - pos - 2)
1626 ref = "/".join(dots) + "/"
1627 else:
1628 ref = ""
1630 sp = " " * row.index("@image")
1631 rows[i] = "\n%s.. image:: %s%s\n%s :align: center\n" % (
1632 sp, ref, img, sp)
1634 elif strow.startswith("@code"):
1635 pos = rows[i].find("@code")
1636 sp = " " * pos
1637 prev = i - 1
1638 while prev > 0 and len(rows[prev].strip(" \n\r\t")) == 0:
1639 prev -= 1
1640 rows[i] = ""
1641 if rows[prev].strip("\n").endswith("."):
1642 rows[prev] += "\n\n%s::\n" % sp
1643 else:
1644 rows[prev] += (":" if rows[prev].endswith(":") else "::")
1645 indent = True
1646 openi = True
1647 beginends["code"] = beginends.get("code", 0) + 1
1649 # basic tags
1650 row = rows[i]
1652 # tag param
1653 look = pars.search(row)
1654 lexxce = exce.search(row)
1655 example = exem.search(row)
1656 faq = faq_.search(row)
1657 nbreg = nb_.search(row)
1659 if look:
1660 rep = look.groups()[0]
1661 sp = look.groups()[1]
1662 name = look.groups()[2]
1663 to = ":param%s%s:" % (sp, name)
1664 rows[i] = row.replace(rep, to)
1666 # it requires an empty line before if the previous line does
1667 # not start by :
1668 if i > 0 and not rows[
1669 i - 1].strip().startswith(":") and len(rows[i - 1].strip()) > 0:
1670 rows[i] = "\n" + rows[i]
1672 elif lexxce:
1673 rep = lexxce.groups()[0]
1674 sp = lexxce.groups()[1]
1675 name = lexxce.groups()[2]
1676 to = ":raises%s%s:" % (sp, name)
1677 rows[i] = row.replace(rep, to)
1679 # it requires an empty line before if the previous line does
1680 # not start by :
1681 if i > 0 and not rows[
1682 i - 1].strip().startswith(":") and len(rows[i - 1].strip()) > 0:
1683 rows[i] = "\n" + rows[i]
1685 elif example:
1686 sp = " " * row.index("@example")
1687 rep = example.groups()[0]
1688 exa = example.groups()[2].replace("[|", "(").replace("|]", ")")
1689 pag = example.groups()[1]
1690 if pag is None:
1691 pag = ""
1692 fil = os.path.splitext(os.path.split(filename)[-1])[0]
1693 fil = re.sub(r'([^a-zA-Z0-9_])', "", fil)
1694 ref = fil + "-l%d" % (i + index_first_line)
1695 ref2 = make_label_index(exa, typstr(example.groups()))
1696 to = "\n\n%s.. _le-%s:\n\n%s.. _le-%s:\n\n%s**Example: %s** \n\n%s.. example(%s%s;;le-%s).\n" % (
1697 sp, ref, sp, ref2, sp, exa, sp, pag, exa, ref)
1698 rows[i] = row.replace(rep, to)
1700 # it requires an empty line before if the previous line does
1701 # not start by :
1702 if i > 0 and not rows[
1703 i - 1].strip().startswith(":") and len(rows[i - 1].strip()) > 0:
1704 rows[i] = "\n" + rows[i]
1705 beginends["example"] = beginends.get("example", 0) + 1
1707 elif faq:
1708 sp = " " * row.index("@FAQ")
1709 rep = faq.groups()[0]
1710 exa = faq.groups()[2].replace("[|", "(").replace("|]", ")")
1711 pag = faq.groups()[1]
1712 if pag is None:
1713 pag = ""
1714 fil = os.path.splitext(os.path.split(filename)[-1])[0]
1715 fil = re.sub(r'([^a-zA-Z0-9_])', "", fil)
1716 ref = fil + "-l%d" % (i + index_first_line)
1717 ref2 = make_label_index(exa, typstr(faq.groups()))
1718 to = "\n\n%s.. _le-%s:\n\n%s.. _le-%s:\n\n%s**FAQ: %s** \n\n%s.. FAQ(%s%s;;le-%s).\n" % (
1719 sp, ref, sp, ref2, sp, exa, sp, pag, exa, ref)
1720 rows[i] = row.replace(rep, to)
1722 # it requires an empty line before if the previous line does
1723 # not start by :
1724 if i > 0 and not rows[
1725 i - 1].strip().startswith(":") and len(rows[i - 1].strip()) > 0:
1726 rows[i] = "\n" + rows[i]
1727 beginends["FAQ"] = beginends.get("FAQ", 0) + 1
1729 elif nbreg: # pragma: no cover
1730 sp = " " * row.index("@NB")
1731 rep = nbreg.groups()[0]
1732 exa = nbreg.groups()[2].replace("[|", "(").replace("|]", ")")
1733 pag = nbreg.groups()[1]
1734 if pag is None:
1735 pag = ""
1736 fil = os.path.splitext(os.path.split(filename)[-1])[0]
1737 fil = re.sub(r'([^a-zA-Z0-9_])', "", fil)
1738 ref = fil + "-l%d" % (i + index_first_line)
1739 ref2 = make_label_index(exa, typstr(nbreg.groups()))
1740 to = "\n\n%s.. _le-%s:\n\n%s.. _le-%s:\n\n%s**NB: %s** \n\n%s.. NB(%s%s;;le-%s).\n" % (
1741 sp, ref, sp, ref2, sp, exa, sp, pag, exa, ref)
1742 rows[i] = row.replace(rep, to)
1744 # it requires an empty line before if the previous line does
1745 # not start by :
1746 if i > 0 and not rows[
1747 i - 1].strip().startswith(":") and len(rows[i - 1].strip()) > 0:
1748 rows[i] = "\n" + rows[i]
1749 beginends["NB"] = beginends.get("NB", 0) + 1
1751 elif "@return" in row:
1752 rows[i] = row.replace("@return", ":return:")
1753 # it requires an empty line before if the previous line does
1754 # not start by :
1755 if i > 0 and not rows[
1756 i - 1].strip().startswith(":") and len(rows[i - 1].strip()) > 0:
1757 rows[i] = "\n" + rows[i]
1759 elif "@rtype" in row:
1760 rows[i] = row.replace("@rtype", ":rtype:")
1761 # it requires an empty line before if the previous line does
1762 # not start by :
1763 if i > 0 and not rows[
1764 i - 1].strip().startswith(":") and len(rows[i - 1].strip()) > 0:
1765 rows[i] = "\n" + rows[i]
1767 elif "@brief" in row:
1768 rows[i] = row.replace("@brief", "").strip()
1769 elif "@file" in row:
1770 rows[i] = row.replace("@file", "").strip()
1772 # loop on references
1773 refl = refe.search(rows[i])
1774 while refl:
1775 see = "see" in refl.groups()[1]
1776 see = "" # " " if see else ""
1777 ty = refl.groups()[4]
1778 name = refl.groups()[-2]
1779 if len(name) == 0:
1780 raise SyntaxError(
1781 "name should be empty: " + typstr(refl.groups()))
1782 rep = refl.groups()[0]
1783 ty = {"cl": "class", "me": "meth", "at": "attr",
1784 "fn": "func", "te": "term", "md": "mod"}[ty]
1785 to = "%s:%s:`%s`" % (see, ty, name)
1786 rows[i] = rows[i].replace(rep, to)
1787 refl = refe.search(rows[i])
1789 if not debug:
1790 for i, row in enumerate(rows):
1791 if "__sphinx__skip__" in row:
1792 break
1793 if "@param" in row or "@return" in row or "@see" in row or "@warning" in row \
1794 or "@todo" in row or "@code" in row or "@endcode" in row or "@brief" in row or "@file" in row \
1795 or "@rtype" in row or "@exception" in row \
1796 or "@example" in row or "@NB" in row or "@endNB" in row or "@endexample" in row:
1797 if not silent: # pragma: no cover
1798 fLOG("#########################")
1799 _private_migrating_doxygen_doc(
1800 debugrows, index_first_line, filename, debug=True)
1801 fLOG("#########################")
1802 mes = " File \"%s\", line %d, in ???\n unable to process: %s \nwhole blocks:\n%s" % (
1803 filename, index_first_line + i + 1, row, "\n".join(rows))
1804 fLOG("[sphinxerror]-D ", mes)
1805 else: # pragma: no cover
1806 mes = " File \"%s\", line %d, in ???\n unable to process: %s \nwhole blocks:\n%s" % (
1807 filename, index_first_line + i + 1, row, "\n".join(rows))
1808 raise SyntaxError(mes) # pragma: no cover
1810 for k, v in beginends.items():
1811 if v != 0: # pragma: no cover
1812 mes = " File \"%s\", line %d, in ???\n unbalanced tag %s: %s \nwhole blocks:\n%s" % (
1813 filename, index_first_line + i + 1, k, row, "\n".join(rows))
1814 fLOG("[sphinxerror]-E ", mes)
1815 raise SyntaxError(mes)
1817 # add githublink
1818 link = [_ for _ in rows if ":githublink:" in _]
1819 if len(link) == 0:
1820 rows.append("")
1821 rows.append("{1}:githublink:`%|py|{0}`".format(
1822 index_first_line, " " * min_indent))
1824 # clean rows
1825 clean_rows = []
1826 for row in rows:
1827 if row.strip():
1828 clean_rows.append(row)
1829 elif len(clean_rows) > 0:
1830 clean_rows.append('')
1831 return clean_rows
1834def doc_checking():
1835 """
1836 Example of a doc string.
1837 """
1838 pass
1841class useless_class_UnicodeStringIOThreadSafe (str):
1843 """avoid conversion problem between str and char,
1844 class protected again Thread issue"""
1846 def __init__(self):
1847 """
1848 creates a lock
1849 """
1850 str.__init__(self)
1851 import threading
1852 self.lock = threading.Lock()