Coverage for pyquickhelper/helpgen/utils_sphinx_doc_helpers.py: 83%
457 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
1"""
2@file
3@brief Various variables and classes used to produce a Sphinx documentation.
4"""
5import inspect
6import os
7import copy
8import re
9import sys
10import importlib
11import traceback
12from ..pandashelper.tblformat import df2rst
13from .helpgen_exceptions import HelpGenException, ImportErrorHelpGen
16#: max length for short summaries
17_length_truncated_doc = 120
20#: template for a module, substring ``__...__`` ought to be replaced
21add_file_rst_template = """
22__FULLNAME_UNDERLINED__
27.. inheritance-diagram:: __FULLNAMENOEXT__
30Short summary
31+++++++++++++
33__DOCUMENTATION__
36__CLASSES__
38__FUNCTIONS__
40__PROPERTIES__
42__STATICMETHODS__
44__METHODS__
46Documentation
47+++++++++++++
49.. automodule:: __FULLNAMENOEXT__
50 :members:
51 :special-members: __init__
52 :show-inheritance:
54__ADDEDMEMBERS__
56"""
58#: fields to be replaced
59add_file_rst_template_cor = {"class": "__CLASSES__",
60 "method": "__METHODS__",
61 "function": "__FUNCTIONS__",
62 "staticmethod": "__STATICMETHODS__",
63 "property": "__PROPERTIES__",
64 }
66#: names for python objects
67add_file_rst_template_title = {"class": "Classes",
68 "method": "Methods",
69 "function": "Functions",
70 "staticmethod": "Static Methods",
71 "property": "Properties",
72 }
74#
75# :platform: Unix, Windows
76# :synopsis: Analyze and reanimate dead parrots.
77# .. moduleauthor:: xx <x@x>
78# .. moduleauthor:: xx <x@x>
79# for autosummary
80# :toctree: __FILENAMENOEXT__/
81#
84def compute_truncated_documentation(doc, length=_length_truncated_doc,
85 raise_exception=False):
86 """
87 Produces a truncated version of a docstring.
89 @param doc doc string
90 @param length approximated length of the truncated docstring
91 @param raise_exception raises an exception when the result is empty and the input is not
92 @return truncated doc string
93 """
94 if len(doc) == 0:
95 return doc
96 else:
97 doc_ = doc
99 if "@brief " in doc:
100 doc = doc.split("@brief ")
101 doc = doc[-1]
102 if ":githublink:" in doc:
103 doc = doc.split(":githublink:")
104 doc = doc[0]
106 doc = doc.strip("\n\r\t ").replace("\t", " ")
108 # we stop at the first ...
109 lines = [li.rstrip() for li in doc.split("\n")]
110 pos = None
111 for i, li in enumerate(lines):
112 lll = li.lstrip()
113 if lll.startswith(".. ") and li.endswith("::"):
114 pos = i
115 break
116 if lll.startswith("* ") or lll.startswith("- "):
117 pos = i
118 break
119 if pos is not None:
120 lines = lines[:pos]
122 # we filter out other stuff
123 def filter_line(line):
124 s = line.strip()
125 if s.startswith(":title:"):
126 return line.replace(":title:", "")
127 elif s.startswith(":tag:") or s.startswith(":lid:"):
128 return ""
129 return line
130 doc = "\n".join(filter_line(line) for line in lines)
131 doc = doc.replace("\n", " ").replace("\r", "").strip("\n\r\t ")
133 for subs in ["@" + "param", "@" + "return", ":param", ":return", ".. ", "::"]:
134 if subs in doc:
135 doc = doc[:doc.find(subs)].strip("\r\t ")
137 if len(doc) >= _length_truncated_doc:
138 spl = doc.split(" ")
139 doc = ""
140 cq, cq2 = 0, 0
141 i = 0
142 while i < len(spl) and (len(doc) < _length_truncated_doc or cq % 2 != 0 or cq2 % 2 != 0):
143 cq += spl[i].count("`")
144 cq2 += spl[i].count("``")
145 doc += spl[i] + " "
146 i += 1
147 doc += "..."
149 doc = re.sub(' +', ' ', doc)
151 if raise_exception and len(doc) == 0:
152 raise ValueError( # pragma: no cover
153 f"bad format for docstring:\n{doc_}")
155 return doc
158class ModuleMemberDoc:
160 """
161 Represents a member in a module.
163 See :epkg:`*py:inspect`.
165 Attributes:
167 * *obj (object)*: object
168 * *type (str)*: type
169 * *cl (object)*: class it belongs to
170 * *name (str)*: name
171 * *module (str)*: module name
172 * *doc (str)*: documentation
173 * *truncdoc (str)*: truncated documentation
174 * *owner (object)*: module
175 """
177 def __init__(self, obj, ty=None, cl=None, name=None, module=None):
178 """
179 @param obj any kind of object
180 @param ty type (if you want to overwrite what the class will choose),
181 this type is a string (class, method, function)
182 @param cl if is a method, class it belongs to
183 @param name name of the object
184 @param module module name if belongs to
185 """
186 if module is None:
187 raise ValueError("module cannot be null.") # pragma: no cover
189 self.owner = module
190 self.obj = obj
191 self.cl = cl
192 if ty is not None:
193 self.type = ty
194 self.name = name
195 self.populate()
197 typstr = str
199 if self.cl is None and self.type in [
200 "method", "staticmethod", "property"]:
201 self.cl = self.obj.__class__
202 if self.cl is None and self.type in [
203 "method", "staticmethod", "property"]:
204 raise TypeError( # pragma: no cover
205 f"N/a method must have a class (not None): {typstr(self.obj)}")
207 def add_prefix(self, prefix):
208 """
209 Adds a prefix (for the documentation).
210 @param prefix string
211 """
212 self.prefix = prefix
214 @property
215 def key(self):
216 """
217 Returns a key to identify it.
218 """
219 return f"{self.type};{self.name}"
221 def populate(self):
222 """
223 Extracts some information about an object.
224 """
225 obj = self.obj
226 ty = self.type if "type" in self.__dict__ else None
227 typstr = str
228 if ty is None:
229 if inspect.isclass(obj):
230 self.type = "class"
231 elif inspect.ismethod(obj):
232 self.type = "method"
233 elif inspect.isfunction(obj) or "built-in function" in str(obj):
234 self.type = "function"
235 elif inspect.isgenerator(obj):
236 self.type = "generator"
237 else:
238 raise TypeError( # pragma: no cover
239 "E/unable to deal with this type: " + typstr(type(obj)))
241 if ty == "method":
242 if isinstance(obj, staticmethod):
243 self.type = "staticmethod"
244 elif isinstance(obj, property):
245 self.type = "property"
246 elif sys.version_info >= (3, 4):
247 # should be replaced by something more robust
248 if len(obj.__code__.co_varnames) == 0:
249 self.type = "staticmethod"
250 elif obj.__code__.co_varnames[0] != 'self':
251 self.type = "staticmethod"
253 # module
254 try:
255 self.module = obj.__module__
256 self.name = obj.__name__
257 except Exception:
258 if self.type in ["property", "staticmethod"]:
259 self.module = self.cl.__module__
260 else:
261 self.module = None
262 if self.name is None:
263 raise IndexError( # pragma: no cover
264 "Unable to find a name for this object type={0}, "
265 "self.type={1}, owner='{2}'".format(
266 type(obj), self.type, self.owner))
268 # full path for the module
269 if self.module is not None:
270 self.fullpath = self.module
271 else:
272 self.fullpath = ""
274 # documentation
275 if self.type == "staticmethod":
276 try:
277 self.doc = obj.__func__.__doc__
278 except Exception as ie: # pragma: no cover
279 try:
280 self.doc = obj.__doc__
281 except Exception as ie2:
282 self.doc = (
283 typstr(ie) + " - " + typstr(ie2) + " \n----------\n " +
284 typstr(dir(obj)))
285 else:
286 try:
287 self.doc = obj.__doc__
288 except Exception as ie: # pragma: no cover
289 self.doc = typstr(ie) + " \n----------\n " + typstr(dir(obj))
291 try:
292 self.file = self.module.__file__
293 except Exception:
294 self.file = ""
296 # truncated documentation
297 if self.doc is not None:
298 self.truncdoc = compute_truncated_documentation(self.doc)
299 else:
300 self.doc = ""
301 self.truncdoc = ""
303 if self.name is None:
304 raise TypeError( # pragma: no cover
305 f"S/name is None for object: {typstr(self.obj)}")
307 def __str__(self):
308 """
309 usual
310 """
311 return "[key={0},clname={1},type={2},module_name={3},file={4}".format(
312 self.key, self.classname, self.type, self.module, self.owner.__file__)
314 def rst_link(self, prefix=None, class_in_bracket=True):
315 """
316 Returns a sphinx link on the object.
318 @param prefix to correct the path with a prefix
319 @param class_in_bracket if True, adds the class in bracket
320 for methods and properties
321 @return a string style, see below
323 String style:
325 ::
327 :%s:`%s <%s>` or
328 :%s:`%s <%s>` (class)
329 """
330 cor = {"function": "func",
331 "method": "meth",
332 "staticmethod": "meth",
333 "property": "meth"}
335 if self.type in ["method", "staticmethod", "property"]:
336 path = f"{self.module}.{self.cl.__name__}.{self.name}"
337 else:
338 path = f"{self.module}.{self.name}"
340 if prefix is not None:
341 path = f"{prefix}.{path}"
343 if self.type in ["method", "staticmethod",
344 "property"] and class_in_bracket:
345 link = ":%s:`%s <%s>` (%s)" % (
346 cor.get(self.type, self.type), self.name, path, self.cl.__name__)
347 else:
348 link = f":{cor.get(self.type, self.type)}:`{self.name} <{path}>`"
349 return link
351 @property
352 def classname(self):
353 """
354 Returns the class name if the object is a method.
356 @return class object
357 """
358 if self.type in ["method", "staticmethod", "property"]:
359 return self.cl
360 else:
361 return None
363 def __cmp__(self, oth):
364 """
365 Comparison operators, compares first the first,
366 second the name (lower case).
368 @param oth other object
369 @return -1, 0 or 1
370 """
371 if self.type == oth.type:
372 ln = self.fullpath + "@@@" + self.name.lower()
373 lo = oth.fullpath + "@@@" + oth.name.lower()
374 c = -1 if ln < lo else (1 if ln > lo else 0)
375 if c == 0 and self.type == "method":
376 ln = self.cl.__name__
377 lo = self.cl.__name__
378 c = -1 if ln < lo else (1 if ln > lo else 0)
379 return c
380 else:
381 return - \
382 1 if self.type < oth.type else (
383 1 if self.type > oth.type else 0)
385 def __lt__(self, oth):
386 """
387 Operator ``<``.
388 """
389 return self.__cmp__(oth) == -1
391 def __eq__(self, oth):
392 """
393 Operator ``==``.
394 """
395 return self.__cmp__(oth) == 0
397 def __gt__(self, oth):
398 """
399 Operator ``>``.
400 """
401 return self.__cmp__(oth) == 1
404class IndexInformation:
406 """
407 Keeps some information to index.
408 """
410 def __init__(self, type, label, name, text, rstfile, fullname):
411 """
412 @param type each type gets an index
413 @param label label used to index
414 @param name name to display
415 @param text text to show as a short description
416 @param rstfile tells which file the index refers to (rst file)
417 @param fullname fullname of a file the rst file describes
418 """
419 self.type = type
420 self.label = label
421 self.name = name
422 self.text = text
423 self.fullname = fullname
424 self.set_rst_file(rstfile)
426 def __str__(self):
427 """
428 usual
429 """
430 return f"{self.label} -- {self.rst_link()}"
432 def set_rst_file(self, rstfile):
433 """
434 Sets the rst file and checks the label is present in it.
436 @param rstfile rst file
437 """
438 self.rstfile = rstfile
439 if rstfile is not None:
440 self.add_label_if_not_present()
442 @property
443 def truncdoc(self):
444 """
445 Returns ``self.text``.
446 """
447 return self.text.replace("\n", " ").replace(
448 "\t", "").replace("\r", "")
450 def add_label_if_not_present(self):
451 """
452 The function checks the label is present in the original file.
453 """
454 if self.rstfile is not None:
455 with open(self.rstfile, "r", encoding="utf8") as f:
456 content = f.read()
457 label = f".. _{self.label}:"
458 if label not in content:
459 content = f"\n{label}\n{content}"
460 with open(self.rstfile, "w", encoding="utf8") as f:
461 f.write(content)
463 @staticmethod
464 def get_label(existing, suggestion):
465 """
466 Returns a new label given the existing ones.
468 @param existing existing labels stored in a dictionary
469 @param suggestion the suggestion will be chosen if it does not exists,
470 ``suggestion + zzz`` otherwise
471 @return string
472 """
473 if existing is None:
474 raise ValueError( # pragma: no cover
475 "existing must not be None")
476 suggestion = suggestion.replace("_", "").replace(".", "")
477 while suggestion in existing:
478 suggestion += "z"
479 return suggestion
481 def rst_link(self):
482 """
483 return a link rst
484 @return rst link
485 """
486 if self.label.startswith("_"):
487 return f":ref:`{self.name} <{self.label[1:]}>`"
488 else:
489 return f":ref:`{self.name} <{self.label}>`"
492class RstFileHelp:
493 """
494 Defines what a rst file and what it describes.
495 """
497 def __init__(self, file, rst, doc):
498 """
499 @param file original filename
500 @param rst produced rst file
501 @param doc documentation if any
502 """
503 self.file = file
504 self.rst = rst
505 self.doc = doc
508def import_module(rootm, filename, log_function, additional_sys_path=None,
509 first_try=True):
510 """
511 Imports a module using its filename.
513 @param rootm root of the module (for relative import)
514 @param filename file name of the module
515 @param log_function logging function
516 @param additional_sys_path additional path to include to ``sys.path`` before
517 importing a module (will be removed afterwards)
518 @param first_try first call to the function (to avoid infinite loop)
519 @return module object, prefix
521 The function can also import compiled modules.
523 .. warning:: It adds the file path at the first
524 position in ``sys.path`` and then deletes it.
525 """
526 if additional_sys_path is None:
527 additional_sys_path = []
528 memo = copy.deepcopy(sys.path)
529 li = filename.replace("\\", "/")
530 sdir = os.path.abspath(os.path.split(li)[0])
531 relpath = os.path.relpath(li, rootm).replace("\\", "/")
532 if "/" in relpath:
533 spl = relpath.split("/")
534 fmod = spl[0] # this is the prefix
535 relpath = "/".join(spl[1:])
536 else:
537 fmod = ""
539 # has init
540 init_ = os.path.join(sdir, "__init__.py")
541 if init_ != filename and not os.path.exists(init_):
542 # no init
543 return f"No __init__.py, unable to import {filename}", fmod
545 # we remove every path ending by "src" except if it is found in PYTHONPATH
546 pythonpath = os.environ.get("PYTHONPATH", None)
547 if pythonpath is not None:
548 sep = ";" if sys.platform.startswith("win") else ":"
549 pypaths = [os.path.normpath(_)
550 for _ in pythonpath.split(sep) if len(_) > 0]
551 else:
552 pypaths = []
553 rem = []
554 for i, p in enumerate(sys.path):
555 if (p.endswith("src") and p not in pypaths) or ".zip" in p:
556 rem.append(i)
557 rem.reverse()
558 for r in rem:
559 del sys.path[r]
561 # Extracts extended extension of the module.
562 if li.endswith(".py"):
563 cpxx = ".py"
564 ext_rem = ".py"
565 elif li.endswith(".pyd"): # pragma: no cover
566 cpxx = ".cp%d%d-" % sys.version_info[:2]
567 search = li.rfind(cpxx)
568 ext_rem = li[search:]
569 else:
570 cpxx = ".cpython-%d%dm-" % sys.version_info[:2]
571 search = li.rfind(cpxx)
572 if search == -1:
573 cpxx = ".cpython-%d%d-" % sys.version_info[:2]
574 search = li.rfind(cpxx)
575 if search == -1:
576 raise ImportErrorHelpGen( # pragma: no cover
577 f"Unable to guess extension from '{li}'.")
578 ext_rem = li[search:]
579 if not ext_rem:
580 raise ValueError( # pragma: no cover
581 f"Unable to guess file extension '{li}'")
582 if ext_rem != ".py":
583 log_function(f"[import_module] found extension='{ext_rem}'")
585 # remove fmod from sys.modules
586 if fmod:
587 addback = []
588 rem = []
589 for n, m in sys.modules.items():
590 if n.startswith(fmod):
591 rem.append(n)
592 addback.append((n, m))
593 else:
594 addback = []
595 relpath.replace(ext_rem, "")
596 rem = []
597 for n, m in sys.modules.items():
598 if n.startswith(relpath):
599 rem.append(n)
600 addback.append((n, m))
602 # we remove the modules
603 # this line is important to remove all modules
604 # from the sources in folder src and not the modified ones
605 # in the documentation folder
606 for r in rem:
607 del sys.modules[r]
609 # full path
610 if rootm is not None:
611 root = rootm
612 tl = relpath
613 fi = tl.replace(ext_rem, "").replace("/", ".")
614 if fmod:
615 fi = fmod + "." + fi
616 context = None
617 if fi.endswith(".__init__"):
618 fi = fi[:-len(".__init__")]
619 else:
620 root = sdir
621 tl = os.path.split(li)[1]
622 fi = tl.replace(ext_rem, "")
623 context = None
625 if additional_sys_path is not None and len(additional_sys_path) > 0:
626 # there is an issue here due to the confusion in the paths
627 # the paths should be removed just after the import
628 sys.path.extend(additional_sys_path) # pragma: no cover
630 sys.path.insert(0, root)
631 try:
632 try:
633 mo = importlib.import_module(fi, context)
634 except ImportError: # pragma: no cover
635 log_function(
636 "[import_module] unable to import module '{0}' fullname "
637 "'{1}'".format(fi, filename))
638 mo_spec = importlib.util.find_spec(fi, context)
639 log_function("[import_module] imported spec", mo_spec)
640 mo = mo_spec.loader.load_module()
641 log_function("[import_module] successful try", mo_spec)
643 if not mo.__file__.replace("\\", "/").endswith(
644 filename.replace("\\", "/").strip("./")): # pragma: no cover
645 namem = os.path.splitext(os.path.split(filename)[-1])[0]
647 if "src" in sys.path:
648 sys.path = [_ for _ in sys.path if _ != "src"]
650 if namem in sys.modules:
651 del sys.modules[namem]
652 # add the context here for relative import
653 # use importlib.import_module with the package argument filled
654 # mo = __import__ (fi)
655 try:
656 mo = importlib.import_module(fi, context)
657 except ImportError:
658 mo = importlib.util.find_spec(fi, context)
660 if not mo.__file__.replace(
661 "\\", "/").endswith(filename.replace("\\", "/").strip("./")):
662 raise ImportError(
663 "The wrong file was imported (2):\nEXP: {0}\nIMP: {1}\n"
664 "PATHS:\n - {2}".format(
665 filename, mo.__file__, "\n - ".join(sys.path)))
666 else:
667 raise ImportError(
668 "The wrong file was imported (1):\nEXP: {0}\nIMP: {1}\n"
669 "PATHS:\n - {2}".format(
670 filename, mo.__file__, "\n - ".join(sys.path)))
672 sys.path = memo
673 log_function(
674 f"[import_module] import '{filename}' successfully", mo.__file__)
675 for n, m in addback:
676 if n not in sys.modules:
677 sys.modules[n] = m
678 return mo, fmod
680 except ImportError as e: # pragma: no cover
681 exp = re.compile("No module named '(.*)'")
682 find = exp.search(str(e))
683 if find:
684 module = find.groups()[0]
685 log_function(
686 "[warning] unable to import module " + module +
687 " --- " + str(e).replace("\n", " "))
689 log_function(" File \"%s\", line %d" % (__file__, 501))
690 log_function("[warning] -- unable to import module (1) ", filename,
691 ",", fi, " in path ", sdir, " Error: ", str(e))
692 log_function(" cwd ", os.getcwd())
693 log_function(" path", sdir)
694 stack = traceback.format_exc()
695 log_function(" executable", sys.executable)
696 log_function(" version", sys.version_info)
697 log_function(" stack:\n", stack)
699 message = ["-----", stack, "-----"]
700 message.append(f" executable: '{sys.executable}'")
701 message.append(f" version: '{sys.version_info}'")
702 message.append(f" platform: '{sys.platform}'")
703 message.append(f" ext_rem='{ext_rem}'")
704 message.append(f" fi='{fi}'")
705 message.append(f" li='{li}'")
706 message.append(f" cpxx='{cpxx}'")
707 message.append("-----")
708 for p in sys.path:
709 message.append(" path: " + p)
710 message.append("-----")
711 for p in sorted(sys.modules):
712 try:
713 m = sys.modules[p].__path__
714 except AttributeError:
715 m = str(sys.modules[p])
716 message.append(f" module: {p}={m}")
718 sys.path = memo
719 for n, m in addback:
720 if n not in sys.modules:
721 sys.modules[n] = m
723 if 'File "<frozen importlib._bootstrap>"' in stack:
724 raise ImportErrorHelpGen(
725 "frozen importlib._bootstrap is an issue:\n" + "\n".join(message)) from e
727 return f"Unable(1) to import {filename}\nError:\n{str(e)}", fmod
729 except SystemError as e: # pragma: no cover
730 log_function("[warning] -- unable to import module (2) ", filename,
731 ",", fi, " in path ", sdir, " Error: ", str(e))
732 stack = traceback.format_exc()
733 log_function(" executable", sys.executable)
734 log_function(" version", sys.version_info)
735 log_function(" stack:\n", stack)
736 sys.path = memo
737 for n, m in addback:
738 if n not in sys.modules:
739 sys.modules[n] = m
740 return f"unable(2) to import {filename}\nError:\n{str(e)}", fmod
742 except KeyError as e: # pragma: no cover
743 if first_try and "KeyError: 'pip._vendor.urllib3.contrib'" in str(e):
744 # Issue with pip 9.0.2
745 return import_module(rootm=rootm, filename=filename, log_function=log_function,
746 additional_sys_path=additional_sys_path,
747 first_try=False)
748 else:
749 log_function("[warning] -- unable to import module (4) ", filename,
750 ",", fi, " in path ", sdir, " Error: ", str(e))
751 stack = traceback.format_exc()
752 log_function(" executable", sys.executable)
753 log_function(" version", sys.version_info)
754 log_function(" stack:\n", stack)
755 sys.path = memo
756 for n, m in addback:
757 if n not in sys.modules:
758 sys.modules[n] = m
759 return f"unable(4) to import {filename}\nError:\n{str(e)}", fmod
761 except Exception as e: # pragma: no cover
762 log_function("[warning] -- unable to import module (3) ", filename,
763 ",", fi, " in path ", sdir, " Error: ", str(e))
764 stack = traceback.format_exc()
765 log_function(" executable", sys.executable)
766 log_function(" version", sys.version_info)
767 log_function(" stack:\n", stack)
768 sys.path = memo
769 for n, m in addback:
770 if n not in sys.modules:
771 sys.modules[n] = m
772 return f"unable(3) to import {filename}\nError:\n{str(e)}", fmod
775def get_module_objects(mod):
776 """
777 Gets all the classes from a module.
779 @param mod module objects
780 @return list of ModuleMemberDoc
781 """
783 # exp = { "__class__":"",
784 # "__dict__":"",
785 # "__doc__":"",
786 # "__format__":"",
787 # "__reduce__":"",
788 # "__reduce_ex__":"",
789 # "__subclasshook__":"",
790 # "__dict__":"",
791 # "__weakref__":""
792 # }
794 cl = []
795 for _, obj in inspect.getmembers(mod):
796 try:
797 stobj = str(obj)
798 except RuntimeError: # pragma: no cover
799 # One issue met in werkzeug
800 # Working outside of request context.
801 stobj = ""
802 if (inspect.isclass(obj) or inspect.isfunction(obj) or
803 inspect.isgenerator(obj) or inspect.ismethod(obj) or
804 ("built-in function" in stobj and not isinstance(obj, dict))):
805 cl.append(ModuleMemberDoc(obj, module=mod))
806 if inspect.isclass(obj):
807 for n, o in inspect.getmembers(obj):
808 try:
809 ok = ModuleMemberDoc(
810 o, "method", cl=obj, name=n, module=mod)
811 if ok.module is not None:
812 cl.append(ok)
813 except Exception as e:
814 if str(e).startswith("S/"):
815 raise e # pragma: no cover
817 res = []
818 for _ in cl:
819 try:
820 # if _.module != None :
821 if _.module == mod.__name__:
822 res.append(_)
823 except Exception: # pragma: no cover
824 pass
826 res.sort()
827 return res
830def process_var_tag(
831 docstring, rst_replace=False, header=None):
832 """
833 Processes a docstring using tag ``@ var``, and return a list of 2-tuple::
835 @ var filename file name
836 @ var utf8 decode in utf8?
837 @ var errors decoding in utf8 can raise some errors
839 @param docstring string
840 @param rst_replace if True, replace the var bloc var a rst bloc
841 @param header header for the table, if None, ``["attribute", "meaning"]``
842 @return a matrix with two columns or a string if rst_replace is True
844 """
845 from pandas import DataFrame
847 if header is None:
848 header = ["attribute", "meaning"]
850 reg = re.compile("[@]var +([_a-zA-Z][a-zA-Z0-9_]*?) +((?:(?!@var).)+)")
852 indent = len(docstring)
853 spl = docstring.split("\n")
854 docstring = []
855 bigrow = ""
856 for line in spl:
857 if len(line.strip("\r \t")) == 0:
858 docstring.append(bigrow)
859 bigrow = ""
860 else:
861 ind = len(line) - len(line.lstrip(" "))
862 indent = min(ind, indent)
863 bigrow += line + "\n"
864 if len(bigrow) > 0:
865 docstring.append(bigrow)
867 values = []
868 if rst_replace:
869 for line in docstring:
870 line2 = line.replace("\n", " ")
871 if "@var" in line2:
872 all = reg.findall(line2)
873 val = []
874 for a in all:
875 val.append(list(a))
876 if len(val) > 0:
877 tbl = DataFrame(columns=header, data=val)
878 rst = df2rst(tbl, list_table=True)
879 if indent > 0:
880 rst = "\n".join((" " * indent) +
881 _ for _ in rst.split("\n"))
882 values.append(rst)
883 else:
884 values.append(line)
885 return "\n".join(values)
886 else:
887 for line in docstring:
888 line = line.replace("\n", " ")
889 if "@var" in line:
890 alls = reg.findall(line)
891 for a in alls:
892 values.append(a)
893 return values
896def make_label_index(title, comment):
897 """
898 Builds a :epkg:`sphinx` label from a string by
899 removing any odd characters.
901 @param title title
902 @param comment add this string in the exception when it raises one
903 @return label
904 """
905 def accept(c):
906 if "a" <= c <= "z":
907 return c
908 if "A" <= c <= "Z":
909 return c
910 if "0" <= c <= "9":
911 return c
912 if c in "-_":
913 return c
914 return ""
916 try:
917 r = "".join(map(accept, title))
918 if len(r) == 0:
919 raise HelpGenException( # pragma: no cover
920 "Unable to interpret this title (empty?): {0} (type: {2})\n"
921 "COMMENT:\n{1}".format(
922 str(title), comment, str(type(title))))
923 return r
924 except TypeError as e: # pragma: no cover
925 raise HelpGenException(
926 "Unable to interpret this title: {0} (type: {2})\nCOMMENT:"
927 "\n{1}".format(
928 str(title), comment, str(type(title)))) from e
931def process_look_for_tag(tag, title, files):
932 """
933 Looks for specific information in all files, collect them
934 into one single page.
936 @param tag tag
937 @param title title of the page
938 @param files list of files to look for
939 @return a list of tuple (page, content of the page)
941 The function is looking for regular expression::
943 .. tag(...).
944 ...
945 .. endtag.
947 They can be split into several pages::
949 .. tag(page::...).
950 ...
951 .. endtag.
953 If the extracted example contains an image (..image:: ../../), the path
954 is fixed too.
956 The function parses the files instead of loading the files as a module.
957 The function needs to replace ``\\\\`` by ``\\``, it does not takes into
958 acount doc string starting with ``r'''``.
959 The function calls @see fn remove_some_indent
960 with ``backslash=True`` to replace double backslashes
961 by simple backslashes.
962 """
963 def noneempty(a):
964 if "___" in a:
965 page, b = a.split("___")
966 return "_" + page, b.lower(), b
967 return "", a.lower(), a
968 repl = "__!LI!NE!__"
969 exp = re.compile(
970 f"[.][.] {tag}[(](.*?);;(.*?)[)][.](.*?)[.][.] end{tag}[.]")
971 exp2 = re.compile(
972 f"[.][.] {tag}[(](.*?)[)][.](.*?)[.][.] end{tag}[.]")
973 coll = []
974 for file in files:
975 if file.file is None:
976 continue
977 if "utils_sphinx_doc.py" in file.file:
978 continue
979 if file.file.endswith(".py"):
980 try:
981 with open(file.file, "r", encoding="utf8") as f:
982 content = f.read()
983 except Exception: # pragma: no cover
984 with open(file.file, "r") as f:
985 content = f.read()
986 content = content.replace("\n", repl)
987 else:
988 content = "Binary file."
990 all = exp.findall(content)
991 all2 = exp2.findall(content)
992 if len(all2) > len(all):
993 raise HelpGenException( # pragma: no cover
994 f"An issue was detected in file {file.file!r}.")
996 coll += [noneempty(a) +
997 (fix_image_page_for_root(c.replace(repl, "\n"), file), b)
998 for a, b, c in all]
1000 coll.sort()
1001 coll = [(_[0],) + _[2:] for _ in coll]
1003 pages = set(_[0] for _ in coll)
1005 pagerows = []
1007 for page in pages:
1008 if page == "":
1009 tit = title
1010 suf = ""
1011 else:
1012 tit = title + ": " + page.strip("_")
1013 suf = page.replace(" ", "").replace("_", "")
1014 suf = re.sub(r'([^a-zA-Z0-9_])', "", suf)
1015 page = re.sub(r'([^a-zA-Z0-9_])', "", page)
1017 rows = ["""
1018 .. _l-{0}{3}:
1020 {1}
1021 {2}
1023 .. contents::
1024 :local:
1026 """.replace(" ", "").format(tag, tit, "=" * len(tit), suf)]
1028 not_expected = os.environ.get(
1029 "USERNAME", os.environ.get("USER", "````````````"))
1030 if not_expected != "jenkins" and not_expected in rows[0]:
1031 raise HelpGenException( # pragma: no cover
1032 "The title is probably wrong (4): {0}\ntag={1}\ntit={2}\n"
1033 "not_expected='{3}'".format(rows[0], tag, tit, not_expected))
1035 for pa, a, b, c in coll:
1036 pan = re.sub(r'([^a-zA-Z0-9_])', "", pa)
1037 if page != pan:
1038 continue
1039 lindex = make_label_index(a, pan)
1040 rows.append("")
1041 rows.append(f".. _lm-{lindex}:")
1042 rows.append("")
1043 rows.append(a)
1044 rows.append("+" * len(a))
1045 rows.append("")
1046 rows.append(remove_some_indent(b, backslash=True))
1047 rows.append("")
1048 spl = c.split("-")
1049 d = f"file {spl[1]}.py" # line, spl[2].lstrip("l"))
1050 rows.append(f"see :ref:`{d} <{c}>`")
1051 rows.append("")
1053 pagerows.append((page, "\n".join(rows)))
1054 return pagerows
1057def fix_image_page_for_root(content, file):
1058 """
1059 Looks for images and fix their path as
1060 if the extract were copied to the root.
1062 @param content extracted content
1063 @param file file where is comes from (unused)
1064 @return content
1065 """
1066 rows = content.split("\n")
1067 for i in range(len(rows)):
1068 row = rows[i]
1069 if ".. image::" in row:
1070 spl = row.split(".. image::")
1071 img = spl[-1]
1072 if "../images" in img:
1073 img = img.lstrip("./ ")
1074 if len(spl) == 1:
1075 row = ".. image:: " + img
1076 else:
1077 row = spl[0] + ".. image:: " + img
1078 rows[i] = row
1079 return "\n".join(rows)
1082def remove_some_indent(s, backslash=False):
1083 """
1084 Brings text to the left.
1086 @param s text
1087 @param backslash if True, replace double backslash by simple backslash
1088 @return text
1089 """
1090 rows = s.split("\n")
1091 mi = len(s)
1092 for lr in rows:
1093 ll = lr.lstrip()
1094 if len(ll) > 0:
1095 d = len(lr) - len(ll)
1096 mi = min(d, mi)
1098 if mi > 0:
1099 keep = []
1100 for _ in rows:
1101 keep.append(_[mi:] if len(_.strip()) > 0 and len(_) > mi else _)
1102 res = "\n".join(keep)
1103 else:
1104 res = s
1106 if backslash:
1107 res = res.replace("\\\\", "\\")
1108 return res
1111def example_function_latex():
1112 """
1113 This function only contains an example with
1114 latex to check it is working fine.
1116 .. exref::
1117 :title: How to display a formula
1119 We want to check this formula to successfully converted.
1121 :math:`\\left \\{ \\begin{array}{l} \\min_{x,y} \\left \\{ x^2 + y^2 - xy + y \\right \\}
1122 \\\\ \\text{sous contrainte} \\; x + 2y = 1 \\end{array}\\right .`
1124 Brackets and backslashes might be an issue.
1125 """
1126 pass