Coverage for src/pyquickhelper/helpgen/utils_sphinx_doc_helpers.py : 95%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2@file
3@brief Various variables and classes used to produce a Sphinx documentation.
4"""
5import inspect
6import os
7import copy
8import re
9import sys
10import importlib
11import traceback
12from ..pandashelper.tblformat import df2rst
13from .helpgen_exceptions import HelpGenException, ImportErrorHelpGen
16#: max length for short summaries
17_length_truncated_doc = 120
20#: template for a module, substring ``__...__`` ought to be replaced
21add_file_rst_template = """
22__FULLNAME_UNDERLINED__
27.. inheritance-diagram:: __FULLNAMENOEXT__
30Short summary
31+++++++++++++
33__DOCUMENTATION__
36__CLASSES__
38__FUNCTIONS__
40__PROPERTIES__
42__STATICMETHODS__
44__METHODS__
46Documentation
47+++++++++++++
49.. automodule:: __FULLNAMENOEXT__
50 :members:
51 :special-members: __init__
52 :show-inheritance:
54__ADDEDMEMBERS__
56"""
58#: fields to be replaced
59add_file_rst_template_cor = {"class": "__CLASSES__",
60 "method": "__METHODS__",
61 "function": "__FUNCTIONS__",
62 "staticmethod": "__STATICMETHODS__",
63 "property": "__PROPERTIES__",
64 }
66#: names for python objects
67add_file_rst_template_title = {"class": "Classes",
68 "method": "Methods",
69 "function": "Functions",
70 "staticmethod": "Static Methods",
71 "property": "Properties",
72 }
74#
75# :platform: Unix, Windows
76# :synopsis: Analyze and reanimate dead parrots.
77# .. moduleauthor:: xx <x@x>
78# .. moduleauthor:: xx <x@x>
79# for autosummary
80# :toctree: __FILENAMENOEXT__/
81#
84def compute_truncated_documentation(doc, length=_length_truncated_doc,
85 raise_exception=False):
86 """
87 Produces a truncated version of a docstring.
89 @param doc doc string
90 @param length approximated length of the truncated docstring
91 @param raise_exception raises an exception when the result is empty and the input is not
92 @return truncated doc string
93 """
94 if len(doc) == 0:
95 return doc
96 else:
97 doc_ = doc
99 if "@brief " in doc:
100 doc = doc.split("@brief ")
101 doc = doc[-1]
102 if ":githublink:" in doc:
103 doc = doc.split(":githublink:")
104 doc = doc[0]
106 doc = doc.strip("\n\r\t ").replace("\t", " ")
108 # we stop at the first ...
109 lines = [li.rstrip() for li in doc.split("\n")]
110 pos = None
111 for i, li in enumerate(lines):
112 lll = li.lstrip()
113 if lll.startswith(".. ") and li.endswith("::"):
114 pos = i
115 break
116 if lll.startswith("* ") or lll.startswith("- "):
117 pos = i
118 break
119 if pos is not None:
120 lines = lines[:pos]
122 # we filter out other stuff
123 def filter_line(line):
124 s = line.strip()
125 if s.startswith(":title:"):
126 return line.replace(":title:", "")
127 elif s.startswith(":tag:") or s.startswith(":lid:"):
128 return ""
129 return line
130 doc = "\n".join(filter_line(line) for line in lines)
131 doc = doc.replace("\n", " ").replace("\r", "").strip("\n\r\t ")
133 for subs in ["@" + "param", "@" + "return", ":param", ":return", ".. ", "::"]:
134 if subs in doc:
135 doc = doc[:doc.find(subs)].strip("\r\t ")
137 if len(doc) >= _length_truncated_doc:
138 spl = doc.split(" ")
139 doc = ""
140 cq, cq2 = 0, 0
141 i = 0
142 while i < len(spl) and (len(doc) < _length_truncated_doc or cq % 2 != 0 or cq2 % 2 != 0):
143 cq += spl[i].count("`")
144 cq2 += spl[i].count("``")
145 doc += spl[i] + " "
146 i += 1
147 doc += "..."
149 doc = re.sub(' +', ' ', doc)
151 if raise_exception and len(doc) == 0:
152 raise ValueError( # pragma: no cover
153 "bad format for docstring:\n{}".format(doc_))
155 return doc
158class ModuleMemberDoc:
160 """
161 Represents a member in a module.
163 See :epkg:`*py:inspect`.
165 Attributes:
167 * *obj (object)*: object
168 * *type (str)*: type
169 * *cl (object)*: class it belongs to
170 * *name (str)*: name
171 * *module (str)*: module name
172 * *doc (str)*: documentation
173 * *truncdoc (str)*: truncated documentation
174 * *owner (object)*: module
175 """
177 def __init__(self, obj, ty=None, cl=None, name=None, module=None):
178 """
179 @param obj any kind of object
180 @param ty type (if you want to overwrite what the class will choose),
181 this type is a string (class, method, function)
182 @param cl if is a method, class it belongs to
183 @param name name of the object
184 @param module module name if belongs to
185 """
186 if module is None:
187 raise ValueError("module cannot be null.") # pragma: no cover
189 self.owner = module
190 self.obj = obj
191 self.cl = cl
192 if ty is not None:
193 self.type = ty
194 self.name = name
195 self.populate()
197 typstr = str
199 if self.cl is None and self.type in [
200 "method", "staticmethod", "property"]:
201 self.cl = self.obj.__class__
202 if self.cl is None and self.type in [
203 "method", "staticmethod", "property"]:
204 raise TypeError( # pragma: no cover
205 "N/a method must have a class (not None): %s" % typstr(self.obj))
207 def add_prefix(self, prefix):
208 """
209 Adds a prefix (for the documentation).
210 @param prefix string
211 """
212 self.prefix = prefix
214 @property
215 def key(self):
216 """
217 Returns a key to identify it.
218 """
219 return "%s;%s" % (self.type, self.name)
221 def populate(self):
222 """
223 Extracts some information about an object.
224 """
225 obj = self.obj
226 ty = self.type if "type" in self.__dict__ else None
227 typstr = str
228 if ty is None:
229 if inspect.isclass(obj):
230 self.type = "class"
231 elif inspect.ismethod(obj):
232 self.type = "method"
233 elif inspect.isfunction(obj) or "built-in function" in str(obj):
234 self.type = "function"
235 elif inspect.isgenerator(obj):
236 self.type = "generator"
237 else:
238 raise TypeError( # pragma: no cover
239 "E/unable to deal with this type: " + typstr(type(obj)))
241 if ty == "method":
242 if isinstance(obj, staticmethod):
243 self.type = "staticmethod"
244 elif isinstance(obj, property):
245 self.type = "property"
246 elif sys.version_info >= (3, 4):
247 # should be replaced by something more robust
248 if len(obj.__code__.co_varnames) == 0:
249 self.type = "staticmethod"
250 elif obj.__code__.co_varnames[0] != 'self':
251 self.type = "staticmethod"
253 # module
254 try:
255 self.module = obj.__module__
256 self.name = obj.__name__
257 except Exception:
258 if self.type in ["property", "staticmethod"]:
259 self.module = self.cl.__module__
260 else:
261 self.module = None
262 if self.name is None:
263 raise IndexError( # pragma: no cover
264 "Unable to find a name for this object type={0}, "
265 "self.type={1}, owner='{2}'".format(
266 type(obj), self.type, self.owner))
268 # full path for the module
269 if self.module is not None:
270 self.fullpath = self.module
271 else:
272 self.fullpath = ""
274 # documentation
275 if self.type == "staticmethod":
276 try:
277 self.doc = obj.__func__.__doc__
278 except Exception as ie: # pragma: no cover
279 try:
280 self.doc = obj.__doc__
281 except Exception as ie2:
282 self.doc = (
283 typstr(ie) + " - " + typstr(ie2) + " \n----------\n " +
284 typstr(dir(obj)))
285 else:
286 try:
287 self.doc = obj.__doc__
288 except Exception as ie: # pragma: no cover
289 self.doc = typstr(ie) + " \n----------\n " + typstr(dir(obj))
291 try:
292 self.file = self.module.__file__
293 except Exception:
294 self.file = ""
296 # truncated documentation
297 if self.doc is not None:
298 self.truncdoc = compute_truncated_documentation(self.doc)
299 else:
300 self.doc = ""
301 self.truncdoc = ""
303 if self.name is None:
304 raise TypeError( # pragma: no cover
305 "S/name is None for object: %s" % typstr(self.obj))
307 def __str__(self):
308 """
309 usual
310 """
311 return "[key={0},clname={1},type={2},module_name={3},file={4}".format(
312 self.key, self.classname, self.type, self.module, self.owner.__file__)
314 def rst_link(self, prefix=None, class_in_bracket=True):
315 """
316 Returns a sphinx link on the object.
318 @param prefix to correct the path with a prefix
319 @param class_in_bracket if True, adds the class in bracket
320 for methods and properties
321 @return a string style, see below
323 String style:
325 ::
327 :%s:`%s <%s>` or
328 :%s:`%s <%s>` (class)
329 """
330 cor = {"function": "func",
331 "method": "meth",
332 "staticmethod": "meth",
333 "property": "meth"}
335 if self.type in ["method", "staticmethod", "property"]:
336 path = "%s.%s.%s" % (self.module, self.cl.__name__, self.name)
337 else:
338 path = "%s.%s" % (self.module, self.name)
340 if prefix is not None:
341 path = "%s.%s" % (prefix, path)
343 if self.type in ["method", "staticmethod",
344 "property"] and class_in_bracket:
345 link = ":%s:`%s <%s>` (%s)" % (
346 cor.get(self.type, self.type), self.name, path, self.cl.__name__)
347 else:
348 link = ":%s:`%s <%s>`" % (
349 cor.get(self.type, self.type), self.name, path)
350 return link
352 @property
353 def classname(self):
354 """
355 Returns the class name if the object is a method.
357 @return class object
358 """
359 if self.type in ["method", "staticmethod", "property"]:
360 return self.cl
361 else:
362 return None
364 def __cmp__(self, oth):
365 """
366 Comparison operators, compares first the first,
367 second the name (lower case).
369 @param oth other object
370 @return -1, 0 or 1
371 """
372 if self.type == oth.type:
373 ln = self.fullpath + "@@@" + self.name.lower()
374 lo = oth.fullpath + "@@@" + oth.name.lower()
375 c = -1 if ln < lo else (1 if ln > lo else 0)
376 if c == 0 and self.type == "method":
377 ln = self.cl.__name__
378 lo = self.cl.__name__
379 c = -1 if ln < lo else (1 if ln > lo else 0)
380 return c
381 else:
382 return - \
383 1 if self.type < oth.type else (
384 1 if self.type > oth.type else 0)
386 def __lt__(self, oth):
387 """
388 Operator ``<``.
389 """
390 return self.__cmp__(oth) == -1
392 def __eq__(self, oth):
393 """
394 Operator ``==``.
395 """
396 return self.__cmp__(oth) == 0
398 def __gt__(self, oth):
399 """
400 Operator ``>``.
401 """
402 return self.__cmp__(oth) == 1
405class IndexInformation:
407 """
408 Keeps some information to index.
409 """
411 def __init__(self, type, label, name, text, rstfile, fullname):
412 """
413 @param type each type gets an index
414 @param label label used to index
415 @param name name to display
416 @param text text to show as a short description
417 @param rstfile tells which file the index refers to (rst file)
418 @param fullname fullname of a file the rst file describes
419 """
420 self.type = type
421 self.label = label
422 self.name = name
423 self.text = text
424 self.fullname = fullname
425 self.set_rst_file(rstfile)
427 def __str__(self):
428 """
429 usual
430 """
431 return "%s -- %s" % (self.label, self.rst_link())
433 def set_rst_file(self, rstfile):
434 """
435 Sets the rst file and checks the label is present in it.
437 @param rstfile rst file
438 """
439 self.rstfile = rstfile
440 if rstfile is not None:
441 self.add_label_if_not_present()
443 @property
444 def truncdoc(self):
445 """
446 Returns ``self.text``.
447 """
448 return self.text.replace("\n", " ").replace(
449 "\t", "").replace("\r", "")
451 def add_label_if_not_present(self):
452 """
453 The function checks the label is present in the original file.
454 """
455 if self.rstfile is not None:
456 with open(self.rstfile, "r", encoding="utf8") as f:
457 content = f.read()
458 label = ".. _%s:" % self.label
459 if label not in content:
460 content = "\n%s\n%s" % (label, content)
461 with open(self.rstfile, "w", encoding="utf8") as f:
462 f.write(content)
464 @staticmethod
465 def get_label(existing, suggestion):
466 """
467 Returns a new label given the existing ones.
469 @param existing existing labels stored in a dictionary
470 @param suggestion the suggestion will be chosen if it does not exists,
471 ``suggestion + zzz`` otherwise
472 @return string
473 """
474 if existing is None:
475 raise ValueError( # pragma: no cover
476 "existing must not be None")
477 suggestion = suggestion.replace("_", "").replace(".", "")
478 while suggestion in existing:
479 suggestion += "z"
480 return suggestion
482 def rst_link(self):
483 """
484 return a link rst
485 @return rst link
486 """
487 if self.label.startswith("_"):
488 return ":ref:`%s <%s>`" % (self.name, self.label[1:])
489 else:
490 return ":ref:`%s <%s>`" % (self.name, self.label)
493class RstFileHelp:
494 """
495 Defines what a rst file and what it describes.
496 """
498 def __init__(self, file, rst, doc):
499 """
500 @param file original filename
501 @param rst produced rst file
502 @param doc documentation if any
503 """
504 self.file = file
505 self.rst = rst
506 self.doc = doc
509def import_module(rootm, filename, log_function, additional_sys_path=None,
510 first_try=True):
511 """
512 Imports a module using its filename.
514 @param rootm root of the module (for relative import)
515 @param filename file name of the module
516 @param log_function logging function
517 @param additional_sys_path additional path to include to ``sys.path`` before
518 importing a module (will be removed afterwards)
519 @param first_try first call to the function (to avoid infinite loop)
520 @return module object, prefix
522 The function can also import compiled modules.
524 .. warning:: It adds the file path at the first
525 position in ``sys.path`` and then deletes it.
526 """
527 if additional_sys_path is None:
528 additional_sys_path = []
529 memo = copy.deepcopy(sys.path)
530 li = filename.replace("\\", "/")
531 sdir = os.path.abspath(os.path.split(li)[0])
532 relpath = os.path.relpath(li, rootm).replace("\\", "/")
533 if "/" in relpath:
534 spl = relpath.split("/")
535 fmod = spl[0] # this is the prefix
536 relpath = "/".join(spl[1:])
537 else:
538 fmod = ""
540 # has init
541 init_ = os.path.join(sdir, "__init__.py")
542 if init_ != filename and not os.path.exists(init_):
543 # no init
544 return "No __init__.py, unable to import %s" % (filename), fmod
546 # we remove every path ending by "src" except if it is found in PYTHONPATH
547 pythonpath = os.environ.get("PYTHONPATH", None)
548 if pythonpath is not None:
549 sep = ";" if sys.platform.startswith("win") else ":"
550 pypaths = [os.path.normpath(_)
551 for _ in pythonpath.split(sep) if len(_) > 0]
552 else:
553 pypaths = []
554 rem = []
555 for i, p in enumerate(sys.path):
556 if (p.endswith("src") and p not in pypaths) or ".zip" in p:
557 rem.append(i)
558 rem.reverse()
559 for r in rem:
560 del sys.path[r]
562 # Extracts extended extension of the module.
563 if li.endswith(".py"):
564 cpxx = ".py"
565 ext_rem = ".py"
566 elif li.endswith(".pyd"):
567 cpxx = ".cp%d%d-" % sys.version_info[:2]
568 search = li.rfind(cpxx)
569 ext_rem = li[search:]
570 else:
571 cpxx = ".cpython-%d%dm-" % sys.version_info[:2]
572 search = li.rfind(cpxx)
573 if search == -1:
574 cpxx = ".cpython-%d%d-" % sys.version_info[:2]
575 search = li.rfind(cpxx)
576 if search == -1:
577 raise ImportErrorHelpGen(
578 "Unable to guess extension from '{}'.".format(li))
579 ext_rem = li[search:]
580 if not ext_rem:
581 raise ValueError( # pragma: no cover
582 "Unable to guess file extension '{0}'".format(li))
583 if ext_rem != ".py":
584 log_function("[import_module] found extension='{0}'".format(ext_rem))
586 # remove fmod from sys.modules
587 if fmod:
588 addback = []
589 rem = []
590 for n, m in sys.modules.items():
591 if n.startswith(fmod):
592 rem.append(n)
593 addback.append((n, m))
594 else:
595 addback = []
596 relpath.replace(ext_rem, "")
597 rem = []
598 for n, m in sys.modules.items():
599 if n.startswith(relpath):
600 rem.append(n)
601 addback.append((n, m))
603 # we remove the modules
604 # this line is important to remove all modules
605 # from the sources in folder src and not the modified ones
606 # in the documentation folder
607 for r in rem:
608 del sys.modules[r]
610 # full path
611 if rootm is not None:
612 root = rootm
613 tl = relpath
614 fi = tl.replace(ext_rem, "").replace("/", ".")
615 if fmod:
616 fi = fmod + "." + fi
617 context = None
618 if fi.endswith(".__init__"):
619 fi = fi[:-len(".__init__")]
620 else:
621 root = sdir
622 tl = os.path.split(li)[1]
623 fi = tl.replace(ext_rem, "")
624 context = None
626 if additional_sys_path is not None and len(additional_sys_path) > 0:
627 # there is an issue here due to the confusion in the paths
628 # the paths should be removed just after the import
629 sys.path.extend(additional_sys_path)
631 sys.path.insert(0, root)
632 try:
633 try:
634 mo = importlib.import_module(fi, context)
635 except ImportError: # pragma: no cover
636 log_function(
637 "[import_module] unable to import module '{0}' fullname "
638 "'{1}'".format(fi, filename))
639 mo_spec = importlib.util.find_spec(fi, context)
640 log_function("[import_module] imported spec", mo_spec)
641 mo = mo_spec.loader.load_module()
642 log_function("[import_module] successful try", mo_spec)
644 if not mo.__file__.replace("\\", "/").endswith(
645 filename.replace("\\", "/").strip("./")): # pragma: no cover
646 namem = os.path.splitext(os.path.split(filename)[-1])[0]
648 if "src" in sys.path:
649 sys.path = [_ for _ in sys.path if _ != "src"]
651 if namem in sys.modules:
652 del sys.modules[namem]
653 # add the context here for relative import
654 # use importlib.import_module with the package argument filled
655 # mo = __import__ (fi)
656 try:
657 mo = importlib.import_module(fi, context)
658 except ImportError:
659 mo = importlib.util.find_spec(fi, context)
661 if not mo.__file__.replace(
662 "\\", "/").endswith(filename.replace("\\", "/").strip("./")):
663 raise ImportError(
664 "The wrong file was imported (2):\nEXP: {0}\nIMP: {1}\n"
665 "PATHS:\n - {2}".format(
666 filename, mo.__file__, "\n - ".join(sys.path)))
667 else:
668 raise ImportError(
669 "The wrong file was imported (1):\nEXP: {0}\nIMP: {1}\n"
670 "PATHS:\n - {2}".format(
671 filename, mo.__file__, "\n - ".join(sys.path)))
673 sys.path = memo
674 log_function("[import_module] import '{0}' successfully".format(
675 filename), mo.__file__)
676 for n, m in addback:
677 if n not in sys.modules:
678 sys.modules[n] = m
679 return mo, fmod
681 except ImportError as e: # pragma: no cover
682 exp = re.compile("No module named '(.*)'")
683 find = exp.search(str(e))
684 if find:
685 module = find.groups()[0]
686 log_function(
687 "[warning] unable to import module " + module +
688 " --- " + str(e).replace("\n", " "))
690 log_function(" File \"%s\", line %d" % (__file__, 501))
691 log_function("[warning] -- unable to import module (1) ", filename,
692 ",", fi, " in path ", sdir, " Error: ", str(e))
693 log_function(" cwd ", os.getcwd())
694 log_function(" path", sdir)
695 stack = traceback.format_exc()
696 log_function(" executable", sys.executable)
697 log_function(" version", sys.version_info)
698 log_function(" stack:\n", stack)
700 message = ["-----", stack, "-----"]
701 message.append(" executable: '{0}'".format(sys.executable))
702 message.append(" version: '{0}'".format(sys.version_info))
703 message.append(" platform: '{0}'".format(sys.platform))
704 message.append(" ext_rem='{0}'".format(ext_rem))
705 message.append(" fi='{0}'".format(fi))
706 message.append(" li='{0}'".format(li))
707 message.append(" cpxx='{0}'".format(cpxx))
708 message.append("-----")
709 for p in sys.path:
710 message.append(" path: " + p)
711 message.append("-----")
712 for p in sorted(sys.modules):
713 try:
714 m = sys.modules[p].__path__
715 except AttributeError:
716 m = str(sys.modules[p])
717 message.append(" module: {0}={1}".format(p, m))
719 sys.path = memo
720 for n, m in addback:
721 if n not in sys.modules:
722 sys.modules[n] = m
724 if 'File "<frozen importlib._bootstrap>"' in stack:
725 raise ImportErrorHelpGen(
726 "frozen importlib._bootstrap is an issue:\n" + "\n".join(message)) from e
728 return "Unable(1) to import %s\nError:\n%s" % (filename, str(e)), fmod
730 except SystemError as e: # pragma: no cover
731 log_function("[warning] -- unable to import module (2) ", filename,
732 ",", fi, " in path ", sdir, " Error: ", str(e))
733 stack = traceback.format_exc()
734 log_function(" executable", sys.executable)
735 log_function(" version", sys.version_info)
736 log_function(" stack:\n", stack)
737 sys.path = memo
738 for n, m in addback:
739 if n not in sys.modules:
740 sys.modules[n] = m
741 return "unable(2) to import %s\nError:\n%s" % (filename, str(e)), fmod
743 except KeyError as e: # pragma: no cover
744 if first_try and "KeyError: 'pip._vendor.urllib3.contrib'" in str(e):
745 # Issue with pip 9.0.2
746 return import_module(rootm=rootm, filename=filename, log_function=log_function,
747 additional_sys_path=additional_sys_path,
748 first_try=False)
749 else:
750 log_function("[warning] -- unable to import module (4) ", filename,
751 ",", fi, " in path ", sdir, " Error: ", str(e))
752 stack = traceback.format_exc()
753 log_function(" executable", sys.executable)
754 log_function(" version", sys.version_info)
755 log_function(" stack:\n", stack)
756 sys.path = memo
757 for n, m in addback:
758 if n not in sys.modules:
759 sys.modules[n] = m
760 return "unable(4) to import %s\nError:\n%s" % (filename, str(e)), fmod
762 except Exception as e: # pragma: no cover
763 log_function("[warning] -- unable to import module (3) ", filename,
764 ",", fi, " in path ", sdir, " Error: ", str(e))
765 stack = traceback.format_exc()
766 log_function(" executable", sys.executable)
767 log_function(" version", sys.version_info)
768 log_function(" stack:\n", stack)
769 sys.path = memo
770 for n, m in addback:
771 if n not in sys.modules:
772 sys.modules[n] = m
773 return "unable(3) to import %s\nError:\n%s" % (filename, str(e)), fmod
776def get_module_objects(mod):
777 """
778 Gets all the classes from a module.
780 @param mod module objects
781 @return list of ModuleMemberDoc
782 """
784 # exp = { "__class__":"",
785 # "__dict__":"",
786 # "__doc__":"",
787 # "__format__":"",
788 # "__reduce__":"",
789 # "__reduce_ex__":"",
790 # "__subclasshook__":"",
791 # "__dict__":"",
792 # "__weakref__":""
793 # }
795 cl = []
796 for _, obj in inspect.getmembers(mod):
797 try:
798 stobj = str(obj)
799 except RuntimeError: # pragma: no cover
800 # One issue met in werkzeug
801 # Working outside of request context.
802 stobj = ""
803 if (inspect.isclass(obj) or inspect.isfunction(obj) or
804 inspect.isgenerator(obj) or inspect.ismethod(obj) or
805 ("built-in function" in stobj and not isinstance(obj, dict))):
806 cl.append(ModuleMemberDoc(obj, module=mod))
807 if inspect.isclass(obj):
808 for n, o in inspect.getmembers(obj):
809 try:
810 ok = ModuleMemberDoc(
811 o, "method", cl=obj, name=n, module=mod)
812 if ok.module is not None:
813 cl.append(ok)
814 except Exception as e:
815 if str(e).startswith("S/"):
816 raise e # pragma: no cover
818 res = []
819 for _ in cl:
820 try:
821 # if _.module != None :
822 if _.module == mod.__name__:
823 res.append(_)
824 except Exception: # pragma: no cover
825 pass
827 res.sort()
828 return res
831def process_var_tag(
832 docstring, rst_replace=False, header=None):
833 """
834 Processes a docstring using tag ``@ var``, and return a list of 2-tuple::
836 @ var filename file name
837 @ var utf8 decode in utf8?
838 @ var errors decoding in utf8 can raise some errors
840 @param docstring string
841 @param rst_replace if True, replace the var bloc var a rst bloc
842 @param header header for the table, if None, ``["attribute", "meaning"]``
843 @return a matrix with two columns or a string if rst_replace is True
845 """
846 from pandas import DataFrame
848 if header is None:
849 header = ["attribute", "meaning"]
851 reg = re.compile("[@]var +([_a-zA-Z][a-zA-Z0-9_]*?) +((?:(?!@var).)+)")
853 indent = len(docstring)
854 spl = docstring.split("\n")
855 docstring = []
856 bigrow = ""
857 for line in spl:
858 if len(line.strip("\r \t")) == 0:
859 docstring.append(bigrow)
860 bigrow = ""
861 else:
862 ind = len(line) - len(line.lstrip(" "))
863 indent = min(ind, indent)
864 bigrow += line + "\n"
865 if len(bigrow) > 0:
866 docstring.append(bigrow)
868 values = []
869 if rst_replace:
870 for line in docstring:
871 line2 = line.replace("\n", " ")
872 if "@var" in line2:
873 all = reg.findall(line2)
874 val = []
875 for a in all:
876 val.append(list(a))
877 if len(val) > 0:
878 tbl = DataFrame(columns=header, data=val)
879 rst = df2rst(tbl, list_table=True)
880 if indent > 0:
881 rst = "\n".join((" " * indent) +
882 _ for _ in rst.split("\n"))
883 values.append(rst)
884 else:
885 values.append(line)
886 return "\n".join(values)
887 else:
888 for line in docstring:
889 line = line.replace("\n", " ")
890 if "@var" in line:
891 alls = reg.findall(line)
892 for a in alls:
893 values.append(a)
894 return values
897def make_label_index(title, comment):
898 """
899 Builds a :epkg:`sphinx` label from a string by
900 removing any odd characters.
902 @param title title
903 @param comment add this string in the exception when it raises one
904 @return label
905 """
906 def accept(c):
907 if "a" <= c <= "z":
908 return c
909 if "A" <= c <= "Z":
910 return c
911 if "0" <= c <= "9":
912 return c
913 if c in "-_":
914 return c
915 return ""
917 try:
918 r = "".join(map(accept, title))
919 if len(r) == 0:
920 typstr = str
921 raise HelpGenException(
922 "Unable to interpret this title (empty?): {0} (type: {2})\n"
923 "COMMENT:\n{1}".format(
924 typstr(title), comment, typstr(type(title))))
925 return r
926 except TypeError as e: # pragma: no cover
927 typstr = str
928 raise HelpGenException(
929 "Unable to interpret this title: {0} (type: {2})\nCOMMENT:"
930 "\n{1}".format(
931 typstr(title), comment, typstr(type(title)))) from e
934def process_look_for_tag(tag, title, files):
935 """
936 Looks for specific information in all files, collect them
937 into one single page.
939 @param tag tag
940 @param title title of the page
941 @param files list of files to look for
942 @return a list of tuple (page, content of the page)
944 The function is looking for regular expression::
946 .. tag(...).
947 ...
948 .. endtag.
950 They can be split into several pages::
952 .. tag(page::...).
953 ...
954 .. endtag.
956 If the extracted example contains an image (..image:: ../../), the path
957 is fixed too.
959 The function parses the files instead of loading the files as a module.
960 The function needs to replace ``\\\\`` by ``\\``, it does not takes into
961 acount doc string starting with ``r'''``.
962 The function calls @see fn remove_some_indent
963 with ``backslash=True`` to replace double backslashes
964 by simple backslashes.
965 """
966 def noneempty(a):
967 if "___" in a:
968 page, b = a.split("___")
969 return "_" + page, b.lower(), b
970 return "", a.lower(), a
971 repl = "__!LI!NE!__"
972 exp = re.compile(
973 "[.][.] %s[(](.*?);;(.*?)[)][.](.*?)[.][.] end%s[.]" % (tag, tag))
974 exp2 = re.compile(
975 "[.][.] %s[(](.*?)[)][.](.*?)[.][.] end%s[.]" % (tag, tag))
976 coll = []
977 for file in files:
978 if file.file is None:
979 continue
980 if "utils_sphinx_doc.py" in file.file:
981 continue
982 if file.file.endswith(".py"):
983 try:
984 with open(file.file, "r", encoding="utf8") as f:
985 content = f.read()
986 except Exception:
987 with open(file.file, "r") as f:
988 content = f.read()
989 content = content.replace("\n", repl)
990 else:
991 content = "Binary file."
993 all = exp.findall(content)
994 all2 = exp2.findall(content)
995 if len(all2) > len(all):
996 raise HelpGenException( # pragma: no cover
997 "An issue was detected in file %r." % file.file)
999 coll += [noneempty(a) +
1000 (fix_image_page_for_root(c.replace(repl, "\n"), file), b)
1001 for a, b, c in all]
1003 coll.sort()
1004 coll = [(_[0],) + _[2:] for _ in coll]
1006 pages = set(_[0] for _ in coll)
1008 pagerows = []
1010 for page in pages:
1011 if page == "":
1012 tit = title
1013 suf = ""
1014 else:
1015 tit = title + ": " + page.strip("_")
1016 suf = page.replace(" ", "").replace("_", "")
1017 suf = re.sub(r'([^a-zA-Z0-9_])', "", suf)
1018 page = re.sub(r'([^a-zA-Z0-9_])', "", page)
1020 rows = ["""
1021 .. _l-{0}{3}:
1023 {1}
1024 {2}
1026 .. contents::
1027 :local:
1029 """.replace(" ", "").format(tag, tit, "=" * len(tit), suf)]
1031 not_expected = os.environ.get(
1032 "USERNAME", os.environ.get("USER", "````````````"))
1033 if not_expected != "jenkins" and not_expected in rows[0]:
1034 raise HelpGenException(
1035 "The title is probably wrong (4): {0}\ntag={1}\ntit={2}\n"
1036 "not_expected='{3}'".format(rows[0], tag, tit, not_expected))
1038 for pa, a, b, c in coll:
1039 pan = re.sub(r'([^a-zA-Z0-9_])', "", pa)
1040 if page != pan:
1041 continue
1042 lindex = make_label_index(a, pan)
1043 rows.append("")
1044 rows.append(".. _lm-{0}:".format(lindex))
1045 rows.append("")
1046 rows.append(a)
1047 rows.append("+" * len(a))
1048 rows.append("")
1049 rows.append(remove_some_indent(b, backslash=True))
1050 rows.append("")
1051 spl = c.split("-")
1052 d = "file {0}.py".format(spl[1]) # line, spl[2].lstrip("l"))
1053 rows.append("see :ref:`%s <%s>`" % (d, c))
1054 rows.append("")
1056 pagerows.append((page, "\n".join(rows)))
1057 return pagerows
1060def fix_image_page_for_root(content, file):
1061 """
1062 Looks for images and fix their path as
1063 if the extract were copied to the root.
1065 @param content extracted content
1066 @param file file where is comes from (unused)
1067 @return content
1068 """
1069 rows = content.split("\n")
1070 for i in range(len(rows)):
1071 row = rows[i]
1072 if ".. image::" in row:
1073 spl = row.split(".. image::")
1074 img = spl[-1]
1075 if "../images" in img:
1076 img = img.lstrip("./ ")
1077 if len(spl) == 1:
1078 row = ".. image:: " + img
1079 else:
1080 row = spl[0] + ".. image:: " + img
1081 rows[i] = row
1082 return "\n".join(rows)
1085def remove_some_indent(s, backslash=False):
1086 """
1087 Brings text to the left.
1089 @param s text
1090 @param backslash if True, replace double backslash by simple backslash
1091 @return text
1092 """
1093 rows = s.split("\n")
1094 mi = len(s)
1095 for lr in rows:
1096 ll = lr.lstrip()
1097 if len(ll) > 0:
1098 d = len(lr) - len(ll)
1099 mi = min(d, mi)
1101 if mi > 0:
1102 keep = []
1103 for _ in rows:
1104 keep.append(_[mi:] if len(_.strip()) > 0 and len(_) > mi else _)
1105 res = "\n".join(keep)
1106 else:
1107 res = s
1109 if backslash:
1110 res = res.replace("\\\\", "\\")
1111 return res
1114def example_function_latex():
1115 """
1116 This function only contains an example with
1117 latex to check it is working fine.
1119 .. exref::
1120 :title: How to display a formula
1122 We want to check this formula to successfully converted.
1124 :math:`\\left \\{ \\begin{array}{l} \\min_{x,y} \\left \\{ x^2 + y^2 - xy + y \\right \\}
1125 \\\\ \\text{sous contrainte} \\; x + 2y = 1 \\end{array}\\right .`
1127 Brackets and backslashes might be an issue.
1128 """
1129 pass