Source code for pyquickhelper.helpgen.sphinx_main_helper

# -*- coding: utf-8 -*-
Contains helpers for the main function :func:`generate_help_sphinx <pyquickhelper.helpgen.sphinx_main.generate_help_sphinx>`.

import os
import sys
import datetime
import shutil
import subprocess
import re

from ..loghelper import run_cmd, RunCmdException, fLOG
from ..loghelper.run_cmd import parse_exception_message
from ..loghelper.pyrepo_helper import SourceRepository
from ..pandashelper.tblformat import df2rst
from ..filehelper import explore_folder_iterfile
from .utils_sphinx_doc_helpers import HelpGenException
from .post_process import post_process_latex_output
from .process_notebooks import find_pdflatex

template_examples = """

List of programs

.. toctree::
   :maxdepth: 2

.. autosummary::
   :toctree: %s/
   :template: modules.rst

Another list


[docs]def setup_environment_for_help(fLOG=fLOG): """ Modifies environment variables to be able to use external tools such as :epkg:`Inkscape`. :githublink:`%|py|46` """ if sys.platform.startswith("win"): prog = os.environ["ProgramFiles"] inkscape = os.path.join(prog, "Inkscape") if not os.path.exists(inkscape): raise FileNotFoundError( "Inkscape is not installed, expected at: {0}".format(inkscape)) path = os.environ["PATH"] if inkscape not in path: fLOG("SETUP: add path to %path%", inkscape) os.environ["PATH"] = path + ";" + inkscape else: pass
[docs]def get_executables_path(): """ Returns the paths to :epkg:`Python`, :epkg:`Python` Scripts. :return: a list of paths :githublink:`%|py|67` """ res = [os.path.split(sys.executable)[0]] res.extend([os.path.join(res[-1], "Scripts"), os.path.join(res[-1], "bin")]) return res
[docs]def my_date_conversion(sdate): """ Converts a date into a datetime. :param sdate: string :return: date :githublink:`%|py|80` """ first = sdate.split(" ")[0] trois = first.replace(".", "-").replace("/", "-").split("-") return datetime.datetime(int(trois[0]), int(trois[1]), int(trois[2]))
[docs]def produce_code_graph_changes(df): """ Returns the code for a graph which counts the number of changes per week over the last year. :param df: dataframe (has a column date with format ``YYYY-MM-DD``) :return: graph The call to :epkg:`datetime.datetime.strptime` introduced exceptions:: File "<frozen importlib._bootstrap>", line 2212, in _find_and_load_unlocked File "<frozen importlib._bootstrap>", line 321, in _call_with_frames_removed File "<frozen importlib._bootstrap>", line 2254, in _gcd_import File "<frozen importlib._bootstrap>", line 2237, in _find_and_load File "<frozen importlib._bootstrap>", line 2224, in _find_and_load_unlocked when generating the documentation for another project. The reason is still unclear. It was replaced by a custom function. :githublink:`%|py|105` """ def year_week(x): dt = datetime.datetime(x.year, x.month, return dt.isocalendar()[:2] def to_str(x): year, week = year_week(x) return "%d-w%02d" % (year, week) df = df.copy() df["dt"] = df.apply(lambda r: my_date_conversion(r["date"]), axis=1) df = df[["dt"]] now = last = now - datetime.timedelta(365) df = df[df.dt >= last] df["week"] = df['dt'].apply(to_str) df["commits"] = 1 val = [] for alldays in range(0, 365): a = now - datetime.timedelta(alldays) val.append({"dt": a, "week": to_str(a), "commits": 0}) # we move pandas here because it imports matplotlib # which is not always wise when you need to modify the backend import pandas df = pandas.concat([df, pandas.DataFrame(val)], sort=True) gr = df[["week", "commits"]].groupby("week", as_index=False).sum() xl = list(gr["week"]) x = list(range(len(xl))) y = list(gr["commits"]) typstr = str code = """ import matplotlib.pyplot as plt x = __X__ y = __Y__ xl = __XL__ plt.close('all')'ggplot') fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10, 4)), y) tig = ax.get_xticks() labs = [] for t in tig: if t in x: labs.append(xl[x.index(t)]) else: labs.append("") ax.set_xticklabels(labs) ax.grid(True) ax.set_title("commits") """.replace(" ", "") \ .replace("__X__", typstr(x)) \ .replace("__XL__", typstr(xl)) \ .replace("__Y__", typstr(y)) return code
[docs]def generate_changes_repo(chan, source, exception_if_empty=True, filter_commit=lambda c: c.strip() != "documentation", fLOG=fLOG, modify_commit=None): """ Generates a :epkg:`RST` tables containing the changes stored by a :epkg:`SVN` or :epkg:`GIT` repository, the outcome is stored in a file. The log comment must start with ``*`` to be taken into account. :param chan: filename to write (or None if you don't need to) :param source: source folder to get changes for :param exception_if_empty: raises an exception if empty :param filter_commit: function which accepts a commit to show on the documentation (based on the comment) :param fLOG: logging function :param modify_commit: function which rewrite the commit text (see below) :return: string (rst tables with the changes) :epkg:`pandas` is not imported in the function itself but at the beginning of the module. It seemed to cause soe weird exceptions when generating the documentation for another module:: File "<frozen importlib._bootstrap>", line 2212, in _find_and_load_unlocked File "<frozen importlib._bootstrap>", line 321, in _call_with_frames_removed File "<frozen importlib._bootstrap>", line 2254, in _gcd_import File "<frozen importlib._bootstrap>", line 2237, in _find_and_load File "<frozen importlib._bootstrap>", line 2224, in _find_and_load_unlocked Doing that helps. The cause still remains obscure. If not None, function *modify_commit* is called the following way (see below). *nbch* is the commit number. *date* can be returned as a datetime or a string. :: nbch, date, author, comment = modify_commit(nbch, date, author, comment) :githublink:`%|py|201` """ # builds the changes files try: src = SourceRepository(commandline=True) logs = src.log(path=source) except Exception as eee: if exception_if_empty: fLOG("[sphinxerror]-9 unable to retrieve log from " + source) raise HelpGenException( "unable to retrieve log in " + source + "\n" + str(eee)) from eee logs = [("none", 0,, "-")] fLOG("[sphinxerror]-8", eee) if len(logs) == 0: fLOG("[sphinxerror]-7 unable to retrieve log from " + source) if exception_if_empty: raise HelpGenException("retrieved logs are empty in " + source) else: fLOG("info, retrieved ", len(logs), " commits") rows = [] rows.append( """\n.. _l-changes:\n\n\nChanges\n=======\n\n__CODEGRAPH__\n\nList of recent changes:\n""") typstr = str values = [] for i, row in enumerate(logs): n = len(logs) - i author, nbch, date, comment = row[:4] last = row[-1] if last.startswith("http"): nbch = "`%s <%s>`_" % (typstr(nbch), last) if filter_commit(comment): if modify_commit is not None: nbch, date, author, comment = modify_commit( nbch, date, author, comment) if isinstance(date, datetime.datetime): ds = "%04d-%02d-%02d" % (date.year, date.month, else: ds = date if isinstance(nbch, int): values.append( ["%d" % n, "%04d" % nbch, "%s" % ds, author, comment.strip("*")]) else: values.append( ["%d" % n, "%s" % nbch, "%s" % ds, author, comment.strip("*")]) if len(values) == 0 and exception_if_empty: raise HelpGenException( "Logs were not empty but there was no comment starting with '*' from '{0}'\n".format(source) + "\n".join([typstr(_) for _ in logs])) if len(values) > 0: import pandas tbl = pandas.DataFrame( columns=["#", "change number", "date", "author", "comment"], data=values) rows.append( "\n\n" + df2rst(tbl, list_table=True) + "\n\n") final = "\n".join(rows) if len(values) > 0: code = produce_code_graph_changes(tbl) code = code.split("\n") code = [" " + _ for _ in code] code = "\n".join(code) code = ".. plot::\n" + code + "\n" final = final.replace("__CODEGRAPH__", code) if chan is not None: with open(chan, "w", encoding="utf8") as f: f.write(final) return final
[docs]def compile_latex_output_final(root, latex_path, doall, afile=None, latex_book=False, fLOG=fLOG, custom_latex_processing=None, remove_unicode=False): """ Compiles the :epkg:`latex` documents. :param root: root :param latex_path: path to the compiler :param doall: do more transformation of the latex file before compiling it :param afile: process a specific file :param latex_book: do some customized transformation for a book :param fLOG: logging function :param custom_latex_processing: function which does some post processing of the full latex file :param remove_unicode: remove unicode characters before compiling it .. faqreq: :title: The PDF is corrupted, SVG are not there :epkg:`SVG` graphs are not well processed by the latex compilation. It usually goes through the following instruction: :: \\sphinxincludegraphics{{seance4_projection_population_correction_51_0}.svg} And produces the following error: :: ! LaTeX Error: Unknown graphics extension: .svg. This function does not stop if the latex compilation but if the PDF is corrupted, the log should be checked to see the errors. :githublink:`%|py|310` """ latex_exe = find_pdflatex(latex_path) processed = 0 tried = [] for subfolder in ['latex', 'elatex']: build = os.path.join(root, "_doc", "sphinxdoc", "build", subfolder) if not os.path.exists(build): build = root tried.append(build) for tex in os.listdir(build): if tex.endswith(".tex") and (afile is None or afile in tex): processed += 1 file = os.path.join(build, tex) if doall: # -interaction=batchmode c = '"{0}" "{1}" -max-print-line=900 -buf-size=10000000 -output-directory="{2}"'.format( latex_exe, file, build) else: c = '"{0}" "{1}" -max-print-line=900 -buf-size=10000000 -interaction=nonstopmode -output-directory="{2}"'.format( latex_exe, file, build) fLOG("[compile_latex_output_final] LATEX compilation (c)", c) post_process_latex_output(file, doall, latex_book=latex_book, fLOG=fLOG, custom_latex_processing=custom_latex_processing, remove_unicode=remove_unicode) if sys.platform.startswith("win"): change_path = None else: # On Linux the parameter --output-directory is sometimes ignored. # And it only works from the current directory. change_path = os.path.split(file)[0] try: out, err = run_cmd(c, wait=True, log_error=False, catch_exit=True, communicate=False, tell_if_no_output=120, fLOG=fLOG, prefix_log="[latex] ", change_path=change_path) except Exception as e: # An exception is raised when the return code is an error. We # check that PDF file was written. out, err = parse_exception_message(e) if err is not None and len(err) == 0 and out is not None and "Output written" in out: # The output was produced. We ignore the return code. fLOG("WARNINGS: Latex compilation had warnings:", c) else: raise OSError( "Unable to execute\n{0}".format(c)) from e if len(err) > 0 and "Output written on " not in out: raise HelpGenException( "CMD:\n{0}\n[sphinxerror]-6\n{1}\n---OUT:---\n{2}".format(c, err, out)) # second compilation fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") fLOG("~~~~ LATEX compilation (d)", c) fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") try: out, err = run_cmd( c, wait=True, log_error=False, communicate=False, fLOG=fLOG, tell_if_no_output=600, prefix_log="[latex] ", change_path=change_path) except (subprocess.CalledProcessError, RunCmdException): fLOG("[sphinxerror]-5 LATEX ERROR: check the logs") err = "" out = "" fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") if len(err) > 0 and "Output written on " not in out: raise HelpGenException(err) if processed == 0: raise FileNotFoundError("Unable to find any latex file in folders\n{0}".format( "\n".join(tried)))
[docs]def replace_placeholder_by_recent_blogpost(all_tocs, plist, placeholder, nb_post=5, fLOG=fLOG): """ Replaces a place holder by a list of blog post. :param all_tocs: list of files to look into :param plist: list of blog post :param placeholder: place holder to replace :param nb_post: number of blog post to display :param fLOG: logging function :githublink:`%|py|388` """ def make_link(post): name = os.path.splitext(os.path.split(post.FileName)[-1])[0] s = """<a href="{{ pathto('',1) }}/blog/%s/%s.html">%s - %s</a>""" % ( post.Year, name, post.Date, post.Title) return s end = min(nb_post, len(plist)) for toc in all_tocs: with open(toc, "r", encoding="utf8") as f: content = if placeholder in content: fLOG(" *** update", toc) links = [make_link(post) for post in plist[:end]] content = content.replace(placeholder, "\n<br />".join(links)) with open(toc, "w", encoding="utf8") as f: f.write(content)
_pattern_images = ".*(([.]png)|([.]gif])|([.]jpeg])|([.]jpg])|([.]svg]))$"
[docs]def enumerate_copy_images_for_slides(src, dest, pattern=_pattern_images): """ Copies images, initial intent was for slides, once converted into html, link to images are relative to the folder which contains them, we copy the images from ``_images`` to ``_downloads``. :param src: sources :param dest: destination :param pattern: see :func:`explore_folder_iterfile <pyquickhelper.filehelper.synchelper.explore_folder_iterfile>` :return: enumerator of copied files :githublink:`%|py|421` """ iter = explore_folder_iterfile(src, pattern=pattern) for img in iter: d = os.path.join(dest, os.path.split(img)[-1]) if os.path.exists(d): os.remove(d) shutil.copy(img, dest) yield d
[docs]def format_history(src, dest, format="basic"): """ Formats history based on module `releases <>`_. :param src: source history (file) :param dest: destination (file) .. versionchanged:: 1.7 Parameter *format* was added. :epkg:`Sphinx` extension *release* is no longer used but the formatting is still available. :githublink:`%|py|442` """ with open(src, "r", encoding="utf-8") as f: lines = f.readlines() new_lines = [] if format == "release": tag = None for i in range(0, len(lines)): line = lines[i].rstrip("\r\t\n ") if line.startswith("===") and i > 0: rel = lines[i - 1].rstrip("\r\t\n ") if "." in rel: del new_lines[-1] res = "* :release:`{0}`".format(rel) res = res.replace("(", "<").replace(")", ">") if new_lines[-1].startswith("==="): new_lines.append("") new_lines.append(res) tag = None else: new_lines.append(line) elif len(line) > 0: if line.startswith("**"): ll = line.lower().strip("*") if ll in ('bug', 'bugfix', 'bugfixes'): tag = "bug" elif ll in ('features', 'feature'): tag = "feature" elif ll in ('support', 'support'): tag = "support" else: raise ValueError( "Line {0}, unable to infer tag from '{1}'".format(i, line)) else: nline = line.lstrip("* ") if nline.startswith("`"): if tag is None: tag = 'issue' res = "* :{0}:{1}".format(tag, nline) if new_lines[-1].startswith("==="): new_lines.append("") new_lines.append(res) else: new_lines.append(line) if line.startswith(".. _"): new_lines.append("") elif format == "basic": reg = re.compile("(.*?)`([0-9]+)`:(.*?)[(]([-0-9]{10})[)]") for line in lines: match = if match: gr = match.groups() new_line = "{0}:issue:`{1}`:{2}({3})".format(*gr) new_lines.append(new_line) else: new_lines.append(line.strip("\n\r")) else: raise ValueError("Unexpected value for format '{0}'".format(format)) with open(dest, "w", encoding="utf-8") as f: f.write("\n".join(new_lines))