Coverage for src/pyquickhelper/benchhelper/benchmark.py : 94%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2@file
3@brief Helpers to benchmark something
4"""
5import os
6from datetime import datetime
7from time import perf_counter
8import pickle
9from ..loghelper import noLOG, CustomLog, fLOGFormat
10from ..loghelper.flog import get_relative_path
11from ..pandashelper import df2rst
12from ..texthelper import apply_template
15class BenchMark:
16 """
17 Class to help benchmarking. You should overwrite method
18 *init*, *bench*, *end*, *graphs*.
19 """
21 def __init__(self, name, clog=None, fLOG=noLOG, path_to_images=".",
22 cache_file=None, pickle_module=None, progressbar=None,
23 **params):
24 """
25 @param name name of the test
26 @param clog @see cl CustomLog or string
27 @param fLOG logging function
28 @param params extra parameters
29 @param path_to_images path to images
30 @param cache_file cache file
31 @param pickle_module pickle or dill if you need to serialize functions
32 @param progressbar relies on *tqdm*, example *tnrange*
34 If *cache_file* is specified, the class will store the results of the
35 method :meth:`bench <pyquickhelper.benchhelper.benchmark.GridBenchMark.bench>`.
36 On a second run, the function load the cache
37 and run modified or new run (in *params_list*).
38 """
39 self._fLOG = fLOG
40 self._name = name
42 if isinstance(clog, CustomLog):
43 self._clog = clog
44 elif clog is None:
45 self._clog = None
46 else:
47 self._clog = CustomLog(clog)
48 self._params = params
49 self._path_to_images = path_to_images
50 self._cache_file = cache_file
51 self._pickle = pickle_module if pickle_module is not None else pickle
52 self._progressbar = progressbar
53 self._tracelogs = []
55 ##
56 # methods to overwrite
57 ##
59 def init(self):
60 """
61 Initialisation. Overwrite this method.
62 """
63 raise NotImplementedError(
64 "It should be overwritten.") # pragma: no cover
66 def bench(self, **params):
67 """
68 Runs the benchmark. Overwrite this method.
70 @param params parameters
71 @return metrics as a dictionary, appendix as a dictionary
73 The results of this method will be cached if a *cache_file* was specified in the constructor.
74 """
75 raise NotImplementedError(
76 "It should be overwritten.") # pragma: no cover
78 def end(self):
79 """
80 Cleans. Overwrites this method.
81 """
82 raise NotImplementedError(
83 "It should be overwritten.") # pragma: no cover
85 def graphs(self, path_to_images):
86 """
87 Builds graphs after the benchmark was run.
89 @param path_to_images path to images
90 @return a list of LocalGraph
92 Every returned graph must contain a function which creates
93 the graph. The function must accepts two parameters *ax* and
94 *text*. Example:
96 ::
98 def local_graph(ax=None, text=True, figsize=(5,5)):
99 vx = ...
100 vy = ...
101 btrys = set(df["_btry"])
102 ymin = df[vy].min()
103 ymax = df[vy].max()
104 decy = (ymax - ymin) / 50
105 colors = cm.rainbow(numpy.linspace(0, 1, len(btrys)))
106 if len(btrys) == 0:
107 raise ValueError("The benchmark is empty.")
108 if ax is None:
109 fig, ax = plt.subplots(1, 1, figsize=figsize)
110 ax.grid(True)
111 for i, btry in enumerate(sorted(btrys)):
112 subset = df[df["_btry"]==btry]
113 if subset.shape[0] > 0:
114 subset.plot(x=vx, y=vy, kind="scatter", label=btry, ax=ax, color=colors[i])
115 if text:
116 tx = subset[vx].mean()
117 ty = subset[vy].mean()
118 ax.text(tx, ty + decy, btry, size='small',
119 color=colors[i], ha='center', va='bottom')
120 ax.set_xlabel(vx)
121 ax.set_ylabel(vy)
122 return ax
123 """
124 return []
126 def uncache(self, cache):
127 """
128 overwrite this method to uncache some previous run
129 """
130 pass
132 ##
133 # end of methods to overwrite
134 ##
136 class LocalGraph:
137 """
138 Information about graphs.
139 """
141 def __init__(self, func_gen, filename=None, title=None, root=None):
142 """
143 @param func_gen function generating the graph
144 @param filename filename
145 @param title title
146 @param root path should be relative to this one
147 """
148 if func_gen is None:
149 raise ValueError("func_gen cannot be None") # pragma: no cover
150 if filename is not None:
151 self.filename = filename
152 if title is not None:
153 self.title = title
154 self.root = root
155 self.func_gen = func_gen
157 def plot(self, ax=None, text=True, **kwargs):
158 """
159 Draws the graph again.
161 @param ax axis
162 @param text add text on the graph
163 @param kwargs additional parameters
164 @return axis
165 """
166 return self.func_gen(ax=ax, text=text, **kwargs)
168 def add(self, name, value):
169 """
170 Adds an attribute.
172 @param name name of the attribute
173 @param value value
174 """
175 setattr(self, name, value)
177 def to_html(self):
178 """
179 Renders as :epkg:`HTML`.
180 """
181 # deal with relatif path.
182 if hasattr(self, "filename"):
183 attr = {}
184 for k in {"title", "alt", "width", "height"}:
185 if k not in attr and hasattr(self, k):
186 attr[k if k != "title" else "alt"] = getattr(self, k)
187 merge = " ".join('{0}="{1}"'.format(k, v)
188 for k, v in attr.items())
189 if self.root is not None:
190 filename = get_relative_path(
191 self.root, self.filename, exists=False, absolute=False)
192 else:
193 filename = self.filename
194 filename = filename.replace("\\", "/")
195 return '<img src="{0}" {1}/>'.format(filename, merge)
196 else:
197 raise NotImplementedError(
198 "only files are allowed") # pragma: no cover
200 def to_rst(self):
201 """
202 Renders as :ekg:`rst`.
203 """
204 # do not consider width or height
205 # deal with relatif path
206 if hasattr(self, "filename"):
207 if self.root is not None:
208 filename = get_relative_path(
209 self.root, self.filename, exists=False, absolute=False)
210 else:
211 filename = self.filename
212 filename = filename.replace("\\", "/")
213 return '.. image:: {0}'.format(filename)
214 else:
215 raise NotImplementedError(
216 "only files are allowed") # pragma: no cover
218 @property
219 def Name(self):
220 """
221 Returns the name of the benchmark.
222 """
223 return self._name
225 def fLOG(self, *args, **kwargs):
226 """
227 Logs something.
228 """
229 self._tracelogs.append(fLOGFormat("\n", *args, **kwargs).strip("\n"))
230 if self._clog:
231 self._clog(*args, **kwargs)
232 if self._fLOG:
233 self._fLOG(*args, **kwargs)
234 if hasattr(self, "_progressbars") and self._progressbars and len(self._progressbars) > 0:
235 br = self._progressbars[-1]
236 br.set_description(fLOGFormat( # pylint: disable=C0207
237 "\n", *args, **kwargs).strip("\n").split("\n")[0])
238 br.refresh()
240 def run(self, params_list):
241 """
242 Runs the benchmark.
244 @param params_list list of dictionaries
245 """
246 if not isinstance(params_list, list):
247 raise TypeError("params_list must be a list") # pragma: no cover
248 for di in params_list:
249 if not isinstance(di, dict):
250 raise TypeError( # pragma: no cover
251 "params_list must be a list of dictionaries")
253 # shared variables
254 cached = {}
255 meta = dict(level="BenchMark", name=self.Name, nb=len(
256 params_list), time_begin=datetime.now())
257 self._metadata = []
258 self._metadata.append(meta)
259 nb_cached = 0
261 # cache
262 def cache_():
263 "local function"
264 if self._cache_file is not None and os.path.exists(self._cache_file):
265 self.fLOG("[BenchMark.run] retrieve cache '{0}'".format(
266 self._cache_file))
267 with open(self._cache_file, "rb") as f:
268 cached.update(self._pickle.load(f))
269 self.fLOG("[BenchMark.run] number of cached run: {0}".format(
270 len(cached["params_list"])))
271 else:
272 if self._cache_file is not None:
273 self.fLOG("[BenchMark.run] cache not found '{0}'".format(
274 self._cache_file))
275 cached.update(dict(metrics=[], appendix=[], params_list=[]))
276 self.uncache(cached)
278 # run
279 def run_(pgar):
280 "local function"
281 nonlocal nb_cached
282 self._metrics = []
283 self._appendix = []
285 self.fLOG("[BenchMark.run] init {0} do".format(self.Name))
286 self.init()
287 self.fLOG("[BenchMark.run] init {0} done".format(self.Name))
288 self.fLOG("[BenchMark.run] start {0}".format(self.Name))
290 for i in pgbar:
291 di = params_list[i]
293 # check the cache
294 if i < len(cached["params_list"]) and cached["params_list"][i] == di:
295 can = True
296 for v in cached.values():
297 if i >= len(v):
298 # cannot cache
299 can = False
300 break
302 if can:
303 # can, it checks a file is present
304 look = "{0}.{1}.clean_cache".format(
305 self._cache_file, cached["metrics"][i]["_btry"])
306 if not os.path.exists(look):
307 can = False
308 self.fLOG(
309 "[BenchMark.run] file '{0}' was not found --> run again.".format(look))
310 if can:
311 self._metrics.append(cached["metrics"][i])
312 self._appendix.append(cached["appendix"][i])
313 self.fLOG(
314 "[BenchMark.run] retrieved cached {0}/{1}: {2}".format(i + 1, len(params_list), di))
315 self.fLOG(
316 "[BenchMark.run] file '{0}' was found.".format(look))
317 nb_cached += 1
318 continue
320 # cache is available
322 # no cache
323 self.fLOG(
324 "[BenchMark.run] {0}/{1}: {2}".format(i + 1, len(params_list), di))
325 dt = datetime.now()
326 cl = perf_counter()
327 tu = self.bench(**di)
328 cl = perf_counter() - cl
330 if isinstance(tu, tuple):
331 tus = [tu]
332 elif isinstance(tu, list):
333 tus = tu
334 else:
335 raise TypeError( # pragma: no cover
336 "return of method bench must be a tuple of a list")
338 # checkings
339 for tu in tus:
340 met, app = tu
341 if len(tu) != 2:
342 raise TypeError( # pragma: no cover
343 "Method run should return a tuple with 2 elements.")
344 if "_btry" not in met:
345 raise KeyError( # pragma: no cover
346 "Metrics should contain key '_btry'.")
347 if "_btry" not in app:
348 raise KeyError( # pragma: no cover
349 "Appendix should contain key '_btry'.")
351 for met, app in tus:
352 met["_date"] = dt
353 dt = datetime.now() - dt
354 if not isinstance(met, dict):
355 raise TypeError( # pragma: no cover
356 "metrics should be a dictionary")
357 if "_time" in met:
358 raise KeyError( # pragma: no cover
359 "key _time should not be the returned metrics")
360 if "_span" in met:
361 raise KeyError( # pragma: no cover
362 "key _span should not be the returned metrics")
363 if "_i" in met:
364 raise KeyError( # pragma: no cover
365 "key _i should not be in the returned metrics")
366 if "_name" in met:
367 raise KeyError( # pragma: no cover
368 "key _name should not be the returned metrics")
369 met["_time"] = cl
370 met["_span"] = dt
371 met["_i"] = i
372 met["_name"] = self.Name
373 self._metrics.append(met)
374 app["_i"] = i
375 self._appendix.append(app)
376 self.fLOG(
377 "[BenchMark.run] {0}/{1} end {2}".format(i + 1, len(params_list), met))
379 def graph_():
380 "local function"
381 self.fLOG("[BenchMark.run] graph {0} do".format(self.Name))
382 self._graphs = self.graphs(self._path_to_images)
383 if self._graphs is None or not isinstance(self._graphs, list):
384 raise TypeError( # pragma: no cover
385 "Method graphs does not return anything.")
386 for tu in self._graphs:
387 if not isinstance(tu, self.LocalGraph):
388 raise TypeError( # pragma: no cover
389 "Method graphs should return a list of LocalGraph.")
390 self.fLOG("[BenchMark.run] graph {0} done".format(self.Name))
391 self.fLOG("[BenchMark.run] Received {0} graphs.".format(
392 len(self._graphs)))
393 self.fLOG("[BenchMark.run] end {0} do".format(self.Name))
394 self.end()
395 self.fLOG("[BenchMark.run] end {0} done".format(self.Name))
396 meta["time_end"] = datetime.now()
397 meta["nb_cached"] = nb_cached
399 # write information about run experiments
400 def final_():
401 "local function"
402 if self._cache_file is not None:
403 self.fLOG("[BenchMark.run] save cache '{0}'".format(
404 self._cache_file))
405 cached = dict(metrics=self._metrics,
406 appendix=self._appendix, params_list=params_list)
407 with open(self._cache_file, "wb") as f:
408 self._pickle.dump(cached, f)
409 for di in self._metrics:
410 look = "{0}.{1}.clean_cache".format(
411 self._cache_file, di["_btry"])
412 with open(look, "w") as f:
413 f.write(
414 "Remove this file if you want to force a new run.")
415 self.fLOG("[BenchMark.run] wrote '{0}'.".format(look))
417 self.fLOG("[BenchMark.run] done.")
419 progress = self._progressbar if self._progressbar is not None else range
420 functions = [cache_, run_, graph_, final_]
421 pgbar0 = progress(0, len(functions))
422 if self._progressbar:
423 self._progressbars = [pgbar0]
424 for i in pgbar0:
425 if i == 1:
426 pgbar = progress(len(params_list))
427 if self._progressbar:
428 self._progressbars.append(pgbar)
429 functions[i](pgbar)
430 if self._progressbar:
431 self._progressbars.pop()
432 else:
433 functions[i]()
435 self._progressbars = None
436 return self._metrics, self._metadata
438 @property
439 def Metrics(self):
440 """
441 Returns the metrics.
442 """
443 if not hasattr(self, "_metrics"):
444 raise KeyError( # pragma: no cover
445 "Method run was not run, no metrics was found.")
446 return self._metrics
448 @property
449 def Metadata(self):
450 """
451 Returns the metrics.
452 """
453 if not hasattr(self, "_metadata"):
454 raise KeyError( # pragma: no cover
455 "Method run was not run, no metadata was found.")
456 return self._metadata
458 @property
459 def Appendix(self):
460 """
461 Returns the metrics.
462 """
463 if not hasattr(self, "_appendix"):
464 raise KeyError( # pragma: no cover
465 "Method run was not run, no metadata was found.")
466 return self._appendix
468 def to_df(self, convert=False, add_link=False, format="html"):
469 """
470 Converts the metrics into a dataframe.
472 @param convert if True, calls method *_convert* on each cell
473 @param add_link add hyperlink
474 @param format format for hyperlinks (html or rst)
475 @return dataframe
476 """
477 import pandas
478 df = pandas.DataFrame(self.Metrics)
479 if convert:
480 for c, d in zip(df.columns, df.dtypes):
481 cols = []
482 for i in range(df.shape[0]):
483 cols.append(self._convert(df, i, c, d, df.loc[i, c]))
484 df[c] = cols
485 col1 = list(sorted(_ for _ in df.columns if _.startswith("_")))
486 col2 = list(sorted(_ for _ in df.columns if not _.startswith("_")))
487 df = df[col1 + col2]
488 if add_link and "_i" in df.columns:
489 if format == "html":
490 if "_btry" in df.columns:
491 df["_btry"] = df.apply(
492 lambda row: '<a href="#{0}">{1}</a>'.format(row["_i"], row["_btry"]), axis=1)
493 df["_i"] = df["_i"].apply(
494 lambda s: '<a href="#{0}">{0}</a>'.format(s))
495 elif format == "rst":
496 if "_btry" in df.columns:
497 df["_btry"] = df.apply(
498 lambda row: ':ref:`{1} <l-{2}-{0}>`'.format(row["_i"], row["_btry"], self.Name), axis=1)
499 df["_i"] = df["_i"].apply(
500 lambda s: ':ref:`{0} <l-{1}-{0}>`'.format(s, self.Name))
501 else:
502 raise ValueError( # pragma: no cover
503 "Format should be rst or html.")
504 return df
506 def meta_to_df(self, convert=False, add_link=False, format="html"):
507 """
508 Converts meta data into a dataframe
510 @param convert if True, calls method *_convert* on each cell
511 @param add_link add hyperlink
512 @param format format for hyperlinks (html or rst)
513 @return dataframe
514 """
515 import pandas
516 df = pandas.DataFrame(self.Metadata)
517 if convert:
518 for c, d in zip(df.columns, df.dtypes):
519 cols = []
520 for i in range(df.shape[0]):
521 cols.append(self._convert(df, i, c, d, df.loc[i, c]))
522 df[c] = cols
523 col1 = list(sorted(_ for _ in df.columns if _.startswith("_")))
524 col2 = list(sorted(_ for _ in df.columns if not _.startswith("_")))
525 if add_link and "_i" in df.columns:
526 if format == "html":
527 if "_btry" in df.columns:
528 df["_btry"] = df.apply(
529 lambda row: '<a href="#{0}">{1}</a>'.format(row["_i"], row["_btry"]), axis=1)
530 df["_i"] = df["_i"].apply(
531 lambda s: '<a href="#{0}">{0}</a>'.format(s))
532 elif format == "rst":
533 if "_btry" in df.columns:
534 df["_btry"] = df.apply(
535 lambda row: ':ref:`{1} <l-{2}-{0}>`'.format(row["_i"], row["_btry"], self.Name), axis=1)
536 df["_i"] = df["_i"].apply(
537 lambda s: ':ref:`{0} <l-{1}-{0}>'.format(s, self.Name))
538 else:
539 raise ValueError( # pragma: no cover
540 "Format should be rst or html.")
541 return df[col1 + col2]
543 def report(self, css=None, template_html=None, template_rst=None, engine="mako", filecsv=None,
544 filehtml=None, filerst=None, params_html=None, title=None, description=None):
545 """
546 Produces a report.
548 @param css css (will take the default one if empty)
549 @param template_html template HTML (:epkg:`mako` or :epkg:`jinja2`)
550 @param template_rst template RST (:epkg:`mako` or :epkg:`jinja2`)
551 @param engine ``'mako``' or '``jinja2'``
552 @param filehtml report will written in this file if not None
553 @param filecsv metrics will be written as a flat table
554 @param filerst metrics will be written as a RST table
555 @param params_html parameter to send to function :epkg:`pandas:DataFrame.to_html`
556 @param title title (Name if any)
557 @param description add a description
558 @return dictionary {format: content}
560 You can define your own template by looking into the default ones
561 defines in this class (see the bottom of this file).
562 By default, HTML and RST report are generated.
563 """
564 if template_html is None:
565 template_html = BenchMark.default_template_html
566 if template_rst is None:
567 template_rst = BenchMark.default_template_rst
568 if css is None:
569 css = BenchMark.default_css
570 if params_html is None:
571 params_html = dict()
572 if title is None:
573 title = self.Name # pragma: no cover
574 if "escape" not in params_html:
575 params_html["escape"] = False
577 for gr in self.Graphs:
578 gr.add("root", os.path.dirname(filehtml))
580 # I don't like that too much as it is not multithreaded.
581 # Avoid truncation.
582 import pandas
584 if description is None:
585 description = "" # pragma: no cover
587 contents = {'df': self.to_df()}
589 # HTML
590 if template_html is not None and len(template_html) > 0:
591 old_width = pandas.get_option('display.max_colwidth')
592 pandas.set_option('display.max_colwidth', None)
593 res = apply_template(template_html, dict(description=description, title=title,
594 css=css, bench=self, params_html=params_html))
595 # Restore previous value.
596 pandas.set_option('display.max_colwidth', old_width)
598 if filehtml is not None:
599 with open(filehtml, "w", encoding="utf-8") as f:
600 f.write(res)
601 contents["html"] = res
603 # RST
604 if template_rst is not None and len(template_rst) > 0:
605 old_width = pandas.get_option('display.max_colwidth')
606 pandas.set_option('display.max_colwidth', None)
608 res = apply_template(template_rst, dict(description=description,
609 title=title, bench=self, df2rst=df2rst))
611 # Restore previous value.
612 pandas.set_option('display.max_colwidth', old_width)
614 with open(filerst, "w", encoding="utf-8") as f:
615 f.write(res)
616 contents["rst"] = res
618 # CSV
619 if filecsv is not None:
620 contents['df'].to_csv(
621 filecsv, encoding="utf-8", index=False, sep="\t")
622 return contents
624 def _convert(self, df, i, col, ty, value):
625 """
626 Converts a value knowing its column, its type
627 into something readable.
629 @param df dataframe
630 @param i line index
631 @param col column name
632 @param ty type
633 @param value value to convert
634 @return value
635 """
636 return value
638 @property
639 def Graphs(self):
640 """
641 Returns images of graphs.
642 """
643 if not hasattr(self, "_graphs"):
644 raise KeyError("unable to find _graphs") # pragma: no cover
645 return self._graphs
647 default_css = """
648 .datagrid table { border-collapse: collapse; border-spacing: 0; width: 100%;
649 table-layout: fixed; font-family: Verdana; font-size: 12px;
650 word-wrap: break-word; }
652 .datagrid thead {
653 cursor: pointer;
654 background: #c9dff0;
655 }
656 .datagrid thead tr th {
657 font-weight: bold;
658 padding: 12px 30px;
659 padding-left: 12px;
660 }
661 .datagrid thead tr th span {
662 padding-right: 10px;
663 background-repeat: no-repeat;
664 text-align: left;
665 }
667 .datagrid tbody tr {
668 color: #555;
669 }
670 .datagrid tbody td {
671 text-align: center;
672 padding: 10px 5px;
673 }
674 .datagrid tbody th {
675 text-align: left;
676 padding: 10px 5px;
677 }
678 """.replace(" ", "")
680 default_template_html = """
681 <html>
682 <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
683 <style>
684 ${css}
685 </style>
686 <body>
687 <h1>${title}</h1>
688 ${description}
689 <ul>
690 <li><a href="#metadata">Metadata</a></li>
691 <li><a href="#metrics">Metrics</a></li>
692 <li><a href="#graphs">Graphs</a></li>
693 <li><a href="#appendix">Appendix</a></li>
694 </ul>
695 <h2 id="metadata">Metadata</h2>
696 <div class="datagrid">
697 ${bench.meta_to_df(convert=True, add_link=True).to_html(**params_html)}
698 </div>
699 <h2 id="metrics">Metrics</h2>
700 <div class="datagrid">
701 ${bench.to_df(convert=True, add_link=True).to_html(**params_html)}
702 </div>
703 % if len(bench.Graphs) > 0:
704 <h2 id="graphs">Graphs</h2>
705 % for gr in bench.Graphs:
706 ${gr.to_html()}
707 % endfor
708 % endif
709 % if len(bench.Appendix) > 0:
710 <h2 id="appendix">Appendix</h2>
711 <div class="appendix">
712 % for met, app in zip(bench.Metrics, bench.Appendix):
713 <h3 id="${app["_i"]}">${app["_btry"]}</h3>
714 <ul>
715 % for k, v in sorted(app.items()):
716 % if isinstance(v, str) and "\\n" in v:
717 <li>I <b>${k}</b>: <pre>${v}</pre></li>
718 % else:
719 <li>I <b>${k}</b>: ${v}</li>
720 % endif
721 % endfor
722 % for k, v in sorted(met.items()):
723 % if isinstance(v, str) and "\\n" in v:
724 <li>M <b>${k}</b>: <pre>${v}</pre></li>
725 % else:
726 <li>M <b>${k}</b>: ${v}</li>
727 % endif
728 % endfor
729 </ul>
730 % endfor
731 % endif
732 </div>
733 </body>
734 </html>
735 """.replace(" ", "")
737 default_template_rst = """
739 .. _lb-${bench.Name}:
741 ${title}
742 ${"=" * len(title)}
744 .. contents::
745 :local:
747 ${description}
749 Metadata
750 --------
752 ${df2rst(bench.meta_to_df(convert=True, add_link=True, format="rst"), index=True, list_table=True)}
754 Metrics
755 --------
757 ${df2rst(bench.to_df(convert=True, add_link=True, format="rst"), index=True, list_table=True)}
759 % if len(bench.Graphs) > 0:
761 Graphs
762 ------
764 % for gr in bench.Graphs:
765 ${gr.to_rst()}
766 % endfor
768 % endif
770 % if len(bench.Appendix) > 0:
772 Appendix
773 --------
775 % for met, app in zip(bench.Metrics, bench.Appendix):
777 .. _l-${bench.Name}-${app["_i"]}:
779 ${app["_btry"]}
780 ${"+" * len(app["_btry"])}
782 % for k, v in sorted(app.items()):
783 % if isinstance(v, str) and "\\n" in v:
784 * I **${k}**:
785 ::
787 ${"\\n ".join(v.split("\\n"))}
789 % else:
790 * M **${k}**: ${v}
791 % endif
792 % endfor
794 % for k, v in sorted(met.items()):
795 % if isinstance(v, str) and "\\n" in v:
796 * **${k}**:
797 ::
799 ${"\\n ".join(v.split("\\n"))}
801 % else:
802 * **${k}**: ${v}
803 % endif
804 % endfor
806 % endfor
807 % endif
808 """.replace(" ", "")