Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding:utf-8 -*-
2"""
3@file
4@brief This file defines a simple local server delivering generating documentation.
5"""
6import sys
7import os
8import subprocess
9import copy
10import datetime
11try:
12 from urllib.parse import urlparse, parse_qs
13except ImportError: # pragma: no cover
14 from urlparse import urlparse, parse_qs
15from threading import Thread
16try:
17 from http.server import BaseHTTPRequestHandler, HTTPServer
18except ImportError: # pragma: no cover
19 from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
21if __name__ == "__main__": # pragma: no cover
22 path_ = os.path.normpath(os.path.abspath(
23 os.path.join(os.path.split(__file__)[0], "..", "..", "..", "src")))
24 if path_ not in sys.path:
25 sys.path.append(path_)
26 path_ = os.path.normpath(os.path.abspath(os.path.join(os.path.split(__file__)[0],
27 "..", "..", "..", "..", "pyquickhelper", "src")))
28 if path_ not in sys.path:
29 sys.path.append(path_)
30 from pyquickhelper.loghelper import fLOG, get_url_content
31else:
32 from ..loghelper.flog import fLOG
33 from ..loghelper.url_helper import get_url_content
36class DocumentationHandler(BaseHTTPRequestHandler):
38 """
39 Define a simple handler used by HTTPServer,
40 it just serves local content.
42 """
44 mappings = {"__fetchurl__": "http://",
45 "__shutdown__": "shut://",
46 }
48 html_header = """
49 <?xml version="1.0" encoding="utf-8"?>
50 <html>
51 <head>
52 <title>%s</title>
53 </head>
54 <body>
55 """.replace(" ", "")
57 html_footer = """
58 </body>
59 </html>
60 """.replace(" ", "")
62 cache = {}
63 cache_attributes = {}
64 cache_refresh = datetime.timedelta(1)
66 def LOG(self, *args, **kwargs):
67 """
68 logging function
69 """
70 fLOG(*args, **kwargs)
72 @staticmethod
73 def add_mapping(key, value):
74 """
75 Adds a mapping associated to a local path to watch.
77 @param key key in ``http://locahost:8008/key/``
78 @param value local path
80 Python documentation says list are protected against
81 multithreading (concurrent accesses).
82 If you run the server multiple times, the mappings stays because it
83 is a static variable.
84 """
85 value = os.path.normpath(value)
86 if not os.path.exists(value):
87 raise FileNotFoundError(value) # pragma: no cover
88 DocumentationHandler.mappings[key] = value
90 @staticmethod
91 def get_mappings():
92 """
93 Returns a copy of the mappings.
95 @return dictionary of mappings
96 """
97 return copy.copy(DocumentationHandler.mappings)
99 def __init__(self, request, client_address, server):
100 """
101 Regular constructor, an instance is created for each request,
102 do not store any data for a longer time than a request.
103 """
104 BaseHTTPRequestHandler.__init__(self, request, client_address, server)
106 def do_GET(self):
107 """
108 What to do is case of GET request.
109 """
110 parsed_path = urlparse(self.path)
111 self.serve_content(parsed_path, "GET")
113 def do_POST(self):
114 """
115 What to do is case of POST request.
116 """
117 parsed_path = urlparse.urlparse(self.path)
118 self.serve_content(parsed_path)
120 def do_redirect(self, path="/index.html"):
121 """
122 Redirection when url is just the website.
124 @param path path to redirect to (a string)
125 """
126 self.send_response(301)
127 self.send_header('Location', path)
128 self.end_headers()
130 media_types = {
131 ".js": ('application/javascript', 'r'),
132 ".css": ("text/css", 'r'),
133 ".html": ('text/html', 'r'),
134 ".py": ('text/html', 'execute'),
135 ".png": ('image/png', 'rb'),
136 ".jpeg": ('image/jpeg', 'rb'),
137 ".jpg": ('image/jpeg', 'rb'),
138 ".ico": ('image/x-icon', 'rb'),
139 ".gif": ('image/gif', 'rb'),
140 ".eot": ('application/vnd.ms-fontobject', 'rb'),
141 ".ttf": ('application/font-sfnt', 'rb'),
142 ".otf": ('font/opentype', 'rb'),
143 ".svg": ('image/svg+xml', 'r'),
144 ".woff": ('application/font-wof', 'rb'),
145 }
147 @staticmethod
148 def get_ftype(apath):
149 """
150 defines the header to send (type of files) based on path
151 @param apath location (a string)
152 @return htype, ftype (html, css, ...)
154 If a type is missing, you should look for the ``MIME TYPE``
155 on a search engine.
157 See also `media-types <http://www.iana.org/assignments/media-types/media-types.xhtml>`_
158 """
159 ext = "." + apath.split(".")[-1]
160 htype, ftype = DocumentationHandler.media_types.get(ext, ('', ''))
161 return htype, ftype
163 def send_headers(self, path):
164 """
165 defines the header to send (type of files) based on path
166 @param path location (a string)
167 @return type (html, css, ...)
168 """
169 htype, ftype = self.get_ftype(path)
171 if htype != '':
172 self.send_header('Content-type', htype)
173 self.end_headers()
174 else:
175 self.send_header('Content-type', 'text/plain')
176 self.end_headers()
177 return ftype
179 def get_file_content(self, localpath, ftype, path=None):
180 """
181 Returns the content of a local file.
183 @param localpath local filename
184 @param ftype r or rb
185 @param path if != None, the filename will be path/localpath
186 @return content
188 This function implements a simple cache mechanism.
189 """
190 if path is not None:
191 tlocalpath = os.path.join(path, localpath)
192 else:
193 tlocalpath = localpath
195 content = DocumentationHandler.get_from_cache(tlocalpath)
196 if content is not None:
197 self.LOG("serves cached", tlocalpath)
198 return content
200 if ftype in ("r", "execute"):
201 if not os.path.exists(
202 tlocalpath) and "_static/bootswatch" in tlocalpath:
203 access = tlocalpath.replace("bootswatch", "bootstrap")
204 else:
205 access = tlocalpath
207 if not os.path.exists(access):
208 self.LOG("** w,unable to find: ", access)
209 return None
211 self.LOG("reading file ", access)
212 with open(access, "r", encoding="utf8") as f:
213 content = f.read()
214 DocumentationHandler.update_cache(tlocalpath, content)
215 return content
216 else:
217 if not os.path.exists(
218 tlocalpath) and "_static/bootswatch" in tlocalpath:
219 access = tlocalpath.replace("bootswatch", "bootstrap")
220 else:
221 access = tlocalpath
223 if not os.path.exists(access):
224 self.LOG("** w,unable to find: ", access)
225 return None
227 self.LOG("reading file ", access)
228 with open(tlocalpath, "rb") as f:
229 content = f.read()
230 DocumentationHandler.update_cache(tlocalpath, content)
231 return content
233 @staticmethod
234 def get_from_cache(key):
235 """
236 Retrieves a file from the cache if it was cached,
237 it the file was added later than a day, it returns None.
239 @param key key
240 @return content or None if None found or too old
241 """
242 content = DocumentationHandler.cache.get(key, None)
243 if content is None:
244 return content
246 att = DocumentationHandler.cache_attributes[key]
247 delta = datetime.datetime.now() - att["date"]
248 if delta > DocumentationHandler.cache_refresh:
249 del DocumentationHandler.cache[key]
250 del DocumentationHandler.cache_attributes[key]
251 return None
252 else:
253 DocumentationHandler.cache_attributes[key]["nb"] += 1
254 return content
256 @staticmethod
257 def update_cache(key, content):
258 """
259 Updates the cache.
261 @param key key
262 @param content content to place
263 """
264 if len(DocumentationHandler.cache) < 5000:
265 # we do not clean here as the cache is shared by every session/user
266 # it would not be safe
267 # unless we add protection
268 # self.clean_cache(1000)
269 pass
271 # this one first as a document existence is checked by using cache
272 DocumentationHandler.cache_attributes[key] = {"nb": 1,
273 "date": datetime.datetime.now()}
274 DocumentationHandler.cache[key] = content
276 @staticmethod
277 def _print_cache(n=20):
278 """
279 Displays the most requested files.
280 """
281 al = [(v["nb"], k)
282 for k, v in DocumentationHandler.cache_attributes.items() if v["nb"] > 1]
283 for i, doc in enumerate(sorted(al, reverse=True)):
284 if i >= n:
285 break
286 print("cache: {0} - {1}".format(*doc))
288 @staticmethod
289 def execute(localpath):
290 """
291 Locally executes a python script.
293 @param localpath local python script
294 @return output, error
295 """
296 exe = subprocess.Popen([sys.executable, localpath],
297 stdout=subprocess.PIPE,
298 stderr=subprocess.PIPE)
299 out, error = exe.communicate()
300 return out, error
302 def feed(self, anys, script_python=False, params=None):
303 """
304 Displays something.
306 @param anys string
307 @param script_python if True, the function processes script sections
308 @param params extra parameters when a script must be executed (should be a dictionary)
310 A script section looks like:
312 ::
314 <script type="text/python">
315 from pandas import DataFrame
316 pars = [ { "key":k, "value":v } for k,v in params ]
317 tbl = DataFrame (pars)
318 print ( tbl.to_html(class_table="myclasstable") )
319 </script>
321 The server does not interpret Python, to do that, you need to use
322 `pyrsslocal <http://www.xavierdupre.fr/app/pyrsslocal/helpsphinx/index.html>`_.
323 """
324 if isinstance(anys, bytes):
325 if script_python:
326 raise SystemError( # pragma: no cover
327 "** w,unable to execute script from bytes")
328 self.wfile.write(anys)
329 else:
330 if script_python:
331 #any = self.process_scripts(any, params)
332 raise NotImplementedError( # pragma: no cover
333 "unable to execute a python script")
334 text = anys.encode("utf-8")
335 self.wfile.write(text)
337 def shutdown(self):
338 """
339 Shuts down the service.
340 """
341 raise NotImplementedError() # pragma: no cover
343 def serve_content(self, cpath, method="GET"):
344 """
345 Tells what to do based on the path. The function intercepts the
346 path /localfile/, otherwise it calls ``serve_content_web``.
348 If you type ``http://localhost:8080/root/file``,
349 assuming ``root`` is mapped to a local folder.
350 It will display this file.
352 @param cpath ParseResult
353 @param method GET or POST
354 """
355 if cpath.path == "" or cpath.path == "/":
356 params = parse_qs(cpath.query)
357 self.serve_main_page()
358 else:
359 params = parse_qs(cpath.query)
360 params["__path__"] = cpath
362 # fullurl = cpath.geturl()
363 fullfile = cpath.path
364 params["__url__"] = cpath
365 spl = fullfile.strip("/").split("/")
367 project = spl[0]
368 link = "/".join(spl[1:])
369 value = DocumentationHandler.mappings.get(project, None)
371 if value is None:
372 self.LOG("can't serve", cpath)
373 self.LOG("with params", params)
374 self.send_response(404)
375 #raise KeyError("unable to find a mapping associated to: " + project + "\nURL:\n" + url + "\nPARAMS:\n" + str(params))
377 elif value == "shut://":
378 self.LOG("call shutdown")
379 self.shutdown()
381 elif value == "http://":
382 self.send_response(200)
383 self.send_headers("debug.html")
384 url = cpath.path.replace("/%s/" % project, "")
385 try:
386 content = get_url_content(url)
387 except Exception as e: # pragma: no cover
388 content = "<html><body>ERROR (2): %s</body></html>" % e
389 self.feed(content, False, params={})
391 else:
392 if ".." in link:
393 # we avoid that case to prevent users from digging others paths
394 # than the mapped ones, just in that the browser does not
395 # remove them
396 self.send_error(404)
397 self.feed("Requested resource %s unavailable" % link)
398 else:
399 # we do not expect the documentation to point to the root
400 # it must be relative paths
401 localpath = link.lstrip("/")
402 if localpath in [None, "/", ""]:
403 localpath = "index.html"
404 fullpath = os.path.join(value, localpath)
405 self.LOG("localpath ", fullpath, os.path.isfile(fullpath))
407 self.send_response(200)
408 _, ftype = self.get_ftype(localpath)
410 execute = eval(params.get("execute", ["True"])[0])
411 spath = params.get("path", [None])[0]
412 # keep = eval(params.get("keep", ["False"])[0])
414 if ftype != 'execute' or not execute:
415 content = self.get_file_content(fullpath, ftype, spath)
416 if content is None:
417 self.LOG("** w,unable to get file for key:", spath)
418 self.send_error(404)
419 self.feed(
420 "Requested resource %s unavailable" % localpath)
421 else:
422 ext = os.path.splitext(localpath)[-1].lower()
423 if ext in [
424 ".py", ".c", ".cpp", ".hpp", ".h", ".r", ".sql", ".java"]:
425 self.send_headers(".html")
426 self.feed(
427 DocumentationHandler.html_code_renderer(localpath, content))
428 elif ext in [".html"]:
429 content = DocumentationHandler.process_html_path(
430 project, content)
431 self.send_headers(localpath)
432 self.feed(content)
433 else:
434 self.send_headers(localpath)
435 self.feed(content)
436 else:
437 self.LOG("execute file ", localpath)
438 out, err = DocumentationHandler.execute(localpath)
439 if len(err) > 0:
440 self.send_error(404)
441 self.feed(
442 "Requested resource %s unavailable" % localpath)
443 else:
444 self.send_headers(localpath)
445 self.feed(out)
447 @staticmethod
448 def process_html_path(project, content):
449 """
450 Processes a :epkg:`HTML` content, replaces path which are relative
451 to the root and not the project.
453 @param project project, ex: ``pyquickhelper``
454 @param content page content
455 @return modified content
456 """
457 #content = content.replace(' src="',' src="' + project + '/')
458 #content = content.replace(' href="',' href="' + project + '/')
459 return content
461 @staticmethod
462 def html_code_renderer(localpath, content):
463 """
464 Produces a html code for code.
466 @param localpath local path to file (local or not)
467 @param content content of the file
468 @return html string
469 """
470 res = [DocumentationHandler.html_header % (localpath)]
471 res.append("<pre class=\"prettyprint\">")
472 res.append(content.replace("<", "<").replace(">", ">"))
473 res.append(DocumentationHandler.html_footer)
474 return "\n".join(res)
476 def serve_content_web(self, path, method, params):
477 """
478 Functions to overload (executed after serve_content).
480 @param path ParseResult
481 @param method GET or POST
482 @param params params parsed from the url + others
483 """
484 self.send_response(200)
485 self.send_headers("")
486 self.feed("** w,unable to serve content for url: " +
487 path.geturl() + "\n" + str(params) + "\n")
488 self.send_error(404)
490 def serve_main_page(self): # pragma: no cover
491 """
492 Displays all the mapping for the default path.
493 """
494 rows = ["<html><body>"]
495 rows.append("<h1>Documentation Server</h1>")
496 rows.append("<ul>")
497 for k, _ in sorted(DocumentationHandler.mappings.items()):
498 if not k.startswith("_"):
499 row = '<li><a href="{0}/">{0}</a></li>'.format(k)
500 rows.append(row)
501 rows.append("</ul></body></html>")
502 content = "\n".join(rows)
504 self.send_response(200)
505 self.send_headers(".html")
506 self.feed(content)
509class DocumentationThreadServer (Thread):
511 """
512 defines a thread which holds a web server
514 @var server the server of run
515 """
517 def __init__(self, server):
518 """
519 @param server to run
520 """
521 Thread.__init__(self)
522 self.server = server
524 def run(self):
525 """
526 Runs the server.
527 """
528 self.server.serve_forever()
530 def shutdown(self):
531 """
532 Shuts down the server, if it does not work, you can still kill
533 the thread:
535 ::
537 self.kill()
538 """
539 self.server.shutdown()
540 self.server.server_close()
543def run_doc_server(server, mappings, thread=False, port=8079):
544 """
545 Runs the server.
547 @param server if None, it becomes ``HTTPServer(('localhost', 8080), DocumentationHandler)``
548 @param mappings prefixes with local folders (dictionary)
549 @param thread if True, the server is run in a thread
550 and the function returns right away,
551 otherwise, it runs the server.
552 @param port port to use
553 @return server if thread is False, the thread otherwise (the thread is started)
555 .. faqref::
556 :title: How to run a local server which serves the documentation?
558 The following code will create a local server: `http://localhost:8079/pyquickhelper/ <http://localhost:8079/pyquickhelper/>`_.
560 ::
562 this_fold = os.path.dirname(pyquickhelper.serverdoc.documentation_server.__file__)
563 this_path = os.path.abspath( os.path.join( this_fold,
564 "..", "..", "..", "dist", "html") )
565 run_doc_server(None, mappings = { "pyquickhelper": this_path } )
567 The same server can serves more than one project.
568 More than one mappings can be sent.
569 """
570 for k, v in mappings.items():
571 DocumentationHandler.add_mapping(k, v)
573 if server is None:
574 server = HTTPServer(('localhost', port), DocumentationHandler)
575 elif isinstance(server, str):
576 server = HTTPServer((server, port), DocumentationHandler)
577 elif not isinstance(server, HTTPServer):
578 raise TypeError( # pragma: no cover
579 "unexpected type for server: " + str(type(server)))
581 if thread:
582 th = DocumentationThreadServer(server)
583 th.start()
584 return th
585 else: # pragma: no cover
586 server.serve_forever()
587 return server
590if __name__ == '__main__': # pragma: no cover
592 run_server = True
593 if run_server:
594 # http://localhost:8079/pyquickhelper/
595 this_fold = os.path.abspath(os.path.dirname(__file__))
596 this_fold2 = os.path.join(
597 this_fold, "..", "..", "..", "..", "ensae_teaching_cs", "dist", "html3")
598 this_fold = os.path.join(this_fold, "..", "..", "..", "dist", "html")
599 fLOG(OutputPrint=True)
600 fLOG("running server")
601 run_doc_server(None, mappings={"pyquickhelper": this_fold,
602 "ensae_teaching_cs": this_fold2})
603 fLOG("end running server")