Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding:utf-8 -*- 

2""" 

3@file 

4@brief This file defines a simple local server delivering generating documentation. 

5""" 

6import sys 

7import os 

8import subprocess 

9import copy 

10import datetime 

11try: 

12 from urllib.parse import urlparse, parse_qs 

13except ImportError: # pragma: no cover 

14 from urlparse import urlparse, parse_qs 

15from threading import Thread 

16try: 

17 from http.server import BaseHTTPRequestHandler, HTTPServer 

18except ImportError: # pragma: no cover 

19 from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer 

20 

21if __name__ == "__main__": # pragma: no cover 

22 path_ = os.path.normpath(os.path.abspath( 

23 os.path.join(os.path.split(__file__)[0], "..", "..", "..", "src"))) 

24 if path_ not in sys.path: 

25 sys.path.append(path_) 

26 path_ = os.path.normpath(os.path.abspath(os.path.join(os.path.split(__file__)[0], 

27 "..", "..", "..", "..", "pyquickhelper", "src"))) 

28 if path_ not in sys.path: 

29 sys.path.append(path_) 

30 from pyquickhelper.loghelper import fLOG, get_url_content 

31else: 

32 from ..loghelper.flog import fLOG 

33 from ..loghelper.url_helper import get_url_content 

34 

35 

36class DocumentationHandler(BaseHTTPRequestHandler): 

37 

38 """ 

39 Define a simple handler used by HTTPServer, 

40 it just serves local content. 

41 

42 """ 

43 

44 mappings = {"__fetchurl__": "http://", 

45 "__shutdown__": "shut://", 

46 } 

47 

48 html_header = """ 

49 <?xml version="1.0" encoding="utf-8"?> 

50 <html> 

51 <head> 

52 <title>%s</title> 

53 </head> 

54 <body> 

55 """.replace(" ", "") 

56 

57 html_footer = """ 

58 </body> 

59 </html> 

60 """.replace(" ", "") 

61 

62 cache = {} 

63 cache_attributes = {} 

64 cache_refresh = datetime.timedelta(1) 

65 

66 def LOG(self, *args, **kwargs): 

67 """ 

68 logging function 

69 """ 

70 fLOG(*args, **kwargs) 

71 

72 @staticmethod 

73 def add_mapping(key, value): 

74 """ 

75 Adds a mapping associated to a local path to watch. 

76 

77 @param key key in ``http://locahost:8008/key/`` 

78 @param value local path 

79 

80 Python documentation says list are protected against 

81 multithreading (concurrent accesses). 

82 If you run the server multiple times, the mappings stays because it 

83 is a static variable. 

84 """ 

85 value = os.path.normpath(value) 

86 if not os.path.exists(value): 

87 raise FileNotFoundError(value) # pragma: no cover 

88 DocumentationHandler.mappings[key] = value 

89 

90 @staticmethod 

91 def get_mappings(): 

92 """ 

93 Returns a copy of the mappings. 

94 

95 @return dictionary of mappings 

96 """ 

97 return copy.copy(DocumentationHandler.mappings) 

98 

99 def __init__(self, request, client_address, server): 

100 """ 

101 Regular constructor, an instance is created for each request, 

102 do not store any data for a longer time than a request. 

103 """ 

104 BaseHTTPRequestHandler.__init__(self, request, client_address, server) 

105 

106 def do_GET(self): 

107 """ 

108 What to do is case of GET request. 

109 """ 

110 parsed_path = urlparse(self.path) 

111 self.serve_content(parsed_path, "GET") 

112 

113 def do_POST(self): 

114 """ 

115 What to do is case of POST request. 

116 """ 

117 parsed_path = urlparse.urlparse(self.path) 

118 self.serve_content(parsed_path) 

119 

120 def do_redirect(self, path="/index.html"): 

121 """ 

122 Redirection when url is just the website. 

123 

124 @param path path to redirect to (a string) 

125 """ 

126 self.send_response(301) 

127 self.send_header('Location', path) 

128 self.end_headers() 

129 

130 media_types = { 

131 ".js": ('application/javascript', 'r'), 

132 ".css": ("text/css", 'r'), 

133 ".html": ('text/html', 'r'), 

134 ".py": ('text/html', 'execute'), 

135 ".png": ('image/png', 'rb'), 

136 ".jpeg": ('image/jpeg', 'rb'), 

137 ".jpg": ('image/jpeg', 'rb'), 

138 ".ico": ('image/x-icon', 'rb'), 

139 ".gif": ('image/gif', 'rb'), 

140 ".eot": ('application/vnd.ms-fontobject', 'rb'), 

141 ".ttf": ('application/font-sfnt', 'rb'), 

142 ".otf": ('font/opentype', 'rb'), 

143 ".svg": ('image/svg+xml', 'r'), 

144 ".woff": ('application/font-wof', 'rb'), 

145 } 

146 

147 @staticmethod 

148 def get_ftype(apath): 

149 """ 

150 defines the header to send (type of files) based on path 

151 @param apath location (a string) 

152 @return htype, ftype (html, css, ...) 

153 

154 If a type is missing, you should look for the ``MIME TYPE`` 

155 on a search engine. 

156 

157 See also `media-types <http://www.iana.org/assignments/media-types/media-types.xhtml>`_ 

158 """ 

159 ext = "." + apath.split(".")[-1] 

160 htype, ftype = DocumentationHandler.media_types.get(ext, ('', '')) 

161 return htype, ftype 

162 

163 def send_headers(self, path): 

164 """ 

165 defines the header to send (type of files) based on path 

166 @param path location (a string) 

167 @return type (html, css, ...) 

168 """ 

169 htype, ftype = self.get_ftype(path) 

170 

171 if htype != '': 

172 self.send_header('Content-type', htype) 

173 self.end_headers() 

174 else: 

175 self.send_header('Content-type', 'text/plain') 

176 self.end_headers() 

177 return ftype 

178 

179 def get_file_content(self, localpath, ftype, path=None): 

180 """ 

181 Returns the content of a local file. 

182 

183 @param localpath local filename 

184 @param ftype r or rb 

185 @param path if != None, the filename will be path/localpath 

186 @return content 

187 

188 This function implements a simple cache mechanism. 

189 """ 

190 if path is not None: 

191 tlocalpath = os.path.join(path, localpath) 

192 else: 

193 tlocalpath = localpath 

194 

195 content = DocumentationHandler.get_from_cache(tlocalpath) 

196 if content is not None: 

197 self.LOG("serves cached", tlocalpath) 

198 return content 

199 

200 if ftype in ("r", "execute"): 

201 if not os.path.exists( 

202 tlocalpath) and "_static/bootswatch" in tlocalpath: 

203 access = tlocalpath.replace("bootswatch", "bootstrap") 

204 else: 

205 access = tlocalpath 

206 

207 if not os.path.exists(access): 

208 self.LOG("** w,unable to find: ", access) 

209 return None 

210 

211 self.LOG("reading file ", access) 

212 with open(access, "r", encoding="utf8") as f: 

213 content = f.read() 

214 DocumentationHandler.update_cache(tlocalpath, content) 

215 return content 

216 else: 

217 if not os.path.exists( 

218 tlocalpath) and "_static/bootswatch" in tlocalpath: 

219 access = tlocalpath.replace("bootswatch", "bootstrap") 

220 else: 

221 access = tlocalpath 

222 

223 if not os.path.exists(access): 

224 self.LOG("** w,unable to find: ", access) 

225 return None 

226 

227 self.LOG("reading file ", access) 

228 with open(tlocalpath, "rb") as f: 

229 content = f.read() 

230 DocumentationHandler.update_cache(tlocalpath, content) 

231 return content 

232 

233 @staticmethod 

234 def get_from_cache(key): 

235 """ 

236 Retrieves a file from the cache if it was cached, 

237 it the file was added later than a day, it returns None. 

238 

239 @param key key 

240 @return content or None if None found or too old 

241 """ 

242 content = DocumentationHandler.cache.get(key, None) 

243 if content is None: 

244 return content 

245 

246 att = DocumentationHandler.cache_attributes[key] 

247 delta = datetime.datetime.now() - att["date"] 

248 if delta > DocumentationHandler.cache_refresh: 

249 del DocumentationHandler.cache[key] 

250 del DocumentationHandler.cache_attributes[key] 

251 return None 

252 else: 

253 DocumentationHandler.cache_attributes[key]["nb"] += 1 

254 return content 

255 

256 @staticmethod 

257 def update_cache(key, content): 

258 """ 

259 Updates the cache. 

260 

261 @param key key 

262 @param content content to place 

263 """ 

264 if len(DocumentationHandler.cache) < 5000: 

265 # we do not clean here as the cache is shared by every session/user 

266 # it would not be safe 

267 # unless we add protection 

268 # self.clean_cache(1000) 

269 pass 

270 

271 # this one first as a document existence is checked by using cache 

272 DocumentationHandler.cache_attributes[key] = {"nb": 1, 

273 "date": datetime.datetime.now()} 

274 DocumentationHandler.cache[key] = content 

275 

276 @staticmethod 

277 def _print_cache(n=20): 

278 """ 

279 Displays the most requested files. 

280 """ 

281 al = [(v["nb"], k) 

282 for k, v in DocumentationHandler.cache_attributes.items() if v["nb"] > 1] 

283 for i, doc in enumerate(sorted(al, reverse=True)): 

284 if i >= n: 

285 break 

286 print("cache: {0} - {1}".format(*doc)) 

287 

288 @staticmethod 

289 def execute(localpath): 

290 """ 

291 Locally executes a python script. 

292 

293 @param localpath local python script 

294 @return output, error 

295 """ 

296 exe = subprocess.Popen([sys.executable, localpath], 

297 stdout=subprocess.PIPE, 

298 stderr=subprocess.PIPE) 

299 out, error = exe.communicate() 

300 return out, error 

301 

302 def feed(self, anys, script_python=False, params=None): 

303 """ 

304 Displays something. 

305 

306 @param anys string 

307 @param script_python if True, the function processes script sections 

308 @param params extra parameters when a script must be executed (should be a dictionary) 

309 

310 A script section looks like: 

311 

312 :: 

313 

314 <script type="text/python"> 

315 from pandas import DataFrame 

316 pars = [ { "key":k, "value":v } for k,v in params ] 

317 tbl = DataFrame (pars) 

318 print ( tbl.to_html(class_table="myclasstable") ) 

319 </script> 

320 

321 The server does not interpret Python, to do that, you need to use 

322 `pyrsslocal <http://www.xavierdupre.fr/app/pyrsslocal/helpsphinx/index.html>`_. 

323 """ 

324 if isinstance(anys, bytes): 

325 if script_python: 

326 raise SystemError( # pragma: no cover 

327 "** w,unable to execute script from bytes") 

328 self.wfile.write(anys) 

329 else: 

330 if script_python: 

331 #any = self.process_scripts(any, params) 

332 raise NotImplementedError( # pragma: no cover 

333 "unable to execute a python script") 

334 text = anys.encode("utf-8") 

335 self.wfile.write(text) 

336 

337 def shutdown(self): 

338 """ 

339 Shuts down the service. 

340 """ 

341 raise NotImplementedError() # pragma: no cover 

342 

343 def serve_content(self, cpath, method="GET"): 

344 """ 

345 Tells what to do based on the path. The function intercepts the 

346 path /localfile/, otherwise it calls ``serve_content_web``. 

347 

348 If you type ``http://localhost:8080/root/file``, 

349 assuming ``root`` is mapped to a local folder. 

350 It will display this file. 

351 

352 @param cpath ParseResult 

353 @param method GET or POST 

354 """ 

355 if cpath.path == "" or cpath.path == "/": 

356 params = parse_qs(cpath.query) 

357 self.serve_main_page() 

358 else: 

359 params = parse_qs(cpath.query) 

360 params["__path__"] = cpath 

361 

362 # fullurl = cpath.geturl() 

363 fullfile = cpath.path 

364 params["__url__"] = cpath 

365 spl = fullfile.strip("/").split("/") 

366 

367 project = spl[0] 

368 link = "/".join(spl[1:]) 

369 value = DocumentationHandler.mappings.get(project, None) 

370 

371 if value is None: 

372 self.LOG("can't serve", cpath) 

373 self.LOG("with params", params) 

374 self.send_response(404) 

375 #raise KeyError("unable to find a mapping associated to: " + project + "\nURL:\n" + url + "\nPARAMS:\n" + str(params)) 

376 

377 elif value == "shut://": 

378 self.LOG("call shutdown") 

379 self.shutdown() 

380 

381 elif value == "http://": 

382 self.send_response(200) 

383 self.send_headers("debug.html") 

384 url = cpath.path.replace("/%s/" % project, "") 

385 try: 

386 content = get_url_content(url) 

387 except Exception as e: # pragma: no cover 

388 content = "<html><body>ERROR (2): %s</body></html>" % e 

389 self.feed(content, False, params={}) 

390 

391 else: 

392 if ".." in link: 

393 # we avoid that case to prevent users from digging others paths 

394 # than the mapped ones, just in that the browser does not 

395 # remove them 

396 self.send_error(404) 

397 self.feed("Requested resource %s unavailable" % link) 

398 else: 

399 # we do not expect the documentation to point to the root 

400 # it must be relative paths 

401 localpath = link.lstrip("/") 

402 if localpath in [None, "/", ""]: 

403 localpath = "index.html" 

404 fullpath = os.path.join(value, localpath) 

405 self.LOG("localpath ", fullpath, os.path.isfile(fullpath)) 

406 

407 self.send_response(200) 

408 _, ftype = self.get_ftype(localpath) 

409 

410 execute = eval(params.get("execute", ["True"])[0]) 

411 spath = params.get("path", [None])[0] 

412 # keep = eval(params.get("keep", ["False"])[0]) 

413 

414 if ftype != 'execute' or not execute: 

415 content = self.get_file_content(fullpath, ftype, spath) 

416 if content is None: 

417 self.LOG("** w,unable to get file for key:", spath) 

418 self.send_error(404) 

419 self.feed( 

420 "Requested resource %s unavailable" % localpath) 

421 else: 

422 ext = os.path.splitext(localpath)[-1].lower() 

423 if ext in [ 

424 ".py", ".c", ".cpp", ".hpp", ".h", ".r", ".sql", ".java"]: 

425 self.send_headers(".html") 

426 self.feed( 

427 DocumentationHandler.html_code_renderer(localpath, content)) 

428 elif ext in [".html"]: 

429 content = DocumentationHandler.process_html_path( 

430 project, content) 

431 self.send_headers(localpath) 

432 self.feed(content) 

433 else: 

434 self.send_headers(localpath) 

435 self.feed(content) 

436 else: 

437 self.LOG("execute file ", localpath) 

438 out, err = DocumentationHandler.execute(localpath) 

439 if len(err) > 0: 

440 self.send_error(404) 

441 self.feed( 

442 "Requested resource %s unavailable" % localpath) 

443 else: 

444 self.send_headers(localpath) 

445 self.feed(out) 

446 

447 @staticmethod 

448 def process_html_path(project, content): 

449 """ 

450 Processes a :epkg:`HTML` content, replaces path which are relative 

451 to the root and not the project. 

452 

453 @param project project, ex: ``pyquickhelper`` 

454 @param content page content 

455 @return modified content 

456 """ 

457 #content = content.replace(' src="',' src="' + project + '/') 

458 #content = content.replace(' href="',' href="' + project + '/') 

459 return content 

460 

461 @staticmethod 

462 def html_code_renderer(localpath, content): 

463 """ 

464 Produces a html code for code. 

465 

466 @param localpath local path to file (local or not) 

467 @param content content of the file 

468 @return html string 

469 """ 

470 res = [DocumentationHandler.html_header % (localpath)] 

471 res.append("<pre class=\"prettyprint\">") 

472 res.append(content.replace("<", "&lt;").replace(">", "&gt;")) 

473 res.append(DocumentationHandler.html_footer) 

474 return "\n".join(res) 

475 

476 def serve_content_web(self, path, method, params): 

477 """ 

478 Functions to overload (executed after serve_content). 

479 

480 @param path ParseResult 

481 @param method GET or POST 

482 @param params params parsed from the url + others 

483 """ 

484 self.send_response(200) 

485 self.send_headers("") 

486 self.feed("** w,unable to serve content for url: " + 

487 path.geturl() + "\n" + str(params) + "\n") 

488 self.send_error(404) 

489 

490 def serve_main_page(self): # pragma: no cover 

491 """ 

492 Displays all the mapping for the default path. 

493 """ 

494 rows = ["<html><body>"] 

495 rows.append("<h1>Documentation Server</h1>") 

496 rows.append("<ul>") 

497 for k, _ in sorted(DocumentationHandler.mappings.items()): 

498 if not k.startswith("_"): 

499 row = '<li><a href="{0}/">{0}</a></li>'.format(k) 

500 rows.append(row) 

501 rows.append("</ul></body></html>") 

502 content = "\n".join(rows) 

503 

504 self.send_response(200) 

505 self.send_headers(".html") 

506 self.feed(content) 

507 

508 

509class DocumentationThreadServer (Thread): 

510 

511 """ 

512 defines a thread which holds a web server 

513 

514 @var server the server of run 

515 """ 

516 

517 def __init__(self, server): 

518 """ 

519 @param server to run 

520 """ 

521 Thread.__init__(self) 

522 self.server = server 

523 

524 def run(self): 

525 """ 

526 Runs the server. 

527 """ 

528 self.server.serve_forever() 

529 

530 def shutdown(self): 

531 """ 

532 Shuts down the server, if it does not work, you can still kill 

533 the thread: 

534 

535 :: 

536 

537 self.kill() 

538 """ 

539 self.server.shutdown() 

540 self.server.server_close() 

541 

542 

543def run_doc_server(server, mappings, thread=False, port=8079): 

544 """ 

545 Runs the server. 

546 

547 @param server if None, it becomes ``HTTPServer(('localhost', 8080), DocumentationHandler)`` 

548 @param mappings prefixes with local folders (dictionary) 

549 @param thread if True, the server is run in a thread 

550 and the function returns right away, 

551 otherwise, it runs the server. 

552 @param port port to use 

553 @return server if thread is False, the thread otherwise (the thread is started) 

554 

555 .. faqref:: 

556 :title: How to run a local server which serves the documentation? 

557 

558 The following code will create a local server: `http://localhost:8079/pyquickhelper/ <http://localhost:8079/pyquickhelper/>`_. 

559 

560 :: 

561 

562 this_fold = os.path.dirname(pyquickhelper.serverdoc.documentation_server.__file__) 

563 this_path = os.path.abspath( os.path.join( this_fold, 

564 "..", "..", "..", "dist", "html") ) 

565 run_doc_server(None, mappings = { "pyquickhelper": this_path } ) 

566 

567 The same server can serves more than one project. 

568 More than one mappings can be sent. 

569 """ 

570 for k, v in mappings.items(): 

571 DocumentationHandler.add_mapping(k, v) 

572 

573 if server is None: 

574 server = HTTPServer(('localhost', port), DocumentationHandler) 

575 elif isinstance(server, str): 

576 server = HTTPServer((server, port), DocumentationHandler) 

577 elif not isinstance(server, HTTPServer): 

578 raise TypeError( # pragma: no cover 

579 "unexpected type for server: " + str(type(server))) 

580 

581 if thread: 

582 th = DocumentationThreadServer(server) 

583 th.start() 

584 return th 

585 else: # pragma: no cover 

586 server.serve_forever() 

587 return server 

588 

589 

590if __name__ == '__main__': # pragma: no cover 

591 

592 run_server = True 

593 if run_server: 

594 # http://localhost:8079/pyquickhelper/ 

595 this_fold = os.path.abspath(os.path.dirname(__file__)) 

596 this_fold2 = os.path.join( 

597 this_fold, "..", "..", "..", "..", "ensae_teaching_cs", "dist", "html3") 

598 this_fold = os.path.join(this_fold, "..", "..", "..", "dist", "html") 

599 fLOG(OutputPrint=True) 

600 fLOG("running server") 

601 run_doc_server(None, mappings={"pyquickhelper": this_fold, 

602 "ensae_teaching_cs": this_fold2}) 

603 fLOG("end running server")