Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2""" 

3@file 

4@brief Helpers to publish the documentation of :epkg:`python` to a website. 

5""" 

6import sys 

7import os 

8from pyquickhelper.filehelper import TransferFTP, FileTreeNode, FolderTransferFTP 

9from pyquickhelper.filehelper.ftp_transfer_files import content_as_binary as pqh_content_as_binary 

10from .teaching_modules import get_teaching_modules 

11 

12 

13def trigger_on_specific_strings(content, filename=None, force_allow=None): 

14 """ 

15 Looks for specific string such as 

16 *USERNAME*, *USERDNSDOMAIN*, *HOMEPATH*, *USERNAME*, *COMPUTERNAME*, 

17 *LOGONSERVER*, *USER*, *HOME*, *LOGNAME* 

18 and returns None if it was found or modifies the content to remove it. 

19 

20 @param content content of a file 

21 @param filename only used when an exception is raised 

22 @param force_allow allow these expressions even if they seem to be credentials 

23 @return modified content 

24 """ 

25 strep = [('somewhere', 'somewhere'), 

26 ('somewhere', 'somewhere')] 

27 for env in ['USERNAME', 'USER']: 

28 if env in os.environ and os.environ[env] != "jenkins": 

29 for sub in ["_data", "GitHub"]: 

30 strep.extend([(r"C:\\%s\\__home_\\%s\\" % (os.environ[env], sub), "somewhere"), 

31 ("C:\\%s\\__home_\\%s\\" % 

32 (os.environ[env], sub), "somewhere"), 

33 ("C:\\%s\\__home_\\%s\\" % 

34 (os.environ[env], sub), "somewhere"), 

35 ("C:%s__home_%s" % 

36 (os.environ[env], sub), "somewhere"), 

37 ("%s__home_%s" % 

38 (os.environ[env], sub), "somewhere") 

39 ]) 

40 for s, b in strep: 

41 if s in content: 

42 content = content.replace(s, b) 

43 

44 if force_allow is None: 

45 force_allow = set() 

46 else: 

47 force_allow = set(force_allow) 

48 lower_content = content.lower() 

49 for st in ["USERNAME", "USERDNSDOMAIN", "HOMEPATH", "USERNAME", 

50 "COMPUTERNAME", "LOGONSERVER", "USER", 'HOME', 'LOGNAME']: 

51 if st in os.environ: 

52 s = os.environ[st].lower() 

53 if s == 'jenkins': 

54 continue 

55 if s in ('administrateur', 'administrator'): 

56 continue 

57 if s not in force_allow and s in lower_content: 

58 raise Exception( 

59 'string {0}:{1} was found in\n File "{2}", line 1'.format(st, s, filename)) 

60 return content 

61 

62 

63def content_as_binary(filename): 

64 """ 

65 Overloads function `content_as_finary 

66 <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/filehelper/ftp_transfer_files.html? 

67 highlight=content_as_binary#pyquickhelper.filehelper.ftp_transfer_files.content_as_binary>`_ from 

68 `pyquickhelper <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/>`_. 

69 Determines if filename is binary or None before transfering it. 

70 

71 @param filename filename 

72 @return boolean 

73 """ 

74 if pqh_content_as_binary(filename): 

75 return True 

76 ff = os.path.split(filename)[-1] 

77 if ff == "footer.html": 

78 return True 

79 return False 

80 

81 

82def text_transform(ftpp, filename, content): 

83 """ 

84 If filename is *rss.xml*, 

85 replaces the string *__BLOG_ROOT__* by *self._root_web*. 

86 

87 @param ftpp object FolderTransferFTP 

88 @param filename filename 

89 @param content content of the file 

90 @return new content 

91 """ 

92 if filename.endswith("rss.xml"): 

93 web = ftpp._root_web 

94 if not web.startswith("http://"): 

95 web = "http://" + web.strip("/") 

96 ftpp.fLOG("[text_transform] replace __BLOG_ROOT__ by ", web) 

97 return content.replace("__BLOG_ROOT__", web) 

98 else: 

99 return content 

100 

101 

102def publish_documentation(docs, ftpsite=None, login=None, password=None, 

103 footer_html=None, content_filter=trigger_on_specific_strings, 

104 is_binary=content_as_binary, force_allow=None, 

105 delay=0.5, exc=False, ftps='FTP', 

106 page_transform=None, fLOG=print): 

107 """ 

108 Publishes the documentation and the setups of a python module on a webiste, 

109 it assumes the modules is organized the same way as :epkg:`pyquickhelper`. 

110 

111 @param docs list of dictionaries (see below) 

112 @param ftpsite something like ``ftp.something.`` 

113 @param login login 

114 @param password password 

115 @param footer_html append this HTML code to any uploaded page (such a javascript code to count the audience) 

116 @param content_filter filter the content of a file (it raises an exception if the result is None), 

117 appies only on text files 

118 @param is_binary a function to tell if a content of a file is binary or not 

119 @param force_allow a file is not published if it contains something which looks like credentials 

120 except if this string is part of *force_allow* 

121 @param delay delay between file transferring (in average) 

122 @param exc raise exception if not able to transfer 

123 @param ftps use protocol FTP, TLS, or SFTP 

124 @param page_transform function which transforms 

125 the page before uploading it, 

126 @see fn text_transform 

127 @param fLOG logging function 

128 

129 *docs* is a list of dictionaries which must contain for each folder 

130 to transfer: 

131 

132 - ``local``: local folder 

133 - ``root_local``: local paths will be related to this root 

134 - ``root_web``: prefix to add to the remote paths 

135 - ``status_file``: a file where the function populates the transfered files and some information about them 

136 

137 A local file is composed by ``<local_root>/<relative_path>``, it 

138 will be uploaded to ``<web_root>/<relative_path>``. 

139 """ 

140 

141 params = {"ftpsite": ftpsite, 

142 "login": login, 

143 "password": password} 

144 

145 nbnone = len([v for k, v in params.items() if v is None or len(v) == 0]) 

146 if nbnone > 0: 

147 raise ValueError( 

148 "One of the following parameters is not specified:\n{0}".format(params)) 

149 

150 nbnone = [v for k, v in params.items() if v is None or len(v) == 0] 

151 if len(nbnone) > 0: 

152 raise Exception("one of the parameters is None:\n" + str(nbnone)) 

153 

154 password = params["password"] 

155 login = params["login"] 

156 ftpsite = params["ftpsite"] 

157 

158 filter_out = "([/\\\\]((moduletoc.html)|(blogtoc.html)|(searchbox.html)))|([.]buildinfo)|([.]pyc)" 

159 

160 ftp = TransferFTP(ftpsite, login, password, ftps=ftps, fLOG=fLOG) 

161 

162 if page_transform is None: 

163 fct_transform = text_transform 

164 else: 

165 

166 def combined_transform(ftpp, filename, content): 

167 text_transform(ftpp, filename, content) 

168 page_transform(ftpp, filename, content) 

169 

170 fct_transform = combined_transform 

171 

172 for project in docs: 

173 

174 fLOG("######################################################################") 

175 for k, v in sorted(project.items()): 

176 fLOG("[publish_documentation] loop {}='{}'".format(k, v)) 

177 

178 location = project["local"] 

179 root_local = project["root_local"] 

180 root_web = project["root_web"] 

181 

182 sfile = project["status_file"] 

183 rootw = project["root_web"] 

184 

185 # documentation + setup 

186 fLOG("[publish_documentation] location='{}'".format(location)) 

187 

188 ftn = FileTreeNode(root_local) 

189 fftp = FolderTransferFTP(ftn, ftp, sfile, root_web=rootw, fLOG=fLOG, footer_html=footer_html, 

190 content_filter=content_filter, is_binary=is_binary, 

191 text_transform=fct_transform, filter_out=filter_out, 

192 force_allow=force_allow, exc=exc) 

193 

194 fftp.start_transfering(delay=delay) 

195 

196 ftn = FileTreeNode(os.path.join(root_local, ".."), 

197 filter=lambda root, path, f, dir: not dir) 

198 fftp = FolderTransferFTP(ftn, ftp, sfile, 

199 root_web=root_web.replace("helpsphinx", ""), fLOG=fLOG, 

200 footer_html=footer_html, content_filter=content_filter, 

201 is_binary=is_binary, text_transform=fct_transform, 

202 filter_out=filter_out) 

203 

204 fftp.start_transfering() 

205 

206 ftp.close() 

207 

208 

209def publish_teachings_to_web(login, ftpsite="ftp.xavierdupre.fr", google_id=None, # pylint: disable=W0102 

210 location="c:\\jenkins\\pymy\\%s\\%s%s\\dist\\%s", 

211 rootw="/www/htdocs/app/%s/%s", 

212 folder_status=".", 

213 layout=[("html", "helpsphinx")], 

214 modules=None, password=None, force_allow=None, 

215 suffix=("_UT_%d%d_std" % sys.version_info[:2],), 

216 delay=0.5, exc=False, exc_transfer=False, 

217 transfer=True, additional_projects=None, 

218 ftps='FTP', page_transform=None, fLOG=print): 

219 """ 

220 Copies the documentation to the website. 

221 

222 @param login login 

223 @param ftpsite ftp site 

224 @param google_id google_id 

225 @param location location of Jenkins build 

226 @param rootw root on ftp site 

227 @param folder_status folder status 

228 @param modules list of modules to publish, if None, use @see fn get_teaching_modules 

229 @param password if None, if will asked 

230 @param layout last part of the folders 

231 @param suffix suffixes to append to the project name 

232 @param force_allow allow to publish files even if they contain these strings 

233 whereas they seem to be credentials 

234 @param delay delay between two files being transferred 

235 @param exc raise exception if not found (True) or skip (False) 

236 @param exc_transfer raise an exception if cannot be transfered 

237 @param transfer starts transfering, otherwise returns the list of 

238 transfering task to do 

239 @param additional_projects additional projects 

240 @param ftps use protocol FTP, TLS, or SFTP 

241 @param page_transform function which transforms a page before uploading it, 

242 @see fn text_transform 

243 @param fLOG logging function 

244 

245 Example of use:: 

246 

247 import sys 

248 import os 

249 from pyquickhelper.filehelper import TransferFTP, FileTreeNode, FolderTransferFTP 

250 from tkinterquickhelper.funcwin import open_window_params 

251 from ensae_teaching_cs.automation.ftp_publish_helper import publish_teachings_to_web 

252 

253 login = "..." 

254 website = "ftp...." 

255 rootw = "/www/htdocs/app/%s/%s" 

256 

257 password = None 

258 

259 publish_teachings_to_web(login, ftpsite=website, 

260 google_id="google_id", 

261 location="<something>\\\\%s\\\\%s%s\\\\dist\\\\%s", 

262 rootw=rootw, 

263 folder_status=os.path.abspath("."), 

264 password=password) 

265 

266 Example of an additional projects: 

267 

268 :: 

269 

270 other_projects = [dict(status_file="status_projects.txt"), 

271 root_local="...", root_web="...")] 

272 """ 

273 if modules is None: 

274 modules = get_teaching_modules() 

275 modules = [m.split(':', maxsplit=1)[0] for m in modules] 

276 

277 if google_id is None: 

278 google_id = "" 

279 else: 

280 footer = """ 

281 <script src="http://www.google-analytics.com/urchin.js" type="text/javascript"></script> 

282 <script type="text/javascript"> 

283 _uacct = "{}"; 

284 urchinTracker(); 

285 </script> 

286 """.format(google_id) 

287 

288 if password is None and transfer: 

289 raise ValueError("password is empty") 

290 

291 location = os.path.abspath(location) 

292 if folder_status is None: 

293 folder_status = os.path.abspath(os.path.dirname(__file__)) 

294 else: 

295 folder_status = os.path.abspath(folder_status) 

296 

297 if not isinstance(suffix, (tuple, list)): 

298 suffix = [suffix] 

299 

300 projects = [] 

301 for module in modules: 

302 fLOG( 

303 "[ensae_teaching_cs] PUBLISH '{0}' - layout '{1}'".format(module, layout)) 

304 for lay in layout: 

305 for suf in suffix: 

306 root = os.path.abspath(location % 

307 (module, module, suf, lay[0])) 

308 if transfer and os.path.exists(root): 

309 break 

310 if transfer and not os.path.exists(root): 

311 if exc: 

312 p = os.path.abspath(location % 

313 (module, module, suffix[0], lay[0])) 

314 raise FileNotFoundError( 

315 "First tried '{0}'\n last tried '{1}'".format(root, p)) 

316 else: 

317 continue 

318 fLOG(" ", root) 

319 rw = rootw % (module, lay[1]) 

320 

321 project = dict(status_file=os.path.join(folder_status, "status_%s.txt" % module), 

322 local=root, root_local=root, root_web=rw) 

323 projects.append(project) 

324 

325 if module == "ensae_teaching_cs": 

326 lay = [_ for _ in layout if _[0] == "html"][0] 

327 if transfer and not os.path.exists(root): 

328 if exc: 

329 raise FileNotFoundError(root) 

330 else: 

331 continue 

332 

333 def _update_path(pth): 

334 for a, b in [("\\build", "\\build3"), 

335 ("\\html", "\\html3"), 

336 ("/build", "/build3"), 

337 ("/html", "/html3")]: 

338 pth = pth.replace(a, b) 

339 return pth 

340 

341 local = _update_path(root) 

342 fLOG("[ensae_teaching_cs] checking folder '{0}'".format(local)) 

343 fLOG("[ensae_teaching_cs] root is '{0}'".format(root)) 

344 if os.path.exists(local): 

345 fLOG("[ensae_teaching_cs] found '{0}'".format(local)) 

346 project = dict(status_file=os.path.join(folder_status, "status_3_%s.txt" % module), 

347 local=local, root_local=local, 

348 root_web=(rootw % (module, lay[1])).replace("_no_clean", "").replace("/helpsphinx", "/helpsphinx3")) 

349 projects.append(project) 

350 project = dict(status_file=os.path.join(folder_status, "status_2_%s.txt" % module), 

351 local=local, root_local=local, 

352 root_web=(rootw % (module, lay[1])).replace("_no_clean", "").replace("/helpsphinx", "/helpsphinx2")) 

353 projects.append(project) 

354 else: 

355 fLOG("[ensae_teaching_cs] looking into DOC1, DOC3") 

356 root1 = root.replace("_UT_", "_DOC1_") 

357 if transfer: 

358 if not os.path.exists(root1): 

359 if exc: 

360 raise FileNotFoundError(root1) 

361 else: 

362 pass 

363 else: 

364 project = dict(status_file=os.path.join(folder_status, "status_doc1_%s.txt" % module), 

365 local=root1, root_local=root1, 

366 root_web=(rootw % (module, lay[1])).replace("_no_clean", "")) 

367 projects.append(project) 

368 

369 root3 = root.replace("_UT_", "_DOC3_").replace("html", "html3") 

370 if transfer: 

371 if not os.path.exists(root3): 

372 if exc: 

373 raise FileNotFoundError(root3) 

374 else: 

375 project = dict(status_file=os.path.join(folder_status, "status_doc3_%s.txt" % module), 

376 local=root3, root_local=root3, 

377 root_web=(rootw % (module, lay[1])).replace("_no_clean", "").replace("/helpsphinx", "/helpsphinx3")) 

378 projects.append(project) 

379 project = dict(status_file=os.path.join(folder_status, "status_doc2_%s.txt" % module), 

380 local=root3, root_local=root3, 

381 root_web=(rootw % (module, lay[1])).replace("_no_clean", "").replace("/helpsphinx", "/helpsphinx2")) 

382 projects.append(project) 

383 

384 # publish 

385 if additional_projects: 

386 for proj in additional_projects: 

387 if 'root_local' not in proj: 

388 if 'folder' not in proj: 

389 raise KeyError( 

390 "Key 'folder' or 'root_local' must be specified in {}.".format(proj)) 

391 proj = proj.copy() 

392 proj['root_local'] = proj['folder'] 

393 if 'folder' not in proj: 

394 proj = proj.copy() 

395 proj['folder'] = proj['root_local'] 

396 if 'local' not in proj: 

397 proj = proj.copy() 

398 proj['local'] = os.path.dirname(proj['root_local']) 

399 projects.append(proj) 

400 

401 if transfer: 

402 publish_documentation(projects, ftpsite=ftpsite, login=login, password=password, 

403 footer_html=footer, force_allow=force_allow, delay=delay, 

404 exc=exc_transfer, ftps=ftps, page_transform=page_transform, 

405 fLOG=fLOG) 

406 return projects