Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Helpers to publish the documentation of :epkg:`python` to a website.
5"""
6import sys
7import os
8from pyquickhelper.filehelper import TransferFTP, FileTreeNode, FolderTransferFTP
9from pyquickhelper.filehelper.ftp_transfer_files import content_as_binary as pqh_content_as_binary
10from .teaching_modules import get_teaching_modules
13def trigger_on_specific_strings(content, filename=None, force_allow=None):
14 """
15 Looks for specific string such as
16 *USERNAME*, *USERDNSDOMAIN*, *HOMEPATH*, *USERNAME*, *COMPUTERNAME*,
17 *LOGONSERVER*, *USER*, *HOME*, *LOGNAME*
18 and returns None if it was found or modifies the content to remove it.
20 @param content content of a file
21 @param filename only used when an exception is raised
22 @param force_allow allow these expressions even if they seem to be credentials
23 @return modified content
24 """
25 strep = [('somewhere', 'somewhere'),
26 ('somewhere', 'somewhere')]
27 for env in ['USERNAME', 'USER']:
28 if env in os.environ and os.environ[env] != "jenkins":
29 for sub in ["_data", "GitHub"]:
30 strep.extend([(r"C:\\%s\\__home_\\%s\\" % (os.environ[env], sub), "somewhere"),
31 ("C:\\%s\\__home_\\%s\\" %
32 (os.environ[env], sub), "somewhere"),
33 ("C:\\%s\\__home_\\%s\\" %
34 (os.environ[env], sub), "somewhere"),
35 ("C:%s__home_%s" %
36 (os.environ[env], sub), "somewhere"),
37 ("%s__home_%s" %
38 (os.environ[env], sub), "somewhere")
39 ])
40 for s, b in strep:
41 if s in content:
42 content = content.replace(s, b)
44 if force_allow is None:
45 force_allow = set()
46 else:
47 force_allow = set(force_allow)
48 lower_content = content.lower()
49 for st in ["USERNAME", "USERDNSDOMAIN", "HOMEPATH", "USERNAME",
50 "COMPUTERNAME", "LOGONSERVER", "USER", 'HOME', 'LOGNAME']:
51 if st in os.environ:
52 s = os.environ[st].lower()
53 if s == 'jenkins':
54 continue
55 if s in ('administrateur', 'administrator'):
56 continue
57 if s not in force_allow and s in lower_content:
58 raise Exception(
59 'string {0}:{1} was found in\n File "{2}", line 1'.format(st, s, filename))
60 return content
63def content_as_binary(filename):
64 """
65 Overloads function `content_as_finary
66 <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/filehelper/ftp_transfer_files.html?
67 highlight=content_as_binary#pyquickhelper.filehelper.ftp_transfer_files.content_as_binary>`_ from
68 `pyquickhelper <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/>`_.
69 Determines if filename is binary or None before transfering it.
71 @param filename filename
72 @return boolean
73 """
74 if pqh_content_as_binary(filename):
75 return True
76 ff = os.path.split(filename)[-1]
77 if ff == "footer.html":
78 return True
79 return False
82def text_transform(ftpp, filename, content):
83 """
84 If filename is *rss.xml*,
85 replaces the string *__BLOG_ROOT__* by *self._root_web*.
87 @param ftpp object FolderTransferFTP
88 @param filename filename
89 @param content content of the file
90 @return new content
91 """
92 if filename.endswith("rss.xml"):
93 web = ftpp._root_web
94 if not web.startswith("http://"):
95 web = "http://" + web.strip("/")
96 ftpp.fLOG("[text_transform] replace __BLOG_ROOT__ by ", web)
97 return content.replace("__BLOG_ROOT__", web)
98 else:
99 return content
102def publish_documentation(docs, ftpsite=None, login=None, password=None,
103 footer_html=None, content_filter=trigger_on_specific_strings,
104 is_binary=content_as_binary, force_allow=None,
105 delay=0.5, exc=False, ftps='FTP',
106 page_transform=None, fLOG=print):
107 """
108 Publishes the documentation and the setups of a python module on a webiste,
109 it assumes the modules is organized the same way as :epkg:`pyquickhelper`.
111 @param docs list of dictionaries (see below)
112 @param ftpsite something like ``ftp.something.``
113 @param login login
114 @param password password
115 @param footer_html append this HTML code to any uploaded page (such a javascript code to count the audience)
116 @param content_filter filter the content of a file (it raises an exception if the result is None),
117 appies only on text files
118 @param is_binary a function to tell if a content of a file is binary or not
119 @param force_allow a file is not published if it contains something which looks like credentials
120 except if this string is part of *force_allow*
121 @param delay delay between file transferring (in average)
122 @param exc raise exception if not able to transfer
123 @param ftps use protocol FTP, TLS, or SFTP
124 @param page_transform function which transforms
125 the page before uploading it,
126 @see fn text_transform
127 @param fLOG logging function
129 *docs* is a list of dictionaries which must contain for each folder
130 to transfer:
132 - ``local``: local folder
133 - ``root_local``: local paths will be related to this root
134 - ``root_web``: prefix to add to the remote paths
135 - ``status_file``: a file where the function populates the transfered files and some information about them
137 A local file is composed by ``<local_root>/<relative_path>``, it
138 will be uploaded to ``<web_root>/<relative_path>``.
139 """
141 params = {"ftpsite": ftpsite,
142 "login": login,
143 "password": password}
145 nbnone = len([v for k, v in params.items() if v is None or len(v) == 0])
146 if nbnone > 0:
147 raise ValueError(
148 "One of the following parameters is not specified:\n{0}".format(params))
150 nbnone = [v for k, v in params.items() if v is None or len(v) == 0]
151 if len(nbnone) > 0:
152 raise Exception("one of the parameters is None:\n" + str(nbnone))
154 password = params["password"]
155 login = params["login"]
156 ftpsite = params["ftpsite"]
158 filter_out = "([/\\\\]((moduletoc.html)|(blogtoc.html)|(searchbox.html)))|([.]buildinfo)|([.]pyc)"
160 ftp = TransferFTP(ftpsite, login, password, ftps=ftps, fLOG=fLOG)
162 if page_transform is None:
163 fct_transform = text_transform
164 else:
166 def combined_transform(ftpp, filename, content):
167 text_transform(ftpp, filename, content)
168 page_transform(ftpp, filename, content)
170 fct_transform = combined_transform
172 for project in docs:
174 fLOG("######################################################################")
175 for k, v in sorted(project.items()):
176 fLOG("[publish_documentation] loop {}='{}'".format(k, v))
178 location = project["local"]
179 root_local = project["root_local"]
180 root_web = project["root_web"]
182 sfile = project["status_file"]
183 rootw = project["root_web"]
185 # documentation + setup
186 fLOG("[publish_documentation] location='{}'".format(location))
188 ftn = FileTreeNode(root_local)
189 fftp = FolderTransferFTP(ftn, ftp, sfile, root_web=rootw, fLOG=fLOG, footer_html=footer_html,
190 content_filter=content_filter, is_binary=is_binary,
191 text_transform=fct_transform, filter_out=filter_out,
192 force_allow=force_allow, exc=exc)
194 fftp.start_transfering(delay=delay)
196 ftn = FileTreeNode(os.path.join(root_local, ".."),
197 filter=lambda root, path, f, dir: not dir)
198 fftp = FolderTransferFTP(ftn, ftp, sfile,
199 root_web=root_web.replace("helpsphinx", ""), fLOG=fLOG,
200 footer_html=footer_html, content_filter=content_filter,
201 is_binary=is_binary, text_transform=fct_transform,
202 filter_out=filter_out)
204 fftp.start_transfering()
206 ftp.close()
209def publish_teachings_to_web(login, ftpsite="ftp.xavierdupre.fr", google_id=None, # pylint: disable=W0102
210 location="c:\\jenkins\\pymy\\%s\\%s%s\\dist\\%s",
211 rootw="/www/htdocs/app/%s/%s",
212 folder_status=".",
213 layout=[("html", "helpsphinx")],
214 modules=None, password=None, force_allow=None,
215 suffix=("_UT_%d%d_std" % sys.version_info[:2],),
216 delay=0.5, exc=False, exc_transfer=False,
217 transfer=True, additional_projects=None,
218 ftps='FTP', page_transform=None, fLOG=print):
219 """
220 Copies the documentation to the website.
222 @param login login
223 @param ftpsite ftp site
224 @param google_id google_id
225 @param location location of Jenkins build
226 @param rootw root on ftp site
227 @param folder_status folder status
228 @param modules list of modules to publish, if None, use @see fn get_teaching_modules
229 @param password if None, if will asked
230 @param layout last part of the folders
231 @param suffix suffixes to append to the project name
232 @param force_allow allow to publish files even if they contain these strings
233 whereas they seem to be credentials
234 @param delay delay between two files being transferred
235 @param exc raise exception if not found (True) or skip (False)
236 @param exc_transfer raise an exception if cannot be transfered
237 @param transfer starts transfering, otherwise returns the list of
238 transfering task to do
239 @param additional_projects additional projects
240 @param ftps use protocol FTP, TLS, or SFTP
241 @param page_transform function which transforms a page before uploading it,
242 @see fn text_transform
243 @param fLOG logging function
245 Example of use::
247 import sys
248 import os
249 from pyquickhelper.filehelper import TransferFTP, FileTreeNode, FolderTransferFTP
250 from tkinterquickhelper.funcwin import open_window_params
251 from ensae_teaching_cs.automation.ftp_publish_helper import publish_teachings_to_web
253 login = "..."
254 website = "ftp...."
255 rootw = "/www/htdocs/app/%s/%s"
257 password = None
259 publish_teachings_to_web(login, ftpsite=website,
260 google_id="google_id",
261 location="<something>\\\\%s\\\\%s%s\\\\dist\\\\%s",
262 rootw=rootw,
263 folder_status=os.path.abspath("."),
264 password=password)
266 Example of an additional projects:
268 ::
270 other_projects = [dict(status_file="status_projects.txt"),
271 root_local="...", root_web="...")]
272 """
273 if modules is None:
274 modules = get_teaching_modules()
275 modules = [m.split(':', maxsplit=1)[0] for m in modules]
277 if google_id is None:
278 google_id = ""
279 else:
280 footer = """
281 <script src="http://www.google-analytics.com/urchin.js" type="text/javascript"></script>
282 <script type="text/javascript">
283 _uacct = "{}";
284 urchinTracker();
285 </script>
286 """.format(google_id)
288 if password is None and transfer:
289 raise ValueError("password is empty")
291 location = os.path.abspath(location)
292 if folder_status is None:
293 folder_status = os.path.abspath(os.path.dirname(__file__))
294 else:
295 folder_status = os.path.abspath(folder_status)
297 if not isinstance(suffix, (tuple, list)):
298 suffix = [suffix]
300 projects = []
301 for module in modules:
302 fLOG(
303 "[ensae_teaching_cs] PUBLISH '{0}' - layout '{1}'".format(module, layout))
304 for lay in layout:
305 for suf in suffix:
306 root = os.path.abspath(location %
307 (module, module, suf, lay[0]))
308 if transfer and os.path.exists(root):
309 break
310 if transfer and not os.path.exists(root):
311 if exc:
312 p = os.path.abspath(location %
313 (module, module, suffix[0], lay[0]))
314 raise FileNotFoundError(
315 "First tried '{0}'\n last tried '{1}'".format(root, p))
316 else:
317 continue
318 fLOG(" ", root)
319 rw = rootw % (module, lay[1])
321 project = dict(status_file=os.path.join(folder_status, "status_%s.txt" % module),
322 local=root, root_local=root, root_web=rw)
323 projects.append(project)
325 if module == "ensae_teaching_cs":
326 lay = [_ for _ in layout if _[0] == "html"][0]
327 if transfer and not os.path.exists(root):
328 if exc:
329 raise FileNotFoundError(root)
330 else:
331 continue
333 def _update_path(pth):
334 for a, b in [("\\build", "\\build3"),
335 ("\\html", "\\html3"),
336 ("/build", "/build3"),
337 ("/html", "/html3")]:
338 pth = pth.replace(a, b)
339 return pth
341 local = _update_path(root)
342 fLOG("[ensae_teaching_cs] checking folder '{0}'".format(local))
343 fLOG("[ensae_teaching_cs] root is '{0}'".format(root))
344 if os.path.exists(local):
345 fLOG("[ensae_teaching_cs] found '{0}'".format(local))
346 project = dict(status_file=os.path.join(folder_status, "status_3_%s.txt" % module),
347 local=local, root_local=local,
348 root_web=(rootw % (module, lay[1])).replace("_no_clean", "").replace("/helpsphinx", "/helpsphinx3"))
349 projects.append(project)
350 project = dict(status_file=os.path.join(folder_status, "status_2_%s.txt" % module),
351 local=local, root_local=local,
352 root_web=(rootw % (module, lay[1])).replace("_no_clean", "").replace("/helpsphinx", "/helpsphinx2"))
353 projects.append(project)
354 else:
355 fLOG("[ensae_teaching_cs] looking into DOC1, DOC3")
356 root1 = root.replace("_UT_", "_DOC1_")
357 if transfer:
358 if not os.path.exists(root1):
359 if exc:
360 raise FileNotFoundError(root1)
361 else:
362 pass
363 else:
364 project = dict(status_file=os.path.join(folder_status, "status_doc1_%s.txt" % module),
365 local=root1, root_local=root1,
366 root_web=(rootw % (module, lay[1])).replace("_no_clean", ""))
367 projects.append(project)
369 root3 = root.replace("_UT_", "_DOC3_").replace("html", "html3")
370 if transfer:
371 if not os.path.exists(root3):
372 if exc:
373 raise FileNotFoundError(root3)
374 else:
375 project = dict(status_file=os.path.join(folder_status, "status_doc3_%s.txt" % module),
376 local=root3, root_local=root3,
377 root_web=(rootw % (module, lay[1])).replace("_no_clean", "").replace("/helpsphinx", "/helpsphinx3"))
378 projects.append(project)
379 project = dict(status_file=os.path.join(folder_status, "status_doc2_%s.txt" % module),
380 local=root3, root_local=root3,
381 root_web=(rootw % (module, lay[1])).replace("_no_clean", "").replace("/helpsphinx", "/helpsphinx2"))
382 projects.append(project)
384 # publish
385 if additional_projects:
386 for proj in additional_projects:
387 if 'root_local' not in proj:
388 if 'folder' not in proj:
389 raise KeyError(
390 "Key 'folder' or 'root_local' must be specified in {}.".format(proj))
391 proj = proj.copy()
392 proj['root_local'] = proj['folder']
393 if 'folder' not in proj:
394 proj = proj.copy()
395 proj['folder'] = proj['root_local']
396 if 'local' not in proj:
397 proj = proj.copy()
398 proj['local'] = os.path.dirname(proj['root_local'])
399 projects.append(proj)
401 if transfer:
402 publish_documentation(projects, ftpsite=ftpsite, login=login, password=password,
403 footer_html=footer, force_allow=force_allow, delay=delay,
404 exc=exc_transfer, ftps=ftps, page_transform=page_transform,
405 fLOG=fLOG)
406 return projects