Coverage for pyquickhelper/pycode/py3to2.py: 100%
128 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
1"""
2@file
3@brief Helper to convert a script written in Python 3 to Python 2
4"""
6import os
7import re
8import shutil
9from ..filehelper.synchelper import explore_folder_iterfile
10from ..loghelper.flog import noLOG
11from .default_regular_expression import _setup_pattern_copy
14class Convert3to2Exception(Exception):
16 """
17 exception raised for an exception happening during the conversion
18 """
19 pass
22def py3to2_convert_tree(folder, dest, encoding="utf8", pattern=".*[.]py$",
23 pattern_copy=_setup_pattern_copy,
24 unittest_modules=None, fLOG=noLOG):
25 """
26 Converts files in a folder and its subfolders from python 3 to python 2,
27 the function only considers python script (verifying *pattern*).
29 @param folder folder
30 @param dest destination
31 @param encoding all files will be saved with this encoding
32 @param pattern pattern to find source code
33 @param pattern_copy copy these files, do not modify them
34 @param fLOG logging function
35 @param unittest_modules modules used during unit tests but not installed
36 @return list of copied files
38 If a folder does not exists, it will create it.
39 The function excludes all files in subfolders
40 starting by ``dist``, ``_doc``, ``build``, ``extensions``, ``nbextensions``.
41 The function also exclude subfolders inside
42 subfolders following the pattern ``ut_.*``.
44 There are some issues difficult to solve with strings.
45 Python 2.7 is not friendly with strings. Some needed pieces of code::
47 if sys.version_info[0]==2:
48 from codecs import open
50 You can also read blog post :ref:`b-migration-py2py3`.
52 The variable *unittest_modules* indicates the list of
53 modules which are not installed in :epkg:`Python` distribution
54 but still used and placed in the same folder as the same which
55 has to converted.
57 *unittest_modules* can be either a list or a tuple ``(module, alias)``.
58 Then the alias appears instead of the module name.
60 The function does not convert the exception
61 `FileNotFoundError <https://docs.python.org/3/library/exceptions.html>`_
62 which only exists in Python 3. The module will fail in version 2.7
63 if this exception is raised.
65 The following page
66 `Cheat Sheet: Writing Python 2-3 compatible code
67 <http://python-future.org/compatible_idioms.html>`_
68 gives the difference between the two versions of Python
69 and how to write compatible code.
70 """
71 exclude = ("temp_", "dist", "_doc", "build", "extensions",
72 "nbextensions", "dist_module27", "_virtualenv", "_venv")
73 reg = re.compile(".*/ut_.*/.*/.*")
75 conv = []
76 for file in explore_folder_iterfile(folder, pattern=pattern):
77 full = os.path.join(folder, file)
78 if "site-packages" in full:
79 continue
80 file = os.path.relpath(file, folder)
82 # undesired sub folders
83 ex = False
84 for exc in exclude:
85 if file.startswith(exc) or "\\temp_" in file or \
86 "/temp_" in file or "dist_module27" in file:
87 ex = True
88 break
89 if ex:
90 continue
92 # subfolders inside unit tests folder
93 lfile = file.replace("\\", "/")
94 if reg.search(lfile):
95 continue
97 py2 = py3to2_convert(full, unittest_modules)
98 destfile = os.path.join(dest, file)
99 dirname = os.path.dirname(destfile)
100 if not os.path.exists(dirname):
101 os.makedirs(dirname)
102 with open(destfile, "w", encoding="utf8") as f:
103 f.write(py2)
104 conv.append(destfile)
106 for file in explore_folder_iterfile(folder, pattern=pattern_copy):
107 full = os.path.join(folder, file)
108 file = os.path.relpath(file, folder)
110 # undesired sub folders
111 ex = False
112 for exc in exclude:
113 if file.startswith(exc) or "\\temp_" in file or \
114 "/temp_" in file or "dist_module27" in file:
115 ex = True
116 break
117 if ex:
118 continue
120 destfile = os.path.join(dest, file)
121 dirname = os.path.dirname(destfile)
122 if not os.path.exists(dirname):
123 os.makedirs(dirname)
124 shutil.copy(full, dirname)
125 conv.append(destfile)
127 fLOG("py3to2_convert_tree, copied", len(conv), "files")
129 return conv
132def py3to2_convert(script, unittest_modules):
133 """
134 converts a script into from python 3 to python 2
136 @param script script or filename
137 @param unittest_modules modules used during unit test but not installed,
138 @see fn py3to2_convert_tree
139 @return string
141 See see @fn py3to2_convert_tree for more information.
142 """
143 if os.path.exists(script):
144 try:
145 with open(script, "r", encoding="utf8") as f:
146 content = f.read()
147 except (UnicodeEncodeError, UnicodeDecodeError): # pragma: no cover
148 with open(script, "r") as f:
149 content = f.read()
151 else:
152 content = script # pragma: no cover
154 # start processing
155 content = py3to2_remove_raise_from(content)
157 # unicode
158 if ("install_requires=" in content or "package_data" in content) and "setup" in content:
159 # we skip the file setup.py as it raises an error
160 pass
161 else:
162 try:
163 content = py3to2_future(content)
164 except Convert3to2Exception as e: # pragma: no cover
165 raise Convert3to2Exception(
166 f'unable to convert a file due to unicode issue.\n File "{script}", line 1') from e
168 # some other modification
169 content = content.replace("from queue import", "from Queue import")
170 content = content.replace("nonlocal ", "# nonlocal ")
172 # long and unicode
173 content = content.replace("int #long#", "long")
174 content = content.replace("int # long#", "long")
175 content = content.replace("str #unicode#", "unicode")
176 content = content.replace("str # unicode#", "unicode")
177 content = content.replace(
178 "Programming Language :: Python :: 3", "Programming Language :: Python :: 2")
179 content = content.replace(', sep="\\t")', ', sep="\\t".encode("ascii"))')
181 # imported modules
182 if unittest_modules is not None:
183 content = py3to2_imported_local_modules(content, unittest_modules)
185 # end
186 return content
189def py3to2_future(content):
190 """
191 checks that import ``from __future__ import unicode_literals``
192 is always present, the function assumes it is a python code
194 @param content file content
195 @return new content
196 """
197 find = "from __future__ import unicode_literals"
198 if find in content and f'"{find}"' not in content:
199 # the second condition avoid to raise this
200 # exception when parsing this file
201 # this case should only happen for this file
202 raise Convert3to2Exception( # pragma: no cover
203 "unable to convert a file")
205 lines = content.split("\n")
206 position = 0
207 incomment = None
208 while (position < len(lines) and not lines[position].startswith("import ") and
209 not lines[position].startswith("from ") and
210 not lines[position].startswith("def ") and
211 not lines[position].startswith("class ")):
212 if incomment is None:
213 if lines[position].startswith("'''"):
214 incomment = "'''" # pragma: no cover
215 elif lines[position].startswith('"""'):
216 incomment = '"""'
217 else:
218 if lines[position].endswith("'''"): # pragma: no cover
219 incomment = None
220 position += 1
221 break
222 if lines[position].endswith('"""'):
223 incomment = None
224 position += 1
225 break
226 position += 1
228 if position < len(lines):
229 lines[position] = f"{find}\n{lines[position]}"
230 return "\n".join(lines)
233def py3to2_remove_raise_from(content):
234 """
235 Removes expression such as: ``raise Exception ("...") from e``.
236 The function is very basic. It should be done with a grammar.
238 @param content file content
239 @return script
240 """
241 lines = content.split("\n")
242 r = None
243 for i, line in enumerate(lines):
244 if " raise " in line:
245 r = i
246 if " from " in line and r is not None:
247 spl = line.split(" from ")
248 if len(spl[0].strip(" \n")) > 0:
249 lines[i] = line = spl[0] + "# from " + " - ".join(spl[1:])
251 if r is not None and i > r + 3:
252 r = None
254 return "\n".join(lines)
257def py3to2_imported_local_modules(content, unittest_modules):
258 """
259 See function @see fn py3to2_convert_tree
260 and documentation about parameter *unittest_modules*.
262 @param content script or filename
263 @param unittest_modules modules used during unit test but not installed,
264 @see fn py3to2_convert_tree
265 """
266 lines = content.split("\n")
267 for modname in unittest_modules:
268 if isinstance(modname, tuple):
269 modname, alias = modname # pragma: no cover
270 else:
271 alias = modname
273 s1 = f'"{modname}"'
274 s2 = f"'{modname}'"
275 s3 = f"import {modname}"
276 s4 = f'"{modname.upper()}"'
277 s4_rep = f'"{modname.upper()}27"'
279 if (s1 in content or s2 in content or s4 in content) and s3 in content:
280 for i, line in enumerate(lines):
281 if " in " in line or "ModuleInstall" in line:
282 continue
283 if s1 in line:
284 line = line.replace(
285 s1, f'"..", "{alias}", "dist_module27"')
286 lines[i] = line
287 elif s2 in line:
288 line = line.replace( # pragma: no cover
289 s2, f"'..', '{alias}', 'dist_module27'")
290 lines[i] = line # pragma: no cover
291 elif s4 in line:
292 line = line.replace(s4, s4_rep)
293 lines[i] = line
294 return "\n".join(lines)