Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1""" 

2@file 

3@brief Produces a build file for a module following *pyquickhelper* design. 

4""" 

5 

6import sys 

7import os 

8import uuid 

9import re 

10from .windows_scripts import windows_error, windows_prefix, windows_setup, windows_notebook 

11from .windows_scripts import windows_publish, windows_publish_doc, windows_pypi, setup_script_dependency_py 

12from .windows_scripts import windows_prefix_27, windows_unittest27, copy_dist_to_local_pypi 

13from .windows_scripts import windows_any_setup_command, windows_blogpost, windows_docserver, windows_build_setup, windows_build 

14from .windows_scripts import pyproj_template, copy_sphinx_to_dist 

15from ..filehelper.file_info import is_file_string 

16 

17#: nickname for no folder 

18_default_nofolder = "__NOFOLDERSHOULDNOTEXIST%d%d__" % sys.version_info[:2] 

19 

20 

21def choose_path(*paths): 

22 """ 

23 Returns the first path which exists in the list. 

24 

25 @param paths list of paths 

26 @return a path 

27 """ 

28 found = None 

29 for path in paths: 

30 if "{" in path: 

31 if found is None: 

32 root = os.path.dirname(path) 

33 if not root: 

34 root = '.' 

35 founds = [os.path.join(root, _) for _ in os.listdir(root)] 

36 founds.sort(reverse=True) 

37 reg = re.compile(path.replace("\\", "\\\\")) 

38 found = [(_, reg.search(_)) for _ in founds] 

39 found = [_ for _ in found if _[1]] 

40 if len(found) > 0: 

41 full = found[0][0] 

42 return full 

43 elif os.path.exists(path): 

44 return path 

45 if paths[-1] != _default_nofolder: 

46 raise FileNotFoundError("No path exist in: " + ", ".join(paths)) 

47 return _default_nofolder 

48 

49 

50#: default values, to be replaced in the build script 

51#: ``'c:\\python36-x64'`` --> appveyor 

52#: ``'c:\\python36_x64'`` --> custom installation 

53 

54default_values = { 

55 "windows": { 

56 "__PY27_X64__": choose_path("c:\\Python27_x64", "c:\\Python27", "c:\\Anaconda2", "c:\\Anaconda", _default_nofolder), 

57 "__PY35_X64__": choose_path("c:\\Python35[0-9]{1}_x64", "c:\\Python35_x64", 'c:\\python35-x64', _default_nofolder), 

58 "__PY36_X64__": choose_path("c:\\Python36[0-9]{1}_x64", "c:\\Python36_x64", "c:\\Python36-x64", _default_nofolder), 

59 "__PY37_X64__": choose_path("c:\\Python37[0-9]{1}_x64", "c:\\Python37_x64", "c:\\Python37-x64", _default_nofolder), 

60 "__PY38_X64__": choose_path("c:\\Python38[0-9]{1}_x64", "c:\\Python38_x64", "c:\\Python38-x64", _default_nofolder), 

61 }, 

62} 

63 

64 

65def private_path_choice(path): 

66 """ 

67 Custom logic to reference other currently developped modules. 

68 """ 

69 s = path 

70 current = '%current%' if sys.platform.startswith('win') else '~' 

71 if "/" in s or "\\" in s: 

72 return s 

73 elif 'ROOT' in s: 

74 return os.path.join(current, "..", s.replace('ROOT', '')) 

75 elif 'BLIB' in s: 

76 return os.path.join(current, "..", s.replace('BLIB', ''), "build", "lib") 

77 elif 'NSRC' in s: 

78 return os.path.join(current, "..", s.replace("NSRC", '')) 

79 else: 

80 return os.path.join(current, "..", s, "src") 

81 

82 

83def private_replacement_(script, paths, key="__ADDITIONAL_LOCAL_PATH__"): 

84 """ 

85 Less copy/paste. 

86 """ 

87 unique_paths = [] 

88 for p in paths: 

89 if p not in unique_paths: 

90 unique_paths.append(p) 

91 rows = [private_path_choice(_) for _ in unique_paths] 

92 sep = ";" if sys.platform.startswith("win") else ":" 

93 rep = sep + sep.join(rows) 

94 script = script.replace(key, rep) 

95 return script 

96 

97 

98def private_script_replacements(script, module, requirements, port, raise_exception=True, 

99 platform=sys.platform, default_engine_paths=None, 

100 additional_local_path=None): 

101 """ 

102 Runs last replacements. 

103 

104 @param script script or list of scripts 

105 @param module module name 

106 @param requirements requirements - (list or 2-uple of lists) 

107 @param port port 

108 @param raise_exception raise an exception if there is an error, otherwise, return None 

109 @param platform platform 

110 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below. 

111 @param additional_local_path additional local path to add to PYTHONPATH 

112 @return modified script 

113 

114 An example for *default_engine_paths*:: 

115 

116 default_engine_paths = { 

117 "windows": { 

118 "__PY35__": None, 

119 "__PY35_X64__": "c:\\Python352_x64", 

120 "__PY36_X64__": "c:\\Python365_x64", 

121 "__PY37_X64__": "c:\\Python370_x64", 

122 "__PY27_X64__": "c:\\Anaconda2", 

123 }, 

124 } 

125 

126 Parameter *requirements* can a list of requirements, 

127 we assume these requirements are available from a local PyPi server. 

128 There can be extra requirements obtained from PiPy. In that case, 

129 those can be specified as a tuple *(requirements_local, requirements_pipy)*. 

130 

131 With Python 3.5, I get the following error on Windows:: 

132 

133 Using base prefix 'c:\\\\python35_x64' 

134 New python executable in c:\\jenkins\\pymy\\py35_pyquickhelper\\_virtualenv\\ 

135 pyquickhelper_virpy35_22316CE015_22316CE015\\Scripts\\python.exe 

136 ERROR: The executable c:\\jenkins\\pymy\\py35_pyquickhelper\\_virtualenv\\ 

137 pyquickhelper_virpy35_22316CE015_22316CE015\\Scripts\\python.exe is not functioning 

138 ERROR: It thinks sys.prefix is 'c:\\\\jenkins\\\\pymy\\\\py35_pyquickhelper' (should be 

139 'c:\\\\jenkins\\\\pymy\\\\py36_pyquickhelper\\\\_virtualenv\\\\pyquickhelper_virpy35_22316ce015_22316ce015') 

140 ERROR: virtualenv is not compatible with this system or executable 

141 Note: some Windows users have reported this error when they installed Python for 

142 "Only this user" or have multiple versions of Python installed. Copying the appropriate PythonXX.dll 

143 to the virtualenv Scripts/ directory may fix this problem. 

144 

145 The function replaces ``rem _PATH_VIRTUAL_ENV_`` 

146 with an instruction to copy these DLLs. 

147 Parameter *requirements* can be a list or a tuple. 

148 """ 

149 global default_values 

150 if default_engine_paths is None: 

151 default_engine_paths = default_values 

152 

153 if isinstance(script, list): 

154 return [private_script_replacements(s, module, requirements, 

155 port, raise_exception, platform, 

156 default_engine_paths=default_engine_paths) for s in script] 

157 

158 if platform.startswith("win"): 

159 plat = "windows" 

160 global _default_nofolder 

161 def_values = default_engine_paths 

162 

163 values = [v for v in def_values[ 

164 plat].values() if v is not None and v != _default_nofolder] 

165 if raise_exception and len(values) != len(set(values)): 

166 raise FileNotFoundError("One path is wrong among:\n" + 

167 "\n".join("{0}={1}".format(k, v) for k, v in def_values[plat].items())) 

168 

169 if module is not None: 

170 script = script.replace("__MODULE__", module) 

171 

172 for k, v in def_values[plat].items(): 

173 script = script.replace(k, v) 

174 

175 # requirements 

176 if requirements is not None: 

177 if isinstance(requirements, list): 

178 requirements_pipy = [] 

179 requirements_local = requirements 

180 else: 

181 requirements_local, requirements_pipy = requirements 

182 

183 if requirements_pipy is None: 

184 requirements_pipy = [] 

185 if requirements_local is None: 

186 requirements_local = [] 

187 

188 cj = "%jenkinspythonpip%" if "jenkinspythonpip" in script else "%pythonpip%" 

189 patternr = "install {0}" 

190 patternl = "install --no-cache-dir --index http://localhost:{0}/simple/ {1}" 

191 rows = [] 

192 for r in requirements_pipy: 

193 r = cj + " " + patternr.format(r) 

194 rows.append(r) 

195 for r in requirements_local: 

196 r = cj + " " + patternl.format(port, r) 

197 rows.append(r) 

198 

199 reqs = "\n".join(rows) 

200 else: 

201 reqs = "" 

202 

203 script = script.replace("__REQUIREMENTS__", reqs) \ 

204 .replace("__PORT__", str(port)) \ 

205 .replace("__USERNAME__", os.environ.get("USERNAME", os.environ.get("USER", "UNKNOWN-USER"))) 

206 

207 if "__ADDITIONAL_LOCAL_PATH__" in script: 

208 paths = [] 

209 if additional_local_path is not None and len(additional_local_path) > 0: 

210 paths.extend(additional_local_path) 

211 if len(paths) > 0: 

212 script = private_replacement_( 

213 script, paths, key="__ADDITIONAL_LOCAL_PATH__") 

214 else: 

215 script = script.replace("__ADDITIONAL_LOCAL_PATH__", "") 

216 

217 if "rem _PATH_VIRTUAL_ENV_" in script: 

218 script = script.replace( 

219 "rem _PATH_VIRTUAL_ENV_", "rem nothing to do here") 

220 

221 return script 

222 

223 else: 

224 if raise_exception: 

225 raise NotImplementedError( 

226 "not implemented yet for this platform %s" % sys.platform) 

227 return None 

228 

229 

230def get_build_script(module, requirements=None, port=8067, default_engine_paths=None, 

231 additional_local_path=None): 

232 """ 

233 Builds the build script which builds the setup, run the unit tests 

234 and the documentation. 

235 

236 @param module module name 

237 @param requirements list of dependencies (not in your python distribution) 

238 @param port port for the local pypi_server which gives the dependencies 

239 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below. 

240 @param additional_local_path additional paths to add to PYTHONPATH 

241 @return scripts 

242 """ 

243 if requirements is None: 

244 requirements = [] 

245 return private_script_replacements(windows_build, module, requirements, port, 

246 default_engine_paths=default_engine_paths, 

247 additional_local_path=additional_local_path) 

248 

249 

250def get_script_command(command, module, requirements, port=8067, platform=sys.platform, 

251 default_engine_paths=None, additional_local_path=None): 

252 """ 

253 Produces a script which runs a command available through the setup. 

254 

255 @param command command to run 

256 @param module module name 

257 @param requirements list of dependencies (not in your python distribution) 

258 @param port port for the local pypi_server which gives the dependencies 

259 @param platform platform (only Windows) 

260 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below. 

261 @param additional_local_path additional local path to add before running command ``setup.py <command>`` 

262 @return scripts 

263 

264 The available list of commands is given by function @see fn process_standard_options_for_setup. 

265 """ 

266 if not platform.startswith("win"): 

267 raise NotImplementedError("not yet available on linux") 

268 global windows_error, windows_prefix, windows_setup 

269 rows = [windows_prefix] 

270 

271 if additional_local_path is not None and len(additional_local_path): 

272 addp = "set PYTHONPATH=%PYTHONPATH%;" + \ 

273 ";".join(private_path_choice(_) for _ in additional_local_path) 

274 else: 

275 addp = "" 

276 rows.append(windows_setup.replace( 

277 "rem set PYTHONPATH=additional_path", addp) + " " + command) 

278 rows.append(windows_error) 

279 sc = "\n".join(rows) 

280 res = private_script_replacements( 

281 sc, module, requirements, port, default_engine_paths=default_engine_paths, 

282 additional_local_path=additional_local_path) 

283 if sys.platform.startswith("win"): 

284 if command == "copy27": 

285 res = """ 

286 if exist dist_module27 ( 

287 rmdir /Q /S dist_module27 

288 if %errorlevel% neq 0 exit /b %errorlevel% 

289 ) 

290 """.replace(" ", "") + res 

291 elif command == "clean_space": 

292 # Run the test which test pep8 and convert the convert the 

293 # notebooks. 

294 res += """ 

295 if not exist _unittests\\ut_module\\test_code_style.py goto end: 

296 %pythonexe% -u _unittests\\ut_module\\test_code_style.py -v 

297 if %errorlevel% neq 0 exit /b %errorlevel% 

298 

299 if not exist _unittests\\ut_module\\test_convert_notebooks.py goto end: 

300 %pythonexe% -u _unittests\\ut_module\\test_convert_notebooks.py 

301 if %errorlevel% neq 0 exit /b %errorlevel% 

302 ) 

303 """.replace(" ", "") + res 

304 return res 

305 

306 

307def get_extra_script_command(command, module, requirements, port=8067, blog_list=None, platform=sys.platform, 

308 default_engine_paths=None, unit_test_folder=None, unittest_modules=None, 

309 additional_notebook_path=None, additional_local_path=None): 

310 """ 

311 Produces a script which runs the notebook, a documentation server, which 

312 publishes... 

313 

314 @param command command to run (*notebook*, *publish*, *publish_doc*, *local_pypi*, *setupdep*, 

315 *run27*, *build27*, *copy_dist*, *any_setup_command*, *lab*) 

316 @param module module name 

317 @param requirements list of dependencies (not in your python distribution) 

318 @param port port for the local pypi_server which gives the dependencies 

319 @param blog_list list of blog to listen for this module (usually stored in 

320 ``module.__blog__``) 

321 @param platform platform (only Windows) 

322 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below. 

323 @param unit_test_folder unit test folders, used for command ``run27`` 

324 @param additional_notebook_path additional paths to add when running the script launching the notebooks 

325 @param additional_local_path additional paths to add when running a local command 

326 @param unittest_modules list of modules to be used during unit tests 

327 @return scripts 

328 

329 The available list of commands is given by function @see fn process_standard_options_for_setup. 

330 """ 

331 if not platform.startswith("win"): 

332 raise NotImplementedError("linux not yet available") 

333 

334 script = None 

335 if command == "notebook": 

336 script = windows_notebook 

337 elif command == "lab": 

338 script = windows_notebook.replace("jupyter-notebook", "jupyter-lab") 

339 elif command == "publish": 

340 script = "\n".join([windows_prefix, windows_publish]) 

341 elif command == "publish_doc": 

342 script = "\n".join([windows_prefix, windows_publish_doc]) 

343 elif command == "local_pypi": 

344 script = "\n".join([windows_prefix, windows_pypi]) 

345 elif command == "run27": 

346 script = "\n".join( 

347 [windows_prefix_27, windows_unittest27, windows_error]) 

348 if unit_test_folder is None: 

349 raise FileNotFoundError( 

350 "the unit test folder must be specified and cannot be None") 

351 if not os.path.exists(unit_test_folder): 

352 raise FileNotFoundError( 

353 "the unit test folder must exist: " + unit_test_folder) 

354 ut_ = [("%pythonexe27%\\..\\Scripts\\nosetests.exe -w " + _) 

355 for _ in os.listdir(unit_test_folder) if _.startswith("ut_")] 

356 stut = "\nif %errorlevel% neq 0 exit /b %errorlevel%\n".join(ut_) 

357 script = script.replace("__LOOP_UNITTEST_FOLDERS__", stut) 

358 elif command == "build27": 

359 script = "\n".join([windows_prefix_27, "cd dist_module27", "rmdir /S /Q dist", 

360 windows_setup.replace( 

361 "exe%", "exe27%") + " bdist_wheel", 

362 windows_error, "cd ..", "copy dist_module27\\dist\\*.whl dist"]) 

363 elif command == "copy_dist": 

364 script = copy_dist_to_local_pypi 

365 elif command == "copy_sphinx": 

366 script = copy_sphinx_to_dist 

367 elif command == "setupdep": 

368 script = setup_script_dependency_py 

369 elif command == "any_setup_command": 

370 script = windows_any_setup_command 

371 elif command == "build_dist": 

372 script = windows_build_setup 

373 elif command == "history": 

374 script = "\n".join( 

375 [windows_prefix, '\n%pythonexe% %current%setup.py history\n']) 

376 else: 

377 raise Exception("unable to interpret command: " + command) 

378 

379 # additional paths 

380 if "__ADDITIONAL_LOCAL_PATH__" in script: 

381 paths = [] 

382 if command in ("notebook", "lab") and additional_notebook_path is not None and len(additional_notebook_path) > 0: 

383 paths.extend(additional_notebook_path) 

384 if unittest_modules is not None and len(unittest_modules) > 0: 

385 paths.extend(unittest_modules) 

386 if additional_local_path is not None and len(additional_local_path) > 0: 

387 paths.extend(additional_local_path) 

388 if len(paths) > 0: 

389 script = private_replacement_( 

390 script, paths, key="__ADDITIONAL_LOCAL_PATH__") 

391 else: 

392 script = script.replace("__ADDITIONAL_LOCAL_PATH__", "") 

393 

394 script = script.replace("__ADDITIONAL_NOTEBOOK_PATH__", "") 

395 

396 # common post-processing 

397 if script is None: 

398 raise Exception("unexpected command: " + command) 

399 return private_script_replacements(script, module, requirements, port, default_engine_paths=default_engine_paths) 

400 

401 

402def get_script_module(command, platform=sys.platform, blog_list=None, 

403 default_engine_paths=None): 

404 """ 

405 Produces a script which runs the notebook, a documentation server, which 

406 publishes and other scripts. 

407 

408 @param command command to run (*blog*) 

409 @param platform platform (only Windows) 

410 @param blog_list list of blog to listen for this module (usually stored in ``module.__blog__``) 

411 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below. 

412 @return scripts 

413 

414 The available list of commands is given by function @see fn process_standard_options_for_setup. 

415 """ 

416 prefix_setup = "" 

417 filename = os.path.abspath(__file__) 

418 if "site-packages" not in filename: 

419 folder = os.path.normpath( 

420 os.path.join(os.path.dirname(filename), "..", "..")) 

421 prefix_setup = """ 

422 import sys 

423 import os 

424 sys.path.append(r"{0}") 

425 sys.path.append(r"{1}") 

426 sys.path.append(r"{2}") 

427 """.replace(" ", "").format(folder, 

428 folder.replace( 

429 "pyquickhelper", "pyensae"), 

430 folder.replace( 

431 "pyquickhelper", "pyrsslocal") 

432 ) 

433 

434 script = None 

435 if command == "blog": 

436 if blog_list is None: 

437 return None 

438 else: 

439 list_xml = blog_list.strip("\n\r\t ") 

440 if '<?xml version="1.0" encoding="UTF-8"?>' not in list_xml and is_file_string(list_xml) and os.path.exists(list_xml): 

441 with open(list_xml, "r", encoding="utf8") as f: 

442 list_xml = f.read() 

443 if "<body>" not in list_xml: 

444 raise ValueError("Wrong XML format:\n{0}".format(list_xml)) 

445 script = [("auto_rss_list.xml", list_xml)] 

446 script.append(("auto_rss_server.py", prefix_setup + """ 

447 from pyquickhelper.pycode.blog_helper import rss_update_run_server 

448 rss_update_run_server("auto_rss_database.db3", "auto_rss_list.xml") 

449 """.replace(" ", ""))) 

450 if platform.startswith("win"): 

451 script.append("\n".join([windows_prefix, windows_blogpost])) 

452 elif command == "doc": 

453 script = [] 

454 script.append(("auto_doc_server.py", prefix_setup + """ 

455 # address http://localhost:8079/ 

456 from pyquickhelper import fLOG 

457 from pyquickhelper.serverdoc import run_doc_server, get_jenkins_mappings 

458 fLOG(OutputPrint=True) 

459 fLOG("running documentation server") 

460 thisfile = os.path.dirname(__file__) 

461 mappings = get_jenkins_mappings(os.path.join(thisfile, "..")) 

462 fLOG("goto", "http://localhost:8079/") 

463 for k,v in sorted(mappings.items()): 

464 fLOG(k,"-->",v) 

465 run_doc_server(None, mappings=mappings) 

466 """.replace(" ", ""))) 

467 if platform.startswith("win"): 

468 script.append("\n".join([windows_prefix, "rem http://localhost:8079/", 

469 windows_docserver])) 

470 else: 

471 raise Exception("unable to interpret command: " + command) 

472 

473 # common post-processing 

474 for i, item in enumerate(script): 

475 if isinstance(item, tuple): 

476 ext = os.path.splitext(item[0]) 

477 if ext == ".py": 

478 s = private_script_replacements( 

479 item[1], None, None, None, default_engine_paths=default_engine_paths) 

480 script[i] = (item[0], s) 

481 else: 

482 script[i] = private_script_replacements( 

483 item, None, None, None, default_engine_paths=default_engine_paths) 

484 return script 

485 

486 

487def get_pyproj_project(name, file_list): 

488 """ 

489 returns a string which corresponds to a pyproj project 

490 

491 @param name project name 

492 @param file_list file_list 

493 @return string 

494 """ 

495 guid = uuid.uuid3(uuid.NAMESPACE_DNS, name) 

496 folders = list(_ for _ in sorted(set(os.path.dirname(f) 

497 for f in file_list)) if len(_) > 0) 

498 sfold = "\n".join(' <Folder Include="%s\" />' % _ for _ in folders) 

499 sfiles = "\n".join(' <Compile Include="%s\" />' % _ for _ in file_list) 

500 

501 script = pyproj_template.replace("__GUID__", str(guid)) \ 

502 .replace("__NAME__", name) \ 

503 .replace("__INCLUDEFILES__", sfiles) \ 

504 .replace("__INCLUDEFOLDERS__", sfold) 

505 return script