Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1""" 

2@file 

3@brief Produces a build file for a module following *pyquickhelper* design. 

4""" 

5 

6import sys 

7import os 

8import uuid 

9import re 

10from .windows_scripts import windows_error, windows_prefix, windows_setup, windows_notebook 

11from .windows_scripts import windows_publish, windows_publish_doc, windows_pypi, setup_script_dependency_py 

12from .windows_scripts import windows_prefix_27, windows_unittest27, copy_dist_to_local_pypi 

13from .windows_scripts import windows_any_setup_command, windows_blogpost, windows_docserver, windows_build_setup, windows_build 

14from .windows_scripts import pyproj_template, copy_sphinx_to_dist 

15from ..filehelper.file_info import is_file_string 

16 

17#: nickname for no folder 

18_default_nofolder = "__NOFOLDERSHOULDNOTEXIST%d%d__" % sys.version_info[:2] 

19 

20 

21def choose_path(*paths): 

22 """ 

23 Returns the first path which exists in the list. 

24 

25 @param paths list of paths 

26 @return a path 

27 """ 

28 found = None 

29 for path in paths: 

30 if "{" in path: 

31 if found is None: 

32 root = os.path.dirname(path) 

33 if not root: 

34 root = '.' 

35 founds = [os.path.join(root, _) for _ in os.listdir(root)] 

36 founds.sort(reverse=True) 

37 reg = re.compile(path.replace("\\", "\\\\")) 

38 found = [(_, reg.search(_)) for _ in founds] 

39 found = [_ for _ in found if _[1]] 

40 if len(found) > 0: # pragma: no cover 

41 full = found[0][0] 

42 return full 

43 elif os.path.exists(path): 

44 return path # pragma: no cover 

45 if paths[-1] != _default_nofolder: 

46 raise FileNotFoundError( # pragma: no cover 

47 "No path exist in: " + ", ".join(paths)) 

48 return _default_nofolder 

49 

50 

51#: default values, to be replaced in the build script 

52#: ``'c:\\python39x64'`` --> appveyor 

53#: ``'c:\\python39_x64'`` --> custom installation 

54 

55default_values = { 

56 "windows": { 

57 "__PY36_X64__": choose_path("c:\\Python36[0-9]{1}_x64", "c:\\Python36_x64", "c:\\Python36-x64", _default_nofolder), 

58 "__PY37_X64__": choose_path("c:\\Python37[0-9]{1}_x64", "c:\\Python37_x64", "c:\\Python37-x64", _default_nofolder), 

59 "__PY38_X64__": choose_path("c:\\Python38[0-9]{1}_x64", "c:\\Python38_x64", "c:\\Python38-x64", _default_nofolder), 

60 "__PY39_X64__": choose_path("c:\\Python39[0-9]{1}_x64", "c:\\Python38_x64", "c:\\Python39-x64", _default_nofolder), 

61 }, 

62} 

63 

64 

65def private_path_choice(path): 

66 """ 

67 Custom logic to reference other currently developped modules. 

68 """ 

69 s = path 

70 current = '%current%' if sys.platform.startswith('win') else '~' 

71 if "/" in s or "\\" in s: 

72 return s # pragma: no cover 

73 if 'ROOT' in s: 

74 return os.path.join(current, "..", s.replace('ROOT', '')) 

75 if 'BLIB' in s: 

76 return os.path.join(current, "..", s.replace('BLIB', ''), "build", "lib") 

77 if 'NSRC' in s: 

78 return os.path.join(current, "..", s.replace("NSRC", '')) # pragma: no cover 

79 return os.path.join(current, "..", s, "src") 

80 

81 

82def private_replacement_(script, paths, key="__ADDITIONAL_LOCAL_PATH__"): 

83 """ 

84 Less copy/paste. 

85 """ 

86 unique_paths = [] 

87 for p in paths: 

88 if p not in unique_paths: 

89 unique_paths.append(p) 

90 rows = [private_path_choice(_) for _ in unique_paths] 

91 sep = ";" if sys.platform.startswith("win") else ":" 

92 rep = sep + sep.join(rows) 

93 script = script.replace(key, rep) 

94 return script 

95 

96 

97def private_script_replacements(script, module, requirements, port, raise_exception=True, 

98 platform=sys.platform, default_engine_paths=None, 

99 additional_local_path=None): 

100 """ 

101 Runs last replacements. 

102 

103 @param script script or list of scripts 

104 @param module module name 

105 @param requirements requirements - (list or 2-uple of lists) 

106 @param port port 

107 @param raise_exception raise an exception if there is an error, otherwise, return None 

108 @param platform platform 

109 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below. 

110 @param additional_local_path additional local path to add to PYTHONPATH 

111 @return modified script 

112 

113 An example for *default_engine_paths*:: 

114 

115 default_engine_paths = { 

116 "windows": { 

117 "__PY35__": None, 

118 "__PY36_X64__": "c:\\Python365_x64", 

119 "__PY37_X64__": "c:\\Python372_x64", 

120 "__PY38_X64__": "c:\\Python387_x64", 

121 "__PY39_X64__": "c:\\Python391_x64", 

122 }, 

123 } 

124 

125 Parameter *requirements* can a list of requirements, 

126 we assume these requirements are available from a local PyPi server. 

127 There can be extra requirements obtained from PiPy. In that case, 

128 those can be specified as a tuple *(requirements_local, requirements_pipy)*. 

129 

130 The function replaces ``rem _PATH_VIRTUAL_ENV_`` 

131 with an instruction to copy these DLLs. 

132 Parameter *requirements* can be a list or a tuple. 

133 """ 

134 global default_values 

135 if default_engine_paths is None: 

136 default_engine_paths = default_values 

137 

138 if isinstance(script, list): 

139 return [private_script_replacements(s, module, requirements, 

140 port, raise_exception, platform, 

141 default_engine_paths=default_engine_paths) for s in script] 

142 

143 if platform.startswith("win"): 

144 plat = "windows" 

145 global _default_nofolder 

146 def_values = default_engine_paths 

147 

148 values = [v for v in def_values[ 

149 plat].values() if v is not None and v != _default_nofolder] 

150 if raise_exception and len(values) != len(set(values)): 

151 raise FileNotFoundError( # pragma: no cover 

152 "One path is wrong among:\n %s" % ( 

153 "\n".join("{0}={1}".format(k, v) 

154 for k, v in def_values[plat].items()))) 

155 

156 if module is not None: 

157 script = script.replace("__MODULE__", module) 

158 

159 for k, v in def_values[plat].items(): 

160 script = script.replace(k, v) 

161 

162 # requirements 

163 if requirements is not None: 

164 if isinstance(requirements, list): 

165 requirements_pipy = [] 

166 requirements_local = requirements 

167 else: 

168 requirements_local, requirements_pipy = requirements 

169 

170 if requirements_pipy is None: 

171 requirements_pipy = [] 

172 if requirements_local is None: 

173 requirements_local = [] 

174 

175 cj = "%jenkinspythonpip%" if "jenkinspythonpip" in script else "%pythonpip%" 

176 patternr = "install {0}" 

177 patternl = "install --no-cache-dir --index http://localhost:{0}/simple/ {1}" 

178 rows = [] 

179 for r in requirements_pipy: 

180 r = cj + " " + patternr.format(r) 

181 rows.append(r) 

182 for r in requirements_local: 

183 r = cj + " " + patternl.format(port, r) 

184 rows.append(r) 

185 

186 reqs = "\n".join(rows) 

187 else: 

188 reqs = "" 

189 

190 script = script.replace("__REQUIREMENTS__", reqs) \ 

191 .replace("__PORT__", str(port)) \ 

192 .replace("__USERNAME__", os.environ.get("USERNAME", os.environ.get("USER", "UNKNOWN-USER"))) 

193 

194 if "__ADDITIONAL_LOCAL_PATH__" in script: 

195 paths = [] 

196 if additional_local_path is not None and len(additional_local_path) > 0: 

197 paths.extend(additional_local_path) 

198 if len(paths) > 0: 

199 script = private_replacement_( 

200 script, paths, key="__ADDITIONAL_LOCAL_PATH__") 

201 else: 

202 script = script.replace("__ADDITIONAL_LOCAL_PATH__", "") 

203 

204 if "rem _PATH_VIRTUAL_ENV_" in script: 

205 script = script.replace( 

206 "rem _PATH_VIRTUAL_ENV_", "rem nothing to do here") 

207 

208 return script 

209 

210 else: 

211 if raise_exception: 

212 raise NotImplementedError( 

213 "not implemented yet for this platform %s" % sys.platform) 

214 return None 

215 

216 

217def get_build_script(module, requirements=None, port=8067, default_engine_paths=None, 

218 additional_local_path=None): 

219 """ 

220 Builds the build script which builds the setup, run the unit tests 

221 and the documentation. 

222 

223 @param module module name 

224 @param requirements list of dependencies (not in your python distribution) 

225 @param port port for the local pypi_server which gives the dependencies 

226 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below. 

227 @param additional_local_path additional paths to add to PYTHONPATH 

228 @return scripts 

229 """ 

230 if requirements is None: 

231 requirements = [] 

232 return private_script_replacements(windows_build, module, requirements, port, 

233 default_engine_paths=default_engine_paths, 

234 additional_local_path=additional_local_path) 

235 

236 

237def get_script_command(command, module, requirements, port=8067, platform=sys.platform, 

238 default_engine_paths=None, 

239 additional_local_path=None): # pragma: no cover 

240 """ 

241 Produces a script which runs a command available through the setup. 

242 

243 @param command command to run 

244 @param module module name 

245 @param requirements list of dependencies (not in your python distribution) 

246 @param port port for the local pypi_server which gives the dependencies 

247 @param platform platform (only Windows) 

248 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below. 

249 @param additional_local_path additional local path to add before running command ``setup.py <command>`` 

250 @return scripts 

251 

252 The available list of commands is given by function @see fn process_standard_options_for_setup. 

253 """ 

254 if not platform.startswith("win"): 

255 raise NotImplementedError( # pragma: no cover 

256 "not yet available on linux") 

257 global windows_error, windows_prefix, windows_setup 

258 rows = [windows_prefix] 

259 

260 if additional_local_path is not None and len(additional_local_path): 

261 addp = "set PYTHONPATH=%PYTHONPATH%;" + \ 

262 ";".join(private_path_choice(_) for _ in additional_local_path) 

263 else: 

264 addp = "" 

265 rows.append(windows_setup.replace( 

266 "rem set PYTHONPATH=additional_path", addp) + " " + command) 

267 rows.append(windows_error) 

268 sc = "\n".join(rows) 

269 res = private_script_replacements( 

270 sc, module, requirements, port, default_engine_paths=default_engine_paths, 

271 additional_local_path=additional_local_path) 

272 if sys.platform.startswith("win"): 

273 if command == "copy27": 

274 res = """ 

275 if exist dist_module27 ( 

276 rmdir /Q /S dist_module27 

277 if %errorlevel% neq 0 exit /b %errorlevel% 

278 ) 

279 """.replace(" ", "") + res 

280 elif command == "clean_space": 

281 # Run the test which test pep8 and convert the convert the 

282 # notebooks. 

283 res += """ 

284 if not exist _unittests\\ut_module\\test_code_style.py goto end: 

285 %pythonexe% -u _unittests\\ut_module\\test_code_style.py -v 

286 if %errorlevel% neq 0 exit /b %errorlevel% 

287 

288 if not exist _unittests\\ut_module\\test_convert_notebooks.py goto end: 

289 %pythonexe% -u _unittests\\ut_module\\test_convert_notebooks.py 

290 if %errorlevel% neq 0 exit /b %errorlevel% 

291 ) 

292 """.replace(" ", "") + res 

293 return res 

294 

295 

296def get_extra_script_command(command, module, requirements, port=8067, blog_list=None, platform=sys.platform, 

297 default_engine_paths=None, unit_test_folder=None, unittest_modules=None, 

298 additional_notebook_path=None, 

299 additional_local_path=None): # pragma: no cover 

300 """ 

301 Produces a script which runs the notebook, a documentation server, which 

302 publishes... 

303 

304 @param command command to run (*notebook*, *publish*, *publish_doc*, *local_pypi*, *setupdep*, 

305 *run27*, *build27*, *copy_dist*, *any_setup_command*, *lab*) 

306 @param module module name 

307 @param requirements list of dependencies (not in your python distribution) 

308 @param port port for the local pypi_server which gives the dependencies 

309 @param blog_list list of blog to listen for this module (usually stored in 

310 ``module.__blog__``) 

311 @param platform platform (only Windows) 

312 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below. 

313 @param unit_test_folder unit test folders, used for command ``run27`` 

314 @param additional_notebook_path additional paths to add when running the script launching the notebooks 

315 @param additional_local_path additional paths to add when running a local command 

316 @param unittest_modules list of modules to be used during unit tests 

317 @return scripts 

318 

319 The available list of commands is given by function @see fn process_standard_options_for_setup. 

320 """ 

321 if not platform.startswith("win"): 

322 raise NotImplementedError("linux not yet available") 

323 

324 script = None 

325 if command == "notebook": 

326 script = windows_notebook 

327 elif command == "lab": 

328 script = windows_notebook.replace("jupyter-notebook", "jupyter-lab") 

329 elif command == "publish": 

330 script = "\n".join([windows_prefix, windows_publish]) 

331 elif command == "publish_doc": 

332 script = "\n".join([windows_prefix, windows_publish_doc]) 

333 elif command == "local_pypi": 

334 script = "\n".join([windows_prefix, windows_pypi]) 

335 elif command == "run27": 

336 script = "\n".join( 

337 [windows_prefix_27, windows_unittest27, windows_error]) 

338 if unit_test_folder is None: 

339 raise FileNotFoundError( 

340 "the unit test folder must be specified and cannot be None") 

341 if not os.path.exists(unit_test_folder): 

342 raise FileNotFoundError( 

343 "the unit test folder must exist: " + unit_test_folder) 

344 ut_ = [("%pythonexe27%\\..\\Scripts\\nosetests.exe -w " + _) 

345 for _ in os.listdir(unit_test_folder) if _.startswith("ut_")] 

346 stut = "\nif %errorlevel% neq 0 exit /b %errorlevel%\n".join(ut_) 

347 script = script.replace("__LOOP_UNITTEST_FOLDERS__", stut) 

348 elif command == "build27": 

349 script = "\n".join([windows_prefix_27, "cd dist_module27", "rmdir /S /Q dist", 

350 windows_setup.replace( 

351 "exe%", "exe27%") + " bdist_wheel", 

352 windows_error, "cd ..", "copy dist_module27\\dist\\*.whl dist"]) 

353 elif command == "copy_dist": 

354 script = copy_dist_to_local_pypi 

355 elif command == "copy_sphinx": 

356 script = copy_sphinx_to_dist 

357 elif command == "setupdep": 

358 script = setup_script_dependency_py 

359 elif command == "any_setup_command": 

360 script = windows_any_setup_command 

361 elif command == "build_dist": 

362 script = windows_build_setup 

363 elif command == "history": 

364 script = "\n".join( 

365 [windows_prefix, '\n%pythonexe% %current%setup.py history\n']) 

366 else: 

367 raise Exception("unable to interpret command: " + command) 

368 

369 # additional paths 

370 if "__ADDITIONAL_LOCAL_PATH__" in script: 

371 paths = [] 

372 if command in ("notebook", "lab") and additional_notebook_path is not None and len(additional_notebook_path) > 0: 

373 paths.extend(additional_notebook_path) 

374 if unittest_modules is not None and len(unittest_modules) > 0: 

375 paths.extend(unittest_modules) 

376 if additional_local_path is not None and len(additional_local_path) > 0: 

377 paths.extend(additional_local_path) 

378 if len(paths) > 0: 

379 script = private_replacement_( 

380 script, paths, key="__ADDITIONAL_LOCAL_PATH__") 

381 else: 

382 script = script.replace("__ADDITIONAL_LOCAL_PATH__", "") 

383 

384 script = script.replace("__ADDITIONAL_NOTEBOOK_PATH__", "") 

385 

386 # common post-processing 

387 if script is None: 

388 raise Exception("unexpected command: " + command) 

389 return private_script_replacements(script, module, requirements, port, default_engine_paths=default_engine_paths) 

390 

391 

392def get_script_module(command, platform=sys.platform, blog_list=None, 

393 default_engine_paths=None): 

394 """ 

395 Produces a script which runs the notebook, a documentation server, which 

396 publishes and other scripts. 

397 

398 @param command command to run (*blog*) 

399 @param platform platform (only Windows) 

400 @param blog_list list of blog to listen for this module (usually stored in ``module.__blog__``) 

401 @param default_engine_paths define the default location for python engine, should be dictionary *{ engine: path }*, see below. 

402 @return scripts 

403 

404 The available list of commands is given by function @see fn process_standard_options_for_setup. 

405 """ 

406 prefix_setup = "" 

407 filename = os.path.abspath(__file__) 

408 if "site-packages" not in filename: 

409 folder = os.path.normpath( 

410 os.path.join(os.path.dirname(filename), "..", "..")) 

411 prefix_setup = """ 

412 import sys 

413 import os 

414 sys.path.append(r"{0}") 

415 sys.path.append(r"{1}") 

416 sys.path.append(r"{2}") 

417 """.replace(" ", "").format(folder, 

418 folder.replace( 

419 "pyquickhelper", "pyensae"), 

420 folder.replace( 

421 "pyquickhelper", "pyrsslocal") 

422 ) 

423 

424 script = None 

425 if command == "blog": 

426 if blog_list is None: 

427 return None 

428 else: 

429 list_xml = blog_list.strip("\n\r\t ") 

430 if '<?xml version="1.0" encoding="UTF-8"?>' not in list_xml and is_file_string(list_xml) and os.path.exists(list_xml): 

431 with open(list_xml, "r", encoding="utf8") as f: 

432 list_xml = f.read() 

433 if "<body>" not in list_xml: 

434 raise ValueError( # pragma: no cover 

435 "Wrong XML format:\n{0}".format(list_xml)) 

436 script = [("auto_rss_list.xml", list_xml)] 

437 script.append(("auto_rss_server.py", prefix_setup + """ 

438 from pyquickhelper.pycode.blog_helper import rss_update_run_server 

439 rss_update_run_server("auto_rss_database.db3", "auto_rss_list.xml") 

440 """.replace(" ", ""))) 

441 if platform.startswith("win"): 

442 script.append("\n".join([windows_prefix, windows_blogpost])) 

443 elif command == "doc": 

444 script = [] 

445 script.append(("auto_doc_server.py", prefix_setup + """ 

446 # address http://localhost:8079/ 

447 from pyquickhelper import fLOG 

448 from pyquickhelper.server import run_doc_server, get_jenkins_mappings 

449 fLOG(OutputPrint=True) 

450 fLOG("running documentation server") 

451 thisfile = os.path.dirname(__file__) 

452 mappings = get_jenkins_mappings(os.path.join(thisfile, "..")) 

453 fLOG("goto", "http://localhost:8079/") 

454 for k,v in sorted(mappings.items()): 

455 fLOG(k,"-->",v) 

456 run_doc_server(None, mappings=mappings) 

457 """.replace(" ", ""))) 

458 if platform.startswith("win"): 

459 script.append("\n".join([windows_prefix, "rem http://localhost:8079/", 

460 windows_docserver])) 

461 else: 

462 raise RuntimeError( # pragma: no cover 

463 "Unable to interpret command: %r" % command) 

464 

465 # common post-processing 

466 for i, item in enumerate(script): 

467 if isinstance(item, tuple): 

468 ext = os.path.splitext(item[0]) 

469 if ext == ".py": 

470 s = private_script_replacements( 

471 item[1], None, None, None, default_engine_paths=default_engine_paths) 

472 script[i] = (item[0], s) 

473 else: 

474 script[i] = private_script_replacements( 

475 item, None, None, None, default_engine_paths=default_engine_paths) 

476 return script 

477 

478 

479def get_pyproj_project(name, file_list): 

480 """ 

481 returns a string which corresponds to a pyproj project 

482 

483 @param name project name 

484 @param file_list file_list 

485 @return string 

486 """ 

487 guid = uuid.uuid3(uuid.NAMESPACE_DNS, name) 

488 folders = list(_ for _ in sorted(set(os.path.dirname(f) 

489 for f in file_list)) if len(_) > 0) 

490 sfold = "\n".join(' <Folder Include="%s\" />' % _ for _ in folders) 

491 sfiles = "\n".join(' <Compile Include="%s\" />' % _ for _ in file_list) 

492 

493 script = pyproj_template.replace("__GUID__", str(guid)) \ 

494 .replace("__NAME__", name) \ 

495 .replace("__INCLUDEFILES__", sfiles) \ 

496 .replace("__INCLUDEFOLDERS__", sfold) 

497 return script