Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2""" 

3@file 

4@brief Implements function @see fn run_cmd. 

5 

6.. versionadded:: 1.1 

7""" 

8from __future__ import print_function 

9import sys 

10import os 

11import time 

12import subprocess 

13import threading 

14import warnings 

15import shlex 

16if sys.version_info[0] == 2: 

17 import Queue as queue 

18else: 

19 import queue 

20 

21 

22class RunCmdException(Exception): 

23 """ 

24 raised by function @see fn run_cmd 

25 """ 

26 pass 

27 

28 

29def get_interpreter_path(): 

30 """ 

31 return the interpreter path 

32 """ 

33 if sys.platform.startswith("win"): 

34 return sys.executable.replace("pythonw.exe", "python.exe") 

35 else: 

36 return sys.executable 

37 

38 

39def split_cmp_command(cmd, remove_quotes=True): 

40 """ 

41 splits a command line 

42 @param cmd command line 

43 @param remove_quotes True by default 

44 @return list 

45 """ 

46 if isinstance(cmd, str # unicode# 

47 ): 

48 return shlex.split(cmd) 

49 else: 

50 return cmd 

51 

52 

53def decode_outerr(outerr, encoding, encerror, msg): 

54 """ 

55 decode the output or the error after running a command line instructions 

56 

57 @param outerr output or error 

58 @param encoding encoding (if None, it is replaced by ascii) 

59 @param encerror how to handle errors 

60 @param msg to add to the exception message 

61 @return converted string 

62 

63 .. versionchanged:: 1.4 

64 If *encoding* is None, it is replaced by ``'ascii'``. 

65 """ 

66 if encoding is None: 

67 encoding = "ascii" 

68 typstr = str # unicode# 

69 if not isinstance(outerr, bytes): 

70 raise TypeError( 

71 "only able to decode bytes, not " + typstr(type(outerr))) 

72 try: 

73 out = outerr.decode(encoding, errors=encerror) 

74 return out 

75 except UnicodeDecodeError as exu: 

76 try: 

77 out = outerr.decode( 

78 "utf8" if encoding != "utf8" else "latin-1", errors=encerror) 

79 return out 

80 except Exception as e: 

81 out = outerr.decode(encoding, errors='ignore') 

82 raise Exception("issue with cmd (" + encoding + "):" + 

83 typstr(msg) + "\n" + typstr(exu) + "\n-----\n" + out) from e 

84 raise Exception("complete issue with cmd:" + typstr(msg)) 

85 

86 

87def skip_run_cmd(cmd, sin="", shell=True, wait=False, log_error=True, 

88 stop_running_if=None, encerror="ignore", 

89 encoding="utf8", change_path=None, communicate=True, 

90 preprocess=True, timeout=None, catch_exit=False, fLOG=None, 

91 timeout_listen=None, tell_if_no_output=None): 

92 """ 

93 has the same signature as @see fn run_cmd but does nothing 

94 

95 .. versionadded:: 1.0 

96 """ 

97 return "", "" 

98 

99 

100def run_cmd_private(cmd, sin="", shell=True, wait=False, log_error=True, 

101 stop_running_if=None, encerror="ignore", encoding="utf8", 

102 change_path=None, communicate=True, preprocess=True, timeout=None, 

103 catch_exit=False, fLOG=None, tell_if_no_output=None): 

104 """ 

105 run a command line and wait for the result 

106 @param cmd command line 

107 @param sin sin: what must be written on the standard input 

108 @param shell if True, cmd is a shell command (and no command window is opened) 

109 @param wait call ``proc.wait`` 

110 @param log_error if log_error, call fLOG (error) 

111 @param stop_running_if the function stops waiting if some condition is fulfilled. 

112 The function received the last line from the logs. 

113 Signature: ``stop_waiting_if(last_out, last_err) -> bool``. 

114 The function must return True to stop waiting. 

115 This function can also be used to intercept the standard output 

116 and the standard error while running. 

117 @param encerror encoding errors (ignore by default) while converting the output into a string 

118 @param encoding encoding of the output 

119 @param change_path change the current path if not None (put it back after the execution) 

120 @param communicate use method `communicate <https://docs.python.org/3/library/subprocess.html#subprocess.Popen.communicate>`_ 

121 which is supposed to be safer, 

122 parameter ``wait`` must be True 

123 @param preprocess preprocess the command line if necessary (not available on Windows) (False to disable that option) 

124 @param timeout when data is sent to stdin (``sin``), a timeout is needed to avoid 

125 waiting for ever (*timeout* is in seconds) 

126 @param catch_exit catch *SystemExit* exception 

127 @param fLOG logging function (if not None, bypass others parameters) 

128 @param tell_if_no_output tells if there is no output every *tell_if_no_output* seconds 

129 @return content of stdout, stdres (only if wait is True) 

130 

131 .. exref:: 

132 :title: Run a program using the command line 

133 

134 :: 

135 

136 from pyquickhelper.loghelper import run_cmd 

137 out,err = run_cmd( "python setup.py install", wait=True) 

138 

139 If you are using this function to run git function, parameter ``shell`` must be True. 

140 

141 .. versionadded:: 1.1 

142 """ 

143 if fLOG is not None: 

144 fLOG("execute", cmd) 

145 

146 if sys.platform.startswith("win"): 

147 cmdl = cmd 

148 else: 

149 cmdl = split_cmp_command(cmd) if preprocess else cmd 

150 

151 if catch_exit: 

152 try: 

153 pproc = subprocess.Popen(cmdl, 

154 shell=shell, 

155 stdin=subprocess.PIPE if ( 

156 sin and len(sin) > 0) else None, 

157 stdout=subprocess.PIPE if wait else None, 

158 stderr=subprocess.PIPE if wait else None, 

159 cwd=change_path) 

160 except SystemExit as e: 

161 raise RunCmdException("SystemExit raised (1)") from e 

162 

163 else: 

164 pproc = subprocess.Popen(cmdl, 

165 shell=shell, 

166 stdin=subprocess.PIPE if ( 

167 sin and len(sin) > 0) else None, 

168 stdout=subprocess.PIPE if wait else None, 

169 stderr=subprocess.PIPE if wait else None, 

170 cwd=change_path) 

171 

172 if isinstance(cmd, list): 

173 cmd = " ".join(cmd) 

174 

175 if wait: 

176 skip_out_err = False 

177 out = [] 

178 err = [] 

179 err_read = False 

180 skip_waiting = False 

181 

182 if communicate: 

183 # communicate is True 

184 if tell_if_no_output is not None: 

185 raise NotImplementedError( 

186 "tell_if_no_output is not implemented when communicate is True") 

187 if stop_running_if is not None: 

188 raise NotImplementedError( 

189 "stop_running_if is not implemented when communicate is True") 

190 input = None if (sin is None or len(sin) > 0) else sin.encode() 

191 if input is not None and len(input) > 0: 

192 if fLOG is not None: 

193 fLOG("input", [input]) 

194 

195 if catch_exit: 

196 try: 

197 if sys.version_info[0] == 2: 

198 if timeout is not None: 

199 raise NotImplementedError( 

200 "timeout is only available with Python 3") 

201 stdoutdata, stderrdata = pproc.communicate(input=input) 

202 else: 

203 stdoutdata, stderrdata = pproc.communicate( 

204 input=input, timeout=timeout) 

205 except SystemExit as e: 

206 raise RunCmdException("SystemExit raised (2)") from e 

207 else: 

208 if sys.version_info[0] == 2: 

209 if timeout is not None: 

210 warnings.warn( 

211 "[run_cmd_private] timeout is only available with Python 3") 

212 stdoutdata, stderrdata = pproc.communicate(input=input) 

213 else: 

214 stdoutdata, stderrdata = pproc.communicate( 

215 input=input, timeout=timeout) 

216 

217 out = decode_outerr(stdoutdata, encoding, encerror, cmd) 

218 err = decode_outerr(stderrdata, encoding, encerror, cmd) 

219 else: 

220 # communicate is False: use of threads 

221 if sin is not None and len(sin) > 0: 

222 raise Exception( 

223 "communicate should be True to send something on stdin") 

224 stdout, stderr = pproc.stdout, pproc.stderr 

225 

226 begin = time.perf_counter() 

227 last_update = begin 

228 # with threads 

229 (stdoutReader, stdoutQueue) = _AsyncLineReader.getForFd( 

230 stdout, catch_exit=catch_exit) 

231 (stderrReader, stderrQueue) = _AsyncLineReader.getForFd( 

232 stderr, catch_exit=catch_exit) 

233 runloop = True 

234 

235 while (not stdoutReader.eof() or not stderrReader.eof()) and runloop: 

236 while not stdoutQueue.empty(): 

237 line = stdoutQueue.get() 

238 decol = decode_outerr( 

239 line, encoding, encerror, cmd) 

240 if fLOG is not None: 

241 fLOG(decol.strip("\n\r")) 

242 out.append(decol.strip("\n\r")) 

243 last_update = time.perf_counter() 

244 if stop_running_if is not None and stop_running_if(decol, None): 

245 runloop = False 

246 break 

247 

248 while not stderrQueue.empty(): 

249 line = stderrQueue.get() 

250 decol = decode_outerr( 

251 line, encoding, encerror, cmd) 

252 if fLOG is not None: 

253 fLOG(decol.strip("\n\r")) 

254 err.append(decol.strip("\n\r")) 

255 last_update = time.perf_counter() 

256 if stop_running_if is not None and stop_running_if(None, decol): 

257 runloop = False 

258 break 

259 time.sleep(0.05) 

260 

261 delta = time.perf_counter() - last_update 

262 if tell_if_no_output is not None and delta >= tell_if_no_output: 

263 if fLOG is not None: 

264 fLOG("[run_cmd] No update in {0} seconds for cmd: {1}".format( 

265 "%5.1f" % (last_update - begin), cmd)) 

266 last_update = time.perf_counter() 

267 full_delta = time.perf_counter() - begin 

268 if timeout is not None and full_delta > timeout: 

269 runloop = False 

270 if fLOG is not None: 

271 fLOG("[run_cmd] Timeout after {0} seconds for cmd: {1}".format( 

272 "%5.1f" % full_delta, cmd)) 

273 break 

274 

275 if runloop: 

276 # Waiting for async readers to finish... 

277 stdoutReader.join() 

278 stderrReader.join() 

279 

280 # Waiting for process to exit... 

281 returnCode = pproc.wait() 

282 err_read = True 

283 

284 if returnCode != 0: 

285 try: 

286 # we try to close the ressources 

287 stdout.close() 

288 stderr.close() 

289 except Exception as e: 

290 warnings.warn("Unable to close stdout and sterr.") 

291 if catch_exit: 

292 raise RunCmdException("SystemExit raised with error code {0}\nOUT:\n{1}\nERR-0:\n{2}".format( 

293 returnCode, "\n".join(out), "\n".join(err))) 

294 raise subprocess.CalledProcessError(returnCode, cmd) 

295 

296 if not skip_waiting: 

297 pproc.wait() 

298 else: 

299 out.append("[run_cmd] killing process.") 

300 if fLOG is not None: 

301 fLOG( 

302 "[run_cmd] killing process because stop_running_if returned True.") 

303 pproc.kill() 

304 err_read = True 

305 if fLOG is not None: 

306 fLOG("[run_cmd] process killed.") 

307 skip_out_err = True 

308 

309 out = "\n".join(out) 

310 if skip_out_err: 

311 err = "Process killed." 

312 else: 

313 if err_read: 

314 err = "\n".join(err) 

315 else: 

316 temp = err = stderr.read() 

317 try: 

318 err = decode_outerr(temp, encoding, encerror, cmd) 

319 except Exception: 

320 err = decode_outerr(temp, encoding, "ignore", cmd) 

321 stdout.close() 

322 stderr.close() 

323 

324 # same path for whether communicate is False or True 

325 err = err.replace("\r\n", "\n") 

326 if fLOG is not None: 

327 fLOG("end of execution", cmd) 

328 

329 if len(err) > 0 and log_error and fLOG is not None: 

330 fLOG("error (log)\n%s" % err) 

331 

332 if sys.platform.startswith("win"): 

333 return out.replace("\r\n", "\n"), err.replace("\r\n", "\n") 

334 else: 

335 return out, err 

336 else: 

337 

338 return "", "" 

339 

340 

341class _AsyncLineReader(threading.Thread): 

342 

343 def __init__(self, fd, outputQueue, catch_exit): 

344 threading.Thread.__init__(self) 

345 

346 assert isinstance(outputQueue, queue.Queue) 

347 assert callable(fd.readline) 

348 

349 self.fd = fd 

350 self.catch_exit = catch_exit 

351 self.outputQueue = outputQueue 

352 

353 def run(self): 

354 if self.catch_exit: 

355 try: 

356 for _ in map(self.outputQueue.put, iter(self.fd.readline, b'')): 

357 pass 

358 except SystemExit as e: 

359 self.outputQueue.put(str(e)) 

360 raise RunCmdException("SystemExit raised (3)") from e 

361 else: 

362 for _ in map(self.outputQueue.put, iter(self.fd.readline, b'')): 

363 pass 

364 

365 def eof(self): 

366 return not self.is_alive() and self.outputQueue.empty() 

367 

368 @classmethod 

369 def getForFd(cls, fd, start=True, catch_exit=False): 

370 q = queue.Queue() 

371 reader = cls(fd, q, catch_exit) 

372 

373 if start: 

374 reader.start() 

375 

376 return reader, q 

377 

378 

379def run_cmd_old(cmd, sin="", shell=False, wait=False, log_error=True, 

380 secure=None, stop_waiting_if=None, do_not_log=False, 

381 encerror="ignore", encoding="utf8", cwd=None, fLOG=print): 

382 """ 

383 run a command line and wait for the result 

384 @param cmd command line 

385 @param sin sin, what must be written on the standard input 

386 @param shell if True, cmd is a shell command (and no command window is opened) 

387 @param wait call proc.wait 

388 @param log_error if log_error, call fLOG (error) 

389 @param secure if secure is a string (a valid filename), the function stores the output in a file 

390 and reads it continuously 

391 @param stop_waiting_if the function stops waiting if some condition is fulfilled. 

392 The function received the last line from the logs. 

393 @param do_not_log do not log the output 

394 @param encerror encoding errors (ignore by default) while converting the output into a string 

395 @param encoding encoding of the output 

396 @param cwd current folder 

397 @param fLOG logging function 

398 @return content of stdout, stderr (only if wait is True) 

399 

400 

401 .. faqref:: 

402 :title: Exception when installing a module 

403 

404 This error can occur when a module is installed on a virtual environment 

405 created before *pip* was updated on the main distribution. 

406 The solution consists in removing the virtual environment and create it again. 

407 

408 :: 

409 

410 c:\\Python34_x64vir\\install\\Scripts\\python -u setup.py install 

411 running install 

412 running bdist_egg 

413 running egg_info 

414 Traceback (most recent call last): 

415 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2785, in _dep_map 

416 return self.__dep_map 

417 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2642, in __getattr__ 

418 raise AttributeError(attr) 

419 AttributeError: _DistInfoDistribution__dep_map 

420 

421 During handling of the above exception, another exception occurred: 

422 

423 Traceback (most recent call last): 

424 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2776, in _parsed_pkg_info 

425 return self._pkg_info 

426 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2642, in __getattr__ 

427 raise AttributeError(attr) 

428 AttributeError: _pkg_info 

429 

430 During handling of the above exception, another exception occurred: 

431 

432 Traceback (most recent call last): 

433 File "setup.py", line 169, in <module> 

434 package_data=package_data, 

435 File "C:\\Python34_x64\\Lib\\distutils\\core.py", line 148, in setup 

436 dist.run_commands() 

437 File "C:\\Python34_x64\\Lib\\distutils\\dist.py", line 955, in run_commands 

438 self.run_command(cmd) 

439 File "C:\\Python34_x64\\Lib\\distutils\\dist.py", line 974, in run_command 

440 cmd_obj.run() 

441 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\setuptools\\command\\install.py", line 67, in run 

442 self.do_egg_install() 

443 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\setuptools\\command\\install.py", line 109, in do_egg_install 

444 self.run_command('bdist_egg') 

445 File "C:\\Python34_x64\\Lib\\distutils\\cmd.py", line 313, in run_command 

446 self.distribution.run_command(command) 

447 File "C:\\Python34_x64\\Lib\\distutils\\dist.py", line 974, in run_command 

448 cmd_obj.run() 

449 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\setuptools\\command\\bdist_egg.py", line 151, in run 

450 self.run_command("egg_info") 

451 File "C:\\Python34_x64\\Lib\\distutils\\cmd.py", line 313, in run_command 

452 self.distribution.run_command(command) 

453 File "C:\\Python34_x64\\Lib\\distutils\\dist.py", line 974, in run_command 

454 cmd_obj.run() 

455 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\setuptools\\command\\egg_info.py", line 171, in run 

456 ep.require(installer=installer) 

457 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2355, in require 

458 items = working_set.resolve(reqs, env, installer) 

459 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 835, in resolve 

460 new_requirements = dist.requires(req.extras)[::-1] 

461 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2586, in requires 

462 dm = self._dep_map 

463 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2787, in _dep_map 

464 self.__dep_map = self._compute_dependencies() 

465 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2809, in _compute_dependencies 

466 for req in self._parsed_pkg_info.get_all('Requires-Dist') or []: 

467 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2778, in _parsed_pkg_info 

468 metadata = self.get_metadata(self.PKG_INFO) 

469 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 1993, in get_metadata 

470 raise KeyError("No metadata except PKG-INFO is available") 

471 KeyError: 'No metadata except PKG-INFO is available' 

472 """ 

473 if sin is not None and sin != "": 

474 raise NotImplementedError("sin is not used") 

475 

476 if secure is not None: 

477 if not do_not_log: 

478 fLOG("secure=", secure) 

479 with open(secure, "w") as f: 

480 f.write("") 

481 add = ">%s" % secure 

482 if isinstance(cmd, str): 

483 cmd += " " + add 

484 else: 

485 cmd.append(add) 

486 if not do_not_log and fLOG is not None: 

487 fLOG("execute ", cmd) 

488 

489 if sys.platform.startswith("win"): 

490 

491 startupinfo = subprocess.STARTUPINFO() 

492 startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW 

493 

494 try: 

495 proc = subprocess.Popen(cmd, shell=shell, 

496 stdout=subprocess.PIPE, stderr=subprocess.PIPE, 

497 startupinfo=startupinfo, cwd=cwd) 

498 except FileNotFoundError as e: 

499 raise RunCmdException("unable to run CMD:\n{0}".format(cmd)) from e 

500 else: 

501 try: 

502 proc = subprocess.Popen(split_cmp_command(cmd), shell=shell, 

503 stdout=subprocess.PIPE, stderr=subprocess.PIPE, 

504 cwd=cwd) 

505 except FileNotFoundError as e: 

506 raise RunCmdException("unable to run CMD:\n{0}".format(cmd)) from e 

507 if wait: 

508 

509 out = [] 

510 skip_waiting = False 

511 

512 if secure is None: 

513 for line in proc.stdout: 

514 if not do_not_log and fLOG is not None: 

515 fLOG(line.decode(encoding, errors=encerror).strip("\n")) 

516 try: 

517 out.append( 

518 line.decode( 

519 encoding, 

520 errors=encerror).strip("\n")) 

521 except UnicodeDecodeError as exu: 

522 raise RunCmdException( 

523 "issue with cmd:" + 

524 str(cmd) + 

525 "\n" + 

526 str(exu)) 

527 if proc.stdout.closed: 

528 break 

529 if stop_waiting_if is not None and stop_waiting_if( 

530 line.decode("utf8", errors=encerror)): 

531 skip_waiting = True 

532 break 

533 else: 

534 last = [] 

535 while proc.poll() is None: 

536 if os.path.exists(secure): 

537 with open(secure, "r") as f: 

538 lines = f.readlines() 

539 if len(lines) > len(last): 

540 for line in lines[len(last):]: 

541 if not do_not_log and fLOG is not None: 

542 fLOG(line.strip("\n")) 

543 out.append(line.strip("\n")) 

544 last = lines 

545 if stop_waiting_if is not None and len( 

546 last) > 0 and stop_waiting_if(last[-1]): 

547 skip_waiting = True 

548 break 

549 time.sleep(0.1) 

550 

551 if not skip_waiting: 

552 proc.wait() 

553 

554 out = "\n".join(out) 

555 err = proc.stderr.read().decode(encoding, errors=encerror) 

556 if not do_not_log and fLOG is not None: 

557 fLOG("end of execution ", cmd) 

558 if len(err) > 0 and log_error and not do_not_log and fLOG is not None: 

559 fLOG("error (log)\n%s" % err) 

560 # return bytes.decode (out, errors="ignore"), bytes.decode(err, 

561 # errors="ignore") 

562 proc.stdout.close() 

563 proc.stderr.close() 

564 

565 return out, err 

566 else: 

567 return "", ""