Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2""" 

3@file 

4@brief Implements function @see fn run_cmd. 

5""" 

6import sys 

7import os 

8import time 

9import subprocess 

10import threading 

11import warnings 

12import re 

13import queue 

14from .flog_fake_classes import PQHException 

15 

16 

17class RunCmdException(Exception): 

18 """ 

19 Raised by function @see fn run_cmd. 

20 """ 

21 pass 

22 

23 

24def get_interpreter_path(): 

25 """ 

26 Returns the interpreter path. 

27 """ 

28 if sys.platform.startswith("win"): 

29 return sys.executable.replace( # pragma: no cover 

30 "pythonw.exe", "python.exe") 

31 else: 

32 return sys.executable 

33 

34 

35def split_cmp_command(cmd, remove_quotes=True): 

36 """ 

37 Splits a command line. 

38 

39 @param cmd command line 

40 @param remove_quotes True by default 

41 @return list 

42 """ 

43 if isinstance(cmd, str): 

44 spl = cmd.split() 

45 res = [] 

46 for s in spl: 

47 if len(res) == 0: 

48 res.append(s) 

49 elif res[-1].startswith('"') and not res[-1].endswith('"'): 

50 res[-1] += " " + s 

51 else: 

52 res.append(s) 

53 if remove_quotes: 

54 nres = [] 

55 for _ in res: 

56 if _.startswith('"') and _.endswith('"'): 

57 nres.append(_.strip('"')) 

58 else: 

59 nres.append(_) 

60 return nres 

61 else: 

62 return res 

63 else: 

64 return cmd 

65 

66 

67def decode_outerr(outerr, encoding, encerror, msg): 

68 """ 

69 Decodes the output or the error after running a command line instructions. 

70 

71 @param outerr output or error 

72 @param encoding encoding (if None, it is replaced by ascii) 

73 @param encerror how to handle errors 

74 @param msg to add to the exception message 

75 @return converted string 

76 """ 

77 if encoding is None: 

78 encoding = "ascii" 

79 typstr = str 

80 if not isinstance(outerr, bytes): 

81 raise TypeError( # pragma: no cover 

82 "only able to decode bytes, not " + typstr(type(outerr))) 

83 try: 

84 out = outerr.decode(encoding, errors=encerror) 

85 return out 

86 except UnicodeDecodeError as exu: 

87 try: 

88 out = outerr.decode( 

89 "utf8" if encoding != "utf8" else "latin-1", errors=encerror) 

90 return out 

91 except Exception as e: # pragma: no cover 

92 out = outerr.decode(encoding, errors='ignore') 

93 raise Exception("issue with cmd (" + encoding + "):" + 

94 typstr(msg) + "\n" + typstr(exu) + "\n-----\n" + out) from e 

95 raise Exception( # pragma: no cover 

96 "complete issue with cmd:" + typstr(msg)) 

97 

98 

99def skip_run_cmd(cmd, sin="", shell=True, wait=False, log_error=True, 

100 stop_running_if=None, encerror="ignore", 

101 encoding="utf8", change_path=None, communicate=True, 

102 preprocess=True, timeout=None, catch_exit=False, fLOG=None, 

103 timeout_listen=None, tell_if_no_output=None, prefix_log=None): 

104 """ 

105 Has the same signature as @see fn run_cmd but does nothing. 

106 """ 

107 return "", "" 

108 

109 

110def run_cmd(cmd, sin="", shell=sys.platform.startswith("win"), wait=False, log_error=True, 

111 stop_running_if=None, encerror="ignore", encoding="utf8", 

112 change_path=None, communicate=True, preprocess=True, timeout=None, 

113 catch_exit=False, fLOG=None, tell_if_no_output=None, prefix_log=None): 

114 """ 

115 Runs a command line and wait for the result. 

116 

117 @param cmd command line 

118 @param sin sin: what must be written on the standard input 

119 @param shell if True, cmd is a shell command (and no command window is opened) 

120 @param wait call ``proc.wait`` 

121 @param log_error if log_error, call fLOG (error) 

122 @param stop_running_if the function stops waiting if some condition is fulfilled. 

123 The function received the last line from the logs. 

124 Signature: ``stop_waiting_if(last_out, last_err) -> bool``. 

125 The function must return True to stop waiting. 

126 This function can also be used to intercept the standard output 

127 and the standard error while running. 

128 @param encerror encoding errors (ignore by default) while converting the output into a string 

129 @param encoding encoding of the output 

130 @param change_path change the current path if not None (put it back after the execution) 

131 @param communicate use method `communicate 

132 <https://docs.python.org/3/library/subprocess.html#subprocess.Popen.communicate>`_ 

133 which is supposed to be safer, parameter ``wait`` must be True 

134 @param preprocess preprocess the command line if necessary (not available on Windows) 

135 (False to disable that option) 

136 @param timeout when data is sent to stdin (``sin``), a timeout is needed to avoid 

137 waiting for ever (*timeout* is in seconds) 

138 @param catch_exit catch *SystemExit* exception 

139 @param fLOG logging function (if not None, bypass others parameters) 

140 @param tell_if_no_output tells if there is no output every *tell_if_no_output* seconds 

141 @param prefix_log add a prefix to a line before printing it 

142 @return content of stdout, stdres (only if wait is True) 

143 

144 .. exref:: 

145 :title: Run a program using the command line 

146 

147 :: 

148 

149 from pyquickhelper.loghelper import run_cmd 

150 out, err = run_cmd("python setup.py install", wait=True) 

151 

152 If you are using this function to run :epkg:`git` function, parameter ``shell`` must be True. 

153 The function catches *SystemExit* exception. 

154 See `Constantly print Subprocess output while process is running 

155 <http://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running/4417735>`_. 

156 If *wait* is False, the function returns the started process. 

157 ``__exit__`` should be called if wait if False. 

158 Parameter *prefix_log* was added. 

159 """ 

160 if prefix_log is None: 

161 prefix_log = "" 

162 if fLOG is not None: 

163 if isinstance(cmd, (list, tuple)): 

164 fLOG( # pragma: no cover 

165 prefix_log + "[run_cmd] execute", " ".join(cmd)) 

166 else: 

167 fLOG(prefix_log + "[run_cmd] execute", cmd) 

168 

169 if change_path is not None: 

170 current = os.getcwd() 

171 os.chdir(change_path) 

172 

173 if sys.platform.startswith("win"): 

174 cmdl = cmd 

175 else: 

176 cmdl = split_cmp_command(cmd) if preprocess else cmd 

177 

178 if catch_exit: 

179 try: 

180 pproc = subprocess.Popen(cmdl, 

181 shell=shell, 

182 stdin=subprocess.PIPE if sin is not None and len( 

183 sin) > 0 else None, 

184 stdout=subprocess.PIPE if wait else None, 

185 stderr=subprocess.PIPE if wait else None) 

186 except SystemExit as e: 

187 if change_path is not None: # pragma: no cover 

188 os.chdir(current) 

189 raise RunCmdException( # pragma: no cover 

190 "SystemExit raised (1)") from e 

191 

192 else: 

193 pproc = subprocess.Popen(cmdl, 

194 shell=shell, 

195 stdin=subprocess.PIPE if sin is not None and len( 

196 sin) > 0 else None, 

197 stdout=subprocess.PIPE if wait else None, 

198 stderr=subprocess.PIPE if wait else None) 

199 

200 pproc.__enter__() 

201 if isinstance(cmd, list): 

202 cmd = " ".join(cmd) 

203 

204 if wait: 

205 skip_out_err = False 

206 out = [] 

207 err = [] 

208 err_read = False 

209 skip_waiting = False 

210 

211 if communicate: 

212 # communicate is True 

213 if tell_if_no_output is not None: 

214 raise NotImplementedError( 

215 "tell_if_no_output is not implemented when communicate is True") 

216 if stop_running_if is not None: 

217 raise NotImplementedError( 

218 "stop_running_if is not implemented when communicate is True") 

219 input = sin if sin is None else sin.encode() 

220 if input is not None and len(input) > 0: 

221 if fLOG is not None: 

222 fLOG(prefix_log + "[run_cmd] input", [input]) 

223 

224 if catch_exit: 

225 try: 

226 stdoutdata, stderrdata = pproc.communicate( 

227 input=input, timeout=timeout) 

228 except SystemExit as e: # pragma: no cover 

229 if change_path is not None: 

230 os.chdir(current) 

231 raise RunCmdException("SystemExit raised (2)") from e 

232 else: 

233 stdoutdata, stderrdata = pproc.communicate( 

234 input=input, timeout=timeout) 

235 

236 out = decode_outerr(stdoutdata, encoding, encerror, cmd) 

237 err = decode_outerr(stderrdata, encoding, encerror, cmd) 

238 else: 

239 # communicate is False: use of threads 

240 if sin is not None and len(sin) > 0: 

241 if change_path is not None: 

242 os.chdir(current) # pragma: no cover 

243 raise Exception( 

244 "communicate should be True to send something on stdin") 

245 stdout, stderr = pproc.stdout, pproc.stderr 

246 

247 begin = time.perf_counter() 

248 last_update = begin 

249 # with threads 

250 (stdoutReader, stdoutQueue) = _AsyncLineReader.getForFd( 

251 stdout, catch_exit=catch_exit) 

252 (stderrReader, stderrQueue) = _AsyncLineReader.getForFd( 

253 stderr, catch_exit=catch_exit) 

254 runloop = True 

255 

256 while (not stdoutReader.eof() or not stderrReader.eof()) and runloop: 

257 while not stdoutQueue.empty(): 

258 line = stdoutQueue.get() 

259 decol = decode_outerr( 

260 line, encoding, encerror, cmd) 

261 sdecol = decol.strip("\n\r") 

262 if fLOG is not None: 

263 fLOG(prefix_log + sdecol) 

264 out.append(sdecol) 

265 last_update = time.perf_counter() 

266 if stop_running_if is not None and stop_running_if(decol, None): 

267 runloop = False 

268 break 

269 

270 while not stderrQueue.empty(): 

271 line = stderrQueue.get() 

272 decol = decode_outerr( 

273 line, encoding, encerror, cmd) 

274 sdecol = decol.strip("\n\r") 

275 if fLOG is not None: 

276 fLOG(prefix_log + sdecol) 

277 err.append(sdecol) 

278 last_update = time.perf_counter() 

279 if stop_running_if is not None and stop_running_if(None, decol): 

280 runloop = False 

281 break 

282 time.sleep(0.05) 

283 

284 delta = time.perf_counter() - last_update 

285 if tell_if_no_output is not None and delta >= tell_if_no_output: 

286 fLOG( # pragma: no cover 

287 prefix_log + "[run_cmd] No update in {0} seconds for cmd: {1}".format( 

288 "%5.1f" % (last_update - begin), cmd)) 

289 last_update = time.perf_counter() # pragma: no cover 

290 full_delta = time.perf_counter() - begin 

291 if timeout is not None and full_delta > timeout: 

292 runloop = False # pragma: no cover 

293 fLOG( # pragma: no cover 

294 prefix_log + "[run_cmd] Timeout after {0} seconds for cmd: {1}".format( 

295 "%5.1f" % full_delta, cmd)) 

296 break # pragma: no cover 

297 

298 if runloop: 

299 # Waiting for async readers to finish... 

300 stdoutReader.join() 

301 stderrReader.join() 

302 

303 # Waiting for process to exit... 

304 returnCode = pproc.wait() 

305 err_read = True 

306 

307 if returnCode != 0: # pragma: no cover 

308 if change_path is not None: 

309 os.chdir(current) 

310 try: 

311 # we try to close the ressources 

312 stdout.close() 

313 stderr.close() 

314 except Exception as e: 

315 warnings.warn( 

316 "Unable to close stdout and sterr.", RuntimeWarning) 

317 if catch_exit: 

318 mes = "SystemExit raised with error code {0}\nCMD:\n{1}\nCWD:\n{2}\n#---OUT---#\n{3}\n#---ERR---#\n{4}" 

319 raise RunCmdException(mes.format( 

320 returnCode, cmd, os.getcwd(), "\n".join(out), "\n".join(err))) 

321 raise subprocess.CalledProcessError(returnCode, cmd) 

322 

323 if not skip_waiting: 

324 pproc.wait() 

325 else: # pragma: no cover 

326 out.append("[run_cmd] killing process.") 

327 fLOG( 

328 prefix_log + "[run_cmd] killing process because stop_running_if returned True.") 

329 pproc.kill() 

330 err_read = True 

331 fLOG(prefix_log + "[run_cmd] process killed.") 

332 skip_out_err = True 

333 

334 out = "\n".join(out) 

335 if skip_out_err: 

336 err = "Process killed." # pragma: no cover 

337 else: 

338 if err_read: 

339 err = "\n".join(err) 

340 else: # pragma: no cover 

341 temp = err = stderr.read() 

342 try: 

343 err = decode_outerr(temp, encoding, encerror, cmd) 

344 except Exception: 

345 err = decode_outerr(temp, encoding, "ignore", cmd) 

346 stdout.close() 

347 stderr.close() 

348 

349 # same path for whether communicate is False or True 

350 err = err.replace("\r\n", "\n") 

351 if fLOG is not None: 

352 fLOG(prefix_log + "end of execution", cmd) 

353 

354 if len(err) > 0 and log_error and fLOG is not None: 

355 if "\n" in err: 

356 fLOG(prefix_log + "[run_cmd] stderr (log)") 

357 for eline in err.split("\n"): 

358 fLOG(prefix_log + eline) 

359 else: 

360 fLOG( # pragma: no cover 

361 prefix_log + "[run_cmd] stderr (log)\n%s" % err) 

362 

363 if change_path is not None: 

364 os.chdir(current) 

365 

366 pproc.__exit__(None, None, None) 

367 if sys.platform.startswith("win"): # pragma: no cover 

368 if err is not None: 

369 err = err.strip("\n\r\t ") 

370 return out.replace("\r\n", "\n"), err.replace("\r\n", "\n") 

371 else: 

372 if err is not None: 

373 err = err.strip("\n\r\t ") 

374 return out, err 

375 else: 

376 

377 if change_path is not None: 

378 os.chdir(current) # pragma: no cover 

379 

380 return pproc, None 

381 

382 

383def parse_exception_message(exc): 

384 """ 

385 Parses the message embedded in an exception and returns the standard output and error 

386 if it can be found. 

387 

388 @param exc exception coming from @see fn run_cmd 

389 @return out, err 

390 """ 

391 mes = str(exc) 

392 reg = re.compile(".*#---OUT---#(.*)#---ERR---#(.*)", re.DOTALL) 

393 find = reg.search(mes.replace("\r", "")) 

394 if find: # pragma: no cover 

395 gr = find.groups() 

396 out, err = gr[0], gr[1] 

397 return out.strip("\n "), err.strip("\n ") 

398 else: 

399 return None, None 

400 

401 

402def run_script(script, *args, **kwargs): 

403 """ 

404 Runs a script. 

405 

406 @param script script to execute or command line starting with ``-m`` 

407 @param args other parameters 

408 @param kwargs sent to @see fn run_cmd 

409 @return out,err: content of stdout stream and stderr stream 

410 

411 Allows command line starting with ``-m``. 

412 """ 

413 if not script.startswith('-m') and not os.path.exists(script): 

414 raise PQHException("file %s not found" % script) 

415 py = get_interpreter_path() 

416 cmd = "%s %s" % (py, script) 

417 if len(args) > 0: 

418 typstr = str 

419 cmd += " " + " ".join([typstr(x) for x in args]) 

420 out, err = run_cmd(cmd, **kwargs) 

421 return out, err 

422 

423 

424class _AsyncLineReader(threading.Thread): 

425 

426 def __init__(self, fd, outputQueue, catch_exit): 

427 threading.Thread.__init__(self) 

428 

429 assert isinstance(outputQueue, queue.Queue) 

430 assert callable(fd.readline) 

431 

432 self.fd = fd 

433 self.catch_exit = catch_exit 

434 self.outputQueue = outputQueue 

435 

436 def run(self): 

437 if self.catch_exit: 

438 try: 

439 for _ in map(self.outputQueue.put, iter(self.fd.readline, b'')): 

440 pass 

441 except SystemExit as e: # pragma: no cover 

442 self.outputQueue.put(str(e)) 

443 raise RunCmdException("SystemExit raised (3)") from e 

444 else: 

445 for _ in map(self.outputQueue.put, iter(self.fd.readline, b'')): 

446 pass 

447 

448 def eof(self): 

449 return not self.is_alive() and self.outputQueue.empty() 

450 

451 @classmethod 

452 def getForFd(cls, fd, start=True, catch_exit=False): 

453 q = queue.Queue() 

454 reader = cls(fd, q, catch_exit) 

455 

456 if start: 

457 reader.start() 

458 

459 return reader, q