Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2""" 

3@file 

4@brief Implements function @see fn run_cmd. 

5""" 

6import sys 

7import os 

8import time 

9import subprocess 

10import threading 

11import warnings 

12import re 

13import queue 

14from .flog_fake_classes import PQHException 

15 

16 

17class RunCmdException(Exception): 

18 """ 

19 Raised by function @see fn run_cmd. 

20 """ 

21 pass 

22 

23 

24def get_interpreter_path(): 

25 """ 

26 Returns the interpreter path. 

27 """ 

28 if sys.platform.startswith("win"): 

29 return sys.executable.replace("pythonw.exe", "python.exe") 

30 else: 

31 return sys.executable 

32 

33 

34def split_cmp_command(cmd, remove_quotes=True): 

35 """ 

36 Splits a command line. 

37 

38 @param cmd command line 

39 @param remove_quotes True by default 

40 @return list 

41 """ 

42 if isinstance(cmd, str): 

43 spl = cmd.split() 

44 res = [] 

45 for s in spl: 

46 if len(res) == 0: 

47 res.append(s) 

48 elif res[-1].startswith('"') and not res[-1].endswith('"'): 

49 res[-1] += " " + s 

50 else: 

51 res.append(s) 

52 if remove_quotes: 

53 nres = [] 

54 for _ in res: 

55 if _.startswith('"') and _.endswith('"'): 

56 nres.append(_.strip('"')) 

57 else: 

58 nres.append(_) 

59 return nres 

60 else: 

61 return res 

62 else: 

63 return cmd 

64 

65 

66def decode_outerr(outerr, encoding, encerror, msg): 

67 """ 

68 Decodes the output or the error after running a command line instructions. 

69 

70 @param outerr output or error 

71 @param encoding encoding (if None, it is replaced by ascii) 

72 @param encerror how to handle errors 

73 @param msg to add to the exception message 

74 @return converted string 

75 """ 

76 if encoding is None: 

77 encoding = "ascii" 

78 typstr = str 

79 if not isinstance(outerr, bytes): 

80 raise TypeError( 

81 "only able to decode bytes, not " + typstr(type(outerr))) 

82 try: 

83 out = outerr.decode(encoding, errors=encerror) 

84 return out 

85 except UnicodeDecodeError as exu: 

86 try: 

87 out = outerr.decode( 

88 "utf8" if encoding != "utf8" else "latin-1", errors=encerror) 

89 return out 

90 except Exception as e: 

91 out = outerr.decode(encoding, errors='ignore') 

92 raise Exception("issue with cmd (" + encoding + "):" + 

93 typstr(msg) + "\n" + typstr(exu) + "\n-----\n" + out) from e 

94 raise Exception("complete issue with cmd:" + typstr(msg)) 

95 

96 

97def skip_run_cmd(cmd, sin="", shell=True, wait=False, log_error=True, 

98 stop_running_if=None, encerror="ignore", 

99 encoding="utf8", change_path=None, communicate=True, 

100 preprocess=True, timeout=None, catch_exit=False, fLOG=None, 

101 timeout_listen=None, tell_if_no_output=None, prefix_log=None): 

102 """ 

103 Has the same signature as @see fn run_cmd but does nothing. 

104 """ 

105 return "", "" 

106 

107 

108def run_cmd(cmd, sin="", shell=sys.platform.startswith("win"), wait=False, log_error=True, 

109 stop_running_if=None, encerror="ignore", encoding="utf8", 

110 change_path=None, communicate=True, preprocess=True, timeout=None, 

111 catch_exit=False, fLOG=None, tell_if_no_output=None, prefix_log=None): 

112 """ 

113 Runs a command line and wait for the result. 

114 

115 @param cmd command line 

116 @param sin sin: what must be written on the standard input 

117 @param shell if True, cmd is a shell command (and no command window is opened) 

118 @param wait call ``proc.wait`` 

119 @param log_error if log_error, call fLOG (error) 

120 @param stop_running_if the function stops waiting if some condition is fulfilled. 

121 The function received the last line from the logs. 

122 Signature: ``stop_waiting_if(last_out, last_err) -> bool``. 

123 The function must return True to stop waiting. 

124 This function can also be used to intercept the standard output 

125 and the standard error while running. 

126 @param encerror encoding errors (ignore by default) while converting the output into a string 

127 @param encoding encoding of the output 

128 @param change_path change the current path if not None (put it back after the execution) 

129 @param communicate use method `communicate 

130 <https://docs.python.org/3/library/subprocess.html#subprocess.Popen.communicate>`_ 

131 which is supposed to be safer, parameter ``wait`` must be True 

132 @param preprocess preprocess the command line if necessary (not available on Windows) 

133 (False to disable that option) 

134 @param timeout when data is sent to stdin (``sin``), a timeout is needed to avoid 

135 waiting for ever (*timeout* is in seconds) 

136 @param catch_exit catch *SystemExit* exception 

137 @param fLOG logging function (if not None, bypass others parameters) 

138 @param tell_if_no_output tells if there is no output every *tell_if_no_output* seconds 

139 @param prefix_log add a prefix to a line before printing it 

140 @return content of stdout, stdres (only if wait is True) 

141 

142 .. exref:: 

143 :title: Run a program using the command line 

144 

145 :: 

146 

147 from pyquickhelper.loghelper import run_cmd 

148 out, err = run_cmd("python setup.py install", wait=True) 

149 

150 If you are using this function to run :epkg:`git` function, parameter ``shell`` must be True. 

151 The function catches *SystemExit* exception. 

152 See `Constantly print Subprocess output while process is running 

153 <http://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running/4417735>`_. 

154 If *wait* is False, the function returns the started process. 

155 ``__exit__`` should be called if wait if False. 

156 Parameter *prefix_log* was added. 

157 """ 

158 if prefix_log is None: 

159 prefix_log = "" 

160 if fLOG is not None: 

161 if isinstance(cmd, (list, tuple)): 

162 fLOG(prefix_log + "[run_cmd] execute", " ".join(cmd)) 

163 else: 

164 fLOG(prefix_log + "[run_cmd] execute", cmd) 

165 

166 if change_path is not None: 

167 current = os.getcwd() 

168 os.chdir(change_path) 

169 

170 if sys.platform.startswith("win"): 

171 cmdl = cmd 

172 else: 

173 cmdl = split_cmp_command(cmd) if preprocess else cmd 

174 

175 if catch_exit: 

176 try: 

177 pproc = subprocess.Popen(cmdl, 

178 shell=shell, 

179 stdin=subprocess.PIPE if sin is not None and len( 

180 sin) > 0 else None, 

181 stdout=subprocess.PIPE if wait else None, 

182 stderr=subprocess.PIPE if wait else None) 

183 except SystemExit as e: 

184 if change_path is not None: 

185 os.chdir(current) 

186 raise RunCmdException("SystemExit raised (1)") from e 

187 

188 else: 

189 pproc = subprocess.Popen(cmdl, 

190 shell=shell, 

191 stdin=subprocess.PIPE if sin is not None and len( 

192 sin) > 0 else None, 

193 stdout=subprocess.PIPE if wait else None, 

194 stderr=subprocess.PIPE if wait else None) 

195 

196 pproc.__enter__() 

197 if isinstance(cmd, list): 

198 cmd = " ".join(cmd) 

199 

200 if wait: 

201 skip_out_err = False 

202 out = [] 

203 err = [] 

204 err_read = False 

205 skip_waiting = False 

206 

207 if communicate: 

208 # communicate is True 

209 if tell_if_no_output is not None: 

210 raise NotImplementedError( 

211 "tell_if_no_output is not implemented when communicate is True") 

212 if stop_running_if is not None: 

213 raise NotImplementedError( 

214 "stop_running_if is not implemented when communicate is True") 

215 input = sin if sin is None else sin.encode() 

216 if input is not None and len(input) > 0: 

217 if fLOG is not None: 

218 fLOG(prefix_log + "[run_cmd] input", [input]) 

219 

220 if catch_exit: 

221 try: 

222 stdoutdata, stderrdata = pproc.communicate( 

223 input=input, timeout=timeout) 

224 except SystemExit as e: 

225 if change_path is not None: 

226 os.chdir(current) 

227 raise RunCmdException("SystemExit raised (2)") from e 

228 else: 

229 stdoutdata, stderrdata = pproc.communicate( 

230 input=input, timeout=timeout) 

231 

232 out = decode_outerr(stdoutdata, encoding, encerror, cmd) 

233 err = decode_outerr(stderrdata, encoding, encerror, cmd) 

234 else: 

235 # communicate is False: use of threads 

236 if sin is not None and len(sin) > 0: 

237 if change_path is not None: 

238 os.chdir(current) 

239 raise Exception( 

240 "communicate should be True to send something on stdin") 

241 stdout, stderr = pproc.stdout, pproc.stderr 

242 

243 begin = time.perf_counter() 

244 last_update = begin 

245 # with threads 

246 (stdoutReader, stdoutQueue) = _AsyncLineReader.getForFd( 

247 stdout, catch_exit=catch_exit) 

248 (stderrReader, stderrQueue) = _AsyncLineReader.getForFd( 

249 stderr, catch_exit=catch_exit) 

250 runloop = True 

251 

252 while (not stdoutReader.eof() or not stderrReader.eof()) and runloop: 

253 while not stdoutQueue.empty(): 

254 line = stdoutQueue.get() 

255 decol = decode_outerr( 

256 line, encoding, encerror, cmd) 

257 sdecol = decol.strip("\n\r") 

258 if fLOG is not None: 

259 fLOG(prefix_log + sdecol) 

260 out.append(sdecol) 

261 last_update = time.perf_counter() 

262 if stop_running_if is not None and stop_running_if(decol, None): 

263 runloop = False 

264 break 

265 

266 while not stderrQueue.empty(): 

267 line = stderrQueue.get() 

268 decol = decode_outerr( 

269 line, encoding, encerror, cmd) 

270 sdecol = decol.strip("\n\r") 

271 if fLOG is not None: 

272 fLOG(prefix_log + sdecol) 

273 err.append(sdecol) 

274 last_update = time.perf_counter() 

275 if stop_running_if is not None and stop_running_if(None, decol): 

276 runloop = False 

277 break 

278 time.sleep(0.05) 

279 

280 delta = time.perf_counter() - last_update 

281 if tell_if_no_output is not None and delta >= tell_if_no_output: 

282 fLOG(prefix_log + "[run_cmd] No update in {0} seconds for cmd: {1}".format( 

283 "%5.1f" % (last_update - begin), cmd)) 

284 last_update = time.perf_counter() 

285 full_delta = time.perf_counter() - begin 

286 if timeout is not None and full_delta > timeout: 

287 runloop = False 

288 fLOG(prefix_log + "[run_cmd] Timeout after {0} seconds for cmd: {1}".format( 

289 "%5.1f" % full_delta, cmd)) 

290 break 

291 

292 if runloop: 

293 # Waiting for async readers to finish... 

294 stdoutReader.join() 

295 stderrReader.join() 

296 

297 # Waiting for process to exit... 

298 returnCode = pproc.wait() 

299 err_read = True 

300 

301 if returnCode != 0: 

302 if change_path is not None: 

303 os.chdir(current) 

304 try: 

305 # we try to close the ressources 

306 stdout.close() 

307 stderr.close() 

308 except Exception as e: 

309 warnings.warn( 

310 "Unable to close stdout and sterr.", RuntimeWarning) 

311 if catch_exit: 

312 mes = "SystemExit raised with error code {0}\nCMD:\n{1}\nCWD:\n{2}\n#---OUT---#\n{3}\n#---ERR---#\n{4}" 

313 raise RunCmdException(mes.format( 

314 returnCode, cmd, os.getcwd(), "\n".join(out), "\n".join(err))) 

315 raise subprocess.CalledProcessError(returnCode, cmd) 

316 

317 if not skip_waiting: 

318 pproc.wait() 

319 else: 

320 out.append("[run_cmd] killing process.") 

321 fLOG( 

322 prefix_log + "[run_cmd] killing process because stop_running_if returned True.") 

323 pproc.kill() 

324 err_read = True 

325 fLOG(prefix_log + "[run_cmd] process killed.") 

326 skip_out_err = True 

327 

328 out = "\n".join(out) 

329 if skip_out_err: 

330 err = "Process killed." 

331 else: 

332 if err_read: 

333 err = "\n".join(err) 

334 else: 

335 temp = err = stderr.read() 

336 try: 

337 err = decode_outerr(temp, encoding, encerror, cmd) 

338 except Exception: 

339 err = decode_outerr(temp, encoding, "ignore", cmd) 

340 stdout.close() 

341 stderr.close() 

342 

343 # same path for whether communicate is False or True 

344 err = err.replace("\r\n", "\n") 

345 if fLOG is not None: 

346 fLOG(prefix_log + "end of execution", cmd) 

347 

348 if len(err) > 0 and log_error and fLOG is not None: 

349 if "\n" in err: 

350 fLOG(prefix_log + "[run_cmd] stderr (log)") 

351 for eline in err.split("\n"): 

352 fLOG(prefix_log + eline) 

353 else: 

354 fLOG(prefix_log + "[run_cmd] stderr (log)\n%s" % err) 

355 

356 if change_path is not None: 

357 os.chdir(current) 

358 

359 pproc.__exit__(None, None, None) 

360 if sys.platform.startswith("win"): 

361 if err is not None: 

362 err = err.strip("\n\r\t ") 

363 return out.replace("\r\n", "\n"), err.replace("\r\n", "\n") 

364 else: 

365 if err is not None: 

366 err = err.strip("\n\r\t ") 

367 return out, err 

368 else: 

369 

370 if change_path is not None: 

371 os.chdir(current) 

372 

373 return pproc, None 

374 

375 

376def parse_exception_message(exc): 

377 """ 

378 Parses the message embedded in an exception and returns the standard output and error 

379 if it can be found. 

380 

381 @param exc exception coming from @see fn run_cmd 

382 @return out, err 

383 """ 

384 mes = str(exc) 

385 reg = re.compile(".*#---OUT---#(.*)#---ERR---#(.*)", re.DOTALL) 

386 find = reg.search(mes.replace("\r", "")) 

387 if find: 

388 gr = find.groups() 

389 out, err = gr[0], gr[1] 

390 return out.strip("\n "), err.strip("\n ") 

391 else: 

392 return None, None 

393 

394 

395def run_script(script, *args, **kwargs): 

396 """ 

397 Runs a script. 

398 

399 @param script script to execute or command line starting with ``-m`` 

400 @param args other parameters 

401 @param kwargs sent to @see fn run_cmd 

402 @return out,err: content of stdout stream and stderr stream 

403 

404 .. versionchanged:: 1.8 

405 Add *kwargs*, allows command line starting with ``-m``. 

406 """ 

407 if not script.startswith('-m') and not os.path.exists(script): 

408 raise PQHException("file %s not found" % script) 

409 py = get_interpreter_path() 

410 cmd = "%s %s" % (py, script) 

411 if len(args) > 0: 

412 typstr = str 

413 cmd += " " + " ".join([typstr(x) for x in args]) 

414 out, err = run_cmd(cmd, **kwargs) 

415 return out, err 

416 

417 

418class _AsyncLineReader(threading.Thread): 

419 

420 def __init__(self, fd, outputQueue, catch_exit): 

421 threading.Thread.__init__(self) 

422 

423 assert isinstance(outputQueue, queue.Queue) 

424 assert callable(fd.readline) 

425 

426 self.fd = fd 

427 self.catch_exit = catch_exit 

428 self.outputQueue = outputQueue 

429 

430 def run(self): 

431 if self.catch_exit: 

432 try: 

433 for _ in map(self.outputQueue.put, iter(self.fd.readline, b'')): 

434 pass 

435 except SystemExit as e: 

436 self.outputQueue.put(str(e)) 

437 raise RunCmdException("SystemExit raised (3)") from e 

438 else: 

439 for _ in map(self.outputQueue.put, iter(self.fd.readline, b'')): 

440 pass 

441 

442 def eof(self): 

443 return not self.is_alive() and self.outputQueue.empty() 

444 

445 @classmethod 

446 def getForFd(cls, fd, start=True, catch_exit=False): 

447 q = queue.Queue() 

448 reader = cls(fd, q, catch_exit) 

449 

450 if start: 

451 reader.start() 

452 

453 return reader, q