Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Implements function @see fn run_cmd.
6.. versionadded:: 1.1
7"""
8from __future__ import print_function
9import sys
10import os
11import time
12import subprocess
13import threading
14import warnings
15import shlex
16if sys.version_info[0] == 2:
17 import Queue as queue
18else:
19 import queue
22class RunCmdException(Exception):
23 """
24 raised by function @see fn run_cmd
25 """
26 pass
29def get_interpreter_path():
30 """
31 return the interpreter path
32 """
33 if sys.platform.startswith("win"):
34 return sys.executable.replace("pythonw.exe", "python.exe")
35 else:
36 return sys.executable
39def split_cmp_command(cmd, remove_quotes=True):
40 """
41 splits a command line
42 @param cmd command line
43 @param remove_quotes True by default
44 @return list
45 """
46 if isinstance(cmd, str # unicode#
47 ):
48 return shlex.split(cmd)
49 else:
50 return cmd
53def decode_outerr(outerr, encoding, encerror, msg):
54 """
55 decode the output or the error after running a command line instructions
57 @param outerr output or error
58 @param encoding encoding (if None, it is replaced by ascii)
59 @param encerror how to handle errors
60 @param msg to add to the exception message
61 @return converted string
63 .. versionchanged:: 1.4
64 If *encoding* is None, it is replaced by ``'ascii'``.
65 """
66 if encoding is None:
67 encoding = "ascii"
68 typstr = str # unicode#
69 if not isinstance(outerr, bytes):
70 raise TypeError(
71 "only able to decode bytes, not " + typstr(type(outerr)))
72 try:
73 out = outerr.decode(encoding, errors=encerror)
74 return out
75 except UnicodeDecodeError as exu:
76 try:
77 out = outerr.decode(
78 "utf8" if encoding != "utf8" else "latin-1", errors=encerror)
79 return out
80 except Exception as e:
81 out = outerr.decode(encoding, errors='ignore')
82 raise Exception("issue with cmd (" + encoding + "):" +
83 typstr(msg) + "\n" + typstr(exu) + "\n-----\n" + out) from e
84 raise Exception("complete issue with cmd:" + typstr(msg))
87def skip_run_cmd(cmd, sin="", shell=True, wait=False, log_error=True,
88 stop_running_if=None, encerror="ignore",
89 encoding="utf8", change_path=None, communicate=True,
90 preprocess=True, timeout=None, catch_exit=False, fLOG=None,
91 timeout_listen=None, tell_if_no_output=None):
92 """
93 has the same signature as @see fn run_cmd but does nothing
95 .. versionadded:: 1.0
96 """
97 return "", ""
100def run_cmd_private(cmd, sin="", shell=True, wait=False, log_error=True,
101 stop_running_if=None, encerror="ignore", encoding="utf8",
102 change_path=None, communicate=True, preprocess=True, timeout=None,
103 catch_exit=False, fLOG=None, tell_if_no_output=None):
104 """
105 run a command line and wait for the result
106 @param cmd command line
107 @param sin sin: what must be written on the standard input
108 @param shell if True, cmd is a shell command (and no command window is opened)
109 @param wait call ``proc.wait``
110 @param log_error if log_error, call fLOG (error)
111 @param stop_running_if the function stops waiting if some condition is fulfilled.
112 The function received the last line from the logs.
113 Signature: ``stop_waiting_if(last_out, last_err) -> bool``.
114 The function must return True to stop waiting.
115 This function can also be used to intercept the standard output
116 and the standard error while running.
117 @param encerror encoding errors (ignore by default) while converting the output into a string
118 @param encoding encoding of the output
119 @param change_path change the current path if not None (put it back after the execution)
120 @param communicate use method `communicate <https://docs.python.org/3/library/subprocess.html#subprocess.Popen.communicate>`_
121 which is supposed to be safer,
122 parameter ``wait`` must be True
123 @param preprocess preprocess the command line if necessary (not available on Windows) (False to disable that option)
124 @param timeout when data is sent to stdin (``sin``), a timeout is needed to avoid
125 waiting for ever (*timeout* is in seconds)
126 @param catch_exit catch *SystemExit* exception
127 @param fLOG logging function (if not None, bypass others parameters)
128 @param tell_if_no_output tells if there is no output every *tell_if_no_output* seconds
129 @return content of stdout, stdres (only if wait is True)
131 .. exref::
132 :title: Run a program using the command line
134 ::
136 from pyquickhelper.loghelper import run_cmd
137 out,err = run_cmd( "python setup.py install", wait=True)
139 If you are using this function to run git function, parameter ``shell`` must be True.
141 .. versionadded:: 1.1
142 """
143 if fLOG is not None:
144 fLOG("execute", cmd)
146 if sys.platform.startswith("win"):
147 cmdl = cmd
148 else:
149 cmdl = split_cmp_command(cmd) if preprocess else cmd
151 if catch_exit:
152 try:
153 pproc = subprocess.Popen(cmdl,
154 shell=shell,
155 stdin=subprocess.PIPE if (
156 sin and len(sin) > 0) else None,
157 stdout=subprocess.PIPE if wait else None,
158 stderr=subprocess.PIPE if wait else None,
159 cwd=change_path)
160 except SystemExit as e:
161 raise RunCmdException("SystemExit raised (1)") from e
163 else:
164 pproc = subprocess.Popen(cmdl,
165 shell=shell,
166 stdin=subprocess.PIPE if (
167 sin and len(sin) > 0) else None,
168 stdout=subprocess.PIPE if wait else None,
169 stderr=subprocess.PIPE if wait else None,
170 cwd=change_path)
172 if isinstance(cmd, list):
173 cmd = " ".join(cmd)
175 if wait:
176 skip_out_err = False
177 out = []
178 err = []
179 err_read = False
180 skip_waiting = False
182 if communicate:
183 # communicate is True
184 if tell_if_no_output is not None:
185 raise NotImplementedError(
186 "tell_if_no_output is not implemented when communicate is True")
187 if stop_running_if is not None:
188 raise NotImplementedError(
189 "stop_running_if is not implemented when communicate is True")
190 input = None if (sin is None or len(sin) > 0) else sin.encode()
191 if input is not None and len(input) > 0:
192 if fLOG is not None:
193 fLOG("input", [input])
195 if catch_exit:
196 try:
197 if sys.version_info[0] == 2:
198 if timeout is not None:
199 raise NotImplementedError(
200 "timeout is only available with Python 3")
201 stdoutdata, stderrdata = pproc.communicate(input=input)
202 else:
203 stdoutdata, stderrdata = pproc.communicate(
204 input=input, timeout=timeout)
205 except SystemExit as e:
206 raise RunCmdException("SystemExit raised (2)") from e
207 else:
208 if sys.version_info[0] == 2:
209 if timeout is not None:
210 warnings.warn(
211 "[run_cmd_private] timeout is only available with Python 3")
212 stdoutdata, stderrdata = pproc.communicate(input=input)
213 else:
214 stdoutdata, stderrdata = pproc.communicate(
215 input=input, timeout=timeout)
217 out = decode_outerr(stdoutdata, encoding, encerror, cmd)
218 err = decode_outerr(stderrdata, encoding, encerror, cmd)
219 else:
220 # communicate is False: use of threads
221 if sin is not None and len(sin) > 0:
222 raise Exception(
223 "communicate should be True to send something on stdin")
224 stdout, stderr = pproc.stdout, pproc.stderr
226 begin = time.perf_counter()
227 last_update = begin
228 # with threads
229 (stdoutReader, stdoutQueue) = _AsyncLineReader.getForFd(
230 stdout, catch_exit=catch_exit)
231 (stderrReader, stderrQueue) = _AsyncLineReader.getForFd(
232 stderr, catch_exit=catch_exit)
233 runloop = True
235 while (not stdoutReader.eof() or not stderrReader.eof()) and runloop:
236 while not stdoutQueue.empty():
237 line = stdoutQueue.get()
238 decol = decode_outerr(
239 line, encoding, encerror, cmd)
240 if fLOG is not None:
241 fLOG(decol.strip("\n\r"))
242 out.append(decol.strip("\n\r"))
243 last_update = time.perf_counter()
244 if stop_running_if is not None and stop_running_if(decol, None):
245 runloop = False
246 break
248 while not stderrQueue.empty():
249 line = stderrQueue.get()
250 decol = decode_outerr(
251 line, encoding, encerror, cmd)
252 if fLOG is not None:
253 fLOG(decol.strip("\n\r"))
254 err.append(decol.strip("\n\r"))
255 last_update = time.perf_counter()
256 if stop_running_if is not None and stop_running_if(None, decol):
257 runloop = False
258 break
259 time.sleep(0.05)
261 delta = time.perf_counter() - last_update
262 if tell_if_no_output is not None and delta >= tell_if_no_output:
263 if fLOG is not None:
264 fLOG("[run_cmd] No update in {0} seconds for cmd: {1}".format(
265 "%5.1f" % (last_update - begin), cmd))
266 last_update = time.perf_counter()
267 full_delta = time.perf_counter() - begin
268 if timeout is not None and full_delta > timeout:
269 runloop = False
270 if fLOG is not None:
271 fLOG("[run_cmd] Timeout after {0} seconds for cmd: {1}".format(
272 "%5.1f" % full_delta, cmd))
273 break
275 if runloop:
276 # Waiting for async readers to finish...
277 stdoutReader.join()
278 stderrReader.join()
280 # Waiting for process to exit...
281 returnCode = pproc.wait()
282 err_read = True
284 if returnCode != 0:
285 try:
286 # we try to close the ressources
287 stdout.close()
288 stderr.close()
289 except Exception as e:
290 warnings.warn("Unable to close stdout and sterr.")
291 if catch_exit:
292 raise RunCmdException("SystemExit raised with error code {0}\nOUT:\n{1}\nERR-0:\n{2}".format(
293 returnCode, "\n".join(out), "\n".join(err)))
294 raise subprocess.CalledProcessError(returnCode, cmd)
296 if not skip_waiting:
297 pproc.wait()
298 else:
299 out.append("[run_cmd] killing process.")
300 if fLOG is not None:
301 fLOG(
302 "[run_cmd] killing process because stop_running_if returned True.")
303 pproc.kill()
304 err_read = True
305 if fLOG is not None:
306 fLOG("[run_cmd] process killed.")
307 skip_out_err = True
309 out = "\n".join(out)
310 if skip_out_err:
311 err = "Process killed."
312 else:
313 if err_read:
314 err = "\n".join(err)
315 else:
316 temp = err = stderr.read()
317 try:
318 err = decode_outerr(temp, encoding, encerror, cmd)
319 except Exception:
320 err = decode_outerr(temp, encoding, "ignore", cmd)
321 stdout.close()
322 stderr.close()
324 # same path for whether communicate is False or True
325 err = err.replace("\r\n", "\n")
326 if fLOG is not None:
327 fLOG("end of execution", cmd)
329 if len(err) > 0 and log_error and fLOG is not None:
330 fLOG("error (log)\n%s" % err)
332 if sys.platform.startswith("win"):
333 return out.replace("\r\n", "\n"), err.replace("\r\n", "\n")
334 else:
335 return out, err
336 else:
338 return "", ""
341class _AsyncLineReader(threading.Thread):
343 def __init__(self, fd, outputQueue, catch_exit):
344 threading.Thread.__init__(self)
346 assert isinstance(outputQueue, queue.Queue)
347 assert callable(fd.readline)
349 self.fd = fd
350 self.catch_exit = catch_exit
351 self.outputQueue = outputQueue
353 def run(self):
354 if self.catch_exit:
355 try:
356 for _ in map(self.outputQueue.put, iter(self.fd.readline, b'')):
357 pass
358 except SystemExit as e:
359 self.outputQueue.put(str(e))
360 raise RunCmdException("SystemExit raised (3)") from e
361 else:
362 for _ in map(self.outputQueue.put, iter(self.fd.readline, b'')):
363 pass
365 def eof(self):
366 return not self.is_alive() and self.outputQueue.empty()
368 @classmethod
369 def getForFd(cls, fd, start=True, catch_exit=False):
370 q = queue.Queue()
371 reader = cls(fd, q, catch_exit)
373 if start:
374 reader.start()
376 return reader, q
379def run_cmd_old(cmd, sin="", shell=False, wait=False, log_error=True,
380 secure=None, stop_waiting_if=None, do_not_log=False,
381 encerror="ignore", encoding="utf8", cwd=None, fLOG=print):
382 """
383 run a command line and wait for the result
384 @param cmd command line
385 @param sin sin, what must be written on the standard input
386 @param shell if True, cmd is a shell command (and no command window is opened)
387 @param wait call proc.wait
388 @param log_error if log_error, call fLOG (error)
389 @param secure if secure is a string (a valid filename), the function stores the output in a file
390 and reads it continuously
391 @param stop_waiting_if the function stops waiting if some condition is fulfilled.
392 The function received the last line from the logs.
393 @param do_not_log do not log the output
394 @param encerror encoding errors (ignore by default) while converting the output into a string
395 @param encoding encoding of the output
396 @param cwd current folder
397 @param fLOG logging function
398 @return content of stdout, stderr (only if wait is True)
401 .. faqref::
402 :title: Exception when installing a module
404 This error can occur when a module is installed on a virtual environment
405 created before *pip* was updated on the main distribution.
406 The solution consists in removing the virtual environment and create it again.
408 ::
410 c:\\Python34_x64vir\\install\\Scripts\\python -u setup.py install
411 running install
412 running bdist_egg
413 running egg_info
414 Traceback (most recent call last):
415 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2785, in _dep_map
416 return self.__dep_map
417 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2642, in __getattr__
418 raise AttributeError(attr)
419 AttributeError: _DistInfoDistribution__dep_map
421 During handling of the above exception, another exception occurred:
423 Traceback (most recent call last):
424 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2776, in _parsed_pkg_info
425 return self._pkg_info
426 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2642, in __getattr__
427 raise AttributeError(attr)
428 AttributeError: _pkg_info
430 During handling of the above exception, another exception occurred:
432 Traceback (most recent call last):
433 File "setup.py", line 169, in <module>
434 package_data=package_data,
435 File "C:\\Python34_x64\\Lib\\distutils\\core.py", line 148, in setup
436 dist.run_commands()
437 File "C:\\Python34_x64\\Lib\\distutils\\dist.py", line 955, in run_commands
438 self.run_command(cmd)
439 File "C:\\Python34_x64\\Lib\\distutils\\dist.py", line 974, in run_command
440 cmd_obj.run()
441 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\setuptools\\command\\install.py", line 67, in run
442 self.do_egg_install()
443 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\setuptools\\command\\install.py", line 109, in do_egg_install
444 self.run_command('bdist_egg')
445 File "C:\\Python34_x64\\Lib\\distutils\\cmd.py", line 313, in run_command
446 self.distribution.run_command(command)
447 File "C:\\Python34_x64\\Lib\\distutils\\dist.py", line 974, in run_command
448 cmd_obj.run()
449 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\setuptools\\command\\bdist_egg.py", line 151, in run
450 self.run_command("egg_info")
451 File "C:\\Python34_x64\\Lib\\distutils\\cmd.py", line 313, in run_command
452 self.distribution.run_command(command)
453 File "C:\\Python34_x64\\Lib\\distutils\\dist.py", line 974, in run_command
454 cmd_obj.run()
455 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\setuptools\\command\\egg_info.py", line 171, in run
456 ep.require(installer=installer)
457 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2355, in require
458 items = working_set.resolve(reqs, env, installer)
459 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 835, in resolve
460 new_requirements = dist.requires(req.extras)[::-1]
461 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2586, in requires
462 dm = self._dep_map
463 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2787, in _dep_map
464 self.__dep_map = self._compute_dependencies()
465 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2809, in _compute_dependencies
466 for req in self._parsed_pkg_info.get_all('Requires-Dist') or []:
467 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 2778, in _parsed_pkg_info
468 metadata = self.get_metadata(self.PKG_INFO)
469 File "c:\\Python34_x64vir\\install\\lib\\site-packages\\pkg_resources\\__init__.py", line 1993, in get_metadata
470 raise KeyError("No metadata except PKG-INFO is available")
471 KeyError: 'No metadata except PKG-INFO is available'
472 """
473 if sin is not None and sin != "":
474 raise NotImplementedError("sin is not used")
476 if secure is not None:
477 if not do_not_log:
478 fLOG("secure=", secure)
479 with open(secure, "w") as f:
480 f.write("")
481 add = ">%s" % secure
482 if isinstance(cmd, str):
483 cmd += " " + add
484 else:
485 cmd.append(add)
486 if not do_not_log and fLOG is not None:
487 fLOG("execute ", cmd)
489 if sys.platform.startswith("win"):
491 startupinfo = subprocess.STARTUPINFO()
492 startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
494 try:
495 proc = subprocess.Popen(cmd, shell=shell,
496 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
497 startupinfo=startupinfo, cwd=cwd)
498 except FileNotFoundError as e:
499 raise RunCmdException("unable to run CMD:\n{0}".format(cmd)) from e
500 else:
501 try:
502 proc = subprocess.Popen(split_cmp_command(cmd), shell=shell,
503 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
504 cwd=cwd)
505 except FileNotFoundError as e:
506 raise RunCmdException("unable to run CMD:\n{0}".format(cmd)) from e
507 if wait:
509 out = []
510 skip_waiting = False
512 if secure is None:
513 for line in proc.stdout:
514 if not do_not_log and fLOG is not None:
515 fLOG(line.decode(encoding, errors=encerror).strip("\n"))
516 try:
517 out.append(
518 line.decode(
519 encoding,
520 errors=encerror).strip("\n"))
521 except UnicodeDecodeError as exu:
522 raise RunCmdException(
523 "issue with cmd:" +
524 str(cmd) +
525 "\n" +
526 str(exu))
527 if proc.stdout.closed:
528 break
529 if stop_waiting_if is not None and stop_waiting_if(
530 line.decode("utf8", errors=encerror)):
531 skip_waiting = True
532 break
533 else:
534 last = []
535 while proc.poll() is None:
536 if os.path.exists(secure):
537 with open(secure, "r") as f:
538 lines = f.readlines()
539 if len(lines) > len(last):
540 for line in lines[len(last):]:
541 if not do_not_log and fLOG is not None:
542 fLOG(line.strip("\n"))
543 out.append(line.strip("\n"))
544 last = lines
545 if stop_waiting_if is not None and len(
546 last) > 0 and stop_waiting_if(last[-1]):
547 skip_waiting = True
548 break
549 time.sleep(0.1)
551 if not skip_waiting:
552 proc.wait()
554 out = "\n".join(out)
555 err = proc.stderr.read().decode(encoding, errors=encerror)
556 if not do_not_log and fLOG is not None:
557 fLOG("end of execution ", cmd)
558 if len(err) > 0 and log_error and not do_not_log and fLOG is not None:
559 fLOG("error (log)\n%s" % err)
560 # return bytes.decode (out, errors="ignore"), bytes.decode(err,
561 # errors="ignore")
562 proc.stdout.close()
563 proc.stderr.close()
565 return out, err
566 else:
567 return "", ""