1#!/usr/bin/env python
2
3r"""
4This module provides command execution functions such as cmd_fnc and cmd_fnc_u.
5"""
6
7import os
8import sys
9import subprocess
10import collections
11import signal
12import time
13
14import gen_print as gp
15import gen_valid as gv
16import gen_misc as gm
17
18robot_env = gp.robot_env
19
20if robot_env:
21    import gen_robot_print as grp
22    from robot.libraries.BuiltIn import BuiltIn
23
24
25# cmd_fnc and cmd_fnc_u should now be considered deprecated.  shell_cmd and
26# t_shell_cmd should be used instead.
27def cmd_fnc(cmd_buf,
28            quiet=None,
29            test_mode=None,
30            debug=0,
31            print_output=1,
32            show_err=1,
33            return_stderr=0,
34            ignore_err=1):
35    r"""
36    Run the given command in a shell and return the shell return code and the
37    output.
38
39    Description of arguments:
40    cmd_buf                         The command string to be run in a shell.
41    quiet                           Indicates whether this function should run
42                                    the print_issuing() function which prints
43                                    "Issuing: <cmd string>" to stdout.
44    test_mode                       If test_mode is set, this function will
45                                    not actually run the command.  If
46                                    print_output is set, it will print
47                                    "(test_mode) Issuing: <cmd string>" to
48                                    stdout.
49    debug                           If debug is set, this function will print
50                                    extra debug info.
51    print_output                    If this is set, this function will print
52                                    the stdout/stderr generated by the shell
53                                    command.
54    show_err                        If show_err is set, this function will
55                                    print a standardized error report if the
56                                    shell command returns non-zero.
57    return_stderr                   If return_stderr is set, this function
58                                    will process the stdout and stderr streams
59                                    from the shell command separately.  It
60                                    will also return stderr in addition to the
61                                    return code and the stdout.
62    """
63
64    # Determine default values.
65    quiet = int(gm.global_default(quiet, 0))
66    test_mode = int(gm.global_default(test_mode, 0))
67
68    if debug:
69        gp.print_vars(cmd_buf, quiet, test_mode, debug)
70
71    err_msg = gv.svalid_value(cmd_buf)
72    if err_msg != "":
73        raise ValueError(err_msg)
74
75    if not quiet:
76        gp.pissuing(cmd_buf, test_mode)
77
78    if test_mode:
79        if return_stderr:
80            return 0, "", ""
81        else:
82            return 0, ""
83
84    if return_stderr:
85        err_buf = ""
86        stderr = subprocess.PIPE
87    else:
88        stderr = subprocess.STDOUT
89
90    sub_proc = subprocess.Popen(cmd_buf,
91                                bufsize=1,
92                                shell=True,
93                                executable='/bin/bash',
94                                stdout=subprocess.PIPE,
95                                stderr=stderr)
96    out_buf = ""
97    if return_stderr:
98        for line in sub_proc.stderr:
99            err_buf += line
100            if not print_output:
101                continue
102            if robot_env:
103                grp.rprint(line)
104            else:
105                sys.stdout.write(line)
106    for line in sub_proc.stdout:
107        out_buf += line
108        if not print_output:
109            continue
110        if robot_env:
111            grp.rprint(line)
112        else:
113            sys.stdout.write(line)
114    if print_output and not robot_env:
115        sys.stdout.flush()
116    sub_proc.communicate()
117    shell_rc = sub_proc.returncode
118    if shell_rc != 0:
119        err_msg = "The prior shell command failed.\n"
120        err_msg += gp.sprint_var(shell_rc, 1)
121        if not print_output:
122            err_msg += "out_buf:\n" + out_buf
123
124        if show_err:
125            if robot_env:
126                grp.rprint_error_report(err_msg)
127            else:
128                gp.print_error_report(err_msg)
129        if not ignore_err:
130            if robot_env:
131                BuiltIn().fail(err_msg)
132            else:
133                raise ValueError(err_msg)
134
135    if return_stderr:
136        return shell_rc, out_buf, err_buf
137    else:
138        return shell_rc, out_buf
139
140
141def cmd_fnc_u(cmd_buf,
142              quiet=None,
143              debug=None,
144              print_output=1,
145              show_err=1,
146              return_stderr=0,
147              ignore_err=1):
148    r"""
149    Call cmd_fnc with test_mode=0.  See cmd_fnc (above) for details.
150
151    Note the "u" in "cmd_fnc_u" stands for "unconditional".
152    """
153
154    return cmd_fnc(cmd_buf, test_mode=0, quiet=quiet, debug=debug,
155                   print_output=print_output, show_err=show_err,
156                   return_stderr=return_stderr, ignore_err=ignore_err)
157
158
159def parse_command_string(command_string):
160    r"""
161    Parse a bash command-line command string and return the result as a
162    dictionary of parms.
163
164    This can be useful for answering questions like "What did the user specify
165    as the value for parm x in the command string?".
166
167    This function expects the command string to follow the following posix
168    conventions:
169    - Short parameters:
170      -<parm name><space><arg value>
171    - Long parameters:
172      --<parm name>=<arg value>
173
174    The first item in the string will be considered to be the command.  All
175    values not conforming to the specifications above will be considered
176    positional parms.  If there are multiple parms with the same name, they
177    will be put into a list (see illustration below where "-v" is specified
178    multiple times).
179
180    Description of argument(s):
181    command_string                  The complete command string including all
182                                    parameters and arguments.
183
184    Sample input:
185
186    robot_cmd_buf:                                    robot -v
187    OPENBMC_HOST:dummy1 -v keyword_string:'Set Auto Reboot  no' -v
188    lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot -v
189    quiet:0 -v test_mode:0 -v debug:0
190    --outputdir='/home/user1/status/children/'
191    --output=dummy1.Auto_reboot.170802.124544.output.xml
192    --log=dummy1.Auto_reboot.170802.124544.log.html
193    --report=dummy1.Auto_reboot.170802.124544.report.html
194    /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
195
196    Sample output:
197
198    robot_cmd_buf_dict:
199      robot_cmd_buf_dict[command]:                    robot
200      robot_cmd_buf_dict[v]:
201        robot_cmd_buf_dict[v][0]:                     OPENBMC_HOST:dummy1
202        robot_cmd_buf_dict[v][1]:                     keyword_string:Set Auto
203        Reboot no
204        robot_cmd_buf_dict[v][2]:
205        lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot
206        robot_cmd_buf_dict[v][3]:                     quiet:0
207        robot_cmd_buf_dict[v][4]:                     test_mode:0
208        robot_cmd_buf_dict[v][5]:                     debug:0
209      robot_cmd_buf_dict[outputdir]:
210      /home/user1/status/children/
211      robot_cmd_buf_dict[output]:
212      dummy1.Auto_reboot.170802.124544.output.xml
213      robot_cmd_buf_dict[log]:
214      dummy1.Auto_reboot.170802.124544.log.html
215      robot_cmd_buf_dict[report]:
216      dummy1.Auto_reboot.170802.124544.report.html
217      robot_cmd_buf_dict[positional]:
218      /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
219    """
220
221    # We want the parms in the string broken down the way bash would do it,
222    # so we'll call upon bash to do that by creating a simple inline bash
223    # function.
224    bash_func_def = "function parse { for parm in \"${@}\" ; do" +\
225        " echo $parm ; done ; }"
226
227    rc, outbuf = cmd_fnc_u(bash_func_def + " ; parse " + command_string,
228                           quiet=1, print_output=0)
229    command_string_list = outbuf.rstrip("\n").split("\n")
230
231    command_string_dict = collections.OrderedDict()
232    ix = 1
233    command_string_dict['command'] = command_string_list[0]
234    while ix < len(command_string_list):
235        if command_string_list[ix].startswith("--"):
236            key, value = command_string_list[ix].split("=")
237            key = key.lstrip("-")
238        elif command_string_list[ix].startswith("-"):
239            key = command_string_list[ix].lstrip("-")
240            ix += 1
241            try:
242                value = command_string_list[ix]
243            except IndexError:
244                value = ""
245        else:
246            key = 'positional'
247            value = command_string_list[ix]
248        if key in command_string_dict:
249            if isinstance(command_string_dict[key], str):
250                command_string_dict[key] = [command_string_dict[key]]
251            command_string_dict[key].append(value)
252        else:
253            command_string_dict[key] = value
254        ix += 1
255
256    return command_string_dict
257
258
259# Save the original SIGALRM handler for later restoration by shell_cmd.
260original_sigalrm_handler = signal.getsignal(signal.SIGALRM)
261
262
263def shell_cmd_timed_out(signal_number,
264                        frame):
265    r"""
266    Handle an alarm signal generated during the shell_cmd function.
267    """
268
269    gp.dprint_executing()
270    # Get subprocess pid from shell_cmd's call stack.
271    sub_proc = gp.get_stack_var('sub_proc', 0)
272    pid = sub_proc.pid
273    # Terminate the child process.
274    os.kill(pid, signal.SIGTERM)
275    # Restore the original SIGALRM handler.
276    signal.signal(signal.SIGALRM, original_sigalrm_handler)
277
278    return
279
280
281def shell_cmd(command_string,
282              quiet=None,
283              print_output=1,
284              show_err=1,
285              test_mode=0,
286              time_out=None,
287              max_attempts=1,
288              retry_sleep_time=5,
289              allowed_shell_rcs=[0],
290              ignore_err=None,
291              return_stderr=0,
292              fork=0):
293    r"""
294    Run the given command string in a shell and return a tuple consisting of
295    the shell return code and the output.
296
297    Description of argument(s):
298    command_string                  The command string to be run in a shell
299                                    (e.g. "ls /tmp").
300    quiet                           If set to 0, this function will print
301                                    "Issuing: <cmd string>" to stdout.  When
302                                    the quiet argument is set to None, this
303                                    function will assign a default value by
304                                    searching upward in the stack for the
305                                    quiet variable value.  If no such value is
306                                    found, quiet is set to 0.
307    print_output                    If this is set, this function will print
308                                    the stdout/stderr generated by the shell
309                                    command to stdout.
310    show_err                        If show_err is set, this function will
311                                    print a standardized error report if the
312                                    shell command fails (i.e. if the shell
313                                    command returns a shell_rc that is not in
314                                    allowed_shell_rcs).  Note: Error text is
315                                    only printed if ALL attempts to run the
316                                    command_string fail.  In other words, if
317                                    the command execution is ultimately
318                                    successful, initial failures are hidden.
319    test_mode                       If test_mode is set, this function will
320                                    not actually run the command.  If
321                                    print_output is also set, this function
322                                    will print "(test_mode) Issuing: <cmd
323                                    string>" to stdout.  A caller should call
324                                    shell_cmd directly if they wish to have
325                                    the command string run unconditionally.
326                                    They should call the t_shell_cmd wrapper
327                                    (defined below) if they wish to run the
328                                    command string only if the prevailing
329                                    test_mode variable is set to 0.
330    time_out                        A time-out value expressed in seconds.  If
331                                    the command string has not finished
332                                    executing within <time_out> seconds, it
333                                    will be halted and counted as an error.
334    max_attempts                    The max number of attempts that should be
335                                    made to run the command string.
336    retry_sleep_time                The number of seconds to sleep between
337                                    attempts.
338    allowed_shell_rcs               A list of integers indicating which
339                                    shell_rc values are not to be considered
340                                    errors.
341    ignore_err                      Ignore error means that a failure
342                                    encountered by running the command string
343                                    will not be raised as a python exception.
344                                    When the ignore_err argument is set to
345                                    None, this function will assign a default
346                                    value by searching upward in the stack for
347                                    the ignore_err variable value.  If no such
348                                    value is found, ignore_err is set to 1.
349    return_stderr                   If return_stderr is set, this function
350                                    will process the stdout and stderr streams
351                                    from the shell command separately.  In
352                                    such a case, the tuple returned by this
353                                    function will consist of three values
354                                    rather than just two: rc, stdout, stderr.
355    fork                            Run the command string asynchronously
356                                    (i.e. don't wait for status of the child
357                                    process and don't try to get
358                                    stdout/stderr).
359    """
360
361    # Assign default values to some of the arguments to this function.
362    quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0)))
363    ignore_err = int(gm.dft(ignore_err, gp.get_stack_var('ignore_err', 1)))
364
365    err_msg = gv.svalid_value(command_string)
366    if err_msg != "":
367        raise ValueError(err_msg)
368
369    if not quiet:
370        gp.print_issuing(command_string, test_mode)
371
372    if test_mode:
373        if return_stderr:
374            return 0, "", ""
375        else:
376            return 0, ""
377
378    # Convert each list entry to a signed value.
379    allowed_shell_rcs = [gm.to_signed(x) for x in allowed_shell_rcs]
380
381    if return_stderr:
382        stderr = subprocess.PIPE
383    else:
384        stderr = subprocess.STDOUT
385
386    shell_rc = 0
387    out_buf = ""
388    err_buf = ""
389    # Write all output to func_history_stdout rather than directly to stdout.
390    # This allows us to decide what to print after all attempts to run the
391    # command string have been made.  func_history_stdout will contain the
392    # complete stdout history from the current invocation of this function.
393    func_history_stdout = ""
394    for attempt_num in range(1, max_attempts + 1):
395        sub_proc = subprocess.Popen(command_string,
396                                    bufsize=1,
397                                    shell=True,
398                                    executable='/bin/bash',
399                                    stdout=subprocess.PIPE,
400                                    stderr=stderr)
401        out_buf = ""
402        err_buf = ""
403        # Output from this loop iteration is written to func_stdout for later
404        # processing.
405        func_stdout = ""
406        if fork:
407            break
408        command_timed_out = False
409        if time_out is not None:
410            # Designate a SIGALRM handling function and set alarm.
411            signal.signal(signal.SIGALRM, shell_cmd_timed_out)
412            signal.alarm(time_out)
413        try:
414            if return_stderr:
415                for line in sub_proc.stderr:
416                    err_buf += line
417                    if not print_output:
418                        continue
419                    func_stdout += line
420            for line in sub_proc.stdout:
421                out_buf += line
422                if not print_output:
423                    continue
424                func_stdout += line
425        except IOError:
426            command_timed_out = True
427        sub_proc.communicate()
428        shell_rc = sub_proc.returncode
429        # Restore the original SIGALRM handler and clear the alarm.
430        signal.signal(signal.SIGALRM, original_sigalrm_handler)
431        signal.alarm(0)
432        if shell_rc in allowed_shell_rcs:
433            break
434        err_msg = "The prior shell command failed.\n"
435        if quiet:
436            err_msg += gp.sprint_var(command_string)
437        if command_timed_out:
438            err_msg += gp.sprint_var(command_timed_out)
439            err_msg += gp.sprint_var(time_out)
440            err_msg += gp.sprint_varx("child_pid", sub_proc.pid)
441        err_msg += gp.sprint_var(attempt_num)
442        err_msg += gp.sprint_var(shell_rc, 1)
443        err_msg += gp.sprint_var(allowed_shell_rcs, 1)
444        if not print_output:
445            if return_stderr:
446                err_msg += "err_buf:\n" + err_buf
447            err_msg += "out_buf:\n" + out_buf
448        if show_err:
449            if robot_env:
450                func_stdout += grp.sprint_error_report(err_msg)
451            else:
452                func_stdout += gp.sprint_error_report(err_msg)
453        func_history_stdout += func_stdout
454        if attempt_num < max_attempts:
455            func_history_stdout += gp.sprint_issuing("time.sleep("
456                                                     + str(retry_sleep_time)
457                                                     + ")")
458            time.sleep(retry_sleep_time)
459
460    if shell_rc not in allowed_shell_rcs:
461        func_stdout = func_history_stdout
462
463    if robot_env:
464        grp.rprint(func_stdout)
465    else:
466        sys.stdout.write(func_stdout)
467        sys.stdout.flush()
468
469    if shell_rc not in allowed_shell_rcs:
470        if not ignore_err:
471            if robot_env:
472                BuiltIn().fail(err_msg)
473            else:
474                raise ValueError("The prior shell command failed.\n")
475
476    if return_stderr:
477        return shell_rc, out_buf, err_buf
478    else:
479        return shell_rc, out_buf
480
481
482def t_shell_cmd(command_string, **kwargs):
483    r"""
484    Search upward in the the call stack to obtain the test_mode argument, add
485    it to kwargs and then call shell_cmd and return the result.
486
487    See shell_cmd prolog for details on all arguments.
488    """
489
490    if 'test_mode' in kwargs:
491        error_message = "Programmer error - test_mode is not a valid" +\
492            " argument to this function."
493        gp.print_error_report(error_message)
494        exit(1)
495
496    test_mode = gp.get_stack_var('test_mode',
497                                 int(gp.get_var_value(None, 0, "test_mode")))
498    kwargs['test_mode'] = test_mode
499
500    return shell_cmd(command_string, **kwargs)
501