1#!/usr/bin/env python
2
3r"""
4This module provides command execution functions such as cmd_fnc and cmd_fnc_u.
5"""
6
7import os
8import sys
9import subprocess
10import collections
11import signal
12import time
13
14import gen_print as gp
15import gen_valid as gv
16import gen_misc as gm
17
18robot_env = gp.robot_env
19
20if robot_env:
21    import gen_robot_print as grp
22    from robot.libraries.BuiltIn import BuiltIn
23
24
25# cmd_fnc and cmd_fnc_u should now be considered deprecated.  shell_cmd and
26# t_shell_cmd should be used instead.
27def cmd_fnc(cmd_buf,
28            quiet=None,
29            test_mode=None,
30            debug=0,
31            print_output=1,
32            show_err=1,
33            return_stderr=0,
34            ignore_err=1):
35    r"""
36    Run the given command in a shell and return the shell return code and the
37    output.
38
39    Description of arguments:
40    cmd_buf                         The command string to be run in a shell.
41    quiet                           Indicates whether this function should run
42                                    the print_issuing() function which prints
43                                    "Issuing: <cmd string>" to stdout.
44    test_mode                       If test_mode is set, this function will
45                                    not actually run the command.  If
46                                    print_output is set, it will print
47                                    "(test_mode) Issuing: <cmd string>" to
48                                    stdout.
49    debug                           If debug is set, this function will print
50                                    extra debug info.
51    print_output                    If this is set, this function will print
52                                    the stdout/stderr generated by the shell
53                                    command.
54    show_err                        If show_err is set, this function will
55                                    print a standardized error report if the
56                                    shell command returns non-zero.
57    return_stderr                   If return_stderr is set, this function
58                                    will process the stdout and stderr streams
59                                    from the shell command separately.  It
60                                    will also return stderr in addition to the
61                                    return code and the stdout.
62    """
63
64    # Determine default values.
65    quiet = int(gm.global_default(quiet, 0))
66    test_mode = int(gm.global_default(test_mode, 0))
67
68    if debug:
69        gp.print_vars(cmd_buf, quiet, test_mode, debug)
70
71    err_msg = gv.svalid_value(cmd_buf)
72    if err_msg != "":
73        raise ValueError(err_msg)
74
75    if not quiet:
76        gp.pissuing(cmd_buf, test_mode)
77
78    if test_mode:
79        if return_stderr:
80            return 0, "", ""
81        else:
82            return 0, ""
83
84    if return_stderr:
85        err_buf = ""
86        stderr = subprocess.PIPE
87    else:
88        stderr = subprocess.STDOUT
89
90    sub_proc = subprocess.Popen(cmd_buf,
91                                bufsize=1,
92                                shell=True,
93                                executable='/bin/bash',
94                                stdout=subprocess.PIPE,
95                                stderr=stderr)
96    out_buf = ""
97    if return_stderr:
98        for line in sub_proc.stderr:
99            err_buf += line
100            if not print_output:
101                continue
102            if robot_env:
103                grp.rprint(line)
104            else:
105                sys.stdout.write(line)
106    for line in sub_proc.stdout:
107        out_buf += line
108        if not print_output:
109            continue
110        if robot_env:
111            grp.rprint(line)
112        else:
113            sys.stdout.write(line)
114    if print_output and not robot_env:
115        sys.stdout.flush()
116    sub_proc.communicate()
117    shell_rc = sub_proc.returncode
118    if shell_rc != 0:
119        err_msg = "The prior shell command failed.\n"
120        err_msg += gp.sprint_var(shell_rc, 1)
121        if not print_output:
122            err_msg += "out_buf:\n" + out_buf
123
124        if show_err:
125            if robot_env:
126                grp.rprint_error_report(err_msg)
127            else:
128                gp.print_error_report(err_msg)
129        if not ignore_err:
130            if robot_env:
131                BuiltIn().fail(err_msg)
132            else:
133                raise ValueError(err_msg)
134
135    if return_stderr:
136        return shell_rc, out_buf, err_buf
137    else:
138        return shell_rc, out_buf
139
140
141def cmd_fnc_u(cmd_buf,
142              quiet=None,
143              debug=None,
144              print_output=1,
145              show_err=1,
146              return_stderr=0,
147              ignore_err=1):
148    r"""
149    Call cmd_fnc with test_mode=0.  See cmd_fnc (above) for details.
150
151    Note the "u" in "cmd_fnc_u" stands for "unconditional".
152    """
153
154    return cmd_fnc(cmd_buf, test_mode=0, quiet=quiet, debug=debug,
155                   print_output=print_output, show_err=show_err,
156                   return_stderr=return_stderr, ignore_err=ignore_err)
157
158
159def parse_command_string(command_string):
160    r"""
161    Parse a bash command-line command string and return the result as a
162    dictionary of parms.
163
164    This can be useful for answering questions like "What did the user specify
165    as the value for parm x in the command string?".
166
167    This function expects the command string to follow the following posix
168    conventions:
169    - Short parameters:
170      -<parm name><space><arg value>
171    - Long parameters:
172      --<parm name>=<arg value>
173
174    The first item in the string will be considered to be the command.  All
175    values not conforming to the specifications above will be considered
176    positional parms.  If there are multiple parms with the same name, they
177    will be put into a list (see illustration below where "-v" is specified
178    multiple times).
179
180    Description of argument(s):
181    command_string                  The complete command string including all
182                                    parameters and arguments.
183
184    Sample input:
185
186    robot_cmd_buf:                                    robot -v
187    OPENBMC_HOST:dummy1 -v keyword_string:'Set Auto Reboot  no' -v
188    lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot -v
189    quiet:0 -v test_mode:0 -v debug:0
190    --outputdir='/home/user1/status/children/'
191    --output=dummy1.Auto_reboot.170802.124544.output.xml
192    --log=dummy1.Auto_reboot.170802.124544.log.html
193    --report=dummy1.Auto_reboot.170802.124544.report.html
194    /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
195
196    Sample output:
197
198    robot_cmd_buf_dict:
199      robot_cmd_buf_dict[command]:                    robot
200      robot_cmd_buf_dict[v]:
201        robot_cmd_buf_dict[v][0]:                     OPENBMC_HOST:dummy1
202        robot_cmd_buf_dict[v][1]:                     keyword_string:Set Auto
203        Reboot no
204        robot_cmd_buf_dict[v][2]:
205        lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot
206        robot_cmd_buf_dict[v][3]:                     quiet:0
207        robot_cmd_buf_dict[v][4]:                     test_mode:0
208        robot_cmd_buf_dict[v][5]:                     debug:0
209      robot_cmd_buf_dict[outputdir]:
210      /home/user1/status/children/
211      robot_cmd_buf_dict[output]:
212      dummy1.Auto_reboot.170802.124544.output.xml
213      robot_cmd_buf_dict[log]:
214      dummy1.Auto_reboot.170802.124544.log.html
215      robot_cmd_buf_dict[report]:
216      dummy1.Auto_reboot.170802.124544.report.html
217      robot_cmd_buf_dict[positional]:
218      /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
219    """
220
221    # We want the parms in the string broken down the way bash would do it,
222    # so we'll call upon bash to do that by creating a simple inline bash
223    # function.
224    bash_func_def = "function parse { for parm in \"${@}\" ; do" +\
225        " echo $parm ; done ; }"
226
227    rc, outbuf = cmd_fnc_u(bash_func_def + " ; parse " + command_string,
228                           quiet=1, print_output=0)
229    command_string_list = outbuf.rstrip("\n").split("\n")
230
231    command_string_dict = collections.OrderedDict()
232    ix = 1
233    command_string_dict['command'] = command_string_list[0]
234    while ix < len(command_string_list):
235        if command_string_list[ix].startswith("--"):
236            key, value = command_string_list[ix].split("=")
237            key = key.lstrip("-")
238        elif command_string_list[ix].startswith("-"):
239            key = command_string_list[ix].lstrip("-")
240            ix += 1
241            try:
242                value = command_string_list[ix]
243            except IndexError:
244                value = ""
245        else:
246            key = 'positional'
247            value = command_string_list[ix]
248        if key in command_string_dict:
249            if isinstance(command_string_dict[key], str):
250                command_string_dict[key] = [command_string_dict[key]]
251            command_string_dict[key].append(value)
252        else:
253            command_string_dict[key] = value
254        ix += 1
255
256    return command_string_dict
257
258
259# Save the original SIGALRM handler for later restoration by shell_cmd.
260original_sigalrm_handler = signal.getsignal(signal.SIGALRM)
261
262
263def shell_cmd_timed_out(signal_number,
264                        frame):
265    r"""
266    Handle an alarm signal generated during the shell_cmd function.
267    """
268
269    gp.dprint_executing()
270    # Get subprocess pid from shell_cmd's call stack.
271    sub_proc = gp.get_stack_var('sub_proc', 0)
272    pid = sub_proc.pid
273    # Terminate the child process.
274    os.kill(pid, signal.SIGTERM)
275    # Restore the original SIGALRM handler.
276    signal.signal(signal.SIGALRM, original_sigalrm_handler)
277
278    return
279
280
281def shell_cmd(command_string,
282              quiet=None,
283              print_output=1,
284              show_err=1,
285              test_mode=0,
286              time_out=None,
287              max_attempts=1,
288              retry_sleep_time=5,
289              allowed_shell_rcs=[0],
290              ignore_err=None,
291              return_stderr=0):
292    r"""
293    Run the given command string in a shell and return a tuple consisting of
294    the shell return code and the output.
295
296    Description of argument(s):
297    command_string                  The command string to be run in a shell
298                                    (e.g. "ls /tmp").
299    quiet                           If set to 0, this function will print
300                                    "Issuing: <cmd string>" to stdout.  When
301                                    the quiet argument is set to None, this
302                                    function will assign a default value by
303                                    searching upward in the stack for the
304                                    quiet variable value.  If no such value is
305                                    found, quiet is set to 0.
306    print_output                    If this is set, this function will print
307                                    the stdout/stderr generated by the shell
308                                    command to stdout.
309    show_err                        If show_err is set, this function will
310                                    print a standardized error report if the
311                                    shell command fails (i.e. if the shell
312                                    command returns a shell_rc that is not in
313                                    allowed_shell_rcs).  Note: Error text is
314                                    only printed if ALL attempts to run the
315                                    command_string fail.  In other words, if
316                                    the command execution is ultimately
317                                    successful, initial failures are hidden.
318    test_mode                       If test_mode is set, this function will
319                                    not actually run the command.  If
320                                    print_output is also set, this function
321                                    will print "(test_mode) Issuing: <cmd
322                                    string>" to stdout.  A caller should call
323                                    shell_cmd directly if they wish to have
324                                    the command string run unconditionally.
325                                    They should call the t_shell_cmd wrapper
326                                    (defined below) if they wish to run the
327                                    command string only if the prevailing
328                                    test_mode variable is set to 0.
329    time_out                        A time-out value expressed in seconds.  If
330                                    the command string has not finished
331                                    executing within <time_out> seconds, it
332                                    will be halted and counted as an error.
333    max_attempts                    The max number of attempts that should be
334                                    made to run the command string.
335    retry_sleep_time                The number of seconds to sleep between
336                                    attempts.
337    allowed_shell_rcs               A list of integers indicating which
338                                    shell_rc values are not to be considered
339                                    errors.
340    ignore_err                      Ignore error means that a failure
341                                    encountered by running the command string
342                                    will not be raised as a python exception.
343                                    When the ignore_err argument is set to
344                                    None, this function will assign a default
345                                    value by searching upward in the stack for
346                                    the ignore_err variable value.  If no such
347                                    value is found, ignore_err is set to 1.
348    return_stderr                   If return_stderr is set, this function
349                                    will process the stdout and stderr streams
350                                    from the shell command separately.  In
351                                    such a case, the tuple returned by this
352                                    function will consist of three values
353                                    rather than just two: rc, stdout, stderr.
354    """
355
356    # Assign default values to some of the arguments to this function.
357    quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0)))
358    ignore_err = int(gm.dft(ignore_err, gp.get_stack_var('ignore_err', 1)))
359
360    err_msg = gv.svalid_value(command_string)
361    if err_msg != "":
362        raise ValueError(err_msg)
363
364    if not quiet:
365        gp.print_issuing(command_string, test_mode)
366
367    if test_mode:
368        if return_stderr:
369            return 0, "", ""
370        else:
371            return 0, ""
372
373    # Convert each list entry to a signed value.
374    allowed_shell_rcs = [gm.to_signed(x) for x in allowed_shell_rcs]
375
376    if return_stderr:
377        stderr = subprocess.PIPE
378    else:
379        stderr = subprocess.STDOUT
380
381    shell_rc = 0
382    out_buf = ""
383    err_buf = ""
384    # Write all output to func_history_stdout rather than directly to stdout.
385    # This allows us to decide what to print after all attempts to run the
386    # command string have been made.  func_history_stdout will contain the
387    # complete stdout history from the current invocation of this function.
388    func_history_stdout = ""
389    for attempt_num in range(1, max_attempts + 1):
390        sub_proc = subprocess.Popen(command_string,
391                                    bufsize=1,
392                                    shell=True,
393                                    executable='/bin/bash',
394                                    stdout=subprocess.PIPE,
395                                    stderr=stderr)
396        out_buf = ""
397        err_buf = ""
398        # Output from this loop iteration is written to func_stdout for later
399        # processing.
400        func_stdout = ""
401        command_timed_out = False
402        if time_out is not None:
403            # Designate a SIGALRM handling function and set alarm.
404            signal.signal(signal.SIGALRM, shell_cmd_timed_out)
405            signal.alarm(time_out)
406        try:
407            if return_stderr:
408                for line in sub_proc.stderr:
409                    err_buf += line
410                    if not print_output:
411                        continue
412                    func_stdout += line
413            for line in sub_proc.stdout:
414                out_buf += line
415                if not print_output:
416                    continue
417                func_stdout += line
418        except IOError:
419            command_timed_out = True
420        sub_proc.communicate()
421        shell_rc = sub_proc.returncode
422        # Restore the original SIGALRM handler and clear the alarm.
423        signal.signal(signal.SIGALRM, original_sigalrm_handler)
424        signal.alarm(0)
425        if shell_rc in allowed_shell_rcs:
426            break
427        err_msg = "The prior shell command failed.\n"
428        if command_timed_out:
429            err_msg += gp.sprint_var(command_timed_out)
430            err_msg += gp.sprint_var(time_out)
431            err_msg += gp.sprint_varx("child_pid", sub_proc.pid)
432        err_msg += gp.sprint_var(attempt_num)
433        err_msg += gp.sprint_var(shell_rc, 1)
434        err_msg += gp.sprint_var(allowed_shell_rcs, 1)
435        if not print_output:
436            if return_stderr:
437                err_msg += "err_buf:\n" + err_buf
438            err_msg += "out_buf:\n" + out_buf
439        if show_err:
440            if robot_env:
441                func_stdout += grp.sprint_error_report(err_msg)
442            else:
443                func_stdout += gp.sprint_error_report(err_msg)
444        func_history_stdout += func_stdout
445        if attempt_num < max_attempts:
446            func_history_stdout += gp.sprint_issuing("time.sleep("
447                                                     + str(retry_sleep_time)
448                                                     + ")")
449            time.sleep(retry_sleep_time)
450
451    if shell_rc not in allowed_shell_rcs:
452        func_stdout = func_history_stdout
453
454    if robot_env:
455        grp.rprint(func_stdout)
456    else:
457        sys.stdout.write(func_stdout)
458        sys.stdout.flush()
459
460    if shell_rc not in allowed_shell_rcs:
461        if not ignore_err:
462            if robot_env:
463                BuiltIn().fail(err_msg)
464            else:
465                raise ValueError("The prior shell command failed.\n")
466
467    if return_stderr:
468        return shell_rc, out_buf, err_buf
469    else:
470        return shell_rc, out_buf
471
472
473def t_shell_cmd(command_string, **kwargs):
474    r"""
475    Search upward in the the call stack to obtain the test_mode argument, add
476    it to kwargs and then call shell_cmd and return the result.
477
478    See shell_cmd prolog for details on all arguments.
479    """
480
481    if 'test_mode' in kwargs:
482        error_message = "Programmer error - test_mode is not a valid" +\
483            " argument to this function."
484        gp.print_error_report(error_message)
485        exit(1)
486
487    test_mode = gp.get_stack_var('test_mode',
488                                 int(gp.get_var_value(None, 0, "test_mode")))
489    kwargs['test_mode'] = test_mode
490
491    return shell_cmd(command_string, **kwargs)
492