1#!/usr/bin/env python
2
3r"""
4This module provides command execution functions such as cmd_fnc and cmd_fnc_u.
5"""
6
7import os
8import sys
9import subprocess
10import collections
11import signal
12import time
13
14import gen_print as gp
15import gen_valid as gv
16import gen_misc as gm
17
18robot_env = gp.robot_env
19
20if robot_env:
21    import gen_robot_print as grp
22    from robot.libraries.BuiltIn import BuiltIn
23
24
25# cmd_fnc and cmd_fnc_u should now be considered deprecated.  shell_cmd and
26# t_shell_cmd should be used instead.
27def cmd_fnc(cmd_buf,
28            quiet=None,
29            test_mode=None,
30            debug=0,
31            print_output=1,
32            show_err=1,
33            return_stderr=0,
34            ignore_err=1):
35    r"""
36    Run the given command in a shell and return the shell return code and the
37    output.
38
39    Description of arguments:
40    cmd_buf                         The command string to be run in a shell.
41    quiet                           Indicates whether this function should run
42                                    the print_issuing() function which prints
43                                    "Issuing: <cmd string>" to stdout.
44    test_mode                       If test_mode is set, this function will
45                                    not actually run the command.  If
46                                    print_output is set, it will print
47                                    "(test_mode) Issuing: <cmd string>" to
48                                    stdout.
49    debug                           If debug is set, this function will print
50                                    extra debug info.
51    print_output                    If this is set, this function will print
52                                    the stdout/stderr generated by the shell
53                                    command.
54    show_err                        If show_err is set, this function will
55                                    print a standardized error report if the
56                                    shell command returns non-zero.
57    return_stderr                   If return_stderr is set, this function
58                                    will process the stdout and stderr streams
59                                    from the shell command separately.  It
60                                    will also return stderr in addition to the
61                                    return code and the stdout.
62    """
63
64    # Determine default values.
65    quiet = int(gm.global_default(quiet, 0))
66    test_mode = int(gm.global_default(test_mode, 0))
67
68    if debug:
69        gp.print_vars(cmd_buf, quiet, test_mode, debug)
70
71    err_msg = gv.svalid_value(cmd_buf)
72    if err_msg != "":
73        raise ValueError(err_msg)
74
75    if not quiet:
76        gp.pissuing(cmd_buf, test_mode)
77
78    if test_mode:
79        if return_stderr:
80            return 0, "", ""
81        else:
82            return 0, ""
83
84    if return_stderr:
85        err_buf = ""
86        stderr = subprocess.PIPE
87    else:
88        stderr = subprocess.STDOUT
89
90    sub_proc = subprocess.Popen(cmd_buf,
91                                bufsize=1,
92                                shell=True,
93                                executable='/bin/bash',
94                                stdout=subprocess.PIPE,
95                                stderr=stderr)
96    out_buf = ""
97    if return_stderr:
98        for line in sub_proc.stderr:
99            try:
100                err_buf += line
101            except TypeError:
102                line = line.decode("utf-8")
103                err_buf += line
104            if not print_output:
105                continue
106            if robot_env:
107                grp.rprint(line)
108            else:
109                sys.stdout.write(line)
110    for line in sub_proc.stdout:
111        try:
112            out_buf += line
113        except TypeError:
114            line = line.decode("utf-8")
115            out_buf += line
116        if not print_output:
117            continue
118        if robot_env:
119            grp.rprint(line)
120        else:
121            sys.stdout.write(line)
122    if print_output and not robot_env:
123        sys.stdout.flush()
124    sub_proc.communicate()
125    shell_rc = sub_proc.returncode
126    if shell_rc != 0:
127        err_msg = "The prior shell command failed.\n"
128        err_msg += gp.sprint_var(shell_rc, 1)
129        if not print_output:
130            err_msg += "out_buf:\n" + out_buf
131
132        if show_err:
133            if robot_env:
134                grp.rprint_error_report(err_msg)
135            else:
136                gp.print_error_report(err_msg)
137        if not ignore_err:
138            if robot_env:
139                BuiltIn().fail(err_msg)
140            else:
141                raise ValueError(err_msg)
142
143    if return_stderr:
144        return shell_rc, out_buf, err_buf
145    else:
146        return shell_rc, out_buf
147
148
149def cmd_fnc_u(cmd_buf,
150              quiet=None,
151              debug=None,
152              print_output=1,
153              show_err=1,
154              return_stderr=0,
155              ignore_err=1):
156    r"""
157    Call cmd_fnc with test_mode=0.  See cmd_fnc (above) for details.
158
159    Note the "u" in "cmd_fnc_u" stands for "unconditional".
160    """
161
162    return cmd_fnc(cmd_buf, test_mode=0, quiet=quiet, debug=debug,
163                   print_output=print_output, show_err=show_err,
164                   return_stderr=return_stderr, ignore_err=ignore_err)
165
166
167def parse_command_string(command_string):
168    r"""
169    Parse a bash command-line command string and return the result as a
170    dictionary of parms.
171
172    This can be useful for answering questions like "What did the user specify
173    as the value for parm x in the command string?".
174
175    This function expects the command string to follow the following posix
176    conventions:
177    - Short parameters:
178      -<parm name><space><arg value>
179    - Long parameters:
180      --<parm name>=<arg value>
181
182    The first item in the string will be considered to be the command.  All
183    values not conforming to the specifications above will be considered
184    positional parms.  If there are multiple parms with the same name, they
185    will be put into a list (see illustration below where "-v" is specified
186    multiple times).
187
188    Description of argument(s):
189    command_string                  The complete command string including all
190                                    parameters and arguments.
191
192    Sample input:
193
194    robot_cmd_buf:                                    robot -v
195    OPENBMC_HOST:dummy1 -v keyword_string:'Set Auto Reboot  no' -v
196    lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot -v
197    quiet:0 -v test_mode:0 -v debug:0
198    --outputdir='/home/user1/status/children/'
199    --output=dummy1.Auto_reboot.170802.124544.output.xml
200    --log=dummy1.Auto_reboot.170802.124544.log.html
201    --report=dummy1.Auto_reboot.170802.124544.report.html
202    /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
203
204    Sample output:
205
206    robot_cmd_buf_dict:
207      robot_cmd_buf_dict[command]:                    robot
208      robot_cmd_buf_dict[v]:
209        robot_cmd_buf_dict[v][0]:                     OPENBMC_HOST:dummy1
210        robot_cmd_buf_dict[v][1]:                     keyword_string:Set Auto
211        Reboot no
212        robot_cmd_buf_dict[v][2]:
213        lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot
214        robot_cmd_buf_dict[v][3]:                     quiet:0
215        robot_cmd_buf_dict[v][4]:                     test_mode:0
216        robot_cmd_buf_dict[v][5]:                     debug:0
217      robot_cmd_buf_dict[outputdir]:
218      /home/user1/status/children/
219      robot_cmd_buf_dict[output]:
220      dummy1.Auto_reboot.170802.124544.output.xml
221      robot_cmd_buf_dict[log]:
222      dummy1.Auto_reboot.170802.124544.log.html
223      robot_cmd_buf_dict[report]:
224      dummy1.Auto_reboot.170802.124544.report.html
225      robot_cmd_buf_dict[positional]:
226      /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
227    """
228
229    # We want the parms in the string broken down the way bash would do it,
230    # so we'll call upon bash to do that by creating a simple inline bash
231    # function.
232    bash_func_def = "function parse { for parm in \"${@}\" ; do" +\
233        " echo $parm ; done ; }"
234
235    rc, outbuf = cmd_fnc_u(bash_func_def + " ; parse " + command_string,
236                           quiet=1, print_output=0)
237    command_string_list = outbuf.rstrip("\n").split("\n")
238
239    command_string_dict = collections.OrderedDict()
240    ix = 1
241    command_string_dict['command'] = command_string_list[0]
242    while ix < len(command_string_list):
243        if command_string_list[ix].startswith("--"):
244            key, value = command_string_list[ix].split("=")
245            key = key.lstrip("-")
246        elif command_string_list[ix].startswith("-"):
247            key = command_string_list[ix].lstrip("-")
248            ix += 1
249            try:
250                value = command_string_list[ix]
251            except IndexError:
252                value = ""
253        else:
254            key = 'positional'
255            value = command_string_list[ix]
256        if key in command_string_dict:
257            if isinstance(command_string_dict[key], str):
258                command_string_dict[key] = [command_string_dict[key]]
259            command_string_dict[key].append(value)
260        else:
261            command_string_dict[key] = value
262        ix += 1
263
264    return command_string_dict
265
266
267# Save the original SIGALRM handler for later restoration by shell_cmd.
268original_sigalrm_handler = signal.getsignal(signal.SIGALRM)
269
270
271def shell_cmd_timed_out(signal_number,
272                        frame):
273    r"""
274    Handle an alarm signal generated during the shell_cmd function.
275    """
276
277    gp.dprint_executing()
278    # Get subprocess pid from shell_cmd's call stack.
279    sub_proc = gp.get_stack_var('sub_proc', 0)
280    pid = sub_proc.pid
281    # Terminate the child process.
282    os.kill(pid, signal.SIGTERM)
283    # Restore the original SIGALRM handler.
284    signal.signal(signal.SIGALRM, original_sigalrm_handler)
285
286    return
287
288
289def shell_cmd(command_string,
290              quiet=None,
291              print_output=None,
292              show_err=1,
293              test_mode=0,
294              time_out=None,
295              max_attempts=1,
296              retry_sleep_time=5,
297              allowed_shell_rcs=[0],
298              ignore_err=None,
299              return_stderr=0,
300              fork=0):
301    r"""
302    Run the given command string in a shell and return a tuple consisting of
303    the shell return code and the output.
304
305    Description of argument(s):
306    command_string                  The command string to be run in a shell
307                                    (e.g. "ls /tmp").
308    quiet                           If set to 0, this function will print
309                                    "Issuing: <cmd string>" to stdout.  When
310                                    the quiet argument is set to None, this
311                                    function will assign a default value by
312                                    searching upward in the stack for the
313                                    quiet variable value.  If no such value is
314                                    found, quiet is set to 0.
315    print_output                    If this is set, this function will print
316                                    the stdout/stderr generated by the shell
317                                    command to stdout.
318    show_err                        If show_err is set, this function will
319                                    print a standardized error report if the
320                                    shell command fails (i.e. if the shell
321                                    command returns a shell_rc that is not in
322                                    allowed_shell_rcs).  Note: Error text is
323                                    only printed if ALL attempts to run the
324                                    command_string fail.  In other words, if
325                                    the command execution is ultimately
326                                    successful, initial failures are hidden.
327    test_mode                       If test_mode is set, this function will
328                                    not actually run the command.  If
329                                    print_output is also set, this function
330                                    will print "(test_mode) Issuing: <cmd
331                                    string>" to stdout.  A caller should call
332                                    shell_cmd directly if they wish to have
333                                    the command string run unconditionally.
334                                    They should call the t_shell_cmd wrapper
335                                    (defined below) if they wish to run the
336                                    command string only if the prevailing
337                                    test_mode variable is set to 0.
338    time_out                        A time-out value expressed in seconds.  If
339                                    the command string has not finished
340                                    executing within <time_out> seconds, it
341                                    will be halted and counted as an error.
342    max_attempts                    The max number of attempts that should be
343                                    made to run the command string.
344    retry_sleep_time                The number of seconds to sleep between
345                                    attempts.
346    allowed_shell_rcs               A list of integers indicating which
347                                    shell_rc values are not to be considered
348                                    errors.
349    ignore_err                      Ignore error means that a failure
350                                    encountered by running the command string
351                                    will not be raised as a python exception.
352                                    When the ignore_err argument is set to
353                                    None, this function will assign a default
354                                    value by searching upward in the stack for
355                                    the ignore_err variable value.  If no such
356                                    value is found, ignore_err is set to 1.
357    return_stderr                   If return_stderr is set, this function
358                                    will process the stdout and stderr streams
359                                    from the shell command separately.  In
360                                    such a case, the tuple returned by this
361                                    function will consist of three values
362                                    rather than just two: rc, stdout, stderr.
363    fork                            Run the command string asynchronously
364                                    (i.e. don't wait for status of the child
365                                    process and don't try to get
366                                    stdout/stderr).
367    """
368
369    # Assign default values to some of the arguments to this function.
370    quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0)))
371    print_output = int(gm.dft(print_output, not quiet))
372    show_err = int(show_err)
373    global_ignore_err = gp.get_var_value(ignore_err, 1)
374    stack_ignore_err = gp.get_stack_var('ignore_err', global_ignore_err)
375    ignore_err = int(gm.dft(ignore_err, gm.dft(stack_ignore_err, 1)))
376
377    err_msg = gv.svalid_value(command_string)
378    if err_msg != "":
379        raise ValueError(err_msg)
380
381    if not quiet:
382        gp.print_issuing(command_string, test_mode)
383
384    if test_mode:
385        if return_stderr:
386            return 0, "", ""
387        else:
388            return 0, ""
389
390    # Convert each list entry to a signed value.
391    allowed_shell_rcs = [gm.to_signed(x) for x in allowed_shell_rcs]
392
393    if return_stderr:
394        stderr = subprocess.PIPE
395    else:
396        stderr = subprocess.STDOUT
397
398    shell_rc = 0
399    out_buf = ""
400    err_buf = ""
401    # Write all output to func_history_stdout rather than directly to stdout.
402    # This allows us to decide what to print after all attempts to run the
403    # command string have been made.  func_history_stdout will contain the
404    # complete stdout history from the current invocation of this function.
405    func_history_stdout = ""
406    for attempt_num in range(1, max_attempts + 1):
407        sub_proc = subprocess.Popen(command_string,
408                                    bufsize=1,
409                                    shell=True,
410                                    executable='/bin/bash',
411                                    stdout=subprocess.PIPE,
412                                    stderr=stderr)
413        out_buf = ""
414        err_buf = ""
415        # Output from this loop iteration is written to func_stdout for later
416        # processing.
417        func_stdout = ""
418        if fork:
419            break
420        command_timed_out = False
421        if time_out is not None:
422            # Designate a SIGALRM handling function and set alarm.
423            signal.signal(signal.SIGALRM, shell_cmd_timed_out)
424            signal.alarm(time_out)
425        try:
426            if return_stderr:
427                for line in sub_proc.stderr:
428                    try:
429                        err_buf += line
430                    except TypeError:
431                        line = line.decode("utf-8")
432                        err_buf += line
433                    if not print_output:
434                        continue
435                    func_stdout += line
436            for line in sub_proc.stdout:
437                try:
438                    out_buf += line
439                except TypeError:
440                    line = line.decode("utf-8")
441                    out_buf += line
442                if not print_output:
443                    continue
444                func_stdout += line
445        except IOError:
446            command_timed_out = True
447        sub_proc.communicate()
448        shell_rc = sub_proc.returncode
449        # Restore the original SIGALRM handler and clear the alarm.
450        signal.signal(signal.SIGALRM, original_sigalrm_handler)
451        signal.alarm(0)
452        if shell_rc in allowed_shell_rcs:
453            break
454        err_msg = "The prior shell command failed.\n"
455        if quiet:
456            err_msg += gp.sprint_var(command_string)
457        if command_timed_out:
458            err_msg += gp.sprint_var(command_timed_out)
459            err_msg += gp.sprint_var(time_out)
460            err_msg += gp.sprint_varx("child_pid", sub_proc.pid)
461        err_msg += gp.sprint_var(attempt_num)
462        err_msg += gp.sprint_var(shell_rc, 1)
463        err_msg += gp.sprint_var(allowed_shell_rcs, 1)
464        if not print_output:
465            if return_stderr:
466                err_msg += "err_buf:\n" + err_buf
467            err_msg += "out_buf:\n" + out_buf
468        if show_err:
469            if robot_env:
470                func_stdout += grp.sprint_error_report(err_msg)
471            else:
472                func_stdout += gp.sprint_error_report(err_msg)
473        func_history_stdout += func_stdout
474        if attempt_num < max_attempts:
475            func_history_stdout += gp.sprint_issuing("time.sleep("
476                                                     + str(retry_sleep_time)
477                                                     + ")")
478            time.sleep(retry_sleep_time)
479
480    if shell_rc not in allowed_shell_rcs:
481        func_stdout = func_history_stdout
482
483    if robot_env:
484        grp.rprint(func_stdout)
485    else:
486        sys.stdout.write(func_stdout)
487        sys.stdout.flush()
488
489    if shell_rc not in allowed_shell_rcs:
490        if not ignore_err:
491            if robot_env:
492                BuiltIn().fail(err_msg)
493            else:
494                raise ValueError("The prior shell command failed.\n")
495
496    if return_stderr:
497        return shell_rc, out_buf, err_buf
498    else:
499        return shell_rc, out_buf
500
501
502def t_shell_cmd(command_string, **kwargs):
503    r"""
504    Search upward in the the call stack to obtain the test_mode argument, add
505    it to kwargs and then call shell_cmd and return the result.
506
507    See shell_cmd prolog for details on all arguments.
508    """
509
510    if 'test_mode' in kwargs:
511        error_message = "Programmer error - test_mode is not a valid" +\
512            " argument to this function."
513        gp.print_error_report(error_message)
514        exit(1)
515
516    test_mode = gp.get_stack_var('test_mode',
517                                 int(gp.get_var_value(None, 0, "test_mode")))
518    kwargs['test_mode'] = test_mode
519
520    return shell_cmd(command_string, **kwargs)
521