#!/usr/bin/env python3 r""" This module provides command execution functions such as cmd_fnc and cmd_fnc_u. """ import os import sys import subprocess import collections import signal import time import re import inspect import gen_print as gp import gen_valid as gv import gen_misc as gm import func_args as fa robot_env = gp.robot_env if robot_env: from robot.libraries.BuiltIn import BuiltIn # cmd_fnc and cmd_fnc_u should now be considered deprecated. shell_cmd and t_shell_cmd should be used # instead. def cmd_fnc(cmd_buf, quiet=None, test_mode=None, debug=0, print_output=1, show_err=1, return_stderr=0, ignore_err=1): r""" Run the given command in a shell and return the shell return code and the output. Description of arguments: cmd_buf The command string to be run in a shell. quiet Indicates whether this function should run the print_issuing() function which prints "Issuing: " to stdout. test_mode If test_mode is set, this function will not actually run the command. If print_output is set, it will print "(test_mode) Issuing: " to stdout. debug If debug is set, this function will print extra debug info. print_output If this is set, this function will print the stdout/stderr generated by the shell command. show_err If show_err is set, this function will print a standardized error report if the shell command returns non-zero. return_stderr If return_stderr is set, this function will process the stdout and stderr streams from the shell command separately. It will also return stderr in addition to the return code and the stdout. """ # Determine default values. quiet = int(gm.global_default(quiet, 0)) test_mode = int(gm.global_default(test_mode, 0)) if debug: gp.print_vars(cmd_buf, quiet, test_mode, debug) err_msg = gv.valid_value(cmd_buf) if err_msg != "": raise ValueError(err_msg) if not quiet: gp.pissuing(cmd_buf, test_mode) if test_mode: if return_stderr: return 0, "", "" else: return 0, "" if return_stderr: err_buf = "" stderr = subprocess.PIPE else: stderr = subprocess.STDOUT sub_proc = subprocess.Popen(cmd_buf, bufsize=1, shell=True, universal_newlines=True, executable='/bin/bash', stdout=subprocess.PIPE, stderr=stderr) out_buf = "" if return_stderr: for line in sub_proc.stderr: try: err_buf += line except TypeError: line = line.decode("utf-8") err_buf += line if not print_output: continue gp.gp_print(line) for line in sub_proc.stdout: try: out_buf += line except TypeError: line = line.decode("utf-8") out_buf += line if not print_output: continue gp.gp_print(line) if print_output and not robot_env: sys.stdout.flush() sub_proc.communicate() shell_rc = sub_proc.returncode if shell_rc != 0: err_msg = "The prior shell command failed.\n" err_msg += gp.sprint_var(shell_rc, gp.hexa()) if not print_output: err_msg += "out_buf:\n" + out_buf if show_err: gp.print_error_report(err_msg) if not ignore_err: if robot_env: BuiltIn().fail(err_msg) else: raise ValueError(err_msg) if return_stderr: return shell_rc, out_buf, err_buf else: return shell_rc, out_buf def cmd_fnc_u(cmd_buf, quiet=None, debug=None, print_output=1, show_err=1, return_stderr=0, ignore_err=1): r""" Call cmd_fnc with test_mode=0. See cmd_fnc (above) for details. Note the "u" in "cmd_fnc_u" stands for "unconditional". """ return cmd_fnc(cmd_buf, test_mode=0, quiet=quiet, debug=debug, print_output=print_output, show_err=show_err, return_stderr=return_stderr, ignore_err=ignore_err) def parse_command_string(command_string): r""" Parse a bash command-line command string and return the result as a dictionary of parms. This can be useful for answering questions like "What did the user specify as the value for parm x in the command string?". This function expects the command string to follow the following posix conventions: - Short parameters: - - Long parameters: --= The first item in the string will be considered to be the command. All values not conforming to the specifications above will be considered positional parms. If there are multiple parms with the same name, they will be put into a list (see illustration below where "-v" is specified multiple times). Description of argument(s): command_string The complete command string including all parameters and arguments. Sample input: robot_cmd_buf: robot -v OPENBMC_HOST:dummy1 -v keyword_string:'Set Auto Reboot no' -v lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot -v quiet:0 -v test_mode:0 -v debug:0 --outputdir='/home/user1/status/children/' --output=dummy1.Auto_reboot.170802.124544.output.xml --log=dummy1.Auto_reboot.170802.124544.log.html --report=dummy1.Auto_reboot.170802.124544.report.html /home/user1/git/openbmc-test-automation/extended/run_keyword.robot Sample output: robot_cmd_buf_dict: robot_cmd_buf_dict[command]: robot robot_cmd_buf_dict[v]: robot_cmd_buf_dict[v][0]: OPENBMC_HOST:dummy1 robot_cmd_buf_dict[v][1]: keyword_string:Set Auto Reboot no robot_cmd_buf_dict[v][2]: lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot robot_cmd_buf_dict[v][3]: quiet:0 robot_cmd_buf_dict[v][4]: test_mode:0 robot_cmd_buf_dict[v][5]: debug:0 robot_cmd_buf_dict[outputdir]: /home/user1/status/children/ robot_cmd_buf_dict[output]: dummy1.Auto_reboot.170802.124544.output.xml robot_cmd_buf_dict[log]: dummy1.Auto_reboot.170802.124544.log.html robot_cmd_buf_dict[report]: dummy1.Auto_reboot.170802.124544.report.html robot_cmd_buf_dict[positional]: /home/user1/git/openbmc-test-automation/extended/run_keyword.robot """ # We want the parms in the string broken down the way bash would do it, so we'll call upon bash to do # that by creating a simple inline bash function. bash_func_def = "function parse { for parm in \"${@}\" ; do" +\ " echo $parm ; done ; }" rc, outbuf = cmd_fnc_u(bash_func_def + " ; parse " + command_string, quiet=1, print_output=0) command_string_list = outbuf.rstrip("\n").split("\n") command_string_dict = collections.OrderedDict() ix = 1 command_string_dict['command'] = command_string_list[0] while ix < len(command_string_list): if command_string_list[ix].startswith("--"): key, value = command_string_list[ix].split("=") key = key.lstrip("-") elif command_string_list[ix].startswith("-"): key = command_string_list[ix].lstrip("-") ix += 1 try: value = command_string_list[ix] except IndexError: value = "" else: key = 'positional' value = command_string_list[ix] if key in command_string_dict: if isinstance(command_string_dict[key], str): command_string_dict[key] = [command_string_dict[key]] command_string_dict[key].append(value) else: command_string_dict[key] = value ix += 1 return command_string_dict # Save the original SIGALRM handler for later restoration by shell_cmd. original_sigalrm_handler = signal.getsignal(signal.SIGALRM) def shell_cmd_timed_out(signal_number, frame): r""" Handle an alarm signal generated during the shell_cmd function. """ gp.dprint_executing() global command_timed_out command_timed_out = True # Get subprocess pid from shell_cmd's call stack. sub_proc = gp.get_stack_var('sub_proc', 0) pid = sub_proc.pid gp.dprint_var(pid) # Terminate the child process group. os.killpg(pid, signal.SIGKILL) # Restore the original SIGALRM handler. signal.signal(signal.SIGALRM, original_sigalrm_handler) return def shell_cmd(command_string, quiet=None, print_output=None, show_err=1, test_mode=0, time_out=None, max_attempts=1, retry_sleep_time=5, valid_rcs=[0], ignore_err=None, return_stderr=0, fork=0, error_regexes=None): r""" Run the given command string in a shell and return a tuple consisting of the shell return code and the output. Description of argument(s): command_string The command string to be run in a shell (e.g. "ls /tmp"). quiet If set to 0, this function will print "Issuing: " to stdout. When the quiet argument is set to None, this function will assign a default value by searching upward in the stack for the quiet variable value. If no such value is found, quiet is set to 0. print_output If this is set, this function will print the stdout/stderr generated by the shell command to stdout. show_err If show_err is set, this function will print a standardized error report if the shell command fails (i.e. if the shell command returns a shell_rc that is not in valid_rcs). Note: Error text is only printed if ALL attempts to run the command_string fail. In other words, if the command execution is ultimately successful, initial failures are hidden. test_mode If test_mode is set, this function will not actually run the command. If print_output is also set, this function will print "(test_mode) Issuing: " to stdout. A caller should call shell_cmd directly if they wish to have the command string run unconditionally. They should call the t_shell_cmd wrapper (defined below) if they wish to run the command string only if the prevailing test_mode variable is set to 0. time_out A time-out value expressed in seconds. If the command string has not finished executing within seconds, it will be halted and counted as an error. max_attempts The max number of attempts that should be made to run the command string. retry_sleep_time The number of seconds to sleep between attempts. valid_rcs A list of integers indicating which shell_rc values are not to be considered errors. ignore_err Ignore error means that a failure encountered by running the command string will not be raised as a python exception. When the ignore_err argument is set to None, this function will assign a default value by searching upward in the stack for the ignore_err variable value. If no such value is found, ignore_err is set to 1. return_stderr If return_stderr is set, this function will process the stdout and stderr streams from the shell command separately. In such a case, the tuple returned by this function will consist of three values rather than just two: rc, stdout, stderr. fork Run the command string asynchronously (i.e. don't wait for status of the child process and don't try to get stdout/stderr) and return the Popen object created by the subprocess.popen() function. See the kill_cmd function for details on how to process the popen object. error_regexes A list of regular expressions to be used to identify errors in the command output. If there is a match for any of these regular expressions, the command will be considered a failure and the shell_rc will be set to -1. For example, if error_regexes = ['ERROR:'] and the command output contains 'ERROR: Unrecognized option', it will be counted as an error even if the command returned 0. This is useful when running commands that do not always return non-zero on error. """ err_msg = gv.valid_value(command_string) if err_msg: raise ValueError(err_msg) # Assign default values to some of the arguments to this function. quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0))) print_output = int(gm.dft(print_output, not quiet)) show_err = int(show_err) ignore_err = int(gm.dft(ignore_err, gp.get_stack_var('ignore_err', 1))) gp.qprint_issuing(command_string, test_mode) if test_mode: return (0, "", "") if return_stderr else (0, "") # Convert a string python dictionary definition to a dictionary. valid_rcs = fa.source_to_object(valid_rcs) # Convert each list entry to a signed value. valid_rcs = [gm.to_signed(x) for x in valid_rcs] stderr = subprocess.PIPE if return_stderr else subprocess.STDOUT # Write all output to func_out_history_buf rather than directly to stdout. This allows us to decide # what to print after all attempts to run the command string have been made. func_out_history_buf will # contain the complete history from the current invocation of this function. global command_timed_out command_timed_out = False func_out_history_buf = "" for attempt_num in range(1, max_attempts + 1): sub_proc = subprocess.Popen(command_string, bufsize=1, shell=True, universal_newlines=True, executable='/bin/bash', stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=stderr) if fork: return sub_proc if time_out: command_timed_out = False # Designate a SIGALRM handling function and set alarm. signal.signal(signal.SIGALRM, shell_cmd_timed_out) signal.alarm(time_out) try: stdout_buf, stderr_buf = sub_proc.communicate() except IOError: command_timed_out = True # Restore the original SIGALRM handler and clear the alarm. signal.signal(signal.SIGALRM, original_sigalrm_handler) signal.alarm(0) # Output from this loop iteration is written to func_out_buf for later processing. This can include # stdout, stderr and our own error messages. func_out_buf = "" if print_output: if return_stderr: func_out_buf += stderr_buf func_out_buf += stdout_buf shell_rc = sub_proc.returncode if shell_rc in valid_rcs: # Check output for text indicating there is an error. if error_regexes and re.match('|'.join(error_regexes), stdout_buf): shell_rc = -1 else: break err_msg = "The prior shell command failed.\n" err_msg += gp.sprint_var(attempt_num) err_msg += gp.sprint_vars(command_string, command_timed_out, time_out) err_msg += gp.sprint_varx("child_pid", sub_proc.pid) err_msg += gp.sprint_vars(shell_rc, valid_rcs, fmt=gp.hexa()) if error_regexes: err_msg += gp.sprint_vars(error_regexes) if not print_output: if return_stderr: err_msg += "stderr_buf:\n" + stderr_buf err_msg += "stdout_buf:\n" + stdout_buf if show_err: func_out_buf += gp.sprint_error_report(err_msg) if attempt_num < max_attempts: cmd_buf = "time.sleep(" + str(retry_sleep_time) + ")" if show_err: func_out_buf += gp.sprint_issuing(cmd_buf) exec(cmd_buf) func_out_history_buf += func_out_buf if shell_rc in valid_rcs: gp.gp_print(func_out_buf) else: if show_err: gp.gp_print(func_out_history_buf, stream='stderr') else: # There is no error information to show so just print output from last loop iteration. gp.gp_print(func_out_buf) if not ignore_err: # If the caller has already asked to show error info, avoid repeating that in the failure message. err_msg = "The prior shell command failed.\n" if show_err \ else err_msg if robot_env: BuiltIn().fail(err_msg) else: raise ValueError(err_msg) return (shell_rc, stdout_buf, stderr_buf) if return_stderr \ else (shell_rc, stdout_buf) def t_shell_cmd(command_string, **kwargs): r""" Search upward in the the call stack to obtain the test_mode argument, add it to kwargs and then call shell_cmd and return the result. See shell_cmd prolog for details on all arguments. """ if 'test_mode' in kwargs: error_message = "Programmer error - test_mode is not a valid" +\ " argument to this function." gp.print_error_report(error_message) exit(1) test_mode = int(gp.get_stack_var('test_mode', 0)) kwargs['test_mode'] = test_mode return shell_cmd(command_string, **kwargs) def kill_cmd(popen, sig=signal.SIGTERM): r""" Kill the subprocess represented by the Popen object and return a tuple consisting of the shell return code and the output. This function is meant to be used as the follow-up for a call to shell_cmd(..., fork=1). Example: popen = shell_cmd("some_pgm.py", fork=1) ... shell_rc, output = kill_cmd(popen) Description of argument(s): popen A Popen object returned by the subprocess.Popen() command. sig The signal to be sent to the child process. """ gp.dprint_var(popen.pid) os.killpg(popen.pid, sig) stdout, stderr = popen.communicate() shell_rc = popen.returncode return (shell_rc, stdout, stderr) if stderr else (shell_rc, stdout) def re_order_kwargs(stack_frame_ix, **kwargs): r""" Re-order the kwargs to match the order in which they were specified on a function invocation and return as an ordered dictionary. Note that this re_order_kwargs function should not be necessary in python versions 3.6 and beyond. Example: The caller calls func1 like this: func1('mike', arg1='one', arg2='two', arg3='three') And func1 is defined as follows: def func1(first_arg, **kwargs): kwargs = re_order_kwargs(first_arg_num=2, stack_frame_ix=3, **kwargs) The kwargs dictionary before calling re_order_kwargs (where order is not guaranteed): kwargs: kwargs[arg3]: three kwargs[arg2]: two kwargs[arg1]: one The kwargs dictionary after calling re_order_kwargs: kwargs: kwargs[arg1]: one kwargs[arg2]: two kwargs[arg3]: three Note that the re-ordered kwargs match the order specified on the call to func1. Description of argument(s): stack_frame_ix The stack frame of the function whose kwargs values must be re-ordered. 0 is the stack frame of re_order_kwargs, 1 is the stack from of its caller and so on. kwargs The keyword argument dictionary which is to be re-ordered. """ new_kwargs = collections.OrderedDict() # Get position number of first keyword on the calling line of code. (args, varargs, keywords, locals) =\ inspect.getargvalues(inspect.stack()[stack_frame_ix][0]) first_kwarg_pos = 1 + len(args) if varargs is not None: first_kwarg_pos += len(locals[varargs]) for arg_num in range(first_kwarg_pos, first_kwarg_pos + len(kwargs)): # This will result in an arg_name value such as "arg1='one'". arg_name = gp.get_arg_name(None, arg_num, stack_frame_ix + 2) # Continuing with the prior example, the following line will result # in key being set to 'arg1'. key = arg_name.split('=')[0] new_kwargs[key] = kwargs[key] return new_kwargs def default_arg_delim(arg_dashes): r""" Return the default argument delimiter value for the given arg_dashes value. Note: this function is useful for functions that manipulate bash command line arguments (e.g. --parm=1 or -parm 1). Description of argument(s): arg_dashes The argument dashes specifier (usually, "-" or "--"). """ if arg_dashes == "--": return "=" return " " def create_command_string(command, *pos_parms, **options): r""" Create and return a bash command string consisting of the given arguments formatted as text. The default formatting of options is as follows: