xref: /openbmc/openbmc-test-automation/lib/gen_misc.py (revision 8e6ebd256f7cf3db67a11d895631820d2873dd08)
1 #!/usr/bin/env python3
2 
3 r"""
4 This module provides many valuable functions such as my_parm_file.
5 """
6 
7 import collections
8 import errno
9 import inspect
10 import json
11 import os
12 import random
13 import shutil
14 
15 # sys and os are needed to get the program dir path and program name.
16 import sys
17 import time
18 
19 try:
20     import ConfigParser
21 except ImportError:
22     import configparser
23 try:
24     import StringIO
25 except ImportError:
26     import io
27 
28 import re
29 import socket
30 import tempfile
31 
32 try:
33     import psutil
34 
35     psutil_imported = True
36 except ImportError:
37     psutil_imported = False
38 
39 import gen_cmd as gc
40 import gen_print as gp
41 
42 robot_env = gp.robot_env
43 if robot_env:
44     from robot.libraries.BuiltIn import BuiltIn
45     from robot.utils import DotDict
46 
47 
48 def add_trailing_slash(dir_path):
49     r"""
50     Add a trailing slash to the directory path if it doesn't already have one
51     and return it.
52 
53     Description of arguments:
54     dir_path                        A directory path.
55     """
56 
57     return os.path.normpath(dir_path) + os.path.sep
58 
59 
60 def makedirs(path, mode=0o777, quiet=None):
61     r"""
62     Call os.makedirs with the caller's arguments.
63 
64     This function offers 2 advantages over the base os.makedirs function:
65     1) It will not fail if the directory already exists.
66     2) It will print an "Issuing: os.makedirs" message.
67 
68     Description of argument(s):
69     path                            The path containing the directories to be created.
70     mode                            The mode or permissions to be granted to the created directories.
71     quiet                           Indicates whether this function should run the print_issuing() function.
72     """
73     quiet = int(dft(quiet, gp.get_stack_var("quiet", 0)))
74     gp.qprint_issuing("os.makedirs('" + path + "', mode=" + oct(mode) + ")")
75     try:
76         os.makedirs(path, mode)
77     except OSError:
78         pass
79 
80 
81 def rmtree(path, ignore_errors=False, onerror=None, quiet=None):
82     r"""
83     Call shutil.rmtree with the caller's arguments.
84 
85     This function offers this advantage over the base function:
86     - It will print an "Issuing: shutil.rmtree" message.
87 
88     Description of argument(s):
89     (All parms are passed directly to shutil.rmtree.  See its prolog for details)
90     quiet                           Indicates whether this function should run the print_issuing() function.
91     """
92     quiet = int(dft(quiet, gp.get_stack_var("quiet", 0)))
93     print_string = gp.sprint_executing(max_width=2000)
94     print_string = re.sub(
95         r"Executing: ", "Issuing: shutil.", print_string.rstrip("\n")
96     )
97     gp.qprintn(re.sub(r", quiet[ ]?=.*", ")", print_string))
98     shutil.rmtree(path, ignore_errors, onerror)
99 
100 
101 def chdir(path, quiet=None):
102     r"""
103     Call os.chdir with the caller's arguments.
104 
105     This function offers this advantage over the base os.chdir function:
106     - It will print an "Issuing: os.chdir" message.
107 
108     Description of argument(s):
109     path                            The path of the directory to change to.
110     quiet                           Indicates whether this function should run the print_issuing() function.
111     """
112     quiet = int(dft(quiet, gp.get_stack_var("quiet", 0)))
113     gp.qprint_issuing("os.chdir('" + path + "')")
114     os.chdir(path)
115 
116 
117 def which(file_path):
118     r"""
119     Find the full path of an executable file and return it.
120 
121     The PATH environment variable dictates the results of this function.
122 
123     Description of arguments:
124     file_path                       The relative file path (e.g. "my_file" or "lib/my_file").
125     """
126 
127     shell_rc, out_buf = gc.cmd_fnc_u(
128         "which " + file_path, quiet=1, print_output=0, show_err=0
129     )
130     if shell_rc != 0:
131         error_message = (
132             'Failed to find complete path for file "' + file_path + '".\n'
133         )
134         error_message += gp.sprint_var(shell_rc, gp.hexa())
135         error_message += out_buf
136         if robot_env:
137             BuiltIn().fail(gp.sprint_error(error_message))
138         else:
139             gp.print_error_report(error_message)
140             return False
141 
142     file_path = out_buf.rstrip("\n")
143 
144     return file_path
145 
146 
147 def add_path(new_path, path, position=0):
148     r"""
149     Add new_path to path, provided that path doesn't already contain new_path, and return the result.
150 
151     Example:
152     If PATH has a value of "/bin/user:/lib/user".  The following code:
153 
154     PATH = add_path("/tmp/new_path", PATH)
155 
156     will change PATH to "/tmp/new_path:/bin/user:/lib/user".
157 
158     Description of argument(s):
159     new_path                        The path to be added.  This function will strip the trailing slash.
160     path                            The path value to which the new_path should be added.
161     position                        The position in path where the new_path should be added.  0 means it
162                                     should be added to the beginning, 1 means add it as the 2nd item, etc.
163                                     sys.maxsize means it should be added to the end.
164     """
165 
166     path_list = list(filter(None, path.split(":")))
167     new_path = new_path.rstrip("/")
168     if new_path not in path_list:
169         path_list.insert(int(position), new_path)
170     return ":".join(path_list)
171 
172 
173 def dft(value, default):
174     r"""
175     Return default if value is None.  Otherwise, return value.
176 
177     This is really just shorthand as shown below.
178 
179     dft(value, default)
180 
181     vs
182 
183     default if value is None else value
184 
185     Description of arguments:
186     value                           The value to be returned.
187     default                         The default value to return if value is None.
188     """
189 
190     return default if value is None else value
191 
192 
193 def get_mod_global(var_name, default=None, mod_name="__main__"):
194     r"""
195     Get module global variable value and return it.
196 
197     If we are running in a robot environment, the behavior will default to
198     calling get_variable_value.
199 
200     Description of arguments:
201     var_name                        The name of the variable whose value is sought.
202     default                         The value to return if the global does not exist.
203     mod_name                        The name of the module containing the global variable.
204     """
205 
206     if robot_env:
207         return BuiltIn().get_variable_value("${" + var_name + "}", default)
208 
209     try:
210         module = sys.modules[mod_name]
211     except KeyError:
212         gp.print_error_report(
213             "Programmer error - The mod_name passed to"
214             + " this function is invalid:\n"
215             + gp.sprint_var(mod_name)
216         )
217         raise ValueError("Programmer error.")
218 
219     if default is None:
220         return getattr(module, var_name)
221     else:
222         return getattr(module, var_name, default)
223 
224 
225 def global_default(var_value, default=0):
226     r"""
227     If var_value is not None, return it.  Otherwise, return the global
228     variable of the same name, if it exists.  If not, return default.
229 
230     This is meant for use by functions needing help assigning dynamic default
231     values to their parms.  Example:
232 
233     def func1(parm1=None):
234 
235         parm1 = global_default(parm1, 0)
236 
237     Description of arguments:
238     var_value                       The value being evaluated.
239     default                         The value to be returned if var_value is None AND the global variable of
240                                     the same name does not exist.
241     """
242 
243     var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
244 
245     return dft(var_value, get_mod_global(var_name, 0))
246 
247 
248 def set_mod_global(var_value, mod_name="__main__", var_name=None):
249     r"""
250     Set a global variable for a given module.
251 
252     Description of arguments:
253     var_value                       The value to set in the variable.
254     mod_name                        The name of the module whose variable is to be set.
255     var_name                        The name of the variable to set.  This defaults to the name of the
256                                     variable used for var_value when calling this function.
257     """
258 
259     try:
260         module = sys.modules[mod_name]
261     except KeyError:
262         gp.print_error_report(
263             "Programmer error - The mod_name passed to"
264             + " this function is invalid:\n"
265             + gp.sprint_var(mod_name)
266         )
267         raise ValueError("Programmer error.")
268 
269     if var_name is None:
270         var_name = gp.get_arg_name(None, 1, 2)
271 
272     setattr(module, var_name, var_value)
273 
274 
275 def my_parm_file(prop_file_path):
276     r"""
277     Read a properties file, put the keys/values into a dictionary and return the dictionary.
278 
279     The properties file must have the following format:
280     var_name<= or :>var_value
281     Comment lines (those beginning with a "#") and blank lines are allowed and will be ignored.  Leading and
282     trailing single or double quotes will be stripped from the value.  E.g.
283     var1="This one"
284     Quotes are stripped so the resulting value for var1 is:
285     This one
286 
287     Description of arguments:
288     prop_file_path                  The caller should pass the path to the properties file.
289     """
290 
291     # ConfigParser expects at least one section header in the file (or you get
292     # ConfigParser.MissingSectionHeaderError).  Properties files don't need those so I'll write a dummy
293     # section header.
294 
295     try:
296         string_file = StringIO.StringIO()
297     except NameError:
298         string_file = io.StringIO()
299 
300     # Write the dummy section header to the string file.
301     string_file.write("[dummysection]\n")
302     # Write the entire contents of the properties file to the string file.
303     string_file.write(open(prop_file_path).read())
304     # Rewind the string file.
305     string_file.seek(0, os.SEEK_SET)
306 
307     # Create the ConfigParser object.
308     try:
309         config_parser = ConfigParser.ConfigParser()
310     except NameError:
311         config_parser = configparser.ConfigParser(strict=False)
312     # Make the property names case-sensitive.
313     config_parser.optionxform = str
314     # Read the properties from the string file.
315     config_parser.readfp(string_file)
316     # Return the properties as a dictionary.
317     if robot_env:
318         return DotDict(config_parser.items("dummysection"))
319     else:
320         return collections.OrderedDict(config_parser.items("dummysection"))
321 
322 
323 def file_to_list(file_path, newlines=0, comments=1, trim=0):
324     r"""
325     Return the contents of a file as a list.  Each element of the resulting
326     list is one line from the file.
327 
328     Description of arguments:
329     file_path                       The path to the file (relative or absolute).
330     newlines                        Include newlines from the file in the results.
331     comments                        Include comment lines and blank lines in the results.  Comment lines are
332                                     any that begin with 0 or more spaces followed by the pound sign ("#").
333     trim                            Trim white space from the beginning and end of each line.
334     """
335 
336     lines = []
337     file = open(file_path)
338     for line in file:
339         if not comments:
340             if re.match(r"[ ]*#|^$", line):
341                 continue
342         if not newlines:
343             line = line.rstrip("\n")
344         if trim:
345             line = line.strip()
346         lines.append(line)
347     file.close()
348 
349     return lines
350 
351 
352 def file_to_str(*args, **kwargs):
353     r"""
354     Return the contents of a file as a string.
355 
356     Description of arguments:
357     See file_to_list defined above for description of arguments.
358     """
359 
360     return "\n".join(file_to_list(*args, **kwargs))
361 
362 
363 def append_file(file_path, buffer):
364     r"""
365     Append the data in buffer to the file named in file_path.
366 
367     Description of argument(s):
368     file_path                       The path to a file (e.g. "/tmp/root/file1").
369     buffer                          The buffer of data to be written to the file (e.g. "this and that").
370     """
371 
372     with open(file_path, "a") as file:
373         file.write(buffer)
374 
375 
376 def return_path_list():
377     r"""
378     This function will split the PATH environment variable into a PATH_LIST and return it.  Each element in
379     the list will be normalized and have a trailing slash added.
380     """
381 
382     PATH_LIST = os.environ["PATH"].split(":")
383     PATH_LIST = [os.path.normpath(path) + os.sep for path in PATH_LIST]
384 
385     return PATH_LIST
386 
387 
388 def escape_bash_quotes(buffer):
389     r"""
390     Escape quotes in string and return it.
391 
392     The escape style implemented will be for use on the bash command line.
393 
394     Example:
395     That's all.
396 
397     Result:
398     That'\''s all.
399 
400     The result may then be single quoted on a bash command.  Example:
401 
402     echo 'That'\''s all.'
403 
404     Description of argument(s):
405     buffer                          The string whose quotes are to be escaped.
406     """
407 
408     return re.sub("'", "'\\''", buffer)
409 
410 
411 def quote_bash_parm(parm):
412     r"""
413     Return the bash command line parm with single quotes if they are needed.
414 
415     Description of arguments:
416     parm                            The string to be quoted.
417     """
418 
419     # If any of these characters are found in the parm string, then the string should be quoted.  This list
420     # is by no means complete and should be expanded as needed by the developer of this function.
421     # Spaces
422     # Single or double quotes.
423     # Bash variables (therefore, any string with a "$" may need quoting).
424     # Glob characters: *, ?, []
425     # Extended Glob characters: +, @, !
426     # Bash brace expansion: {}
427     # Tilde expansion: ~
428     # Piped commands: |
429     # Bash re-direction: >, <
430     bash_special_chars = set(" '\"$*?[]+@!{}~|><")
431 
432     if any((char in bash_special_chars) for char in parm):
433         return "'" + escape_bash_quotes(parm) + "'"
434 
435     if parm == "":
436         parm = "''"
437 
438     return parm
439 
440 
441 def get_host_name_ip(host=None, short_name=0):
442     r"""
443     Get the host name and the IP address for the given host and return them as a tuple.
444 
445     Description of argument(s):
446     host                            The host name or IP address to be obtained.
447     short_name                      Include the short host name in the returned tuple, i.e. return host, ip
448                                     and short_host.
449     """
450 
451     host = dft(host, socket.gethostname())
452     host_name = socket.getfqdn(host)
453     try:
454         host_ip = socket.gethostbyname(host)
455     except socket.gaierror as my_gaierror:
456         message = (
457             "Unable to obtain the host name for the following host:"
458             + "\n"
459             + gp.sprint_var(host)
460         )
461         gp.print_error_report(message)
462         raise my_gaierror
463 
464     if short_name:
465         host_short_name = host_name.split(".")[0]
466         return host_name, host_ip, host_short_name
467     else:
468         return host_name, host_ip
469 
470 
471 def pid_active(pid):
472     r"""
473     Return true if pid represents an active pid and false otherwise.
474 
475     Description of argument(s):
476     pid                             The pid whose status is being sought.
477     """
478 
479     try:
480         os.kill(int(pid), 0)
481     except OSError as err:
482         if err.errno == errno.ESRCH:
483             # ESRCH == No such process
484             return False
485         elif err.errno == errno.EPERM:
486             # EPERM clearly means there's a process to deny access to
487             return True
488         else:
489             # According to "man 2 kill" possible error values are
490             # (EINVAL, EPERM, ESRCH)
491             raise
492 
493     return True
494 
495 
496 def to_signed(number, bit_width=None):
497     r"""
498     Convert number to a signed number and return the result.
499 
500     Examples:
501 
502     With the following code:
503 
504     var1 = 0xfffffffffffffff1
505     print_var(var1)
506     print_var(var1, hexa())
507     var1 = to_signed(var1)
508     print_var(var1)
509     print_var(var1, hexa())
510 
511     The following is written to stdout:
512     var1:  18446744073709551601
513     var1:  0x00000000fffffffffffffff1
514     var1:  -15
515     var1:  0xfffffffffffffff1
516 
517     The same code but with var1 set to 0x000000000000007f produces the following:
518     var1:  127
519     var1:  0x000000000000007f
520     var1:  127
521     var1:  0x000000000000007f
522 
523     Description of argument(s):
524     number                          The number to be converted.
525     bit_width                       The number of bits that defines a complete hex value.  Typically, this
526                                     would be a multiple of 32.
527     """
528 
529     if bit_width is None:
530         try:
531             bit_width = gp.bit_length(long(sys.maxsize)) + 1
532         except NameError:
533             bit_width = gp.bit_length(int(sys.maxsize)) + 1
534 
535     if number < 0:
536         return number
537     neg_bit_mask = 2 ** (bit_width - 1)
538     if number & neg_bit_mask:
539         return ((2**bit_width) - number) * -1
540     else:
541         return number
542 
543 
544 def get_child_pids(quiet=1):
545     r"""
546     Get and return a list of pids representing all first-generation processes that are the children of the
547     current process.
548 
549     Example:
550 
551     children = get_child_pids()
552     print_var(children)
553 
554     Output:
555     children:
556       children[0]:           9123
557 
558     Description of argument(s):
559     quiet                           Display output to stdout detailing how this child pids are obtained.
560     """
561 
562     if psutil_imported:
563         # If "import psutil" worked, find child pids using psutil.
564         current_process = psutil.Process()
565         return [x.pid for x in current_process.children(recursive=False)]
566     else:
567         # Otherwise, find child pids using shell commands.
568         print_output = not quiet
569 
570         ps_cmd_buf = (
571             "ps --no-headers --ppid " + str(os.getpid()) + " -o pid,args"
572         )
573         # Route the output of ps to a temporary file for later grepping.  Avoid using " | grep" in the ps
574         # command string because it creates yet another process which is of no interest to the caller.
575         temp = tempfile.NamedTemporaryFile()
576         temp_file_path = temp.name
577         gc.shell_cmd(
578             ps_cmd_buf + " > " + temp_file_path, print_output=print_output
579         )
580         # Sample contents of the temporary file:
581         # 30703 sleep 2
582         # 30795 /bin/bash -c ps --no-headers --ppid 30672 -o pid,args > /tmp/tmpqqorWY
583         # Use egrep to exclude the "ps" process itself from the results collected with the prior shell_cmd
584         # invocation.  Only the other children are of interest to the caller.  Use cut on the grep results to
585         # obtain only the pid column.
586         rc, output = gc.shell_cmd(
587             "egrep -v '"
588             + re.escape(ps_cmd_buf)
589             + "' "
590             + temp_file_path
591             + " | cut -c1-5",
592             print_output=print_output,
593         )
594         # Split the output buffer by line into a list.  Strip each element of extra spaces and convert each
595         # element to an integer.
596         return map(int, map(str.strip, filter(None, output.split("\n"))))
597 
598 
599 def json_loads_multiple(buffer):
600     r"""
601     Convert the contents of the buffer to a JSON array, run json.loads() on it and return the result.
602 
603     The buffer is expected to contain one or more JSON objects.
604 
605     Description of argument(s):
606     buffer                          A string containing several JSON objects.
607     """
608 
609     # Any line consisting of just "}", which indicates the end of an object, should have a comma appended.
610     regex = "([\\r\\n])[\\}]([\\r\\n])"
611     buffer = re.sub(regex, "\\1},\\2", buffer, 1)
612     # Remove the comma from after the final object and place the whole buffer inside square brackets.
613     buffer = "[" + re.sub(",([\r\n])$", "\\1}", buffer, 1) + "]"
614     if gp.robot_env:
615         return json.loads(buffer, object_pairs_hook=DotDict)
616     else:
617         return json.loads(buffer, object_pairs_hook=collections.OrderedDict)
618 
619 
620 def file_date_time_stamp():
621     r"""
622     Return a date/time stamp in the following format: yymmdd.HHMMSS
623 
624     This value is suitable for including in file names.  Example file1.181001.171716.status
625     """
626 
627     return time.strftime("%y%m%d.%H%M%S", time.localtime(time.time()))
628 
629 
630 def get_function_stack():
631     r"""
632     Return a list of all the function names currently in the call stack.
633 
634     This function's name will be at offset 0.  This function's caller's name will be at offset 1 and so on.
635     """
636 
637     return [str(stack_frame[3]) for stack_frame in inspect.stack()]
638 
639 
640 def username():
641     r"""
642     Return the username for the current process.
643     """
644 
645     username = os.environ.get("USER", "")
646     if username != "":
647         return username
648     user_num = str(os.geteuid())
649     try:
650         username = os.getlogin()
651     except OSError:
652         if user_num == "0":
653             username = "root"
654         else:
655             username = "?"
656 
657     return username
658 
659 
660 def version_tuple(version):
661     r"""
662     Convert the version string to a tuple and return it.
663 
664     Description of argument(s):
665     version                         A version string whose format is "n[.n]" (e.g. "3.6.3", "3", etc.).
666     """
667 
668     return tuple(map(int, (version.split("."))))
669 
670 
671 def get_python_version():
672     r"""
673     Get and return the python version.
674     """
675 
676     sys_version = sys.version
677     # Strip out any revision code data (e.g. "3.6.3rc1" will become "3.6.3").
678     sys_version = re.sub("rc[^ ]+", "", sys_version).split(" ")[0]
679     # Remove any non-numerics, etc. (e.g. "2.7.15+" becomes ""2.7.15").
680     return re.sub("[^0-9\\.]", "", sys_version)
681 
682 
683 python_version = version_tuple(get_python_version())
684 ordered_dict_version = version_tuple("3.6")
685 
686 
687 def create_temp_file_path(delim=":", suffix=""):
688     r"""
689     Create a temporary file path and return it.
690 
691     This function is appropriate for users who with to create a temporary file and:
692     1) Have control over when and whether the file is deleted.
693     2) Have the name of the file indicate information such as program name, function name, line, pid, etc.
694     This can be an aid in debugging, cleanup, etc.
695 
696     The dir path portion of the file path will be /tmp/<username>/.  This function will create this directory
697     if it doesn't already exist.
698 
699     This function will NOT create the file.  The file will NOT automatically get deleted.  It is the
700     responsibility of the caller to dispose of it.
701 
702     Example:
703 
704     pgm123.py is run by user 'joe'.  It calls func1 which contains this code:
705 
706     temp_file_path = create_temp_file_path(suffix='suffix1')
707     print_var(temp_file_path)
708 
709     Output:
710 
711     temp_file_path:                 /tmp/joe/pgm123.py:func1:line_55:pid_8199:831848:suffix1
712 
713     Description of argument(s):
714     delim                           A delimiter to be used to separate the sub-components of the file name.
715     suffix                          A suffix to include as the last sub-component of the file name.
716     """
717 
718     temp_dir_path = "/tmp/" + username() + "/"
719     try:
720         os.mkdir(temp_dir_path)
721     except FileExistsError:
722         pass
723 
724     callers_stack_frame = inspect.stack()[1]
725     file_name_elements = [
726         gp.pgm_name,
727         callers_stack_frame.function,
728         "line_" + str(callers_stack_frame.lineno),
729         "pid_" + str(os.getpid()),
730         str(random.randint(0, 1000000)),
731         suffix,
732     ]
733     temp_file_name = delim.join(file_name_elements)
734 
735     temp_file_path = temp_dir_path + temp_file_name
736 
737     return temp_file_path
738 
739 
740 def pause(message="Hit enter to continue..."):
741     r"""
742     Print the message, with time stamp, and pause until the user hits enter.
743 
744     Description of argument(s):
745     message                         The message to be printed to stdout.
746     """
747     gp.print_time(message)
748     try:
749         input()
750     except SyntaxError:
751         pass
752 
753     return
754