1#!/usr/bin/env python3
2
3r"""
4This module provides many valuable functions such as my_parm_file.
5"""
6
7import collections
8import errno
9import inspect
10import json
11import os
12import random
13import shutil
14
15# sys and os are needed to get the program dir path and program name.
16import sys
17import time
18
19try:
20    import ConfigParser
21except ImportError:
22    import configparser
23try:
24    import StringIO
25except ImportError:
26    import io
27
28import re
29import socket
30import tempfile
31
32try:
33    import psutil
34
35    psutil_imported = True
36except ImportError:
37    psutil_imported = False
38
39import gen_cmd as gc
40import gen_print as gp
41
42robot_env = gp.robot_env
43if robot_env:
44    from robot.libraries.BuiltIn import BuiltIn
45    from robot.utils import DotDict
46
47
48def add_trailing_slash(dir_path):
49    r"""
50    Add a trailing slash to the directory path if it doesn't already have one
51    and return it.
52
53    Description of arguments:
54    dir_path                        A directory path.
55    """
56
57    return os.path.normpath(dir_path) + os.path.sep
58
59
60def makedirs(path, mode=0o777, quiet=None):
61    r"""
62    Call os.makedirs with the caller's arguments.
63
64    This function offers 2 advantages over the base os.makedirs function:
65    1) It will not fail if the directory already exists.
66    2) It will print an "Issuing: os.makedirs" message.
67
68    Description of argument(s):
69    path                            The path containing the directories to be created.
70    mode                            The mode or permissions to be granted to the created directories.
71    quiet                           Indicates whether this function should run the print_issuing() function.
72    """
73    quiet = int(dft(quiet, gp.get_stack_var("quiet", 0)))
74    gp.qprint_issuing("os.makedirs('" + path + "', mode=" + oct(mode) + ")")
75    try:
76        os.makedirs(path, mode)
77    except OSError:
78        pass
79
80
81def rmtree(path, ignore_errors=False, onerror=None, quiet=None):
82    r"""
83    Call shutil.rmtree with the caller's arguments.
84
85    This function offers this advantage over the base function:
86    - It will print an "Issuing: shutil.rmtree" message.
87
88    Description of argument(s):
89    (All parms are passed directly to shutil.rmtree.  See its prolog for details)
90    quiet                           Indicates whether this function should run the print_issuing() function.
91    """
92    quiet = int(dft(quiet, gp.get_stack_var("quiet", 0)))
93    print_string = gp.sprint_executing(max_width=2000)
94    print_string = re.sub(
95        r"Executing: ", "Issuing: shutil.", print_string.rstrip("\n")
96    )
97    gp.qprintn(re.sub(r", quiet[ ]?=.*", ")", print_string))
98    shutil.rmtree(path, ignore_errors, onerror)
99
100
101def chdir(path, quiet=None):
102    r"""
103    Call os.chdir with the caller's arguments.
104
105    This function offers this advantage over the base os.chdir function:
106    - It will print an "Issuing: os.chdir" message.
107
108    Description of argument(s):
109    path                            The path of the directory to change to.
110    quiet                           Indicates whether this function should run the print_issuing() function.
111    """
112    quiet = int(dft(quiet, gp.get_stack_var("quiet", 0)))
113    gp.qprint_issuing("os.chdir('" + path + "')")
114    os.chdir(path)
115
116
117def which(file_path):
118    r"""
119    Find the full path of an executable file and return it.
120
121    The PATH environment variable dictates the results of this function.
122
123    Description of arguments:
124    file_path                       The relative file path (e.g. "my_file" or "lib/my_file").
125    """
126
127    shell_rc, out_buf = gc.cmd_fnc_u(
128        "which " + file_path, quiet=1, print_output=0, show_err=0
129    )
130    if shell_rc != 0:
131        error_message = (
132            'Failed to find complete path for file "' + file_path + '".\n'
133        )
134        error_message += gp.sprint_var(shell_rc, gp.hexa())
135        error_message += out_buf
136        if robot_env:
137            BuiltIn().fail(gp.sprint_error(error_message))
138        else:
139            gp.print_error_report(error_message)
140            return False
141
142    file_path = out_buf.rstrip("\n")
143
144    return file_path
145
146
147def add_path(new_path, path, position=0):
148    r"""
149    Add new_path to path, provided that path doesn't already contain new_path, and return the result.
150
151    Example:
152    If PATH has a value of "/bin/user:/lib/user".  The following code:
153
154    PATH = add_path("/tmp/new_path", PATH)
155
156    will change PATH to "/tmp/new_path:/bin/user:/lib/user".
157
158    Description of argument(s):
159    new_path                        The path to be added.  This function will strip the trailing slash.
160    path                            The path value to which the new_path should be added.
161    position                        The position in path where the new_path should be added.  0 means it
162                                    should be added to the beginning, 1 means add it as the 2nd item, etc.
163                                    sys.maxsize means it should be added to the end.
164    """
165
166    path_list = list(filter(None, path.split(":")))
167    new_path = new_path.rstrip("/")
168    if new_path not in path_list:
169        path_list.insert(int(position), new_path)
170    return ":".join(path_list)
171
172
173def dft(value, default):
174    r"""
175    Return default if value is None.  Otherwise, return value.
176
177    This is really just shorthand as shown below.
178
179    dft(value, default)
180
181    vs
182
183    default if value is None else value
184
185    Description of arguments:
186    value                           The value to be returned.
187    default                         The default value to return if value is None.
188    """
189
190    return default if value is None else value
191
192
193def get_mod_global(var_name, default=None, mod_name="__main__"):
194    r"""
195    Get module global variable value and return it.
196
197    If we are running in a robot environment, the behavior will default to
198    calling get_variable_value.
199
200    Description of arguments:
201    var_name                        The name of the variable whose value is sought.
202    default                         The value to return if the global does not exist.
203    mod_name                        The name of the module containing the global variable.
204    """
205
206    if robot_env:
207        return BuiltIn().get_variable_value("${" + var_name + "}", default)
208
209    try:
210        module = sys.modules[mod_name]
211    except KeyError:
212        gp.print_error_report(
213            "Programmer error - The mod_name passed to"
214            + " this function is invalid:\n"
215            + gp.sprint_var(mod_name)
216        )
217        raise ValueError("Programmer error.")
218
219    if default is None:
220        return getattr(module, var_name)
221    else:
222        return getattr(module, var_name, default)
223
224
225def global_default(var_value, default=0):
226    r"""
227    If var_value is not None, return it.  Otherwise, return the global
228    variable of the same name, if it exists.  If not, return default.
229
230    This is meant for use by functions needing help assigning dynamic default
231    values to their parms.  Example:
232
233    def func1(parm1=None):
234
235        parm1 = global_default(parm1, 0)
236
237    Description of arguments:
238    var_value                       The value being evaluated.
239    default                         The value to be returned if var_value is None AND the global variable of
240                                    the same name does not exist.
241    """
242
243    var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
244
245    return dft(var_value, get_mod_global(var_name, 0))
246
247
248def set_mod_global(var_value, mod_name="__main__", var_name=None):
249    r"""
250    Set a global variable for a given module.
251
252    Description of arguments:
253    var_value                       The value to set in the variable.
254    mod_name                        The name of the module whose variable is to be set.
255    var_name                        The name of the variable to set.  This defaults to the name of the
256                                    variable used for var_value when calling this function.
257    """
258
259    try:
260        module = sys.modules[mod_name]
261    except KeyError:
262        gp.print_error_report(
263            "Programmer error - The mod_name passed to"
264            + " this function is invalid:\n"
265            + gp.sprint_var(mod_name)
266        )
267        raise ValueError("Programmer error.")
268
269    if var_name is None:
270        var_name = gp.get_arg_name(None, 1, 2)
271
272    setattr(module, var_name, var_value)
273
274
275def my_parm_file(prop_file_path):
276    r"""
277    Read a properties file, put the keys/values into a dictionary and return the dictionary.
278
279    The properties file must have the following format:
280    var_name<= or :>var_value
281    Comment lines (those beginning with a "#") and blank lines are allowed and will be ignored.  Leading and
282    trailing single or double quotes will be stripped from the value.  E.g.
283    var1="This one"
284    Quotes are stripped so the resulting value for var1 is:
285    This one
286
287    Description of arguments:
288    prop_file_path                  The caller should pass the path to the properties file.
289    """
290
291    # ConfigParser expects at least one section header in the file (or you get
292    # ConfigParser.MissingSectionHeaderError).  Properties files don't need those so I'll write a dummy
293    # section header.
294
295    try:
296        string_file = StringIO.StringIO()
297    except NameError:
298        string_file = io.StringIO()
299
300    # Write the dummy section header to the string file.
301    string_file.write("[dummysection]\n")
302    # Write the entire contents of the properties file to the string file.
303    string_file.write(open(prop_file_path).read())
304    # Rewind the string file.
305    string_file.seek(0, os.SEEK_SET)
306
307    # Create the ConfigParser object.
308    try:
309        config_parser = ConfigParser.ConfigParser()
310    except NameError:
311        config_parser = configparser.ConfigParser(strict=False)
312    # Make the property names case-sensitive.
313    config_parser.optionxform = str
314    # Read the properties from the string file.
315    config_parser.readfp(string_file)
316    # Return the properties as a dictionary.
317    if robot_env:
318        return DotDict(config_parser.items("dummysection"))
319    else:
320        return collections.OrderedDict(config_parser.items("dummysection"))
321
322
323def file_to_list(file_path, newlines=0, comments=1, trim=0):
324    r"""
325    Return the contents of a file as a list.  Each element of the resulting
326    list is one line from the file.
327
328    Description of arguments:
329    file_path                       The path to the file (relative or absolute).
330    newlines                        Include newlines from the file in the results.
331    comments                        Include comment lines and blank lines in the results.  Comment lines are
332                                    any that begin with 0 or more spaces followed by the pound sign ("#").
333    trim                            Trim white space from the beginning and end of each line.
334    """
335
336    lines = []
337    file = open(file_path)
338    for line in file:
339        if not comments:
340            if re.match(r"[ ]*#|^$", line):
341                continue
342        if not newlines:
343            line = line.rstrip("\n")
344        if trim:
345            line = line.strip()
346        lines.append(line)
347    file.close()
348
349    return lines
350
351
352def file_to_str(*args, **kwargs):
353    r"""
354    Return the contents of a file as a string.
355
356    Description of arguments:
357    See file_to_list defined above for description of arguments.
358    """
359
360    return "\n".join(file_to_list(*args, **kwargs))
361
362
363def append_file(file_path, buffer):
364    r"""
365    Append the data in buffer to the file named in file_path.
366
367    Description of argument(s):
368    file_path                       The path to a file (e.g. "/tmp/root/file1").
369    buffer                          The buffer of data to be written to the file (e.g. "this and that").
370    """
371
372    with open(file_path, "a") as file:
373        file.write(buffer)
374
375
376def return_path_list():
377    r"""
378    This function will split the PATH environment variable into a PATH_LIST and return it.  Each element in
379    the list will be normalized and have a trailing slash added.
380    """
381
382    PATH_LIST = os.environ["PATH"].split(":")
383    PATH_LIST = [os.path.normpath(path) + os.sep for path in PATH_LIST]
384
385    return PATH_LIST
386
387
388def escape_bash_quotes(buffer):
389    r"""
390    Escape quotes in string and return it.
391
392    The escape style implemented will be for use on the bash command line.
393
394    Example:
395    That's all.
396
397    Result:
398    That'\''s all.
399
400    The result may then be single quoted on a bash command.  Example:
401
402    echo 'That'\''s all.'
403
404    Description of argument(s):
405    buffer                          The string whose quotes are to be escaped.
406    """
407
408    return re.sub("'", "'\\''", buffer)
409
410
411def quote_bash_parm(parm):
412    r"""
413    Return the bash command line parm with single quotes if they are needed.
414
415    Description of arguments:
416    parm                            The string to be quoted.
417    """
418
419    # If any of these characters are found in the parm string, then the string should be quoted.  This list
420    # is by no means complete and should be expanded as needed by the developer of this function.
421    # Spaces
422    # Single or double quotes.
423    # Bash variables (therefore, any string with a "$" may need quoting).
424    # Glob characters: *, ?, []
425    # Extended Glob characters: +, @, !
426    # Bash brace expansion: {}
427    # Tilde expansion: ~
428    # Piped commands: |
429    # Bash re-direction: >, <
430    bash_special_chars = set(" '\"$*?[]+@!{}~|><")
431
432    if any((char in bash_special_chars) for char in parm):
433        return "'" + escape_bash_quotes(parm) + "'"
434
435    if parm == "":
436        parm = "''"
437
438    return parm
439
440
441def get_host_name_ip(host=None, short_name=0):
442    r"""
443    Get the host name and the IP address for the given host and return them as a tuple.
444
445    Description of argument(s):
446    host                            The host name or IP address to be obtained.
447    short_name                      Include the short host name in the returned tuple, i.e. return host, ip
448                                    and short_host.
449    """
450
451    host = dft(host, socket.gethostname())
452    host_name = socket.getfqdn(host)
453    try:
454        host_ip = socket.gethostbyname(host)
455    except socket.gaierror as my_gaierror:
456        message = (
457            "Unable to obtain the host name for the following host:"
458            + "\n"
459            + gp.sprint_var(host)
460        )
461        gp.print_error_report(message)
462        raise my_gaierror
463
464    if short_name:
465        host_short_name = host_name.split(".")[0]
466        return host_name, host_ip, host_short_name
467    else:
468        return host_name, host_ip
469
470
471def pid_active(pid):
472    r"""
473    Return true if pid represents an active pid and false otherwise.
474
475    Description of argument(s):
476    pid                             The pid whose status is being sought.
477    """
478
479    try:
480        os.kill(int(pid), 0)
481    except OSError as err:
482        if err.errno == errno.ESRCH:
483            # ESRCH == No such process
484            return False
485        elif err.errno == errno.EPERM:
486            # EPERM clearly means there's a process to deny access to
487            return True
488        else:
489            # According to "man 2 kill" possible error values are
490            # (EINVAL, EPERM, ESRCH)
491            raise
492
493    return True
494
495
496def to_signed(number, bit_width=None):
497    r"""
498    Convert number to a signed number and return the result.
499
500    Examples:
501
502    With the following code:
503
504    var1 = 0xfffffffffffffff1
505    print_var(var1)
506    print_var(var1, hexa())
507    var1 = to_signed(var1)
508    print_var(var1)
509    print_var(var1, hexa())
510
511    The following is written to stdout:
512    var1:  18446744073709551601
513    var1:  0x00000000fffffffffffffff1
514    var1:  -15
515    var1:  0xfffffffffffffff1
516
517    The same code but with var1 set to 0x000000000000007f produces the following:
518    var1:  127
519    var1:  0x000000000000007f
520    var1:  127
521    var1:  0x000000000000007f
522
523    Description of argument(s):
524    number                          The number to be converted.
525    bit_width                       The number of bits that defines a complete hex value.  Typically, this
526                                    would be a multiple of 32.
527    """
528
529    if bit_width is None:
530        try:
531            bit_width = gp.bit_length(long(sys.maxsize)) + 1
532        except NameError:
533            bit_width = gp.bit_length(int(sys.maxsize)) + 1
534
535    if number < 0:
536        return number
537    neg_bit_mask = 2 ** (bit_width - 1)
538    if number & neg_bit_mask:
539        return ((2**bit_width) - number) * -1
540    else:
541        return number
542
543
544def get_child_pids(quiet=1):
545    r"""
546    Get and return a list of pids representing all first-generation processes that are the children of the
547    current process.
548
549    Example:
550
551    children = get_child_pids()
552    print_var(children)
553
554    Output:
555    children:
556      children[0]:           9123
557
558    Description of argument(s):
559    quiet                           Display output to stdout detailing how this child pids are obtained.
560    """
561
562    if psutil_imported:
563        # If "import psutil" worked, find child pids using psutil.
564        current_process = psutil.Process()
565        return [x.pid for x in current_process.children(recursive=False)]
566    else:
567        # Otherwise, find child pids using shell commands.
568        print_output = not quiet
569
570        ps_cmd_buf = (
571            "ps --no-headers --ppid " + str(os.getpid()) + " -o pid,args"
572        )
573        # Route the output of ps to a temporary file for later grepping.  Avoid using " | grep" in the ps
574        # command string because it creates yet another process which is of no interest to the caller.
575        temp = tempfile.NamedTemporaryFile()
576        temp_file_path = temp.name
577        gc.shell_cmd(
578            ps_cmd_buf + " > " + temp_file_path, print_output=print_output
579        )
580        # Sample contents of the temporary file:
581        # 30703 sleep 2
582        # 30795 /bin/bash -c ps --no-headers --ppid 30672 -o pid,args > /tmp/tmpqqorWY
583        # Use egrep to exclude the "ps" process itself from the results collected with the prior shell_cmd
584        # invocation.  Only the other children are of interest to the caller.  Use cut on the grep results to
585        # obtain only the pid column.
586        rc, output = gc.shell_cmd(
587            "egrep -v '"
588            + re.escape(ps_cmd_buf)
589            + "' "
590            + temp_file_path
591            + " | cut -c1-5",
592            print_output=print_output,
593        )
594        # Split the output buffer by line into a list.  Strip each element of extra spaces and convert each
595        # element to an integer.
596        return map(int, map(str.strip, filter(None, output.split("\n"))))
597
598
599def json_loads_multiple(buffer):
600    r"""
601    Convert the contents of the buffer to a JSON array, run json.loads() on it and return the result.
602
603    The buffer is expected to contain one or more JSON objects.
604
605    Description of argument(s):
606    buffer                          A string containing several JSON objects.
607    """
608
609    # Any line consisting of just "}", which indicates the end of an object, should have a comma appended.
610    regex = "([\\r\\n])[\\}]([\\r\\n])"
611    buffer = re.sub(regex, "\\1},\\2", buffer, 1)
612    # Remove the comma from after the final object and place the whole buffer inside square brackets.
613    buffer = "[" + re.sub(",([\r\n])$", "\\1}", buffer, 1) + "]"
614    if gp.robot_env:
615        return json.loads(buffer, object_pairs_hook=DotDict)
616    else:
617        return json.loads(buffer, object_pairs_hook=collections.OrderedDict)
618
619
620def file_date_time_stamp():
621    r"""
622    Return a date/time stamp in the following format: yymmdd.HHMMSS
623
624    This value is suitable for including in file names.  Example file1.181001.171716.status
625    """
626
627    return time.strftime("%y%m%d.%H%M%S", time.localtime(time.time()))
628
629
630def get_function_stack():
631    r"""
632    Return a list of all the function names currently in the call stack.
633
634    This function's name will be at offset 0.  This function's caller's name will be at offset 1 and so on.
635    """
636
637    return [str(stack_frame[3]) for stack_frame in inspect.stack()]
638
639
640def username():
641    r"""
642    Return the username for the current process.
643    """
644
645    username = os.environ.get("USER", "")
646    if username != "":
647        return username
648    user_num = str(os.geteuid())
649    try:
650        username = os.getlogin()
651    except OSError:
652        if user_num == "0":
653            username = "root"
654        else:
655            username = "?"
656
657    return username
658
659
660def version_tuple(version):
661    r"""
662    Convert the version string to a tuple and return it.
663
664    Description of argument(s):
665    version                         A version string whose format is "n[.n]" (e.g. "3.6.3", "3", etc.).
666    """
667
668    return tuple(map(int, (version.split("."))))
669
670
671def get_python_version():
672    r"""
673    Get and return the python version.
674    """
675
676    sys_version = sys.version
677    # Strip out any revision code data (e.g. "3.6.3rc1" will become "3.6.3").
678    sys_version = re.sub("rc[^ ]+", "", sys_version).split(" ")[0]
679    # Remove any non-numerics, etc. (e.g. "2.7.15+" becomes ""2.7.15").
680    return re.sub("[^0-9\\.]", "", sys_version)
681
682
683python_version = version_tuple(get_python_version())
684ordered_dict_version = version_tuple("3.6")
685
686
687def create_temp_file_path(delim=":", suffix=""):
688    r"""
689    Create a temporary file path and return it.
690
691    This function is appropriate for users who with to create a temporary file and:
692    1) Have control over when and whether the file is deleted.
693    2) Have the name of the file indicate information such as program name, function name, line, pid, etc.
694    This can be an aid in debugging, cleanup, etc.
695
696    The dir path portion of the file path will be /tmp/<username>/.  This function will create this directory
697    if it doesn't already exist.
698
699    This function will NOT create the file.  The file will NOT automatically get deleted.  It is the
700    responsibility of the caller to dispose of it.
701
702    Example:
703
704    pgm123.py is run by user 'joe'.  It calls func1 which contains this code:
705
706    temp_file_path = create_temp_file_path(suffix='suffix1')
707    print_var(temp_file_path)
708
709    Output:
710
711    temp_file_path:                 /tmp/joe/pgm123.py:func1:line_55:pid_8199:831848:suffix1
712
713    Description of argument(s):
714    delim                           A delimiter to be used to separate the sub-components of the file name.
715    suffix                          A suffix to include as the last sub-component of the file name.
716    """
717
718    temp_dir_path = "/tmp/" + username() + "/"
719    try:
720        os.mkdir(temp_dir_path)
721    except FileExistsError:
722        pass
723
724    callers_stack_frame = inspect.stack()[1]
725    file_name_elements = [
726        gp.pgm_name,
727        callers_stack_frame.function,
728        "line_" + str(callers_stack_frame.lineno),
729        "pid_" + str(os.getpid()),
730        str(random.randint(0, 1000000)),
731        suffix,
732    ]
733    temp_file_name = delim.join(file_name_elements)
734
735    temp_file_path = temp_dir_path + temp_file_name
736
737    return temp_file_path
738
739
740def pause(message="Hit enter to continue..."):
741    r"""
742    Print the message, with time stamp, and pause until the user hits enter.
743
744    Description of argument(s):
745    message                         The message to be printed to stdout.
746    """
747    gp.print_time(message)
748    try:
749        input()
750    except SyntaxError:
751        pass
752
753    return
754