1#!/usr/bin/env python
2
3r"""
4This module provides many valuable functions such as my_parm_file.
5"""
6
7# sys and os are needed to get the program dir path and program name.
8import sys
9import errno
10import os
11import collections
12import json
13import time
14import inspect
15import random
16try:
17    import ConfigParser
18except ImportError:
19    import configparser
20try:
21    import StringIO
22except ImportError:
23    import io
24import re
25import socket
26import tempfile
27try:
28    import psutil
29    psutil_imported = True
30except ImportError:
31    psutil_imported = False
32
33import gen_print as gp
34import gen_cmd as gc
35
36robot_env = gp.robot_env
37if robot_env:
38    from robot.libraries.BuiltIn import BuiltIn
39    from robot.utils import DotDict
40
41
42def add_trailing_slash(dir_path):
43    r"""
44    Add a trailing slash to the directory path if it doesn't already have one
45    and return it.
46
47    Description of arguments:
48    dir_path                        A directory path.
49    """
50
51    return os.path.normpath(dir_path) + os.path.sep
52
53
54def which(file_path):
55    r"""
56    Find the full path of an executable file and return it.
57
58    The PATH environment variable dictates the results of this function.
59
60    Description of arguments:
61    file_path                       The relative file path (e.g. "my_file" or "lib/my_file").
62    """
63
64    shell_rc, out_buf = gc.cmd_fnc_u("which " + file_path, quiet=1,
65                                     print_output=0, show_err=0)
66    if shell_rc != 0:
67        error_message = "Failed to find complete path for file \"" +\
68                        file_path + "\".\n"
69        error_message += gp.sprint_var(shell_rc, gp.hexa())
70        error_message += out_buf
71        if robot_env:
72            BuiltIn().fail(gp.sprint_error(error_message))
73        else:
74            gp.print_error_report(error_message)
75            return False
76
77    file_path = out_buf.rstrip("\n")
78
79    return file_path
80
81
82def add_path(new_path,
83             path,
84             position=0):
85    r"""
86    Add new_path to path, provided that path doesn't already contain new_path, and return the result.
87
88    Example:
89    If PATH has a value of "/bin/user:/lib/user".  The following code:
90
91    PATH = add_path("/tmp/new_path", PATH)
92
93    will change PATH to "/tmp/new_path:/bin/user:/lib/user".
94
95    Description of argument(s):
96    new_path                        The path to be added.  This function will strip the trailing slash.
97    path                            The path value to which the new_path should be added.
98    position                        The position in path where the new_path should be added.  0 means it
99                                    should be added to the beginning, 1 means add it as the 2nd item, etc.
100                                    sys.maxsize means it should be added to the end.
101    """
102
103    path_list = list(filter(None, path.split(":")))
104    new_path = new_path.rstrip("/")
105    if new_path not in path_list:
106        path_list.insert(int(position), new_path)
107    return ":".join(path_list)
108
109
110def dft(value, default):
111    r"""
112    Return default if value is None.  Otherwise, return value.
113
114    This is really just shorthand as shown below.
115
116    dft(value, default)
117
118    vs
119
120    default if value is None else value
121
122    Description of arguments:
123    value                           The value to be returned.
124    default                         The default value to return if value is None.
125    """
126
127    return default if value is None else value
128
129
130def get_mod_global(var_name,
131                   default=None,
132                   mod_name="__main__"):
133    r"""
134    Get module global variable value and return it.
135
136    If we are running in a robot environment, the behavior will default to
137    calling get_variable_value.
138
139    Description of arguments:
140    var_name                        The name of the variable whose value is sought.
141    default                         The value to return if the global does not exist.
142    mod_name                        The name of the module containing the global variable.
143    """
144
145    if robot_env:
146        return BuiltIn().get_variable_value("${" + var_name + "}", default)
147
148    try:
149        module = sys.modules[mod_name]
150    except KeyError:
151        gp.print_error_report("Programmer error - The mod_name passed to"
152                              + " this function is invalid:\n"
153                              + gp.sprint_var(mod_name))
154        raise ValueError('Programmer error.')
155
156    if default is None:
157        return getattr(module, var_name)
158    else:
159        return getattr(module, var_name, default)
160
161
162def global_default(var_value,
163                   default=0):
164    r"""
165    If var_value is not None, return it.  Otherwise, return the global
166    variable of the same name, if it exists.  If not, return default.
167
168    This is meant for use by functions needing help assigning dynamic default
169    values to their parms.  Example:
170
171    def func1(parm1=None):
172
173        parm1 = global_default(parm1, 0)
174
175    Description of arguments:
176    var_value                       The value being evaluated.
177    default                         The value to be returned if var_value is None AND the global variable of
178                                    the same name does not exist.
179    """
180
181    var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
182
183    return dft(var_value, get_mod_global(var_name, 0))
184
185
186def set_mod_global(var_value,
187                   mod_name="__main__",
188                   var_name=None):
189    r"""
190    Set a global variable for a given module.
191
192    Description of arguments:
193    var_value                       The value to set in the variable.
194    mod_name                        The name of the module whose variable is to be set.
195    var_name                        The name of the variable to set.  This defaults to the name of the
196                                    variable used for var_value when calling this function.
197    """
198
199    try:
200        module = sys.modules[mod_name]
201    except KeyError:
202        gp.print_error_report("Programmer error - The mod_name passed to"
203                              + " this function is invalid:\n"
204                              + gp.sprint_var(mod_name))
205        raise ValueError('Programmer error.')
206
207    if var_name is None:
208        var_name = gp.get_arg_name(None, 1, 2)
209
210    setattr(module, var_name, var_value)
211
212
213def my_parm_file(prop_file_path):
214    r"""
215    Read a properties file, put the keys/values into a dictionary and return the dictionary.
216
217    The properties file must have the following format:
218    var_name<= or :>var_value
219    Comment lines (those beginning with a "#") and blank lines are allowed and will be ignored.  Leading and
220    trailing single or double quotes will be stripped from the value.  E.g.
221    var1="This one"
222    Quotes are stripped so the resulting value for var1 is:
223    This one
224
225    Description of arguments:
226    prop_file_path                  The caller should pass the path to the properties file.
227    """
228
229    # ConfigParser expects at least one section header in the file (or you get
230    # ConfigParser.MissingSectionHeaderError).  Properties files don't need those so I'll write a dummy
231    # section header.
232
233    try:
234        string_file = StringIO.StringIO()
235    except NameError:
236        string_file = io.StringIO()
237
238    # Write the dummy section header to the string file.
239    string_file.write('[dummysection]\n')
240    # Write the entire contents of the properties file to the string file.
241    string_file.write(open(prop_file_path).read())
242    # Rewind the string file.
243    string_file.seek(0, os.SEEK_SET)
244
245    # Create the ConfigParser object.
246    try:
247        config_parser = ConfigParser.ConfigParser()
248    except NameError:
249        config_parser = configparser.ConfigParser(strict=False)
250    # Make the property names case-sensitive.
251    config_parser.optionxform = str
252    # Read the properties from the string file.
253    config_parser.readfp(string_file)
254    # Return the properties as a dictionary.
255    if robot_env:
256        return DotDict(config_parser.items('dummysection'))
257    else:
258        return collections.OrderedDict(config_parser.items('dummysection'))
259
260
261def file_to_list(file_path,
262                 newlines=0,
263                 comments=1,
264                 trim=0):
265    r"""
266    Return the contents of a file as a list.  Each element of the resulting
267    list is one line from the file.
268
269    Description of arguments:
270    file_path                       The path to the file (relative or absolute).
271    newlines                        Include newlines from the file in the results.
272    comments                        Include comment lines and blank lines in the results.  Comment lines are
273                                    any that begin with 0 or more spaces followed by the pound sign ("#").
274    trim                            Trim white space from the beginning and end of each line.
275    """
276
277    lines = []
278    file = open(file_path)
279    for line in file:
280        if not comments:
281            if re.match(r"[ ]*#|^$", line):
282                continue
283        if not newlines:
284            line = line.rstrip("\n")
285        if trim:
286            line = line.strip()
287        lines.append(line)
288    file.close()
289
290    return lines
291
292
293def file_to_str(*args, **kwargs):
294    r"""
295    Return the contents of a file as a string.
296
297    Description of arguments:
298    See file_to_list defined above for description of arguments.
299    """
300
301    return '\n'.join(file_to_list(*args, **kwargs))
302
303
304def append_file(file_path, buffer):
305    r"""
306    Append the data in buffer to the file named in file_path.
307
308    Description of argument(s):
309    file_path                       The path to a file (e.g. "/tmp/root/file1").
310    buffer                          The buffer of data to be written to the file (e.g. "this and that").
311    """
312
313    with open(file_path, "a") as file:
314        file.write(buffer)
315
316
317def return_path_list():
318    r"""
319    This function will split the PATH environment variable into a PATH_LIST and return it.  Each element in
320    the list will be normalized and have a trailing slash added.
321    """
322
323    PATH_LIST = os.environ['PATH'].split(":")
324    PATH_LIST = [os.path.normpath(path) + os.sep for path in PATH_LIST]
325
326    return PATH_LIST
327
328
329def escape_bash_quotes(buffer):
330    r"""
331    Escape quotes in string and return it.
332
333    The escape style implemented will be for use on the bash command line.
334
335    Example:
336    That's all.
337
338    Result:
339    That'\''s all.
340
341    The result may then be single quoted on a bash command.  Example:
342
343    echo 'That'\''s all.'
344
345    Description of argument(s):
346    buffer                          The string whose quotes are to be escaped.
347    """
348
349    return re.sub("\'", "\'\\\'\'", buffer)
350
351
352def quote_bash_parm(parm):
353    r"""
354    Return the bash command line parm with single quotes if they are needed.
355
356    Description of arguments:
357    parm                            The string to be quoted.
358    """
359
360    # If any of these characters are found in the parm string, then the string should be quoted.  This list
361    # is by no means complete and should be expanded as needed by the developer of this function.
362    # Spaces
363    # Single or double quotes.
364    # Bash variables (therefore, any string with a "$" may need quoting).
365    # Glob characters: *, ?, []
366    # Extended Glob characters: +, @, !
367    # Bash brace expansion: {}
368    # Tilde expansion: ~
369    # Piped commands: |
370    # Bash re-direction: >, <
371    bash_special_chars = set(' \'"$*?[]+@!{}~|><')
372
373    if any((char in bash_special_chars) for char in parm):
374        return "'" + escape_bash_quotes(parm) + "'"
375
376    if parm == '':
377        parm = "''"
378
379    return parm
380
381
382def get_host_name_ip(host=None,
383                     short_name=0):
384    r"""
385    Get the host name and the IP address for the given host and return them as a tuple.
386
387    Description of argument(s):
388    host                            The host name or IP address to be obtained.
389    short_name                      Include the short host name in the returned tuple, i.e. return host, ip
390                                    and short_host.
391    """
392
393    host = dft(host, socket.gethostname())
394    host_name = socket.getfqdn(host)
395    try:
396        host_ip = socket.gethostbyname(host)
397    except socket.gaierror as my_gaierror:
398        message = "Unable to obtain the host name for the following host:" +\
399                  "\n" + gp.sprint_var(host)
400        gp.print_error_report(message)
401        raise my_gaierror
402
403    if short_name:
404        host_short_name = host_name.split(".")[0]
405        return host_name, host_ip, host_short_name
406    else:
407        return host_name, host_ip
408
409
410def pid_active(pid):
411    r"""
412    Return true if pid represents an active pid and false otherwise.
413
414    Description of argument(s):
415    pid                             The pid whose status is being sought.
416    """
417
418    try:
419        os.kill(int(pid), 0)
420    except OSError as err:
421        if err.errno == errno.ESRCH:
422            # ESRCH == No such process
423            return False
424        elif err.errno == errno.EPERM:
425            # EPERM clearly means there's a process to deny access to
426            return True
427        else:
428            # According to "man 2 kill" possible error values are
429            # (EINVAL, EPERM, ESRCH)
430            raise
431
432    return True
433
434
435def to_signed(number,
436              bit_width=None):
437    r"""
438    Convert number to a signed number and return the result.
439
440    Examples:
441
442    With the following code:
443
444    var1 = 0xfffffffffffffff1
445    print_var(var1)
446    print_var(var1, hexa())
447    var1 = to_signed(var1)
448    print_var(var1)
449    print_var(var1, hexa())
450
451    The following is written to stdout:
452    var1:  18446744073709551601
453    var1:  0x00000000fffffffffffffff1
454    var1:  -15
455    var1:  0xfffffffffffffff1
456
457    The same code but with var1 set to 0x000000000000007f produces the following:
458    var1:  127
459    var1:  0x000000000000007f
460    var1:  127
461    var1:  0x000000000000007f
462
463    Description of argument(s):
464    number                          The number to be converted.
465    bit_width                       The number of bits that defines a complete hex value.  Typically, this
466                                    would be a multiple of 32.
467    """
468
469    if bit_width is None:
470        try:
471            bit_width = gp.bit_length(long(sys.maxsize)) + 1
472        except NameError:
473            bit_width = gp.bit_length(int(sys.maxsize)) + 1
474
475    if number < 0:
476        return number
477    neg_bit_mask = 2**(bit_width - 1)
478    if number & neg_bit_mask:
479        return ((2**bit_width) - number) * -1
480    else:
481        return number
482
483
484def get_child_pids(quiet=1):
485
486    r"""
487    Get and return a list of pids representing all first-generation processes that are the children of the
488    current process.
489
490    Example:
491
492    children = get_child_pids()
493    print_var(children)
494
495    Output:
496    children:
497      children[0]:           9123
498
499    Description of argument(s):
500    quiet                           Display output to stdout detailing how this child pids are obtained.
501    """
502
503    if psutil_imported:
504        # If "import psutil" worked, find child pids using psutil.
505        current_process = psutil.Process()
506        return [x.pid for x in current_process.children(recursive=False)]
507    else:
508        # Otherwise, find child pids using shell commands.
509        print_output = not quiet
510
511        ps_cmd_buf = "ps --no-headers --ppid " + str(os.getpid()) +\
512            " -o pid,args"
513        # Route the output of ps to a temporary file for later grepping.  Avoid using " | grep" in the ps
514        # command string because it creates yet another process which is of no interest to the caller.
515        temp = tempfile.NamedTemporaryFile()
516        temp_file_path = temp.name
517        gc.shell_cmd(ps_cmd_buf + " > " + temp_file_path,
518                     print_output=print_output)
519        # Sample contents of the temporary file:
520        # 30703 sleep 2
521        # 30795 /bin/bash -c ps --no-headers --ppid 30672 -o pid,args > /tmp/tmpqqorWY
522        # Use egrep to exclude the "ps" process itself from the results collected with the prior shell_cmd
523        # invocation.  Only the other children are of interest to the caller.  Use cut on the grep results to
524        # obtain only the pid column.
525        rc, output = \
526            gc.shell_cmd("egrep -v '" + re.escape(ps_cmd_buf) + "' "
527                         + temp_file_path + " | cut -c1-5",
528                         print_output=print_output)
529        # Split the output buffer by line into a list.  Strip each element of extra spaces and convert each
530        # element to an integer.
531        return map(int, map(str.strip, filter(None, output.split("\n"))))
532
533
534def json_loads_multiple(buffer):
535    r"""
536    Convert the contents of the buffer to a JSON array, run json.loads() on it and return the result.
537
538    The buffer is expected to contain one or more JSON objects.
539
540    Description of argument(s):
541    buffer                          A string containing several JSON objects.
542    """
543
544    # Any line consisting of just "}", which indicates the end of an object, should have a comma appended.
545    regex = "([\\r\\n])[\\}]([\\r\\n])"
546    buffer = re.sub(regex, "\\1},\\2", buffer, 1)
547    # Remove the comma from after the final object and place the whole buffer inside square brackets.
548    buffer = "[" + re.sub(",([\r\n])$", "\\1}", buffer, 1) + "]"
549    if gp.robot_env:
550        return json.loads(buffer, object_pairs_hook=DotDict)
551    else:
552        return json.loads(buffer, object_pairs_hook=collections.OrderedDict)
553
554
555def file_date_time_stamp():
556    r"""
557    Return a date/time stamp in the following format: yymmdd.HHMMSS
558
559    This value is suitable for including in file names.  Example file1.181001.171716.status
560    """
561
562    return time.strftime("%y%m%d.%H%M%S", time.localtime(time.time()))
563
564
565def get_function_stack():
566    r"""
567    Return a list of all the function names currently in the call stack.
568
569    This function's name will be at offset 0.  This function's caller's name will be at offset 1 and so on.
570    """
571
572    return [str(stack_frame[3]) for stack_frame in inspect.stack()]
573
574
575def username():
576    r"""
577    Return the username for the current process.
578    """
579
580    username = os.environ.get("USER", "")
581    if username != "":
582        return username
583    user_num = str(os.geteuid())
584    try:
585        username = os.getlogin()
586    except OSError:
587        if user_num == "0":
588            username = "root"
589        else:
590            username = "?"
591
592    return username
593
594
595def version_tuple(version):
596    r"""
597    Convert the version string to a tuple and return it.
598
599    Description of argument(s):
600    version                         A version string whose format is "n[.n]" (e.g. "3.6.3", "3", etc.).
601    """
602
603    return tuple(map(int, (version.split("."))))
604
605
606def get_python_version():
607    r"""
608    Get and return the python version.
609    """
610
611    sys_version = sys.version
612    # Strip out any revision code data (e.g. "3.6.3rc1" will become "3.6.3").
613    sys_version = re.sub("rc[^ ]+", "", sys_version).split(" ")[0]
614    # Remove any non-numerics, etc. (e.g. "2.7.15+" becomes ""2.7.15").
615    return re.sub("[^0-9\\.]", "", sys_version)
616
617
618python_version = \
619    version_tuple(get_python_version())
620ordered_dict_version = version_tuple("3.6")
621
622
623def create_temp_file_path(delim=":", suffix=""):
624    r"""
625    Create a temporary file path and return it.
626
627    This function is appropriate for users who with to create a temporary file and:
628    1) Have control over when and whether the file is deleted.
629    2) Have the name of the file indicate information such as program name, function name, line, pid, etc.
630    This can be an aid in debugging, cleanup, etc.
631
632    The dir path portion of the file path will be /tmp/<username>/.  This function will create this directory
633    if it doesn't already exist.
634
635    This function will NOT create the file.  The file will NOT automatically get deleted.  It is the
636    responsibility of the caller to dispose of it.
637
638    Example:
639
640    pgm123.py is run by user 'joe'.  It calls func1 which contains this code:
641
642    temp_file_path = create_temp_file_path(suffix='suffix1')
643    print_var(temp_file_path)
644
645    Output:
646
647    temp_file_path:                 /tmp/joe/pgm123.py:func1:line_55:pid_8199:831848:suffix1
648
649    Description of argument(s):
650    delim                           A delimiter to be used to separate the sub-components of the file name.
651    suffix                          A suffix to include as the last sub-component of the file name.
652    """
653
654    temp_dir_path = "/tmp/" + username() + "/"
655    try:
656        os.mkdir(temp_dir_path)
657    except FileExistsError:
658        pass
659
660    callers_stack_frame = inspect.stack()[1]
661    file_name_elements = \
662        [
663            gp.pgm_name, callers_stack_frame.function, "line_" + str(callers_stack_frame.lineno),
664            "pid_" + str(os.getpid()), str(random.randint(0, 1000000)), suffix
665        ]
666    temp_file_name = delim.join(file_name_elements)
667
668    temp_file_path = temp_dir_path + temp_file_name
669
670    return temp_file_path
671