1#!/usr/bin/env python
2
3r"""
4This module provides many valuable functions such as my_parm_file.
5"""
6
7# sys and os are needed to get the program dir path and program name.
8import sys
9import errno
10import os
11import shutil
12import collections
13import json
14import time
15import inspect
16import random
17try:
18    import ConfigParser
19except ImportError:
20    import configparser
21try:
22    import StringIO
23except ImportError:
24    import io
25import re
26import socket
27import tempfile
28try:
29    import psutil
30    psutil_imported = True
31except ImportError:
32    psutil_imported = False
33
34import gen_print as gp
35import gen_cmd as gc
36
37robot_env = gp.robot_env
38if robot_env:
39    from robot.libraries.BuiltIn import BuiltIn
40    from robot.utils import DotDict
41
42
43def add_trailing_slash(dir_path):
44    r"""
45    Add a trailing slash to the directory path if it doesn't already have one
46    and return it.
47
48    Description of arguments:
49    dir_path                        A directory path.
50    """
51
52    return os.path.normpath(dir_path) + os.path.sep
53
54
55def makedirs(path, mode=0o777, quiet=None):
56    r"""
57    Call os.makedirs with the caller's arguments.
58
59    This function offers 2 advantages over the base os.makedirs function:
60    1) It will not fail if the directory already exists.
61    2) It will print an "Issuing: os.makedirs" message.
62
63    Description of argument(s):
64    path                            The path containing the directories to be created.
65    mode                            The mode or permissions to be granted to the created directories.
66    quiet                           Indicates whether this function should run the print_issuing() function.
67    """
68    quiet = int(dft(quiet, gp.get_stack_var('quiet', 0)))
69    gp.qprint_issuing("os.makedirs('" + path + "', mode=" + oct(mode) + ")")
70    try:
71        os.makedirs(path, mode)
72    except OSError:
73        pass
74
75
76def rmtree(path, ignore_errors=False, onerror=None, quiet=None):
77    r"""
78    Call shutil.rmtree with the caller's arguments.
79
80    This function offers this advantage over the base function:
81    - It will print an "Issuing: shutil.rmtree" message.
82
83    Description of argument(s):
84    (All parms are passed directly to shutil.rmtree.  See its prolog for details)
85    quiet                           Indicates whether this function should run the print_issuing() function.
86    """
87    quiet = int(dft(quiet, gp.get_stack_var('quiet', 0)))
88    print_string = gp.sprint_executing(max_width=2000)
89    print_string = re.sub(r"Executing: ", "Issuing: shutil.", print_string.rstrip("\n"))
90    gp.qprintn(re.sub(r", quiet[ ]?=.*", ")", print_string))
91    shutil.rmtree(path, ignore_errors, onerror)
92
93
94def chdir(path, quiet=None):
95    r"""
96    Call os.chdir with the caller's arguments.
97
98    This function offers this advantage over the base os.chdir function:
99    - It will print an "Issuing: os.chdir" message.
100
101    Description of argument(s):
102    path                            The path of the directory to change to.
103    quiet                           Indicates whether this function should run the print_issuing() function.
104    """
105    quiet = int(dft(quiet, gp.get_stack_var('quiet', 0)))
106    gp.qprint_issuing("os.chdir('" + path + "')")
107    os.chdir(path)
108
109
110def which(file_path):
111    r"""
112    Find the full path of an executable file and return it.
113
114    The PATH environment variable dictates the results of this function.
115
116    Description of arguments:
117    file_path                       The relative file path (e.g. "my_file" or "lib/my_file").
118    """
119
120    shell_rc, out_buf = gc.cmd_fnc_u("which " + file_path, quiet=1,
121                                     print_output=0, show_err=0)
122    if shell_rc != 0:
123        error_message = "Failed to find complete path for file \"" +\
124                        file_path + "\".\n"
125        error_message += gp.sprint_var(shell_rc, gp.hexa())
126        error_message += out_buf
127        if robot_env:
128            BuiltIn().fail(gp.sprint_error(error_message))
129        else:
130            gp.print_error_report(error_message)
131            return False
132
133    file_path = out_buf.rstrip("\n")
134
135    return file_path
136
137
138def add_path(new_path,
139             path,
140             position=0):
141    r"""
142    Add new_path to path, provided that path doesn't already contain new_path, and return the result.
143
144    Example:
145    If PATH has a value of "/bin/user:/lib/user".  The following code:
146
147    PATH = add_path("/tmp/new_path", PATH)
148
149    will change PATH to "/tmp/new_path:/bin/user:/lib/user".
150
151    Description of argument(s):
152    new_path                        The path to be added.  This function will strip the trailing slash.
153    path                            The path value to which the new_path should be added.
154    position                        The position in path where the new_path should be added.  0 means it
155                                    should be added to the beginning, 1 means add it as the 2nd item, etc.
156                                    sys.maxsize means it should be added to the end.
157    """
158
159    path_list = list(filter(None, path.split(":")))
160    new_path = new_path.rstrip("/")
161    if new_path not in path_list:
162        path_list.insert(int(position), new_path)
163    return ":".join(path_list)
164
165
166def dft(value, default):
167    r"""
168    Return default if value is None.  Otherwise, return value.
169
170    This is really just shorthand as shown below.
171
172    dft(value, default)
173
174    vs
175
176    default if value is None else value
177
178    Description of arguments:
179    value                           The value to be returned.
180    default                         The default value to return if value is None.
181    """
182
183    return default if value is None else value
184
185
186def get_mod_global(var_name,
187                   default=None,
188                   mod_name="__main__"):
189    r"""
190    Get module global variable value and return it.
191
192    If we are running in a robot environment, the behavior will default to
193    calling get_variable_value.
194
195    Description of arguments:
196    var_name                        The name of the variable whose value is sought.
197    default                         The value to return if the global does not exist.
198    mod_name                        The name of the module containing the global variable.
199    """
200
201    if robot_env:
202        return BuiltIn().get_variable_value("${" + var_name + "}", default)
203
204    try:
205        module = sys.modules[mod_name]
206    except KeyError:
207        gp.print_error_report("Programmer error - The mod_name passed to"
208                              + " this function is invalid:\n"
209                              + gp.sprint_var(mod_name))
210        raise ValueError('Programmer error.')
211
212    if default is None:
213        return getattr(module, var_name)
214    else:
215        return getattr(module, var_name, default)
216
217
218def global_default(var_value,
219                   default=0):
220    r"""
221    If var_value is not None, return it.  Otherwise, return the global
222    variable of the same name, if it exists.  If not, return default.
223
224    This is meant for use by functions needing help assigning dynamic default
225    values to their parms.  Example:
226
227    def func1(parm1=None):
228
229        parm1 = global_default(parm1, 0)
230
231    Description of arguments:
232    var_value                       The value being evaluated.
233    default                         The value to be returned if var_value is None AND the global variable of
234                                    the same name does not exist.
235    """
236
237    var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
238
239    return dft(var_value, get_mod_global(var_name, 0))
240
241
242def set_mod_global(var_value,
243                   mod_name="__main__",
244                   var_name=None):
245    r"""
246    Set a global variable for a given module.
247
248    Description of arguments:
249    var_value                       The value to set in the variable.
250    mod_name                        The name of the module whose variable is to be set.
251    var_name                        The name of the variable to set.  This defaults to the name of the
252                                    variable used for var_value when calling this function.
253    """
254
255    try:
256        module = sys.modules[mod_name]
257    except KeyError:
258        gp.print_error_report("Programmer error - The mod_name passed to"
259                              + " this function is invalid:\n"
260                              + gp.sprint_var(mod_name))
261        raise ValueError('Programmer error.')
262
263    if var_name is None:
264        var_name = gp.get_arg_name(None, 1, 2)
265
266    setattr(module, var_name, var_value)
267
268
269def my_parm_file(prop_file_path):
270    r"""
271    Read a properties file, put the keys/values into a dictionary and return the dictionary.
272
273    The properties file must have the following format:
274    var_name<= or :>var_value
275    Comment lines (those beginning with a "#") and blank lines are allowed and will be ignored.  Leading and
276    trailing single or double quotes will be stripped from the value.  E.g.
277    var1="This one"
278    Quotes are stripped so the resulting value for var1 is:
279    This one
280
281    Description of arguments:
282    prop_file_path                  The caller should pass the path to the properties file.
283    """
284
285    # ConfigParser expects at least one section header in the file (or you get
286    # ConfigParser.MissingSectionHeaderError).  Properties files don't need those so I'll write a dummy
287    # section header.
288
289    try:
290        string_file = StringIO.StringIO()
291    except NameError:
292        string_file = io.StringIO()
293
294    # Write the dummy section header to the string file.
295    string_file.write('[dummysection]\n')
296    # Write the entire contents of the properties file to the string file.
297    string_file.write(open(prop_file_path).read())
298    # Rewind the string file.
299    string_file.seek(0, os.SEEK_SET)
300
301    # Create the ConfigParser object.
302    try:
303        config_parser = ConfigParser.ConfigParser()
304    except NameError:
305        config_parser = configparser.ConfigParser(strict=False)
306    # Make the property names case-sensitive.
307    config_parser.optionxform = str
308    # Read the properties from the string file.
309    config_parser.readfp(string_file)
310    # Return the properties as a dictionary.
311    if robot_env:
312        return DotDict(config_parser.items('dummysection'))
313    else:
314        return collections.OrderedDict(config_parser.items('dummysection'))
315
316
317def file_to_list(file_path,
318                 newlines=0,
319                 comments=1,
320                 trim=0):
321    r"""
322    Return the contents of a file as a list.  Each element of the resulting
323    list is one line from the file.
324
325    Description of arguments:
326    file_path                       The path to the file (relative or absolute).
327    newlines                        Include newlines from the file in the results.
328    comments                        Include comment lines and blank lines in the results.  Comment lines are
329                                    any that begin with 0 or more spaces followed by the pound sign ("#").
330    trim                            Trim white space from the beginning and end of each line.
331    """
332
333    lines = []
334    file = open(file_path)
335    for line in file:
336        if not comments:
337            if re.match(r"[ ]*#|^$", line):
338                continue
339        if not newlines:
340            line = line.rstrip("\n")
341        if trim:
342            line = line.strip()
343        lines.append(line)
344    file.close()
345
346    return lines
347
348
349def file_to_str(*args, **kwargs):
350    r"""
351    Return the contents of a file as a string.
352
353    Description of arguments:
354    See file_to_list defined above for description of arguments.
355    """
356
357    return '\n'.join(file_to_list(*args, **kwargs))
358
359
360def append_file(file_path, buffer):
361    r"""
362    Append the data in buffer to the file named in file_path.
363
364    Description of argument(s):
365    file_path                       The path to a file (e.g. "/tmp/root/file1").
366    buffer                          The buffer of data to be written to the file (e.g. "this and that").
367    """
368
369    with open(file_path, "a") as file:
370        file.write(buffer)
371
372
373def return_path_list():
374    r"""
375    This function will split the PATH environment variable into a PATH_LIST and return it.  Each element in
376    the list will be normalized and have a trailing slash added.
377    """
378
379    PATH_LIST = os.environ['PATH'].split(":")
380    PATH_LIST = [os.path.normpath(path) + os.sep for path in PATH_LIST]
381
382    return PATH_LIST
383
384
385def escape_bash_quotes(buffer):
386    r"""
387    Escape quotes in string and return it.
388
389    The escape style implemented will be for use on the bash command line.
390
391    Example:
392    That's all.
393
394    Result:
395    That'\''s all.
396
397    The result may then be single quoted on a bash command.  Example:
398
399    echo 'That'\''s all.'
400
401    Description of argument(s):
402    buffer                          The string whose quotes are to be escaped.
403    """
404
405    return re.sub("\'", "\'\\\'\'", buffer)
406
407
408def quote_bash_parm(parm):
409    r"""
410    Return the bash command line parm with single quotes if they are needed.
411
412    Description of arguments:
413    parm                            The string to be quoted.
414    """
415
416    # If any of these characters are found in the parm string, then the string should be quoted.  This list
417    # is by no means complete and should be expanded as needed by the developer of this function.
418    # Spaces
419    # Single or double quotes.
420    # Bash variables (therefore, any string with a "$" may need quoting).
421    # Glob characters: *, ?, []
422    # Extended Glob characters: +, @, !
423    # Bash brace expansion: {}
424    # Tilde expansion: ~
425    # Piped commands: |
426    # Bash re-direction: >, <
427    bash_special_chars = set(' \'"$*?[]+@!{}~|><')
428
429    if any((char in bash_special_chars) for char in parm):
430        return "'" + escape_bash_quotes(parm) + "'"
431
432    if parm == '':
433        parm = "''"
434
435    return parm
436
437
438def get_host_name_ip(host=None,
439                     short_name=0):
440    r"""
441    Get the host name and the IP address for the given host and return them as a tuple.
442
443    Description of argument(s):
444    host                            The host name or IP address to be obtained.
445    short_name                      Include the short host name in the returned tuple, i.e. return host, ip
446                                    and short_host.
447    """
448
449    host = dft(host, socket.gethostname())
450    host_name = socket.getfqdn(host)
451    try:
452        host_ip = socket.gethostbyname(host)
453    except socket.gaierror as my_gaierror:
454        message = "Unable to obtain the host name for the following host:" +\
455                  "\n" + gp.sprint_var(host)
456        gp.print_error_report(message)
457        raise my_gaierror
458
459    if short_name:
460        host_short_name = host_name.split(".")[0]
461        return host_name, host_ip, host_short_name
462    else:
463        return host_name, host_ip
464
465
466def pid_active(pid):
467    r"""
468    Return true if pid represents an active pid and false otherwise.
469
470    Description of argument(s):
471    pid                             The pid whose status is being sought.
472    """
473
474    try:
475        os.kill(int(pid), 0)
476    except OSError as err:
477        if err.errno == errno.ESRCH:
478            # ESRCH == No such process
479            return False
480        elif err.errno == errno.EPERM:
481            # EPERM clearly means there's a process to deny access to
482            return True
483        else:
484            # According to "man 2 kill" possible error values are
485            # (EINVAL, EPERM, ESRCH)
486            raise
487
488    return True
489
490
491def to_signed(number,
492              bit_width=None):
493    r"""
494    Convert number to a signed number and return the result.
495
496    Examples:
497
498    With the following code:
499
500    var1 = 0xfffffffffffffff1
501    print_var(var1)
502    print_var(var1, hexa())
503    var1 = to_signed(var1)
504    print_var(var1)
505    print_var(var1, hexa())
506
507    The following is written to stdout:
508    var1:  18446744073709551601
509    var1:  0x00000000fffffffffffffff1
510    var1:  -15
511    var1:  0xfffffffffffffff1
512
513    The same code but with var1 set to 0x000000000000007f produces the following:
514    var1:  127
515    var1:  0x000000000000007f
516    var1:  127
517    var1:  0x000000000000007f
518
519    Description of argument(s):
520    number                          The number to be converted.
521    bit_width                       The number of bits that defines a complete hex value.  Typically, this
522                                    would be a multiple of 32.
523    """
524
525    if bit_width is None:
526        try:
527            bit_width = gp.bit_length(long(sys.maxsize)) + 1
528        except NameError:
529            bit_width = gp.bit_length(int(sys.maxsize)) + 1
530
531    if number < 0:
532        return number
533    neg_bit_mask = 2**(bit_width - 1)
534    if number & neg_bit_mask:
535        return ((2**bit_width) - number) * -1
536    else:
537        return number
538
539
540def get_child_pids(quiet=1):
541
542    r"""
543    Get and return a list of pids representing all first-generation processes that are the children of the
544    current process.
545
546    Example:
547
548    children = get_child_pids()
549    print_var(children)
550
551    Output:
552    children:
553      children[0]:           9123
554
555    Description of argument(s):
556    quiet                           Display output to stdout detailing how this child pids are obtained.
557    """
558
559    if psutil_imported:
560        # If "import psutil" worked, find child pids using psutil.
561        current_process = psutil.Process()
562        return [x.pid for x in current_process.children(recursive=False)]
563    else:
564        # Otherwise, find child pids using shell commands.
565        print_output = not quiet
566
567        ps_cmd_buf = "ps --no-headers --ppid " + str(os.getpid()) +\
568            " -o pid,args"
569        # Route the output of ps to a temporary file for later grepping.  Avoid using " | grep" in the ps
570        # command string because it creates yet another process which is of no interest to the caller.
571        temp = tempfile.NamedTemporaryFile()
572        temp_file_path = temp.name
573        gc.shell_cmd(ps_cmd_buf + " > " + temp_file_path,
574                     print_output=print_output)
575        # Sample contents of the temporary file:
576        # 30703 sleep 2
577        # 30795 /bin/bash -c ps --no-headers --ppid 30672 -o pid,args > /tmp/tmpqqorWY
578        # Use egrep to exclude the "ps" process itself from the results collected with the prior shell_cmd
579        # invocation.  Only the other children are of interest to the caller.  Use cut on the grep results to
580        # obtain only the pid column.
581        rc, output = \
582            gc.shell_cmd("egrep -v '" + re.escape(ps_cmd_buf) + "' "
583                         + temp_file_path + " | cut -c1-5",
584                         print_output=print_output)
585        # Split the output buffer by line into a list.  Strip each element of extra spaces and convert each
586        # element to an integer.
587        return map(int, map(str.strip, filter(None, output.split("\n"))))
588
589
590def json_loads_multiple(buffer):
591    r"""
592    Convert the contents of the buffer to a JSON array, run json.loads() on it and return the result.
593
594    The buffer is expected to contain one or more JSON objects.
595
596    Description of argument(s):
597    buffer                          A string containing several JSON objects.
598    """
599
600    # Any line consisting of just "}", which indicates the end of an object, should have a comma appended.
601    regex = "([\\r\\n])[\\}]([\\r\\n])"
602    buffer = re.sub(regex, "\\1},\\2", buffer, 1)
603    # Remove the comma from after the final object and place the whole buffer inside square brackets.
604    buffer = "[" + re.sub(",([\r\n])$", "\\1}", buffer, 1) + "]"
605    if gp.robot_env:
606        return json.loads(buffer, object_pairs_hook=DotDict)
607    else:
608        return json.loads(buffer, object_pairs_hook=collections.OrderedDict)
609
610
611def file_date_time_stamp():
612    r"""
613    Return a date/time stamp in the following format: yymmdd.HHMMSS
614
615    This value is suitable for including in file names.  Example file1.181001.171716.status
616    """
617
618    return time.strftime("%y%m%d.%H%M%S", time.localtime(time.time()))
619
620
621def get_function_stack():
622    r"""
623    Return a list of all the function names currently in the call stack.
624
625    This function's name will be at offset 0.  This function's caller's name will be at offset 1 and so on.
626    """
627
628    return [str(stack_frame[3]) for stack_frame in inspect.stack()]
629
630
631def username():
632    r"""
633    Return the username for the current process.
634    """
635
636    username = os.environ.get("USER", "")
637    if username != "":
638        return username
639    user_num = str(os.geteuid())
640    try:
641        username = os.getlogin()
642    except OSError:
643        if user_num == "0":
644            username = "root"
645        else:
646            username = "?"
647
648    return username
649
650
651def version_tuple(version):
652    r"""
653    Convert the version string to a tuple and return it.
654
655    Description of argument(s):
656    version                         A version string whose format is "n[.n]" (e.g. "3.6.3", "3", etc.).
657    """
658
659    return tuple(map(int, (version.split("."))))
660
661
662def get_python_version():
663    r"""
664    Get and return the python version.
665    """
666
667    sys_version = sys.version
668    # Strip out any revision code data (e.g. "3.6.3rc1" will become "3.6.3").
669    sys_version = re.sub("rc[^ ]+", "", sys_version).split(" ")[0]
670    # Remove any non-numerics, etc. (e.g. "2.7.15+" becomes ""2.7.15").
671    return re.sub("[^0-9\\.]", "", sys_version)
672
673
674python_version = \
675    version_tuple(get_python_version())
676ordered_dict_version = version_tuple("3.6")
677
678
679def create_temp_file_path(delim=":", suffix=""):
680    r"""
681    Create a temporary file path and return it.
682
683    This function is appropriate for users who with to create a temporary file and:
684    1) Have control over when and whether the file is deleted.
685    2) Have the name of the file indicate information such as program name, function name, line, pid, etc.
686    This can be an aid in debugging, cleanup, etc.
687
688    The dir path portion of the file path will be /tmp/<username>/.  This function will create this directory
689    if it doesn't already exist.
690
691    This function will NOT create the file.  The file will NOT automatically get deleted.  It is the
692    responsibility of the caller to dispose of it.
693
694    Example:
695
696    pgm123.py is run by user 'joe'.  It calls func1 which contains this code:
697
698    temp_file_path = create_temp_file_path(suffix='suffix1')
699    print_var(temp_file_path)
700
701    Output:
702
703    temp_file_path:                 /tmp/joe/pgm123.py:func1:line_55:pid_8199:831848:suffix1
704
705    Description of argument(s):
706    delim                           A delimiter to be used to separate the sub-components of the file name.
707    suffix                          A suffix to include as the last sub-component of the file name.
708    """
709
710    temp_dir_path = "/tmp/" + username() + "/"
711    try:
712        os.mkdir(temp_dir_path)
713    except FileExistsError:
714        pass
715
716    callers_stack_frame = inspect.stack()[1]
717    file_name_elements = \
718        [
719            gp.pgm_name, callers_stack_frame.function, "line_" + str(callers_stack_frame.lineno),
720            "pid_" + str(os.getpid()), str(random.randint(0, 1000000)), suffix
721        ]
722    temp_file_name = delim.join(file_name_elements)
723
724    temp_file_path = temp_dir_path + temp_file_name
725
726    return temp_file_path
727
728
729def pause(message="Hit enter to continue..."):
730    r"""
731    Print the message, with time stamp, and pause until the user hits enter.
732
733    Description of argument(s):
734    message                         The message to be printed to stdout.
735    """
736    gp.print_time(message)
737    try:
738        input()
739    except SyntaxError:
740        pass
741
742    return
743