1#!/usr/bin/env python
2
3r"""
4This module provides many valuable functions such as my_parm_file.
5"""
6
7# sys and os are needed to get the program dir path and program name.
8import sys
9import errno
10import os
11import collections
12import json
13import time
14import inspect
15import random
16try:
17    import ConfigParser
18except ImportError:
19    import configparser
20try:
21    import StringIO
22except ImportError:
23    import io
24import re
25import socket
26import tempfile
27try:
28    import psutil
29    psutil_imported = True
30except ImportError:
31    psutil_imported = False
32
33import gen_print as gp
34import gen_cmd as gc
35
36robot_env = gp.robot_env
37if robot_env:
38    from robot.libraries.BuiltIn import BuiltIn
39    from robot.utils import DotDict
40
41
42def add_trailing_slash(dir_path):
43    r"""
44    Add a trailing slash to the directory path if it doesn't already have one
45    and return it.
46
47    Description of arguments:
48    dir_path                        A directory path.
49    """
50
51    return os.path.normpath(dir_path) + os.path.sep
52
53
54def makedirs(path, mode=0o777, quiet=None):
55    r"""
56    Call os.makedirs with the caller's arguments.
57
58    This function offers 2 advantages over the base os.makedirs function:
59    1) It will not fail if the directory already exists.
60    2) It will print an "Issuing: os.makedirs" message.
61
62    Description of argument(s):
63    path                            The path containing the directories to be created.
64    mode                            The mode or permissions to be granted to the created directories.
65    quiet                           Indicates whether this function should run the print_issuing() function.
66    """
67    quiet = int(dft(quiet, gp.get_stack_var('quiet', 0)))
68    gp.qprint_issuing("os.makedirs('" + path + "', mode=" + oct(mode) + ")")
69    try:
70        os.makedirs(path, mode)
71    except FileExistsError:
72        pass
73
74
75def chdir(path, quiet=None):
76    r"""
77    Call os.chdir with the caller's arguments.
78
79    This function offers this advantage over the base os.chdir function:
80    - It will print an "Issuing: os.chdir" message.
81
82    Description of argument(s):
83    path                            The path of the directory to change to.
84    quiet                           Indicates whether this function should run the print_issuing() function.
85    """
86    quiet = int(dft(quiet, gp.get_stack_var('quiet', 0)))
87    gp.qprint_issuing("os.chdir('" + path + "')")
88    os.chdir(path)
89
90
91def which(file_path):
92    r"""
93    Find the full path of an executable file and return it.
94
95    The PATH environment variable dictates the results of this function.
96
97    Description of arguments:
98    file_path                       The relative file path (e.g. "my_file" or "lib/my_file").
99    """
100
101    shell_rc, out_buf = gc.cmd_fnc_u("which " + file_path, quiet=1,
102                                     print_output=0, show_err=0)
103    if shell_rc != 0:
104        error_message = "Failed to find complete path for file \"" +\
105                        file_path + "\".\n"
106        error_message += gp.sprint_var(shell_rc, gp.hexa())
107        error_message += out_buf
108        if robot_env:
109            BuiltIn().fail(gp.sprint_error(error_message))
110        else:
111            gp.print_error_report(error_message)
112            return False
113
114    file_path = out_buf.rstrip("\n")
115
116    return file_path
117
118
119def add_path(new_path,
120             path,
121             position=0):
122    r"""
123    Add new_path to path, provided that path doesn't already contain new_path, and return the result.
124
125    Example:
126    If PATH has a value of "/bin/user:/lib/user".  The following code:
127
128    PATH = add_path("/tmp/new_path", PATH)
129
130    will change PATH to "/tmp/new_path:/bin/user:/lib/user".
131
132    Description of argument(s):
133    new_path                        The path to be added.  This function will strip the trailing slash.
134    path                            The path value to which the new_path should be added.
135    position                        The position in path where the new_path should be added.  0 means it
136                                    should be added to the beginning, 1 means add it as the 2nd item, etc.
137                                    sys.maxsize means it should be added to the end.
138    """
139
140    path_list = list(filter(None, path.split(":")))
141    new_path = new_path.rstrip("/")
142    if new_path not in path_list:
143        path_list.insert(int(position), new_path)
144    return ":".join(path_list)
145
146
147def dft(value, default):
148    r"""
149    Return default if value is None.  Otherwise, return value.
150
151    This is really just shorthand as shown below.
152
153    dft(value, default)
154
155    vs
156
157    default if value is None else value
158
159    Description of arguments:
160    value                           The value to be returned.
161    default                         The default value to return if value is None.
162    """
163
164    return default if value is None else value
165
166
167def get_mod_global(var_name,
168                   default=None,
169                   mod_name="__main__"):
170    r"""
171    Get module global variable value and return it.
172
173    If we are running in a robot environment, the behavior will default to
174    calling get_variable_value.
175
176    Description of arguments:
177    var_name                        The name of the variable whose value is sought.
178    default                         The value to return if the global does not exist.
179    mod_name                        The name of the module containing the global variable.
180    """
181
182    if robot_env:
183        return BuiltIn().get_variable_value("${" + var_name + "}", default)
184
185    try:
186        module = sys.modules[mod_name]
187    except KeyError:
188        gp.print_error_report("Programmer error - The mod_name passed to"
189                              + " this function is invalid:\n"
190                              + gp.sprint_var(mod_name))
191        raise ValueError('Programmer error.')
192
193    if default is None:
194        return getattr(module, var_name)
195    else:
196        return getattr(module, var_name, default)
197
198
199def global_default(var_value,
200                   default=0):
201    r"""
202    If var_value is not None, return it.  Otherwise, return the global
203    variable of the same name, if it exists.  If not, return default.
204
205    This is meant for use by functions needing help assigning dynamic default
206    values to their parms.  Example:
207
208    def func1(parm1=None):
209
210        parm1 = global_default(parm1, 0)
211
212    Description of arguments:
213    var_value                       The value being evaluated.
214    default                         The value to be returned if var_value is None AND the global variable of
215                                    the same name does not exist.
216    """
217
218    var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
219
220    return dft(var_value, get_mod_global(var_name, 0))
221
222
223def set_mod_global(var_value,
224                   mod_name="__main__",
225                   var_name=None):
226    r"""
227    Set a global variable for a given module.
228
229    Description of arguments:
230    var_value                       The value to set in the variable.
231    mod_name                        The name of the module whose variable is to be set.
232    var_name                        The name of the variable to set.  This defaults to the name of the
233                                    variable used for var_value when calling this function.
234    """
235
236    try:
237        module = sys.modules[mod_name]
238    except KeyError:
239        gp.print_error_report("Programmer error - The mod_name passed to"
240                              + " this function is invalid:\n"
241                              + gp.sprint_var(mod_name))
242        raise ValueError('Programmer error.')
243
244    if var_name is None:
245        var_name = gp.get_arg_name(None, 1, 2)
246
247    setattr(module, var_name, var_value)
248
249
250def my_parm_file(prop_file_path):
251    r"""
252    Read a properties file, put the keys/values into a dictionary and return the dictionary.
253
254    The properties file must have the following format:
255    var_name<= or :>var_value
256    Comment lines (those beginning with a "#") and blank lines are allowed and will be ignored.  Leading and
257    trailing single or double quotes will be stripped from the value.  E.g.
258    var1="This one"
259    Quotes are stripped so the resulting value for var1 is:
260    This one
261
262    Description of arguments:
263    prop_file_path                  The caller should pass the path to the properties file.
264    """
265
266    # ConfigParser expects at least one section header in the file (or you get
267    # ConfigParser.MissingSectionHeaderError).  Properties files don't need those so I'll write a dummy
268    # section header.
269
270    try:
271        string_file = StringIO.StringIO()
272    except NameError:
273        string_file = io.StringIO()
274
275    # Write the dummy section header to the string file.
276    string_file.write('[dummysection]\n')
277    # Write the entire contents of the properties file to the string file.
278    string_file.write(open(prop_file_path).read())
279    # Rewind the string file.
280    string_file.seek(0, os.SEEK_SET)
281
282    # Create the ConfigParser object.
283    try:
284        config_parser = ConfigParser.ConfigParser()
285    except NameError:
286        config_parser = configparser.ConfigParser(strict=False)
287    # Make the property names case-sensitive.
288    config_parser.optionxform = str
289    # Read the properties from the string file.
290    config_parser.readfp(string_file)
291    # Return the properties as a dictionary.
292    if robot_env:
293        return DotDict(config_parser.items('dummysection'))
294    else:
295        return collections.OrderedDict(config_parser.items('dummysection'))
296
297
298def file_to_list(file_path,
299                 newlines=0,
300                 comments=1,
301                 trim=0):
302    r"""
303    Return the contents of a file as a list.  Each element of the resulting
304    list is one line from the file.
305
306    Description of arguments:
307    file_path                       The path to the file (relative or absolute).
308    newlines                        Include newlines from the file in the results.
309    comments                        Include comment lines and blank lines in the results.  Comment lines are
310                                    any that begin with 0 or more spaces followed by the pound sign ("#").
311    trim                            Trim white space from the beginning and end of each line.
312    """
313
314    lines = []
315    file = open(file_path)
316    for line in file:
317        if not comments:
318            if re.match(r"[ ]*#|^$", line):
319                continue
320        if not newlines:
321            line = line.rstrip("\n")
322        if trim:
323            line = line.strip()
324        lines.append(line)
325    file.close()
326
327    return lines
328
329
330def file_to_str(*args, **kwargs):
331    r"""
332    Return the contents of a file as a string.
333
334    Description of arguments:
335    See file_to_list defined above for description of arguments.
336    """
337
338    return '\n'.join(file_to_list(*args, **kwargs))
339
340
341def append_file(file_path, buffer):
342    r"""
343    Append the data in buffer to the file named in file_path.
344
345    Description of argument(s):
346    file_path                       The path to a file (e.g. "/tmp/root/file1").
347    buffer                          The buffer of data to be written to the file (e.g. "this and that").
348    """
349
350    with open(file_path, "a") as file:
351        file.write(buffer)
352
353
354def return_path_list():
355    r"""
356    This function will split the PATH environment variable into a PATH_LIST and return it.  Each element in
357    the list will be normalized and have a trailing slash added.
358    """
359
360    PATH_LIST = os.environ['PATH'].split(":")
361    PATH_LIST = [os.path.normpath(path) + os.sep for path in PATH_LIST]
362
363    return PATH_LIST
364
365
366def escape_bash_quotes(buffer):
367    r"""
368    Escape quotes in string and return it.
369
370    The escape style implemented will be for use on the bash command line.
371
372    Example:
373    That's all.
374
375    Result:
376    That'\''s all.
377
378    The result may then be single quoted on a bash command.  Example:
379
380    echo 'That'\''s all.'
381
382    Description of argument(s):
383    buffer                          The string whose quotes are to be escaped.
384    """
385
386    return re.sub("\'", "\'\\\'\'", buffer)
387
388
389def quote_bash_parm(parm):
390    r"""
391    Return the bash command line parm with single quotes if they are needed.
392
393    Description of arguments:
394    parm                            The string to be quoted.
395    """
396
397    # If any of these characters are found in the parm string, then the string should be quoted.  This list
398    # is by no means complete and should be expanded as needed by the developer of this function.
399    # Spaces
400    # Single or double quotes.
401    # Bash variables (therefore, any string with a "$" may need quoting).
402    # Glob characters: *, ?, []
403    # Extended Glob characters: +, @, !
404    # Bash brace expansion: {}
405    # Tilde expansion: ~
406    # Piped commands: |
407    # Bash re-direction: >, <
408    bash_special_chars = set(' \'"$*?[]+@!{}~|><')
409
410    if any((char in bash_special_chars) for char in parm):
411        return "'" + escape_bash_quotes(parm) + "'"
412
413    if parm == '':
414        parm = "''"
415
416    return parm
417
418
419def get_host_name_ip(host=None,
420                     short_name=0):
421    r"""
422    Get the host name and the IP address for the given host and return them as a tuple.
423
424    Description of argument(s):
425    host                            The host name or IP address to be obtained.
426    short_name                      Include the short host name in the returned tuple, i.e. return host, ip
427                                    and short_host.
428    """
429
430    host = dft(host, socket.gethostname())
431    host_name = socket.getfqdn(host)
432    try:
433        host_ip = socket.gethostbyname(host)
434    except socket.gaierror as my_gaierror:
435        message = "Unable to obtain the host name for the following host:" +\
436                  "\n" + gp.sprint_var(host)
437        gp.print_error_report(message)
438        raise my_gaierror
439
440    if short_name:
441        host_short_name = host_name.split(".")[0]
442        return host_name, host_ip, host_short_name
443    else:
444        return host_name, host_ip
445
446
447def pid_active(pid):
448    r"""
449    Return true if pid represents an active pid and false otherwise.
450
451    Description of argument(s):
452    pid                             The pid whose status is being sought.
453    """
454
455    try:
456        os.kill(int(pid), 0)
457    except OSError as err:
458        if err.errno == errno.ESRCH:
459            # ESRCH == No such process
460            return False
461        elif err.errno == errno.EPERM:
462            # EPERM clearly means there's a process to deny access to
463            return True
464        else:
465            # According to "man 2 kill" possible error values are
466            # (EINVAL, EPERM, ESRCH)
467            raise
468
469    return True
470
471
472def to_signed(number,
473              bit_width=None):
474    r"""
475    Convert number to a signed number and return the result.
476
477    Examples:
478
479    With the following code:
480
481    var1 = 0xfffffffffffffff1
482    print_var(var1)
483    print_var(var1, hexa())
484    var1 = to_signed(var1)
485    print_var(var1)
486    print_var(var1, hexa())
487
488    The following is written to stdout:
489    var1:  18446744073709551601
490    var1:  0x00000000fffffffffffffff1
491    var1:  -15
492    var1:  0xfffffffffffffff1
493
494    The same code but with var1 set to 0x000000000000007f produces the following:
495    var1:  127
496    var1:  0x000000000000007f
497    var1:  127
498    var1:  0x000000000000007f
499
500    Description of argument(s):
501    number                          The number to be converted.
502    bit_width                       The number of bits that defines a complete hex value.  Typically, this
503                                    would be a multiple of 32.
504    """
505
506    if bit_width is None:
507        try:
508            bit_width = gp.bit_length(long(sys.maxsize)) + 1
509        except NameError:
510            bit_width = gp.bit_length(int(sys.maxsize)) + 1
511
512    if number < 0:
513        return number
514    neg_bit_mask = 2**(bit_width - 1)
515    if number & neg_bit_mask:
516        return ((2**bit_width) - number) * -1
517    else:
518        return number
519
520
521def get_child_pids(quiet=1):
522
523    r"""
524    Get and return a list of pids representing all first-generation processes that are the children of the
525    current process.
526
527    Example:
528
529    children = get_child_pids()
530    print_var(children)
531
532    Output:
533    children:
534      children[0]:           9123
535
536    Description of argument(s):
537    quiet                           Display output to stdout detailing how this child pids are obtained.
538    """
539
540    if psutil_imported:
541        # If "import psutil" worked, find child pids using psutil.
542        current_process = psutil.Process()
543        return [x.pid for x in current_process.children(recursive=False)]
544    else:
545        # Otherwise, find child pids using shell commands.
546        print_output = not quiet
547
548        ps_cmd_buf = "ps --no-headers --ppid " + str(os.getpid()) +\
549            " -o pid,args"
550        # Route the output of ps to a temporary file for later grepping.  Avoid using " | grep" in the ps
551        # command string because it creates yet another process which is of no interest to the caller.
552        temp = tempfile.NamedTemporaryFile()
553        temp_file_path = temp.name
554        gc.shell_cmd(ps_cmd_buf + " > " + temp_file_path,
555                     print_output=print_output)
556        # Sample contents of the temporary file:
557        # 30703 sleep 2
558        # 30795 /bin/bash -c ps --no-headers --ppid 30672 -o pid,args > /tmp/tmpqqorWY
559        # Use egrep to exclude the "ps" process itself from the results collected with the prior shell_cmd
560        # invocation.  Only the other children are of interest to the caller.  Use cut on the grep results to
561        # obtain only the pid column.
562        rc, output = \
563            gc.shell_cmd("egrep -v '" + re.escape(ps_cmd_buf) + "' "
564                         + temp_file_path + " | cut -c1-5",
565                         print_output=print_output)
566        # Split the output buffer by line into a list.  Strip each element of extra spaces and convert each
567        # element to an integer.
568        return map(int, map(str.strip, filter(None, output.split("\n"))))
569
570
571def json_loads_multiple(buffer):
572    r"""
573    Convert the contents of the buffer to a JSON array, run json.loads() on it and return the result.
574
575    The buffer is expected to contain one or more JSON objects.
576
577    Description of argument(s):
578    buffer                          A string containing several JSON objects.
579    """
580
581    # Any line consisting of just "}", which indicates the end of an object, should have a comma appended.
582    regex = "([\\r\\n])[\\}]([\\r\\n])"
583    buffer = re.sub(regex, "\\1},\\2", buffer, 1)
584    # Remove the comma from after the final object and place the whole buffer inside square brackets.
585    buffer = "[" + re.sub(",([\r\n])$", "\\1}", buffer, 1) + "]"
586    if gp.robot_env:
587        return json.loads(buffer, object_pairs_hook=DotDict)
588    else:
589        return json.loads(buffer, object_pairs_hook=collections.OrderedDict)
590
591
592def file_date_time_stamp():
593    r"""
594    Return a date/time stamp in the following format: yymmdd.HHMMSS
595
596    This value is suitable for including in file names.  Example file1.181001.171716.status
597    """
598
599    return time.strftime("%y%m%d.%H%M%S", time.localtime(time.time()))
600
601
602def get_function_stack():
603    r"""
604    Return a list of all the function names currently in the call stack.
605
606    This function's name will be at offset 0.  This function's caller's name will be at offset 1 and so on.
607    """
608
609    return [str(stack_frame[3]) for stack_frame in inspect.stack()]
610
611
612def username():
613    r"""
614    Return the username for the current process.
615    """
616
617    username = os.environ.get("USER", "")
618    if username != "":
619        return username
620    user_num = str(os.geteuid())
621    try:
622        username = os.getlogin()
623    except OSError:
624        if user_num == "0":
625            username = "root"
626        else:
627            username = "?"
628
629    return username
630
631
632def version_tuple(version):
633    r"""
634    Convert the version string to a tuple and return it.
635
636    Description of argument(s):
637    version                         A version string whose format is "n[.n]" (e.g. "3.6.3", "3", etc.).
638    """
639
640    return tuple(map(int, (version.split("."))))
641
642
643def get_python_version():
644    r"""
645    Get and return the python version.
646    """
647
648    sys_version = sys.version
649    # Strip out any revision code data (e.g. "3.6.3rc1" will become "3.6.3").
650    sys_version = re.sub("rc[^ ]+", "", sys_version).split(" ")[0]
651    # Remove any non-numerics, etc. (e.g. "2.7.15+" becomes ""2.7.15").
652    return re.sub("[^0-9\\.]", "", sys_version)
653
654
655python_version = \
656    version_tuple(get_python_version())
657ordered_dict_version = version_tuple("3.6")
658
659
660def create_temp_file_path(delim=":", suffix=""):
661    r"""
662    Create a temporary file path and return it.
663
664    This function is appropriate for users who with to create a temporary file and:
665    1) Have control over when and whether the file is deleted.
666    2) Have the name of the file indicate information such as program name, function name, line, pid, etc.
667    This can be an aid in debugging, cleanup, etc.
668
669    The dir path portion of the file path will be /tmp/<username>/.  This function will create this directory
670    if it doesn't already exist.
671
672    This function will NOT create the file.  The file will NOT automatically get deleted.  It is the
673    responsibility of the caller to dispose of it.
674
675    Example:
676
677    pgm123.py is run by user 'joe'.  It calls func1 which contains this code:
678
679    temp_file_path = create_temp_file_path(suffix='suffix1')
680    print_var(temp_file_path)
681
682    Output:
683
684    temp_file_path:                 /tmp/joe/pgm123.py:func1:line_55:pid_8199:831848:suffix1
685
686    Description of argument(s):
687    delim                           A delimiter to be used to separate the sub-components of the file name.
688    suffix                          A suffix to include as the last sub-component of the file name.
689    """
690
691    temp_dir_path = "/tmp/" + username() + "/"
692    try:
693        os.mkdir(temp_dir_path)
694    except FileExistsError:
695        pass
696
697    callers_stack_frame = inspect.stack()[1]
698    file_name_elements = \
699        [
700            gp.pgm_name, callers_stack_frame.function, "line_" + str(callers_stack_frame.lineno),
701            "pid_" + str(os.getpid()), str(random.randint(0, 1000000)), suffix
702        ]
703    temp_file_name = delim.join(file_name_elements)
704
705    temp_file_path = temp_dir_path + temp_file_name
706
707    return temp_file_path
708
709
710def pause(message="Hit enter to continue..."):
711    r"""
712    Print the message, with time stamp, and pause until the user hits enter.
713
714    Description of argument(s):
715    message                         The message to be printed to stdout.
716    """
717    gp.print_time(message)
718    try:
719        input()
720    except SyntaxError:
721        pass
722
723    return
724