1#!/usr/bin/env python3 2 3r""" 4This module provides many valuable functions such as my_parm_file. 5""" 6 7import collections 8import errno 9import inspect 10import json 11import os 12import random 13import shutil 14 15# sys and os are needed to get the program dir path and program name. 16import sys 17import time 18 19try: 20 import ConfigParser 21except ImportError: 22 import configparser 23try: 24 import StringIO 25except ImportError: 26 import io 27 28import re 29import socket 30import tempfile 31 32try: 33 import psutil 34 35 psutil_imported = True 36except ImportError: 37 psutil_imported = False 38 39import gen_cmd as gc 40import gen_print as gp 41 42robot_env = gp.robot_env 43if robot_env: 44 from robot.libraries.BuiltIn import BuiltIn 45 from robot.utils import DotDict 46 47 48def add_trailing_slash(dir_path): 49 r""" 50 Add a trailing slash to the directory path if it doesn't already have one 51 and return it. 52 53 Description of arguments: 54 dir_path A directory path. 55 """ 56 57 return os.path.normpath(dir_path) + os.path.sep 58 59 60def makedirs(path, mode=0o777, quiet=None): 61 r""" 62 Call os.makedirs with the caller's arguments. 63 64 This function offers 2 advantages over the base os.makedirs function: 65 1) It will not fail if the directory already exists. 66 2) It will print an "Issuing: os.makedirs" message. 67 68 Description of argument(s): 69 path The path containing the directories to be created. 70 mode The mode or permissions to be granted to the created directories. 71 quiet Indicates whether this function should run the print_issuing() function. 72 """ 73 quiet = int(dft(quiet, gp.get_stack_var("quiet", 0))) 74 gp.qprint_issuing("os.makedirs('" + path + "', mode=" + oct(mode) + ")") 75 try: 76 os.makedirs(path, mode) 77 except OSError: 78 pass 79 80 81def rmtree(path, ignore_errors=False, onerror=None, quiet=None): 82 r""" 83 Call shutil.rmtree with the caller's arguments. 84 85 This function offers this advantage over the base function: 86 - It will print an "Issuing: shutil.rmtree" message. 87 88 Description of argument(s): 89 (All parms are passed directly to shutil.rmtree. See its prolog for details) 90 quiet Indicates whether this function should run the print_issuing() function. 91 """ 92 quiet = int(dft(quiet, gp.get_stack_var("quiet", 0))) 93 print_string = gp.sprint_executing(max_width=2000) 94 print_string = re.sub( 95 r"Executing: ", "Issuing: shutil.", print_string.rstrip("\n") 96 ) 97 gp.qprintn(re.sub(r", quiet[ ]?=.*", ")", print_string)) 98 shutil.rmtree(path, ignore_errors, onerror) 99 100 101def chdir(path, quiet=None): 102 r""" 103 Call os.chdir with the caller's arguments. 104 105 This function offers this advantage over the base os.chdir function: 106 - It will print an "Issuing: os.chdir" message. 107 108 Description of argument(s): 109 path The path of the directory to change to. 110 quiet Indicates whether this function should run the print_issuing() function. 111 """ 112 quiet = int(dft(quiet, gp.get_stack_var("quiet", 0))) 113 gp.qprint_issuing("os.chdir('" + path + "')") 114 os.chdir(path) 115 116 117def which(file_path): 118 r""" 119 Find the full path of an executable file and return it. 120 121 The PATH environment variable dictates the results of this function. 122 123 Description of arguments: 124 file_path The relative file path (e.g. "my_file" or "lib/my_file"). 125 """ 126 127 shell_rc, out_buf = gc.cmd_fnc_u( 128 "which " + file_path, quiet=1, print_output=0, show_err=0 129 ) 130 if shell_rc != 0: 131 error_message = ( 132 'Failed to find complete path for file "' + file_path + '".\n' 133 ) 134 error_message += gp.sprint_var(shell_rc, gp.hexa()) 135 error_message += out_buf 136 if robot_env: 137 BuiltIn().fail(gp.sprint_error(error_message)) 138 else: 139 gp.print_error_report(error_message) 140 return False 141 142 file_path = out_buf.rstrip("\n") 143 144 return file_path 145 146 147def add_path(new_path, path, position=0): 148 r""" 149 Add new_path to path, provided that path doesn't already contain new_path, and return the result. 150 151 Example: 152 If PATH has a value of "/bin/user:/lib/user". The following code: 153 154 PATH = add_path("/tmp/new_path", PATH) 155 156 will change PATH to "/tmp/new_path:/bin/user:/lib/user". 157 158 Description of argument(s): 159 new_path The path to be added. This function will strip the trailing slash. 160 path The path value to which the new_path should be added. 161 position The position in path where the new_path should be added. 0 means it 162 should be added to the beginning, 1 means add it as the 2nd item, etc. 163 sys.maxsize means it should be added to the end. 164 """ 165 166 path_list = list(filter(None, path.split(":"))) 167 new_path = new_path.rstrip("/") 168 if new_path not in path_list: 169 path_list.insert(int(position), new_path) 170 return ":".join(path_list) 171 172 173def dft(value, default): 174 r""" 175 Return default if value is None. Otherwise, return value. 176 177 This is really just shorthand as shown below. 178 179 dft(value, default) 180 181 vs 182 183 default if value is None else value 184 185 Description of arguments: 186 value The value to be returned. 187 default The default value to return if value is None. 188 """ 189 190 return default if value is None else value 191 192 193def get_mod_global(var_name, default=None, mod_name="__main__"): 194 r""" 195 Get module global variable value and return it. 196 197 If we are running in a robot environment, the behavior will default to 198 calling get_variable_value. 199 200 Description of arguments: 201 var_name The name of the variable whose value is sought. 202 default The value to return if the global does not exist. 203 mod_name The name of the module containing the global variable. 204 """ 205 206 if robot_env: 207 return BuiltIn().get_variable_value("${" + var_name + "}", default) 208 209 try: 210 module = sys.modules[mod_name] 211 except KeyError: 212 gp.print_error_report( 213 "Programmer error - The mod_name passed to" 214 + " this function is invalid:\n" 215 + gp.sprint_var(mod_name) 216 ) 217 raise ValueError("Programmer error.") 218 219 if default is None: 220 return getattr(module, var_name) 221 else: 222 return getattr(module, var_name, default) 223 224 225def global_default(var_value, default=0): 226 r""" 227 If var_value is not None, return it. Otherwise, return the global 228 variable of the same name, if it exists. If not, return default. 229 230 This is meant for use by functions needing help assigning dynamic default 231 values to their parms. Example: 232 233 def func1(parm1=None): 234 235 parm1 = global_default(parm1, 0) 236 237 Description of arguments: 238 var_value The value being evaluated. 239 default The value to be returned if var_value is None AND the global variable of 240 the same name does not exist. 241 """ 242 243 var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 244 245 return dft(var_value, get_mod_global(var_name, 0)) 246 247 248def set_mod_global(var_value, mod_name="__main__", var_name=None): 249 r""" 250 Set a global variable for a given module. 251 252 Description of arguments: 253 var_value The value to set in the variable. 254 mod_name The name of the module whose variable is to be set. 255 var_name The name of the variable to set. This defaults to the name of the 256 variable used for var_value when calling this function. 257 """ 258 259 try: 260 module = sys.modules[mod_name] 261 except KeyError: 262 gp.print_error_report( 263 "Programmer error - The mod_name passed to" 264 + " this function is invalid:\n" 265 + gp.sprint_var(mod_name) 266 ) 267 raise ValueError("Programmer error.") 268 269 if var_name is None: 270 var_name = gp.get_arg_name(None, 1, 2) 271 272 setattr(module, var_name, var_value) 273 274 275def my_parm_file(prop_file_path): 276 r""" 277 Read a properties file, put the keys/values into a dictionary and return the dictionary. 278 279 The properties file must have the following format: 280 var_name<= or :>var_value 281 Comment lines (those beginning with a "#") and blank lines are allowed and will be ignored. Leading and 282 trailing single or double quotes will be stripped from the value. E.g. 283 var1="This one" 284 Quotes are stripped so the resulting value for var1 is: 285 This one 286 287 Description of arguments: 288 prop_file_path The caller should pass the path to the properties file. 289 """ 290 291 # ConfigParser expects at least one section header in the file (or you get 292 # ConfigParser.MissingSectionHeaderError). Properties files don't need those so I'll write a dummy 293 # section header. 294 295 try: 296 string_file = StringIO.StringIO() 297 except NameError: 298 string_file = io.StringIO() 299 300 # Write the dummy section header to the string file. 301 string_file.write("[dummysection]\n") 302 # Write the entire contents of the properties file to the string file. 303 string_file.write(open(prop_file_path).read()) 304 # Rewind the string file. 305 string_file.seek(0, os.SEEK_SET) 306 307 # Create the ConfigParser object. 308 try: 309 config_parser = ConfigParser() 310 except NameError: 311 config_parser = configparser.ConfigParser(strict=False) 312 # Make the property names case-sensitive. 313 config_parser.optionxform = str 314 # Read the properties from the string file. 315 config_parser.read_file(string_file) 316 # Return the properties as a dictionary. 317 if robot_env: 318 return DotDict(config_parser.items("dummysection")) 319 else: 320 return collections.OrderedDict(config_parser.items("dummysection")) 321 322 323def file_to_list(file_path, newlines=0, comments=1, trim=0): 324 r""" 325 Return the contents of a file as a list. Each element of the resulting 326 list is one line from the file. 327 328 Description of arguments: 329 file_path The path to the file (relative or absolute). 330 newlines Include newlines from the file in the results. 331 comments Include comment lines and blank lines in the results. Comment lines are 332 any that begin with 0 or more spaces followed by the pound sign ("#"). 333 trim Trim white space from the beginning and end of each line. 334 """ 335 336 lines = [] 337 file = open(file_path) 338 for line in file: 339 if not comments: 340 if re.match(r"[ ]*#|^$", line): 341 continue 342 if not newlines: 343 line = line.rstrip("\n") 344 if trim: 345 line = line.strip() 346 lines.append(line) 347 file.close() 348 349 return lines 350 351 352def file_to_str(*args, **kwargs): 353 r""" 354 Return the contents of a file as a string. 355 356 Description of arguments: 357 See file_to_list defined above for description of arguments. 358 """ 359 360 return "\n".join(file_to_list(*args, **kwargs)) 361 362 363def append_file(file_path, buffer): 364 r""" 365 Append the data in buffer to the file named in file_path. 366 367 Description of argument(s): 368 file_path The path to a file (e.g. "/tmp/root/file1"). 369 buffer The buffer of data to be written to the file (e.g. "this and that"). 370 """ 371 372 with open(file_path, "a") as file: 373 file.write(buffer) 374 375 376def return_path_list(): 377 r""" 378 This function will split the PATH environment variable into a PATH_LIST and return it. Each element in 379 the list will be normalized and have a trailing slash added. 380 """ 381 382 PATH_LIST = os.environ["PATH"].split(":") 383 PATH_LIST = [os.path.normpath(path) + os.sep for path in PATH_LIST] 384 385 return PATH_LIST 386 387 388def escape_bash_quotes(buffer): 389 r""" 390 Escape quotes in string and return it. 391 392 The escape style implemented will be for use on the bash command line. 393 394 Example: 395 That's all. 396 397 Result: 398 That'\''s all. 399 400 The result may then be single quoted on a bash command. Example: 401 402 echo 'That'\''s all.' 403 404 Description of argument(s): 405 buffer The string whose quotes are to be escaped. 406 """ 407 408 return re.sub("'", "'\\''", buffer) 409 410 411def quote_bash_parm(parm): 412 r""" 413 Return the bash command line parm with single quotes if they are needed. 414 415 Description of arguments: 416 parm The string to be quoted. 417 """ 418 419 # If any of these characters are found in the parm string, then the string should be quoted. This list 420 # is by no means complete and should be expanded as needed by the developer of this function. 421 # Spaces 422 # Single or double quotes. 423 # Bash variables (therefore, any string with a "$" may need quoting). 424 # Glob characters: *, ?, [] 425 # Extended Glob characters: +, @, ! 426 # Bash brace expansion: {} 427 # Tilde expansion: ~ 428 # Piped commands: | 429 # Bash re-direction: >, < 430 bash_special_chars = set(" '\"$*?[]+@!{}~|><") 431 432 if any((char in bash_special_chars) for char in parm): 433 return "'" + escape_bash_quotes(parm) + "'" 434 435 if parm == "": 436 parm = "''" 437 438 return parm 439 440 441def get_host_name_ip(host=None, short_name=0): 442 r""" 443 Get the host name and the IP address for the given host and return them as a tuple. 444 445 Description of argument(s): 446 host The host name or IP address to be obtained. 447 short_name Include the short host name in the returned tuple, i.e. return host, ip 448 and short_host. 449 """ 450 451 if not host: 452 host = socket.gethostname() 453 host_name = socket.getfqdn(host) 454 try: 455 host_ip = get_first_host_addr(host) 456 except socket.gaierror as my_gaierror: 457 message = ( 458 "Unable to obtain the host name for the following host:" 459 + "\n" 460 + gp.sprint_var(host) 461 ) 462 gp.print_error_report(message) 463 raise my_gaierror 464 465 if short_name: 466 host_short_name = host_name.split(".")[0] 467 return host_name, host_ip, host_short_name 468 else: 469 return host_name, host_ip 470 471 472def get_first_host_addr(host): 473 # Prefer IPv4 474 addr_infos = sorted( 475 socket.getaddrinfo(host, 443, proto=socket.IPPROTO_TCP), 476 key=lambda x: x[0], 477 ) 478 # socket.getaddrinfo returns a list of 5-tuples with the 479 # following structure: 480 # (family, type, proto, canonname, sockaddr) 481 # sockaddr structure is (ip, port) 482 # See: https://docs.python.org/3/library/socket.html#socket.getaddrinfo 483 # Take the first tuple for the first IP, 484 # then its 4th element - sockaddr and IP from it 485 return addr_infos[0][4][0] 486 487 488def pid_active(pid): 489 r""" 490 Return true if pid represents an active pid and false otherwise. 491 492 Description of argument(s): 493 pid The pid whose status is being sought. 494 """ 495 496 try: 497 os.kill(int(pid), 0) 498 except OSError as err: 499 if err.errno == errno.ESRCH: 500 # ESRCH == No such process 501 return False 502 elif err.errno == errno.EPERM: 503 # EPERM clearly means there's a process to deny access to 504 return True 505 else: 506 # According to "man 2 kill" possible error values are 507 # (EINVAL, EPERM, ESRCH) 508 raise 509 510 return True 511 512 513def to_signed(number, bit_width=None): 514 r""" 515 Convert number to a signed number and return the result. 516 517 Examples: 518 519 With the following code: 520 521 var1 = 0xfffffffffffffff1 522 print_var(var1) 523 print_var(var1, hexa()) 524 var1 = to_signed(var1) 525 print_var(var1) 526 print_var(var1, hexa()) 527 528 The following is written to stdout: 529 var1: 18446744073709551601 530 var1: 0x00000000fffffffffffffff1 531 var1: -15 532 var1: 0xfffffffffffffff1 533 534 The same code but with var1 set to 0x000000000000007f produces the following: 535 var1: 127 536 var1: 0x000000000000007f 537 var1: 127 538 var1: 0x000000000000007f 539 540 Description of argument(s): 541 number The number to be converted. 542 bit_width The number of bits that defines a complete hex value. Typically, this 543 would be a multiple of 32. 544 """ 545 546 if bit_width is None: 547 try: 548 bit_width = gp.bit_length(long(sys.maxsize)) + 1 549 except NameError: 550 bit_width = gp.bit_length(int(sys.maxsize)) + 1 551 552 if number < 0: 553 return number 554 neg_bit_mask = 2 ** (bit_width - 1) 555 if number & neg_bit_mask: 556 return ((2**bit_width) - number) * -1 557 else: 558 return number 559 560 561def get_child_pids(quiet=1): 562 r""" 563 Get and return a list of pids representing all first-generation processes that are the children of the 564 current process. 565 566 Example: 567 568 children = get_child_pids() 569 print_var(children) 570 571 Output: 572 children: 573 children[0]: 9123 574 575 Description of argument(s): 576 quiet Display output to stdout detailing how this child pids are obtained. 577 """ 578 579 if psutil_imported: 580 # If "import psutil" worked, find child pids using psutil. 581 current_process = psutil.Process() 582 return [x.pid for x in current_process.children(recursive=False)] 583 else: 584 # Otherwise, find child pids using shell commands. 585 print_output = not quiet 586 587 ps_cmd_buf = ( 588 "ps --no-headers --ppid " + str(os.getpid()) + " -o pid,args" 589 ) 590 # Route the output of ps to a temporary file for later grepping. Avoid using " | grep" in the ps 591 # command string because it creates yet another process which is of no interest to the caller. 592 temp = tempfile.NamedTemporaryFile() 593 temp_file_path = temp.name 594 gc.shell_cmd( 595 ps_cmd_buf + " > " + temp_file_path, print_output=print_output 596 ) 597 # Sample contents of the temporary file: 598 # 30703 sleep 2 599 # 30795 /bin/bash -c ps --no-headers --ppid 30672 -o pid,args > /tmp/tmpqqorWY 600 # Use egrep to exclude the "ps" process itself from the results collected with the prior shell_cmd 601 # invocation. Only the other children are of interest to the caller. Use cut on the grep results to 602 # obtain only the pid column. 603 rc, output = gc.shell_cmd( 604 "egrep -v '" 605 + re.escape(ps_cmd_buf) 606 + "' " 607 + temp_file_path 608 + " | cut -c1-5", 609 print_output=print_output, 610 ) 611 # Split the output buffer by line into a list. Strip each element of extra spaces and convert each 612 # element to an integer. 613 return map(int, map(str.strip, filter(None, output.split("\n")))) 614 615 616def json_loads_multiple(buffer): 617 r""" 618 Convert the contents of the buffer to a JSON array, run json.loads() on it and return the result. 619 620 The buffer is expected to contain one or more JSON objects. 621 622 Description of argument(s): 623 buffer A string containing several JSON objects. 624 """ 625 626 # Any line consisting of just "}", which indicates the end of an object, should have a comma appended. 627 regex = "([\\r\\n])[\\}]([\\r\\n])" 628 buffer = re.sub(regex, "\\1},\\2", buffer, 1) 629 # Remove the comma from after the final object and place the whole buffer inside square brackets. 630 buffer = "[" + re.sub(",([\r\n])$", "\\1}", buffer, 1) + "]" 631 if gp.robot_env: 632 return json.loads(buffer, object_pairs_hook=DotDict) 633 else: 634 return json.loads(buffer, object_pairs_hook=collections.OrderedDict) 635 636 637def file_date_time_stamp(): 638 r""" 639 Return a date/time stamp in the following format: yymmdd.HHMMSS 640 641 This value is suitable for including in file names. Example file1.181001.171716.status 642 """ 643 644 return time.strftime("%y%m%d.%H%M%S", time.localtime(time.time())) 645 646 647def get_function_stack(): 648 r""" 649 Return a list of all the function names currently in the call stack. 650 651 This function's name will be at offset 0. This function's caller's name will be at offset 1 and so on. 652 """ 653 654 return [str(stack_frame[3]) for stack_frame in inspect.stack()] 655 656 657def username(): 658 r""" 659 Return the username for the current process. 660 """ 661 662 username = os.environ.get("USER", "") 663 if username != "": 664 return username 665 user_num = str(os.geteuid()) 666 try: 667 username = os.getlogin() 668 except OSError: 669 if user_num == "0": 670 username = "root" 671 else: 672 username = "?" 673 674 return username 675 676 677def version_tuple(version): 678 r""" 679 Convert the version string to a tuple and return it. 680 681 Description of argument(s): 682 version A version string whose format is "n[.n]" (e.g. "3.6.3", "3", etc.). 683 """ 684 685 return tuple(map(int, (version.split(".")))) 686 687 688def get_python_version(): 689 r""" 690 Get and return the python version. 691 """ 692 693 sys_version = sys.version 694 # Strip out any revision code data (e.g. "3.6.3rc1" will become "3.6.3"). 695 sys_version = re.sub("rc[^ ]+", "", sys_version).split(" ")[0] 696 # Remove any non-numerics, etc. (e.g. "2.7.15+" becomes ""2.7.15"). 697 return re.sub("[^0-9\\.]", "", sys_version) 698 699 700python_version = version_tuple(get_python_version()) 701ordered_dict_version = version_tuple("3.6") 702 703 704def create_temp_file_path(delim=":", suffix=""): 705 r""" 706 Create a temporary file path and return it. 707 708 This function is appropriate for users who with to create a temporary file and: 709 1) Have control over when and whether the file is deleted. 710 2) Have the name of the file indicate information such as program name, function name, line, pid, etc. 711 This can be an aid in debugging, cleanup, etc. 712 713 The dir path portion of the file path will be /tmp/<username>/. This function will create this directory 714 if it doesn't already exist. 715 716 This function will NOT create the file. The file will NOT automatically get deleted. It is the 717 responsibility of the caller to dispose of it. 718 719 Example: 720 721 pgm123.py is run by user 'joe'. It calls func1 which contains this code: 722 723 temp_file_path = create_temp_file_path(suffix='suffix1') 724 print_var(temp_file_path) 725 726 Output: 727 728 temp_file_path: /tmp/joe/pgm123.py:func1:line_55:pid_8199:831848:suffix1 729 730 Description of argument(s): 731 delim A delimiter to be used to separate the sub-components of the file name. 732 suffix A suffix to include as the last sub-component of the file name. 733 """ 734 735 temp_dir_path = "/tmp/" + username() + "/" 736 try: 737 os.mkdir(temp_dir_path) 738 except FileExistsError: 739 pass 740 741 callers_stack_frame = inspect.stack()[1] 742 file_name_elements = [ 743 gp.pgm_name, 744 callers_stack_frame.function, 745 "line_" + str(callers_stack_frame.lineno), 746 "pid_" + str(os.getpid()), 747 str(random.randint(0, 1000000)), 748 suffix, 749 ] 750 temp_file_name = delim.join(file_name_elements) 751 752 temp_file_path = temp_dir_path + temp_file_name 753 754 return temp_file_path 755 756 757def pause(message="Hit enter to continue..."): 758 r""" 759 Print the message, with time stamp, and pause until the user hits enter. 760 761 Description of argument(s): 762 message The message to be printed to stdout. 763 """ 764 gp.print_time(message) 765 try: 766 input() 767 except SyntaxError: 768 pass 769 770 return 771