1#!/usr/bin/env python3 2 3r""" 4This module provides many valuable functions such as my_parm_file. 5""" 6 7import collections 8import errno 9import inspect 10import json 11import os 12import random 13import shutil 14 15# sys and os are needed to get the program dir path and program name. 16import sys 17import time 18 19try: 20 import ConfigParser 21except ImportError: 22 import configparser 23try: 24 import StringIO 25except ImportError: 26 import io 27 28import re 29import socket 30import tempfile 31 32try: 33 import psutil 34 35 psutil_imported = True 36except ImportError: 37 psutil_imported = False 38 39import gen_cmd as gc 40import gen_print as gp 41 42robot_env = gp.robot_env 43if robot_env: 44 from robot.libraries.BuiltIn import BuiltIn 45 from robot.utils import DotDict 46 47 48def add_trailing_slash(dir_path): 49 r""" 50 Add a trailing slash to the directory path if it doesn't already have one 51 and return it. 52 53 Description of arguments: 54 dir_path A directory path. 55 """ 56 57 return os.path.normpath(dir_path) + os.path.sep 58 59 60def makedirs(path, mode=0o777, quiet=None): 61 r""" 62 Call os.makedirs with the caller's arguments. 63 64 This function offers 2 advantages over the base os.makedirs function: 65 1) It will not fail if the directory already exists. 66 2) It will print an "Issuing: os.makedirs" message. 67 68 Description of argument(s): 69 path The path containing the directories to be created. 70 mode The mode or permissions to be granted to the created directories. 71 quiet Indicates whether this function should run the print_issuing() function. 72 """ 73 quiet = int(dft(quiet, gp.get_stack_var("quiet", 0))) 74 gp.qprint_issuing("os.makedirs('" + path + "', mode=" + oct(mode) + ")") 75 try: 76 os.makedirs(path, mode) 77 except OSError: 78 pass 79 80 81def rmtree(path, ignore_errors=False, onerror=None, quiet=None): 82 r""" 83 Call shutil.rmtree with the caller's arguments. 84 85 This function offers this advantage over the base function: 86 - It will print an "Issuing: shutil.rmtree" message. 87 88 Description of argument(s): 89 (All parms are passed directly to shutil.rmtree. See its prolog for details) 90 quiet Indicates whether this function should run the print_issuing() function. 91 """ 92 quiet = int(dft(quiet, gp.get_stack_var("quiet", 0))) 93 print_string = gp.sprint_executing(max_width=2000) 94 print_string = re.sub( 95 r"Executing: ", "Issuing: shutil.", print_string.rstrip("\n") 96 ) 97 gp.qprintn(re.sub(r", quiet[ ]?=.*", ")", print_string)) 98 shutil.rmtree(path, ignore_errors, onerror) 99 100 101def chdir(path, quiet=None): 102 r""" 103 Call os.chdir with the caller's arguments. 104 105 This function offers this advantage over the base os.chdir function: 106 - It will print an "Issuing: os.chdir" message. 107 108 Description of argument(s): 109 path The path of the directory to change to. 110 quiet Indicates whether this function should run the print_issuing() function. 111 """ 112 quiet = int(dft(quiet, gp.get_stack_var("quiet", 0))) 113 gp.qprint_issuing("os.chdir('" + path + "')") 114 os.chdir(path) 115 116 117def which(file_path): 118 r""" 119 Find the full path of an executable file and return it. 120 121 The PATH environment variable dictates the results of this function. 122 123 Description of arguments: 124 file_path The relative file path (e.g. "my_file" or "lib/my_file"). 125 """ 126 127 shell_rc, out_buf = gc.cmd_fnc_u( 128 "which " + file_path, quiet=1, print_output=0, show_err=0 129 ) 130 if shell_rc != 0: 131 error_message = ( 132 'Failed to find complete path for file "' + file_path + '".\n' 133 ) 134 error_message += gp.sprint_var(shell_rc, gp.hexa()) 135 error_message += out_buf 136 if robot_env: 137 BuiltIn().fail(gp.sprint_error(error_message)) 138 else: 139 gp.print_error_report(error_message) 140 return False 141 142 file_path = out_buf.rstrip("\n") 143 144 return file_path 145 146 147def add_path(new_path, path, position=0): 148 r""" 149 Add new_path to path, provided that path doesn't already contain new_path, and return the result. 150 151 Example: 152 If PATH has a value of "/bin/user:/lib/user". The following code: 153 154 PATH = add_path("/tmp/new_path", PATH) 155 156 will change PATH to "/tmp/new_path:/bin/user:/lib/user". 157 158 Description of argument(s): 159 new_path The path to be added. This function will strip the trailing slash. 160 path The path value to which the new_path should be added. 161 position The position in path where the new_path should be added. 0 means it 162 should be added to the beginning, 1 means add it as the 2nd item, etc. 163 sys.maxsize means it should be added to the end. 164 """ 165 166 path_list = list(filter(None, path.split(":"))) 167 new_path = new_path.rstrip("/") 168 if new_path not in path_list: 169 path_list.insert(int(position), new_path) 170 return ":".join(path_list) 171 172 173def dft(value, default): 174 r""" 175 Return default if value is None. Otherwise, return value. 176 177 This is really just shorthand as shown below. 178 179 dft(value, default) 180 181 vs 182 183 default if value is None else value 184 185 Description of arguments: 186 value The value to be returned. 187 default The default value to return if value is None. 188 """ 189 190 return default if value is None else value 191 192 193def get_mod_global(var_name, default=None, mod_name="__main__"): 194 r""" 195 Get module global variable value and return it. 196 197 If we are running in a robot environment, the behavior will default to 198 calling get_variable_value. 199 200 Description of arguments: 201 var_name The name of the variable whose value is sought. 202 default The value to return if the global does not exist. 203 mod_name The name of the module containing the global variable. 204 """ 205 206 if robot_env: 207 return BuiltIn().get_variable_value("${" + var_name + "}", default) 208 209 try: 210 module = sys.modules[mod_name] 211 except KeyError: 212 gp.print_error_report( 213 "Programmer error - The mod_name passed to" 214 + " this function is invalid:\n" 215 + gp.sprint_var(mod_name) 216 ) 217 raise ValueError("Programmer error.") 218 219 if default is None: 220 return getattr(module, var_name) 221 else: 222 return getattr(module, var_name, default) 223 224 225def global_default(var_value, default=0): 226 r""" 227 If var_value is not None, return it. Otherwise, return the global 228 variable of the same name, if it exists. If not, return default. 229 230 This is meant for use by functions needing help assigning dynamic default 231 values to their parms. Example: 232 233 def func1(parm1=None): 234 235 parm1 = global_default(parm1, 0) 236 237 Description of arguments: 238 var_value The value being evaluated. 239 default The value to be returned if var_value is None AND the global variable of 240 the same name does not exist. 241 """ 242 243 var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 244 245 return dft(var_value, get_mod_global(var_name, 0)) 246 247 248def set_mod_global(var_value, mod_name="__main__", var_name=None): 249 r""" 250 Set a global variable for a given module. 251 252 Description of arguments: 253 var_value The value to set in the variable. 254 mod_name The name of the module whose variable is to be set. 255 var_name The name of the variable to set. This defaults to the name of the 256 variable used for var_value when calling this function. 257 """ 258 259 try: 260 module = sys.modules[mod_name] 261 except KeyError: 262 gp.print_error_report( 263 "Programmer error - The mod_name passed to" 264 + " this function is invalid:\n" 265 + gp.sprint_var(mod_name) 266 ) 267 raise ValueError("Programmer error.") 268 269 if var_name is None: 270 var_name = gp.get_arg_name(None, 1, 2) 271 272 setattr(module, var_name, var_value) 273 274 275def my_parm_file(prop_file_path): 276 r""" 277 Read a properties file, put the keys/values into a dictionary and return the dictionary. 278 279 The properties file must have the following format: 280 var_name<= or :>var_value 281 Comment lines (those beginning with a "#") and blank lines are allowed and will be ignored. Leading and 282 trailing single or double quotes will be stripped from the value. E.g. 283 var1="This one" 284 Quotes are stripped so the resulting value for var1 is: 285 This one 286 287 Description of arguments: 288 prop_file_path The caller should pass the path to the properties file. 289 """ 290 291 # ConfigParser expects at least one section header in the file (or you get 292 # ConfigParser.MissingSectionHeaderError). Properties files don't need those so I'll write a dummy 293 # section header. 294 295 try: 296 string_file = StringIO.StringIO() 297 except NameError: 298 string_file = io.StringIO() 299 300 # Write the dummy section header to the string file. 301 string_file.write("[dummysection]\n") 302 # Write the entire contents of the properties file to the string file. 303 string_file.write(open(prop_file_path).read()) 304 # Rewind the string file. 305 string_file.seek(0, os.SEEK_SET) 306 307 # Create the ConfigParser object. 308 try: 309 config_parser = ConfigParser.ConfigParser() 310 except NameError: 311 config_parser = configparser.ConfigParser(strict=False) 312 # Make the property names case-sensitive. 313 config_parser.optionxform = str 314 # Read the properties from the string file. 315 config_parser.readfp(string_file) 316 # Return the properties as a dictionary. 317 if robot_env: 318 return DotDict(config_parser.items("dummysection")) 319 else: 320 return collections.OrderedDict(config_parser.items("dummysection")) 321 322 323def file_to_list(file_path, newlines=0, comments=1, trim=0): 324 r""" 325 Return the contents of a file as a list. Each element of the resulting 326 list is one line from the file. 327 328 Description of arguments: 329 file_path The path to the file (relative or absolute). 330 newlines Include newlines from the file in the results. 331 comments Include comment lines and blank lines in the results. Comment lines are 332 any that begin with 0 or more spaces followed by the pound sign ("#"). 333 trim Trim white space from the beginning and end of each line. 334 """ 335 336 lines = [] 337 file = open(file_path) 338 for line in file: 339 if not comments: 340 if re.match(r"[ ]*#|^$", line): 341 continue 342 if not newlines: 343 line = line.rstrip("\n") 344 if trim: 345 line = line.strip() 346 lines.append(line) 347 file.close() 348 349 return lines 350 351 352def file_to_str(*args, **kwargs): 353 r""" 354 Return the contents of a file as a string. 355 356 Description of arguments: 357 See file_to_list defined above for description of arguments. 358 """ 359 360 return "\n".join(file_to_list(*args, **kwargs)) 361 362 363def append_file(file_path, buffer): 364 r""" 365 Append the data in buffer to the file named in file_path. 366 367 Description of argument(s): 368 file_path The path to a file (e.g. "/tmp/root/file1"). 369 buffer The buffer of data to be written to the file (e.g. "this and that"). 370 """ 371 372 with open(file_path, "a") as file: 373 file.write(buffer) 374 375 376def return_path_list(): 377 r""" 378 This function will split the PATH environment variable into a PATH_LIST and return it. Each element in 379 the list will be normalized and have a trailing slash added. 380 """ 381 382 PATH_LIST = os.environ["PATH"].split(":") 383 PATH_LIST = [os.path.normpath(path) + os.sep for path in PATH_LIST] 384 385 return PATH_LIST 386 387 388def escape_bash_quotes(buffer): 389 r""" 390 Escape quotes in string and return it. 391 392 The escape style implemented will be for use on the bash command line. 393 394 Example: 395 That's all. 396 397 Result: 398 That'\''s all. 399 400 The result may then be single quoted on a bash command. Example: 401 402 echo 'That'\''s all.' 403 404 Description of argument(s): 405 buffer The string whose quotes are to be escaped. 406 """ 407 408 return re.sub("'", "'\\''", buffer) 409 410 411def quote_bash_parm(parm): 412 r""" 413 Return the bash command line parm with single quotes if they are needed. 414 415 Description of arguments: 416 parm The string to be quoted. 417 """ 418 419 # If any of these characters are found in the parm string, then the string should be quoted. This list 420 # is by no means complete and should be expanded as needed by the developer of this function. 421 # Spaces 422 # Single or double quotes. 423 # Bash variables (therefore, any string with a "$" may need quoting). 424 # Glob characters: *, ?, [] 425 # Extended Glob characters: +, @, ! 426 # Bash brace expansion: {} 427 # Tilde expansion: ~ 428 # Piped commands: | 429 # Bash re-direction: >, < 430 bash_special_chars = set(" '\"$*?[]+@!{}~|><") 431 432 if any((char in bash_special_chars) for char in parm): 433 return "'" + escape_bash_quotes(parm) + "'" 434 435 if parm == "": 436 parm = "''" 437 438 return parm 439 440 441def get_host_name_ip(host=None, short_name=0): 442 r""" 443 Get the host name and the IP address for the given host and return them as a tuple. 444 445 Description of argument(s): 446 host The host name or IP address to be obtained. 447 short_name Include the short host name in the returned tuple, i.e. return host, ip 448 and short_host. 449 """ 450 451 host = dft(host, socket.gethostname()) 452 host_name = socket.getfqdn(host) 453 try: 454 host_ip = socket.gethostbyname(host) 455 except socket.gaierror as my_gaierror: 456 message = ( 457 "Unable to obtain the host name for the following host:" 458 + "\n" 459 + gp.sprint_var(host) 460 ) 461 gp.print_error_report(message) 462 raise my_gaierror 463 464 if short_name: 465 host_short_name = host_name.split(".")[0] 466 return host_name, host_ip, host_short_name 467 else: 468 return host_name, host_ip 469 470 471def pid_active(pid): 472 r""" 473 Return true if pid represents an active pid and false otherwise. 474 475 Description of argument(s): 476 pid The pid whose status is being sought. 477 """ 478 479 try: 480 os.kill(int(pid), 0) 481 except OSError as err: 482 if err.errno == errno.ESRCH: 483 # ESRCH == No such process 484 return False 485 elif err.errno == errno.EPERM: 486 # EPERM clearly means there's a process to deny access to 487 return True 488 else: 489 # According to "man 2 kill" possible error values are 490 # (EINVAL, EPERM, ESRCH) 491 raise 492 493 return True 494 495 496def to_signed(number, bit_width=None): 497 r""" 498 Convert number to a signed number and return the result. 499 500 Examples: 501 502 With the following code: 503 504 var1 = 0xfffffffffffffff1 505 print_var(var1) 506 print_var(var1, hexa()) 507 var1 = to_signed(var1) 508 print_var(var1) 509 print_var(var1, hexa()) 510 511 The following is written to stdout: 512 var1: 18446744073709551601 513 var1: 0x00000000fffffffffffffff1 514 var1: -15 515 var1: 0xfffffffffffffff1 516 517 The same code but with var1 set to 0x000000000000007f produces the following: 518 var1: 127 519 var1: 0x000000000000007f 520 var1: 127 521 var1: 0x000000000000007f 522 523 Description of argument(s): 524 number The number to be converted. 525 bit_width The number of bits that defines a complete hex value. Typically, this 526 would be a multiple of 32. 527 """ 528 529 if bit_width is None: 530 try: 531 bit_width = gp.bit_length(long(sys.maxsize)) + 1 532 except NameError: 533 bit_width = gp.bit_length(int(sys.maxsize)) + 1 534 535 if number < 0: 536 return number 537 neg_bit_mask = 2 ** (bit_width - 1) 538 if number & neg_bit_mask: 539 return ((2**bit_width) - number) * -1 540 else: 541 return number 542 543 544def get_child_pids(quiet=1): 545 r""" 546 Get and return a list of pids representing all first-generation processes that are the children of the 547 current process. 548 549 Example: 550 551 children = get_child_pids() 552 print_var(children) 553 554 Output: 555 children: 556 children[0]: 9123 557 558 Description of argument(s): 559 quiet Display output to stdout detailing how this child pids are obtained. 560 """ 561 562 if psutil_imported: 563 # If "import psutil" worked, find child pids using psutil. 564 current_process = psutil.Process() 565 return [x.pid for x in current_process.children(recursive=False)] 566 else: 567 # Otherwise, find child pids using shell commands. 568 print_output = not quiet 569 570 ps_cmd_buf = ( 571 "ps --no-headers --ppid " + str(os.getpid()) + " -o pid,args" 572 ) 573 # Route the output of ps to a temporary file for later grepping. Avoid using " | grep" in the ps 574 # command string because it creates yet another process which is of no interest to the caller. 575 temp = tempfile.NamedTemporaryFile() 576 temp_file_path = temp.name 577 gc.shell_cmd( 578 ps_cmd_buf + " > " + temp_file_path, print_output=print_output 579 ) 580 # Sample contents of the temporary file: 581 # 30703 sleep 2 582 # 30795 /bin/bash -c ps --no-headers --ppid 30672 -o pid,args > /tmp/tmpqqorWY 583 # Use egrep to exclude the "ps" process itself from the results collected with the prior shell_cmd 584 # invocation. Only the other children are of interest to the caller. Use cut on the grep results to 585 # obtain only the pid column. 586 rc, output = gc.shell_cmd( 587 "egrep -v '" 588 + re.escape(ps_cmd_buf) 589 + "' " 590 + temp_file_path 591 + " | cut -c1-5", 592 print_output=print_output, 593 ) 594 # Split the output buffer by line into a list. Strip each element of extra spaces and convert each 595 # element to an integer. 596 return map(int, map(str.strip, filter(None, output.split("\n")))) 597 598 599def json_loads_multiple(buffer): 600 r""" 601 Convert the contents of the buffer to a JSON array, run json.loads() on it and return the result. 602 603 The buffer is expected to contain one or more JSON objects. 604 605 Description of argument(s): 606 buffer A string containing several JSON objects. 607 """ 608 609 # Any line consisting of just "}", which indicates the end of an object, should have a comma appended. 610 regex = "([\\r\\n])[\\}]([\\r\\n])" 611 buffer = re.sub(regex, "\\1},\\2", buffer, 1) 612 # Remove the comma from after the final object and place the whole buffer inside square brackets. 613 buffer = "[" + re.sub(",([\r\n])$", "\\1}", buffer, 1) + "]" 614 if gp.robot_env: 615 return json.loads(buffer, object_pairs_hook=DotDict) 616 else: 617 return json.loads(buffer, object_pairs_hook=collections.OrderedDict) 618 619 620def file_date_time_stamp(): 621 r""" 622 Return a date/time stamp in the following format: yymmdd.HHMMSS 623 624 This value is suitable for including in file names. Example file1.181001.171716.status 625 """ 626 627 return time.strftime("%y%m%d.%H%M%S", time.localtime(time.time())) 628 629 630def get_function_stack(): 631 r""" 632 Return a list of all the function names currently in the call stack. 633 634 This function's name will be at offset 0. This function's caller's name will be at offset 1 and so on. 635 """ 636 637 return [str(stack_frame[3]) for stack_frame in inspect.stack()] 638 639 640def username(): 641 r""" 642 Return the username for the current process. 643 """ 644 645 username = os.environ.get("USER", "") 646 if username != "": 647 return username 648 user_num = str(os.geteuid()) 649 try: 650 username = os.getlogin() 651 except OSError: 652 if user_num == "0": 653 username = "root" 654 else: 655 username = "?" 656 657 return username 658 659 660def version_tuple(version): 661 r""" 662 Convert the version string to a tuple and return it. 663 664 Description of argument(s): 665 version A version string whose format is "n[.n]" (e.g. "3.6.3", "3", etc.). 666 """ 667 668 return tuple(map(int, (version.split(".")))) 669 670 671def get_python_version(): 672 r""" 673 Get and return the python version. 674 """ 675 676 sys_version = sys.version 677 # Strip out any revision code data (e.g. "3.6.3rc1" will become "3.6.3"). 678 sys_version = re.sub("rc[^ ]+", "", sys_version).split(" ")[0] 679 # Remove any non-numerics, etc. (e.g. "2.7.15+" becomes ""2.7.15"). 680 return re.sub("[^0-9\\.]", "", sys_version) 681 682 683python_version = version_tuple(get_python_version()) 684ordered_dict_version = version_tuple("3.6") 685 686 687def create_temp_file_path(delim=":", suffix=""): 688 r""" 689 Create a temporary file path and return it. 690 691 This function is appropriate for users who with to create a temporary file and: 692 1) Have control over when and whether the file is deleted. 693 2) Have the name of the file indicate information such as program name, function name, line, pid, etc. 694 This can be an aid in debugging, cleanup, etc. 695 696 The dir path portion of the file path will be /tmp/<username>/. This function will create this directory 697 if it doesn't already exist. 698 699 This function will NOT create the file. The file will NOT automatically get deleted. It is the 700 responsibility of the caller to dispose of it. 701 702 Example: 703 704 pgm123.py is run by user 'joe'. It calls func1 which contains this code: 705 706 temp_file_path = create_temp_file_path(suffix='suffix1') 707 print_var(temp_file_path) 708 709 Output: 710 711 temp_file_path: /tmp/joe/pgm123.py:func1:line_55:pid_8199:831848:suffix1 712 713 Description of argument(s): 714 delim A delimiter to be used to separate the sub-components of the file name. 715 suffix A suffix to include as the last sub-component of the file name. 716 """ 717 718 temp_dir_path = "/tmp/" + username() + "/" 719 try: 720 os.mkdir(temp_dir_path) 721 except FileExistsError: 722 pass 723 724 callers_stack_frame = inspect.stack()[1] 725 file_name_elements = [ 726 gp.pgm_name, 727 callers_stack_frame.function, 728 "line_" + str(callers_stack_frame.lineno), 729 "pid_" + str(os.getpid()), 730 str(random.randint(0, 1000000)), 731 suffix, 732 ] 733 temp_file_name = delim.join(file_name_elements) 734 735 temp_file_path = temp_dir_path + temp_file_name 736 737 return temp_file_path 738 739 740def pause(message="Hit enter to continue..."): 741 r""" 742 Print the message, with time stamp, and pause until the user hits enter. 743 744 Description of argument(s): 745 message The message to be printed to stdout. 746 """ 747 gp.print_time(message) 748 try: 749 input() 750 except SyntaxError: 751 pass 752 753 return 754