1#!/usr/bin/env python 2 3r""" 4See class prolog below for details. 5""" 6 7import os 8import sys 9import yaml 10import time 11import platform 12from errno import EACCES, EPERM 13import subprocess 14from ssh_utility import SSHRemoteclient 15 16 17class FFDCCollector: 18 19 r""" 20 Sends commands from configuration file to the targeted system to collect log files. 21 Fetch and store generated files at the specified location. 22 23 """ 24 25 def __init__(self, 26 hostname, 27 username, 28 password, 29 ffdc_config, 30 location, 31 remote_type, 32 remote_protocol): 33 r""" 34 Description of argument(s): 35 36 hostname name/ip of the targeted (remote) system 37 username user on the targeted system with access to FFDC files 38 password password for user on targeted system 39 ffdc_config configuration file listing commands and files for FFDC 40 location where to store collected FFDC 41 remote_type os type of the remote host 42 43 """ 44 if self.verify_script_env(): 45 self.hostname = hostname 46 self.username = username 47 self.password = password 48 self.ffdc_config = ffdc_config 49 self.location = location 50 self.remote_client = None 51 self.ffdc_dir_path = "" 52 self.ffdc_prefix = "" 53 self.target_type = remote_type.upper() 54 self.remote_protocol = remote_protocol.upper() 55 self.start_time = 0 56 self.elapsed_time = '' 57 else: 58 sys.exit(-1) 59 60 # Load default or user define YAML configuration file. 61 with open(self.ffdc_config, 'r') as file: 62 self.ffdc_actions = yaml.load(file, Loader=yaml.FullLoader) 63 64 if self.target_type not in self.ffdc_actions.keys(): 65 print("\n\tERROR: %s is not listed in %s.\n\n" % (self.target_type, self.ffdc_config)) 66 sys.exit(-1) 67 68 def verify_script_env(self): 69 70 # Import to log version 71 import click 72 import paramiko 73 74 run_env_ok = True 75 76 redfishtool_version = self.run_redfishtool('-V').split(' ')[2].strip('\n') 77 ipmitool_version = self.run_ipmitool('-V').split(' ')[2] 78 79 print("\n\t---- Script host environment ----") 80 print("\t{:<10} {:<10}".format('Script hostname', os.uname()[1])) 81 print("\t{:<10} {:<10}".format('Script host os', platform.platform())) 82 print("\t{:<10} {:>10}".format('Python', platform.python_version())) 83 print("\t{:<10} {:>10}".format('PyYAML', yaml.__version__)) 84 print("\t{:<10} {:>10}".format('click', click.__version__)) 85 print("\t{:<10} {:>10}".format('paramiko', paramiko.__version__)) 86 print("\t{:<10} {:>9}".format('redfishtool', redfishtool_version)) 87 print("\t{:<10} {:>12}".format('ipmitool', ipmitool_version)) 88 89 if eval(yaml.__version__.replace('.', ',')) < (5, 4, 1): 90 print("\n\tERROR: Python or python packages do not meet minimum version requirement.") 91 print("\tERROR: PyYAML version 5.4.1 or higher is needed.\n") 92 run_env_ok = False 93 94 print("\t---- End script host environment ----") 95 return run_env_ok 96 97 def target_is_pingable(self): 98 r""" 99 Check if target system is ping-able. 100 101 """ 102 response = os.system("ping -c 1 -w 2 %s 2>&1 >/dev/null" % self.hostname) 103 if response == 0: 104 print("\n\t[Check] %s is ping-able.\t\t [OK]" % self.hostname) 105 return True 106 else: 107 print("\n>>>>>\tERROR: %s is not ping-able. FFDC collection aborted.\n" % self.hostname) 108 sys.exit(-1) 109 110 def collect_ffdc(self): 111 r""" 112 Initiate FFDC Collection depending on requested protocol. 113 114 """ 115 116 print("\n\t---- Start communicating with %s ----" % self.hostname) 117 self.start_time = time.time() 118 working_protocol_list = [] 119 if self.target_is_pingable(): 120 # Check supported protocol ping,ssh, redfish are working. 121 if self.ssh_to_target_system(): 122 working_protocol_list.append("SSH") 123 working_protocol_list.append("SCP") 124 125 # Redfish 126 if self.verify_redfish(): 127 working_protocol_list.append("REDFISH") 128 print("\n\t[Check] %s Redfish Service.\t\t [OK]" % self.hostname) 129 else: 130 print("\n\t[Check] %s Redfish Service.\t\t [NOT AVAILABLE]" % self.hostname) 131 132 if self.verify_ipmi(): 133 working_protocol_list.append("IPMI") 134 print("\n\t[Check] %s IPMI LAN Service.\t\t [OK]" % self.hostname) 135 else: 136 print("\n\t[Check] %s IPMI LAN Service.\t\t [NOT AVAILABLE]" % self.hostname) 137 138 # Verify top level directory exists for storage 139 self.validate_local_store(self.location) 140 print("\n\t---- Completed protocol pre-requisite check ----\n") 141 142 if ((self.remote_protocol not in working_protocol_list) and (self.remote_protocol != 'ALL')): 143 print("\n\tWorking protocol list: %s" % working_protocol_list) 144 print( 145 '>>>>>\tERROR: Requested protocol %s is not in working protocol list.\n' 146 % self.remote_protocol) 147 sys.exit(-1) 148 else: 149 self.generate_ffdc(working_protocol_list) 150 151 def ssh_to_target_system(self): 152 r""" 153 Open a ssh connection to targeted system. 154 155 """ 156 157 self.remoteclient = SSHRemoteclient(self.hostname, 158 self.username, 159 self.password) 160 161 self.remoteclient.ssh_remoteclient_login() 162 print("\n\t[Check] %s SSH connection established.\t [OK]" % self.hostname) 163 164 # Check scp connection. 165 # If scp connection fails, 166 # continue with FFDC generation but skip scp files to local host. 167 self.remoteclient.scp_connection() 168 return True 169 170 def generate_ffdc(self, working_protocol_list): 171 r""" 172 Determine actions based on remote host type 173 174 Description of argument(s): 175 working_protocol_list list of confirmed working protocols to connect to remote host. 176 """ 177 178 print("\n\t---- Executing commands on " + self.hostname + " ----") 179 print("\n\tWorking protocol list: %s" % working_protocol_list) 180 181 # Set prefix values for scp files and directory. 182 # Since the time stamp is at second granularity, these values are set here 183 # to be sure that all files for this run will have same timestamps 184 # and they will be saved in the same directory. 185 # self.location == local system for now 186 self.set_ffdc_defaults() 187 ffdc_actions = self.ffdc_actions 188 189 for machine_type in ffdc_actions.keys(): 190 if self.target_type != machine_type: 191 continue 192 193 print("\tSystem Type: %s" % machine_type) 194 for k, v in ffdc_actions[machine_type].items(): 195 196 if self.remote_protocol != ffdc_actions[machine_type][k]['PROTOCOL'][0] \ 197 and self.remote_protocol != 'ALL': 198 continue 199 200 if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'SSH' \ 201 or ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'SCP': 202 if 'SSH' in working_protocol_list \ 203 or 'SCP' in working_protocol_list: 204 self.protocol_ssh(ffdc_actions, machine_type, k) 205 else: 206 print("\n\tERROR: SSH or SCP is not available for %s." % self.hostname) 207 208 if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'REDFISH': 209 if 'REDFISH' in working_protocol_list: 210 self.protocol_redfish(ffdc_actions, machine_type, k) 211 else: 212 print("\n\tERROR: REDFISH is not available for %s." % self.hostname) 213 214 if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'IPMI': 215 if 'IPMI' in working_protocol_list: 216 self.protocol_ipmi(ffdc_actions, machine_type, k) 217 else: 218 print("\n\tERROR: IMPI is not available for %s." % self.hostname) 219 220 # Close network connection after collecting all files 221 self.elapsed_time = time.strftime("%H:%M:%S", time.gmtime(time.time() - self.start_time)) 222 self.remoteclient.ssh_remoteclient_disconnect() 223 224 def protocol_ssh(self, 225 ffdc_actions, 226 machine_type, 227 sub_type): 228 r""" 229 Perform actions using SSH and SCP protocols. 230 231 Description of argument(s): 232 ffdc_actions List of actions from ffdc_config.yaml. 233 machine_type OS Type of remote host. 234 sub_type Group type of commands. 235 """ 236 237 if sub_type == 'DUMP_LOGS': 238 self.group_copy(ffdc_actions[machine_type][sub_type]) 239 else: 240 self.collect_and_copy_ffdc(ffdc_actions[machine_type][sub_type]) 241 242 def protocol_redfish(self, 243 ffdc_actions, 244 machine_type, 245 sub_type): 246 r""" 247 Perform actions using Redfish protocol. 248 249 Description of argument(s): 250 ffdc_actions List of actions from ffdc_config.yaml. 251 machine_type OS Type of remote host. 252 sub_type Group type of commands. 253 """ 254 255 print("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'REDFISH')) 256 redfish_files_saved = [] 257 progress_counter = 0 258 list_of_URL = ffdc_actions[machine_type][sub_type]['URL'] 259 for index, each_url in enumerate(list_of_URL, start=0): 260 redfish_parm = '-u ' + self.username + ' -p ' + self.password + ' -r ' \ 261 + self.hostname + ' -S Always raw GET ' + each_url 262 263 result = self.run_redfishtool(redfish_parm) 264 if result: 265 try: 266 targ_file = self.get_file_list(ffdc_actions[machine_type][sub_type])[index] 267 except IndexError: 268 targ_file = each_url.split('/')[-1] 269 print("\n\t[WARN] Missing filename to store data from redfish URL %s." % each_url) 270 print("\t[WARN] Data will be stored in %s." % targ_file) 271 272 targ_file_with_path = (self.ffdc_dir_path 273 + self.ffdc_prefix 274 + targ_file) 275 276 # Creates a new file 277 with open(targ_file_with_path, 'w') as fp: 278 fp.write(result) 279 fp.close 280 redfish_files_saved.append(targ_file) 281 282 progress_counter += 1 283 self.print_progress(progress_counter) 284 285 print("\n\t[Run] Commands execution completed.\t\t [OK]") 286 287 for file in redfish_files_saved: 288 print("\n\t\tSuccessfully save file " + file + ".") 289 290 def protocol_ipmi(self, 291 ffdc_actions, 292 machine_type, 293 sub_type): 294 r""" 295 Perform actions using ipmitool over LAN protocol. 296 297 Description of argument(s): 298 ffdc_actions List of actions from ffdc_config.yaml. 299 machine_type OS Type of remote host. 300 sub_type Group type of commands. 301 """ 302 303 print("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'IPMI')) 304 ipmi_files_saved = [] 305 progress_counter = 0 306 list_of_cmd = self.get_command_list(ffdc_actions[machine_type][sub_type]) 307 for index, each_cmd in enumerate(list_of_cmd, start=0): 308 ipmi_parm = '-U ' + self.username + ' -P ' + self.password + ' -H ' \ 309 + self.hostname + ' ' + each_cmd 310 311 result = self.run_ipmitool(ipmi_parm) 312 if result: 313 try: 314 targ_file = self.get_file_list(ffdc_actions[machine_type][sub_type])[index] 315 except IndexError: 316 targ_file = each_cmd.split('/')[-1] 317 print("\n\t[WARN] Missing filename to store data from IPMI %s." % each_cmd) 318 print("\t[WARN] Data will be stored in %s." % targ_file) 319 320 targ_file_with_path = (self.ffdc_dir_path 321 + self.ffdc_prefix 322 + targ_file) 323 324 # Creates a new file 325 with open(targ_file_with_path, 'w') as fp: 326 fp.write(result) 327 fp.close 328 ipmi_files_saved.append(targ_file) 329 330 progress_counter += 1 331 self.print_progress(progress_counter) 332 333 print("\n\t[Run] Commands execution completed.\t\t [OK]") 334 335 for file in ipmi_files_saved: 336 print("\n\t\tSuccessfully save file " + file + ".") 337 338 def collect_and_copy_ffdc(self, 339 ffdc_actions_for_machine_type, 340 form_filename=False): 341 r""" 342 Send commands in ffdc_config file to targeted system. 343 344 Description of argument(s): 345 ffdc_actions_for_machine_type commands and files for the selected remote host type. 346 form_filename if true, pre-pend self.target_type to filename 347 """ 348 349 # Executing commands, , if any 350 self.ssh_execute_ffdc_commands(ffdc_actions_for_machine_type, 351 form_filename) 352 353 # Copying files 354 if self.remoteclient.scpclient: 355 print("\n\n\tCopying FFDC files from remote system %s.\n" % self.hostname) 356 357 # Retrieving files from target system 358 list_of_files = self.get_file_list(ffdc_actions_for_machine_type) 359 self.scp_ffdc(self.ffdc_dir_path, self.ffdc_prefix, form_filename, list_of_files) 360 else: 361 print("\n\n\tSkip copying FFDC files from remote system %s.\n" % self.hostname) 362 363 def get_command_list(self, 364 ffdc_actions_for_machine_type): 365 r""" 366 Fetch list of commands from configuration file 367 368 Description of argument(s): 369 ffdc_actions_for_machine_type commands and files for the selected remote host type. 370 """ 371 try: 372 list_of_commands = ffdc_actions_for_machine_type['COMMANDS'] 373 except KeyError: 374 list_of_commands = [] 375 return list_of_commands 376 377 def get_file_list(self, 378 ffdc_actions_for_machine_type): 379 r""" 380 Fetch list of commands from configuration file 381 382 Description of argument(s): 383 ffdc_actions_for_machine_type commands and files for the selected remote host type. 384 """ 385 try: 386 list_of_files = ffdc_actions_for_machine_type['FILES'] 387 except KeyError: 388 list_of_files = [] 389 return list_of_files 390 391 def ssh_execute_ffdc_commands(self, 392 ffdc_actions_for_machine_type, 393 form_filename=False): 394 r""" 395 Send commands in ffdc_config file to targeted system. 396 397 Description of argument(s): 398 ffdc_actions_for_machine_type commands and files for the selected remote host type. 399 form_filename if true, pre-pend self.target_type to filename 400 """ 401 print("\n\t[Run] Executing commands on %s using %s" 402 % (self.hostname, ffdc_actions_for_machine_type['PROTOCOL'][0])) 403 404 list_of_commands = self.get_command_list(ffdc_actions_for_machine_type) 405 # If command list is empty, returns 406 if not list_of_commands: 407 return 408 409 progress_counter = 0 410 for command in list_of_commands: 411 if isinstance(command, dict): 412 command_txt = next(iter(command)) 413 command_timeout = next(iter(command.values())) 414 elif isinstance(command, str): 415 command_txt = command 416 # Default ssh command timeout 60 seconds 417 command_timeout = 60 418 419 if form_filename: 420 command_txt = str(command_txt % self.target_type) 421 422 err, response = self.remoteclient.execute_command(command_txt, command_timeout) 423 424 progress_counter += 1 425 self.print_progress(progress_counter) 426 427 print("\n\t[Run] Commands execution completed.\t\t [OK]") 428 429 def group_copy(self, 430 ffdc_actions_for_machine_type): 431 r""" 432 scp group of files (wild card) from remote host. 433 434 Description of argument(s): 435 ffdc_actions_for_machine_type commands and files for the selected remote host type. 436 """ 437 438 if self.remoteclient.scpclient: 439 print("\n\tCopying DUMP files from remote system %s.\n" % self.hostname) 440 441 list_of_commands = self.get_command_list(ffdc_actions_for_machine_type) 442 # If command list is empty, returns 443 if not list_of_commands: 444 return 445 446 for command in list_of_commands: 447 try: 448 filename = command.split(' ')[2] 449 except IndexError: 450 print("\t\tInvalid command %s for DUMP_LOGS block." % command) 451 continue 452 453 err, response = self.remoteclient.execute_command(command) 454 455 if response: 456 scp_result = self.remoteclient.scp_file_from_remote(filename, self.ffdc_dir_path) 457 if scp_result: 458 print("\t\tSuccessfully copied from " + self.hostname + ':' + filename) 459 else: 460 print("\t\tThere is no " + filename) 461 462 else: 463 print("\n\n\tSkip copying files from remote system %s.\n" % self.hostname) 464 465 def scp_ffdc(self, 466 targ_dir_path, 467 targ_file_prefix, 468 form_filename, 469 file_list=None, 470 quiet=None): 471 r""" 472 SCP all files in file_dict to the indicated directory on the local system. 473 474 Description of argument(s): 475 targ_dir_path The path of the directory to receive the files. 476 targ_file_prefix Prefix which will be pre-pended to each 477 target file's name. 478 file_dict A dictionary of files to scp from targeted system to this system 479 480 """ 481 482 progress_counter = 0 483 for filename in file_list: 484 if form_filename: 485 filename = str(filename % self.target_type) 486 source_file_path = filename 487 targ_file_path = targ_dir_path + targ_file_prefix + filename.split('/')[-1] 488 489 # If source file name contains wild card, copy filename as is. 490 if '*' in source_file_path: 491 scp_result = self.remoteclient.scp_file_from_remote(source_file_path, self.ffdc_dir_path) 492 else: 493 scp_result = self.remoteclient.scp_file_from_remote(source_file_path, targ_file_path) 494 495 if not quiet: 496 if scp_result: 497 print("\t\tSuccessfully copied from " + self.hostname + ':' + source_file_path + ".\n") 498 else: 499 print("\t\tFail to copy from " + self.hostname + ':' + source_file_path + ".\n") 500 else: 501 progress_counter += 1 502 self.print_progress(progress_counter) 503 504 def set_ffdc_defaults(self): 505 r""" 506 Set a default value for self.ffdc_dir_path and self.ffdc_prefix. 507 Collected ffdc file will be stored in dir /self.location/hostname_timestr/. 508 Individual ffdc file will have timestr_filename. 509 510 Description of class variables: 511 self.ffdc_dir_path The dir path where collected ffdc data files should be put. 512 513 self.ffdc_prefix The prefix to be given to each ffdc file name. 514 515 """ 516 517 timestr = time.strftime("%Y%m%d-%H%M%S") 518 self.ffdc_dir_path = self.location + "/" + self.hostname + "_" + timestr + "/" 519 self.ffdc_prefix = timestr + "_" 520 self.validate_local_store(self.ffdc_dir_path) 521 522 def validate_local_store(self, dir_path): 523 r""" 524 Ensure path exists to store FFDC files locally. 525 526 Description of variable: 527 dir_path The dir path where collected ffdc data files will be stored. 528 529 """ 530 531 if not os.path.exists(dir_path): 532 try: 533 os.mkdir(dir_path, 0o755) 534 except (IOError, OSError) as e: 535 # PermissionError 536 if e.errno == EPERM or e.errno == EACCES: 537 print('>>>>>\tERROR: os.mkdir %s failed with PermissionError.\n' % dir_path) 538 else: 539 print('>>>>>\tERROR: os.mkdir %s failed with %s.\n' % (dir_path, e.strerror)) 540 sys.exit(-1) 541 542 def print_progress(self, progress): 543 r""" 544 Print activity progress + 545 546 Description of variable: 547 progress Progress counter. 548 549 """ 550 551 sys.stdout.write("\r\t" + "+" * progress) 552 sys.stdout.flush() 553 time.sleep(.1) 554 555 def verify_redfish(self): 556 r""" 557 Verify remote host has redfish service active 558 559 """ 560 redfish_parm = '-u ' + self.username + ' -p ' + self.password + ' -r ' \ 561 + self.hostname + ' -S Always raw GET /redfish/v1/' 562 return(self.run_redfishtool(redfish_parm, True)) 563 564 def verify_ipmi(self): 565 r""" 566 Verify remote host has IPMI LAN service active 567 568 """ 569 ipmi_parm = '-U ' + self.username + ' -P ' + self.password + ' -H ' \ 570 + self.hostname + ' power status' 571 return(self.run_ipmitool(ipmi_parm, True)) 572 573 def run_redfishtool(self, 574 parms_string, 575 quiet=False): 576 r""" 577 Run CLI redfishtool 578 579 Description of variable: 580 parms_string redfishtool subcommand and options. 581 quiet do not print redfishtool error message if True 582 """ 583 584 result = subprocess.run(['redfishtool ' + parms_string], 585 stdout=subprocess.PIPE, 586 stderr=subprocess.PIPE, 587 shell=True, 588 universal_newlines=True) 589 590 if result.stderr and not quiet: 591 print('\n\t\tERROR with redfishtool ' + parms_string) 592 print('\t\t' + result.stderr) 593 594 return result.stdout 595 596 def run_ipmitool(self, 597 parms_string, 598 quiet=False): 599 r""" 600 Run CLI IPMI tool. 601 602 Description of variable: 603 parms_string ipmitool subcommand and options. 604 quiet do not print redfishtool error message if True 605 """ 606 607 result = subprocess.run(['ipmitool -I lanplus -C 17 ' + parms_string], 608 stdout=subprocess.PIPE, 609 stderr=subprocess.PIPE, 610 shell=True, 611 universal_newlines=True) 612 613 if result.stderr and not quiet: 614 print('\n\t\tERROR with ipmitool -I lanplus -C 17 ' + parms_string) 615 print('\t\t' + result.stderr) 616 617 return result.stdout 618