1#!/usr/bin/env python 2 3r""" 4See class prolog below for details. 5""" 6 7import os 8import sys 9import yaml 10import time 11import platform 12from errno import EACCES, EPERM 13import subprocess 14from ssh_utility import SSHRemoteclient 15 16 17class FFDCCollector: 18 19 r""" 20 Sends commands from configuration file to the targeted system to collect log files. 21 Fetch and store generated files at the specified location. 22 23 """ 24 25 def __init__(self, 26 hostname, 27 username, 28 password, 29 ffdc_config, 30 location, 31 remote_type, 32 remote_protocol): 33 r""" 34 Description of argument(s): 35 36 hostname name/ip of the targeted (remote) system 37 username user on the targeted system with access to FFDC files 38 password password for user on targeted system 39 ffdc_config configuration file listing commands and files for FFDC 40 location where to store collected FFDC 41 remote_type os type of the remote host 42 43 """ 44 if self.verify_script_env(): 45 self.hostname = hostname 46 self.username = username 47 self.password = password 48 self.ffdc_config = ffdc_config 49 self.location = location 50 self.remote_client = None 51 self.ffdc_dir_path = "" 52 self.ffdc_prefix = "" 53 self.target_type = remote_type.upper() 54 self.remote_protocol = remote_protocol.upper() 55 self.start_time = 0 56 self.elapsed_time = '' 57 else: 58 sys.exit(-1) 59 60 # Load default or user define YAML configuration file. 61 with open(self.ffdc_config, 'r') as file: 62 self.ffdc_actions = yaml.load(file, Loader=yaml.FullLoader) 63 64 if self.target_type not in self.ffdc_actions.keys(): 65 print("\n\tERROR: %s is not listed in %s.\n\n" % (self.target_type, self.ffdc_config)) 66 sys.exit(-1) 67 68 def verify_script_env(self): 69 70 # Import to log version 71 import click 72 import paramiko 73 74 run_env_ok = True 75 76 redfishtool_version = self.run_redfishtool('-V').split(' ')[2].strip('\n') 77 ipmitool_version = self.run_ipmitool('-V').split(' ')[2] 78 79 print("\n\t---- Script host environment ----") 80 print("\t{:<10} {:<10}".format('Script hostname', os.uname()[1])) 81 print("\t{:<10} {:<10}".format('Script host os', platform.platform())) 82 print("\t{:<10} {:>10}".format('Python', platform.python_version())) 83 print("\t{:<10} {:>10}".format('PyYAML', yaml.__version__)) 84 print("\t{:<10} {:>10}".format('click', click.__version__)) 85 print("\t{:<10} {:>10}".format('paramiko', paramiko.__version__)) 86 print("\t{:<10} {:>9}".format('redfishtool', redfishtool_version)) 87 print("\t{:<10} {:>12}".format('ipmitool', ipmitool_version)) 88 89 if eval(yaml.__version__.replace('.', ',')) < (5, 4, 1): 90 print("\n\tERROR: Python or python packages do not meet minimum version requirement.") 91 print("\tERROR: PyYAML version 5.4.1 or higher is needed.\n") 92 run_env_ok = False 93 94 print("\t---- End script host environment ----") 95 return run_env_ok 96 97 def target_is_pingable(self): 98 r""" 99 Check if target system is ping-able. 100 101 """ 102 response = os.system("ping -c 1 -w 2 %s 2>&1 >/dev/null" % self.hostname) 103 if response == 0: 104 print("\n\t[Check] %s is ping-able.\t\t [OK]" % self.hostname) 105 return True 106 else: 107 print("\n>>>>>\tERROR: %s is not ping-able. FFDC collection aborted.\n" % self.hostname) 108 sys.exit(-1) 109 110 def collect_ffdc(self): 111 r""" 112 Initiate FFDC Collection depending on requested protocol. 113 114 """ 115 116 print("\n\t---- Start communicating with %s ----" % self.hostname) 117 self.start_time = time.time() 118 working_protocol_list = [] 119 if self.target_is_pingable(): 120 # Check supported protocol ping,ssh, redfish are working. 121 if self.ssh_to_target_system(): 122 working_protocol_list.append("SSH") 123 working_protocol_list.append("SCP") 124 125 # Redfish 126 if self.verify_redfish(): 127 working_protocol_list.append("REDFISH") 128 print("\n\t[Check] %s Redfish Service.\t\t [OK]" % self.hostname) 129 else: 130 print("\n\t[Check] %s Redfish Service.\t\t [NOT AVAILABLE]" % self.hostname) 131 132 if self.verify_ipmi(): 133 working_protocol_list.append("IPMI") 134 print("\n\t[Check] %s IPMI LAN Service.\t\t [OK]" % self.hostname) 135 else: 136 print("\n\t[Check] %s IPMI LAN Service.\t\t [NOT AVAILABLE]" % self.hostname) 137 138 # Verify top level directory exists for storage 139 self.validate_local_store(self.location) 140 print("\n\t---- Completed protocol pre-requisite check ----\n") 141 142 if ((self.remote_protocol not in working_protocol_list) and (self.remote_protocol != 'ALL')): 143 print("\n\tWorking protocol list: %s" % working_protocol_list) 144 print( 145 '>>>>>\tERROR: Requested protocol %s is not in working protocol list.\n' 146 % self.remote_protocol) 147 sys.exit(-1) 148 else: 149 self.generate_ffdc(working_protocol_list) 150 151 def ssh_to_target_system(self): 152 r""" 153 Open a ssh connection to targeted system. 154 155 """ 156 157 self.remoteclient = SSHRemoteclient(self.hostname, 158 self.username, 159 self.password) 160 161 self.remoteclient.ssh_remoteclient_login() 162 print("\n\t[Check] %s SSH connection established.\t [OK]" % self.hostname) 163 164 # Check scp connection. 165 # If scp connection fails, 166 # continue with FFDC generation but skip scp files to local host. 167 self.remoteclient.scp_connection() 168 return True 169 170 def generate_ffdc(self, working_protocol_list): 171 r""" 172 Determine actions based on remote host type 173 174 Description of argument(s): 175 working_protocol_list list of confirmed working protocols to connect to remote host. 176 """ 177 178 print("\n\t---- Executing commands on " + self.hostname + " ----") 179 print("\n\tWorking protocol list: %s" % working_protocol_list) 180 181 # Set prefix values for scp files and directory. 182 # Since the time stamp is at second granularity, these values are set here 183 # to be sure that all files for this run will have same timestamps 184 # and they will be saved in the same directory. 185 # self.location == local system for now 186 self.set_ffdc_defaults() 187 ffdc_actions = self.ffdc_actions 188 189 for machine_type in ffdc_actions.keys(): 190 if self.target_type != machine_type: 191 continue 192 193 print("\tSystem Type: %s" % machine_type) 194 for k, v in ffdc_actions[machine_type].items(): 195 196 if self.remote_protocol != ffdc_actions[machine_type][k]['PROTOCOL'][0] \ 197 and self.remote_protocol != 'ALL': 198 continue 199 200 if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'SSH': 201 if 'SSH' in working_protocol_list: 202 self.protocol_ssh(ffdc_actions, machine_type, k) 203 else: 204 print("\n\tERROR: SSH is not available for %s." % self.hostname) 205 206 if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'REDFISH': 207 if 'REDFISH' in working_protocol_list: 208 self.protocol_redfish(ffdc_actions, machine_type, k) 209 else: 210 print("\n\tERROR: REDFISH is not available for %s." % self.hostname) 211 212 if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'IPMI': 213 if 'IPMI' in working_protocol_list: 214 self.protocol_ipmi(ffdc_actions, machine_type, k) 215 else: 216 print("\n\tERROR: IMPI is not available for %s." % self.hostname) 217 218 # Close network connection after collecting all files 219 self.elapsed_time = time.strftime("%H:%M:%S", time.gmtime(time.time() - self.start_time)) 220 self.remoteclient.ssh_remoteclient_disconnect() 221 222 def protocol_ssh(self, 223 ffdc_actions, 224 machine_type, 225 sub_type): 226 r""" 227 Perform actions using SSH and SCP protocols. 228 229 Description of argument(s): 230 ffdc_actions List of actions from ffdc_config.yaml. 231 machine_type OS Type of remote host. 232 sub_type Group type of commands. 233 """ 234 235 if sub_type == 'DUMP_LOGS': 236 self.group_copy(ffdc_actions[machine_type][sub_type]) 237 else: 238 self.collect_and_copy_ffdc(ffdc_actions[machine_type][sub_type]) 239 240 def protocol_redfish(self, 241 ffdc_actions, 242 machine_type, 243 sub_type): 244 r""" 245 Perform actions using Redfish protocol. 246 247 Description of argument(s): 248 ffdc_actions List of actions from ffdc_config.yaml. 249 machine_type OS Type of remote host. 250 sub_type Group type of commands. 251 """ 252 253 print("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'REDFISH')) 254 redfish_files_saved = [] 255 progress_counter = 0 256 list_of_URL = ffdc_actions[machine_type][sub_type]['URL'] 257 for index, each_url in enumerate(list_of_URL, start=0): 258 redfish_parm = '-u ' + self.username + ' -p ' + self.password + ' -r ' \ 259 + self.hostname + ' -S Always raw GET ' + each_url 260 261 result = self.run_redfishtool(redfish_parm) 262 if result: 263 try: 264 targ_file = ffdc_actions[machine_type][sub_type]['FILES'][index] 265 except IndexError: 266 targ_file = each_url.split('/')[-1] 267 print("\n\t[WARN] Missing filename to store data from redfish URL %s." % each_url) 268 print("\t[WARN] Data will be stored in %s." % targ_file) 269 270 targ_file_with_path = (self.ffdc_dir_path 271 + self.ffdc_prefix 272 + targ_file) 273 274 # Creates a new file 275 with open(targ_file_with_path, 'w') as fp: 276 fp.write(result) 277 fp.close 278 redfish_files_saved.append(targ_file) 279 280 progress_counter += 1 281 self.print_progress(progress_counter) 282 283 print("\n\t[Run] Commands execution completed.\t\t [OK]") 284 285 for file in redfish_files_saved: 286 print("\n\t\tSuccessfully save file " + file + ".") 287 288 def protocol_ipmi(self, 289 ffdc_actions, 290 machine_type, 291 sub_type): 292 r""" 293 Perform actions using ipmitool over LAN protocol. 294 295 Description of argument(s): 296 ffdc_actions List of actions from ffdc_config.yaml. 297 machine_type OS Type of remote host. 298 sub_type Group type of commands. 299 """ 300 301 print("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'IPMI')) 302 ipmi_files_saved = [] 303 progress_counter = 0 304 list_of_cmd = ffdc_actions[machine_type][sub_type]['COMMANDS'] 305 for index, each_cmd in enumerate(list_of_cmd, start=0): 306 ipmi_parm = '-U ' + self.username + ' -P ' + self.password + ' -H ' \ 307 + self.hostname + ' ' + each_cmd 308 309 result = self.run_ipmitool(ipmi_parm) 310 if result: 311 try: 312 targ_file = ffdc_actions[machine_type][sub_type]['FILES'][index] 313 except IndexError: 314 targ_file = each_cmd.split('/')[-1] 315 print("\n\t[WARN] Missing filename to store data from IPMI %s." % each_cmd) 316 print("\t[WARN] Data will be stored in %s." % targ_file) 317 318 targ_file_with_path = (self.ffdc_dir_path 319 + self.ffdc_prefix 320 + targ_file) 321 322 # Creates a new file 323 with open(targ_file_with_path, 'w') as fp: 324 fp.write(result) 325 fp.close 326 ipmi_files_saved.append(targ_file) 327 328 progress_counter += 1 329 self.print_progress(progress_counter) 330 331 print("\n\t[Run] Commands execution completed.\t\t [OK]") 332 333 for file in ipmi_files_saved: 334 print("\n\t\tSuccessfully save file " + file + ".") 335 336 def collect_and_copy_ffdc(self, 337 ffdc_actions_for_machine_type, 338 form_filename=False): 339 r""" 340 Send commands in ffdc_config file to targeted system. 341 342 Description of argument(s): 343 ffdc_actions_for_machine_type commands and files for the selected remote host type. 344 form_filename if true, pre-pend self.target_type to filename 345 """ 346 347 # Executing commands, , if any 348 self.ssh_execute_ffdc_commands(ffdc_actions_for_machine_type, 349 form_filename) 350 351 # Copying files 352 if self.remoteclient.scpclient: 353 print("\n\n\tCopying FFDC files from remote system %s.\n" % self.hostname) 354 355 # Retrieving files from target system 356 list_of_files = ffdc_actions_for_machine_type['FILES'] 357 self.scp_ffdc(self.ffdc_dir_path, self.ffdc_prefix, form_filename, list_of_files) 358 else: 359 print("\n\n\tSkip copying FFDC files from remote system %s.\n" % self.hostname) 360 361 def ssh_execute_ffdc_commands(self, 362 ffdc_actions_for_machine_type, 363 form_filename=False): 364 r""" 365 Send commands in ffdc_config file to targeted system. 366 367 Description of argument(s): 368 ffdc_actions_for_machine_type commands and files for the selected remote host type. 369 form_filename if true, pre-pend self.target_type to filename 370 """ 371 print("\n\t[Run] Executing commands on %s using %s" 372 % (self.hostname, ffdc_actions_for_machine_type['PROTOCOL'][0])) 373 list_of_commands = ffdc_actions_for_machine_type['COMMANDS'] 374 375 # If command list is empty, returns 376 if not list_of_commands: 377 return 378 379 progress_counter = 0 380 for command in list_of_commands: 381 if isinstance(command, dict): 382 command_txt = next(iter(command)) 383 command_timeout = next(iter(command.values())) 384 elif isinstance(command, str): 385 command_txt = command 386 # Default ssh command timeout 60 seconds 387 command_timeout = 60 388 389 if form_filename: 390 command_txt = str(command_txt % self.target_type) 391 392 self.remoteclient.execute_command(command_txt, command_timeout) 393 progress_counter += 1 394 self.print_progress(progress_counter) 395 396 print("\n\t[Run] Commands execution completed.\t\t [OK]") 397 398 def group_copy(self, 399 ffdc_actions_for_machine_type): 400 r""" 401 scp group of files (wild card) from remote host. 402 403 Description of argument(s): 404 ffdc_actions_for_machine_type commands and files for the selected remote host type. 405 """ 406 407 # Executing commands, if any 408 self.ssh_execute_ffdc_commands(ffdc_actions_for_machine_type) 409 410 if self.remoteclient.scpclient: 411 print("\n\tCopying DUMP files from remote system %s.\n" % self.hostname) 412 413 # Retrieving files from target system, if any 414 list_of_files = ffdc_actions_for_machine_type['FILES'] 415 416 for filename in list_of_files: 417 command = 'ls -AX ' + filename 418 response = self.remoteclient.execute_command(command) 419 # self.remoteclient.scp_file_from_remote() completed without exception, 420 # if any 421 if response: 422 scp_result = self.remoteclient.scp_file_from_remote(filename, self.ffdc_dir_path) 423 if scp_result: 424 print("\t\tSuccessfully copied from " + self.hostname + ':' + filename) 425 else: 426 print("\t\tThere is no " + filename) 427 428 else: 429 print("\n\n\tSkip copying files from remote system %s.\n" % self.hostname) 430 431 def scp_ffdc(self, 432 targ_dir_path, 433 targ_file_prefix, 434 form_filename, 435 file_list=None, 436 quiet=None): 437 r""" 438 SCP all files in file_dict to the indicated directory on the local system. 439 440 Description of argument(s): 441 targ_dir_path The path of the directory to receive the files. 442 targ_file_prefix Prefix which will be pre-pended to each 443 target file's name. 444 file_dict A dictionary of files to scp from targeted system to this system 445 446 """ 447 448 progress_counter = 0 449 for filename in file_list: 450 if form_filename: 451 filename = str(filename % self.target_type) 452 source_file_path = filename 453 targ_file_path = targ_dir_path + targ_file_prefix + filename.split('/')[-1] 454 455 # self.remoteclient.scp_file_from_remote() completed without exception, 456 # add file to the receiving file list. 457 scp_result = self.remoteclient.scp_file_from_remote(source_file_path, targ_file_path) 458 459 if not quiet: 460 if scp_result: 461 print("\t\tSuccessfully copied from " + self.hostname + ':' + source_file_path + ".\n") 462 else: 463 print("\t\tFail to copy from " + self.hostname + ':' + source_file_path + ".\n") 464 else: 465 progress_counter += 1 466 self.print_progress(progress_counter) 467 468 def set_ffdc_defaults(self): 469 r""" 470 Set a default value for self.ffdc_dir_path and self.ffdc_prefix. 471 Collected ffdc file will be stored in dir /self.location/hostname_timestr/. 472 Individual ffdc file will have timestr_filename. 473 474 Description of class variables: 475 self.ffdc_dir_path The dir path where collected ffdc data files should be put. 476 477 self.ffdc_prefix The prefix to be given to each ffdc file name. 478 479 """ 480 481 timestr = time.strftime("%Y%m%d-%H%M%S") 482 self.ffdc_dir_path = self.location + "/" + self.hostname + "_" + timestr + "/" 483 self.ffdc_prefix = timestr + "_" 484 self.validate_local_store(self.ffdc_dir_path) 485 486 def validate_local_store(self, dir_path): 487 r""" 488 Ensure path exists to store FFDC files locally. 489 490 Description of variable: 491 dir_path The dir path where collected ffdc data files will be stored. 492 493 """ 494 495 if not os.path.exists(dir_path): 496 try: 497 os.mkdir(dir_path, 0o755) 498 except (IOError, OSError) as e: 499 # PermissionError 500 if e.errno == EPERM or e.errno == EACCES: 501 print('>>>>>\tERROR: os.mkdir %s failed with PermissionError.\n' % dir_path) 502 else: 503 print('>>>>>\tERROR: os.mkdir %s failed with %s.\n' % (dir_path, e.strerror)) 504 sys.exit(-1) 505 506 def print_progress(self, progress): 507 r""" 508 Print activity progress + 509 510 Description of variable: 511 progress Progress counter. 512 513 """ 514 515 sys.stdout.write("\r\t" + "+" * progress) 516 sys.stdout.flush() 517 time.sleep(.1) 518 519 def verify_redfish(self): 520 r""" 521 Verify remote host has redfish service active 522 523 """ 524 redfish_parm = '-u ' + self.username + ' -p ' + self.password + ' -r ' \ 525 + self.hostname + ' -S Always raw GET /redfish/v1/' 526 return(self.run_redfishtool(redfish_parm, True)) 527 528 def verify_ipmi(self): 529 r""" 530 Verify remote host has IPMI LAN service active 531 532 """ 533 ipmi_parm = '-U ' + self.username + ' -P ' + self.password + ' -H ' \ 534 + self.hostname + ' power status' 535 return(self.run_ipmitool(ipmi_parm, True)) 536 537 def run_redfishtool(self, 538 parms_string, 539 quiet=False): 540 r""" 541 Run CLI redfishtool 542 543 Description of variable: 544 parms_string redfishtool subcommand and options. 545 quiet do not print redfishtool error message if True 546 """ 547 548 result = subprocess.run(['redfishtool ' + parms_string], 549 stdout=subprocess.PIPE, 550 stderr=subprocess.PIPE, 551 shell=True, 552 universal_newlines=True) 553 554 if result.stderr and not quiet: 555 print('\n\t\tERROR with redfishtool ' + parms_string) 556 print('\t\t' + result.stderr) 557 558 return result.stdout 559 560 def run_ipmitool(self, 561 parms_string, 562 quiet=False): 563 r""" 564 Run CLI IPMI tool. 565 566 Description of variable: 567 parms_string ipmitool subcommand and options. 568 quiet do not print redfishtool error message if True 569 """ 570 571 result = subprocess.run(['ipmitool -I lanplus -C 17 ' + parms_string], 572 stdout=subprocess.PIPE, 573 stderr=subprocess.PIPE, 574 shell=True, 575 universal_newlines=True) 576 577 if result.stderr and not quiet: 578 print('\n\t\tERROR with ipmitool -I lanplus -C 17 ' + parms_string) 579 print('\t\t' + result.stderr) 580 581 return result.stdout 582