1#!/usr/bin/env python 2 3r""" 4See class prolog below for details. 5""" 6 7import os 8import sys 9import yaml 10import time 11import platform 12from errno import EACCES, EPERM 13import subprocess 14from ssh_utility import SSHRemoteclient 15 16 17class FFDCCollector: 18 19 r""" 20 Sends commands from configuration file to the targeted system to collect log files. 21 Fetch and store generated files at the specified location. 22 23 """ 24 25 # List of supported OSes. 26 supported_oses = ['OPENBMC', 'RHEL', 'AIX', 'UBUNTU'] 27 28 def __init__(self, 29 hostname, 30 username, 31 password, 32 ffdc_config, 33 location, 34 remote_type, 35 remote_protocol): 36 r""" 37 Description of argument(s): 38 39 hostname name/ip of the targeted (remote) system 40 username user on the targeted system with access to FFDC files 41 password password for user on targeted system 42 ffdc_config configuration file listing commands and files for FFDC 43 location where to store collected FFDC 44 remote_type os type of the remote host 45 46 """ 47 if self.verify_script_env(): 48 self.hostname = hostname 49 self.username = username 50 self.password = password 51 self.ffdc_config = ffdc_config 52 self.location = location 53 self.remote_client = None 54 self.ffdc_dir_path = "" 55 self.ffdc_prefix = "" 56 self.target_type = remote_type.upper() 57 self.remote_protocol = remote_protocol.upper() 58 else: 59 sys.exit(-1) 60 61 def verify_script_env(self): 62 63 # Import to log version 64 import click 65 import paramiko 66 67 run_env_ok = True 68 69 redfishtool_version = self.run_redfishtool('-V').split(' ')[2] 70 71 print("\n\t---- Script host environment ----") 72 print("\t{:<10} {:<10}".format('Script hostname', os.uname()[1])) 73 print("\t{:<10} {:<10}".format('Script host os', platform.platform())) 74 print("\t{:<10} {:>10}".format('Python', platform.python_version())) 75 print("\t{:<10} {:>10}".format('PyYAML', yaml.__version__)) 76 print("\t{:<10} {:>10}".format('click', click.__version__)) 77 print("\t{:<10} {:>10}".format('paramiko', paramiko.__version__)) 78 print("\t{:<10} {:>10}".format('redfishtool', redfishtool_version)) 79 80 if eval(yaml.__version__.replace('.', ',')) < (5, 4, 1): 81 print("\n\tERROR: Python or python packages do not meet minimum version requirement.") 82 print("\tERROR: PyYAML version 5.4.1 or higher is needed.\n") 83 run_env_ok = False 84 85 print("\t---- End script host environment ----") 86 return run_env_ok 87 88 def target_is_pingable(self): 89 r""" 90 Check if target system is ping-able. 91 92 """ 93 response = os.system("ping -c 1 -w 2 %s 2>&1 >/dev/null" % self.hostname) 94 if response == 0: 95 print("\n\t[Check] %s is ping-able.\t\t [OK]" % self.hostname) 96 return True 97 else: 98 print("\n>>>>>\tERROR: %s is not ping-able. FFDC collection aborted.\n" % self.hostname) 99 sys.exit(-1) 100 101 def inspect_target_machine_type(self): 102 r""" 103 Inspect remote host os-release or uname. 104 105 """ 106 command = "cat /etc/os-release" 107 response = self.remoteclient.execute_command(command) 108 if response: 109 print("\n\t[INFO] %s /etc/os-release\n" % self.hostname) 110 print("\t\t %s" % self.find_os_type(response, 'PRETTY_NAME')) 111 identity = self.find_os_type(response, 'ID').split('=')[1].upper() 112 else: 113 response = self.remoteclient.execute_command('uname -a') 114 print("\n\t[INFO] %s uname -a\n" % self.hostname) 115 print("\t\t %s" % ' '.join(response)) 116 identity = self.find_os_type(response, 'AIX').split(' ')[0].upper() 117 118 # If OS does not have /etc/os-release and is not AIX, 119 # script does not yet know what to do. 120 if not identity: 121 print(">>>>>\tERROR: Script does not yet know about %s" % ' '.join(response)) 122 sys.exit(-1) 123 124 if (self.target_type not in identity): 125 126 user_target_type = self.target_type 127 self.target_type = "" 128 for each_os in FFDCCollector.supported_oses: 129 if each_os in identity: 130 self.target_type = each_os 131 break 132 133 # If OS in not one of ['OPENBMC', 'RHEL', 'AIX', 'UBUNTU'] 134 # script does not yet know what to do. 135 if not self.target_type: 136 print(">>>>>\tERROR: Script does not yet know about %s" % identity) 137 sys.exit(-1) 138 139 print("\n\t[WARN] user request %s does not match remote host type %s.\n" 140 % (user_target_type, self.target_type)) 141 print("\t[WARN] FFDC collection continues for %s.\n" % self.target_type) 142 143 def find_os_type(self, 144 listing_from_os, 145 key): 146 147 r""" 148 Return OS information with the requested key 149 150 Description of argument(s): 151 152 listing_from_os list of information returns from OS command 153 key key of the desired data 154 155 """ 156 157 for each_item in listing_from_os: 158 if key in each_item: 159 return each_item 160 return '' 161 162 def collect_ffdc(self): 163 r""" 164 Initiate FFDC Collection depending on requested protocol. 165 166 """ 167 168 print("\n\t---- Start communicating with %s ----" % self.hostname) 169 working_protocol_list = [] 170 if self.target_is_pingable(): 171 # Check supported protocol ping,ssh, redfish are working. 172 if self.ssh_to_target_system(): 173 working_protocol_list.append("SSH") 174 working_protocol_list.append("SCP") 175 176 # Redfish 177 if self.verify_redfish(): 178 working_protocol_list.append("REDFISH") 179 print("\n\t[Check] %s Redfish Service.\t\t [OK]" % self.hostname) 180 else: 181 print("\n\t[Check] %s Redfish Service.\t\t [FAILED]" % self.hostname) 182 183 # Verify top level directory exists for storage 184 self.validate_local_store(self.location) 185 self.inspect_target_machine_type() 186 print("\n\t---- Completed protocol pre-requisite check ----\n") 187 188 if ((self.remote_protocol not in working_protocol_list) and (self.remote_protocol != 'ALL')): 189 print("\n\tWorking protocol list: %s" % working_protocol_list) 190 print( 191 '>>>>>\tERROR: Requested protocol %s is not in working protocol list.\n' 192 % self.remote_protocol) 193 sys.exit(-1) 194 else: 195 self.generate_ffdc(working_protocol_list) 196 197 def ssh_to_target_system(self): 198 r""" 199 Open a ssh connection to targeted system. 200 201 """ 202 203 self.remoteclient = SSHRemoteclient(self.hostname, 204 self.username, 205 self.password) 206 207 self.remoteclient.ssh_remoteclient_login() 208 print("\n\t[Check] %s SSH connection established.\t [OK]" % self.hostname) 209 210 # Check scp connection. 211 # If scp connection fails, 212 # continue with FFDC generation but skip scp files to local host. 213 self.remoteclient.scp_connection() 214 return True 215 216 def generate_ffdc(self, working_protocol_list): 217 r""" 218 Determine actions based on remote host type 219 220 Description of argument(s): 221 working_protocol_list list of confirmed working protocols to connect to remote host. 222 """ 223 224 print("\n\t---- Executing commands on " + self.hostname + " ----") 225 print("\n\tWorking protocol list: %s" % working_protocol_list) 226 with open(self.ffdc_config, 'r') as file: 227 ffdc_actions = yaml.load(file, Loader=yaml.FullLoader) 228 229 # Set prefix values for scp files and directory. 230 # Since the time stamp is at second granularity, these values are set here 231 # to be sure that all files for this run will have same timestamps 232 # and they will be saved in the same directory. 233 # self.location == local system for now 234 self.set_ffdc_defaults() 235 236 for machine_type in ffdc_actions.keys(): 237 238 if machine_type == self.target_type: 239 if self.remote_protocol == 'SSH' or self.remote_protocol == 'ALL': 240 self.protocol_ssh(ffdc_actions, machine_type) 241 242 if self.target_type == 'OPENBMC': 243 if self.remote_protocol == 'REDFISH' or self.remote_protocol == 'ALL': 244 self.protocol_redfish(ffdc_actions, 'OPENBMC_REDFISH') 245 246 # Close network connection after collecting all files 247 self.remoteclient.ssh_remoteclient_disconnect() 248 249 def protocol_ssh(self, 250 ffdc_actions, 251 machine_type): 252 r""" 253 Perform actions using SSH and SCP protocols. 254 255 Description of argument(s): 256 ffdc_actions List of actions from ffdc_config.yaml. 257 machine_type OS Type of remote host. 258 """ 259 260 # For OPENBMC collect general system info. 261 if self.target_type == 'OPENBMC': 262 263 self.collect_and_copy_ffdc(ffdc_actions['GENERAL'], 264 form_filename=True) 265 self.group_copy(ffdc_actions['OPENBMC_DUMPS']) 266 267 # For RHEL and UBUNTU, collect common Linux OS FFDC. 268 if self.target_type == 'RHEL' \ 269 or self.target_type == 'UBUNTU': 270 271 self.collect_and_copy_ffdc(ffdc_actions['LINUX']) 272 273 # Collect remote host specific FFDC. 274 self.collect_and_copy_ffdc(ffdc_actions[machine_type]) 275 276 def protocol_redfish(self, 277 ffdc_actions, 278 machine_type): 279 r""" 280 Perform actions using Redfish protocol. 281 282 Description of argument(s): 283 ffdc_actions List of actions from ffdc_config.yaml. 284 machine_type OS Type of remote host. 285 """ 286 287 print("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'REDFISH')) 288 redfish_files_saved = [] 289 progress_counter = 0 290 list_of_URL = ffdc_actions[machine_type]['URL'] 291 for index, each_url in enumerate(list_of_URL, start=0): 292 redfish_parm = '-u ' + self.username + ' -p ' + self.password + ' -r ' \ 293 + self.hostname + ' -S Always raw GET ' + each_url 294 295 result = self.run_redfishtool(redfish_parm) 296 if result: 297 try: 298 targ_file = ffdc_actions[machine_type]['FILES'][index] 299 except IndexError: 300 targ_file = each_url.split('/')[-1] 301 print("\n\t[WARN] Missing filename to store data from redfish URL %s." % each_url) 302 print("\t[WARN] Data will be stored in %s." % targ_file) 303 304 targ_file_with_path = (self.ffdc_dir_path 305 + self.ffdc_prefix 306 + targ_file) 307 308 # Creates a new file 309 with open(targ_file_with_path, 'w') as fp: 310 fp.write(result) 311 fp.close 312 redfish_files_saved.append(targ_file) 313 314 progress_counter += 1 315 self.print_progress(progress_counter) 316 317 print("\n\t[Run] Commands execution completed.\t\t [OK]") 318 319 for file in redfish_files_saved: 320 print("\n\t\tSuccessfully save file " + file + ".") 321 322 def collect_and_copy_ffdc(self, 323 ffdc_actions_for_machine_type, 324 form_filename=False): 325 r""" 326 Send commands in ffdc_config file to targeted system. 327 328 Description of argument(s): 329 ffdc_actions_for_machine_type commands and files for the selected remote host type. 330 form_filename if true, pre-pend self.target_type to filename 331 """ 332 333 print("\n\t[Run] Executing commands on %s using %s" 334 % (self.hostname, ffdc_actions_for_machine_type['PROTOCOL'][0])) 335 list_of_commands = ffdc_actions_for_machine_type['COMMANDS'] 336 progress_counter = 0 337 for command in list_of_commands: 338 if form_filename: 339 command = str(command % self.target_type) 340 self.remoteclient.execute_command(command) 341 progress_counter += 1 342 self.print_progress(progress_counter) 343 344 print("\n\t[Run] Commands execution completed.\t\t [OK]") 345 346 if self.remoteclient.scpclient: 347 print("\n\n\tCopying FFDC files from remote system %s.\n" % self.hostname) 348 349 # Retrieving files from target system 350 list_of_files = ffdc_actions_for_machine_type['FILES'] 351 self.scp_ffdc(self.ffdc_dir_path, self.ffdc_prefix, form_filename, list_of_files) 352 else: 353 print("\n\n\tSkip copying FFDC files from remote system %s.\n" % self.hostname) 354 355 def group_copy(self, 356 ffdc_actions_for_machine_type): 357 358 r""" 359 scp group of files (wild card) from remote host. 360 361 Description of argument(s): 362 ffdc_actions_for_machine_type commands and files for the selected remote host type. 363 """ 364 if self.remoteclient.scpclient: 365 print("\n\tCopying DUMP files from remote system %s.\n" % self.hostname) 366 367 # Retrieving files from target system, if any 368 list_of_files = ffdc_actions_for_machine_type['FILES'] 369 370 for filename in list_of_files: 371 command = 'ls -AX ' + filename 372 response = self.remoteclient.execute_command(command) 373 # self.remoteclient.scp_file_from_remote() completed without exception, 374 # if any 375 if response: 376 scp_result = self.remoteclient.scp_file_from_remote(filename, self.ffdc_dir_path) 377 if scp_result: 378 print("\t\tSuccessfully copied from " + self.hostname + ':' + filename) 379 else: 380 print("\t\tThere is no " + filename) 381 382 else: 383 print("\n\n\tSkip copying files from remote system %s.\n" % self.hostname) 384 385 def scp_ffdc(self, 386 targ_dir_path, 387 targ_file_prefix, 388 form_filename, 389 file_list=None, 390 quiet=None): 391 r""" 392 SCP all files in file_dict to the indicated directory on the local system. 393 394 Description of argument(s): 395 targ_dir_path The path of the directory to receive the files. 396 targ_file_prefix Prefix which will be pre-pended to each 397 target file's name. 398 file_dict A dictionary of files to scp from targeted system to this system 399 400 """ 401 402 progress_counter = 0 403 for filename in file_list: 404 if form_filename: 405 filename = str(filename % self.target_type) 406 source_file_path = filename 407 targ_file_path = targ_dir_path + targ_file_prefix + filename.split('/')[-1] 408 409 # self.remoteclient.scp_file_from_remote() completed without exception, 410 # add file to the receiving file list. 411 scp_result = self.remoteclient.scp_file_from_remote(source_file_path, targ_file_path) 412 413 if not quiet: 414 if scp_result: 415 print("\t\tSuccessfully copied from " + self.hostname + ':' + source_file_path + ".\n") 416 else: 417 print("\t\tFail to copy from " + self.hostname + ':' + source_file_path + ".\n") 418 else: 419 progress_counter += 1 420 self.print_progress(progress_counter) 421 422 def set_ffdc_defaults(self): 423 r""" 424 Set a default value for self.ffdc_dir_path and self.ffdc_prefix. 425 Collected ffdc file will be stored in dir /self.location/hostname_timestr/. 426 Individual ffdc file will have timestr_filename. 427 428 Description of class variables: 429 self.ffdc_dir_path The dir path where collected ffdc data files should be put. 430 431 self.ffdc_prefix The prefix to be given to each ffdc file name. 432 433 """ 434 435 timestr = time.strftime("%Y%m%d-%H%M%S") 436 self.ffdc_dir_path = self.location + "/" + self.hostname + "_" + timestr + "/" 437 self.ffdc_prefix = timestr + "_" 438 self.validate_local_store(self.ffdc_dir_path) 439 440 def validate_local_store(self, dir_path): 441 r""" 442 Ensure path exists to store FFDC files locally. 443 444 Description of variable: 445 dir_path The dir path where collected ffdc data files will be stored. 446 447 """ 448 449 if not os.path.exists(dir_path): 450 try: 451 os.mkdir(dir_path, 0o755) 452 except (IOError, OSError) as e: 453 # PermissionError 454 if e.errno == EPERM or e.errno == EACCES: 455 print('>>>>>\tERROR: os.mkdir %s failed with PermissionError.\n' % dir_path) 456 else: 457 print('>>>>>\tERROR: os.mkdir %s failed with %s.\n' % (dir_path, e.strerror)) 458 sys.exit(-1) 459 460 def print_progress(self, progress): 461 r""" 462 Print activity progress + 463 464 Description of variable: 465 progress Progress counter. 466 467 """ 468 469 sys.stdout.write("\r\t" + "+" * progress) 470 sys.stdout.flush() 471 time.sleep(.1) 472 473 def verify_redfish(self): 474 r""" 475 Verify remote host has redfish service active 476 477 """ 478 redfish_parm = '-u ' + self.username + ' -p ' + self.password + ' -r ' \ 479 + self.hostname + ' -S Always raw GET /redfish/v1/' 480 return(self.run_redfishtool(redfish_parm, True)) 481 482 def run_redfishtool(self, 483 parms_string, 484 quiet=False): 485 r""" 486 Run CLI redfishtool 487 488 Description of variable: 489 parms_string redfishtool subcommand and options. 490 quiet do not print redfishtool error message if True 491 """ 492 493 result = subprocess.run(['redfishtool ' + parms_string], 494 stdout=subprocess.PIPE, 495 stderr=subprocess.PIPE, 496 shell=True, 497 universal_newlines=True) 498 499 if result.stderr and not quiet: 500 print('\n\t\tERROR with redfishtool ' + parms_string) 501 print('\t\t' + result.stderr) 502 503 return result.stdout 504