xref: /openbmc/openbmc-test-automation/ffdc/ffdc_collector.py (revision ac76512089f08d3e17d24633a664f200aa6c8e51)
1#!/usr/bin/env python
2
3r"""
4See class prolog below for details.
5"""
6
7import os
8import re
9import sys
10import yaml
11import json
12import time
13import logging
14import platform
15from errno import EACCES, EPERM
16import subprocess
17from ssh_utility import SSHRemoteclient
18from telnet_utility import TelnetRemoteclient
19
20
21class FFDCCollector:
22
23    r"""
24    Sends commands from configuration file to the targeted system to collect log files.
25    Fetch and store generated files at the specified location.
26
27    """
28
29    def __init__(self,
30                 hostname,
31                 username,
32                 password,
33                 ffdc_config,
34                 location,
35                 remote_type,
36                 remote_protocol,
37                 env_vars,
38                 econfig,
39                 log_level):
40        r"""
41        Description of argument(s):
42
43        hostname            name/ip of the targeted (remote) system
44        username            user on the targeted system with access to FFDC files
45        password            password for user on targeted system
46        ffdc_config         configuration file listing commands and files for FFDC
47        location            where to store collected FFDC
48        remote_type         os type of the remote host
49        remote_protocol     Protocol to use to collect data
50        env_vars            User define CLI env vars '{"key : "value"}'
51        econfig             User define env vars YAML file
52
53        """
54
55        self.hostname = hostname
56        self.username = username
57        self.password = password
58        # This is for the env vars a user can use in YAML to load it at runtime.
59        # Example YAML:
60        # -COMMANDS:
61        #    - my_command ${hostname}  ${username}   ${password}
62        os.environ['hostname'] = hostname
63        os.environ['username'] = username
64        os.environ['password'] = password
65
66        self.ffdc_config = ffdc_config
67        self.location = location + "/" + remote_type.upper()
68        self.ssh_remoteclient = None
69        self.telnet_remoteclient = None
70        self.ffdc_dir_path = ""
71        self.ffdc_prefix = ""
72        self.target_type = remote_type.upper()
73        self.remote_protocol = remote_protocol.upper()
74        self.start_time = 0
75        self.elapsed_time = ''
76        self.logger = None
77
78        # Set prefix values for scp files and directory.
79        # Since the time stamp is at second granularity, these values are set here
80        # to be sure that all files for this run will have same timestamps
81        # and they will be saved in the same directory.
82        # self.location == local system for now
83        self.set_ffdc_defaults()
84
85        # Logger for this run.  Need to be after set_ffdc_defaults()
86        self.script_logging(getattr(logging, log_level.upper()))
87
88        # Verify top level directory exists for storage
89        self.validate_local_store(self.location)
90
91        if self.verify_script_env():
92            # Load default or user define YAML configuration file.
93            with open(self.ffdc_config, 'r') as file:
94                self.ffdc_actions = yaml.load(file, Loader=yaml.FullLoader)
95
96            if self.target_type not in self.ffdc_actions.keys():
97                self.logger.error(
98                    "\n\tERROR: %s is not listed in %s.\n\n" % (self.target_type, self.ffdc_config))
99                sys.exit(-1)
100        else:
101            sys.exit(-1)
102
103        # Load ENV vars from user.
104        self.logger.info("\n\tENV: User define input YAML variables")
105        self.env_dict = {}
106
107        try:
108            if env_vars:
109                self.env_dict = json.loads(env_vars)
110
111                # Export ENV vars default.
112                for key, value in self.env_dict.items():
113                    os.environ[key] = value
114
115            if econfig:
116                with open(econfig, 'r') as file:
117                    env_config_dict = yaml.load(file, Loader=yaml.FullLoader)
118                # Export ENV vars.
119                for key, value in env_config_dict['env_params'].items():
120                    os.environ[key] = str(value)
121                    self.env_dict[key] = str(value)
122
123        except json.decoder.JSONDecodeError as e:
124            self.logger.error("\n\tERROR: %s " % e)
125            sys.exit(-1)
126
127        # Append default Env.
128        self.env_dict['hostname'] = self.hostname
129        self.env_dict['username'] = self.username
130        self.env_dict['password'] = self.password
131        # This to mask the password from displaying on the console.
132        mask_dict = self.env_dict.copy()
133        for k, v in mask_dict.items():
134            if k.lower().find("password") != -1:
135                hidden_text = []
136                hidden_text.append(v)
137                password_regex = '(' +\
138                    '|'.join([re.escape(x) for x in hidden_text]) + ')'
139                mask_dict[k] = re.sub(password_regex, "********", v)
140        self.logger.info(json.dumps(mask_dict, indent=8, sort_keys=True))
141
142    def verify_script_env(self):
143
144        # Import to log version
145        import click
146        import paramiko
147
148        run_env_ok = True
149
150        redfishtool_version = self.run_redfishtool('-V').split(' ')[2].strip('\n')
151        ipmitool_version = self.run_ipmitool('-V').split(' ')[2]
152
153        self.logger.info("\n\t---- Script host environment ----")
154        self.logger.info("\t{:<10}  {:<10}".format('Script hostname', os.uname()[1]))
155        self.logger.info("\t{:<10}  {:<10}".format('Script host os', platform.platform()))
156        self.logger.info("\t{:<10}  {:>10}".format('Python', platform.python_version()))
157        self.logger.info("\t{:<10}  {:>10}".format('PyYAML', yaml.__version__))
158        self.logger.info("\t{:<10}  {:>10}".format('click', click.__version__))
159        self.logger.info("\t{:<10}  {:>10}".format('paramiko', paramiko.__version__))
160        self.logger.info("\t{:<10}  {:>9}".format('redfishtool', redfishtool_version))
161        self.logger.info("\t{:<10}  {:>12}".format('ipmitool', ipmitool_version))
162
163        if eval(yaml.__version__.replace('.', ',')) < (5, 4, 1):
164            self.logger.error("\n\tERROR: Python or python packages do not meet minimum version requirement.")
165            self.logger.error("\tERROR: PyYAML version 5.4.1 or higher is needed.\n")
166            run_env_ok = False
167
168        self.logger.info("\t---- End script host environment ----")
169        return run_env_ok
170
171    def script_logging(self,
172                       log_level_attr):
173        r"""
174        Create logger
175
176        """
177        self.logger = logging.getLogger()
178        self.logger.setLevel(log_level_attr)
179        log_file_handler = logging.FileHandler(self.ffdc_dir_path + "collector.log")
180
181        stdout_handler = logging.StreamHandler(sys.stdout)
182        self.logger.addHandler(log_file_handler)
183        self.logger.addHandler(stdout_handler)
184
185        # Turn off paramiko INFO logging
186        logging.getLogger("paramiko").setLevel(logging.WARNING)
187
188    def target_is_pingable(self):
189        r"""
190        Check if target system is ping-able.
191
192        """
193        response = os.system("ping -c 1 %s  2>&1 >/dev/null" % self.hostname)
194        if response == 0:
195            self.logger.info("\n\t[Check] %s is ping-able.\t\t [OK]" % self.hostname)
196            return True
197        else:
198            self.logger.error(
199                "\n\tERROR: %s is not ping-able. FFDC collection aborted.\n" % self.hostname)
200            sys.exit(-1)
201
202    def collect_ffdc(self):
203        r"""
204        Initiate FFDC Collection depending on requested protocol.
205
206        """
207
208        self.logger.info("\n\t---- Start communicating with %s ----" % self.hostname)
209        self.start_time = time.time()
210
211        # Find the list of target and protocol supported.
212        check_protocol_list = []
213        config_dict = self.ffdc_actions
214
215        for target_type in config_dict.keys():
216            if self.target_type != target_type:
217                continue
218
219            for k, v in config_dict[target_type].items():
220                if config_dict[target_type][k]['PROTOCOL'][0] not in check_protocol_list:
221                    check_protocol_list.append(config_dict[target_type][k]['PROTOCOL'][0])
222
223        self.logger.info("\n\t %s protocol type: %s" % (self.target_type, check_protocol_list))
224
225        verified_working_protocol = self.verify_protocol(check_protocol_list)
226
227        if verified_working_protocol:
228            self.logger.info("\n\t---- Completed protocol pre-requisite check ----\n")
229
230        # Verify top level directory exists for storage
231        self.validate_local_store(self.location)
232
233        if ((self.remote_protocol not in verified_working_protocol) and (self.remote_protocol != 'ALL')):
234            self.logger.info("\n\tWorking protocol list: %s" % verified_working_protocol)
235            self.logger.error(
236                '\tERROR: Requested protocol %s is not in working protocol list.\n'
237                % self.remote_protocol)
238            sys.exit(-1)
239        else:
240            self.generate_ffdc(verified_working_protocol)
241
242    def ssh_to_target_system(self):
243        r"""
244        Open a ssh connection to targeted system.
245
246        """
247
248        self.ssh_remoteclient = SSHRemoteclient(self.hostname,
249                                                self.username,
250                                                self.password)
251
252        if self.ssh_remoteclient.ssh_remoteclient_login():
253            self.logger.info("\n\t[Check] %s SSH connection established.\t [OK]" % self.hostname)
254
255            # Check scp connection.
256            # If scp connection fails,
257            # continue with FFDC generation but skip scp files to local host.
258            self.ssh_remoteclient.scp_connection()
259            return True
260        else:
261            self.logger.info("\n\t[Check] %s SSH connection.\t [NOT AVAILABLE]" % self.hostname)
262            return False
263
264    def telnet_to_target_system(self):
265        r"""
266        Open a telnet connection to targeted system.
267        """
268        self.telnet_remoteclient = TelnetRemoteclient(self.hostname,
269                                                      self.username,
270                                                      self.password)
271        if self.telnet_remoteclient.tn_remoteclient_login():
272            self.logger.info("\n\t[Check] %s Telnet connection established.\t [OK]" % self.hostname)
273            return True
274        else:
275            self.logger.info("\n\t[Check] %s Telnet connection.\t [NOT AVAILABLE]" % self.hostname)
276            return False
277
278    def generate_ffdc(self, working_protocol_list):
279        r"""
280        Determine actions based on remote host type
281
282        Description of argument(s):
283        working_protocol_list    list of confirmed working protocols to connect to remote host.
284        """
285
286        self.logger.info("\n\t---- Executing commands on " + self.hostname + " ----")
287        self.logger.info("\n\tWorking protocol list: %s" % working_protocol_list)
288
289        config_dict = self.ffdc_actions
290        for target_type in config_dict.keys():
291            if self.target_type != target_type:
292                continue
293
294            self.logger.info("\n\tFFDC Path: %s " % self.ffdc_dir_path)
295            self.logger.info("\tSystem Type: %s" % target_type)
296            for k, v in config_dict[target_type].items():
297
298                if self.remote_protocol not in working_protocol_list \
299                        and self.remote_protocol != 'ALL':
300                    continue
301
302                protocol = config_dict[target_type][k]['PROTOCOL'][0]
303
304                if protocol not in working_protocol_list:
305                    continue
306
307                if protocol == 'SSH' or protocol == 'SCP':
308                    if 'SSH' in working_protocol_list or 'SCP' in working_protocol_list:
309                        self.protocol_ssh(target_type, k)
310                    else:
311                        self.logger.error("\n\tERROR: SSH or SCP is not available for %s." % self.hostname)
312
313                if protocol == 'TELNET':
314                    if protocol in working_protocol_list:
315                        self.protocol_telnet(target_type, k)
316                    else:
317                        self.logger.error("\n\tERROR: TELNET is not available for %s." % self.hostname)
318
319                if protocol == 'REDFISH':
320                    if protocol in working_protocol_list:
321                        self.protocol_redfish(target_type, k)
322                    else:
323                        self.logger.error("\n\tERROR: REDFISH is not available for %s." % self.hostname)
324
325                if protocol == 'IPMI':
326                    if protocol in working_protocol_list:
327                        self.protocol_ipmi(target_type, k)
328                    else:
329                        self.logger.error("\n\tERROR: IPMI is not available for %s." % self.hostname)
330
331                if protocol == 'SHELL':
332                    if protocol in working_protocol_list:
333                        self.protocol_shell_script(target_type, k)
334                    else:
335                        self.logger.error("\n\tERROR: can't execute SHELL script")
336
337        # Close network connection after collecting all files
338        self.elapsed_time = time.strftime("%H:%M:%S", time.gmtime(time.time() - self.start_time))
339        if self.ssh_remoteclient:
340            self.ssh_remoteclient.ssh_remoteclient_disconnect()
341        if self.telnet_remoteclient:
342            self.telnet_remoteclient.tn_remoteclient_disconnect()
343
344    def protocol_ssh(self,
345                     target_type,
346                     sub_type):
347        r"""
348        Perform actions using SSH and SCP protocols.
349
350        Description of argument(s):
351        target_type         OS Type of remote host.
352        sub_type            Group type of commands.
353        """
354
355        if sub_type == 'DUMP_LOGS':
356            self.group_copy(self.ffdc_actions[target_type][sub_type])
357        else:
358            self.collect_and_copy_ffdc(self.ffdc_actions[target_type][sub_type])
359
360    def protocol_telnet(self,
361                        target_type,
362                        sub_type):
363        r"""
364        Perform actions using telnet protocol.
365        Description of argument(s):
366        target_type          OS Type of remote host.
367        """
368        self.logger.info("\n\t[Run] Executing commands on %s using %s" % (self.hostname, 'TELNET'))
369        telnet_files_saved = []
370        progress_counter = 0
371        list_of_commands = self.ffdc_actions[target_type][sub_type]['COMMANDS']
372        for index, each_cmd in enumerate(list_of_commands, start=0):
373            command_txt, command_timeout = self.unpack_command(each_cmd)
374            result = self.telnet_remoteclient.execute_command(command_txt, command_timeout)
375            if result:
376                try:
377                    targ_file = self.ffdc_actions[target_type][sub_type]['FILES'][index]
378                except IndexError:
379                    targ_file = command_txt
380                    self.logger.warning(
381                        "\n\t[WARN] Missing filename to store data from telnet %s." % each_cmd)
382                    self.logger.warning("\t[WARN] Data will be stored in %s." % targ_file)
383                targ_file_with_path = (self.ffdc_dir_path
384                                       + self.ffdc_prefix
385                                       + targ_file)
386                # Creates a new file
387                with open(targ_file_with_path, 'wb') as fp:
388                    fp.write(result)
389                    fp.close
390                    telnet_files_saved.append(targ_file)
391            progress_counter += 1
392            self.print_progress(progress_counter)
393        self.logger.info("\n\t[Run] Commands execution completed.\t\t [OK]")
394        for file in telnet_files_saved:
395            self.logger.info("\n\t\tSuccessfully save file " + file + ".")
396
397    def protocol_redfish(self,
398                         target_type,
399                         sub_type):
400        r"""
401        Perform actions using Redfish protocol.
402
403        Description of argument(s):
404        target_type         OS Type of remote host.
405        sub_type            Group type of commands.
406        """
407
408        self.logger.info("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'REDFISH'))
409        redfish_files_saved = []
410        progress_counter = 0
411        list_of_URL = self.ffdc_actions[target_type][sub_type]['URL']
412        for index, each_url in enumerate(list_of_URL, start=0):
413            redfish_parm = '-u ' + self.username + ' -p ' + self.password + ' -r ' \
414                           + self.hostname + ' -S Always raw GET ' + each_url
415
416            result = self.run_redfishtool(redfish_parm)
417            if result:
418                try:
419                    targ_file = self.get_file_list(self.ffdc_actions[target_type][sub_type])[index]
420                except IndexError:
421                    targ_file = each_url.split('/')[-1]
422                    self.logger.warning(
423                        "\n\t[WARN] Missing filename to store data from redfish URL %s." % each_url)
424                    self.logger.warning("\t[WARN] Data will be stored in %s." % targ_file)
425
426                targ_file_with_path = (self.ffdc_dir_path
427                                       + self.ffdc_prefix
428                                       + targ_file)
429
430                # Creates a new file
431                with open(targ_file_with_path, 'w') as fp:
432                    fp.write(result)
433                    fp.close
434                    redfish_files_saved.append(targ_file)
435
436            progress_counter += 1
437            self.print_progress(progress_counter)
438
439        self.logger.info("\n\t[Run] Commands execution completed.\t\t [OK]")
440
441        for file in redfish_files_saved:
442            self.logger.info("\n\t\tSuccessfully save file " + file + ".")
443
444    def protocol_ipmi(self,
445                      target_type,
446                      sub_type):
447        r"""
448        Perform actions using ipmitool over LAN protocol.
449
450        Description of argument(s):
451        target_type         OS Type of remote host.
452        sub_type            Group type of commands.
453        """
454
455        self.logger.info("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'IPMI'))
456        ipmi_files_saved = []
457        progress_counter = 0
458        list_of_cmd = self.get_command_list(self.ffdc_actions[target_type][sub_type])
459        for index, each_cmd in enumerate(list_of_cmd, start=0):
460            ipmi_parm = '-U ' + self.username + ' -P ' + self.password + ' -H ' \
461                + self.hostname + ' -I lanplus ' + each_cmd
462
463            result = self.run_ipmitool(ipmi_parm)
464            if result:
465                try:
466                    targ_file = self.get_file_list(self.ffdc_actions[target_type][sub_type])[index]
467                except IndexError:
468                    targ_file = each_cmd.split('/')[-1]
469                    self.logger.warning("\n\t[WARN] Missing filename to store data from IPMI %s." % each_cmd)
470                    self.logger.warning("\t[WARN] Data will be stored in %s." % targ_file)
471
472                targ_file_with_path = (self.ffdc_dir_path
473                                       + self.ffdc_prefix
474                                       + targ_file)
475
476                # Creates a new file
477                with open(targ_file_with_path, 'w') as fp:
478                    fp.write(result)
479                    fp.close
480                    ipmi_files_saved.append(targ_file)
481
482            progress_counter += 1
483            self.print_progress(progress_counter)
484
485        self.logger.info("\n\t[Run] Commands execution completed.\t\t [OK]")
486
487        for file in ipmi_files_saved:
488            self.logger.info("\n\t\tSuccessfully save file " + file + ".")
489
490    def collect_and_copy_ffdc(self,
491                              ffdc_actions_for_target_type,
492                              form_filename=False):
493        r"""
494        Send commands in ffdc_config file to targeted system.
495
496        Description of argument(s):
497        ffdc_actions_for_target_type     commands and files for the selected remote host type.
498        form_filename                    if true, pre-pend self.target_type to filename
499        """
500
501        # Executing commands, if any
502        self.ssh_execute_ffdc_commands(ffdc_actions_for_target_type,
503                                       form_filename)
504
505        # Copying files
506        if self.ssh_remoteclient.scpclient:
507            self.logger.info("\n\n\tCopying FFDC files from remote system %s.\n" % self.hostname)
508
509            # Retrieving files from target system
510            list_of_files = self.get_file_list(ffdc_actions_for_target_type)
511            self.scp_ffdc(self.ffdc_dir_path, self.ffdc_prefix, form_filename, list_of_files)
512        else:
513            self.logger.info("\n\n\tSkip copying FFDC files from remote system %s.\n" % self.hostname)
514
515    def get_command_list(self,
516                         ffdc_actions_for_target_type):
517        r"""
518        Fetch list of commands from configuration file
519
520        Description of argument(s):
521        ffdc_actions_for_target_type    commands and files for the selected remote host type.
522        """
523        try:
524            list_of_commands = ffdc_actions_for_target_type['COMMANDS']
525        except KeyError:
526            list_of_commands = []
527        return list_of_commands
528
529    def get_file_list(self,
530                      ffdc_actions_for_target_type):
531        r"""
532        Fetch list of commands from configuration file
533
534        Description of argument(s):
535        ffdc_actions_for_target_type    commands and files for the selected remote host type.
536        """
537        try:
538            list_of_files = ffdc_actions_for_target_type['FILES']
539        except KeyError:
540            list_of_files = []
541        return list_of_files
542
543    def unpack_command(self,
544                       command):
545        r"""
546        Unpack command from config file
547
548        Description of argument(s):
549        command    Command from config file.
550        """
551        if isinstance(command, dict):
552            command_txt = next(iter(command))
553            command_timeout = next(iter(command.values()))
554        elif isinstance(command, str):
555            command_txt = command
556            # Default command timeout 60 seconds
557            command_timeout = 60
558
559        return command_txt, command_timeout
560
561    def ssh_execute_ffdc_commands(self,
562                                  ffdc_actions_for_target_type,
563                                  form_filename=False):
564        r"""
565        Send commands in ffdc_config file to targeted system.
566
567        Description of argument(s):
568        ffdc_actions_for_target_type    commands and files for the selected remote host type.
569        form_filename                    if true, pre-pend self.target_type to filename
570        """
571        self.logger.info("\n\t[Run] Executing commands on %s using %s"
572                         % (self.hostname, ffdc_actions_for_target_type['PROTOCOL'][0]))
573
574        list_of_commands = self.get_command_list(ffdc_actions_for_target_type)
575        # If command list is empty, returns
576        if not list_of_commands:
577            return
578
579        progress_counter = 0
580        for command in list_of_commands:
581            command_txt, command_timeout = self.unpack_command(command)
582
583            if form_filename:
584                command_txt = str(command_txt % self.target_type)
585
586            cmd_exit_code, err, response = \
587                self.ssh_remoteclient.execute_command(command_txt, command_timeout)
588
589            if cmd_exit_code:
590                self.logger.warning(
591                    "\n\t\t[WARN] %s exits with code %s." % (command_txt, str(cmd_exit_code)))
592                self.logger.warning("\t\t[WARN] %s " % err)
593
594            progress_counter += 1
595            self.print_progress(progress_counter)
596
597        self.logger.info("\n\t[Run] Commands execution completed.\t\t [OK]")
598
599    def group_copy(self,
600                   ffdc_actions_for_target_type):
601        r"""
602        scp group of files (wild card) from remote host.
603
604        Description of argument(s):
605        fdc_actions_for_target_type    commands and files for the selected remote host type.
606        """
607
608        if self.ssh_remoteclient.scpclient:
609            self.logger.info("\n\tCopying DUMP files from remote system %s.\n" % self.hostname)
610
611            list_of_commands = self.get_command_list(ffdc_actions_for_target_type)
612            # If command list is empty, returns
613            if not list_of_commands:
614                return
615
616            for command in list_of_commands:
617                try:
618                    filename = command.split(' ')[2]
619                except IndexError:
620                    self.logger.info("\t\tInvalid command %s for DUMP_LOGS block." % command)
621                    continue
622
623                cmd_exit_code, err, response = \
624                    self.ssh_remoteclient.execute_command(command)
625
626                # If file does not exist, code take no action.
627                # cmd_exit_code is ignored for this scenario.
628                if response:
629                    scp_result = self.ssh_remoteclient.scp_file_from_remote(filename, self.ffdc_dir_path)
630                    if scp_result:
631                        self.logger.info("\t\tSuccessfully copied from " + self.hostname + ':' + filename)
632                else:
633                    self.logger.info("\t\tThere is no " + filename)
634
635        else:
636            self.logger.info("\n\n\tSkip copying files from remote system %s.\n" % self.hostname)
637
638    def scp_ffdc(self,
639                 targ_dir_path,
640                 targ_file_prefix,
641                 form_filename,
642                 file_list=None,
643                 quiet=None):
644        r"""
645        SCP all files in file_dict to the indicated directory on the local system.
646
647        Description of argument(s):
648        targ_dir_path                   The path of the directory to receive the files.
649        targ_file_prefix                Prefix which will be pre-pended to each
650                                        target file's name.
651        file_dict                       A dictionary of files to scp from targeted system to this system
652
653        """
654
655        progress_counter = 0
656        for filename in file_list:
657            if form_filename:
658                filename = str(filename % self.target_type)
659            source_file_path = filename
660            targ_file_path = targ_dir_path + targ_file_prefix + filename.split('/')[-1]
661
662            # If source file name contains wild card, copy filename as is.
663            if '*' in source_file_path:
664                scp_result = self.ssh_remoteclient.scp_file_from_remote(source_file_path, self.ffdc_dir_path)
665            else:
666                scp_result = self.ssh_remoteclient.scp_file_from_remote(source_file_path, targ_file_path)
667
668            if not quiet:
669                if scp_result:
670                    self.logger.info(
671                        "\t\tSuccessfully copied from " + self.hostname + ':' + source_file_path + ".\n")
672                else:
673                    self.logger.info(
674                        "\t\tFail to copy from " + self.hostname + ':' + source_file_path + ".\n")
675            else:
676                progress_counter += 1
677                self.print_progress(progress_counter)
678
679    def set_ffdc_defaults(self):
680        r"""
681        Set a default value for self.ffdc_dir_path and self.ffdc_prefix.
682        Collected ffdc file will be stored in dir /self.location/hostname_timestr/.
683        Individual ffdc file will have timestr_filename.
684
685        Description of class variables:
686        self.ffdc_dir_path  The dir path where collected ffdc data files should be put.
687
688        self.ffdc_prefix    The prefix to be given to each ffdc file name.
689
690        """
691
692        timestr = time.strftime("%Y%m%d-%H%M%S")
693        self.ffdc_dir_path = self.location + "/" + self.hostname + "_" + timestr + "/"
694        self.ffdc_prefix = timestr + "_"
695        self.validate_local_store(self.ffdc_dir_path)
696
697    def validate_local_store(self, dir_path):
698        r"""
699        Ensure path exists to store FFDC files locally.
700
701        Description of variable:
702        dir_path  The dir path where collected ffdc data files will be stored.
703
704        """
705
706        if not os.path.exists(dir_path):
707            try:
708                os.makedirs(dir_path, 0o755)
709            except (IOError, OSError) as e:
710                # PermissionError
711                if e.errno == EPERM or e.errno == EACCES:
712                    self.logger.error(
713                        '\tERROR: os.makedirs %s failed with PermissionError.\n' % dir_path)
714                else:
715                    self.logger.error(
716                        '\tERROR: os.makedirs %s failed with %s.\n' % (dir_path, e.strerror))
717                sys.exit(-1)
718
719    def print_progress(self, progress):
720        r"""
721        Print activity progress +
722
723        Description of variable:
724        progress  Progress counter.
725
726        """
727
728        sys.stdout.write("\r\t" + "+" * progress)
729        sys.stdout.flush()
730        time.sleep(.1)
731
732    def verify_redfish(self):
733        r"""
734        Verify remote host has redfish service active
735
736        """
737        redfish_parm = '-u ' + self.username + ' -p ' + self.password + ' -r ' \
738                       + self.hostname + ' -S Always raw GET /redfish/v1/'
739        return(self.run_redfishtool(redfish_parm, True))
740
741    def verify_ipmi(self):
742        r"""
743        Verify remote host has IPMI LAN service active
744
745        """
746        ipmi_parm = '-U ' + self.username + ' -P ' + self.password + ' -H ' \
747            + self.hostname + ' power status -I lanplus'
748        return(self.run_ipmitool(ipmi_parm, True))
749
750    def run_redfishtool(self,
751                        parms_string,
752                        quiet=False):
753        r"""
754        Run CLI redfishtool
755
756        Description of variable:
757        parms_string         redfishtool subcommand and options.
758        quiet                do not print redfishtool error message if True
759        """
760
761        result = subprocess.run(['redfishtool ' + parms_string],
762                                stdout=subprocess.PIPE,
763                                stderr=subprocess.PIPE,
764                                shell=True,
765                                universal_newlines=True)
766
767        if result.stderr and not quiet:
768            self.logger.error('\n\tERROR with redfishtool ' + parms_string)
769            self.logger.error('\n\t%s' % result.stderr.split('\n'))
770
771        return result.stdout
772
773    def run_ipmitool(self,
774                     parms_string,
775                     quiet=False):
776        r"""
777        Run CLI IPMI tool.
778
779        Description of variable:
780        parms_string         ipmitool subcommand and options.
781        quiet                do not print redfishtool error message if True
782        """
783
784        result = subprocess.run(['ipmitool -I lanplus -C 17 ' + parms_string],
785                                stdout=subprocess.PIPE,
786                                stderr=subprocess.PIPE,
787                                shell=True,
788                                universal_newlines=True)
789
790        if result.stderr and not quiet:
791            self.logger.error('\n\t\tERROR with ipmitool -I lanplus -C 17 ' + parms_string)
792            self.logger.error('\t\t' + result.stderr)
793
794        return result.stdout
795
796    def run_shell_script(self,
797                         parms_string,
798                         quiet=False):
799        r"""
800        Run CLI shell script tool.
801
802        Description of variable:
803        parms_string         script command options.
804        quiet                do not print redfishtool error message if True
805        """
806
807        result = subprocess.run([parms_string],
808                                stdout=subprocess.PIPE,
809                                stderr=subprocess.PIPE,
810                                shell=True,
811                                universal_newlines=True)
812
813        if result.stderr and not quiet:
814            self.logger.error('\n\t\tERROR executing %s' % parms_string)
815            self.logger.error('\t\t' + result.stderr)
816
817        return result.stdout
818
819    def protocol_shell_script(self,
820                              target_type,
821                              sub_type):
822        r"""
823        Perform SHELL script execution locally.
824
825        Description of argument(s):
826        target_type         OS Type of remote host.
827        sub_type            Group type of commands.
828        """
829
830        self.logger.info("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'SHELL'))
831        shell_files_saved = []
832        progress_counter = 0
833        list_of_cmd = self.get_command_list(self.ffdc_actions[target_type][sub_type])
834        for index, each_cmd in enumerate(list_of_cmd, start=0):
835
836            result = self.run_shell_script(each_cmd)
837            if result:
838                try:
839                    targ_file = self.get_file_list(self.ffdc_actions[target_type][sub_type])[index]
840                except IndexError:
841                    targ_file = each_cmd.split('/')[-1]
842                    self.logger.warning("\n\t[WARN] Missing filename to store data %s." % each_cmd)
843                    self.logger.warning("\t[WARN] Data will be stored in %s." % targ_file)
844
845                targ_file_with_path = (self.ffdc_dir_path
846                                       + self.ffdc_prefix
847                                       + targ_file)
848
849                # Creates a new file
850                with open(targ_file_with_path, 'w') as fp:
851                    fp.write(result)
852                    fp.close
853                    shell_files_saved.append(targ_file)
854
855            progress_counter += 1
856            self.print_progress(progress_counter)
857
858        self.logger.info("\n\t[Run] Commands execution completed.\t\t [OK]")
859
860        for file in shell_files_saved:
861            self.logger.info("\n\t\tSuccessfully save file " + file + ".")
862
863    def verify_protocol(self, protocol_list):
864        r"""
865        Perform protocol working check.
866
867        Description of argument(s):
868        protocol_list        List of protocol.
869        """
870
871        tmp_list = []
872        if self.target_is_pingable():
873            tmp_list.append("SHELL")
874
875        for protocol in protocol_list:
876            if self.remote_protocol != 'ALL':
877                if self.remote_protocol != protocol:
878                    continue
879
880            # Only check SSH/SCP once for both protocols
881            if protocol == 'SSH' or protocol == 'SCP' and protocol not in tmp_list:
882                if self.ssh_to_target_system():
883                    # Add only what user asked.
884                    if self.remote_protocol != 'ALL':
885                        tmp_list.append(self.remote_protocol)
886                    else:
887                        tmp_list.append('SSH')
888                        tmp_list.append('SCP')
889
890            if protocol == 'TELNET':
891                if self.telnet_to_target_system():
892                    tmp_list.append(protocol)
893
894            if protocol == 'REDFISH':
895                if self.verify_redfish():
896                    tmp_list.append(protocol)
897                    self.logger.info("\n\t[Check] %s Redfish Service.\t\t [OK]" % self.hostname)
898                else:
899                    self.logger.info("\n\t[Check] %s Redfish Service.\t\t [NOT AVAILABLE]" % self.hostname)
900
901            if protocol == 'IPMI':
902                if self.verify_ipmi():
903                    tmp_list.append(protocol)
904                    self.logger.info("\n\t[Check] %s IPMI LAN Service.\t\t [OK]" % self.hostname)
905                else:
906                    self.logger.info("\n\t[Check] %s IPMI LAN Service.\t\t [NOT AVAILABLE]" % self.hostname)
907
908        return tmp_list
909