xref: /openbmc/openbmc-test-automation/ffdc/ffdc_collector.py (revision 8e72e3e4ad81b71ad56db6161a6bc4f8334b436e)
1#!/usr/bin/env python
2
3r"""
4See class prolog below for details.
5"""
6
7import os
8import sys
9import yaml
10import json
11import time
12import logging
13import platform
14from errno import EACCES, EPERM
15import subprocess
16from ssh_utility import SSHRemoteclient
17from telnet_utility import TelnetRemoteclient
18
19
20class FFDCCollector:
21
22    r"""
23    Sends commands from configuration file to the targeted system to collect log files.
24    Fetch and store generated files at the specified location.
25
26    """
27
28    def __init__(self,
29                 hostname,
30                 username,
31                 password,
32                 ffdc_config,
33                 location,
34                 remote_type,
35                 remote_protocol,
36                 env_vars,
37                 log_level):
38        r"""
39        Description of argument(s):
40
41        hostname                name/ip of the targeted (remote) system
42        username                user on the targeted system with access to FFDC files
43        password                password for user on targeted system
44        ffdc_config             configuration file listing commands and files for FFDC
45        location                where to store collected FFDC
46        remote_type             os type of the remote host
47
48        """
49
50        self.hostname = hostname
51        self.username = username
52        self.password = password
53        # This is for the env vars a user can use in YAML to load it at runtime.
54        # Example YAML:
55        # -COMMANDS:
56        #    - my_command ${hostname}  ${username}   ${password}
57        os.environ['hostname'] = hostname
58        os.environ['username'] = username
59        os.environ['password'] = password
60
61        self.ffdc_config = ffdc_config
62        self.location = location + "/" + remote_type.upper()
63        self.ssh_remoteclient = None
64        self.telnet_remoteclient = None
65        self.ffdc_dir_path = ""
66        self.ffdc_prefix = ""
67        self.target_type = remote_type.upper()
68        self.remote_protocol = remote_protocol.upper()
69        self.start_time = 0
70        self.elapsed_time = ''
71        self.logger = None
72
73        # Set prefix values for scp files and directory.
74        # Since the time stamp is at second granularity, these values are set here
75        # to be sure that all files for this run will have same timestamps
76        # and they will be saved in the same directory.
77        # self.location == local system for now
78        self.set_ffdc_defaults()
79
80        # Logger for this run.  Need to be after set_ffdc_defaults()
81        self.script_logging(getattr(logging, log_level.upper()))
82
83        # Verify top level directory exists for storage
84        self.validate_local_store(self.location)
85
86        if self.verify_script_env():
87            # Load default or user define YAML configuration file.
88            with open(self.ffdc_config, 'r') as file:
89                self.ffdc_actions = yaml.load(file, Loader=yaml.FullLoader)
90
91            if self.target_type not in self.ffdc_actions.keys():
92                self.logger.error(
93                    "\n\tERROR: %s is not listed in %s.\n\n" % (self.target_type, self.ffdc_config))
94                sys.exit(-1)
95        else:
96            sys.exit(-1)
97
98        # Load ENV vars from user.
99        self.logger.info("\n\tENV: User define input YAML variables")
100        self.env_dict = {}
101
102        try:
103            if env_vars:
104                self.env_dict = json.loads(env_vars)
105
106                # Export ENV vars default.
107                for key, value in self.env_dict.items():
108                    os.environ[key] = value
109
110        except json.decoder.JSONDecodeError as e:
111            self.logger.error("\n\tERROR: %s " % e)
112            sys.exit(-1)
113
114        # Append default Env.
115        self.env_dict['hostname'] = self.hostname
116        self.env_dict['username'] = self.username
117        self.env_dict['password'] = self.password
118        self.logger.info(json.dumps(self.env_dict, indent=8, sort_keys=True))
119
120    def verify_script_env(self):
121
122        # Import to log version
123        import click
124        import paramiko
125
126        run_env_ok = True
127
128        redfishtool_version = self.run_redfishtool('-V').split(' ')[2].strip('\n')
129        ipmitool_version = self.run_ipmitool('-V').split(' ')[2]
130
131        self.logger.info("\n\t---- Script host environment ----")
132        self.logger.info("\t{:<10}  {:<10}".format('Script hostname', os.uname()[1]))
133        self.logger.info("\t{:<10}  {:<10}".format('Script host os', platform.platform()))
134        self.logger.info("\t{:<10}  {:>10}".format('Python', platform.python_version()))
135        self.logger.info("\t{:<10}  {:>10}".format('PyYAML', yaml.__version__))
136        self.logger.info("\t{:<10}  {:>10}".format('click', click.__version__))
137        self.logger.info("\t{:<10}  {:>10}".format('paramiko', paramiko.__version__))
138        self.logger.info("\t{:<10}  {:>9}".format('redfishtool', redfishtool_version))
139        self.logger.info("\t{:<10}  {:>12}".format('ipmitool', ipmitool_version))
140
141        if eval(yaml.__version__.replace('.', ',')) < (5, 4, 1):
142            self.logger.error("\n\tERROR: Python or python packages do not meet minimum version requirement.")
143            self.logger.error("\tERROR: PyYAML version 5.4.1 or higher is needed.\n")
144            run_env_ok = False
145
146        self.logger.info("\t---- End script host environment ----")
147        return run_env_ok
148
149    def script_logging(self,
150                       log_level_attr):
151        r"""
152        Create logger
153
154        """
155        self.logger = logging.getLogger()
156        self.logger.setLevel(log_level_attr)
157        log_file_handler = logging.FileHandler(self.ffdc_dir_path + "collector.log")
158
159        stdout_handler = logging.StreamHandler(sys.stdout)
160        self.logger.addHandler(log_file_handler)
161        self.logger.addHandler(stdout_handler)
162
163        # Turn off paramiko INFO logging
164        logging.getLogger("paramiko").setLevel(logging.WARNING)
165
166    def target_is_pingable(self):
167        r"""
168        Check if target system is ping-able.
169
170        """
171        response = os.system("ping -c 1 %s  2>&1 >/dev/null" % self.hostname)
172        if response == 0:
173            self.logger.info("\n\t[Check] %s is ping-able.\t\t [OK]" % self.hostname)
174            return True
175        else:
176            self.logger.error(
177                "\n>>>>>\tERROR: %s is not ping-able. FFDC collection aborted.\n" % self.hostname)
178            sys.exit(-1)
179
180    def collect_ffdc(self):
181        r"""
182        Initiate FFDC Collection depending on requested protocol.
183
184        """
185
186        self.logger.info("\n\t---- Start communicating with %s ----" % self.hostname)
187        self.start_time = time.time()
188        working_protocol_list = []
189        if self.target_is_pingable():
190            working_protocol_list.append("SHELL")
191            # Check supported protocol ping,ssh, redfish are working.
192            if self.ssh_to_target_system():
193                working_protocol_list.append("SSH")
194                working_protocol_list.append("SCP")
195
196            # Redfish
197            if self.verify_redfish():
198                working_protocol_list.append("REDFISH")
199                self.logger.info("\n\t[Check] %s Redfish Service.\t\t [OK]" % self.hostname)
200            else:
201                self.logger.info("\n\t[Check] %s Redfish Service.\t\t [NOT AVAILABLE]" % self.hostname)
202
203            # IPMI
204            if self.verify_ipmi():
205                working_protocol_list.append("IPMI")
206                self.logger.info("\n\t[Check] %s IPMI LAN Service.\t\t [OK]" % self.hostname)
207            else:
208                self.logger.info("\n\t[Check] %s IPMI LAN Service.\t\t [NOT AVAILABLE]" % self.hostname)
209
210            # Telnet
211            if self.telnet_to_target_system():
212                working_protocol_list.append("TELNET")
213
214            # Verify top level directory exists for storage
215            self.validate_local_store(self.location)
216            self.logger.info("\n\t---- Completed protocol pre-requisite check ----\n")
217
218            if ((self.remote_protocol not in working_protocol_list) and (self.remote_protocol != 'ALL')):
219                self.logger.info("\n\tWorking protocol list: %s" % working_protocol_list)
220                self.logger.error(
221                    '>>>>>\tERROR: Requested protocol %s is not in working protocol list.\n'
222                    % self.remote_protocol)
223                sys.exit(-1)
224            else:
225                self.generate_ffdc(working_protocol_list)
226
227    def ssh_to_target_system(self):
228        r"""
229        Open a ssh connection to targeted system.
230
231        """
232
233        self.ssh_remoteclient = SSHRemoteclient(self.hostname,
234                                                self.username,
235                                                self.password)
236
237        if self.ssh_remoteclient.ssh_remoteclient_login():
238            self.logger.info("\n\t[Check] %s SSH connection established.\t [OK]" % self.hostname)
239
240            # Check scp connection.
241            # If scp connection fails,
242            # continue with FFDC generation but skip scp files to local host.
243            self.ssh_remoteclient.scp_connection()
244            return True
245        else:
246            self.logger.info("\n\t[Check] %s SSH connection.\t [NOT AVAILABLE]" % self.hostname)
247            return False
248
249    def telnet_to_target_system(self):
250        r"""
251        Open a telnet connection to targeted system.
252        """
253        self.telnet_remoteclient = TelnetRemoteclient(self.hostname,
254                                                      self.username,
255                                                      self.password)
256        if self.telnet_remoteclient.tn_remoteclient_login():
257            self.logger.info("\n\t[Check] %s Telnet connection established.\t [OK]" % self.hostname)
258            return True
259        else:
260            self.logger.info("\n\t[Check] %s Telnet connection.\t [NOT AVAILABLE]" % self.hostname)
261            return False
262
263    def generate_ffdc(self, working_protocol_list):
264        r"""
265        Determine actions based on remote host type
266
267        Description of argument(s):
268        working_protocol_list    list of confirmed working protocols to connect to remote host.
269        """
270
271        self.logger.info("\n\t---- Executing commands on " + self.hostname + " ----")
272        self.logger.info("\n\tWorking protocol list: %s" % working_protocol_list)
273
274        ffdc_actions = self.ffdc_actions
275
276        for machine_type in ffdc_actions.keys():
277            if self.target_type != machine_type:
278                continue
279
280            self.logger.info("\n\tFFDC Path: %s " % self.ffdc_dir_path)
281            self.logger.info("\tSystem Type: %s" % machine_type)
282            for k, v in ffdc_actions[machine_type].items():
283
284                if self.remote_protocol != ffdc_actions[machine_type][k]['PROTOCOL'][0] \
285                        and self.remote_protocol != 'ALL':
286                    continue
287
288                if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'SSH' \
289                   or ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'SCP':
290                    if 'SSH' in working_protocol_list \
291                       or 'SCP' in working_protocol_list:
292                        self.protocol_ssh(ffdc_actions, machine_type, k)
293                    else:
294                        self.logger.error("\n\tERROR: SSH or SCP is not available for %s." % self.hostname)
295
296                if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'TELNET':
297                    if 'TELNET' in working_protocol_list:
298                        self.protocol_telnet(ffdc_actions, machine_type, k)
299                    else:
300                        self.logger.error("\n\tERROR: TELNET is not available for %s." % self.hostname)
301
302                if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'REDFISH':
303                    if 'REDFISH' in working_protocol_list:
304                        self.protocol_redfish(ffdc_actions, machine_type, k)
305                    else:
306                        self.logger.error("\n\tERROR: REDFISH is not available for %s." % self.hostname)
307
308                if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'IPMI':
309                    if 'IPMI' in working_protocol_list:
310                        self.protocol_ipmi(ffdc_actions, machine_type, k)
311                    else:
312                        self.logger.error("\n\tERROR: IPMI is not available for %s." % self.hostname)
313
314                if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'SHELL':
315                    if 'SHELL' in working_protocol_list:
316                        self.protocol_shell_script(ffdc_actions, machine_type, k)
317                    else:
318                        self.logger.error("\n\tERROR: can't execute SHELL script")
319
320        # Close network connection after collecting all files
321        self.elapsed_time = time.strftime("%H:%M:%S", time.gmtime(time.time() - self.start_time))
322        self.ssh_remoteclient.ssh_remoteclient_disconnect()
323        self.telnet_remoteclient.tn_remoteclient_disconnect()
324
325    def protocol_ssh(self,
326                     ffdc_actions,
327                     machine_type,
328                     sub_type):
329        r"""
330        Perform actions using SSH and SCP protocols.
331
332        Description of argument(s):
333        ffdc_actions        List of actions from ffdc_config.yaml.
334        machine_type        OS Type of remote host.
335        sub_type            Group type of commands.
336        """
337
338        if sub_type == 'DUMP_LOGS':
339            self.group_copy(ffdc_actions[machine_type][sub_type])
340        else:
341            self.collect_and_copy_ffdc(ffdc_actions[machine_type][sub_type])
342
343    def protocol_telnet(self,
344                        ffdc_actions,
345                        machine_type,
346                        sub_type):
347        r"""
348        Perform actions using telnet protocol.
349        Description of argument(s):
350        ffdc_actions        List of actions from ffdc_config.yaml.
351        machine_type        OS Type of remote host.
352        """
353        self.logger.info("\n\t[Run] Executing commands on %s using %s" % (self.hostname, 'TELNET'))
354        telnet_files_saved = []
355        progress_counter = 0
356        list_of_commands = ffdc_actions[machine_type][sub_type]['COMMANDS']
357        for index, each_cmd in enumerate(list_of_commands, start=0):
358            command_txt, command_timeout = self.unpack_command(each_cmd)
359            result = self.telnet_remoteclient.execute_command(command_txt, command_timeout)
360            if result:
361                try:
362                    targ_file = ffdc_actions[machine_type][sub_type]['FILES'][index]
363                except IndexError:
364                    targ_file = command_txt
365                    self.logger.warning(
366                        "\n\t[WARN] Missing filename to store data from telnet %s." % each_cmd)
367                    self.logger.warning("\t[WARN] Data will be stored in %s." % targ_file)
368                targ_file_with_path = (self.ffdc_dir_path
369                                       + self.ffdc_prefix
370                                       + targ_file)
371                # Creates a new file
372                with open(targ_file_with_path, 'wb') as fp:
373                    fp.write(result)
374                    fp.close
375                    telnet_files_saved.append(targ_file)
376            progress_counter += 1
377            self.print_progress(progress_counter)
378        self.logger.info("\n\t[Run] Commands execution completed.\t\t [OK]")
379        for file in telnet_files_saved:
380            self.logger.info("\n\t\tSuccessfully save file " + file + ".")
381
382    def protocol_redfish(self,
383                         ffdc_actions,
384                         machine_type,
385                         sub_type):
386        r"""
387        Perform actions using Redfish protocol.
388
389        Description of argument(s):
390        ffdc_actions        List of actions from ffdc_config.yaml.
391        machine_type        OS Type of remote host.
392        sub_type            Group type of commands.
393        """
394
395        self.logger.info("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'REDFISH'))
396        redfish_files_saved = []
397        progress_counter = 0
398        list_of_URL = ffdc_actions[machine_type][sub_type]['URL']
399        for index, each_url in enumerate(list_of_URL, start=0):
400            redfish_parm = '-u ' + self.username + ' -p ' + self.password + ' -r ' \
401                           + self.hostname + ' -S Always raw GET ' + each_url
402
403            result = self.run_redfishtool(redfish_parm)
404            if result:
405                try:
406                    targ_file = self.get_file_list(ffdc_actions[machine_type][sub_type])[index]
407                except IndexError:
408                    targ_file = each_url.split('/')[-1]
409                    self.logger.warning(
410                        "\n\t[WARN] Missing filename to store data from redfish URL %s." % each_url)
411                    self.logger.warning("\t[WARN] Data will be stored in %s." % targ_file)
412
413                targ_file_with_path = (self.ffdc_dir_path
414                                       + self.ffdc_prefix
415                                       + targ_file)
416
417                # Creates a new file
418                with open(targ_file_with_path, 'w') as fp:
419                    fp.write(result)
420                    fp.close
421                    redfish_files_saved.append(targ_file)
422
423            progress_counter += 1
424            self.print_progress(progress_counter)
425
426        self.logger.info("\n\t[Run] Commands execution completed.\t\t [OK]")
427
428        for file in redfish_files_saved:
429            self.logger.info("\n\t\tSuccessfully save file " + file + ".")
430
431    def protocol_ipmi(self,
432                      ffdc_actions,
433                      machine_type,
434                      sub_type):
435        r"""
436        Perform actions using ipmitool over LAN protocol.
437
438        Description of argument(s):
439        ffdc_actions        List of actions from ffdc_config.yaml.
440        machine_type        OS Type of remote host.
441        sub_type            Group type of commands.
442        """
443
444        self.logger.info("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'IPMI'))
445        ipmi_files_saved = []
446        progress_counter = 0
447        list_of_cmd = self.get_command_list(ffdc_actions[machine_type][sub_type])
448        for index, each_cmd in enumerate(list_of_cmd, start=0):
449            ipmi_parm = '-U ' + self.username + ' -P ' + self.password + ' -H ' \
450                + self.hostname + ' -I lanplus ' + each_cmd
451
452            result = self.run_ipmitool(ipmi_parm)
453            if result:
454                try:
455                    targ_file = self.get_file_list(ffdc_actions[machine_type][sub_type])[index]
456                except IndexError:
457                    targ_file = each_cmd.split('/')[-1]
458                    self.logger.warning("\n\t[WARN] Missing filename to store data from IPMI %s." % each_cmd)
459                    self.logger.warning("\t[WARN] Data will be stored in %s." % targ_file)
460
461                targ_file_with_path = (self.ffdc_dir_path
462                                       + self.ffdc_prefix
463                                       + targ_file)
464
465                # Creates a new file
466                with open(targ_file_with_path, 'w') as fp:
467                    fp.write(result)
468                    fp.close
469                    ipmi_files_saved.append(targ_file)
470
471            progress_counter += 1
472            self.print_progress(progress_counter)
473
474        self.logger.info("\n\t[Run] Commands execution completed.\t\t [OK]")
475
476        for file in ipmi_files_saved:
477            self.logger.info("\n\t\tSuccessfully save file " + file + ".")
478
479    def collect_and_copy_ffdc(self,
480                              ffdc_actions_for_machine_type,
481                              form_filename=False):
482        r"""
483        Send commands in ffdc_config file to targeted system.
484
485        Description of argument(s):
486        ffdc_actions_for_machine_type    commands and files for the selected remote host type.
487        form_filename                    if true, pre-pend self.target_type to filename
488        """
489
490        # Executing commands, if any
491        self.ssh_execute_ffdc_commands(ffdc_actions_for_machine_type,
492                                       form_filename)
493
494        # Copying files
495        if self.ssh_remoteclient.scpclient:
496            self.logger.info("\n\n\tCopying FFDC files from remote system %s.\n" % self.hostname)
497
498            # Retrieving files from target system
499            list_of_files = self.get_file_list(ffdc_actions_for_machine_type)
500            self.scp_ffdc(self.ffdc_dir_path, self.ffdc_prefix, form_filename, list_of_files)
501        else:
502            self.logger.info("\n\n\tSkip copying FFDC files from remote system %s.\n" % self.hostname)
503
504    def get_command_list(self,
505                         ffdc_actions_for_machine_type):
506        r"""
507        Fetch list of commands from configuration file
508
509        Description of argument(s):
510        ffdc_actions_for_machine_type    commands and files for the selected remote host type.
511        """
512        try:
513            list_of_commands = ffdc_actions_for_machine_type['COMMANDS']
514        except KeyError:
515            list_of_commands = []
516        return list_of_commands
517
518    def get_file_list(self,
519                      ffdc_actions_for_machine_type):
520        r"""
521        Fetch list of commands from configuration file
522
523        Description of argument(s):
524        ffdc_actions_for_machine_type    commands and files for the selected remote host type.
525        """
526        try:
527            list_of_files = ffdc_actions_for_machine_type['FILES']
528        except KeyError:
529            list_of_files = []
530        return list_of_files
531
532    def unpack_command(self,
533                       command):
534        r"""
535        Unpack command from config file
536
537        Description of argument(s):
538        command    Command from config file.
539        """
540        if isinstance(command, dict):
541            command_txt = next(iter(command))
542            command_timeout = next(iter(command.values()))
543        elif isinstance(command, str):
544            command_txt = command
545            # Default command timeout 60 seconds
546            command_timeout = 60
547
548        return command_txt, command_timeout
549
550    def ssh_execute_ffdc_commands(self,
551                                  ffdc_actions_for_machine_type,
552                                  form_filename=False):
553        r"""
554        Send commands in ffdc_config file to targeted system.
555
556        Description of argument(s):
557        ffdc_actions_for_machine_type    commands and files for the selected remote host type.
558        form_filename                    if true, pre-pend self.target_type to filename
559        """
560        self.logger.info("\n\t[Run] Executing commands on %s using %s"
561                         % (self.hostname, ffdc_actions_for_machine_type['PROTOCOL'][0]))
562
563        list_of_commands = self.get_command_list(ffdc_actions_for_machine_type)
564        # If command list is empty, returns
565        if not list_of_commands:
566            return
567
568        progress_counter = 0
569        for command in list_of_commands:
570            command_txt, command_timeout = self.unpack_command(command)
571
572            if form_filename:
573                command_txt = str(command_txt % self.target_type)
574
575            cmd_exit_code, err, response = \
576                self.ssh_remoteclient.execute_command(command_txt, command_timeout)
577
578            if cmd_exit_code:
579                self.logger.warning(
580                    "\n\t\t[WARN] %s exits with code %s." % (command_txt, str(cmd_exit_code)))
581                self.logger.warning("\t\t[WARN] %s " % err)
582
583            progress_counter += 1
584            self.print_progress(progress_counter)
585
586        self.logger.info("\n\t[Run] Commands execution completed.\t\t [OK]")
587
588    def group_copy(self,
589                   ffdc_actions_for_machine_type):
590        r"""
591        scp group of files (wild card) from remote host.
592
593        Description of argument(s):
594        ffdc_actions_for_machine_type    commands and files for the selected remote host type.
595        """
596
597        if self.ssh_remoteclient.scpclient:
598            self.logger.info("\n\tCopying DUMP files from remote system %s.\n" % self.hostname)
599
600            list_of_commands = self.get_command_list(ffdc_actions_for_machine_type)
601            # If command list is empty, returns
602            if not list_of_commands:
603                return
604
605            for command in list_of_commands:
606                try:
607                    filename = command.split(' ')[2]
608                except IndexError:
609                    self.logger.info("\t\tInvalid command %s for DUMP_LOGS block." % command)
610                    continue
611
612                cmd_exit_code, err, response = \
613                    self.ssh_remoteclient.execute_command(command)
614
615                # If file does not exist, code take no action.
616                # cmd_exit_code is ignored for this scenario.
617                if response:
618                    scp_result = self.ssh_remoteclient.scp_file_from_remote(filename, self.ffdc_dir_path)
619                    if scp_result:
620                        self.logger.info("\t\tSuccessfully copied from " + self.hostname + ':' + filename)
621                else:
622                    self.logger.info("\t\tThere is no " + filename)
623
624        else:
625            self.logger.info("\n\n\tSkip copying files from remote system %s.\n" % self.hostname)
626
627    def scp_ffdc(self,
628                 targ_dir_path,
629                 targ_file_prefix,
630                 form_filename,
631                 file_list=None,
632                 quiet=None):
633        r"""
634        SCP all files in file_dict to the indicated directory on the local system.
635
636        Description of argument(s):
637        targ_dir_path                   The path of the directory to receive the files.
638        targ_file_prefix                Prefix which will be pre-pended to each
639                                        target file's name.
640        file_dict                       A dictionary of files to scp from targeted system to this system
641
642        """
643
644        progress_counter = 0
645        for filename in file_list:
646            if form_filename:
647                filename = str(filename % self.target_type)
648            source_file_path = filename
649            targ_file_path = targ_dir_path + targ_file_prefix + filename.split('/')[-1]
650
651            # If source file name contains wild card, copy filename as is.
652            if '*' in source_file_path:
653                scp_result = self.ssh_remoteclient.scp_file_from_remote(source_file_path, self.ffdc_dir_path)
654            else:
655                scp_result = self.ssh_remoteclient.scp_file_from_remote(source_file_path, targ_file_path)
656
657            if not quiet:
658                if scp_result:
659                    self.logger.info(
660                        "\t\tSuccessfully copied from " + self.hostname + ':' + source_file_path + ".\n")
661                else:
662                    self.logger.info(
663                        "\t\tFail to copy from " + self.hostname + ':' + source_file_path + ".\n")
664            else:
665                progress_counter += 1
666                self.print_progress(progress_counter)
667
668    def set_ffdc_defaults(self):
669        r"""
670        Set a default value for self.ffdc_dir_path and self.ffdc_prefix.
671        Collected ffdc file will be stored in dir /self.location/hostname_timestr/.
672        Individual ffdc file will have timestr_filename.
673
674        Description of class variables:
675        self.ffdc_dir_path  The dir path where collected ffdc data files should be put.
676
677        self.ffdc_prefix    The prefix to be given to each ffdc file name.
678
679        """
680
681        timestr = time.strftime("%Y%m%d-%H%M%S")
682        self.ffdc_dir_path = self.location + "/" + self.hostname + "_" + timestr + "/"
683        self.ffdc_prefix = timestr + "_"
684        self.validate_local_store(self.ffdc_dir_path)
685
686    def validate_local_store(self, dir_path):
687        r"""
688        Ensure path exists to store FFDC files locally.
689
690        Description of variable:
691        dir_path  The dir path where collected ffdc data files will be stored.
692
693        """
694
695        if not os.path.exists(dir_path):
696            try:
697                os.makedirs(dir_path, 0o755)
698            except (IOError, OSError) as e:
699                # PermissionError
700                if e.errno == EPERM or e.errno == EACCES:
701                    self.logger.error(
702                        '>>>>>\tERROR: os.makedirs %s failed with PermissionError.\n' % dir_path)
703                else:
704                    self.logger.error(
705                        '>>>>>\tERROR: os.makedirs %s failed with %s.\n' % (dir_path, e.strerror))
706                sys.exit(-1)
707
708    def print_progress(self, progress):
709        r"""
710        Print activity progress +
711
712        Description of variable:
713        progress  Progress counter.
714
715        """
716
717        sys.stdout.write("\r\t" + "+" * progress)
718        sys.stdout.flush()
719        time.sleep(.1)
720
721    def verify_redfish(self):
722        r"""
723        Verify remote host has redfish service active
724
725        """
726        redfish_parm = '-u ' + self.username + ' -p ' + self.password + ' -r ' \
727                       + self.hostname + ' -S Always raw GET /redfish/v1/'
728        return(self.run_redfishtool(redfish_parm, True))
729
730    def verify_ipmi(self):
731        r"""
732        Verify remote host has IPMI LAN service active
733
734        """
735        ipmi_parm = '-U ' + self.username + ' -P ' + self.password + ' -H ' \
736            + self.hostname + ' power status -I lanplus'
737        return(self.run_ipmitool(ipmi_parm, True))
738
739    def run_redfishtool(self,
740                        parms_string,
741                        quiet=False):
742        r"""
743        Run CLI redfishtool
744
745        Description of variable:
746        parms_string         redfishtool subcommand and options.
747        quiet                do not print redfishtool error message if True
748        """
749
750        result = subprocess.run(['redfishtool ' + parms_string],
751                                stdout=subprocess.PIPE,
752                                stderr=subprocess.PIPE,
753                                shell=True,
754                                universal_newlines=True)
755
756        if result.stderr and not quiet:
757            self.logger.error('\n\t\tERROR with redfishtool ' + parms_string)
758            self.logger.error('\t\t' + result.stderr)
759
760        return result.stdout
761
762    def run_ipmitool(self,
763                     parms_string,
764                     quiet=False):
765        r"""
766        Run CLI IPMI tool.
767
768        Description of variable:
769        parms_string         ipmitool subcommand and options.
770        quiet                do not print redfishtool error message if True
771        """
772
773        result = subprocess.run(['ipmitool -I lanplus -C 17 ' + parms_string],
774                                stdout=subprocess.PIPE,
775                                stderr=subprocess.PIPE,
776                                shell=True,
777                                universal_newlines=True)
778
779        if result.stderr and not quiet:
780            self.logger.error('\n\t\tERROR with ipmitool -I lanplus -C 17 ' + parms_string)
781            self.logger.error('\t\t' + result.stderr)
782
783        return result.stdout
784
785    def run_shell_script(self,
786                         parms_string,
787                         quiet=False):
788        r"""
789        Run CLI shell script tool.
790
791        Description of variable:
792        parms_string         script command options.
793        quiet                do not print redfishtool error message if True
794        """
795
796        result = subprocess.run([parms_string],
797                                stdout=subprocess.PIPE,
798                                stderr=subprocess.PIPE,
799                                shell=True,
800                                universal_newlines=True)
801
802        if result.stderr and not quiet:
803            self.logger.error('\n\t\tERROR executing %s' % parms_string)
804            self.logger.error('\t\t' + result.stderr)
805
806        return result.stdout
807
808    def protocol_shell_script(self,
809                              ffdc_actions,
810                              machine_type,
811                              sub_type):
812        r"""
813        Perform SHELL script execution locally.
814
815        Description of argument(s):
816        ffdc_actions        List of actions from ffdc_config.yaml.
817        machine_type        OS Type of remote host.
818        sub_type            Group type of commands.
819        """
820
821        self.logger.info("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'SHELL'))
822        shell_files_saved = []
823        progress_counter = 0
824        list_of_cmd = self.get_command_list(ffdc_actions[machine_type][sub_type])
825        for index, each_cmd in enumerate(list_of_cmd, start=0):
826
827            result = self.run_shell_script(each_cmd)
828            if result:
829                try:
830                    targ_file = self.get_file_list(ffdc_actions[machine_type][sub_type])[index]
831                except IndexError:
832                    targ_file = each_cmd.split('/')[-1]
833                    self.logger.warning("\n\t[WARN] Missing filename to store data %s." % each_cmd)
834                    self.logger.warning("\t[WARN] Data will be stored in %s." % targ_file)
835
836                targ_file_with_path = (self.ffdc_dir_path
837                                       + self.ffdc_prefix
838                                       + targ_file)
839
840                # Creates a new file
841                with open(targ_file_with_path, 'w') as fp:
842                    fp.write(result)
843                    fp.close
844                    shell_files_saved.append(targ_file)
845
846            progress_counter += 1
847            self.print_progress(progress_counter)
848
849        self.logger.info("\n\t[Run] Commands execution completed.\t\t [OK]")
850
851        for file in shell_files_saved:
852            self.logger.info("\n\t\tSuccessfully save file " + file + ".")
853