#!/usr/bin/env python3 r""" Use robot framework API to extract test result data from output.xml generated by robot tests. For more information on the Robot Framework API, see http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html """ import sys import os import getopt import csv import robot.errors import re import stat import datetime from robot.api import ExecutionResult from robot.result.visitor import ResultVisitor from xml.etree import ElementTree # Remove the python library path to restore with local project path later. save_path_0 = sys.path[0] del sys.path[0] sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib")) from gen_arg import * from gen_print import * from gen_valid import * # Restore sys.path[0]. sys.path.insert(0, save_path_0) this_program = sys.argv[0] info = " For more information: " + this_program + ' -h' if len(sys.argv) == 1: print(info) sys.exit(1) parser = argparse.ArgumentParser( usage=info, description="%(prog)s uses a robot framework API to extract test result\ data from output.xml generated by robot tests. For more information on the\ Robot Framework API, see\ http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html", formatter_class=argparse.ArgumentDefaultsHelpFormatter, prefix_chars='-+') parser.add_argument( '--source', '-s', help='The output.xml robot test result file path. This parameter is \ required.') parser.add_argument( '--dest', '-d', help='The directory path where the generated .csv files will go. This \ parameter is required.') parser.add_argument( '--version_id', help='Driver version of openbmc firmware which was used during test,\ e.g. "v2.1-215-g6e7eacb". This parameter is required.') parser.add_argument( '--platform', help='OpenBMC platform which was used during test,\ e.g. "Witherspoon". This parameter is required.') parser.add_argument( '--level', help='OpenBMC release level which was used during test,\ e.g. "Master", "OBMC920". This parameter is required.') parser.add_argument( '--test_phase', help='Name of testing phase, e.g. "DVT", "SVT", etc.\ This parameter is optional.', default="FVT") parser.add_argument( '--subsystem', help='Name of the subsystem, e.g. "OPENBMC" etc.\ This parameter is optional.', default="OPENBMC") parser.add_argument( '--processor', help='Name of processor, e.g. "P9". This parameter is optional.', default="OPENPOWER") # Populate stock_list with options we want. stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)] def exit_function(signal_number=0, frame=None): r""" Execute whenever the program ends normally or with the signals that we catch (i.e. TERM, INT). """ dprint_executing() dprint_var(signal_number) qprint_pgm_footer() def signal_handler(signal_number, frame): r""" Handle signals. Without a function to catch a SIGTERM or SIGINT, the program would terminate immediately with return code 143 and without calling the exit_function. """ # Our convention is to set up exit_function with atexit.register() so # there is no need to explicitly call exit_function from here. dprint_executing() # Calling exit prevents us from returning to the code that was running # when the signal was received. exit(0) def validate_parms(): r""" Validate program parameters, etc. Return True or False (i.e. pass/fail) accordingly. """ if not valid_file_path(source): return False if not valid_dir_path(dest): return False gen_post_validation(exit_function, signal_handler) return True def parse_output_xml(xml_file_path, csv_dir_path, version_id, platform, level, test_phase, processor): r""" Parse the robot-generated output.xml file and extract various test output data. Put the extracted information into a csv file in the "dest" folder. Description of argument(s): xml_file_path The path to a Robot-generated output.xml file. csv_dir_path The path to the directory that is to contain the .csv files generated by this function. version_id Version of the openbmc firmware (e.g. "v2.1-215-g6e7eacb"). platform Platform of the openbmc system. level Release level of the OpenBMC system (e.g. "Master"). """ # Initialize tallies total_critical_tc = 0 total_critical_passed = 0 total_critical_failed = 0 total_non_critical_tc = 0 total_non_critical_passed = 0 total_non_critical_failed = 0 result = ExecutionResult(xml_file_path) result.configure(stat_config={'suite_stat_level': 2, 'tag_stat_combine': 'tagANDanother'}) stats = result.statistics print("--------------------------------------") try: total_critical_tc = stats.total.critical.passed + stats.total.critical.failed total_critical_passed = stats.total.critical.passed total_critical_failed = stats.total.critical.failed except AttributeError: pass try: total_non_critical_tc = stats.total.passed + stats.total.failed total_non_critical_passed = stats.total.passed total_non_critical_failed = stats.total.failed except AttributeError: pass print("Total Test Count:\t %d" % (total_non_critical_tc + total_critical_tc)) print("Total Critical Test Failed:\t %d" % total_critical_failed) print("Total Critical Test Passed:\t %d" % total_critical_passed) print("Total Non-Critical Test Failed:\t %d" % total_non_critical_failed) print("Total Non-Critical Test Passed:\t %d" % total_non_critical_passed) print("Test Start Time:\t %s" % result.suite.starttime) print("Test End Time:\t\t %s" % result.suite.endtime) print("--------------------------------------") # Use ResultVisitor object and save off the test data info class TestResult(ResultVisitor): def __init__(self): self.testData = [] def visit_test(self, test): self.testData += [test] collectDataObj = TestResult() result.visit(collectDataObj) # Write the result statistics attributes to CSV file l_csvlist = [] # Default Test data l_test_type = test_phase l_pse_rel = 'Master' if level: l_pse_rel = level l_env = 'HW' l_proc = processor l_platform_type = "" l_func_area = "" # System data from XML meta data # l_system_info = get_system_details(xml_file_path) # First let us try to collect information from keyboard input # If keyboard input cannot give both information, then find from xml file. if version_id and platform: l_driver = version_id l_platform_type = platform print("BMC Version_id:%s" % version_id) print("BMC Platform:%s" % platform) else: # System data from XML meta data l_system_info = get_system_details(xml_file_path) l_driver = l_system_info[0] l_platform_type = l_system_info[1] # Driver version id and platform are mandatorily required for CSV file # generation. If any one is not avaulable, exit CSV file generation # process. if l_driver and l_platform_type: print("Driver and system info set.") else: print("Both driver and system info need to be set.\ CSV file is not generated.") sys.exit() # Default header l_header = ['test_start', 'test_end', 'subsys', 'test_type', 'test_result', 'test_name', 'pse_rel', 'driver', 'env', 'proc', 'platform_type', 'test_func_area'] l_csvlist.append(l_header) # Generate CSV file onto the path with current time stamp l_base_dir = csv_dir_path l_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S") # Example: 2017-02-20-08-47-22_Witherspoon.csv l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv" print("Writing data into csv file:%s" % l_csvfile) for testcase in collectDataObj.testData: # Functional Area: Suite Name # Test Name: Test Case Name l_func_area = str(testcase.parent).split(' ', 1)[1] l_test_name = str(testcase) # Test Result pass=0 fail=1 if testcase.status == 'PASS': l_test_result = 0 else: l_test_result = 1 # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S" l_stime = xml_to_csv_time(testcase.starttime) l_etime = xml_to_csv_time(testcase.endtime) # Data Sequence: test_start,test_end,subsys,test_type, # test_result,test_name,pse_rel,driver, # env,proc,platform_type,test_func_area, l_data = [l_stime, l_etime, subsystem, l_test_type, l_test_result, l_test_name, l_pse_rel, l_driver, l_env, l_proc, l_platform_type, l_func_area] l_csvlist.append(l_data) # Open the file and write to the CSV file l_file = open(l_csvfile, "w") l_writer = csv.writer(l_file, lineterminator='\n') l_writer.writerows(l_csvlist) l_file.close() # Set file permissions 666. perm = stat.S_IRUSR + stat.S_IWUSR + stat.S_IRGRP + stat.S_IWGRP + stat.S_IROTH + stat.S_IWOTH os.chmod(l_csvfile, perm) def xml_to_csv_time(xml_datetime): r""" Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format and return it. Description of argument(s): datetime The date in the following format: %Y%m%d %H:%M:%S.%f (This is the format typically found in an XML file.) The date returned will be in the following format: %Y-%m-%d-%H-%M-%S """ # 20170206 05:05:19.342 l_str = datetime.datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f") # 2017-02-06-05-05-19 l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S") return str(l_str) def get_system_details(xml_file_path): r""" Get the system data from output.xml generated by robot and return it. The list returned will be in the following order: [driver,platform] Description of argument(s): xml_file_path The relative or absolute path to the output.xml file. """ bmc_version_id = "" bmc_platform = "" with open(xml_file_path, 'rt') as output: tree = ElementTree.parse(output) for node in tree.iter('msg'): # /etc/os-release output is logged in the XML as msg # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79" if '${output} = VERSION_ID=' in node.text: # Get BMC version (e.g. v1.99.1-96-g2a46570) bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1] # Platform is logged in the XML as msg. # Example: ${bmc_model} = Witherspoon BMC if '${bmc_model} = ' in node.text: bmc_platform = node.text.split(" = ")[1] print_vars(bmc_version_id, bmc_platform) return [str(bmc_version_id), str(bmc_platform)] def main(): if not gen_get_options(parser, stock_list): return False if not validate_parms(): return False qprint_pgm_header() parse_output_xml(source, dest, version_id, platform, level, test_phase, processor) return True # Main if not main(): exit(1)