1# resulttool - common library/utility functions 2# 3# Copyright (c) 2019, Intel Corporation. 4# Copyright (c) 2019, Linux Foundation 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9import os 10import base64 11import zlib 12import json 13import scriptpath 14import copy 15import urllib.request 16import posixpath 17import logging 18scriptpath.add_oe_lib_path() 19 20logger = logging.getLogger('resulttool') 21 22flatten_map = { 23 "oeselftest": [], 24 "runtime": [], 25 "sdk": [], 26 "sdkext": [], 27 "manual": [] 28} 29regression_map = { 30 "oeselftest": ['TEST_TYPE', 'MACHINE'], 31 "runtime": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'IMAGE_PKGTYPE', 'DISTRO'], 32 "sdk": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE'], 33 "sdkext": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE'], 34 "manual": ['TEST_TYPE', 'TEST_MODULE', 'IMAGE_BASENAME', 'MACHINE'] 35} 36store_map = { 37 "oeselftest": ['TEST_TYPE', 'TESTSERIES', 'MACHINE'], 38 "runtime": ['TEST_TYPE', 'DISTRO', 'MACHINE', 'IMAGE_BASENAME'], 39 "sdk": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'], 40 "sdkext": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'], 41 "manual": ['TEST_TYPE', 'TEST_MODULE', 'MACHINE', 'IMAGE_BASENAME'] 42} 43 44rawlog_sections = { 45 "ptestresult.rawlogs": "ptest", 46 "ltpresult.rawlogs": "ltp", 47 "ltpposixresult.rawlogs": "ltpposix" 48} 49 50def is_url(p): 51 """ 52 Helper for determining if the given path is a URL 53 """ 54 return p.startswith('http://') or p.startswith('https://') 55 56extra_configvars = {'TESTSERIES': ''} 57 58# 59# Load the json file and append the results data into the provided results dict 60# 61def append_resultsdata(results, f, configmap=store_map, configvars=extra_configvars): 62 if type(f) is str: 63 if is_url(f): 64 with urllib.request.urlopen(f) as response: 65 data = json.loads(response.read().decode('utf-8')) 66 url = urllib.parse.urlparse(f) 67 testseries = posixpath.basename(posixpath.dirname(url.path)) 68 else: 69 with open(f, "r") as filedata: 70 try: 71 data = json.load(filedata) 72 except json.decoder.JSONDecodeError: 73 print("Cannot decode {}. Possible corruption. Skipping.".format(f)) 74 data = "" 75 testseries = os.path.basename(os.path.dirname(f)) 76 else: 77 data = f 78 for res in data: 79 if "configuration" not in data[res] or "result" not in data[res]: 80 raise ValueError("Test results data without configuration or result section?") 81 for config in configvars: 82 if config == "TESTSERIES" and "TESTSERIES" not in data[res]["configuration"]: 83 data[res]["configuration"]["TESTSERIES"] = testseries 84 continue 85 if config not in data[res]["configuration"]: 86 data[res]["configuration"][config] = configvars[config] 87 testtype = data[res]["configuration"].get("TEST_TYPE") 88 if testtype not in configmap: 89 raise ValueError("Unknown test type %s" % testtype) 90 testpath = "/".join(data[res]["configuration"].get(i) for i in configmap[testtype]) 91 if testpath not in results: 92 results[testpath] = {} 93 results[testpath][res] = data[res] 94 95# 96# Walk a directory and find/load results data 97# or load directly from a file 98# 99def load_resultsdata(source, configmap=store_map, configvars=extra_configvars): 100 results = {} 101 if is_url(source) or os.path.isfile(source): 102 append_resultsdata(results, source, configmap, configvars) 103 return results 104 for root, dirs, files in os.walk(source): 105 for name in files: 106 f = os.path.join(root, name) 107 if name == "testresults.json": 108 append_resultsdata(results, f, configmap, configvars) 109 return results 110 111def filter_resultsdata(results, resultid): 112 newresults = {} 113 for r in results: 114 for i in results[r]: 115 if i == resultsid: 116 newresults[r] = {} 117 newresults[r][i] = results[r][i] 118 return newresults 119 120def strip_logs(results): 121 newresults = copy.deepcopy(results) 122 for res in newresults: 123 if 'result' not in newresults[res]: 124 continue 125 for logtype in rawlog_sections: 126 if logtype in newresults[res]['result']: 127 del newresults[res]['result'][logtype] 128 if 'ptestresult.sections' in newresults[res]['result']: 129 for i in newresults[res]['result']['ptestresult.sections']: 130 if 'log' in newresults[res]['result']['ptestresult.sections'][i]: 131 del newresults[res]['result']['ptestresult.sections'][i]['log'] 132 return newresults 133 134# For timing numbers, crazy amounts of precision don't make sense and just confuse 135# the logs. For numbers over 1, trim to 3 decimal places, for numbers less than 1, 136# trim to 4 significant digits 137def trim_durations(results): 138 for res in results: 139 if 'result' not in results[res]: 140 continue 141 for entry in results[res]['result']: 142 if 'duration' in results[res]['result'][entry]: 143 duration = results[res]['result'][entry]['duration'] 144 if duration > 1: 145 results[res]['result'][entry]['duration'] = float("%.3f" % duration) 146 elif duration < 1: 147 results[res]['result'][entry]['duration'] = float("%.4g" % duration) 148 return results 149 150def handle_cleanups(results): 151 # Remove pointless path duplication from old format reproducibility results 152 for res2 in results: 153 try: 154 section = results[res2]['result']['reproducible']['files'] 155 for pkgtype in section: 156 for filelist in section[pkgtype].copy(): 157 if section[pkgtype][filelist] and type(section[pkgtype][filelist][0]) == dict: 158 newlist = [] 159 for entry in section[pkgtype][filelist]: 160 newlist.append(entry["reference"].split("/./")[1]) 161 section[pkgtype][filelist] = newlist 162 163 except KeyError: 164 pass 165 # Remove pointless duplicate rawlogs data 166 try: 167 del results[res2]['result']['reproducible.rawlogs'] 168 except KeyError: 169 pass 170 171def decode_log(logdata): 172 if isinstance(logdata, str): 173 return logdata 174 elif isinstance(logdata, dict): 175 if "compressed" in logdata: 176 data = logdata.get("compressed") 177 data = base64.b64decode(data.encode("utf-8")) 178 data = zlib.decompress(data) 179 return data.decode("utf-8", errors='ignore') 180 return None 181 182def generic_get_log(sectionname, results, section): 183 if sectionname not in results: 184 return None 185 if section not in results[sectionname]: 186 return None 187 188 ptest = results[sectionname][section] 189 if 'log' not in ptest: 190 return None 191 return decode_log(ptest['log']) 192 193def ptestresult_get_log(results, section): 194 return generic_get_log('ptestresult.sections', results, section) 195 196def generic_get_rawlogs(sectname, results): 197 if sectname not in results: 198 return None 199 if 'log' not in results[sectname]: 200 return None 201 return decode_log(results[sectname]['log']) 202 203def save_resultsdata(results, destdir, fn="testresults.json", ptestjson=False, ptestlogs=False): 204 for res in results: 205 if res: 206 dst = destdir + "/" + res + "/" + fn 207 else: 208 dst = destdir + "/" + fn 209 os.makedirs(os.path.dirname(dst), exist_ok=True) 210 resultsout = results[res] 211 if not ptestjson: 212 resultsout = strip_logs(results[res]) 213 trim_durations(resultsout) 214 handle_cleanups(resultsout) 215 with open(dst, 'w') as f: 216 f.write(json.dumps(resultsout, sort_keys=True, indent=1)) 217 for res2 in results[res]: 218 if ptestlogs and 'result' in results[res][res2]: 219 seriesresults = results[res][res2]['result'] 220 for logtype in rawlog_sections: 221 logdata = generic_get_rawlogs(logtype, seriesresults) 222 if logdata is not None: 223 logger.info("Extracting " + rawlog_sections[logtype] + "-raw.log") 224 with open(dst.replace(fn, rawlog_sections[logtype] + "-raw.log"), "w+") as f: 225 f.write(logdata) 226 if 'ptestresult.sections' in seriesresults: 227 for i in seriesresults['ptestresult.sections']: 228 sectionlog = ptestresult_get_log(seriesresults, i) 229 if sectionlog is not None: 230 with open(dst.replace(fn, "ptest-%s.log" % i), "w+") as f: 231 f.write(sectionlog) 232 233def git_get_result(repo, tags, configmap=store_map): 234 git_objs = [] 235 for tag in tags: 236 files = repo.run_cmd(['ls-tree', "--name-only", "-r", tag]).splitlines() 237 git_objs.extend([tag + ':' + f for f in files if f.endswith("testresults.json")]) 238 239 def parse_json_stream(data): 240 """Parse multiple concatenated JSON objects""" 241 objs = [] 242 json_d = "" 243 for line in data.splitlines(): 244 if line == '}{': 245 json_d += '}' 246 objs.append(json.loads(json_d)) 247 json_d = '{' 248 else: 249 json_d += line 250 objs.append(json.loads(json_d)) 251 return objs 252 253 # Optimize by reading all data with one git command 254 results = {} 255 for obj in parse_json_stream(repo.run_cmd(['show'] + git_objs + ['--'])): 256 append_resultsdata(results, obj, configmap=configmap) 257 258 return results 259 260def test_run_results(results): 261 """ 262 Convenient generator function that iterates over all test runs that have a 263 result section. 264 265 Generates a tuple of: 266 (result json file path, test run name, test run (dict), test run "results" (dict)) 267 for each test run that has a "result" section 268 """ 269 for path in results: 270 for run_name, test_run in results[path].items(): 271 if not 'result' in test_run: 272 continue 273 yield path, run_name, test_run, test_run['result'] 274 275