1SUMMARY = "Updates the NVD CVE database" 2LICENSE = "MIT" 3 4# Important note: 5# This product uses the NVD API but is not endorsed or certified by the NVD. 6 7INHIBIT_DEFAULT_DEPS = "1" 8 9inherit native 10 11deltask do_unpack 12deltask do_patch 13deltask do_configure 14deltask do_compile 15deltask do_install 16deltask do_populate_sysroot 17 18NVDCVE_URL ?= "https://services.nvd.nist.gov/rest/json/cves/2.0" 19 20# If you have a NVD API key (https://nvd.nist.gov/developers/request-an-api-key) 21# then setting this to get higher rate limits. 22NVDCVE_API_KEY ?= "" 23 24# CVE database update interval, in seconds. By default: once a day (24*60*60). 25# Use 0 to force the update 26# Use a negative value to skip the update 27CVE_DB_UPDATE_INTERVAL ?= "86400" 28 29# Number of attmepts for each http query to nvd server before giving up 30CVE_DB_UPDATE_ATTEMPTS ?= "5" 31 32CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_2.db" 33 34CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_2.db" 35 36python () { 37 if not bb.data.inherits_class("cve-check", d): 38 raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.") 39} 40 41python do_fetch() { 42 """ 43 Update NVD database with API 2.0 44 """ 45 import bb.utils 46 import bb.progress 47 import shutil 48 49 bb.utils.export_proxies(d) 50 51 db_file = d.getVar("CVE_CHECK_DB_FILE") 52 db_dir = os.path.dirname(db_file) 53 db_tmp_file = d.getVar("CVE_DB_TEMP_FILE") 54 55 cleanup_db_download(db_file, db_tmp_file) 56 # By default let's update the whole database (since time 0) 57 database_time = 0 58 59 # The NVD database changes once a day, so no need to update more frequently 60 # Allow the user to force-update 61 try: 62 import time 63 update_interval = int(d.getVar("CVE_DB_UPDATE_INTERVAL")) 64 if update_interval < 0: 65 bb.note("CVE database update skipped") 66 return 67 if time.time() - os.path.getmtime(db_file) < update_interval: 68 bb.note("CVE database recently updated, skipping") 69 return 70 database_time = os.path.getmtime(db_file) 71 72 except OSError: 73 pass 74 75 bb.utils.mkdirhier(db_dir) 76 if os.path.exists(db_file): 77 shutil.copy2(db_file, db_tmp_file) 78 79 if update_db_file(db_tmp_file, d, database_time) == True: 80 # Update downloaded correctly, can swap files 81 shutil.move(db_tmp_file, db_file) 82 else: 83 # Update failed, do not modify the database 84 bb.warn("CVE database update failed") 85 os.remove(db_tmp_file) 86} 87 88do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" 89do_fetch[file-checksums] = "" 90do_fetch[vardeps] = "" 91 92def cleanup_db_download(db_file, db_tmp_file): 93 """ 94 Cleanup the download space from possible failed downloads 95 """ 96 97 # Clean up the updates done on the main file 98 # Remove it only if a journal file exists - it means a complete re-download 99 if os.path.exists("{0}-journal".format(db_file)): 100 # If a journal is present the last update might have been interrupted. In that case, 101 # just wipe any leftovers and force the DB to be recreated. 102 os.remove("{0}-journal".format(db_file)) 103 104 if os.path.exists(db_file): 105 os.remove(db_file) 106 107 # Clean-up the temporary file downloads, we can remove both journal 108 # and the temporary database 109 if os.path.exists("{0}-journal".format(db_tmp_file)): 110 # If a journal is present the last update might have been interrupted. In that case, 111 # just wipe any leftovers and force the DB to be recreated. 112 os.remove("{0}-journal".format(db_tmp_file)) 113 114 if os.path.exists(db_tmp_file): 115 os.remove(db_tmp_file) 116 117def nvd_request_wait(attempt, min_wait): 118 return min ( ( (2 * attempt) + min_wait ) , 30) 119 120def nvd_request_next(url, attempts, api_key, args, min_wait): 121 """ 122 Request next part of the NVD dabase 123 """ 124 125 import urllib.request 126 import urllib.parse 127 import gzip 128 import http 129 import time 130 131 request = urllib.request.Request(url + "?" + urllib.parse.urlencode(args)) 132 if api_key: 133 request.add_header("apiKey", api_key) 134 bb.note("Requesting %s" % request.full_url) 135 136 for attempt in range(attempts): 137 try: 138 r = urllib.request.urlopen(request) 139 140 if (r.headers['content-encoding'] == 'gzip'): 141 buf = r.read() 142 raw_data = gzip.decompress(buf) 143 else: 144 raw_data = r.read().decode("utf-8") 145 146 r.close() 147 148 except Exception as e: 149 wait_time = nvd_request_wait(attempt, min_wait) 150 bb.note("CVE database: received error (%s)" % (e)) 151 bb.note("CVE database: retrying download after %d seconds. attempted (%d/%d)" % (wait_time, attempt+1, attempts)) 152 time.sleep(wait_time) 153 pass 154 else: 155 return raw_data 156 else: 157 # We failed at all attempts 158 return None 159 160def update_db_file(db_tmp_file, d, database_time): 161 """ 162 Update the given database file 163 """ 164 import bb.utils, bb.progress 165 import datetime 166 import sqlite3 167 import json 168 169 # Connect to database 170 conn = sqlite3.connect(db_tmp_file) 171 initialize_db(conn) 172 173 req_args = {'startIndex' : 0} 174 175 # The maximum range for time is 120 days 176 # Force a complete update if our range is longer 177 if (database_time != 0): 178 database_date = datetime.datetime.fromtimestamp(database_time, tz=datetime.timezone.utc) 179 today_date = datetime.datetime.now(tz=datetime.timezone.utc) 180 delta = today_date - database_date 181 if delta.days < 120: 182 bb.note("CVE database: performing partial update") 183 req_args['lastModStartDate'] = database_date.isoformat() 184 req_args['lastModEndDate'] = today_date.isoformat() 185 else: 186 bb.note("CVE database: file too old, forcing a full update") 187 188 with bb.progress.ProgressHandler(d) as ph, open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f: 189 190 bb.note("Updating entries") 191 index = 0 192 url = d.getVar("NVDCVE_URL") 193 api_key = d.getVar("NVDCVE_API_KEY") or None 194 attempts = int(d.getVar("CVE_DB_UPDATE_ATTEMPTS")) 195 196 # Recommended by NVD 197 wait_time = 6 198 if api_key: 199 wait_time = 2 200 201 while True: 202 req_args['startIndex'] = index 203 raw_data = nvd_request_next(url, attempts, api_key, req_args, wait_time) 204 if raw_data is None: 205 # We haven't managed to download data 206 return False 207 208 data = json.loads(raw_data) 209 210 index = data["startIndex"] 211 total = data["totalResults"] 212 per_page = data["resultsPerPage"] 213 bb.note("Got %d entries" % per_page) 214 for cve in data["vulnerabilities"]: 215 update_db(conn, cve) 216 217 index += per_page 218 ph.update((float(index) / (total+1)) * 100) 219 if index >= total: 220 break 221 222 # Recommended by NVD 223 time.sleep(wait_time) 224 225 # Update success, set the date to cve_check file. 226 cve_f.write('CVE database update : %s\n\n' % datetime.date.today()) 227 228 conn.commit() 229 conn.close() 230 return True 231 232def initialize_db(conn): 233 with conn: 234 c = conn.cursor() 235 236 c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)") 237 238 c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \ 239 SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT, VECTORSTRING TEXT)") 240 241 c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \ 242 VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \ 243 VERSION_END TEXT, OPERATOR_END TEXT)") 244 c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);") 245 246 c.close() 247 248def parse_node_and_insert(conn, node, cveId): 249 250 def cpe_generator(): 251 for cpe in node.get('cpeMatch', ()): 252 if not cpe['vulnerable']: 253 return 254 cpe23 = cpe.get('criteria') 255 if not cpe23: 256 return 257 cpe23 = cpe23.split(':') 258 if len(cpe23) < 6: 259 return 260 vendor = cpe23[3] 261 product = cpe23[4] 262 version = cpe23[5] 263 264 if cpe23[6] == '*' or cpe23[6] == '-': 265 version_suffix = "" 266 else: 267 version_suffix = "_" + cpe23[6] 268 269 if version != '*' and version != '-': 270 # Version is defined, this is a '=' match 271 yield [cveId, vendor, product, version + version_suffix, '=', '', ''] 272 elif version == '-': 273 # no version information is available 274 yield [cveId, vendor, product, version, '', '', ''] 275 else: 276 # Parse start version, end version and operators 277 op_start = '' 278 op_end = '' 279 v_start = '' 280 v_end = '' 281 282 if 'versionStartIncluding' in cpe: 283 op_start = '>=' 284 v_start = cpe['versionStartIncluding'] 285 286 if 'versionStartExcluding' in cpe: 287 op_start = '>' 288 v_start = cpe['versionStartExcluding'] 289 290 if 'versionEndIncluding' in cpe: 291 op_end = '<=' 292 v_end = cpe['versionEndIncluding'] 293 294 if 'versionEndExcluding' in cpe: 295 op_end = '<' 296 v_end = cpe['versionEndExcluding'] 297 298 if op_start or op_end or v_start or v_end: 299 yield [cveId, vendor, product, v_start, op_start, v_end, op_end] 300 else: 301 # This is no version information, expressed differently. 302 # Save processing by representing as -. 303 yield [cveId, vendor, product, '-', '', '', ''] 304 305 conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator()).close() 306 307def update_db(conn, elt): 308 """ 309 Update a single entry in the on-disk database 310 """ 311 312 accessVector = None 313 vectorString = None 314 cveId = elt['cve']['id'] 315 if elt['cve']['vulnStatus'] == "Rejected": 316 return 317 cveDesc = "" 318 for desc in elt['cve']['descriptions']: 319 if desc['lang'] == 'en': 320 cveDesc = desc['value'] 321 date = elt['cve']['lastModified'] 322 try: 323 accessVector = elt['cve']['metrics']['cvssMetricV2'][0]['cvssData']['accessVector'] 324 vectorString = elt['cve']['metrics']['cvssMetricV2'][0]['cvssData']['vectorString'] 325 cvssv2 = elt['cve']['metrics']['cvssMetricV2'][0]['cvssData']['baseScore'] 326 except KeyError: 327 cvssv2 = 0.0 328 cvssv3 = None 329 try: 330 accessVector = accessVector or elt['cve']['metrics']['cvssMetricV30'][0]['cvssData']['attackVector'] 331 vectorString = vectorString or elt['cve']['metrics']['cvssMetricV30'][0]['cvssData']['vectorString'] 332 cvssv3 = elt['cve']['metrics']['cvssMetricV30'][0]['cvssData']['baseScore'] 333 except KeyError: 334 pass 335 try: 336 accessVector = accessVector or elt['cve']['metrics']['cvssMetricV31'][0]['cvssData']['attackVector'] 337 vectorString = vectorString or elt['cve']['metrics']['cvssMetricV31'][0]['cvssData']['vectorString'] 338 cvssv3 = cvssv3 or elt['cve']['metrics']['cvssMetricV31'][0]['cvssData']['baseScore'] 339 except KeyError: 340 pass 341 accessVector = accessVector or "UNKNOWN" 342 vectorString = vectorString or "UNKNOWN" 343 cvssv3 = cvssv3 or 0.0 344 345 conn.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?, ?)", 346 [cveId, cveDesc, cvssv2, cvssv3, date, accessVector, vectorString]).close() 347 348 try: 349 for config in elt['cve']['configurations']: 350 # This is suboptimal as it doesn't handle AND/OR and negate, but is better than nothing 351 for node in config["nodes"]: 352 parse_node_and_insert(conn, node, cveId) 353 except KeyError: 354 bb.note("CVE %s has no configurations" % cveId) 355 356do_fetch[nostamp] = "1" 357 358EXCLUDE_FROM_WORLD = "1" 359