1SUMMARY = "Updates the NVD CVE database"
2LICENSE = "MIT"
3
4# Important note:
5# This product uses the NVD API but is not endorsed or certified by the NVD.
6
7INHIBIT_DEFAULT_DEPS = "1"
8
9inherit native
10
11deltask do_unpack
12deltask do_patch
13deltask do_configure
14deltask do_compile
15deltask do_install
16deltask do_populate_sysroot
17
18NVDCVE_URL ?= "https://services.nvd.nist.gov/rest/json/cves/2.0"
19
20# If you have a NVD API key (https://nvd.nist.gov/developers/request-an-api-key)
21# then setting this to get higher rate limits.
22NVDCVE_API_KEY ?= ""
23
24# CVE database update interval, in seconds. By default: once a day (24*60*60).
25# Use 0 to force the update
26# Use a negative value to skip the update
27CVE_DB_UPDATE_INTERVAL ?= "86400"
28
29# Number of attmepts for each http query to nvd server before giving up
30CVE_DB_UPDATE_ATTEMPTS ?= "5"
31
32CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_2.db"
33
34CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_2.db"
35
36python () {
37    if not bb.data.inherits_class("cve-check", d):
38        raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.")
39}
40
41python do_fetch() {
42    """
43    Update NVD database with API 2.0
44    """
45    import bb.utils
46    import bb.progress
47    import shutil
48
49    bb.utils.export_proxies(d)
50
51    db_file = d.getVar("CVE_CHECK_DB_FILE")
52    db_dir = os.path.dirname(db_file)
53    db_tmp_file = d.getVar("CVE_DB_TEMP_FILE")
54
55    cleanup_db_download(db_file, db_tmp_file)
56    # By default let's update the whole database (since time 0)
57    database_time = 0
58
59    # The NVD database changes once a day, so no need to update more frequently
60    # Allow the user to force-update
61    try:
62        import time
63        update_interval = int(d.getVar("CVE_DB_UPDATE_INTERVAL"))
64        if update_interval < 0:
65            bb.note("CVE database update skipped")
66            return
67        if time.time() - os.path.getmtime(db_file) < update_interval:
68            bb.note("CVE database recently updated, skipping")
69            return
70        database_time = os.path.getmtime(db_file)
71
72    except OSError:
73        pass
74
75    bb.utils.mkdirhier(db_dir)
76    if os.path.exists(db_file):
77        shutil.copy2(db_file, db_tmp_file)
78
79    if update_db_file(db_tmp_file, d, database_time) == True:
80        # Update downloaded correctly, can swap files
81        shutil.move(db_tmp_file, db_file)
82    else:
83        # Update failed, do not modify the database
84        bb.warn("CVE database update failed")
85        os.remove(db_tmp_file)
86}
87
88do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}"
89do_fetch[file-checksums] = ""
90do_fetch[vardeps] = ""
91
92def cleanup_db_download(db_file, db_tmp_file):
93    """
94    Cleanup the download space from possible failed downloads
95    """
96
97    # Clean up the updates done on the main file
98    # Remove it only if a journal file exists - it means a complete re-download
99    if os.path.exists("{0}-journal".format(db_file)):
100        # If a journal is present the last update might have been interrupted. In that case,
101        # just wipe any leftovers and force the DB to be recreated.
102        os.remove("{0}-journal".format(db_file))
103
104        if os.path.exists(db_file):
105            os.remove(db_file)
106
107    # Clean-up the temporary file downloads, we can remove both journal
108    # and the temporary database
109    if os.path.exists("{0}-journal".format(db_tmp_file)):
110        # If a journal is present the last update might have been interrupted. In that case,
111        # just wipe any leftovers and force the DB to be recreated.
112        os.remove("{0}-journal".format(db_tmp_file))
113
114    if os.path.exists(db_tmp_file):
115        os.remove(db_tmp_file)
116
117def nvd_request_wait(attempt, min_wait):
118    return min ( ( (2 * attempt) + min_wait ) , 30)
119
120def nvd_request_next(url, attempts, api_key, args, min_wait):
121    """
122    Request next part of the NVD dabase
123    """
124
125    import urllib.request
126    import urllib.parse
127    import gzip
128    import http
129    import time
130
131    request = urllib.request.Request(url + "?" + urllib.parse.urlencode(args))
132    if api_key:
133        request.add_header("apiKey", api_key)
134    bb.note("Requesting %s" % request.full_url)
135
136    for attempt in range(attempts):
137        try:
138            r = urllib.request.urlopen(request)
139
140            if (r.headers['content-encoding'] == 'gzip'):
141                buf = r.read()
142                raw_data = gzip.decompress(buf)
143            else:
144                raw_data = r.read().decode("utf-8")
145
146            r.close()
147
148        except Exception as e:
149            wait_time = nvd_request_wait(attempt, min_wait)
150            bb.note("CVE database: received error (%s)" % (e))
151            bb.note("CVE database: retrying download after %d seconds. attempted (%d/%d)" % (wait_time, attempt+1, attempts))
152            time.sleep(wait_time)
153            pass
154        else:
155            return raw_data
156    else:
157        # We failed at all attempts
158        return None
159
160def update_db_file(db_tmp_file, d, database_time):
161    """
162    Update the given database file
163    """
164    import bb.utils, bb.progress
165    import datetime
166    import sqlite3
167    import json
168
169    # Connect to database
170    conn = sqlite3.connect(db_tmp_file)
171    initialize_db(conn)
172
173    req_args = {'startIndex' : 0}
174
175    # The maximum range for time is 120 days
176    # Force a complete update if our range is longer
177    if (database_time != 0):
178        database_date = datetime.datetime.fromtimestamp(database_time, tz=datetime.timezone.utc)
179        today_date = datetime.datetime.now(tz=datetime.timezone.utc)
180        delta = today_date - database_date
181        if delta.days < 120:
182            bb.note("CVE database: performing partial update")
183            req_args['lastModStartDate'] = database_date.isoformat()
184            req_args['lastModEndDate'] = today_date.isoformat()
185        else:
186            bb.note("CVE database: file too old, forcing a full update")
187
188    with bb.progress.ProgressHandler(d) as ph, open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f:
189
190        bb.note("Updating entries")
191        index = 0
192        url = d.getVar("NVDCVE_URL")
193        api_key = d.getVar("NVDCVE_API_KEY") or None
194        attempts = int(d.getVar("CVE_DB_UPDATE_ATTEMPTS"))
195
196        # Recommended by NVD
197        wait_time = 6
198        if api_key:
199            wait_time = 2
200
201        while True:
202            req_args['startIndex'] = index
203            raw_data = nvd_request_next(url, attempts, api_key, req_args, wait_time)
204            if raw_data is None:
205                # We haven't managed to download data
206                return False
207
208            data = json.loads(raw_data)
209
210            index = data["startIndex"]
211            total = data["totalResults"]
212            per_page = data["resultsPerPage"]
213            bb.note("Got %d entries" % per_page)
214            for cve in data["vulnerabilities"]:
215               update_db(conn, cve)
216
217            index += per_page
218            ph.update((float(index) / (total+1)) * 100)
219            if index >= total:
220               break
221
222            # Recommended by NVD
223            time.sleep(wait_time)
224
225        # Update success, set the date to cve_check file.
226        cve_f.write('CVE database update : %s\n\n' % datetime.date.today())
227
228    conn.commit()
229    conn.close()
230    return True
231
232def initialize_db(conn):
233    with conn:
234        c = conn.cursor()
235
236        c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
237
238        c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
239            SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT, VECTORSTRING TEXT)")
240
241        c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \
242            VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \
243            VERSION_END TEXT, OPERATOR_END TEXT)")
244        c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);")
245
246        c.close()
247
248def parse_node_and_insert(conn, node, cveId):
249
250    def cpe_generator():
251        for cpe in node.get('cpeMatch', ()):
252            if not cpe['vulnerable']:
253                return
254            cpe23 = cpe.get('criteria')
255            if not cpe23:
256                return
257            cpe23 = cpe23.split(':')
258            if len(cpe23) < 6:
259                return
260            vendor = cpe23[3]
261            product = cpe23[4]
262            version = cpe23[5]
263
264            if cpe23[6] == '*' or cpe23[6] == '-':
265                version_suffix = ""
266            else:
267                version_suffix = "_" + cpe23[6]
268
269            if version != '*' and version != '-':
270                # Version is defined, this is a '=' match
271                yield [cveId, vendor, product, version + version_suffix, '=', '', '']
272            elif version == '-':
273                # no version information is available
274                yield [cveId, vendor, product, version, '', '', '']
275            else:
276                # Parse start version, end version and operators
277                op_start = ''
278                op_end = ''
279                v_start = ''
280                v_end = ''
281
282                if 'versionStartIncluding' in cpe:
283                    op_start = '>='
284                    v_start = cpe['versionStartIncluding']
285
286                if 'versionStartExcluding' in cpe:
287                    op_start = '>'
288                    v_start = cpe['versionStartExcluding']
289
290                if 'versionEndIncluding' in cpe:
291                    op_end = '<='
292                    v_end = cpe['versionEndIncluding']
293
294                if 'versionEndExcluding' in cpe:
295                    op_end = '<'
296                    v_end = cpe['versionEndExcluding']
297
298                if op_start or op_end or v_start or v_end:
299                    yield [cveId, vendor, product, v_start, op_start, v_end, op_end]
300                else:
301                    # This is no version information, expressed differently.
302                    # Save processing by representing as -.
303                    yield [cveId, vendor, product, '-', '', '', '']
304
305    conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator()).close()
306
307def update_db(conn, elt):
308    """
309    Update a single entry in the on-disk database
310    """
311
312    accessVector = None
313    vectorString = None
314    cveId = elt['cve']['id']
315    if elt['cve']['vulnStatus'] ==  "Rejected":
316        return
317    cveDesc = ""
318    for desc in elt['cve']['descriptions']:
319        if desc['lang'] == 'en':
320            cveDesc = desc['value']
321    date = elt['cve']['lastModified']
322    try:
323        accessVector = elt['cve']['metrics']['cvssMetricV2'][0]['cvssData']['accessVector']
324        vectorString = elt['cve']['metrics']['cvssMetricV2'][0]['cvssData']['vectorString']
325        cvssv2 = elt['cve']['metrics']['cvssMetricV2'][0]['cvssData']['baseScore']
326    except KeyError:
327        cvssv2 = 0.0
328    cvssv3 = None
329    try:
330        accessVector = accessVector or elt['cve']['metrics']['cvssMetricV30'][0]['cvssData']['attackVector']
331        vectorString = vectorString or elt['cve']['metrics']['cvssMetricV30'][0]['cvssData']['vectorString']
332        cvssv3 = elt['cve']['metrics']['cvssMetricV30'][0]['cvssData']['baseScore']
333    except KeyError:
334        pass
335    try:
336        accessVector = accessVector or elt['cve']['metrics']['cvssMetricV31'][0]['cvssData']['attackVector']
337        vectorString = vectorString or elt['cve']['metrics']['cvssMetricV31'][0]['cvssData']['vectorString']
338        cvssv3 = cvssv3 or elt['cve']['metrics']['cvssMetricV31'][0]['cvssData']['baseScore']
339    except KeyError:
340        pass
341    accessVector = accessVector or "UNKNOWN"
342    vectorString = vectorString or "UNKNOWN"
343    cvssv3 = cvssv3 or 0.0
344
345    conn.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?, ?)",
346                [cveId, cveDesc, cvssv2, cvssv3, date, accessVector, vectorString]).close()
347
348    try:
349        for config in elt['cve']['configurations']:
350            # This is suboptimal as it doesn't handle AND/OR and negate, but is better than nothing
351            for node in config["nodes"]:
352                parse_node_and_insert(conn, node, cveId)
353    except KeyError:
354        bb.note("CVE %s has no configurations" % cveId)
355
356do_fetch[nostamp] = "1"
357
358EXCLUDE_FROM_WORLD = "1"
359