1c342db35SBrad Bishop# 2*92b42cb3SPatrick Williams# Copyright OpenEmbedded Contributors 3*92b42cb3SPatrick Williams# 4c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 5c342db35SBrad Bishop# 6c342db35SBrad Bishop 7eb8dc403SDave Cobbleydef create_socket(url, d): 8eb8dc403SDave Cobbley import urllib 9eb8dc403SDave Cobbley from bb.utils import export_proxies 10eb8dc403SDave Cobbley 11eb8dc403SDave Cobbley export_proxies(d) 12eb8dc403SDave Cobbley return urllib.request.urlopen(url) 13eb8dc403SDave Cobbley 14eb8dc403SDave Cobbleydef get_links_from_url(url, d): 15eb8dc403SDave Cobbley "Return all the href links found on the web location" 16eb8dc403SDave Cobbley 17eb8dc403SDave Cobbley from bs4 import BeautifulSoup, SoupStrainer 18eb8dc403SDave Cobbley 19eb8dc403SDave Cobbley soup = BeautifulSoup(create_socket(url,d), "html.parser", parse_only=SoupStrainer("a")) 20eb8dc403SDave Cobbley hyperlinks = [] 21eb8dc403SDave Cobbley for line in soup.find_all('a', href=True): 22eb8dc403SDave Cobbley hyperlinks.append(line['href'].strip('/')) 23eb8dc403SDave Cobbley return hyperlinks 24eb8dc403SDave Cobbley 25eb8dc403SDave Cobbleydef find_latest_numeric_release(url, d): 26eb8dc403SDave Cobbley "Find the latest listed numeric release on the given url" 27eb8dc403SDave Cobbley max=0 28eb8dc403SDave Cobbley maxstr="" 29eb8dc403SDave Cobbley for link in get_links_from_url(url, d): 30eb8dc403SDave Cobbley try: 31595f6308SAndrew Geissler # TODO use bb.utils.vercmp_string_op() 32eb8dc403SDave Cobbley release = float(link) 33eb8dc403SDave Cobbley except: 34eb8dc403SDave Cobbley release = 0 35eb8dc403SDave Cobbley if release > max: 36eb8dc403SDave Cobbley max = release 37eb8dc403SDave Cobbley maxstr = link 38eb8dc403SDave Cobbley return maxstr 39eb8dc403SDave Cobbley 40eb8dc403SDave Cobbleydef is_src_rpm(name): 41eb8dc403SDave Cobbley "Check if the link is pointing to a src.rpm file" 42eb8dc403SDave Cobbley return name.endswith(".src.rpm") 43eb8dc403SDave Cobbley 44eb8dc403SDave Cobbleydef package_name_from_srpm(srpm): 45eb8dc403SDave Cobbley "Strip out the package name from the src.rpm filename" 46eb8dc403SDave Cobbley 47eb8dc403SDave Cobbley # ca-certificates-2016.2.7-1.0.fc24.src.rpm 48eb8dc403SDave Cobbley # ^name ^ver ^release^removed 49eb8dc403SDave Cobbley (name, version, release) = srpm.replace(".src.rpm", "").rsplit("-", 2) 50eb8dc403SDave Cobbley return name 51eb8dc403SDave Cobbley 52eb8dc403SDave Cobbleydef get_source_package_list_from_url(url, section, d): 53eb8dc403SDave Cobbley "Return a sectioned list of package names from a URL list" 54eb8dc403SDave Cobbley 55eb8dc403SDave Cobbley bb.note("Reading %s: %s" % (url, section)) 56eb8dc403SDave Cobbley links = get_links_from_url(url, d) 57eb8dc403SDave Cobbley srpms = filter(is_src_rpm, links) 58eb8dc403SDave Cobbley names_list = map(package_name_from_srpm, srpms) 59eb8dc403SDave Cobbley 60eb8dc403SDave Cobbley new_pkgs = set() 61eb8dc403SDave Cobbley for pkgs in names_list: 62eb8dc403SDave Cobbley new_pkgs.add(pkgs + ":" + section) 63eb8dc403SDave Cobbley return new_pkgs 64eb8dc403SDave Cobbley 65eb8dc403SDave Cobbleydef get_source_package_list_from_url_by_letter(url, section, d): 66eb8dc403SDave Cobbley import string 67eb8dc403SDave Cobbley from urllib.error import HTTPError 68eb8dc403SDave Cobbley packages = set() 69eb8dc403SDave Cobbley for letter in (string.ascii_lowercase + string.digits): 70eb8dc403SDave Cobbley # Not all subfolders may exist, so silently handle 404 71eb8dc403SDave Cobbley try: 72eb8dc403SDave Cobbley packages |= get_source_package_list_from_url(url + "/" + letter, section, d) 73eb8dc403SDave Cobbley except HTTPError as e: 74eb8dc403SDave Cobbley if e.code != 404: raise 75eb8dc403SDave Cobbley return packages 76eb8dc403SDave Cobbley 77eb8dc403SDave Cobbleydef get_latest_released_fedora_source_package_list(d): 78eb8dc403SDave Cobbley "Returns list of all the name os packages in the latest fedora distro" 79eb8dc403SDave Cobbley latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d) 80eb8dc403SDave Cobbley package_names = get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Everything/source/tree/Packages/" % latest, "main", d) 81eb8dc403SDave Cobbley package_names |= get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d) 82eb8dc403SDave Cobbley return latest, package_names 83eb8dc403SDave Cobbley 84eb8dc403SDave Cobbleydef get_latest_released_opensuse_source_package_list(d): 85eb8dc403SDave Cobbley "Returns list of all the name os packages in the latest opensuse distro" 86eb8dc403SDave Cobbley latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/leap", d) 87eb8dc403SDave Cobbley 88eb8dc403SDave Cobbley package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/leap/%s/repo/oss/suse/src/" % latest, "main", d) 89eb8dc403SDave Cobbley package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/leap/%s/oss/src/" % latest, "updates", d) 90eb8dc403SDave Cobbley return latest, package_names 91eb8dc403SDave Cobbley 92eb8dc403SDave Cobbleydef get_latest_released_clear_source_package_list(d): 93eb8dc403SDave Cobbley latest = find_latest_numeric_release("https://download.clearlinux.org/releases/", d) 94eb8dc403SDave Cobbley package_names = get_source_package_list_from_url("https://download.clearlinux.org/releases/%s/clear/source/SRPMS/" % latest, "main", d) 95eb8dc403SDave Cobbley return latest, package_names 96eb8dc403SDave Cobbley 97eb8dc403SDave Cobbleydef find_latest_debian_release(url, d): 98eb8dc403SDave Cobbley "Find the latest listed debian release on the given url" 99eb8dc403SDave Cobbley 100eb8dc403SDave Cobbley releases = [link.replace("Debian", "") 101eb8dc403SDave Cobbley for link in get_links_from_url(url, d) 102eb8dc403SDave Cobbley if link.startswith("Debian")] 103eb8dc403SDave Cobbley releases.sort() 104eb8dc403SDave Cobbley try: 105eb8dc403SDave Cobbley return releases[-1] 106eb8dc403SDave Cobbley except: 107eb8dc403SDave Cobbley return "_NotFound_" 108eb8dc403SDave Cobbley 109eb8dc403SDave Cobbleydef get_debian_style_source_package_list(url, section, d): 110eb8dc403SDave Cobbley "Return the list of package-names stored in the debian style Sources.gz file" 111eb8dc403SDave Cobbley import gzip 112eb8dc403SDave Cobbley 113eb8dc403SDave Cobbley package_names = set() 114eb8dc403SDave Cobbley for line in gzip.open(create_socket(url, d), mode="rt"): 115eb8dc403SDave Cobbley if line.startswith("Package:"): 116eb8dc403SDave Cobbley pkg = line.split(":", 1)[1].strip() 117eb8dc403SDave Cobbley package_names.add(pkg + ":" + section) 118eb8dc403SDave Cobbley return package_names 119eb8dc403SDave Cobbley 120eb8dc403SDave Cobbleydef get_latest_released_debian_source_package_list(d): 121eb8dc403SDave Cobbley "Returns list of all the name of packages in the latest debian distro" 122eb8dc403SDave Cobbley latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d) 123eb8dc403SDave Cobbley url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" 124eb8dc403SDave Cobbley package_names = get_debian_style_source_package_list(url, "main", d) 125eb8dc403SDave Cobbley url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" 126eb8dc403SDave Cobbley package_names |= get_debian_style_source_package_list(url, "updates", d) 127eb8dc403SDave Cobbley return latest, package_names 128eb8dc403SDave Cobbley 129eb8dc403SDave Cobbleydef find_latest_ubuntu_release(url, d): 130eb8dc403SDave Cobbley """ 131eb8dc403SDave Cobbley Find the latest listed Ubuntu release on the given ubuntu/dists/ URL. 132eb8dc403SDave Cobbley 133eb8dc403SDave Cobbley To avoid matching development releases look for distributions that have 134eb8dc403SDave Cobbley updates, so the resulting distro could be any supported release. 135eb8dc403SDave Cobbley """ 136eb8dc403SDave Cobbley url += "?C=M;O=D" # Descending Sort by Last Modified 137eb8dc403SDave Cobbley for link in get_links_from_url(url, d): 138eb8dc403SDave Cobbley if "-updates" in link: 139eb8dc403SDave Cobbley distro = link.replace("-updates", "") 140eb8dc403SDave Cobbley return distro 141eb8dc403SDave Cobbley return "_NotFound_" 142eb8dc403SDave Cobbley 143eb8dc403SDave Cobbleydef get_latest_released_ubuntu_source_package_list(d): 144eb8dc403SDave Cobbley "Returns list of all the name os packages in the latest ubuntu distro" 145eb8dc403SDave Cobbley latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d) 146eb8dc403SDave Cobbley url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest 147eb8dc403SDave Cobbley package_names = get_debian_style_source_package_list(url, "main", d) 148eb8dc403SDave Cobbley url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest 149eb8dc403SDave Cobbley package_names |= get_debian_style_source_package_list(url, "updates", d) 150eb8dc403SDave Cobbley return latest, package_names 151eb8dc403SDave Cobbley 152eb8dc403SDave Cobbleydef create_distro_packages_list(distro_check_dir, d): 153eb8dc403SDave Cobbley import shutil 154eb8dc403SDave Cobbley 155eb8dc403SDave Cobbley pkglst_dir = os.path.join(distro_check_dir, "package_lists") 156eb8dc403SDave Cobbley bb.utils.remove(pkglst_dir, True) 157eb8dc403SDave Cobbley bb.utils.mkdirhier(pkglst_dir) 158eb8dc403SDave Cobbley 159eb8dc403SDave Cobbley per_distro_functions = ( 160eb8dc403SDave Cobbley ("Debian", get_latest_released_debian_source_package_list), 161eb8dc403SDave Cobbley ("Ubuntu", get_latest_released_ubuntu_source_package_list), 162eb8dc403SDave Cobbley ("Fedora", get_latest_released_fedora_source_package_list), 163eb8dc403SDave Cobbley ("openSUSE", get_latest_released_opensuse_source_package_list), 164eb8dc403SDave Cobbley ("Clear", get_latest_released_clear_source_package_list), 165eb8dc403SDave Cobbley ) 166eb8dc403SDave Cobbley 167eb8dc403SDave Cobbley for name, fetcher_func in per_distro_functions: 168eb8dc403SDave Cobbley try: 169eb8dc403SDave Cobbley release, package_list = fetcher_func(d) 170eb8dc403SDave Cobbley except Exception as e: 171eb8dc403SDave Cobbley bb.warn("Cannot fetch packages for %s: %s" % (name, e)) 172eb8dc403SDave Cobbley bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list))) 173eb8dc403SDave Cobbley if len(package_list) == 0: 174eb8dc403SDave Cobbley bb.error("Didn't fetch any packages for %s %s" % (name, release)) 175eb8dc403SDave Cobbley 176eb8dc403SDave Cobbley package_list_file = os.path.join(pkglst_dir, name + "-" + release) 177eb8dc403SDave Cobbley with open(package_list_file, 'w') as f: 178eb8dc403SDave Cobbley for pkg in sorted(package_list): 179eb8dc403SDave Cobbley f.write(pkg + "\n") 180eb8dc403SDave Cobbley 181eb8dc403SDave Cobbleydef update_distro_data(distro_check_dir, datetime, d): 182eb8dc403SDave Cobbley """ 183eb8dc403SDave Cobbley If distro packages list data is old then rebuild it. 184eb8dc403SDave Cobbley The operations has to be protected by a lock so that 185eb8dc403SDave Cobbley only one thread performes it at a time. 186eb8dc403SDave Cobbley """ 187eb8dc403SDave Cobbley if not os.path.isdir (distro_check_dir): 188eb8dc403SDave Cobbley try: 189eb8dc403SDave Cobbley bb.note ("Making new directory: %s" % distro_check_dir) 190eb8dc403SDave Cobbley os.makedirs (distro_check_dir) 191eb8dc403SDave Cobbley except OSError: 192eb8dc403SDave Cobbley raise Exception('Unable to create directory %s' % (distro_check_dir)) 193eb8dc403SDave Cobbley 194eb8dc403SDave Cobbley 195eb8dc403SDave Cobbley datetime_file = os.path.join(distro_check_dir, "build_datetime") 196eb8dc403SDave Cobbley saved_datetime = "_invalid_" 197eb8dc403SDave Cobbley import fcntl 198eb8dc403SDave Cobbley try: 199eb8dc403SDave Cobbley if not os.path.exists(datetime_file): 200eb8dc403SDave Cobbley open(datetime_file, 'w+').close() # touch the file so that the next open won't fail 201eb8dc403SDave Cobbley 202eb8dc403SDave Cobbley f = open(datetime_file, "r+") 203eb8dc403SDave Cobbley fcntl.lockf(f, fcntl.LOCK_EX) 204eb8dc403SDave Cobbley saved_datetime = f.read() 205eb8dc403SDave Cobbley if saved_datetime[0:8] != datetime[0:8]: 206eb8dc403SDave Cobbley bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime)) 207eb8dc403SDave Cobbley bb.note("Regenerating distro package lists") 208eb8dc403SDave Cobbley create_distro_packages_list(distro_check_dir, d) 209eb8dc403SDave Cobbley f.seek(0) 210eb8dc403SDave Cobbley f.write(datetime) 211eb8dc403SDave Cobbley 212eb8dc403SDave Cobbley except OSError as e: 213eb8dc403SDave Cobbley raise Exception('Unable to open timestamp: %s' % e) 214eb8dc403SDave Cobbley finally: 215eb8dc403SDave Cobbley fcntl.lockf(f, fcntl.LOCK_UN) 216eb8dc403SDave Cobbley f.close() 217eb8dc403SDave Cobbley 218eb8dc403SDave Cobbleydef compare_in_distro_packages_list(distro_check_dir, d): 219eb8dc403SDave Cobbley if not os.path.isdir(distro_check_dir): 220eb8dc403SDave Cobbley raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed") 221eb8dc403SDave Cobbley 222eb8dc403SDave Cobbley localdata = bb.data.createCopy(d) 223eb8dc403SDave Cobbley pkglst_dir = os.path.join(distro_check_dir, "package_lists") 224eb8dc403SDave Cobbley matching_distros = [] 225eb8dc403SDave Cobbley pn = recipe_name = d.getVar('PN') 226eb8dc403SDave Cobbley bb.note("Checking: %s" % pn) 227eb8dc403SDave Cobbley 228eb8dc403SDave Cobbley if pn.find("-native") != -1: 229eb8dc403SDave Cobbley pnstripped = pn.split("-native") 230eb8dc403SDave Cobbley localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) 231eb8dc403SDave Cobbley recipe_name = pnstripped[0] 232eb8dc403SDave Cobbley 233eb8dc403SDave Cobbley if pn.startswith("nativesdk-"): 234eb8dc403SDave Cobbley pnstripped = pn.split("nativesdk-") 235eb8dc403SDave Cobbley localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES')) 236eb8dc403SDave Cobbley recipe_name = pnstripped[1] 237eb8dc403SDave Cobbley 238eb8dc403SDave Cobbley if pn.find("-cross") != -1: 239eb8dc403SDave Cobbley pnstripped = pn.split("-cross") 240eb8dc403SDave Cobbley localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) 241eb8dc403SDave Cobbley recipe_name = pnstripped[0] 242eb8dc403SDave Cobbley 243eb8dc403SDave Cobbley if pn.find("-initial") != -1: 244eb8dc403SDave Cobbley pnstripped = pn.split("-initial") 245eb8dc403SDave Cobbley localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) 246eb8dc403SDave Cobbley recipe_name = pnstripped[0] 247eb8dc403SDave Cobbley 248eb8dc403SDave Cobbley bb.note("Recipe: %s" % recipe_name) 249eb8dc403SDave Cobbley 250eb8dc403SDave Cobbley distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'}) 251eb8dc403SDave Cobbley tmp = localdata.getVar('DISTRO_PN_ALIAS') or "" 252eb8dc403SDave Cobbley for str in tmp.split(): 253eb8dc403SDave Cobbley if str and str.find("=") == -1 and distro_exceptions[str]: 254eb8dc403SDave Cobbley matching_distros.append(str) 255eb8dc403SDave Cobbley 256eb8dc403SDave Cobbley distro_pn_aliases = {} 257eb8dc403SDave Cobbley for str in tmp.split(): 258eb8dc403SDave Cobbley if "=" in str: 259eb8dc403SDave Cobbley (dist, pn_alias) = str.split('=') 260eb8dc403SDave Cobbley distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() 261eb8dc403SDave Cobbley 262eb8dc403SDave Cobbley for file in os.listdir(pkglst_dir): 263eb8dc403SDave Cobbley (distro, distro_release) = file.split("-") 264eb8dc403SDave Cobbley f = open(os.path.join(pkglst_dir, file), "r") 265eb8dc403SDave Cobbley for line in f: 266eb8dc403SDave Cobbley (pkg, section) = line.split(":") 267eb8dc403SDave Cobbley if distro.lower() in distro_pn_aliases: 268eb8dc403SDave Cobbley pn = distro_pn_aliases[distro.lower()] 269eb8dc403SDave Cobbley else: 270eb8dc403SDave Cobbley pn = recipe_name 271eb8dc403SDave Cobbley if pn == pkg: 272eb8dc403SDave Cobbley matching_distros.append(distro + "-" + section[:-1]) # strip the \n at the end 273eb8dc403SDave Cobbley f.close() 274eb8dc403SDave Cobbley break 275eb8dc403SDave Cobbley f.close() 276eb8dc403SDave Cobbley 277eb8dc403SDave Cobbley for item in tmp.split(): 278eb8dc403SDave Cobbley matching_distros.append(item) 279eb8dc403SDave Cobbley bb.note("Matching: %s" % matching_distros) 280eb8dc403SDave Cobbley return matching_distros 281eb8dc403SDave Cobbley 282eb8dc403SDave Cobbleydef create_log_file(d, logname): 283eb8dc403SDave Cobbley logpath = d.getVar('LOG_DIR') 284eb8dc403SDave Cobbley bb.utils.mkdirhier(logpath) 285eb8dc403SDave Cobbley logfn, logsuffix = os.path.splitext(logname) 286eb8dc403SDave Cobbley logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME'), logsuffix)) 287eb8dc403SDave Cobbley if not os.path.exists(logfile): 288eb8dc403SDave Cobbley slogfile = os.path.join(logpath, logname) 289eb8dc403SDave Cobbley if os.path.exists(slogfile): 290eb8dc403SDave Cobbley os.remove(slogfile) 291eb8dc403SDave Cobbley open(logfile, 'w+').close() 292eb8dc403SDave Cobbley os.symlink(logfile, slogfile) 293eb8dc403SDave Cobbley d.setVar('LOG_FILE', logfile) 294eb8dc403SDave Cobbley return logfile 295eb8dc403SDave Cobbley 296eb8dc403SDave Cobbley 297eb8dc403SDave Cobbleydef save_distro_check_result(result, datetime, result_file, d): 298eb8dc403SDave Cobbley pn = d.getVar('PN') 299eb8dc403SDave Cobbley logdir = d.getVar('LOG_DIR') 300eb8dc403SDave Cobbley if not logdir: 301eb8dc403SDave Cobbley bb.error("LOG_DIR variable is not defined, can't write the distro_check results") 302eb8dc403SDave Cobbley return 303eb8dc403SDave Cobbley bb.utils.mkdirhier(logdir) 304eb8dc403SDave Cobbley 305eb8dc403SDave Cobbley line = pn 306eb8dc403SDave Cobbley for i in result: 307eb8dc403SDave Cobbley line = line + "," + i 308eb8dc403SDave Cobbley f = open(result_file, "a") 309eb8dc403SDave Cobbley import fcntl 310eb8dc403SDave Cobbley fcntl.lockf(f, fcntl.LOCK_EX) 311eb8dc403SDave Cobbley f.seek(0, os.SEEK_END) # seek to the end of file 312eb8dc403SDave Cobbley f.write(line + "\n") 313eb8dc403SDave Cobbley fcntl.lockf(f, fcntl.LOCK_UN) 314eb8dc403SDave Cobbley f.close() 315