1# 2# SPDX-License-Identifier: GPL-2.0-only 3# 4 5def create_socket(url, d): 6 import urllib 7 from bb.utils import export_proxies 8 9 export_proxies(d) 10 return urllib.request.urlopen(url) 11 12def get_links_from_url(url, d): 13 "Return all the href links found on the web location" 14 15 from bs4 import BeautifulSoup, SoupStrainer 16 17 soup = BeautifulSoup(create_socket(url,d), "html.parser", parse_only=SoupStrainer("a")) 18 hyperlinks = [] 19 for line in soup.find_all('a', href=True): 20 hyperlinks.append(line['href'].strip('/')) 21 return hyperlinks 22 23def find_latest_numeric_release(url, d): 24 "Find the latest listed numeric release on the given url" 25 max=0 26 maxstr="" 27 for link in get_links_from_url(url, d): 28 try: 29 # TODO use LooseVersion 30 release = float(link) 31 except: 32 release = 0 33 if release > max: 34 max = release 35 maxstr = link 36 return maxstr 37 38def is_src_rpm(name): 39 "Check if the link is pointing to a src.rpm file" 40 return name.endswith(".src.rpm") 41 42def package_name_from_srpm(srpm): 43 "Strip out the package name from the src.rpm filename" 44 45 # ca-certificates-2016.2.7-1.0.fc24.src.rpm 46 # ^name ^ver ^release^removed 47 (name, version, release) = srpm.replace(".src.rpm", "").rsplit("-", 2) 48 return name 49 50def get_source_package_list_from_url(url, section, d): 51 "Return a sectioned list of package names from a URL list" 52 53 bb.note("Reading %s: %s" % (url, section)) 54 links = get_links_from_url(url, d) 55 srpms = filter(is_src_rpm, links) 56 names_list = map(package_name_from_srpm, srpms) 57 58 new_pkgs = set() 59 for pkgs in names_list: 60 new_pkgs.add(pkgs + ":" + section) 61 return new_pkgs 62 63def get_source_package_list_from_url_by_letter(url, section, d): 64 import string 65 from urllib.error import HTTPError 66 packages = set() 67 for letter in (string.ascii_lowercase + string.digits): 68 # Not all subfolders may exist, so silently handle 404 69 try: 70 packages |= get_source_package_list_from_url(url + "/" + letter, section, d) 71 except HTTPError as e: 72 if e.code != 404: raise 73 return packages 74 75def get_latest_released_fedora_source_package_list(d): 76 "Returns list of all the name os packages in the latest fedora distro" 77 latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d) 78 package_names = get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Everything/source/tree/Packages/" % latest, "main", d) 79 package_names |= get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d) 80 return latest, package_names 81 82def get_latest_released_opensuse_source_package_list(d): 83 "Returns list of all the name os packages in the latest opensuse distro" 84 latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/leap", d) 85 86 package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/leap/%s/repo/oss/suse/src/" % latest, "main", d) 87 package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/leap/%s/oss/src/" % latest, "updates", d) 88 return latest, package_names 89 90def get_latest_released_clear_source_package_list(d): 91 latest = find_latest_numeric_release("https://download.clearlinux.org/releases/", d) 92 package_names = get_source_package_list_from_url("https://download.clearlinux.org/releases/%s/clear/source/SRPMS/" % latest, "main", d) 93 return latest, package_names 94 95def find_latest_debian_release(url, d): 96 "Find the latest listed debian release on the given url" 97 98 releases = [link.replace("Debian", "") 99 for link in get_links_from_url(url, d) 100 if link.startswith("Debian")] 101 releases.sort() 102 try: 103 return releases[-1] 104 except: 105 return "_NotFound_" 106 107def get_debian_style_source_package_list(url, section, d): 108 "Return the list of package-names stored in the debian style Sources.gz file" 109 import gzip 110 111 package_names = set() 112 for line in gzip.open(create_socket(url, d), mode="rt"): 113 if line.startswith("Package:"): 114 pkg = line.split(":", 1)[1].strip() 115 package_names.add(pkg + ":" + section) 116 return package_names 117 118def get_latest_released_debian_source_package_list(d): 119 "Returns list of all the name of packages in the latest debian distro" 120 latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d) 121 url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" 122 package_names = get_debian_style_source_package_list(url, "main", d) 123 url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" 124 package_names |= get_debian_style_source_package_list(url, "updates", d) 125 return latest, package_names 126 127def find_latest_ubuntu_release(url, d): 128 """ 129 Find the latest listed Ubuntu release on the given ubuntu/dists/ URL. 130 131 To avoid matching development releases look for distributions that have 132 updates, so the resulting distro could be any supported release. 133 """ 134 url += "?C=M;O=D" # Descending Sort by Last Modified 135 for link in get_links_from_url(url, d): 136 if "-updates" in link: 137 distro = link.replace("-updates", "") 138 return distro 139 return "_NotFound_" 140 141def get_latest_released_ubuntu_source_package_list(d): 142 "Returns list of all the name os packages in the latest ubuntu distro" 143 latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d) 144 url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest 145 package_names = get_debian_style_source_package_list(url, "main", d) 146 url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest 147 package_names |= get_debian_style_source_package_list(url, "updates", d) 148 return latest, package_names 149 150def create_distro_packages_list(distro_check_dir, d): 151 import shutil 152 153 pkglst_dir = os.path.join(distro_check_dir, "package_lists") 154 bb.utils.remove(pkglst_dir, True) 155 bb.utils.mkdirhier(pkglst_dir) 156 157 per_distro_functions = ( 158 ("Debian", get_latest_released_debian_source_package_list), 159 ("Ubuntu", get_latest_released_ubuntu_source_package_list), 160 ("Fedora", get_latest_released_fedora_source_package_list), 161 ("openSUSE", get_latest_released_opensuse_source_package_list), 162 ("Clear", get_latest_released_clear_source_package_list), 163 ) 164 165 for name, fetcher_func in per_distro_functions: 166 try: 167 release, package_list = fetcher_func(d) 168 except Exception as e: 169 bb.warn("Cannot fetch packages for %s: %s" % (name, e)) 170 bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list))) 171 if len(package_list) == 0: 172 bb.error("Didn't fetch any packages for %s %s" % (name, release)) 173 174 package_list_file = os.path.join(pkglst_dir, name + "-" + release) 175 with open(package_list_file, 'w') as f: 176 for pkg in sorted(package_list): 177 f.write(pkg + "\n") 178 179def update_distro_data(distro_check_dir, datetime, d): 180 """ 181 If distro packages list data is old then rebuild it. 182 The operations has to be protected by a lock so that 183 only one thread performes it at a time. 184 """ 185 if not os.path.isdir (distro_check_dir): 186 try: 187 bb.note ("Making new directory: %s" % distro_check_dir) 188 os.makedirs (distro_check_dir) 189 except OSError: 190 raise Exception('Unable to create directory %s' % (distro_check_dir)) 191 192 193 datetime_file = os.path.join(distro_check_dir, "build_datetime") 194 saved_datetime = "_invalid_" 195 import fcntl 196 try: 197 if not os.path.exists(datetime_file): 198 open(datetime_file, 'w+').close() # touch the file so that the next open won't fail 199 200 f = open(datetime_file, "r+") 201 fcntl.lockf(f, fcntl.LOCK_EX) 202 saved_datetime = f.read() 203 if saved_datetime[0:8] != datetime[0:8]: 204 bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime)) 205 bb.note("Regenerating distro package lists") 206 create_distro_packages_list(distro_check_dir, d) 207 f.seek(0) 208 f.write(datetime) 209 210 except OSError as e: 211 raise Exception('Unable to open timestamp: %s' % e) 212 finally: 213 fcntl.lockf(f, fcntl.LOCK_UN) 214 f.close() 215 216def compare_in_distro_packages_list(distro_check_dir, d): 217 if not os.path.isdir(distro_check_dir): 218 raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed") 219 220 localdata = bb.data.createCopy(d) 221 pkglst_dir = os.path.join(distro_check_dir, "package_lists") 222 matching_distros = [] 223 pn = recipe_name = d.getVar('PN') 224 bb.note("Checking: %s" % pn) 225 226 if pn.find("-native") != -1: 227 pnstripped = pn.split("-native") 228 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) 229 recipe_name = pnstripped[0] 230 231 if pn.startswith("nativesdk-"): 232 pnstripped = pn.split("nativesdk-") 233 localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES')) 234 recipe_name = pnstripped[1] 235 236 if pn.find("-cross") != -1: 237 pnstripped = pn.split("-cross") 238 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) 239 recipe_name = pnstripped[0] 240 241 if pn.find("-initial") != -1: 242 pnstripped = pn.split("-initial") 243 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) 244 recipe_name = pnstripped[0] 245 246 bb.note("Recipe: %s" % recipe_name) 247 248 distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'}) 249 tmp = localdata.getVar('DISTRO_PN_ALIAS') or "" 250 for str in tmp.split(): 251 if str and str.find("=") == -1 and distro_exceptions[str]: 252 matching_distros.append(str) 253 254 distro_pn_aliases = {} 255 for str in tmp.split(): 256 if "=" in str: 257 (dist, pn_alias) = str.split('=') 258 distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() 259 260 for file in os.listdir(pkglst_dir): 261 (distro, distro_release) = file.split("-") 262 f = open(os.path.join(pkglst_dir, file), "r") 263 for line in f: 264 (pkg, section) = line.split(":") 265 if distro.lower() in distro_pn_aliases: 266 pn = distro_pn_aliases[distro.lower()] 267 else: 268 pn = recipe_name 269 if pn == pkg: 270 matching_distros.append(distro + "-" + section[:-1]) # strip the \n at the end 271 f.close() 272 break 273 f.close() 274 275 for item in tmp.split(): 276 matching_distros.append(item) 277 bb.note("Matching: %s" % matching_distros) 278 return matching_distros 279 280def create_log_file(d, logname): 281 logpath = d.getVar('LOG_DIR') 282 bb.utils.mkdirhier(logpath) 283 logfn, logsuffix = os.path.splitext(logname) 284 logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME'), logsuffix)) 285 if not os.path.exists(logfile): 286 slogfile = os.path.join(logpath, logname) 287 if os.path.exists(slogfile): 288 os.remove(slogfile) 289 open(logfile, 'w+').close() 290 os.symlink(logfile, slogfile) 291 d.setVar('LOG_FILE', logfile) 292 return logfile 293 294 295def save_distro_check_result(result, datetime, result_file, d): 296 pn = d.getVar('PN') 297 logdir = d.getVar('LOG_DIR') 298 if not logdir: 299 bb.error("LOG_DIR variable is not defined, can't write the distro_check results") 300 return 301 bb.utils.mkdirhier(logdir) 302 303 line = pn 304 for i in result: 305 line = line + "," + i 306 f = open(result_file, "a") 307 import fcntl 308 fcntl.lockf(f, fcntl.LOCK_EX) 309 f.seek(0, os.SEEK_END) # seek to the end of file 310 f.write(line + "\n") 311 fcntl.lockf(f, fcntl.LOCK_UN) 312 f.close() 313