1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6 7def create_socket(url, d): 8 import urllib 9 from bb.utils import export_proxies 10 11 export_proxies(d) 12 return urllib.request.urlopen(url) 13 14def get_links_from_url(url, d): 15 "Return all the href links found on the web location" 16 17 from bs4 import BeautifulSoup, SoupStrainer 18 19 soup = BeautifulSoup(create_socket(url,d), "html.parser", parse_only=SoupStrainer("a")) 20 hyperlinks = [] 21 for line in soup.find_all('a', href=True): 22 hyperlinks.append(line['href'].strip('/')) 23 return hyperlinks 24 25def find_latest_numeric_release(url, d): 26 "Find the latest listed numeric release on the given url" 27 max=0 28 maxstr="" 29 for link in get_links_from_url(url, d): 30 try: 31 # TODO use bb.utils.vercmp_string_op() 32 release = float(link) 33 except: 34 release = 0 35 if release > max: 36 max = release 37 maxstr = link 38 return maxstr 39 40def is_src_rpm(name): 41 "Check if the link is pointing to a src.rpm file" 42 return name.endswith(".src.rpm") 43 44def package_name_from_srpm(srpm): 45 "Strip out the package name from the src.rpm filename" 46 47 # ca-certificates-2016.2.7-1.0.fc24.src.rpm 48 # ^name ^ver ^release^removed 49 (name, version, release) = srpm.replace(".src.rpm", "").rsplit("-", 2) 50 return name 51 52def get_source_package_list_from_url(url, section, d): 53 "Return a sectioned list of package names from a URL list" 54 55 bb.note("Reading %s: %s" % (url, section)) 56 links = get_links_from_url(url, d) 57 srpms = filter(is_src_rpm, links) 58 names_list = map(package_name_from_srpm, srpms) 59 60 new_pkgs = set() 61 for pkgs in names_list: 62 new_pkgs.add(pkgs + ":" + section) 63 return new_pkgs 64 65def get_source_package_list_from_url_by_letter(url, section, d): 66 import string 67 from urllib.error import HTTPError 68 packages = set() 69 for letter in (string.ascii_lowercase + string.digits): 70 # Not all subfolders may exist, so silently handle 404 71 try: 72 packages |= get_source_package_list_from_url(url + "/" + letter, section, d) 73 except HTTPError as e: 74 if e.code != 404: raise 75 return packages 76 77def get_latest_released_fedora_source_package_list(d): 78 "Returns list of all the name os packages in the latest fedora distro" 79 latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d) 80 package_names = get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Everything/source/tree/Packages/" % latest, "main", d) 81 package_names |= get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d) 82 return latest, package_names 83 84def get_latest_released_opensuse_source_package_list(d): 85 "Returns list of all the name os packages in the latest opensuse distro" 86 latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/leap", d) 87 88 package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/leap/%s/repo/oss/suse/src/" % latest, "main", d) 89 package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/leap/%s/oss/src/" % latest, "updates", d) 90 return latest, package_names 91 92def get_latest_released_clear_source_package_list(d): 93 latest = find_latest_numeric_release("https://download.clearlinux.org/releases/", d) 94 package_names = get_source_package_list_from_url("https://download.clearlinux.org/releases/%s/clear/source/SRPMS/" % latest, "main", d) 95 return latest, package_names 96 97def find_latest_debian_release(url, d): 98 "Find the latest listed debian release on the given url" 99 100 releases = [link.replace("Debian", "") 101 for link in get_links_from_url(url, d) 102 if link.startswith("Debian")] 103 releases.sort() 104 try: 105 return releases[-1] 106 except: 107 return "_NotFound_" 108 109def get_debian_style_source_package_list(url, section, d): 110 "Return the list of package-names stored in the debian style Sources.gz file" 111 import gzip 112 113 package_names = set() 114 for line in gzip.open(create_socket(url, d), mode="rt"): 115 if line.startswith("Package:"): 116 pkg = line.split(":", 1)[1].strip() 117 package_names.add(pkg + ":" + section) 118 return package_names 119 120def get_latest_released_debian_source_package_list(d): 121 "Returns list of all the name of packages in the latest debian distro" 122 latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d) 123 url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" 124 package_names = get_debian_style_source_package_list(url, "main", d) 125 url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" 126 package_names |= get_debian_style_source_package_list(url, "updates", d) 127 return latest, package_names 128 129def find_latest_ubuntu_release(url, d): 130 """ 131 Find the latest listed Ubuntu release on the given ubuntu/dists/ URL. 132 133 To avoid matching development releases look for distributions that have 134 updates, so the resulting distro could be any supported release. 135 """ 136 url += "?C=M;O=D" # Descending Sort by Last Modified 137 for link in get_links_from_url(url, d): 138 if "-updates" in link: 139 distro = link.replace("-updates", "") 140 return distro 141 return "_NotFound_" 142 143def get_latest_released_ubuntu_source_package_list(d): 144 "Returns list of all the name os packages in the latest ubuntu distro" 145 latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d) 146 url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest 147 package_names = get_debian_style_source_package_list(url, "main", d) 148 url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest 149 package_names |= get_debian_style_source_package_list(url, "updates", d) 150 return latest, package_names 151 152def create_distro_packages_list(distro_check_dir, d): 153 import shutil 154 155 pkglst_dir = os.path.join(distro_check_dir, "package_lists") 156 bb.utils.remove(pkglst_dir, True) 157 bb.utils.mkdirhier(pkglst_dir) 158 159 per_distro_functions = ( 160 ("Debian", get_latest_released_debian_source_package_list), 161 ("Ubuntu", get_latest_released_ubuntu_source_package_list), 162 ("Fedora", get_latest_released_fedora_source_package_list), 163 ("openSUSE", get_latest_released_opensuse_source_package_list), 164 ("Clear", get_latest_released_clear_source_package_list), 165 ) 166 167 for name, fetcher_func in per_distro_functions: 168 try: 169 release, package_list = fetcher_func(d) 170 except Exception as e: 171 bb.warn("Cannot fetch packages for %s: %s" % (name, e)) 172 bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list))) 173 if len(package_list) == 0: 174 bb.error("Didn't fetch any packages for %s %s" % (name, release)) 175 176 package_list_file = os.path.join(pkglst_dir, name + "-" + release) 177 with open(package_list_file, 'w') as f: 178 for pkg in sorted(package_list): 179 f.write(pkg + "\n") 180 181def update_distro_data(distro_check_dir, datetime, d): 182 """ 183 If distro packages list data is old then rebuild it. 184 The operations has to be protected by a lock so that 185 only one thread performes it at a time. 186 """ 187 if not os.path.isdir (distro_check_dir): 188 try: 189 bb.note ("Making new directory: %s" % distro_check_dir) 190 os.makedirs (distro_check_dir) 191 except OSError: 192 raise Exception('Unable to create directory %s' % (distro_check_dir)) 193 194 195 datetime_file = os.path.join(distro_check_dir, "build_datetime") 196 saved_datetime = "_invalid_" 197 import fcntl 198 try: 199 if not os.path.exists(datetime_file): 200 open(datetime_file, 'w+').close() # touch the file so that the next open won't fail 201 202 f = open(datetime_file, "r+") 203 fcntl.lockf(f, fcntl.LOCK_EX) 204 saved_datetime = f.read() 205 if saved_datetime[0:8] != datetime[0:8]: 206 bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime)) 207 bb.note("Regenerating distro package lists") 208 create_distro_packages_list(distro_check_dir, d) 209 f.seek(0) 210 f.write(datetime) 211 212 except OSError as e: 213 raise Exception('Unable to open timestamp: %s' % e) 214 finally: 215 fcntl.lockf(f, fcntl.LOCK_UN) 216 f.close() 217 218def compare_in_distro_packages_list(distro_check_dir, d): 219 if not os.path.isdir(distro_check_dir): 220 raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed") 221 222 localdata = bb.data.createCopy(d) 223 pkglst_dir = os.path.join(distro_check_dir, "package_lists") 224 matching_distros = [] 225 pn = recipe_name = d.getVar('PN') 226 bb.note("Checking: %s" % pn) 227 228 if pn.find("-native") != -1: 229 pnstripped = pn.split("-native") 230 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) 231 recipe_name = pnstripped[0] 232 233 if pn.startswith("nativesdk-"): 234 pnstripped = pn.split("nativesdk-") 235 localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES')) 236 recipe_name = pnstripped[1] 237 238 if pn.find("-cross") != -1: 239 pnstripped = pn.split("-cross") 240 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) 241 recipe_name = pnstripped[0] 242 243 if pn.find("-initial") != -1: 244 pnstripped = pn.split("-initial") 245 localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) 246 recipe_name = pnstripped[0] 247 248 bb.note("Recipe: %s" % recipe_name) 249 250 distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'}) 251 tmp = localdata.getVar('DISTRO_PN_ALIAS') or "" 252 for str in tmp.split(): 253 if str and str.find("=") == -1 and distro_exceptions[str]: 254 matching_distros.append(str) 255 256 distro_pn_aliases = {} 257 for str in tmp.split(): 258 if "=" in str: 259 (dist, pn_alias) = str.split('=') 260 distro_pn_aliases[dist.strip().lower()] = pn_alias.strip() 261 262 for file in os.listdir(pkglst_dir): 263 (distro, distro_release) = file.split("-") 264 f = open(os.path.join(pkglst_dir, file), "r") 265 for line in f: 266 (pkg, section) = line.split(":") 267 if distro.lower() in distro_pn_aliases: 268 pn = distro_pn_aliases[distro.lower()] 269 else: 270 pn = recipe_name 271 if pn == pkg: 272 matching_distros.append(distro + "-" + section[:-1]) # strip the \n at the end 273 f.close() 274 break 275 f.close() 276 277 for item in tmp.split(): 278 matching_distros.append(item) 279 bb.note("Matching: %s" % matching_distros) 280 return matching_distros 281 282def create_log_file(d, logname): 283 logpath = d.getVar('LOG_DIR') 284 bb.utils.mkdirhier(logpath) 285 logfn, logsuffix = os.path.splitext(logname) 286 logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME'), logsuffix)) 287 if not os.path.exists(logfile): 288 slogfile = os.path.join(logpath, logname) 289 if os.path.exists(slogfile): 290 os.remove(slogfile) 291 open(logfile, 'w+').close() 292 os.symlink(logfile, slogfile) 293 d.setVar('LOG_FILE', logfile) 294 return logfile 295 296 297def save_distro_check_result(result, datetime, result_file, d): 298 pn = d.getVar('PN') 299 logdir = d.getVar('LOG_DIR') 300 if not logdir: 301 bb.error("LOG_DIR variable is not defined, can't write the distro_check results") 302 return 303 bb.utils.mkdirhier(logdir) 304 305 line = pn 306 for i in result: 307 line = line + "," + i 308 f = open(result_file, "a") 309 import fcntl 310 fcntl.lockf(f, fcntl.LOCK_EX) 311 f.seek(0, os.SEEK_END) # seek to the end of file 312 f.write(line + "\n") 313 fcntl.lockf(f, fcntl.LOCK_UN) 314 f.close() 315