1""" 2BitBake 'Fetch' git implementation 3 4git fetcher support the SRC_URI with format of: 5SRC_URI = "git://some.host/somepath;OptionA=xxx;OptionB=xxx;..." 6 7Supported SRC_URI options are: 8 9- branch 10 The git branch to retrieve from. The default is "master" 11 12 This option also supports multiple branch fetching, with branches 13 separated by commas. In multiple branches case, the name option 14 must have the same number of names to match the branches, which is 15 used to specify the SRC_REV for the branch 16 e.g: 17 SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY" 18 SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx" 19 SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY" 20 21- tag 22 The git tag to retrieve. The default is "master" 23 24- protocol 25 The method to use to access the repository. Common options are "git", 26 "http", "https", "file", "ssh" and "rsync". The default is "git". 27 28- rebaseable 29 rebaseable indicates that the upstream git repo may rebase in the future, 30 and current revision may disappear from upstream repo. This option will 31 remind fetcher to preserve local cache carefully for future use. 32 The default value is "0", set rebaseable=1 for rebaseable git repo. 33 34- nocheckout 35 Don't checkout source code when unpacking. set this option for the recipe 36 who has its own routine to checkout code. 37 The default is "0", set nocheckout=1 if needed. 38 39- bareclone 40 Create a bare clone of the source code and don't checkout the source code 41 when unpacking. Set this option for the recipe who has its own routine to 42 checkout code and tracking branch requirements. 43 The default is "0", set bareclone=1 if needed. 44 45- nobranch 46 Don't check the SHA validation for branch. set this option for the recipe 47 referring to commit which is valid in tag instead of branch. 48 The default is "0", set nobranch=1 if needed. 49 50- usehead 51 For local git:// urls to use the current branch HEAD as the revision for use with 52 AUTOREV. Implies nobranch. 53 54""" 55 56# Copyright (C) 2005 Richard Purdie 57# 58# SPDX-License-Identifier: GPL-2.0-only 59# 60 61import collections 62import errno 63import fnmatch 64import os 65import re 66import shlex 67import subprocess 68import tempfile 69import bb 70import bb.progress 71from bb.fetch2 import FetchMethod 72from bb.fetch2 import runfetchcmd 73from bb.fetch2 import logger 74 75 76class GitProgressHandler(bb.progress.LineFilterProgressHandler): 77 """Extract progress information from git output""" 78 def __init__(self, d): 79 self._buffer = '' 80 self._count = 0 81 super(GitProgressHandler, self).__init__(d) 82 # Send an initial progress event so the bar gets shown 83 self._fire_progress(-1) 84 85 def write(self, string): 86 self._buffer += string 87 stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas'] 88 stage_weights = [0.2, 0.05, 0.5, 0.25] 89 stagenum = 0 90 for i, stage in reversed(list(enumerate(stages))): 91 if stage in self._buffer: 92 stagenum = i 93 self._buffer = '' 94 break 95 self._status = stages[stagenum] 96 percs = re.findall(r'(\d+)%', string) 97 if percs: 98 progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100))) 99 rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string) 100 if rates: 101 rate = rates[-1] 102 else: 103 rate = None 104 self.update(progress, rate) 105 else: 106 if stagenum == 0: 107 percs = re.findall(r': (\d+)', string) 108 if percs: 109 count = int(percs[-1]) 110 if count > self._count: 111 self._count = count 112 self._fire_progress(-count) 113 super(GitProgressHandler, self).write(string) 114 115 116class Git(FetchMethod): 117 bitbake_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.join(os.path.abspath(__file__))), '..', '..', '..')) 118 make_shallow_path = os.path.join(bitbake_dir, 'bin', 'git-make-shallow') 119 120 """Class to fetch a module or modules from git repositories""" 121 def init(self, d): 122 pass 123 124 def supports(self, ud, d): 125 """ 126 Check to see if a given url can be fetched with git. 127 """ 128 return ud.type in ['git'] 129 130 def supports_checksum(self, urldata): 131 return False 132 133 def urldata_init(self, ud, d): 134 """ 135 init git specific variable within url data 136 so that the git method like latest_revision() can work 137 """ 138 if 'protocol' in ud.parm: 139 ud.proto = ud.parm['protocol'] 140 elif not ud.host: 141 ud.proto = 'file' 142 else: 143 ud.proto = "git" 144 145 if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): 146 raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) 147 148 ud.nocheckout = ud.parm.get("nocheckout","0") == "1" 149 150 ud.rebaseable = ud.parm.get("rebaseable","0") == "1" 151 152 ud.nobranch = ud.parm.get("nobranch","0") == "1" 153 154 # usehead implies nobranch 155 ud.usehead = ud.parm.get("usehead","0") == "1" 156 if ud.usehead: 157 if ud.proto != "file": 158 raise bb.fetch2.ParameterError("The usehead option is only for use with local ('protocol=file') git repositories", ud.url) 159 ud.nobranch = 1 160 161 # bareclone implies nocheckout 162 ud.bareclone = ud.parm.get("bareclone","0") == "1" 163 if ud.bareclone: 164 ud.nocheckout = 1 165 166 ud.unresolvedrev = {} 167 branches = ud.parm.get("branch", "master").split(',') 168 if len(branches) != len(ud.names): 169 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) 170 171 ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1" 172 173 ud.cloneflags = "-n" 174 if not ud.noshared: 175 ud.cloneflags += " -s" 176 if ud.bareclone: 177 ud.cloneflags += " --mirror" 178 179 ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1" 180 ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split() 181 182 depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH") 183 if depth_default is not None: 184 try: 185 depth_default = int(depth_default or 0) 186 except ValueError: 187 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default) 188 else: 189 if depth_default < 0: 190 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % depth_default) 191 else: 192 depth_default = 1 193 ud.shallow_depths = collections.defaultdict(lambda: depth_default) 194 195 revs_default = d.getVar("BB_GIT_SHALLOW_REVS") 196 ud.shallow_revs = [] 197 ud.branches = {} 198 for pos, name in enumerate(ud.names): 199 branch = branches[pos] 200 ud.branches[name] = branch 201 ud.unresolvedrev[name] = branch 202 203 shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name) 204 if shallow_depth is not None: 205 try: 206 shallow_depth = int(shallow_depth or 0) 207 except ValueError: 208 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) 209 else: 210 if shallow_depth < 0: 211 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) 212 ud.shallow_depths[name] = shallow_depth 213 214 revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name) 215 if revs is not None: 216 ud.shallow_revs.extend(revs.split()) 217 elif revs_default is not None: 218 ud.shallow_revs.extend(revs_default.split()) 219 220 if (ud.shallow and 221 not ud.shallow_revs and 222 all(ud.shallow_depths[n] == 0 for n in ud.names)): 223 # Shallow disabled for this URL 224 ud.shallow = False 225 226 if ud.usehead: 227 # When usehead is set let's associate 'HEAD' with the unresolved 228 # rev of this repository. This will get resolved into a revision 229 # later. If an actual revision happens to have also been provided 230 # then this setting will be overridden. 231 for name in ud.names: 232 ud.unresolvedrev[name] = 'HEAD' 233 234 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0 -c gc.autoDetach=false" 235 236 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0" 237 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable 238 ud.write_shallow_tarballs = (d.getVar("BB_GENERATE_SHALLOW_TARBALLS") or write_tarballs) != "0" 239 240 ud.setup_revisions(d) 241 242 for name in ud.names: 243 # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one 244 if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): 245 if ud.revisions[name]: 246 ud.unresolvedrev[name] = ud.revisions[name] 247 ud.revisions[name] = self.latest_revision(ud, d, name) 248 249 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) 250 if gitsrcname.startswith('.'): 251 gitsrcname = gitsrcname[1:] 252 253 # for rebaseable git repo, it is necessary to keep mirror tar ball 254 # per revision, so that even the revision disappears from the 255 # upstream repo in the future, the mirror will remain intact and still 256 # contains the revision 257 if ud.rebaseable: 258 for name in ud.names: 259 gitsrcname = gitsrcname + '_' + ud.revisions[name] 260 261 dl_dir = d.getVar("DL_DIR") 262 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2") 263 ud.clonedir = os.path.join(gitdir, gitsrcname) 264 ud.localfile = ud.clonedir 265 266 mirrortarball = 'git2_%s.tar.gz' % gitsrcname 267 ud.fullmirror = os.path.join(dl_dir, mirrortarball) 268 ud.mirrortarballs = [mirrortarball] 269 if ud.shallow: 270 tarballname = gitsrcname 271 if ud.bareclone: 272 tarballname = "%s_bare" % tarballname 273 274 if ud.shallow_revs: 275 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs))) 276 277 for name, revision in sorted(ud.revisions.items()): 278 tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7]) 279 depth = ud.shallow_depths[name] 280 if depth: 281 tarballname = "%s-%s" % (tarballname, depth) 282 283 shallow_refs = [] 284 if not ud.nobranch: 285 shallow_refs.extend(ud.branches.values()) 286 if ud.shallow_extra_refs: 287 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs) 288 if shallow_refs: 289 tarballname = "%s_%s" % (tarballname, "_".join(sorted(shallow_refs)).replace('/', '.')) 290 291 fetcher = self.__class__.__name__.lower() 292 ud.shallowtarball = '%sshallow_%s.tar.gz' % (fetcher, tarballname) 293 ud.fullshallow = os.path.join(dl_dir, ud.shallowtarball) 294 ud.mirrortarballs.insert(0, ud.shallowtarball) 295 296 def localpath(self, ud, d): 297 return ud.clonedir 298 299 def need_update(self, ud, d): 300 return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud) 301 302 def clonedir_need_update(self, ud, d): 303 if not os.path.exists(ud.clonedir): 304 return True 305 if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d): 306 return True 307 for name in ud.names: 308 if not self._contains_ref(ud, d, name, ud.clonedir): 309 return True 310 return False 311 312 def clonedir_need_shallow_revs(self, ud, d): 313 for rev in ud.shallow_revs: 314 try: 315 runfetchcmd('%s rev-parse -q --verify %s' % (ud.basecmd, rev), d, quiet=True, workdir=ud.clonedir) 316 except bb.fetch2.FetchError: 317 return rev 318 return None 319 320 def shallow_tarball_need_update(self, ud): 321 return ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow) 322 323 def tarball_need_update(self, ud): 324 return ud.write_tarballs and not os.path.exists(ud.fullmirror) 325 326 def try_premirror(self, ud, d): 327 # If we don't do this, updating an existing checkout with only premirrors 328 # is not possible 329 if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): 330 return True 331 if os.path.exists(ud.clonedir): 332 return False 333 return True 334 335 def download(self, ud, d): 336 """Fetch url""" 337 338 # A current clone is preferred to either tarball, a shallow tarball is 339 # preferred to an out of date clone, and a missing clone will use 340 # either tarball. 341 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): 342 ud.localpath = ud.fullshallow 343 return 344 elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir): 345 bb.utils.mkdirhier(ud.clonedir) 346 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) 347 348 repourl = self._get_repo_url(ud) 349 350 # If the repo still doesn't exist, fallback to cloning it 351 if not os.path.exists(ud.clonedir): 352 # We do this since git will use a "-l" option automatically for local urls where possible 353 if repourl.startswith("file://"): 354 repourl = repourl[7:] 355 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir) 356 if ud.proto.lower() != 'file': 357 bb.fetch2.check_network_access(d, clone_cmd, ud.url) 358 progresshandler = GitProgressHandler(d) 359 runfetchcmd(clone_cmd, d, log=progresshandler) 360 361 # Update the checkout if needed 362 if self.clonedir_need_update(ud, d): 363 output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir) 364 if "origin" in output: 365 runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) 366 367 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) 368 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) 369 if ud.proto.lower() != 'file': 370 bb.fetch2.check_network_access(d, fetch_cmd, ud.url) 371 progresshandler = GitProgressHandler(d) 372 runfetchcmd(fetch_cmd, d, log=progresshandler, workdir=ud.clonedir) 373 runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir) 374 runfetchcmd("%s pack-refs --all" % ud.basecmd, d, workdir=ud.clonedir) 375 runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d, workdir=ud.clonedir) 376 try: 377 os.unlink(ud.fullmirror) 378 except OSError as exc: 379 if exc.errno != errno.ENOENT: 380 raise 381 382 for name in ud.names: 383 if not self._contains_ref(ud, d, name, ud.clonedir): 384 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name])) 385 386 if ud.shallow and ud.write_shallow_tarballs: 387 missing_rev = self.clonedir_need_shallow_revs(ud, d) 388 if missing_rev: 389 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) 390 391 if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud): 392 # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching 393 # of all LFS blobs needed at the the srcrev. 394 # 395 # It would be nice to just do this inline here by running 'git-lfs fetch' 396 # on the bare clonedir, but that operation requires a working copy on some 397 # releases of Git LFS. 398 tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) 399 try: 400 # Do the checkout. This implicitly involves a Git LFS fetch. 401 Git.unpack(self, ud, tmpdir, d) 402 403 # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into 404 # the bare clonedir. 405 # 406 # As this procedure is invoked repeatedly on incremental fetches as 407 # a recipe's SRCREV is bumped throughout its lifetime, this will 408 # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs 409 # corresponding to all the blobs reachable from the different revs 410 # fetched across time. 411 # 412 # Only do this if the unpack resulted in a .git/lfs directory being 413 # created; this only happens if at least one blob needed to be 414 # downloaded. 415 if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")): 416 runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir) 417 finally: 418 bb.utils.remove(tmpdir, recurse=True) 419 420 def build_mirror_data(self, ud, d): 421 if ud.shallow and ud.write_shallow_tarballs: 422 if not os.path.exists(ud.fullshallow): 423 if os.path.islink(ud.fullshallow): 424 os.unlink(ud.fullshallow) 425 tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) 426 shallowclone = os.path.join(tempdir, 'git') 427 try: 428 self.clone_shallow_local(ud, shallowclone, d) 429 430 logger.info("Creating tarball of git repository") 431 runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone) 432 runfetchcmd("touch %s.done" % ud.fullshallow, d) 433 finally: 434 bb.utils.remove(tempdir, recurse=True) 435 elif ud.write_tarballs and not os.path.exists(ud.fullmirror): 436 if os.path.islink(ud.fullmirror): 437 os.unlink(ud.fullmirror) 438 439 logger.info("Creating tarball of git repository") 440 runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir) 441 runfetchcmd("touch %s.done" % ud.fullmirror, d) 442 443 def clone_shallow_local(self, ud, dest, d): 444 """Clone the repo and make it shallow. 445 446 The upstream url of the new clone isn't set at this time, as it'll be 447 set correctly when unpacked.""" 448 runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d) 449 450 to_parse, shallow_branches = [], [] 451 for name in ud.names: 452 revision = ud.revisions[name] 453 depth = ud.shallow_depths[name] 454 if depth: 455 to_parse.append('%s~%d^{}' % (revision, depth - 1)) 456 457 # For nobranch, we need a ref, otherwise the commits will be 458 # removed, and for non-nobranch, we truncate the branch to our 459 # srcrev, to avoid keeping unnecessary history beyond that. 460 branch = ud.branches[name] 461 if ud.nobranch: 462 ref = "refs/shallow/%s" % name 463 elif ud.bareclone: 464 ref = "refs/heads/%s" % branch 465 else: 466 ref = "refs/remotes/origin/%s" % branch 467 468 shallow_branches.append(ref) 469 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest) 470 471 # Map srcrev+depths to revisions 472 parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest) 473 474 # Resolve specified revisions 475 parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest) 476 shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines() 477 478 # Apply extra ref wildcards 479 all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd, 480 d, workdir=dest).splitlines() 481 for r in ud.shallow_extra_refs: 482 if not ud.bareclone: 483 r = r.replace('refs/heads/', 'refs/remotes/origin/') 484 485 if '*' in r: 486 matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs) 487 shallow_branches.extend(matches) 488 else: 489 shallow_branches.append(r) 490 491 # Make the repository shallow 492 shallow_cmd = [self.make_shallow_path, '-s'] 493 for b in shallow_branches: 494 shallow_cmd.append('-r') 495 shallow_cmd.append(b) 496 shallow_cmd.extend(shallow_revisions) 497 runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest) 498 499 def unpack(self, ud, destdir, d): 500 """ unpack the downloaded src to destdir""" 501 502 subdir = ud.parm.get("subpath", "") 503 if subdir != "": 504 readpathspec = ":%s" % subdir 505 def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/')) 506 else: 507 readpathspec = "" 508 def_destsuffix = "git/" 509 510 destsuffix = ud.parm.get("destsuffix", def_destsuffix) 511 destdir = ud.destdir = os.path.join(destdir, destsuffix) 512 if os.path.exists(destdir): 513 bb.utils.prunedir(destdir) 514 515 need_lfs = self._need_lfs(ud) 516 517 if not need_lfs: 518 ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd 519 520 source_found = False 521 source_error = [] 522 523 if not source_found: 524 clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) 525 if clonedir_is_up_to_date: 526 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) 527 source_found = True 528 else: 529 source_error.append("clone directory not available or not up to date: " + ud.clonedir) 530 531 if not source_found: 532 if ud.shallow: 533 if os.path.exists(ud.fullshallow): 534 bb.utils.mkdirhier(destdir) 535 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir) 536 source_found = True 537 else: 538 source_error.append("shallow clone not available: " + ud.fullshallow) 539 else: 540 source_error.append("shallow clone not enabled") 541 542 if not source_found: 543 raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url) 544 545 repourl = self._get_repo_url(ud) 546 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir) 547 548 if self._contains_lfs(ud, d, destdir): 549 if need_lfs and not self._find_git_lfs(d): 550 raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl)) 551 elif not need_lfs: 552 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) 553 554 if not ud.nocheckout: 555 if subdir != "": 556 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d, 557 workdir=destdir) 558 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) 559 elif not ud.nobranch: 560 branchname = ud.branches[ud.names[0]] 561 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \ 562 ud.revisions[ud.names[0]]), d, workdir=destdir) 563 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \ 564 branchname), d, workdir=destdir) 565 else: 566 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir) 567 568 return True 569 570 def clean(self, ud, d): 571 """ clean the git directory """ 572 573 to_remove = [ud.localpath, ud.fullmirror, ud.fullmirror + ".done"] 574 # The localpath is a symlink to clonedir when it is cloned from a 575 # mirror, so remove both of them. 576 if os.path.islink(ud.localpath): 577 clonedir = os.path.realpath(ud.localpath) 578 to_remove.append(clonedir) 579 580 for r in to_remove: 581 if os.path.exists(r): 582 bb.note('Removing %s' % r) 583 bb.utils.remove(r, True) 584 585 def supports_srcrev(self): 586 return True 587 588 def _contains_ref(self, ud, d, name, wd): 589 cmd = "" 590 if ud.nobranch: 591 cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % ( 592 ud.basecmd, ud.revisions[name]) 593 else: 594 cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % ( 595 ud.basecmd, ud.revisions[name], ud.branches[name]) 596 try: 597 output = runfetchcmd(cmd, d, quiet=True, workdir=wd) 598 except bb.fetch2.FetchError: 599 return False 600 if len(output.split()) > 1: 601 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) 602 return output.split()[0] != "0" 603 604 def _need_lfs(self, ud): 605 return ud.parm.get("lfs", "1") == "1" 606 607 def _contains_lfs(self, ud, d, wd): 608 """ 609 Check if the repository has 'lfs' (large file) content 610 """ 611 612 if not ud.nobranch: 613 branchname = ud.branches[ud.names[0]] 614 else: 615 branchname = "master" 616 617 # The bare clonedir doesn't use the remote names; it has the branch immediately. 618 if wd == ud.clonedir: 619 refname = ud.branches[ud.names[0]] 620 else: 621 refname = "origin/%s" % ud.branches[ud.names[0]] 622 623 cmd = "%s grep lfs %s:.gitattributes | wc -l" % ( 624 ud.basecmd, refname) 625 626 try: 627 output = runfetchcmd(cmd, d, quiet=True, workdir=wd) 628 if int(output) > 0: 629 return True 630 except (bb.fetch2.FetchError,ValueError): 631 pass 632 return False 633 634 def _find_git_lfs(self, d): 635 """ 636 Return True if git-lfs can be found, False otherwise. 637 """ 638 import shutil 639 return shutil.which("git-lfs", path=d.getVar('PATH')) is not None 640 641 def _get_repo_url(self, ud): 642 """ 643 Return the repository URL 644 """ 645 # Note that we do not support passwords directly in the git urls. There are several 646 # reasons. SRC_URI can be written out to things like buildhistory and people don't 647 # want to leak passwords like that. Its also all too easy to share metadata without 648 # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as 649 # alternatives so we will not take patches adding password support here. 650 if ud.user: 651 username = ud.user + '@' 652 else: 653 username = "" 654 return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path) 655 656 def _revision_key(self, ud, d, name): 657 """ 658 Return a unique key for the url 659 """ 660 # Collapse adjacent slashes 661 slash_re = re.compile(r"/+") 662 return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name] 663 664 def _lsremote(self, ud, d, search): 665 """ 666 Run git ls-remote with the specified search string 667 """ 668 # Prevent recursion e.g. in OE if SRCPV is in PV, PV is in WORKDIR, 669 # and WORKDIR is in PATH (as a result of RSS), our call to 670 # runfetchcmd() exports PATH so this function will get called again (!) 671 # In this scenario the return call of the function isn't actually 672 # important - WORKDIR isn't needed in PATH to call git ls-remote 673 # anyway. 674 if d.getVar('_BB_GIT_IN_LSREMOTE', False): 675 return '' 676 d.setVar('_BB_GIT_IN_LSREMOTE', '1') 677 try: 678 repourl = self._get_repo_url(ud) 679 cmd = "%s ls-remote %s %s" % \ 680 (ud.basecmd, shlex.quote(repourl), search) 681 if ud.proto.lower() != 'file': 682 bb.fetch2.check_network_access(d, cmd, repourl) 683 output = runfetchcmd(cmd, d, True) 684 if not output: 685 raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url) 686 finally: 687 d.delVar('_BB_GIT_IN_LSREMOTE') 688 return output 689 690 def _latest_revision(self, ud, d, name): 691 """ 692 Compute the HEAD revision for the url 693 """ 694 output = self._lsremote(ud, d, "") 695 # Tags of the form ^{} may not work, need to fallback to other form 696 if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead: 697 head = ud.unresolvedrev[name] 698 tag = ud.unresolvedrev[name] 699 else: 700 head = "refs/heads/%s" % ud.unresolvedrev[name] 701 tag = "refs/tags/%s" % ud.unresolvedrev[name] 702 for s in [head, tag + "^{}", tag]: 703 for l in output.strip().split('\n'): 704 sha1, ref = l.split() 705 if s == ref: 706 return sha1 707 raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \ 708 (ud.unresolvedrev[name], ud.host+ud.path)) 709 710 def latest_versionstring(self, ud, d): 711 """ 712 Compute the latest release name like "x.y.x" in "x.y.x+gitHASH" 713 by searching through the tags output of ls-remote, comparing 714 versions and returning the highest match. 715 """ 716 pupver = ('', '') 717 718 tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)") 719 try: 720 output = self._lsremote(ud, d, "refs/tags/*") 721 except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: 722 bb.note("Could not list remote: %s" % str(e)) 723 return pupver 724 725 verstring = "" 726 revision = "" 727 for line in output.split("\n"): 728 if not line: 729 break 730 731 tag_head = line.split("/")[-1] 732 # Ignore non-released branches 733 m = re.search(r"(alpha|beta|rc|final)+", tag_head) 734 if m: 735 continue 736 737 # search for version in the line 738 tag = tagregex.search(tag_head) 739 if tag is None: 740 continue 741 742 tag = tag.group('pver') 743 tag = tag.replace("_", ".") 744 745 if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0: 746 continue 747 748 verstring = tag 749 revision = line.split()[0] 750 pupver = (verstring, revision) 751 752 return pupver 753 754 def _build_revision(self, ud, d, name): 755 return ud.revisions[name] 756 757 def gitpkgv_revision(self, ud, d, name): 758 """ 759 Return a sortable revision number by counting commits in the history 760 Based on gitpkgv.bblass in meta-openembedded 761 """ 762 rev = self._build_revision(ud, d, name) 763 localpath = ud.localpath 764 rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev) 765 if not os.path.exists(localpath): 766 commits = None 767 else: 768 if not os.path.exists(rev_file) or not os.path.getsize(rev_file): 769 from pipes import quote 770 commits = bb.fetch2.runfetchcmd( 771 "git rev-list %s -- | wc -l" % quote(rev), 772 d, quiet=True).strip().lstrip('0') 773 if commits: 774 open(rev_file, "w").write("%d\n" % int(commits)) 775 else: 776 commits = open(rev_file, "r").readline(128).strip() 777 if commits: 778 return False, "%s+%s" % (commits, rev[:7]) 779 else: 780 return True, str(rev) 781 782 def checkstatus(self, fetch, ud, d): 783 try: 784 self._lsremote(ud, d, "") 785 return True 786 except bb.fetch2.FetchError: 787 return False 788