1import oe.path 2 3class NotFoundError(bb.BBHandledException): 4 def __init__(self, path): 5 self.path = path 6 7 def __str__(self): 8 return "Error: %s not found." % self.path 9 10class CmdError(bb.BBHandledException): 11 def __init__(self, exitstatus, output): 12 self.status = exitstatus 13 self.output = output 14 15 def __str__(self): 16 return "Command Error: exit status: %d Output:\n%s" % (self.status, self.output) 17 18 19def runcmd(args, dir = None): 20 import pipes 21 22 if dir: 23 olddir = os.path.abspath(os.curdir) 24 if not os.path.exists(dir): 25 raise NotFoundError(dir) 26 os.chdir(dir) 27 # print("cwd: %s -> %s" % (olddir, dir)) 28 29 try: 30 args = [ pipes.quote(str(arg)) for arg in args ] 31 cmd = " ".join(args) 32 # print("cmd: %s" % cmd) 33 (exitstatus, output) = oe.utils.getstatusoutput(cmd) 34 if exitstatus != 0: 35 raise CmdError(exitstatus >> 8, output) 36 return output 37 38 finally: 39 if dir: 40 os.chdir(olddir) 41 42class PatchError(Exception): 43 def __init__(self, msg): 44 self.msg = msg 45 46 def __str__(self): 47 return "Patch Error: %s" % self.msg 48 49class PatchSet(object): 50 defaults = { 51 "strippath": 1 52 } 53 54 def __init__(self, dir, d): 55 self.dir = dir 56 self.d = d 57 self.patches = [] 58 self._current = None 59 60 def current(self): 61 return self._current 62 63 def Clean(self): 64 """ 65 Clean out the patch set. Generally includes unapplying all 66 patches and wiping out all associated metadata. 67 """ 68 raise NotImplementedError() 69 70 def Import(self, patch, force): 71 if not patch.get("file"): 72 if not patch.get("remote"): 73 raise PatchError("Patch file must be specified in patch import.") 74 else: 75 patch["file"] = bb.fetch2.localpath(patch["remote"], self.d) 76 77 for param in PatchSet.defaults: 78 if not patch.get(param): 79 patch[param] = PatchSet.defaults[param] 80 81 if patch.get("remote"): 82 patch["file"] = bb.data.expand(bb.fetch2.localpath(patch["remote"], self.d), self.d) 83 84 patch["filemd5"] = bb.utils.md5_file(patch["file"]) 85 86 def Push(self, force): 87 raise NotImplementedError() 88 89 def Pop(self, force): 90 raise NotImplementedError() 91 92 def Refresh(self, remote = None, all = None): 93 raise NotImplementedError() 94 95 @staticmethod 96 def getPatchedFiles(patchfile, striplevel, srcdir=None): 97 """ 98 Read a patch file and determine which files it will modify. 99 Params: 100 patchfile: the patch file to read 101 striplevel: the strip level at which the patch is going to be applied 102 srcdir: optional path to join onto the patched file paths 103 Returns: 104 A list of tuples of file path and change mode ('A' for add, 105 'D' for delete or 'M' for modify) 106 """ 107 108 def patchedpath(patchline): 109 filepth = patchline.split()[1] 110 if filepth.endswith('/dev/null'): 111 return '/dev/null' 112 filesplit = filepth.split(os.sep) 113 if striplevel > len(filesplit): 114 bb.error('Patch %s has invalid strip level %d' % (patchfile, striplevel)) 115 return None 116 return os.sep.join(filesplit[striplevel:]) 117 118 copiedmode = False 119 filelist = [] 120 with open(patchfile) as f: 121 for line in f: 122 if line.startswith('--- '): 123 patchpth = patchedpath(line) 124 if not patchpth: 125 break 126 if copiedmode: 127 addedfile = patchpth 128 else: 129 removedfile = patchpth 130 elif line.startswith('+++ '): 131 addedfile = patchedpath(line) 132 if not addedfile: 133 break 134 elif line.startswith('*** '): 135 copiedmode = True 136 removedfile = patchedpath(line) 137 if not removedfile: 138 break 139 else: 140 removedfile = None 141 addedfile = None 142 143 if addedfile and removedfile: 144 if removedfile == '/dev/null': 145 mode = 'A' 146 elif addedfile == '/dev/null': 147 mode = 'D' 148 else: 149 mode = 'M' 150 if srcdir: 151 fullpath = os.path.abspath(os.path.join(srcdir, addedfile)) 152 else: 153 fullpath = addedfile 154 filelist.append((fullpath, mode)) 155 156 return filelist 157 158 159class PatchTree(PatchSet): 160 def __init__(self, dir, d): 161 PatchSet.__init__(self, dir, d) 162 self.patchdir = os.path.join(self.dir, 'patches') 163 self.seriespath = os.path.join(self.dir, 'patches', 'series') 164 bb.utils.mkdirhier(self.patchdir) 165 166 def _appendPatchFile(self, patch, strippath): 167 with open(self.seriespath, 'a') as f: 168 f.write(os.path.basename(patch) + "," + strippath + "\n") 169 shellcmd = ["cat", patch, ">" , self.patchdir + "/" + os.path.basename(patch)] 170 runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) 171 172 def _removePatch(self, p): 173 patch = {} 174 patch['file'] = p.split(",")[0] 175 patch['strippath'] = p.split(",")[1] 176 self._applypatch(patch, False, True) 177 178 def _removePatchFile(self, all = False): 179 if not os.path.exists(self.seriespath): 180 return 181 with open(self.seriespath, 'r+') as f: 182 patches = f.readlines() 183 if all: 184 for p in reversed(patches): 185 self._removePatch(os.path.join(self.patchdir, p.strip())) 186 patches = [] 187 else: 188 self._removePatch(os.path.join(self.patchdir, patches[-1].strip())) 189 patches.pop() 190 with open(self.seriespath, 'w') as f: 191 for p in patches: 192 f.write(p) 193 194 def Import(self, patch, force = None): 195 """""" 196 PatchSet.Import(self, patch, force) 197 198 if self._current is not None: 199 i = self._current + 1 200 else: 201 i = 0 202 self.patches.insert(i, patch) 203 204 def _applypatch(self, patch, force = False, reverse = False, run = True): 205 shellcmd = ["cat", patch['file'], "|", "patch", "-p", patch['strippath']] 206 if reverse: 207 shellcmd.append('-R') 208 209 if not run: 210 return "sh" + "-c" + " ".join(shellcmd) 211 212 if not force: 213 shellcmd.append('--dry-run') 214 215 output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) 216 217 if force: 218 return 219 220 shellcmd.pop(len(shellcmd) - 1) 221 output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) 222 223 if not reverse: 224 self._appendPatchFile(patch['file'], patch['strippath']) 225 226 return output 227 228 def Push(self, force = False, all = False, run = True): 229 bb.note("self._current is %s" % self._current) 230 bb.note("patches is %s" % self.patches) 231 if all: 232 for i in self.patches: 233 bb.note("applying patch %s" % i) 234 self._applypatch(i, force) 235 self._current = i 236 else: 237 if self._current is not None: 238 next = self._current + 1 239 else: 240 next = 0 241 242 bb.note("applying patch %s" % self.patches[next]) 243 ret = self._applypatch(self.patches[next], force) 244 245 self._current = next 246 return ret 247 248 def Pop(self, force = None, all = None): 249 if all: 250 self._removePatchFile(True) 251 self._current = None 252 else: 253 self._removePatchFile(False) 254 255 if self._current == 0: 256 self._current = None 257 258 if self._current is not None: 259 self._current = self._current - 1 260 261 def Clean(self): 262 """""" 263 self.Pop(all=True) 264 265class GitApplyTree(PatchTree): 266 patch_line_prefix = '%% original patch' 267 268 def __init__(self, dir, d): 269 PatchTree.__init__(self, dir, d) 270 271 @staticmethod 272 def extractPatchHeader(patchfile): 273 """ 274 Extract just the header lines from the top of a patch file 275 """ 276 lines = [] 277 with open(patchfile, 'r') as f: 278 for line in f.readlines(): 279 if line.startswith('Index: ') or line.startswith('diff -') or line.startswith('---'): 280 break 281 lines.append(line) 282 return lines 283 284 @staticmethod 285 def prepareCommit(patchfile): 286 """ 287 Prepare a git commit command line based on the header from a patch file 288 (typically this is useful for patches that cannot be applied with "git am" due to formatting) 289 """ 290 import tempfile 291 import re 292 author_re = re.compile('[\S ]+ <\S+@\S+\.\S+>') 293 # Process patch header and extract useful information 294 lines = GitApplyTree.extractPatchHeader(patchfile) 295 outlines = [] 296 author = None 297 date = None 298 for line in lines: 299 if line.startswith('Subject: '): 300 subject = line.split(':', 1)[1] 301 # Remove any [PATCH][oe-core] etc. 302 subject = re.sub(r'\[.+?\]\s*', '', subject) 303 outlines.insert(0, '%s\n\n' % subject.strip()) 304 continue 305 if line.startswith('From: ') or line.startswith('Author: '): 306 authorval = line.split(':', 1)[1].strip().replace('"', '') 307 # git is fussy about author formatting i.e. it must be Name <email@domain> 308 if author_re.match(authorval): 309 author = authorval 310 continue 311 if line.startswith('Date: '): 312 if date is None: 313 dateval = line.split(':', 1)[1].strip() 314 # Very crude check for date format, since git will blow up if it's not in the right 315 # format. Without e.g. a python-dateutils dependency we can't do a whole lot more 316 if len(dateval) > 12: 317 date = dateval 318 continue 319 if line.startswith('Signed-off-by: '): 320 authorval = line.split(':', 1)[1].strip().replace('"', '') 321 # git is fussy about author formatting i.e. it must be Name <email@domain> 322 if author_re.match(authorval): 323 author = authorval 324 outlines.append(line) 325 # Write out commit message to a file 326 with tempfile.NamedTemporaryFile('w', delete=False) as tf: 327 tmpfile = tf.name 328 for line in outlines: 329 tf.write(line) 330 # Prepare git command 331 cmd = ["git", "commit", "-F", tmpfile] 332 # git doesn't like plain email addresses as authors 333 if author and '<' in author: 334 cmd.append('--author="%s"' % author) 335 if date: 336 cmd.append('--date="%s"' % date) 337 return (tmpfile, cmd) 338 339 @staticmethod 340 def extractPatches(tree, startcommit, outdir, paths=None): 341 import tempfile 342 import shutil 343 tempdir = tempfile.mkdtemp(prefix='oepatch') 344 try: 345 shellcmd = ["git", "format-patch", startcommit, "-o", tempdir] 346 if paths: 347 shellcmd.append('--') 348 shellcmd.extend(paths) 349 out = runcmd(["sh", "-c", " ".join(shellcmd)], tree) 350 if out: 351 for srcfile in out.split(): 352 patchlines = [] 353 outfile = None 354 with open(srcfile, 'r') as f: 355 for line in f: 356 if line.startswith(GitApplyTree.patch_line_prefix): 357 outfile = line.split()[-1].strip() 358 continue 359 patchlines.append(line) 360 if not outfile: 361 outfile = os.path.basename(srcfile) 362 with open(os.path.join(outdir, outfile), 'w') as of: 363 for line in patchlines: 364 of.write(line) 365 finally: 366 shutil.rmtree(tempdir) 367 368 def _applypatch(self, patch, force = False, reverse = False, run = True): 369 import shutil 370 371 def _applypatchhelper(shellcmd, patch, force = False, reverse = False, run = True): 372 if reverse: 373 shellcmd.append('-R') 374 375 shellcmd.append(patch['file']) 376 377 if not run: 378 return "sh" + "-c" + " ".join(shellcmd) 379 380 return runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) 381 382 # Add hooks which add a pointer to the original patch file name in the commit message 383 reporoot = (runcmd("git rev-parse --show-toplevel".split(), self.dir) or '').strip() 384 if not reporoot: 385 raise Exception("Cannot get repository root for directory %s" % self.dir) 386 commithook = os.path.join(reporoot, '.git', 'hooks', 'commit-msg') 387 commithook_backup = commithook + '.devtool-orig' 388 applyhook = os.path.join(reporoot, '.git', 'hooks', 'applypatch-msg') 389 applyhook_backup = applyhook + '.devtool-orig' 390 if os.path.exists(commithook): 391 shutil.move(commithook, commithook_backup) 392 if os.path.exists(applyhook): 393 shutil.move(applyhook, applyhook_backup) 394 with open(commithook, 'w') as f: 395 # NOTE: the formatting here is significant; if you change it you'll also need to 396 # change other places which read it back 397 f.write('echo >> $1\n') 398 f.write('echo "%s: $PATCHFILE" >> $1\n' % GitApplyTree.patch_line_prefix) 399 os.chmod(commithook, 0755) 400 shutil.copy2(commithook, applyhook) 401 try: 402 patchfilevar = 'PATCHFILE="%s"' % os.path.basename(patch['file']) 403 try: 404 shellcmd = [patchfilevar, "git", "--work-tree=%s" % reporoot, "am", "-3", "--keep-cr", "-p%s" % patch['strippath']] 405 return _applypatchhelper(shellcmd, patch, force, reverse, run) 406 except CmdError: 407 # Need to abort the git am, or we'll still be within it at the end 408 try: 409 shellcmd = ["git", "--work-tree=%s" % reporoot, "am", "--abort"] 410 runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) 411 except CmdError: 412 pass 413 # git am won't always clean up after itself, sadly, so... 414 shellcmd = ["git", "--work-tree=%s" % reporoot, "reset", "--hard", "HEAD"] 415 runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) 416 # Also need to take care of any stray untracked files 417 shellcmd = ["git", "--work-tree=%s" % reporoot, "clean", "-f"] 418 runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) 419 420 # Fall back to git apply 421 shellcmd = ["git", "--git-dir=%s" % reporoot, "apply", "-p%s" % patch['strippath']] 422 try: 423 output = _applypatchhelper(shellcmd, patch, force, reverse, run) 424 except CmdError: 425 # Fall back to patch 426 output = PatchTree._applypatch(self, patch, force, reverse, run) 427 # Add all files 428 shellcmd = ["git", "add", "-f", "-A", "."] 429 output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) 430 # Exclude the patches directory 431 shellcmd = ["git", "reset", "HEAD", self.patchdir] 432 output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) 433 # Commit the result 434 (tmpfile, shellcmd) = self.prepareCommit(patch['file']) 435 try: 436 shellcmd.insert(0, patchfilevar) 437 output += runcmd(["sh", "-c", " ".join(shellcmd)], self.dir) 438 finally: 439 os.remove(tmpfile) 440 return output 441 finally: 442 os.remove(commithook) 443 os.remove(applyhook) 444 if os.path.exists(commithook_backup): 445 shutil.move(commithook_backup, commithook) 446 if os.path.exists(applyhook_backup): 447 shutil.move(applyhook_backup, applyhook) 448 449 450class QuiltTree(PatchSet): 451 def _runcmd(self, args, run = True): 452 quiltrc = self.d.getVar('QUILTRCFILE', True) 453 if not run: 454 return ["quilt"] + ["--quiltrc"] + [quiltrc] + args 455 runcmd(["quilt"] + ["--quiltrc"] + [quiltrc] + args, self.dir) 456 457 def _quiltpatchpath(self, file): 458 return os.path.join(self.dir, "patches", os.path.basename(file)) 459 460 461 def __init__(self, dir, d): 462 PatchSet.__init__(self, dir, d) 463 self.initialized = False 464 p = os.path.join(self.dir, 'patches') 465 if not os.path.exists(p): 466 os.makedirs(p) 467 468 def Clean(self): 469 try: 470 self._runcmd(["pop", "-a", "-f"]) 471 oe.path.remove(os.path.join(self.dir, "patches","series")) 472 except Exception: 473 pass 474 self.initialized = True 475 476 def InitFromDir(self): 477 # read series -> self.patches 478 seriespath = os.path.join(self.dir, 'patches', 'series') 479 if not os.path.exists(self.dir): 480 raise NotFoundError(self.dir) 481 if os.path.exists(seriespath): 482 with open(seriespath, 'r') as f: 483 for line in f.readlines(): 484 patch = {} 485 parts = line.strip().split() 486 patch["quiltfile"] = self._quiltpatchpath(parts[0]) 487 patch["quiltfilemd5"] = bb.utils.md5_file(patch["quiltfile"]) 488 if len(parts) > 1: 489 patch["strippath"] = parts[1][2:] 490 self.patches.append(patch) 491 492 # determine which patches are applied -> self._current 493 try: 494 output = runcmd(["quilt", "applied"], self.dir) 495 except CmdError: 496 import sys 497 if sys.exc_value.output.strip() == "No patches applied": 498 return 499 else: 500 raise 501 output = [val for val in output.split('\n') if not val.startswith('#')] 502 for patch in self.patches: 503 if os.path.basename(patch["quiltfile"]) == output[-1]: 504 self._current = self.patches.index(patch) 505 self.initialized = True 506 507 def Import(self, patch, force = None): 508 if not self.initialized: 509 self.InitFromDir() 510 PatchSet.Import(self, patch, force) 511 oe.path.symlink(patch["file"], self._quiltpatchpath(patch["file"]), force=True) 512 with open(os.path.join(self.dir, "patches", "series"), "a") as f: 513 f.write(os.path.basename(patch["file"]) + " -p" + patch["strippath"] + "\n") 514 patch["quiltfile"] = self._quiltpatchpath(patch["file"]) 515 patch["quiltfilemd5"] = bb.utils.md5_file(patch["quiltfile"]) 516 517 # TODO: determine if the file being imported: 518 # 1) is already imported, and is the same 519 # 2) is already imported, but differs 520 521 self.patches.insert(self._current or 0, patch) 522 523 524 def Push(self, force = False, all = False, run = True): 525 # quilt push [-f] 526 527 args = ["push"] 528 if force: 529 args.append("-f") 530 if all: 531 args.append("-a") 532 if not run: 533 return self._runcmd(args, run) 534 535 self._runcmd(args) 536 537 if self._current is not None: 538 self._current = self._current + 1 539 else: 540 self._current = 0 541 542 def Pop(self, force = None, all = None): 543 # quilt pop [-f] 544 args = ["pop"] 545 if force: 546 args.append("-f") 547 if all: 548 args.append("-a") 549 550 self._runcmd(args) 551 552 if self._current == 0: 553 self._current = None 554 555 if self._current is not None: 556 self._current = self._current - 1 557 558 def Refresh(self, **kwargs): 559 if kwargs.get("remote"): 560 patch = self.patches[kwargs["patch"]] 561 if not patch: 562 raise PatchError("No patch found at index %s in patchset." % kwargs["patch"]) 563 (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(patch["remote"]) 564 if type == "file": 565 import shutil 566 if not patch.get("file") and patch.get("remote"): 567 patch["file"] = bb.fetch2.localpath(patch["remote"], self.d) 568 569 shutil.copyfile(patch["quiltfile"], patch["file"]) 570 else: 571 raise PatchError("Unable to do a remote refresh of %s, unsupported remote url scheme %s." % (os.path.basename(patch["quiltfile"]), type)) 572 else: 573 # quilt refresh 574 args = ["refresh"] 575 if kwargs.get("quiltfile"): 576 args.append(os.path.basename(kwargs["quiltfile"])) 577 elif kwargs.get("patch"): 578 args.append(os.path.basename(self.patches[kwargs["patch"]]["quiltfile"])) 579 self._runcmd(args) 580 581class Resolver(object): 582 def __init__(self, patchset, terminal): 583 raise NotImplementedError() 584 585 def Resolve(self): 586 raise NotImplementedError() 587 588 def Revert(self): 589 raise NotImplementedError() 590 591 def Finalize(self): 592 raise NotImplementedError() 593 594class NOOPResolver(Resolver): 595 def __init__(self, patchset, terminal): 596 self.patchset = patchset 597 self.terminal = terminal 598 599 def Resolve(self): 600 olddir = os.path.abspath(os.curdir) 601 os.chdir(self.patchset.dir) 602 try: 603 self.patchset.Push() 604 except Exception: 605 import sys 606 os.chdir(olddir) 607 raise 608 609# Patch resolver which relies on the user doing all the work involved in the 610# resolution, with the exception of refreshing the remote copy of the patch 611# files (the urls). 612class UserResolver(Resolver): 613 def __init__(self, patchset, terminal): 614 self.patchset = patchset 615 self.terminal = terminal 616 617 # Force a push in the patchset, then drop to a shell for the user to 618 # resolve any rejected hunks 619 def Resolve(self): 620 olddir = os.path.abspath(os.curdir) 621 os.chdir(self.patchset.dir) 622 try: 623 self.patchset.Push(False) 624 except CmdError as v: 625 # Patch application failed 626 patchcmd = self.patchset.Push(True, False, False) 627 628 t = self.patchset.d.getVar('T', True) 629 if not t: 630 bb.msg.fatal("Build", "T not set") 631 bb.utils.mkdirhier(t) 632 import random 633 rcfile = "%s/bashrc.%s.%s" % (t, str(os.getpid()), random.random()) 634 with open(rcfile, "w") as f: 635 f.write("echo '*** Manual patch resolution mode ***'\n") 636 f.write("echo 'Dropping to a shell, so patch rejects can be fixed manually.'\n") 637 f.write("echo 'Run \"quilt refresh\" when patch is corrected, press CTRL+D to exit.'\n") 638 f.write("echo ''\n") 639 f.write(" ".join(patchcmd) + "\n") 640 os.chmod(rcfile, 0775) 641 642 self.terminal("bash --rcfile " + rcfile, 'Patch Rejects: Please fix patch rejects manually', self.patchset.d) 643 644 # Construct a new PatchSet after the user's changes, compare the 645 # sets, checking patches for modifications, and doing a remote 646 # refresh on each. 647 oldpatchset = self.patchset 648 self.patchset = oldpatchset.__class__(self.patchset.dir, self.patchset.d) 649 650 for patch in self.patchset.patches: 651 oldpatch = None 652 for opatch in oldpatchset.patches: 653 if opatch["quiltfile"] == patch["quiltfile"]: 654 oldpatch = opatch 655 656 if oldpatch: 657 patch["remote"] = oldpatch["remote"] 658 if patch["quiltfile"] == oldpatch["quiltfile"]: 659 if patch["quiltfilemd5"] != oldpatch["quiltfilemd5"]: 660 bb.note("Patch %s has changed, updating remote url %s" % (os.path.basename(patch["quiltfile"]), patch["remote"])) 661 # user change? remote refresh 662 self.patchset.Refresh(remote=True, patch=self.patchset.patches.index(patch)) 663 else: 664 # User did not fix the problem. Abort. 665 raise PatchError("Patch application failed, and user did not fix and refresh the patch.") 666 except Exception: 667 os.chdir(olddir) 668 raise 669 os.chdir(olddir) 670