1# 2# BitBake Tests for the Fetcher (fetch2/) 3# 4# Copyright (C) 2012 Richard Purdie 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9import contextlib 10import shutil 11import unittest 12import unittest.mock 13import urllib.parse 14import hashlib 15import tempfile 16import collections 17import os 18import signal 19import tarfile 20from bb.fetch2 import URI 21from bb.fetch2 import FetchMethod 22import bb 23import bb.utils 24from bb.tests.support.httpserver import HTTPService 25 26def skipIfNoNetwork(): 27 if os.environ.get("BB_SKIP_NETTESTS") == "yes": 28 return unittest.skip("network test") 29 return lambda f: f 30 31 32@contextlib.contextmanager 33def hide_directory(directory): 34 """Hide the given directory and restore it after the context is left""" 35 temp_name = directory + ".bak" 36 os.rename(directory, temp_name) 37 try: 38 yield 39 finally: 40 os.rename(temp_name, directory) 41 42 43class TestTimeout(Exception): 44 # Indicate to pytest that this is not a test suite 45 __test__ = False 46 47class Timeout(): 48 49 def __init__(self, seconds): 50 self.seconds = seconds 51 52 def handle_timeout(self, signum, frame): 53 raise TestTimeout("Test failed: timeout reached") 54 55 def __enter__(self): 56 signal.signal(signal.SIGALRM, self.handle_timeout) 57 signal.alarm(self.seconds) 58 59 def __exit__(self, exc_type, exc_val, exc_tb): 60 signal.alarm(0) 61 62class URITest(unittest.TestCase): 63 test_uris = { 64 "http://www.google.com/index.html" : { 65 'uri': 'http://www.google.com/index.html', 66 'scheme': 'http', 67 'hostname': 'www.google.com', 68 'port': None, 69 'hostport': 'www.google.com', 70 'path': '/index.html', 71 'userinfo': '', 72 'username': '', 73 'password': '', 74 'params': {}, 75 'query': {}, 76 'relative': False 77 }, 78 "http://www.google.com/index.html;param1=value1" : { 79 'uri': 'http://www.google.com/index.html;param1=value1', 80 'scheme': 'http', 81 'hostname': 'www.google.com', 82 'port': None, 83 'hostport': 'www.google.com', 84 'path': '/index.html', 85 'userinfo': '', 86 'username': '', 87 'password': '', 88 'params': { 89 'param1': 'value1' 90 }, 91 'query': {}, 92 'relative': False 93 }, 94 "http://www.example.org/index.html?param1=value1" : { 95 'uri': 'http://www.example.org/index.html?param1=value1', 96 'scheme': 'http', 97 'hostname': 'www.example.org', 98 'port': None, 99 'hostport': 'www.example.org', 100 'path': '/index.html', 101 'userinfo': '', 102 'username': '', 103 'password': '', 104 'params': {}, 105 'query': { 106 'param1': 'value1' 107 }, 108 'relative': False 109 }, 110 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : { 111 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 112 'scheme': 'http', 113 'hostname': 'www.example.org', 114 'port': None, 115 'hostport': 'www.example.org', 116 'path': '/index.html', 117 'userinfo': '', 118 'username': '', 119 'password': '', 120 'params': { 121 'param2': 'value2' 122 }, 123 'query': { 124 'qparam1': 'qvalue1' 125 }, 126 'relative': False 127 }, 128 # Check that trailing semicolons are handled correctly 129 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2;" : { 130 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 131 'scheme': 'http', 132 'hostname': 'www.example.org', 133 'port': None, 134 'hostport': 'www.example.org', 135 'path': '/index.html', 136 'userinfo': '', 137 'username': '', 138 'password': '', 139 'params': { 140 'param2': 'value2' 141 }, 142 'query': { 143 'qparam1': 'qvalue1' 144 }, 145 'relative': False 146 }, 147 "http://www.example.com:8080/index.html" : { 148 'uri': 'http://www.example.com:8080/index.html', 149 'scheme': 'http', 150 'hostname': 'www.example.com', 151 'port': 8080, 152 'hostport': 'www.example.com:8080', 153 'path': '/index.html', 154 'userinfo': '', 155 'username': '', 156 'password': '', 157 'params': {}, 158 'query': {}, 159 'relative': False 160 }, 161 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : { 162 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg', 163 'scheme': 'cvs', 164 'hostname': 'cvs.handhelds.org', 165 'port': None, 166 'hostport': 'cvs.handhelds.org', 167 'path': '/cvs', 168 'userinfo': 'anoncvs', 169 'username': 'anoncvs', 170 'password': '', 171 'params': { 172 'module': 'familiar/dist/ipkg' 173 }, 174 'query': {}, 175 'relative': False 176 }, 177 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": { 178 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg', 179 'scheme': 'cvs', 180 'hostname': 'cvs.handhelds.org', 181 'port': None, 182 'hostport': 'cvs.handhelds.org', 183 'path': '/cvs', 184 'userinfo': 'anoncvs:anonymous', 185 'username': 'anoncvs', 186 'password': 'anonymous', 187 'params': collections.OrderedDict([ 188 ('tag', 'V0-99-81'), 189 ('module', 'familiar/dist/ipkg') 190 ]), 191 'query': {}, 192 'relative': False 193 }, 194 "file://example.diff": { # NOTE: Not RFC compliant! 195 'uri': 'file:example.diff', 196 'scheme': 'file', 197 'hostname': '', 198 'port': None, 199 'hostport': '', 200 'path': 'example.diff', 201 'userinfo': '', 202 'username': '', 203 'password': '', 204 'params': {}, 205 'query': {}, 206 'relative': True 207 }, 208 "file:example.diff": { # NOTE: RFC compliant version of the former 209 'uri': 'file:example.diff', 210 'scheme': 'file', 211 'hostname': '', 212 'port': None, 213 'hostport': '', 214 'path': 'example.diff', 215 'userinfo': '', 216 'userinfo': '', 217 'username': '', 218 'password': '', 219 'params': {}, 220 'query': {}, 221 'relative': True 222 }, 223 "file:///tmp/example.diff": { 224 'uri': 'file:///tmp/example.diff', 225 'scheme': 'file', 226 'hostname': '', 227 'port': None, 228 'hostport': '', 229 'path': '/tmp/example.diff', 230 'userinfo': '', 231 'userinfo': '', 232 'username': '', 233 'password': '', 234 'params': {}, 235 'query': {}, 236 'relative': False 237 }, 238 "git:///path/example.git": { 239 'uri': 'git:///path/example.git', 240 'scheme': 'git', 241 'hostname': '', 242 'port': None, 243 'hostport': '', 244 'path': '/path/example.git', 245 'userinfo': '', 246 'userinfo': '', 247 'username': '', 248 'password': '', 249 'params': {}, 250 'query': {}, 251 'relative': False 252 }, 253 "git:path/example.git": { 254 'uri': 'git:path/example.git', 255 'scheme': 'git', 256 'hostname': '', 257 'port': None, 258 'hostport': '', 259 'path': 'path/example.git', 260 'userinfo': '', 261 'userinfo': '', 262 'username': '', 263 'password': '', 264 'params': {}, 265 'query': {}, 266 'relative': True 267 }, 268 "git://example.net/path/example.git": { 269 'uri': 'git://example.net/path/example.git', 270 'scheme': 'git', 271 'hostname': 'example.net', 272 'port': None, 273 'hostport': 'example.net', 274 'path': '/path/example.git', 275 'userinfo': '', 276 'userinfo': '', 277 'username': '', 278 'password': '', 279 'params': {}, 280 'query': {}, 281 'relative': False 282 }, 283 "git://tfs-example.org:22/tfs/example%20path/example.git": { 284 'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git', 285 'scheme': 'git', 286 'hostname': 'tfs-example.org', 287 'port': 22, 288 'hostport': 'tfs-example.org:22', 289 'path': '/tfs/example path/example.git', 290 'userinfo': '', 291 'userinfo': '', 292 'username': '', 293 'password': '', 294 'params': {}, 295 'query': {}, 296 'relative': False 297 }, 298 "http://somesite.net;someparam=1": { 299 'uri': 'http://somesite.net;someparam=1', 300 'scheme': 'http', 301 'hostname': 'somesite.net', 302 'port': None, 303 'hostport': 'somesite.net', 304 'path': '', 305 'userinfo': '', 306 'userinfo': '', 307 'username': '', 308 'password': '', 309 'params': {"someparam" : "1"}, 310 'query': {}, 311 'relative': False 312 }, 313 "file://somelocation;someparam=1": { 314 'uri': 'file:somelocation;someparam=1', 315 'scheme': 'file', 316 'hostname': '', 317 'port': None, 318 'hostport': '', 319 'path': 'somelocation', 320 'userinfo': '', 321 'userinfo': '', 322 'username': '', 323 'password': '', 324 'params': {"someparam" : "1"}, 325 'query': {}, 326 'relative': True 327 }, 328 "https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip": { 329 'uri': 'https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip', 330 'scheme': 'https', 331 'hostname': 'www.innodisk.com', 332 'port': None, 333 'hostport': 'www.innodisk.com', 334 'path': '/Download_file', 335 'userinfo': '', 336 'userinfo': '', 337 'username': '', 338 'password': '', 339 'params': {"downloadfilename" : "EGPL-T101.zip"}, 340 'query': {"9BE0BF6657": None}, 341 'relative': False 342 }, 343 "file://example@.service": { 344 'uri': 'file:example%40.service', 345 'scheme': 'file', 346 'hostname': '', 347 'port': None, 348 'hostport': '', 349 'path': 'example@.service', 350 'userinfo': '', 351 'userinfo': '', 352 'username': '', 353 'password': '', 354 'params': {}, 355 'query': {}, 356 'relative': True 357 } 358 359 } 360 361 def test_uri(self): 362 for test_uri, ref in self.test_uris.items(): 363 uri = URI(test_uri) 364 365 self.assertEqual(str(uri), ref['uri']) 366 367 # expected attributes 368 self.assertEqual(uri.scheme, ref['scheme']) 369 370 self.assertEqual(uri.userinfo, ref['userinfo']) 371 self.assertEqual(uri.username, ref['username']) 372 self.assertEqual(uri.password, ref['password']) 373 374 self.assertEqual(uri.hostname, ref['hostname']) 375 self.assertEqual(uri.port, ref['port']) 376 self.assertEqual(uri.hostport, ref['hostport']) 377 378 self.assertEqual(uri.path, ref['path']) 379 self.assertEqual(uri.params, ref['params']) 380 381 self.assertEqual(uri.relative, ref['relative']) 382 383 def test_dict(self): 384 for test in self.test_uris.values(): 385 uri = URI() 386 387 self.assertEqual(uri.scheme, '') 388 self.assertEqual(uri.userinfo, '') 389 self.assertEqual(uri.username, '') 390 self.assertEqual(uri.password, '') 391 self.assertEqual(uri.hostname, '') 392 self.assertEqual(uri.port, None) 393 self.assertEqual(uri.path, '') 394 self.assertEqual(uri.params, {}) 395 396 397 uri.scheme = test['scheme'] 398 self.assertEqual(uri.scheme, test['scheme']) 399 400 uri.userinfo = test['userinfo'] 401 self.assertEqual(uri.userinfo, test['userinfo']) 402 self.assertEqual(uri.username, test['username']) 403 self.assertEqual(uri.password, test['password']) 404 405 # make sure changing the values doesn't do anything unexpected 406 uri.username = 'changeme' 407 self.assertEqual(uri.username, 'changeme') 408 self.assertEqual(uri.password, test['password']) 409 uri.password = 'insecure' 410 self.assertEqual(uri.username, 'changeme') 411 self.assertEqual(uri.password, 'insecure') 412 413 # reset back after our trickery 414 uri.userinfo = test['userinfo'] 415 self.assertEqual(uri.userinfo, test['userinfo']) 416 self.assertEqual(uri.username, test['username']) 417 self.assertEqual(uri.password, test['password']) 418 419 uri.hostname = test['hostname'] 420 self.assertEqual(uri.hostname, test['hostname']) 421 self.assertEqual(uri.hostport, test['hostname']) 422 423 uri.port = test['port'] 424 self.assertEqual(uri.port, test['port']) 425 self.assertEqual(uri.hostport, test['hostport']) 426 427 uri.path = test['path'] 428 self.assertEqual(uri.path, test['path']) 429 430 uri.params = test['params'] 431 self.assertEqual(uri.params, test['params']) 432 433 uri.query = test['query'] 434 self.assertEqual(uri.query, test['query']) 435 436 self.assertEqual(str(uri), test['uri']) 437 438 uri.params = {} 439 self.assertEqual(uri.params, {}) 440 self.assertEqual(str(uri), (str(uri).split(";"))[0]) 441 442class FetcherTest(unittest.TestCase): 443 444 def setUp(self): 445 self.origdir = os.getcwd() 446 self.d = bb.data.init() 447 self.tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") 448 self.dldir = os.path.join(self.tempdir, "download") 449 os.mkdir(self.dldir) 450 self.d.setVar("DL_DIR", self.dldir) 451 self.unpackdir = os.path.join(self.tempdir, "unpacked") 452 os.mkdir(self.unpackdir) 453 persistdir = os.path.join(self.tempdir, "persistdata") 454 self.d.setVar("PERSISTENT_DIR", persistdir) 455 456 def tearDown(self): 457 os.chdir(self.origdir) 458 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 459 print("Not cleaning up %s. Please remove manually." % self.tempdir) 460 else: 461 bb.process.run('chmod u+rw -R %s' % self.tempdir) 462 bb.utils.prunedir(self.tempdir) 463 464 def git(self, cmd, cwd=None): 465 if isinstance(cmd, str): 466 cmd = 'git -c safe.bareRepository=all ' + cmd 467 else: 468 cmd = ['git', '-c', 'safe.bareRepository=all'] + cmd 469 if cwd is None: 470 cwd = self.gitdir 471 return bb.process.run(cmd, cwd=cwd)[0] 472 473 def git_init(self, cwd=None): 474 self.git('init', cwd=cwd) 475 # Explicitly set initial branch to master as 476 # a common setup is to use other default 477 # branch than master. 478 self.git(['checkout', '-b', 'master'], cwd=cwd) 479 480 try: 481 self.git(['config', 'user.email'], cwd=cwd) 482 except bb.process.ExecutionError: 483 self.git(['config', 'user.email', 'you@example.com'], cwd=cwd) 484 485 try: 486 self.git(['config', 'user.name'], cwd=cwd) 487 except bb.process.ExecutionError: 488 self.git(['config', 'user.name', 'Your Name'], cwd=cwd) 489 490class MirrorUriTest(FetcherTest): 491 492 replaceuris = { 493 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/.*", "http://somewhere.org/somedir/") 494 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz", 495 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 496 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 497 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 498 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 499 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") 500 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 501 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake") 502 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", 503 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") 504 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 505 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") 506 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 507 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") 508 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 509 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") 510 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 511 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist") 512 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", 513 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/") 514 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2", 515 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 516 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 517 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 518 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 519 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") 520 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 521 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org") 522 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 523 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/") 524 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 525 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http") 526 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 527 ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http") 528 : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 529 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=git;branch=master", "git://someserver.org/bitbake", "git://someotherserver.org/bitbake;protocol=https") 530 : "git://someotherserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=https;branch=master", 531 ("gitsm://git.qemu.org/git/seabios.git/;protocol=https;name=roms/seabios;subpath=roms/seabios;bareclone=1;nobranch=1;rev=1234567890123456789012345678901234567890", "gitsm://.*/.*", "http://petalinux.xilinx.com/sswreleases/rel-v${XILINX_VER_MAIN}/downloads") : "http://petalinux.xilinx.com/sswreleases/rel-v%24%7BXILINX_VER_MAIN%7D/downloads/git2_git.qemu.org.git.seabios.git..tar.gz", 532 ("https://somewhere.org/example/1.0.0/example;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/PATH") 533 : "file:///mirror/example/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 534 ("https://somewhere.org/example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/some-example-1.0.0.tgz") 535 : "file:///mirror/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 536 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", r"git://(?!internal\.git\.server).*/.*", "http://somewhere.org/somedir/") 537 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz", 538 ("git://internal.git.server.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", r"git://(?!internal\.git\.server).*/.*", "http://somewhere.org/somedir/") 539 : None, 540 541 #Renaming files doesn't work 542 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz" 543 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 544 } 545 546 mirrorvar = "http://.*/.* file:///somepath/downloads/ " \ 547 "git://someserver.org/bitbake git://git.openembedded.org/bitbake " \ 548 "https?://.*/.* file:///someotherpath/downloads/ " \ 549 "svn://svn.server1.com/ svn://svn.server2.com/" 550 551 def test_urireplace(self): 552 self.d.setVar("FILESPATH", ".") 553 for k, v in self.replaceuris.items(): 554 ud = bb.fetch.FetchData(k[0], self.d) 555 ud.setup_localpath(self.d) 556 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2])) 557 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d) 558 self.assertEqual([v] if v else [], newuris) 559 560 def test_urilist1(self): 561 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 562 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 563 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 564 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 565 566 def test_urilist2(self): 567 # Catch https:// -> files:// bug 568 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 569 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 570 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 571 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 572 573 def test_urilistsvn(self): 574 # Catch svn:// -> svn:// bug 575 fetcher = bb.fetch.FetchData("svn://svn.server1.com/isource/svnroot/reponame/tags/tagname;module=path_in_tagnamefolder;protocol=https;rev=2", self.d) 576 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 577 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 578 self.assertEqual(uris, ['svn://svn.server2.com/isource/svnroot/reponame/tags/tagname;module=path_in_tagnamefolder;protocol=https;rev=2']) 579 580 def test_mirror_of_mirror(self): 581 # Test if mirror of a mirror works 582 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/" 583 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/" 584 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 585 mirrors = bb.fetch2.mirror_from_string(mirrorvar) 586 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 587 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 588 'file:///someotherpath/downloads/bitbake-1.0.tar.gz', 589 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', 590 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) 591 592 recmirrorvar = "https://.*/[^/]* http://aaaa/A/A/A/ " \ 593 "https://.*/[^/]* https://bbbb/B/B/B/" 594 595 def test_recursive(self): 596 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 597 mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar) 598 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 599 self.assertEqual(uris, ['http://aaaa/A/A/A/bitbake/bitbake-1.0.tar.gz', 600 'https://bbbb/B/B/B/bitbake/bitbake-1.0.tar.gz', 601 'http://aaaa/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz']) 602 603 604class GitDownloadDirectoryNamingTest(FetcherTest): 605 def setUp(self): 606 super(GitDownloadDirectoryNamingTest, self).setUp() 607 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 608 self.recipe_dir = "git.openembedded.org.bitbake" 609 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 610 self.mirror_dir = "github.com.openembedded.bitbake.git" 611 612 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 613 614 def setup_mirror_rewrite(self): 615 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 616 617 @skipIfNoNetwork() 618 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self): 619 self.setup_mirror_rewrite() 620 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 621 622 fetcher.download() 623 624 dir = os.listdir(self.dldir + "/git2") 625 self.assertIn(self.recipe_dir, dir) 626 627 @skipIfNoNetwork() 628 def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self): 629 self.setup_mirror_rewrite() 630 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 631 632 fetcher.download() 633 634 dir = os.listdir(self.dldir + "/git2") 635 self.assertIn(self.mirror_dir, dir) 636 self.assertIn(self.recipe_dir, dir) 637 638 @skipIfNoNetwork() 639 def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self): 640 self.setup_mirror_rewrite() 641 fetcher = bb.fetch.Fetch([self.mirror_url], self.d) 642 fetcher.download() 643 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 644 645 fetcher.download() 646 647 dir = os.listdir(self.dldir + "/git2") 648 self.assertIn(self.mirror_dir, dir) 649 self.assertIn(self.recipe_dir, dir) 650 651 652class TarballNamingTest(FetcherTest): 653 def setUp(self): 654 super(TarballNamingTest, self).setUp() 655 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 656 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 657 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 658 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz" 659 660 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 661 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 662 663 def setup_mirror_rewrite(self): 664 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 665 666 @skipIfNoNetwork() 667 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self): 668 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 669 670 fetcher.download() 671 672 dir = os.listdir(self.dldir) 673 self.assertIn(self.recipe_tarball, dir) 674 675 @skipIfNoNetwork() 676 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 677 self.setup_mirror_rewrite() 678 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 679 680 fetcher.download() 681 682 dir = os.listdir(self.dldir) 683 self.assertIn(self.mirror_tarball, dir) 684 685 686class GitShallowTarballNamingTest(FetcherTest): 687 def setUp(self): 688 super(GitShallowTarballNamingTest, self).setUp() 689 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 690 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz" 691 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 692 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz" 693 694 self.d.setVar('BB_GIT_SHALLOW', '1') 695 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 696 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 697 698 def setup_mirror_rewrite(self): 699 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 700 701 @skipIfNoNetwork() 702 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self): 703 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 704 705 fetcher.download() 706 707 dir = os.listdir(self.dldir) 708 self.assertIn(self.recipe_tarball, dir) 709 710 @skipIfNoNetwork() 711 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 712 self.setup_mirror_rewrite() 713 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 714 715 fetcher.download() 716 717 dir = os.listdir(self.dldir) 718 self.assertIn(self.mirror_tarball, dir) 719 720 721class CleanTarballTest(FetcherTest): 722 def setUp(self): 723 super(CleanTarballTest, self).setUp() 724 self.recipe_url = "git://git.openembedded.org/bitbake;protocol=https;branch=master" 725 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 726 727 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 728 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 729 730 @skipIfNoNetwork() 731 def test_that_the_tarball_contents_does_not_leak_info(self): 732 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 733 734 fetcher.download() 735 736 fetcher.unpack(self.unpackdir) 737 mtime = bb.process.run('git log --all -1 --format=%ct', 738 cwd=os.path.join(self.unpackdir, 'git')) 739 self.assertEqual(len(mtime), 2) 740 mtime = int(mtime[0]) 741 742 archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball)) 743 self.assertNotEqual(len(archive.members), 0) 744 for member in archive.members: 745 if member.name == ".": 746 continue 747 self.assertEqual(member.uname, 'oe', "user name for %s differs" % member.name) 748 self.assertEqual(member.uid, 0, "uid for %s differs" % member.name) 749 self.assertEqual(member.gname, 'oe', "group name for %s differs" % member.name) 750 self.assertEqual(member.gid, 0, "gid for %s differs" % member.name) 751 self.assertEqual(member.mtime, mtime, "mtime for %s differs" % member.name) 752 753 754class FetcherLocalTest(FetcherTest): 755 def setUp(self): 756 def touch(fn): 757 with open(fn, 'a'): 758 os.utime(fn, None) 759 760 super(FetcherLocalTest, self).setUp() 761 self.localsrcdir = os.path.join(self.tempdir, 'localsrc') 762 os.makedirs(self.localsrcdir) 763 touch(os.path.join(self.localsrcdir, 'a')) 764 touch(os.path.join(self.localsrcdir, 'b')) 765 touch(os.path.join(self.localsrcdir, 'c@d')) 766 os.makedirs(os.path.join(self.localsrcdir, 'dir')) 767 touch(os.path.join(self.localsrcdir, 'dir', 'c')) 768 touch(os.path.join(self.localsrcdir, 'dir', 'd')) 769 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir')) 770 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e')) 771 touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device')) 772 bb.process.run('tar cf archive.tar -C dir .', cwd=self.localsrcdir) 773 bb.process.run('tar czf archive.tar.gz -C dir .', cwd=self.localsrcdir) 774 bb.process.run('tar cjf archive.tar.bz2 -C dir .', cwd=self.localsrcdir) 775 self.d.setVar("FILESPATH", self.localsrcdir) 776 777 def fetchUnpack(self, uris): 778 fetcher = bb.fetch.Fetch(uris, self.d) 779 fetcher.download() 780 fetcher.unpack(self.unpackdir) 781 flst = [] 782 for root, dirs, files in os.walk(self.unpackdir): 783 for f in files: 784 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir)) 785 flst.sort() 786 return flst 787 788 def test_local_checksum_fails_no_file(self): 789 self.d.setVar("SRC_URI", "file://404") 790 with self.assertRaises(bb.BBHandledException): 791 bb.fetch.get_checksum_file_list(self.d) 792 793 def test_local(self): 794 tree = self.fetchUnpack(['file://a', 'file://dir/c']) 795 self.assertEqual(tree, ['a', 'dir/c']) 796 797 def test_local_at(self): 798 tree = self.fetchUnpack(['file://c@d']) 799 self.assertEqual(tree, ['c@d']) 800 801 def test_local_backslash(self): 802 tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device']) 803 self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device']) 804 805 def test_local_wildcard(self): 806 with self.assertRaises(bb.fetch2.ParameterError): 807 tree = self.fetchUnpack(['file://a', 'file://dir/*']) 808 809 def test_local_dir(self): 810 tree = self.fetchUnpack(['file://a', 'file://dir']) 811 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e']) 812 813 def test_local_subdir(self): 814 tree = self.fetchUnpack(['file://dir/subdir']) 815 self.assertEqual(tree, ['dir/subdir/e']) 816 817 def test_local_subdir_file(self): 818 tree = self.fetchUnpack(['file://dir/subdir/e']) 819 self.assertEqual(tree, ['dir/subdir/e']) 820 821 def test_local_subdirparam(self): 822 tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo']) 823 self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e']) 824 825 def test_local_deepsubdirparam(self): 826 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar']) 827 self.assertEqual(tree, ['bar/dir/subdir/e']) 828 829 def test_local_absolutedir(self): 830 # Unpacking to an absolute path that is a subdirectory of the root 831 # should work 832 tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')]) 833 834 # Unpacking to an absolute path outside of the root should fail 835 with self.assertRaises(bb.fetch2.UnpackError): 836 self.fetchUnpack(['file://a;subdir=/bin/sh']) 837 838 def test_local_striplevel(self): 839 tree = self.fetchUnpack(['file://archive.tar;subdir=bar;striplevel=1']) 840 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 841 842 def test_local_striplevel_gzip(self): 843 tree = self.fetchUnpack(['file://archive.tar.gz;subdir=bar;striplevel=1']) 844 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 845 846 def test_local_striplevel_bzip2(self): 847 tree = self.fetchUnpack(['file://archive.tar.bz2;subdir=bar;striplevel=1']) 848 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 849 850 def dummyGitTest(self, suffix): 851 # Create dummy local Git repo 852 src_dir = tempfile.mkdtemp(dir=self.tempdir, 853 prefix='gitfetch_localusehead_') 854 self.gitdir = os.path.abspath(src_dir) 855 self.git_init() 856 self.git(['commit', '--allow-empty', '-m', 'Dummy commit']) 857 # Use other branch than master 858 self.git(['checkout', '-b', 'my-devel']) 859 self.git(['commit', '--allow-empty', '-m', 'Dummy commit 2']) 860 orig_rev = self.git(['rev-parse', 'HEAD']).strip() 861 862 # Fetch and check revision 863 self.d.setVar("SRCREV", "AUTOINC") 864 self.d.setVar("__BBSRCREV_SEEN", "1") 865 url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix 866 fetcher = bb.fetch.Fetch([url], self.d) 867 fetcher.download() 868 fetcher.unpack(self.unpackdir) 869 unpack_rev = self.git(['rev-parse', 'HEAD'], 870 cwd=os.path.join(self.unpackdir, 'git')).strip() 871 self.assertEqual(orig_rev, unpack_rev) 872 873 def test_local_gitfetch_usehead(self): 874 self.dummyGitTest("usehead=1") 875 876 def test_local_gitfetch_usehead_withname(self): 877 self.dummyGitTest("usehead=1;name=newName") 878 879 def test_local_gitfetch_shared(self): 880 self.dummyGitTest("usehead=1;name=sharedName") 881 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 882 self.assertTrue(os.path.exists(alt)) 883 884 def test_local_gitfetch_noshared(self): 885 self.d.setVar('BB_GIT_NOSHARED', '1') 886 self.unpackdir += '_noshared' 887 self.dummyGitTest("usehead=1;name=noSharedName") 888 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 889 self.assertFalse(os.path.exists(alt)) 890 891class FetcherNoNetworkTest(FetcherTest): 892 def setUp(self): 893 super().setUp() 894 # all test cases are based on not having network 895 self.d.setVar("BB_NO_NETWORK", "1") 896 897 def test_missing(self): 898 string = "this is a test file\n".encode("utf-8") 899 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 900 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 901 902 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 903 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 904 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 905 with self.assertRaises(bb.fetch2.NetworkAccess): 906 fetcher.download() 907 908 def test_valid_missing_donestamp(self): 909 # create the file in the download directory with correct hash 910 string = "this is a test file\n".encode("utf-8") 911 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f: 912 f.write(string) 913 914 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 915 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 916 917 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 918 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 919 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 920 fetcher.download() 921 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 922 923 def test_invalid_missing_donestamp(self): 924 # create an invalid file in the download directory with incorrect hash 925 string = "this is a test file\n".encode("utf-8") 926 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 927 pass 928 929 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 930 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 931 932 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 933 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 934 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 935 with self.assertRaises(bb.fetch2.NetworkAccess): 936 fetcher.download() 937 # the existing file should not exist or should have be moved to "bad-checksum" 938 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 939 940 def test_nochecksums_missing(self): 941 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 942 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 943 # ssh fetch does not support checksums 944 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 945 # attempts to download with missing donestamp 946 with self.assertRaises(bb.fetch2.NetworkAccess): 947 fetcher.download() 948 949 def test_nochecksums_missing_donestamp(self): 950 # create a file in the download directory 951 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 952 pass 953 954 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 955 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 956 # ssh fetch does not support checksums 957 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 958 # attempts to download with missing donestamp 959 with self.assertRaises(bb.fetch2.NetworkAccess): 960 fetcher.download() 961 962 def test_nochecksums_has_donestamp(self): 963 # create a file in the download directory with the donestamp 964 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 965 pass 966 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 967 pass 968 969 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 970 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 971 # ssh fetch does not support checksums 972 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 973 # should not fetch 974 fetcher.download() 975 # both files should still exist 976 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 977 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 978 979 def test_nochecksums_missing_has_donestamp(self): 980 # create a file in the download directory with the donestamp 981 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 982 pass 983 984 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 985 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 986 # ssh fetch does not support checksums 987 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 988 with self.assertRaises(bb.fetch2.NetworkAccess): 989 fetcher.download() 990 # both files should still exist 991 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 992 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 993 994class FetcherNetworkTest(FetcherTest): 995 @skipIfNoNetwork() 996 def test_fetch(self): 997 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 998 fetcher.download() 999 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1000 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892) 1001 self.d.setVar("BB_NO_NETWORK", "1") 1002 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 1003 fetcher.download() 1004 fetcher.unpack(self.unpackdir) 1005 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9) 1006 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9) 1007 1008 @skipIfNoNetwork() 1009 def test_fetch_mirror(self): 1010 self.d.setVar("MIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 1011 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 1012 fetcher.download() 1013 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1014 1015 @skipIfNoNetwork() 1016 def test_fetch_mirror_of_mirror(self): 1017 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake") 1018 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 1019 fetcher.download() 1020 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1021 1022 @skipIfNoNetwork() 1023 def test_fetch_file_mirror_of_mirror(self): 1024 self.d.setVar("FILESPATH", ".") 1025 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake") 1026 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 1027 os.mkdir(self.dldir + "/some2where") 1028 fetcher.download() 1029 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1030 1031 @skipIfNoNetwork() 1032 def test_fetch_premirror(self): 1033 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 1034 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 1035 fetcher.download() 1036 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1037 1038 @skipIfNoNetwork() 1039 def test_fetch_specify_downloadfilename(self): 1040 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) 1041 fetcher.download() 1042 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) 1043 1044 @skipIfNoNetwork() 1045 def test_fetch_premirror_specify_downloadfilename_regex_uri(self): 1046 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/") 1047 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1048 fetcher.download() 1049 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1050 1051 @skipIfNoNetwork() 1052 # BZ13039 1053 def test_fetch_premirror_specify_downloadfilename_specific_uri(self): 1054 self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake") 1055 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1056 fetcher.download() 1057 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1058 1059 @skipIfNoNetwork() 1060 def test_fetch_premirror_use_downloadfilename_to_fetch(self): 1061 # Ensure downloadfilename is used when fetching from premirror. 1062 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 1063 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1064 fetcher.download() 1065 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1066 1067 @skipIfNoNetwork() 1068 def gitfetcher(self, url1, url2): 1069 def checkrevision(self, fetcher): 1070 fetcher.unpack(self.unpackdir) 1071 revision = self.git(['rev-parse', 'HEAD'], 1072 cwd=os.path.join(self.unpackdir, 'git')).strip() 1073 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 1074 1075 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 1076 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 1077 fetcher = bb.fetch.Fetch([url1], self.d) 1078 fetcher.download() 1079 checkrevision(self, fetcher) 1080 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works 1081 bb.utils.prunedir(self.dldir + "/git2/") 1082 bb.utils.prunedir(self.unpackdir) 1083 self.d.setVar("BB_NO_NETWORK", "1") 1084 fetcher = bb.fetch.Fetch([url2], self.d) 1085 fetcher.download() 1086 checkrevision(self, fetcher) 1087 1088 @skipIfNoNetwork() 1089 def test_gitfetch(self): 1090 url1 = url2 = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 1091 self.gitfetcher(url1, url2) 1092 1093 @skipIfNoNetwork() 1094 def test_gitfetch_goodsrcrev(self): 1095 # SRCREV is set but matches rev= parameter 1096 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https" 1097 self.gitfetcher(url1, url2) 1098 1099 @skipIfNoNetwork() 1100 def test_gitfetch_badsrcrev(self): 1101 # SRCREV is set but does not match rev= parameter 1102 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https" 1103 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1104 1105 @skipIfNoNetwork() 1106 def test_gitfetch_usehead(self): 1107 # Since self.gitfetcher() sets SRCREV we expect this to override 1108 # `usehead=1' and instead fetch the specified SRCREV. See 1109 # test_local_gitfetch_usehead() for a positive use of the usehead 1110 # feature. 1111 url = "git://git.openembedded.org/bitbake;usehead=1;branch=master;protocol=https" 1112 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1113 1114 @skipIfNoNetwork() 1115 def test_gitfetch_usehead_withname(self): 1116 # Since self.gitfetcher() sets SRCREV we expect this to override 1117 # `usehead=1' and instead fetch the specified SRCREV. See 1118 # test_local_gitfetch_usehead() for a positive use of the usehead 1119 # feature. 1120 url = "git://git.openembedded.org/bitbake;usehead=1;name=newName;branch=master;protocol=https" 1121 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1122 1123 @skipIfNoNetwork() 1124 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self): 1125 recipeurl = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 1126 mirrorurl = "git://someserver.org/bitbake;branch=master;protocol=https" 1127 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1128 self.gitfetcher(recipeurl, mirrorurl) 1129 1130 @skipIfNoNetwork() 1131 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self): 1132 recipeurl = "git://someserver.org/bitbake;branch=master;protocol=https" 1133 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1134 self.gitfetcher(recipeurl, recipeurl) 1135 1136 @skipIfNoNetwork() 1137 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self): 1138 realurl = "https://git.openembedded.org/bitbake" 1139 recipeurl = "git://someserver.org/bitbake;protocol=https;branch=master" 1140 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git") 1141 os.chdir(self.tempdir) 1142 self.git(['clone', realurl, self.sourcedir], cwd=self.tempdir) 1143 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipeurl, self.sourcedir)) 1144 self.gitfetcher(recipeurl, recipeurl) 1145 1146 @skipIfNoNetwork() 1147 def test_git_submodule(self): 1148 # URL with ssh submodules 1149 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master;protocol=https" 1150 # Original URL (comment this if you have ssh access to git.yoctoproject.org) 1151 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master;protocol=https" 1152 fetcher = bb.fetch.Fetch([url], self.d) 1153 fetcher.download() 1154 # Previous cwd has been deleted 1155 os.chdir(os.path.dirname(self.unpackdir)) 1156 fetcher.unpack(self.unpackdir) 1157 1158 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1159 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing') 1160 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing') 1161 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present') 1162 1163 # Only when we're running the extended test with a submodule's submodule, can we check this. 1164 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')): 1165 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing') 1166 1167 @skipIfNoNetwork() 1168 def test_git_submodule_restricted_network_premirrors(self): 1169 # this test is to ensure that premirrors will be tried in restricted network 1170 # that is, BB_ALLOWED_NETWORKS does not contain the domain the url uses 1171 url = "gitsm://github.com/grpc/grpc.git;protocol=https;name=grpc;branch=v1.60.x;rev=0ef13a7555dbaadd4633399242524129eef5e231" 1172 # create a download directory to be used as premirror later 1173 tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") 1174 dl_premirror = os.path.join(tempdir, "download-premirror") 1175 os.mkdir(dl_premirror) 1176 self.d.setVar("DL_DIR", dl_premirror) 1177 fetcher = bb.fetch.Fetch([url], self.d) 1178 fetcher.download() 1179 # now use the premirror in restricted network 1180 self.d.setVar("DL_DIR", self.dldir) 1181 self.d.setVar("PREMIRRORS", "gitsm://.*/.* gitsm://%s/git2/MIRRORNAME;protocol=file" % dl_premirror) 1182 self.d.setVar("BB_ALLOWED_NETWORKS", "*.some.domain") 1183 fetcher = bb.fetch.Fetch([url], self.d) 1184 fetcher.download() 1185 1186 @skipIfNoNetwork() 1187 def test_git_submodule_dbus_broker(self): 1188 # The following external repositories have show failures in fetch and unpack operations 1189 # We want to avoid regressions! 1190 url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main" 1191 fetcher = bb.fetch.Fetch([url], self.d) 1192 fetcher.download() 1193 # Previous cwd has been deleted 1194 os.chdir(os.path.dirname(self.unpackdir)) 1195 fetcher.unpack(self.unpackdir) 1196 1197 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1198 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"') 1199 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"') 1200 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"') 1201 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"') 1202 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"') 1203 1204 @skipIfNoNetwork() 1205 def test_git_submodule_CLI11(self): 1206 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main" 1207 fetcher = bb.fetch.Fetch([url], self.d) 1208 fetcher.download() 1209 # Previous cwd has been deleted 1210 os.chdir(os.path.dirname(self.unpackdir)) 1211 fetcher.unpack(self.unpackdir) 1212 1213 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1214 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1215 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1216 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1217 1218 @skipIfNoNetwork() 1219 def test_git_submodule_update_CLI11(self): 1220 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """ 1221 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main" 1222 fetcher = bb.fetch.Fetch([url], self.d) 1223 fetcher.download() 1224 1225 # CLI11 that pulls in a newer nlohmann-json 1226 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main" 1227 fetcher = bb.fetch.Fetch([url], self.d) 1228 fetcher.download() 1229 # Previous cwd has been deleted 1230 os.chdir(os.path.dirname(self.unpackdir)) 1231 fetcher.unpack(self.unpackdir) 1232 1233 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1234 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1235 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1236 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1237 1238 @skipIfNoNetwork() 1239 def test_git_submodule_aktualizr(self): 1240 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" 1241 fetcher = bb.fetch.Fetch([url], self.d) 1242 fetcher.download() 1243 # Previous cwd has been deleted 1244 os.chdir(os.path.dirname(self.unpackdir)) 1245 fetcher.unpack(self.unpackdir) 1246 1247 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1248 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"') 1249 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"') 1250 self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly") 1251 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"') 1252 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"') 1253 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"') 1254 1255 @skipIfNoNetwork() 1256 def test_git_submodule_iotedge(self): 1257 """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """ 1258 1259 # This repository also has submodules where the module (name), path and url do not align 1260 url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699;branch=main" 1261 fetcher = bb.fetch.Fetch([url], self.d) 1262 fetcher.download() 1263 # Previous cwd has been deleted 1264 os.chdir(os.path.dirname(self.unpackdir)) 1265 fetcher.unpack(self.unpackdir) 1266 1267 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1268 1269 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/README.md')), msg='Missing submodule checkout') 1270 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/ctest/README.md')), msg='Missing submodule checkout') 1271 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1272 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1273 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1274 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1275 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/README.md')), msg='Missing submodule checkout') 1276 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/README.md')), msg='Missing submodule checkout') 1277 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/ctest/README.md')), msg='Missing submodule checkout') 1278 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1279 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1280 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1281 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1282 1283 @skipIfNoNetwork() 1284 def test_git_submodule_reference_to_parent(self): 1285 self.recipe_url = "gitsm://github.com/gflags/gflags.git;protocol=https;branch=master" 1286 self.d.setVar("SRCREV", "14e1138441bbbb584160cb1c0a0426ec1bac35f1") 1287 with Timeout(60): 1288 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 1289 with self.assertRaises(bb.fetch2.FetchError): 1290 fetcher.download() 1291 1292class SVNTest(FetcherTest): 1293 def skipIfNoSvn(): 1294 if not shutil.which("svn"): 1295 return unittest.skip("svn not installed, tests being skipped") 1296 1297 if not shutil.which("svnadmin"): 1298 return unittest.skip("svnadmin not installed, tests being skipped") 1299 1300 return lambda f: f 1301 1302 @skipIfNoSvn() 1303 def setUp(self): 1304 """ Create a local repository """ 1305 1306 super(SVNTest, self).setUp() 1307 1308 # Create something we can fetch 1309 src_dir = tempfile.mkdtemp(dir=self.tempdir, 1310 prefix='svnfetch_srcdir_') 1311 src_dir = os.path.abspath(src_dir) 1312 bb.process.run("echo readme > README.md", cwd=src_dir) 1313 1314 # Store it in a local SVN repository 1315 repo_dir = tempfile.mkdtemp(dir=self.tempdir, 1316 prefix='svnfetch_localrepo_') 1317 repo_dir = os.path.abspath(repo_dir) 1318 bb.process.run("svnadmin create project", cwd=repo_dir) 1319 1320 self.repo_url = "file://%s/project" % repo_dir 1321 bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url), 1322 cwd=repo_dir) 1323 1324 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir) 1325 # Github won't emulate SVN anymore (see https://github.blog/2023-01-20-sunsetting-subversion-support/) 1326 # Use still accessible svn repo (only trunk to avoid longer downloads) 1327 bb.process.run("svn propset svn:externals 'bitbake https://svn.apache.org/repos/asf/serf/trunk' .", 1328 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1329 bb.process.run("svn commit --non-interactive -m 'Add external'", 1330 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1331 1332 self.src_dir = src_dir 1333 self.repo_dir = repo_dir 1334 1335 @skipIfNoSvn() 1336 def tearDown(self): 1337 os.chdir(self.origdir) 1338 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 1339 print("Not cleaning up %s. Please remove manually." % self.tempdir) 1340 else: 1341 bb.utils.prunedir(self.tempdir) 1342 1343 @skipIfNoSvn() 1344 @skipIfNoNetwork() 1345 def test_noexternal_svn(self): 1346 # Always match the rev count from setUp (currently rev 2) 1347 url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '') 1348 fetcher = bb.fetch.Fetch([url], self.d) 1349 fetcher.download() 1350 os.chdir(os.path.dirname(self.unpackdir)) 1351 fetcher.unpack(self.unpackdir) 1352 1353 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1354 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1355 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should NOT exist") 1356 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should NOT exit") 1357 1358 @skipIfNoSvn() 1359 def test_external_svn(self): 1360 # Always match the rev count from setUp (currently rev 2) 1361 url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '') 1362 fetcher = bb.fetch.Fetch([url], self.d) 1363 fetcher.download() 1364 os.chdir(os.path.dirname(self.unpackdir)) 1365 fetcher.unpack(self.unpackdir) 1366 1367 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1368 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1369 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should exist") 1370 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should exit") 1371 1372class TrustedNetworksTest(FetcherTest): 1373 def test_trusted_network(self): 1374 # Ensure trusted_network returns False when the host IS in the list. 1375 url = "git://Someserver.org/foo;rev=1;branch=master" 1376 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org") 1377 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1378 1379 def test_wild_trusted_network(self): 1380 # Ensure trusted_network returns true when the *.host IS in the list. 1381 url = "git://Someserver.org/foo;rev=1;branch=master" 1382 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1383 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1384 1385 def test_prefix_wild_trusted_network(self): 1386 # Ensure trusted_network returns true when the prefix matches *.host. 1387 url = "git://git.Someserver.org/foo;rev=1;branch=master" 1388 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1389 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1390 1391 def test_two_prefix_wild_trusted_network(self): 1392 # Ensure trusted_network returns true when the prefix matches *.host. 1393 url = "git://something.git.Someserver.org/foo;rev=1;branch=master" 1394 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1395 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1396 1397 def test_port_trusted_network(self): 1398 # Ensure trusted_network returns True, even if the url specifies a port. 1399 url = "git://someserver.org:8080/foo;rev=1;branch=master" 1400 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org") 1401 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1402 1403 def test_untrusted_network(self): 1404 # Ensure trusted_network returns False when the host is NOT in the list. 1405 url = "git://someserver.org/foo;rev=1;branch=master" 1406 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1407 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1408 1409 def test_wild_untrusted_network(self): 1410 # Ensure trusted_network returns False when the host is NOT in the list. 1411 url = "git://*.someserver.org/foo;rev=1;branch=master" 1412 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1413 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1414 1415class URLHandle(unittest.TestCase): 1416 # Quote password as per RFC3986 1417 password = urllib.parse.quote(r"!#$%^&*()-_={}[]\|:?,.<>~`", r"!$&'/()*+,;=") 1418 datatable = { 1419 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}), 1420 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}), 1421 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])), 1422 "git://git.openembedded.org/bitbake;branch=@foo;protocol=https" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo', 'protocol' : 'https'}), 1423 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}), 1424 "file://example@.service": ('file', '', 'example@.service', '', '', {}), 1425 "https://somesite.com/somerepo.git;user=anyUser:idtoken=1234" : ('https', 'somesite.com', '/somerepo.git', '', '', {'user': 'anyUser:idtoken=1234'}), 1426 'git://s.o-me_ONE:%s@git.openembedded.org/bitbake;branch=main;protocol=https' % password: ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', password, {'branch': 'main', 'protocol' : 'https'}), 1427 } 1428 # we require a pathname to encodeurl but users can still pass such urls to 1429 # decodeurl and we need to handle them 1430 decodedata = datatable.copy() 1431 decodedata.update({ 1432 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}), 1433 "npmsw://some.registry.url;package=@pkg;version=latest": ('npmsw', 'some.registry.url', '/', '', '', {'package': '@pkg', 'version': 'latest'}), 1434 }) 1435 1436 def test_decodeurl(self): 1437 for k, v in self.decodedata.items(): 1438 result = bb.fetch.decodeurl(k) 1439 self.assertEqual(result, v) 1440 1441 def test_encodeurl(self): 1442 for k, v in self.datatable.items(): 1443 result = bb.fetch.encodeurl(v) 1444 if result.startswith("file:"): 1445 result = urllib.parse.unquote(result) 1446 self.assertEqual(result, k) 1447 1448class FetchLatestVersionTest(FetcherTest): 1449 1450 test_git_uris = { 1451 # version pattern "X.Y.Z" 1452 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "", "") 1453 : "1.99.4", 1454 # version pattern "vX.Y" 1455 # mirror of git.infradead.org since network issues interfered with testing 1456 ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master;protocol=https", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "", "") 1457 : "1.5.0", 1458 # version pattern "pkg_name-X.Y" 1459 # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing 1460 ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master;protocol=https", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "", "") 1461 : "1.0", 1462 # version pattern "pkg_name-vX.Y.Z" 1463 ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master;protocol=https", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "", "") 1464 : "1.4.0", 1465 # combination version pattern 1466 ("sysprof", "git://git.yoctoproject.org/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "", "") 1467 : "1.2.0", 1468 ("u-boot-mkimage", "git://source.denx.de/u-boot/u-boot.git;branch=master;protocol=https", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "") 1469 : "2014.01", 1470 # version pattern "yyyymmdd" 1471 ("mobile-broadband-provider-info", "git://git.yoctoproject.org/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "", "") 1472 : "20120614", 1473 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX 1474 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing 1475 ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master;protocol=https", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))", "") 1476 : "0.4.3", 1477 ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master;protocol=https", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))", "") 1478 : "11.0.0", 1479 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))", "") 1480 : "1.3.59", 1481 ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))", "") 1482 : "3.82+dbg0.9", 1483 ("sysdig", "git://github.com/draios/sysdig.git;branch=dev;protocol=https", "4fb6288275f567f63515df0ff0a6518043ecfa9b", r"^(?P<pver>\d+(\.\d+)+)", "10.0.0") 1484 : "0.28.0", 1485 } 1486 1487 WgetTestData = collections.namedtuple("WgetTestData", ["pn", "path", "pv", "check_uri", "check_regex"], defaults=[None, None, None]) 1488 test_wget_uris = { 1489 # 1490 # packages with versions inside directory name 1491 # 1492 # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2 1493 WgetTestData("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2") 1494 : "2.24.2", 1495 # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz 1496 WgetTestData("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz") 1497 : "1.6.0", 1498 # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz 1499 WgetTestData("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz") 1500 : "2.8.12.1", 1501 # https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz 1502 WgetTestData("libxml2", "/software/libxml2/2.9/libxml2-2.9.14.tar.xz") 1503 : "2.10.3", 1504 # 1505 # packages with versions only in current directory 1506 # 1507 # https://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2 1508 WgetTestData("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2") 1509 : "2.19", 1510 # https://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2 1511 WgetTestData("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2") 1512 : "20120814", 1513 # 1514 # packages with "99" in the name of possible version 1515 # 1516 # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz 1517 WgetTestData("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz") 1518 : "5.0", 1519 # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2 1520 WgetTestData("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2") 1521 : "1.15.1", 1522 # 1523 # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX 1524 # 1525 # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2 1526 # https://github.com/apple/cups/releases 1527 WgetTestData("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", check_uri="/apple/cups/releases", check_regex=r"(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz") 1528 : "2.0.0", 1529 # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz 1530 # http://ftp.debian.org/debian/pool/main/d/db5.3/ 1531 WgetTestData("db", "/berkeley-db/db-5.3.21.tar.gz", check_uri="/debian/pool/main/d/db5.3/", check_regex=r"(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz") 1532 : "5.3.10", 1533 # 1534 # packages where the tarball compression changed in the new version 1535 # 1536 # http://ftp.debian.org/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz 1537 WgetTestData("minicom", "/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz") 1538 : "2.8", 1539 1540 # 1541 # packages where the path doesn't actually contain the filename, so downloadfilename should be respected 1542 # 1543 WgetTestData("miniupnpd", "/software/miniupnp/download.php?file=miniupnpd_2.1.20191006.tar.gz;downloadfilename=miniupnpd_2.1.20191006.tar.gz", pv="2.1.20191006", check_uri="/software/miniupnp/download.php", check_regex=r"miniupnpd-(?P<pver>\d+(\.\d+)+)\.tar") 1544 : "2.3.7", 1545 } 1546 1547 test_crate_uris = { 1548 # basic example; version pattern "A.B.C+cargo-D.E.F" 1549 ("cargo-c", "crate://crates.io/cargo-c/0.9.18+cargo-0.69") 1550 : "0.9.29" 1551 } 1552 1553 @skipIfNoNetwork() 1554 def test_git_latest_versionstring(self): 1555 for k, v in self.test_git_uris.items(): 1556 with self.subTest(pn=k[0]): 1557 self.d.setVar("PN", k[0]) 1558 self.d.setVar("SRCREV", k[2]) 1559 self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3]) 1560 ud = bb.fetch2.FetchData(k[1], self.d) 1561 pupver= ud.method.latest_versionstring(ud, self.d) 1562 verstring = pupver[0] 1563 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1564 r = bb.utils.vercmp_string(v, verstring) 1565 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1566 if k[4]: 1567 r = bb.utils.vercmp_string(verstring, k[4]) 1568 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], verstring, k[4])) 1569 1570 def test_wget_latest_versionstring(self): 1571 testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata" 1572 server = HTTPService(testdata, host="127.0.0.1") 1573 server.start() 1574 port = server.port 1575 try: 1576 for data, v in self.test_wget_uris.items(): 1577 with self.subTest(pn=data.pn): 1578 self.d.setVar("PN", data.pn) 1579 self.d.setVar("PV", data.pv) 1580 if data.check_uri: 1581 checkuri = "http://127.0.0.1:%s/%s" % (port, data.check_uri) 1582 self.d.setVar("UPSTREAM_CHECK_URI", checkuri) 1583 if data.check_regex: 1584 self.d.setVar("UPSTREAM_CHECK_REGEX", data.check_regex) 1585 1586 url = "http://127.0.0.1:%s/%s" % (port, data.path) 1587 ud = bb.fetch2.FetchData(url, self.d) 1588 pupver = ud.method.latest_versionstring(ud, self.d) 1589 verstring = pupver[0] 1590 self.assertTrue(verstring, msg="Could not find upstream version for %s" % data.pn) 1591 r = bb.utils.vercmp_string(v, verstring) 1592 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (data.pn, v, verstring)) 1593 finally: 1594 server.stop() 1595 1596 @skipIfNoNetwork() 1597 def test_crate_latest_versionstring(self): 1598 for k, v in self.test_crate_uris.items(): 1599 with self.subTest(pn=k[0]): 1600 self.d.setVar("PN", k[0]) 1601 ud = bb.fetch2.FetchData(k[1], self.d) 1602 pupver = ud.method.latest_versionstring(ud, self.d) 1603 verstring = pupver[0] 1604 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1605 r = bb.utils.vercmp_string(v, verstring) 1606 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1607 1608class FetchCheckStatusTest(FetcherTest): 1609 test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz", 1610 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz", 1611 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz", 1612 "https://yoctoproject.org/", 1613 "https://docs.yoctoproject.org", 1614 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", 1615 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", 1616 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz", 1617 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD 1618 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz" 1619 ] 1620 1621 @skipIfNoNetwork() 1622 def test_wget_checkstatus(self): 1623 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d) 1624 for u in self.test_wget_uris: 1625 with self.subTest(url=u): 1626 ud = fetch.ud[u] 1627 m = ud.method 1628 ret = m.checkstatus(fetch, ud, self.d) 1629 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1630 1631 @skipIfNoNetwork() 1632 def test_wget_checkstatus_connection_cache(self): 1633 from bb.fetch2 import FetchConnectionCache 1634 1635 connection_cache = FetchConnectionCache() 1636 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d, 1637 connection_cache = connection_cache) 1638 1639 for u in self.test_wget_uris: 1640 with self.subTest(url=u): 1641 ud = fetch.ud[u] 1642 m = ud.method 1643 ret = m.checkstatus(fetch, ud, self.d) 1644 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1645 1646 connection_cache.close_connections() 1647 1648 1649class GitMakeShallowTest(FetcherTest): 1650 def setUp(self): 1651 FetcherTest.setUp(self) 1652 self.gitdir = os.path.join(self.tempdir, 'gitshallow') 1653 bb.utils.mkdirhier(self.gitdir) 1654 self.git_init() 1655 1656 def assertRefs(self, expected_refs): 1657 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines() 1658 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines() 1659 self.assertEqual(sorted(full_expected), sorted(actual_refs)) 1660 1661 def assertRevCount(self, expected_count, args=None): 1662 if args is None: 1663 args = ['HEAD'] 1664 revs = self.git(['rev-list'] + args) 1665 actual_count = len(revs.splitlines()) 1666 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1667 1668 def make_shallow(self, args=None): 1669 if args is None: 1670 args = ['HEAD'] 1671 return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir) 1672 1673 def add_empty_file(self, path, msg=None): 1674 if msg is None: 1675 msg = path 1676 open(os.path.join(self.gitdir, path), 'w').close() 1677 self.git(['add', path]) 1678 self.git(['commit', '-m', msg, path]) 1679 1680 def test_make_shallow_single_branch_no_merge(self): 1681 self.add_empty_file('a') 1682 self.add_empty_file('b') 1683 self.assertRevCount(2) 1684 self.make_shallow() 1685 self.assertRevCount(1) 1686 1687 def test_make_shallow_single_branch_one_merge(self): 1688 self.add_empty_file('a') 1689 self.add_empty_file('b') 1690 self.git('checkout -b a_branch') 1691 self.add_empty_file('c') 1692 self.git('checkout master') 1693 self.add_empty_file('d') 1694 self.git('merge --no-ff --no-edit a_branch') 1695 self.git('branch -d a_branch') 1696 self.add_empty_file('e') 1697 self.assertRevCount(6) 1698 self.make_shallow(['HEAD~2']) 1699 self.assertRevCount(5) 1700 1701 def test_make_shallow_at_merge(self): 1702 self.add_empty_file('a') 1703 self.git('checkout -b a_branch') 1704 self.add_empty_file('b') 1705 self.git('checkout master') 1706 self.git('merge --no-ff --no-edit a_branch') 1707 self.git('branch -d a_branch') 1708 self.assertRevCount(3) 1709 self.make_shallow() 1710 self.assertRevCount(1) 1711 1712 def test_make_shallow_annotated_tag(self): 1713 self.add_empty_file('a') 1714 self.add_empty_file('b') 1715 self.git('tag -a -m a_tag a_tag') 1716 self.assertRevCount(2) 1717 self.make_shallow(['a_tag']) 1718 self.assertRevCount(1) 1719 1720 def test_make_shallow_multi_ref(self): 1721 self.add_empty_file('a') 1722 self.add_empty_file('b') 1723 self.git('checkout -b a_branch') 1724 self.add_empty_file('c') 1725 self.git('checkout master') 1726 self.add_empty_file('d') 1727 self.git('checkout -b a_branch_2') 1728 self.add_empty_file('a_tag') 1729 self.git('tag a_tag') 1730 self.git('checkout master') 1731 self.git('branch -D a_branch_2') 1732 self.add_empty_file('e') 1733 self.assertRevCount(6, ['--all']) 1734 self.make_shallow() 1735 self.assertRevCount(5, ['--all']) 1736 1737 def test_make_shallow_multi_ref_trim(self): 1738 self.add_empty_file('a') 1739 self.git('checkout -b a_branch') 1740 self.add_empty_file('c') 1741 self.git('checkout master') 1742 self.assertRevCount(1) 1743 self.assertRevCount(2, ['--all']) 1744 self.assertRefs(['master', 'a_branch']) 1745 self.make_shallow(['-r', 'master', 'HEAD']) 1746 self.assertRevCount(1, ['--all']) 1747 self.assertRefs(['master']) 1748 1749 def test_make_shallow_noop(self): 1750 self.add_empty_file('a') 1751 self.assertRevCount(1) 1752 self.make_shallow() 1753 self.assertRevCount(1) 1754 1755 @skipIfNoNetwork() 1756 def test_make_shallow_bitbake(self): 1757 self.git('remote add origin https://github.com/openembedded/bitbake') 1758 self.git('fetch --tags origin') 1759 orig_revs = len(self.git('rev-list --all').splitlines()) 1760 self.make_shallow(['refs/tags/1.10.0']) 1761 self.assertRevCount(orig_revs - 1746, ['--all']) 1762 1763class GitShallowTest(FetcherTest): 1764 def setUp(self): 1765 FetcherTest.setUp(self) 1766 self.gitdir = os.path.join(self.tempdir, 'git') 1767 self.srcdir = os.path.join(self.tempdir, 'gitsource') 1768 1769 bb.utils.mkdirhier(self.srcdir) 1770 self.git_init(cwd=self.srcdir) 1771 self.d.setVar('WORKDIR', self.tempdir) 1772 self.d.setVar('S', self.gitdir) 1773 self.d.delVar('PREMIRRORS') 1774 self.d.delVar('MIRRORS') 1775 1776 uri = 'git://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1777 self.d.setVar('SRC_URI', uri) 1778 self.d.setVar('SRCREV', '${AUTOREV}') 1779 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 1780 1781 self.d.setVar('BB_GIT_SHALLOW', '1') 1782 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0') 1783 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 1784 self.d.setVar("__BBSRCREV_SEEN", "1") 1785 1786 def assertRefs(self, expected_refs, cwd=None): 1787 if cwd is None: 1788 cwd = self.gitdir 1789 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines() 1790 # Resolve references into the same format as the comparision (needed by git 2.48 onwards) 1791 actual_refs = self.git(['rev-parse', '--symbolic-full-name'] + actual_refs, cwd=cwd).splitlines() 1792 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines() 1793 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs))) 1794 1795 def assertRevCount(self, expected_count, args=None, cwd=None): 1796 if args is None: 1797 args = ['HEAD'] 1798 if cwd is None: 1799 cwd = self.gitdir 1800 revs = self.git(['rev-list'] + args, cwd=cwd) 1801 actual_count = len(revs.splitlines()) 1802 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1803 1804 def add_empty_file(self, path, cwd=None, msg=None): 1805 if msg is None: 1806 msg = path 1807 if cwd is None: 1808 cwd = self.srcdir 1809 open(os.path.join(cwd, path), 'w').close() 1810 self.git(['add', path], cwd) 1811 self.git(['commit', '-m', msg, path], cwd) 1812 1813 def fetch(self, uri=None): 1814 if uri is None: 1815 uris = self.d.getVar('SRC_URI').split() 1816 uri = uris[0] 1817 d = self.d 1818 else: 1819 d = self.d.createCopy() 1820 d.setVar('SRC_URI', uri) 1821 uri = d.expand(uri) 1822 uris = [uri] 1823 1824 fetcher = bb.fetch2.Fetch(uris, d) 1825 fetcher.download() 1826 ud = fetcher.ud[uri] 1827 return fetcher, ud 1828 1829 def fetch_and_unpack(self, uri=None): 1830 fetcher, ud = self.fetch(uri) 1831 fetcher.unpack(self.d.getVar('WORKDIR')) 1832 assert os.path.exists(self.d.getVar('S')) 1833 return fetcher, ud 1834 1835 def fetch_shallow(self, uri=None, disabled=False, keepclone=False): 1836 """Fetch a uri, generating a shallow tarball, then unpack using it""" 1837 fetcher, ud = self.fetch_and_unpack(uri) 1838 1839 # Confirm that the unpacked repo is unshallow 1840 if not disabled: 1841 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0])) 1842 1843 # fetch and unpack, from the shallow tarball 1844 bb.utils.remove(self.gitdir, recurse=True) 1845 if os.path.exists(ud.clonedir): 1846 bb.process.run('chmod u+w -R "%s"' % ud.clonedir) 1847 bb.utils.remove(ud.clonedir, recurse=True) 1848 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True) 1849 1850 # confirm that the unpacked repo is used when no git clone or git 1851 # mirror tarball is available 1852 fetcher, ud = self.fetch_and_unpack(uri) 1853 if not disabled: 1854 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir 1855 else: 1856 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir 1857 return fetcher, ud 1858 1859 def test_shallow_disabled(self): 1860 self.add_empty_file('a') 1861 self.add_empty_file('b') 1862 self.assertRevCount(2, cwd=self.srcdir) 1863 1864 self.d.setVar('BB_GIT_SHALLOW', '0') 1865 self.fetch_shallow(disabled=True) 1866 self.assertRevCount(2) 1867 1868 def test_shallow_nobranch(self): 1869 self.add_empty_file('a') 1870 self.add_empty_file('b') 1871 self.assertRevCount(2, cwd=self.srcdir) 1872 1873 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip() 1874 self.d.setVar('SRCREV', srcrev) 1875 uri = self.d.getVar('SRC_URI').split()[0] 1876 uri = '%s;nobranch=1;bare=1' % uri 1877 1878 self.fetch_shallow(uri) 1879 self.assertRevCount(1) 1880 1881 # shallow refs are used to ensure the srcrev sticks around when we 1882 # have no other branches referencing it 1883 self.assertRefs(['refs/shallow/default']) 1884 1885 def test_shallow_default_depth_1(self): 1886 # Create initial git repo 1887 self.add_empty_file('a') 1888 self.add_empty_file('b') 1889 self.assertRevCount(2, cwd=self.srcdir) 1890 1891 self.fetch_shallow() 1892 self.assertRevCount(1) 1893 1894 def test_shallow_depth_0_disables(self): 1895 self.add_empty_file('a') 1896 self.add_empty_file('b') 1897 self.assertRevCount(2, cwd=self.srcdir) 1898 1899 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1900 self.fetch_shallow(disabled=True) 1901 self.assertRevCount(2) 1902 1903 def test_shallow_depth_default_override(self): 1904 self.add_empty_file('a') 1905 self.add_empty_file('b') 1906 self.assertRevCount(2, cwd=self.srcdir) 1907 1908 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2') 1909 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1') 1910 self.fetch_shallow() 1911 self.assertRevCount(1) 1912 1913 def test_shallow_depth_default_override_disable(self): 1914 self.add_empty_file('a') 1915 self.add_empty_file('b') 1916 self.add_empty_file('c') 1917 self.assertRevCount(3, cwd=self.srcdir) 1918 1919 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1920 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2') 1921 self.fetch_shallow() 1922 self.assertRevCount(2) 1923 1924 def test_current_shallow_out_of_date_clone(self): 1925 # Create initial git repo 1926 self.add_empty_file('a') 1927 self.add_empty_file('b') 1928 self.add_empty_file('c') 1929 self.assertRevCount(3, cwd=self.srcdir) 1930 1931 # Clone without tarball 1932 self.d.setVar('BB_GIT_SHALLOW', '0') 1933 fetcher, ud = self.fetch() 1934 1935 # Clone and generate mirror tarball 1936 self.d.setVar('BB_GIT_SHALLOW', '1') 1937 fetcher, ud = self.fetch() 1938 1939 # Ensure we have a current mirror tarball, but an out of date clone 1940 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir) 1941 self.assertRevCount(2, cwd=ud.clonedir) 1942 1943 # Fetch and unpack, from the current tarball, not the out of date clone 1944 bb.utils.remove(self.gitdir, recurse=True) 1945 fetcher, ud = self.fetch() 1946 fetcher.unpack(self.d.getVar('WORKDIR')) 1947 self.assertRevCount(1) 1948 assert os.path.exists(os.path.join(self.d.getVar('WORKDIR'), 'git', 'c')) 1949 1950 def test_shallow_single_branch_no_merge(self): 1951 self.add_empty_file('a') 1952 self.add_empty_file('b') 1953 self.assertRevCount(2, cwd=self.srcdir) 1954 1955 self.fetch_shallow() 1956 self.assertRevCount(1) 1957 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1958 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1959 1960 def test_shallow_no_dangling(self): 1961 self.add_empty_file('a') 1962 self.add_empty_file('b') 1963 self.assertRevCount(2, cwd=self.srcdir) 1964 1965 self.fetch_shallow() 1966 self.assertRevCount(1) 1967 assert not self.git('fsck --dangling') 1968 1969 def test_shallow_srcrev_branch_truncation(self): 1970 self.add_empty_file('a') 1971 self.add_empty_file('b') 1972 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip() 1973 self.add_empty_file('c') 1974 self.assertRevCount(3, cwd=self.srcdir) 1975 1976 self.d.setVar('SRCREV', b_commit) 1977 self.fetch_shallow() 1978 1979 # The 'c' commit was removed entirely, and 'a' was removed from history 1980 self.assertRevCount(1, ['--all']) 1981 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit) 1982 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1983 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1984 assert not os.path.exists(os.path.join(self.gitdir, 'c')) 1985 1986 def test_shallow_ref_pruning(self): 1987 self.add_empty_file('a') 1988 self.add_empty_file('b') 1989 self.git('branch a_branch', cwd=self.srcdir) 1990 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 1991 self.assertRevCount(2, cwd=self.srcdir) 1992 1993 self.fetch_shallow() 1994 1995 self.assertRefs(['master', 'origin/master']) 1996 self.assertRevCount(1) 1997 1998 def test_shallow_submodules(self): 1999 self.add_empty_file('a') 2000 self.add_empty_file('b') 2001 2002 smdir = os.path.join(self.tempdir, 'gitsubmodule') 2003 bb.utils.mkdirhier(smdir) 2004 self.git_init(cwd=smdir) 2005 # Make this look like it was cloned from a remote... 2006 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 2007 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 2008 self.add_empty_file('asub', cwd=smdir) 2009 self.add_empty_file('bsub', cwd=smdir) 2010 2011 self.git('submodule init', cwd=self.srcdir) 2012 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 2013 self.git('submodule update', cwd=self.srcdir) 2014 self.git('commit -m submodule -a', cwd=self.srcdir) 2015 2016 uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 2017 fetcher, ud = self.fetch_shallow(uri) 2018 2019 # Verify the main repository is shallow 2020 self.assertRevCount(1) 2021 2022 # Verify the gitsubmodule directory is present 2023 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 2024 2025 # Verify the submodule is also shallow 2026 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 2027 2028 def test_shallow_submodule_mirrors(self): 2029 self.add_empty_file('a') 2030 self.add_empty_file('b') 2031 2032 smdir = os.path.join(self.tempdir, 'gitsubmodule') 2033 bb.utils.mkdirhier(smdir) 2034 self.git_init(cwd=smdir) 2035 # Make this look like it was cloned from a remote... 2036 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 2037 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 2038 self.add_empty_file('asub', cwd=smdir) 2039 self.add_empty_file('bsub', cwd=smdir) 2040 2041 self.git('submodule init', cwd=self.srcdir) 2042 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 2043 self.git('submodule update', cwd=self.srcdir) 2044 self.git('commit -m submodule -a', cwd=self.srcdir) 2045 2046 uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 2047 2048 # Fetch once to generate the shallow tarball 2049 fetcher, ud = self.fetch(uri) 2050 2051 # Set up the mirror 2052 mirrordir = os.path.join(self.tempdir, 'mirror') 2053 bb.utils.rename(self.dldir, mirrordir) 2054 self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/' % mirrordir) 2055 2056 # Fetch from the mirror 2057 bb.utils.remove(self.dldir, recurse=True) 2058 bb.utils.remove(self.gitdir, recurse=True) 2059 self.fetch_and_unpack(uri) 2060 2061 # Verify the main repository is shallow 2062 self.assertRevCount(1) 2063 2064 # Verify the gitsubmodule directory is present 2065 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 2066 2067 # Verify the submodule is also shallow 2068 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 2069 2070 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')): 2071 def test_shallow_annex(self): 2072 self.add_empty_file('a') 2073 self.add_empty_file('b') 2074 self.git('annex init', cwd=self.srcdir) 2075 open(os.path.join(self.srcdir, 'c'), 'w').close() 2076 self.git('annex add c', cwd=self.srcdir) 2077 self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir) 2078 bb.process.run('chmod u+w -R %s' % self.srcdir) 2079 2080 uri = 'gitannex://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 2081 fetcher, ud = self.fetch_shallow(uri) 2082 2083 self.assertRevCount(1) 2084 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0] 2085 assert os.path.exists(os.path.join(self.gitdir, 'c')) 2086 2087 def test_shallow_clone_preferred_over_shallow(self): 2088 self.add_empty_file('a') 2089 self.add_empty_file('b') 2090 2091 # Fetch once to generate the shallow tarball 2092 self.d.setVar('BB_GIT_SHALLOW', '0') 2093 fetcher, ud = self.fetch() 2094 2095 # Fetch and unpack with both the clonedir and shallow tarball available 2096 bb.utils.remove(self.gitdir, recurse=True) 2097 self.d.setVar('BB_GIT_SHALLOW', '1') 2098 fetcher, ud = self.fetch_and_unpack() 2099 2100 # The unpacked tree should *not* be shallow 2101 self.assertRevCount(2) 2102 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')) 2103 2104 def test_shallow_mirrors(self): 2105 self.add_empty_file('a') 2106 self.add_empty_file('b') 2107 2108 # Fetch once to generate the shallow tarball 2109 fetcher, ud = self.fetch() 2110 mirrortarball = ud.mirrortarballs[0] 2111 assert os.path.exists(os.path.join(self.dldir, mirrortarball)) 2112 2113 # Set up the mirror 2114 mirrordir = os.path.join(self.tempdir, 'mirror') 2115 bb.utils.mkdirhier(mirrordir) 2116 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % mirrordir) 2117 2118 bb.utils.rename(os.path.join(self.dldir, mirrortarball), 2119 os.path.join(mirrordir, mirrortarball)) 2120 2121 # Fetch from the mirror 2122 bb.utils.remove(self.dldir, recurse=True) 2123 bb.utils.remove(self.gitdir, recurse=True) 2124 self.fetch_and_unpack() 2125 self.assertRevCount(1) 2126 2127 def test_shallow_invalid_depth(self): 2128 self.add_empty_file('a') 2129 self.add_empty_file('b') 2130 2131 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12') 2132 with self.assertRaises(bb.fetch2.FetchError): 2133 self.fetch() 2134 2135 def test_shallow_invalid_depth_default(self): 2136 self.add_empty_file('a') 2137 self.add_empty_file('b') 2138 2139 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12') 2140 with self.assertRaises(bb.fetch2.FetchError): 2141 self.fetch() 2142 2143 def test_shallow_extra_refs(self): 2144 self.add_empty_file('a') 2145 self.add_empty_file('b') 2146 self.git('branch a_branch', cwd=self.srcdir) 2147 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 2148 self.assertRevCount(2, cwd=self.srcdir) 2149 2150 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch') 2151 self.fetch_shallow() 2152 2153 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2154 self.assertRevCount(1) 2155 2156 def test_shallow_extra_refs_wildcard(self): 2157 self.add_empty_file('a') 2158 self.add_empty_file('b') 2159 self.git('branch a_branch', cwd=self.srcdir) 2160 self.git('tag v1.0', cwd=self.srcdir) 2161 self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir) 2162 self.assertRevCount(2, cwd=self.srcdir) 2163 2164 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2165 self.fetch_shallow() 2166 2167 self.assertRefs(['master', 'origin/master', 'v1.0']) 2168 self.assertRevCount(1) 2169 2170 def test_shallow_missing_extra_refs(self): 2171 self.add_empty_file('a') 2172 self.add_empty_file('b') 2173 2174 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo') 2175 with self.assertRaises(bb.fetch2.FetchError): 2176 self.fetch() 2177 2178 def test_shallow_missing_extra_refs_wildcard(self): 2179 self.add_empty_file('a') 2180 self.add_empty_file('b') 2181 2182 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2183 self.fetch() 2184 2185 def test_shallow_remove_revs(self): 2186 # Create initial git repo 2187 self.add_empty_file('a') 2188 self.add_empty_file('b') 2189 self.git('checkout -b a_branch', cwd=self.srcdir) 2190 self.add_empty_file('c') 2191 self.add_empty_file('d') 2192 self.git('checkout master', cwd=self.srcdir) 2193 self.git('tag v0.0 a_branch', cwd=self.srcdir) 2194 self.add_empty_file('e') 2195 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2196 self.git('branch -d a_branch', cwd=self.srcdir) 2197 self.add_empty_file('f') 2198 self.assertRevCount(7, cwd=self.srcdir) 2199 2200 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2201 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2202 2203 self.fetch_shallow() 2204 2205 self.assertRevCount(2) 2206 2207 def test_shallow_invalid_revs(self): 2208 self.add_empty_file('a') 2209 self.add_empty_file('b') 2210 2211 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2212 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2213 2214 with self.assertRaises(bb.fetch2.FetchError): 2215 self.fetch() 2216 2217 def test_shallow_fetch_missing_revs(self): 2218 self.add_empty_file('a') 2219 self.add_empty_file('b') 2220 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2221 self.git('tag v0.0 master', cwd=self.srcdir) 2222 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2223 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2224 2225 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm: 2226 self.fetch_shallow() 2227 self.assertIn("fatal: no commits selected for shallow requests", cm.output[0]) 2228 2229 def test_shallow_fetch_missing_revs_fails(self): 2230 self.add_empty_file('a') 2231 self.add_empty_file('b') 2232 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2233 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2234 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2235 2236 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm: 2237 self.fetch_shallow() 2238 self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0]) 2239 2240 @skipIfNoNetwork() 2241 def test_git_shallow_fetch_premirrors(self): 2242 url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 2243 2244 # Create a separate premirror directory within tempdir 2245 premirror = os.path.join(self.tempdir, "premirror") 2246 os.mkdir(premirror) 2247 2248 # Fetch a non-shallow clone into the premirror subdir 2249 self.d.setVar('BB_GIT_SHALLOW', '0') 2250 self.d.setVar("DL_DIR", premirror) 2251 fetcher, ud = self.fetch(url) 2252 2253 # Fetch a shallow clone from the premirror subdir with unpacking 2254 # using the original recipe URL and the premirror mapping 2255 self.d.setVar('BB_GIT_SHALLOW', '1') 2256 self.d.setVar("DL_DIR", self.dldir) 2257 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2258 self.d.setVar('BB_NO_NETWORK', '1') 2259 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0') 2260 self.d.setVar("PREMIRRORS", "git://.*/.* git://{0};protocol=file".format(premirror + "/git2/" + ud.host + ud.path.replace("/", "."))) 2261 fetcher = self.fetch_and_unpack(url) 2262 2263 # Verify that the unpacked sources are shallow clones 2264 self.assertRevCount(1) 2265 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')) 2266 2267 @skipIfNoNetwork() 2268 def test_bitbake(self): 2269 self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir) 2270 self.git('config core.bare true', cwd=self.srcdir) 2271 self.git('fetch', cwd=self.srcdir) 2272 2273 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2274 # Note that the 1.10.0 tag is annotated, so this also tests 2275 # reference of an annotated vs unannotated tag 2276 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0') 2277 2278 self.fetch_shallow() 2279 2280 # Confirm that the history of 1.10.0 was removed 2281 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines()) 2282 revs = len(self.git('rev-list master').splitlines()) 2283 self.assertNotEqual(orig_revs, revs) 2284 self.assertRefs(['master', 'origin/master']) 2285 self.assertRevCount(orig_revs - 1760) 2286 2287 def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self): 2288 self.add_empty_file('a') 2289 fetcher, ud = self.fetch() 2290 bb.utils.remove(self.gitdir, recurse=True) 2291 bb.utils.remove(self.dldir, recurse=True) 2292 2293 with self.assertRaises(bb.fetch2.UnpackError) as context: 2294 fetcher.unpack(self.d.getVar('WORKDIR')) 2295 2296 self.assertIn("No up to date source found", context.exception.msg) 2297 self.assertIn("clone directory not available or not up to date", context.exception.msg) 2298 2299 def test_shallow_check_is_shallow(self): 2300 self.add_empty_file('a') 2301 self.add_empty_file('b') 2302 2303 # Fetch and unpack without the clonedir and *only* shallow tarball available 2304 bb.utils.remove(self.gitdir, recurse=True) 2305 fetcher, ud = self.fetch_and_unpack() 2306 2307 # The unpacked tree *should* be shallow 2308 self.assertRevCount(1) 2309 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')) 2310 2311 def test_shallow_succeeds_with_tag_containing_slash(self): 2312 self.add_empty_file('a') 2313 self.add_empty_file('b') 2314 self.git('tag t1/t2/t3', cwd=self.srcdir) 2315 self.assertRevCount(2, cwd=self.srcdir) 2316 2317 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip() 2318 self.d.setVar('SRCREV', srcrev) 2319 uri = self.d.getVar('SRC_URI').split()[0] 2320 uri = '%s;tag=t1/t2/t3' % uri 2321 self.fetch_shallow(uri) 2322 self.assertRevCount(1) 2323 2324class GitLfsTest(FetcherTest): 2325 def skipIfNoGitLFS(): 2326 if not shutil.which('git-lfs'): 2327 return unittest.skip('git-lfs not installed') 2328 return lambda f: f 2329 2330 def setUp(self): 2331 FetcherTest.setUp(self) 2332 2333 self.gitdir = os.path.join(self.tempdir, 'git') 2334 self.srcdir = os.path.join(self.tempdir, 'gitsource') 2335 2336 self.d.setVar('WORKDIR', self.tempdir) 2337 self.d.setVar('S', self.gitdir) 2338 self.d.delVar('PREMIRRORS') 2339 self.d.delVar('MIRRORS') 2340 2341 self.d.setVar('SRCREV', '${AUTOREV}') 2342 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 2343 self.d.setVar("__BBSRCREV_SEEN", "1") 2344 2345 bb.utils.mkdirhier(self.srcdir) 2346 self.git_init(cwd=self.srcdir) 2347 self.commit_file('.gitattributes', '*.mp3 filter=lfs -text') 2348 2349 def commit(self, *, cwd=None): 2350 cwd = cwd or self.srcdir 2351 self.git(["commit", "-m", "Change"], cwd=cwd) 2352 return self.git(["rev-parse", "HEAD"], cwd=cwd).strip() 2353 2354 def commit_file(self, filename, content, *, cwd=None): 2355 cwd = cwd or self.srcdir 2356 2357 with open(os.path.join(cwd, filename), "w") as f: 2358 f.write(content) 2359 self.git(["add", filename], cwd=cwd) 2360 return self.commit(cwd=cwd) 2361 2362 def fetch(self, uri=None, download=True): 2363 uris = self.d.getVar('SRC_URI').split() 2364 uri = uris[0] 2365 d = self.d 2366 2367 fetcher = bb.fetch2.Fetch(uris, d) 2368 if download: 2369 fetcher.download() 2370 ud = fetcher.ud[uri] 2371 return fetcher, ud 2372 2373 def get_real_git_lfs_file(self): 2374 self.d.setVar('PATH', os.environ.get('PATH')) 2375 fetcher, ud = self.fetch() 2376 fetcher.unpack(self.d.getVar('WORKDIR')) 2377 unpacked_lfs_file = os.path.join(self.d.getVar('WORKDIR'), 'git', "Cat_poster_1.jpg") 2378 return unpacked_lfs_file 2379 2380 @skipIfNoGitLFS() 2381 def test_gitsm_lfs(self): 2382 """Test that the gitsm fetcher caches objects stored via LFS""" 2383 self.git(["lfs", "install", "--local"], cwd=self.srcdir) 2384 2385 def fetch_and_verify(revision, filename, content): 2386 self.d.setVar('SRCREV', revision) 2387 fetcher, ud = self.fetch() 2388 2389 with hide_directory(submoduledir), hide_directory(self.srcdir): 2390 workdir = self.d.getVar('WORKDIR') 2391 fetcher.unpack(workdir) 2392 2393 with open(os.path.join(workdir, "git", filename)) as f: 2394 self.assertEqual(f.read(), content) 2395 2396 # Create the git repository that will later be used as a submodule 2397 submoduledir = self.tempdir + "/submodule" 2398 bb.utils.mkdirhier(submoduledir) 2399 self.git_init(submoduledir) 2400 self.git(["lfs", "install", "--local"], cwd=submoduledir) 2401 self.commit_file('.gitattributes', '*.mp3 filter=lfs -text', cwd=submoduledir) 2402 2403 submodule_commit_1 = self.commit_file("a.mp3", "submodule version 1", cwd=submoduledir) 2404 _ = self.commit_file("a.mp3", "submodule version 2", cwd=submoduledir) 2405 2406 # Add the submodule to the repository at its current HEAD revision 2407 self.git(["-c", "protocol.file.allow=always", "submodule", "add", submoduledir, "submodule"], 2408 cwd=self.srcdir) 2409 base_commit_1 = self.commit() 2410 2411 # Let the submodule point at a different revision 2412 self.git(["checkout", submodule_commit_1], self.srcdir + "/submodule") 2413 self.git(["add", "submodule"], cwd=self.srcdir) 2414 base_commit_2 = self.commit() 2415 2416 # Add a LFS file to the repository 2417 base_commit_3 = self.commit_file("a.mp3", "version 1") 2418 # Update the added LFS file 2419 base_commit_4 = self.commit_file("a.mp3", "version 2") 2420 2421 self.d.setVar('SRC_URI', "gitsm://%s;protocol=file;lfs=1;branch=master" % self.srcdir) 2422 2423 # Verify that LFS objects referenced from submodules are fetched and checked out 2424 fetch_and_verify(base_commit_1, "submodule/a.mp3", "submodule version 2") 2425 # Verify that the repository inside the download cache of a submodile is extended with any 2426 # additional LFS objects needed when checking out a different revision. 2427 fetch_and_verify(base_commit_2, "submodule/a.mp3", "submodule version 1") 2428 # Verify that LFS objects referenced from the base repository are fetched and checked out 2429 fetch_and_verify(base_commit_3, "a.mp3", "version 1") 2430 # Verify that the cached repository is extended with any additional LFS objects required 2431 # when checking out a different revision. 2432 fetch_and_verify(base_commit_4, "a.mp3", "version 2") 2433 2434 @skipIfNoGitLFS() 2435 def test_gitsm_lfs_disabled(self): 2436 """Test that the gitsm fetcher does not use LFS when explicitly disabled""" 2437 self.git(["lfs", "install", "--local"], cwd=self.srcdir) 2438 2439 def fetch_and_verify(revision, filename, content): 2440 self.d.setVar('SRCREV', revision) 2441 fetcher, ud = self.fetch() 2442 2443 with hide_directory(submoduledir), hide_directory(self.srcdir): 2444 workdir = self.d.getVar('WORKDIR') 2445 fetcher.unpack(workdir) 2446 2447 with open(os.path.join(workdir, "git", filename)) as f: 2448 # Assume that LFS did not perform smudging when the expected content is 2449 # missing. 2450 self.assertNotEqual(f.read(), content) 2451 2452 # Create the git repository that will later be used as a submodule 2453 submoduledir = self.tempdir + "/submodule" 2454 bb.utils.mkdirhier(submoduledir) 2455 self.git_init(submoduledir) 2456 self.git(["lfs", "install", "--local"], cwd=submoduledir) 2457 self.commit_file('.gitattributes', '*.mp3 filter=lfs -text', cwd=submoduledir) 2458 2459 submodule_commit_1 = self.commit_file("a.mp3", "submodule version 1", cwd=submoduledir) 2460 2461 # Add the submodule to the repository at its current HEAD revision 2462 self.git(["-c", "protocol.file.allow=always", "submodule", "add", submoduledir, "submodule"], 2463 cwd=self.srcdir) 2464 base_commit_1 = self.commit() 2465 2466 # Add a LFS file to the repository 2467 base_commit_2 = self.commit_file("a.mp3", "version 1") 2468 2469 self.d.setVar('SRC_URI', "gitsm://%s;protocol=file;lfs=1;branch=master;lfs=0" % self.srcdir) 2470 2471 # Verify that LFS objects referenced from submodules are not fetched nor checked out 2472 fetch_and_verify(base_commit_1, "submodule/a.mp3", "submodule version 1") 2473 # Verify that the LFS objects referenced from the base repository are not fetched nor 2474 # checked out 2475 fetch_and_verify(base_commit_2, "a.mp3", "version 1") 2476 2477 @skipIfNoGitLFS() 2478 def test_fetch_lfs_on_srcrev_change(self): 2479 """Test if fetch downloads missing LFS objects when a different revision within an existing repository is requested""" 2480 self.git(["lfs", "install", "--local"], cwd=self.srcdir) 2481 2482 def fetch_and_verify(revision, filename, content): 2483 self.d.setVar('SRCREV', revision) 2484 fetcher, ud = self.fetch() 2485 2486 with hide_directory(self.srcdir): 2487 workdir = self.d.getVar('WORKDIR') 2488 fetcher.unpack(workdir) 2489 2490 with open(os.path.join(workdir, "git", filename)) as f: 2491 self.assertEqual(f.read(), content) 2492 2493 commit_1 = self.commit_file("a.mp3", "version 1") 2494 commit_2 = self.commit_file("a.mp3", "version 2") 2495 2496 self.d.setVar('SRC_URI', "git://%s;protocol=file;lfs=1;branch=master" % self.srcdir) 2497 2498 # Seed the local download folder by fetching the latest commit and verifying that the LFS contents are 2499 # available even when the upstream repository disappears. 2500 fetch_and_verify(commit_2, "a.mp3", "version 2") 2501 # Verify that even when an older revision is fetched, the needed LFS objects are fetched into the download 2502 # folder. 2503 fetch_and_verify(commit_1, "a.mp3", "version 1") 2504 2505 @skipIfNoGitLFS() 2506 @skipIfNoNetwork() 2507 def test_real_git_lfs_repo_succeeds_without_lfs_param(self): 2508 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master") 2509 f = self.get_real_git_lfs_file() 2510 self.assertTrue(os.path.exists(f)) 2511 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) 2512 2513 @skipIfNoGitLFS() 2514 @skipIfNoNetwork() 2515 def test_real_git_lfs_repo_succeeds(self): 2516 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=1") 2517 f = self.get_real_git_lfs_file() 2518 self.assertTrue(os.path.exists(f)) 2519 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) 2520 2521 @skipIfNoGitLFS() 2522 @skipIfNoNetwork() 2523 def test_real_git_lfs_repo_skips(self): 2524 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=0") 2525 f = self.get_real_git_lfs_file() 2526 # This is the actual non-smudged placeholder file on the repo if git-lfs does not run 2527 lfs_file = ( 2528 'version https://git-lfs.github.com/spec/v1\n' 2529 'oid sha256:34be66b1a39a1955b46a12588df9d5f6fc1da790e05cf01f3c7422f4bbbdc26b\n' 2530 'size 11423554\n' 2531 ) 2532 2533 with open(f) as fh: 2534 self.assertEqual(lfs_file, fh.read()) 2535 2536 @skipIfNoGitLFS() 2537 def test_lfs_enabled(self): 2538 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir 2539 self.d.setVar('SRC_URI', uri) 2540 2541 # With git-lfs installed, test that we can fetch and unpack 2542 fetcher, ud = self.fetch() 2543 shutil.rmtree(self.gitdir, ignore_errors=True) 2544 fetcher.unpack(self.d.getVar('WORKDIR')) 2545 2546 @skipIfNoGitLFS() 2547 def test_lfs_disabled(self): 2548 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir 2549 self.d.setVar('SRC_URI', uri) 2550 2551 # Verify that the fetcher can survive even if the source 2552 # repository has Git LFS usage configured. 2553 fetcher, ud = self.fetch() 2554 fetcher.unpack(self.d.getVar('WORKDIR')) 2555 2556 @skipIfNoGitLFS() 2557 def test_lfs_enabled_not_installed_during_unpack(self): 2558 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir 2559 self.d.setVar('SRC_URI', uri) 2560 2561 # Careful: suppress initial attempt at downloading 2562 fetcher, ud = self.fetch(uri=None, download=False) 2563 2564 fetcher.download() 2565 # If git-lfs cannot be found, the unpack should throw an error 2566 with self.assertRaises(bb.fetch2.FetchError): 2567 with unittest.mock.patch("shutil.which", return_value=None): 2568 shutil.rmtree(self.gitdir, ignore_errors=True) 2569 fetcher.unpack(self.d.getVar('WORKDIR')) 2570 2571 def test_lfs_enabled_not_installed(self): 2572 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir 2573 self.d.setVar('SRC_URI', uri) 2574 2575 # Careful: suppress initial attempt at downloading 2576 fetcher, ud = self.fetch(uri=None, download=False) 2577 2578 # If git-lfs cannot be found, the download should throw an error 2579 with unittest.mock.patch("shutil.which", return_value=None): 2580 with self.assertRaises(bb.fetch2.FetchError): 2581 fetcher.download() 2582 2583 def test_lfs_disabled_not_installed(self): 2584 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir 2585 self.d.setVar('SRC_URI', uri) 2586 2587 # Careful: suppress initial attempt at downloading 2588 fetcher, ud = self.fetch(uri=None, download=False) 2589 2590 # Even if git-lfs cannot be found, the download / unpack should be successful 2591 with unittest.mock.patch("shutil.which", return_value=None): 2592 fetcher.download() 2593 shutil.rmtree(self.gitdir, ignore_errors=True) 2594 fetcher.unpack(self.d.getVar('WORKDIR')) 2595 2596 def test_lfs_enabled_not_installed_but_not_needed(self): 2597 srcdir = os.path.join(self.tempdir, "emptygit") 2598 bb.utils.mkdirhier(srcdir) 2599 self.git_init(srcdir) 2600 self.commit_file("test", "test content", cwd=srcdir) 2601 2602 uri = 'git://%s;protocol=file;lfs=1;branch=master' % srcdir 2603 self.d.setVar('SRC_URI', uri) 2604 2605 # Careful: suppress initial attempt at downloading 2606 fetcher, ud = self.fetch(uri=None, download=False) 2607 2608 # It shouldnt't matter that git-lfs cannot be found as the repository configuration does not 2609 # specify any LFS filters. 2610 with unittest.mock.patch("shutil.which", return_value=None): 2611 fetcher.download() 2612 shutil.rmtree(self.gitdir, ignore_errors=True) 2613 fetcher.unpack(self.d.getVar('WORKDIR')) 2614 2615class GitURLWithSpacesTest(FetcherTest): 2616 test_git_urls = { 2617 "git://tfs-example.org:22/tfs/example%20path/example.git;branch=master" : { 2618 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git;branch=master', 2619 'repo_url': 'git://tfs-example.org:22/tfs/example%20path/example.git', 2620 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git', 2621 'path': '/tfs/example path/example.git' 2622 }, 2623 "git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master" : { 2624 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master', 2625 'repo_url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git', 2626 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git', 2627 'path': '/tfs/example path/example repo.git' 2628 } 2629 } 2630 2631 def test_urls(self): 2632 2633 # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo 2634 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 2635 2636 for test_git_url, ref in self.test_git_urls.items(): 2637 2638 fetcher = bb.fetch.Fetch([test_git_url], self.d) 2639 ud = fetcher.ud[fetcher.urls[0]] 2640 2641 self.assertEqual(ud.url, ref['url']) 2642 self.assertEqual(ud.path, ref['path']) 2643 self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2644 self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2645 self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock')) 2646 self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2647 self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz')) 2648 self.assertEqual(ud.method._get_repo_url(ud), ref['repo_url']) 2649 2650class CrateTest(FetcherTest): 2651 @skipIfNoNetwork() 2652 def test_crate_url(self): 2653 2654 uri = "crate://crates.io/glob/0.2.11" 2655 self.d.setVar('SRC_URI', uri) 2656 2657 uris = self.d.getVar('SRC_URI').split() 2658 d = self.d 2659 2660 fetcher = bb.fetch2.Fetch(uris, self.d) 2661 ud = fetcher.ud[fetcher.urls[0]] 2662 2663 self.assertIn("name", ud.parm) 2664 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2665 self.assertIn("downloadfilename", ud.parm) 2666 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2667 2668 fetcher.download() 2669 fetcher.unpack(self.tempdir) 2670 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2671 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) 2672 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2673 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2674 2675 @skipIfNoNetwork() 2676 def test_crate_url_matching_recipe(self): 2677 2678 self.d.setVar('BP', 'glob-0.2.11') 2679 2680 uri = "crate://crates.io/glob/0.2.11" 2681 self.d.setVar('SRC_URI', uri) 2682 2683 uris = self.d.getVar('SRC_URI').split() 2684 d = self.d 2685 2686 fetcher = bb.fetch2.Fetch(uris, self.d) 2687 ud = fetcher.ud[fetcher.urls[0]] 2688 2689 self.assertIn("name", ud.parm) 2690 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2691 self.assertIn("downloadfilename", ud.parm) 2692 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2693 2694 fetcher.download() 2695 fetcher.unpack(self.tempdir) 2696 self.assertEqual(sorted(os.listdir(self.tempdir)), ['download', 'glob-0.2.11', 'unpacked']) 2697 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) 2698 self.assertTrue(os.path.exists(self.tempdir + "/glob-0.2.11/src/lib.rs")) 2699 2700 @skipIfNoNetwork() 2701 def test_crate_url_params(self): 2702 2703 uri = "crate://crates.io/aho-corasick/0.7.20;name=aho-corasick-renamed" 2704 self.d.setVar('SRC_URI', uri) 2705 2706 uris = self.d.getVar('SRC_URI').split() 2707 d = self.d 2708 2709 fetcher = bb.fetch2.Fetch(uris, self.d) 2710 ud = fetcher.ud[fetcher.urls[0]] 2711 2712 self.assertIn("name", ud.parm) 2713 self.assertEqual(ud.parm["name"], "aho-corasick-renamed") 2714 self.assertIn("downloadfilename", ud.parm) 2715 self.assertEqual(ud.parm["downloadfilename"], "aho-corasick-0.7.20.crate") 2716 2717 fetcher.download() 2718 fetcher.unpack(self.tempdir) 2719 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2720 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['aho-corasick-0.7.20.crate', 'aho-corasick-0.7.20.crate.done']) 2721 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/.cargo-checksum.json")) 2722 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/src/lib.rs")) 2723 2724 @skipIfNoNetwork() 2725 def test_crate_url_multi(self): 2726 2727 uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35" 2728 self.d.setVar('SRC_URI', uri) 2729 2730 uris = self.d.getVar('SRC_URI').split() 2731 d = self.d 2732 2733 fetcher = bb.fetch2.Fetch(uris, self.d) 2734 ud = fetcher.ud[fetcher.urls[0]] 2735 2736 self.assertIn("name", ud.parm) 2737 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2738 self.assertIn("downloadfilename", ud.parm) 2739 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2740 2741 ud = fetcher.ud[fetcher.urls[1]] 2742 self.assertIn("name", ud.parm) 2743 self.assertEqual(ud.parm["name"], "time-0.1.35") 2744 self.assertIn("downloadfilename", ud.parm) 2745 self.assertEqual(ud.parm["downloadfilename"], "time-0.1.35.crate") 2746 2747 fetcher.download() 2748 fetcher.unpack(self.tempdir) 2749 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2750 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done', 'time-0.1.35.crate', 'time-0.1.35.crate.done']) 2751 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2752 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2753 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json")) 2754 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs")) 2755 2756 @skipIfNoNetwork() 2757 def test_crate_incorrect_cksum(self): 2758 uri = "crate://crates.io/aho-corasick/0.7.20" 2759 self.d.setVar('SRC_URI', uri) 2760 self.d.setVarFlag("SRC_URI", "aho-corasick-0.7.20.sha256sum", hashlib.sha256("Invalid".encode("utf-8")).hexdigest()) 2761 2762 uris = self.d.getVar('SRC_URI').split() 2763 2764 fetcher = bb.fetch2.Fetch(uris, self.d) 2765 with self.assertRaisesRegex(bb.fetch2.FetchError, "Fetcher failure for URL"): 2766 fetcher.download() 2767 2768class NPMTest(FetcherTest): 2769 def skipIfNoNpm(): 2770 if not shutil.which('npm'): 2771 return unittest.skip('npm not installed') 2772 return lambda f: f 2773 2774 @skipIfNoNpm() 2775 @skipIfNoNetwork() 2776 def test_npm(self): 2777 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2778 fetcher = bb.fetch.Fetch(urls, self.d) 2779 ud = fetcher.ud[fetcher.urls[0]] 2780 fetcher.download() 2781 self.assertTrue(os.path.exists(ud.localpath)) 2782 self.assertTrue(os.path.exists(ud.localpath + '.done')) 2783 self.assertTrue(os.path.exists(ud.resolvefile)) 2784 fetcher.unpack(self.unpackdir) 2785 unpackdir = os.path.join(self.unpackdir, 'npm') 2786 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2787 2788 @skipIfNoNpm() 2789 @skipIfNoNetwork() 2790 def test_npm_bad_checksum(self): 2791 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2792 # Fetch once to get a tarball 2793 fetcher = bb.fetch.Fetch(urls, self.d) 2794 ud = fetcher.ud[fetcher.urls[0]] 2795 fetcher.download() 2796 self.assertTrue(os.path.exists(ud.localpath)) 2797 # Modify the tarball 2798 bad = b'bad checksum' 2799 with open(ud.localpath, 'wb') as f: 2800 f.write(bad) 2801 # Verify that the tarball is fetched again 2802 fetcher.download() 2803 badsum = hashlib.sha512(bad).hexdigest() 2804 self.assertTrue(os.path.exists(ud.localpath + '_bad-checksum_' + badsum)) 2805 self.assertTrue(os.path.exists(ud.localpath)) 2806 2807 @skipIfNoNpm() 2808 @skipIfNoNetwork() 2809 def test_npm_premirrors(self): 2810 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2811 # Fetch once to get a tarball 2812 fetcher = bb.fetch.Fetch(urls, self.d) 2813 ud = fetcher.ud[fetcher.urls[0]] 2814 fetcher.download() 2815 self.assertTrue(os.path.exists(ud.localpath)) 2816 2817 # Setup the mirror by renaming the download directory 2818 mirrordir = os.path.join(self.tempdir, 'mirror') 2819 bb.utils.rename(self.dldir, mirrordir) 2820 os.mkdir(self.dldir) 2821 2822 # Configure the premirror to be used 2823 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/npm2' % mirrordir) 2824 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2825 2826 # Fetch again 2827 self.assertFalse(os.path.exists(ud.localpath)) 2828 # The npm fetcher doesn't handle that the .resolved file disappears 2829 # while the fetcher object exists, which it does when we rename the 2830 # download directory to "mirror" above. Thus we need a new fetcher to go 2831 # with the now empty download directory. 2832 fetcher = bb.fetch.Fetch(urls, self.d) 2833 ud = fetcher.ud[fetcher.urls[0]] 2834 fetcher.download() 2835 self.assertTrue(os.path.exists(ud.localpath)) 2836 2837 @skipIfNoNpm() 2838 @skipIfNoNetwork() 2839 def test_npm_premirrors_with_specified_filename(self): 2840 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2841 # Fetch once to get a tarball 2842 fetcher = bb.fetch.Fetch(urls, self.d) 2843 ud = fetcher.ud[fetcher.urls[0]] 2844 fetcher.download() 2845 self.assertTrue(os.path.exists(ud.localpath)) 2846 # Setup the mirror 2847 mirrordir = os.path.join(self.tempdir, 'mirror') 2848 bb.utils.mkdirhier(mirrordir) 2849 mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath)) 2850 os.replace(ud.localpath, mirrorfilename) 2851 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s' % mirrorfilename) 2852 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2853 # Fetch again 2854 self.assertFalse(os.path.exists(ud.localpath)) 2855 fetcher.download() 2856 self.assertTrue(os.path.exists(ud.localpath)) 2857 2858 @skipIfNoNpm() 2859 @skipIfNoNetwork() 2860 def test_npm_mirrors(self): 2861 # Fetch once to get a tarball 2862 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2863 fetcher = bb.fetch.Fetch(urls, self.d) 2864 ud = fetcher.ud[fetcher.urls[0]] 2865 fetcher.download() 2866 self.assertTrue(os.path.exists(ud.localpath)) 2867 # Setup the mirror 2868 mirrordir = os.path.join(self.tempdir, 'mirror') 2869 bb.utils.mkdirhier(mirrordir) 2870 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 2871 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 2872 # Update the resolved url to an invalid url 2873 with open(ud.resolvefile, 'r') as f: 2874 url = f.read() 2875 uri = URI(url) 2876 uri.path = '/invalid' 2877 with open(ud.resolvefile, 'w') as f: 2878 f.write(str(uri)) 2879 # Fetch again 2880 self.assertFalse(os.path.exists(ud.localpath)) 2881 fetcher.download() 2882 self.assertTrue(os.path.exists(ud.localpath)) 2883 2884 @skipIfNoNpm() 2885 @skipIfNoNetwork() 2886 def test_npm_destsuffix_downloadfilename(self): 2887 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz'] 2888 fetcher = bb.fetch.Fetch(urls, self.d) 2889 fetcher.download() 2890 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'foo-bar.tgz'))) 2891 fetcher.unpack(self.unpackdir) 2892 unpackdir = os.path.join(self.unpackdir, 'foo', 'bar') 2893 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2894 2895 def test_npm_no_network_no_tarball(self): 2896 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2897 self.d.setVar('BB_NO_NETWORK', '1') 2898 fetcher = bb.fetch.Fetch(urls, self.d) 2899 with self.assertRaises(bb.fetch2.NetworkAccess): 2900 fetcher.download() 2901 2902 @skipIfNoNpm() 2903 @skipIfNoNetwork() 2904 def test_npm_no_network_with_tarball(self): 2905 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2906 # Fetch once to get a tarball 2907 fetcher = bb.fetch.Fetch(urls, self.d) 2908 fetcher.download() 2909 # Disable network access 2910 self.d.setVar('BB_NO_NETWORK', '1') 2911 # Fetch again 2912 fetcher.download() 2913 fetcher.unpack(self.unpackdir) 2914 unpackdir = os.path.join(self.unpackdir, 'npm') 2915 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2916 2917 @skipIfNoNpm() 2918 @skipIfNoNetwork() 2919 def test_npm_registry_alternate(self): 2920 urls = ['npm://skimdb.npmjs.com;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2921 fetcher = bb.fetch.Fetch(urls, self.d) 2922 fetcher.download() 2923 fetcher.unpack(self.unpackdir) 2924 unpackdir = os.path.join(self.unpackdir, 'npm') 2925 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2926 2927 @skipIfNoNpm() 2928 @skipIfNoNetwork() 2929 def test_npm_version_latest(self): 2930 url = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest'] 2931 fetcher = bb.fetch.Fetch(url, self.d) 2932 fetcher.download() 2933 fetcher.unpack(self.unpackdir) 2934 unpackdir = os.path.join(self.unpackdir, 'npm') 2935 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2936 2937 @skipIfNoNpm() 2938 @skipIfNoNetwork() 2939 def test_npm_registry_invalid(self): 2940 urls = ['npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2941 fetcher = bb.fetch.Fetch(urls, self.d) 2942 with self.assertRaises(bb.fetch2.FetchError): 2943 fetcher.download() 2944 2945 @skipIfNoNpm() 2946 @skipIfNoNetwork() 2947 def test_npm_package_invalid(self): 2948 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0'] 2949 fetcher = bb.fetch.Fetch(urls, self.d) 2950 with self.assertRaises(bb.fetch2.FetchError): 2951 fetcher.download() 2952 2953 @skipIfNoNpm() 2954 @skipIfNoNetwork() 2955 def test_npm_version_invalid(self): 2956 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid'] 2957 with self.assertRaises(bb.fetch2.ParameterError): 2958 fetcher = bb.fetch.Fetch(urls, self.d) 2959 2960 @skipIfNoNpm() 2961 @skipIfNoNetwork() 2962 def test_npm_registry_none(self): 2963 urls = ['npm://;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2964 with self.assertRaises(bb.fetch2.MalformedUrl): 2965 fetcher = bb.fetch.Fetch(urls, self.d) 2966 2967 @skipIfNoNpm() 2968 @skipIfNoNetwork() 2969 def test_npm_package_none(self): 2970 urls = ['npm://registry.npmjs.org;version=1.0.0'] 2971 with self.assertRaises(bb.fetch2.MissingParameterError): 2972 fetcher = bb.fetch.Fetch(urls, self.d) 2973 2974 @skipIfNoNpm() 2975 @skipIfNoNetwork() 2976 def test_npm_version_none(self): 2977 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example'] 2978 with self.assertRaises(bb.fetch2.MissingParameterError): 2979 fetcher = bb.fetch.Fetch(urls, self.d) 2980 2981 def create_shrinkwrap_file(self, data): 2982 import json 2983 datadir = os.path.join(self.tempdir, 'data') 2984 swfile = os.path.join(datadir, 'npm-shrinkwrap.json') 2985 bb.utils.mkdirhier(datadir) 2986 with open(swfile, 'w') as f: 2987 json.dump(data, f) 2988 return swfile 2989 2990 @skipIfNoNetwork() 2991 def test_npmsw(self): 2992 swfile = self.create_shrinkwrap_file({ 2993 'packages': { 2994 'node_modules/array-flatten': { 2995 'version': '1.1.1', 2996 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2997 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=', 2998 'dependencies': { 2999 'content-type': "1.0.4" 3000 } 3001 }, 3002 'node_modules/array-flatten/node_modules/content-type': { 3003 'version': '1.0.4', 3004 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 3005 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 3006 'dependencies': { 3007 'cookie': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09' 3008 } 3009 }, 3010 'node_modules/array-flatten/node_modules/content-type/node_modules/cookie': { 3011 'resolved': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09' 3012 } 3013 } 3014 }) 3015 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3016 fetcher.download() 3017 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 3018 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 3019 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 3020 fetcher.unpack(self.unpackdir) 3021 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm-shrinkwrap.json'))) 3022 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'package.json'))) 3023 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json'))) 3024 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json'))) 3025 3026 @skipIfNoNetwork() 3027 def test_npmsw_git(self): 3028 swfile = self.create_shrinkwrap_file({ 3029 'packages': { 3030 'node_modules/cookie': { 3031 'resolved': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09' 3032 } 3033 } 3034 }) 3035 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3036 fetcher.download() 3037 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 3038 3039 @skipIfNoNetwork() 3040 def test_npmsw_dev(self): 3041 swfile = self.create_shrinkwrap_file({ 3042 'packages': { 3043 'node_modules/array-flatten': { 3044 'version': '1.1.1', 3045 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3046 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3047 }, 3048 'node_modules/content-type': { 3049 'version': '1.0.4', 3050 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 3051 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 3052 'dev': True 3053 } 3054 } 3055 }) 3056 # Fetch with dev disabled 3057 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3058 fetcher.download() 3059 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 3060 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 3061 # Fetch with dev enabled 3062 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d) 3063 fetcher.download() 3064 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 3065 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 3066 3067 @skipIfNoNetwork() 3068 def test_npmsw_destsuffix(self): 3069 swfile = self.create_shrinkwrap_file({ 3070 'packages': { 3071 'node_modules/array-flatten': { 3072 'version': '1.1.1', 3073 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3074 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3075 } 3076 } 3077 }) 3078 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d) 3079 fetcher.download() 3080 fetcher.unpack(self.unpackdir) 3081 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json'))) 3082 3083 def test_npmsw_no_network_no_tarball(self): 3084 swfile = self.create_shrinkwrap_file({ 3085 'packages': { 3086 'node_modules/array-flatten': { 3087 'version': '1.1.1', 3088 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3089 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3090 } 3091 } 3092 }) 3093 self.d.setVar('BB_NO_NETWORK', '1') 3094 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3095 with self.assertRaises(bb.fetch2.NetworkAccess): 3096 fetcher.download() 3097 3098 @skipIfNoNpm() 3099 @skipIfNoNetwork() 3100 def test_npmsw_no_network_with_tarball(self): 3101 # Fetch once to get a tarball 3102 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 3103 fetcher.download() 3104 # Disable network access 3105 self.d.setVar('BB_NO_NETWORK', '1') 3106 # Fetch again 3107 swfile = self.create_shrinkwrap_file({ 3108 'packages': { 3109 'node_modules/array-flatten': { 3110 'version': '1.1.1', 3111 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3112 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3113 } 3114 } 3115 }) 3116 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3117 fetcher.download() 3118 fetcher.unpack(self.unpackdir) 3119 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'package.json'))) 3120 3121 @skipIfNoNetwork() 3122 def test_npmsw_npm_reusability(self): 3123 # Fetch once with npmsw 3124 swfile = self.create_shrinkwrap_file({ 3125 'packages': { 3126 'node_modules/array-flatten': { 3127 'version': '1.1.1', 3128 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3129 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3130 } 3131 } 3132 }) 3133 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3134 fetcher.download() 3135 # Disable network access 3136 self.d.setVar('BB_NO_NETWORK', '1') 3137 # Fetch again with npm 3138 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 3139 fetcher.download() 3140 fetcher.unpack(self.unpackdir) 3141 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json'))) 3142 3143 @skipIfNoNetwork() 3144 def test_npmsw_bad_checksum(self): 3145 # Try to fetch with bad checksum 3146 swfile = self.create_shrinkwrap_file({ 3147 'packages': { 3148 'node_modules/array-flatten': { 3149 'version': '1.1.1', 3150 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3151 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg=' 3152 } 3153 } 3154 }) 3155 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3156 with self.assertRaises(bb.fetch2.FetchError): 3157 fetcher.download() 3158 # Fetch correctly to get a tarball 3159 swfile = self.create_shrinkwrap_file({ 3160 'packages': { 3161 'node_modules/array-flatten': { 3162 'version': '1.1.1', 3163 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3164 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3165 } 3166 } 3167 }) 3168 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3169 fetcher.download() 3170 localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz') 3171 self.assertTrue(os.path.exists(localpath)) 3172 # Modify the tarball 3173 bad = b'bad checksum' 3174 with open(localpath, 'wb') as f: 3175 f.write(bad) 3176 # Verify that the tarball is fetched again 3177 fetcher.download() 3178 badsum = hashlib.sha1(bad).hexdigest() 3179 self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum)) 3180 self.assertTrue(os.path.exists(localpath)) 3181 3182 @skipIfNoNpm() 3183 @skipIfNoNetwork() 3184 def test_npmsw_premirrors(self): 3185 # Fetch once to get a tarball 3186 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 3187 ud = fetcher.ud[fetcher.urls[0]] 3188 fetcher.download() 3189 self.assertTrue(os.path.exists(ud.localpath)) 3190 # Setup the mirror 3191 mirrordir = os.path.join(self.tempdir, 'mirror') 3192 bb.utils.mkdirhier(mirrordir) 3193 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 3194 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 3195 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 3196 # Fetch again 3197 self.assertFalse(os.path.exists(ud.localpath)) 3198 swfile = self.create_shrinkwrap_file({ 3199 'packages': { 3200 'node_modules/array-flatten': { 3201 'version': '1.1.1', 3202 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3203 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3204 } 3205 } 3206 }) 3207 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3208 fetcher.download() 3209 self.assertTrue(os.path.exists(ud.localpath)) 3210 3211 @skipIfNoNpm() 3212 @skipIfNoNetwork() 3213 def test_npmsw_mirrors(self): 3214 # Fetch once to get a tarball 3215 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 3216 ud = fetcher.ud[fetcher.urls[0]] 3217 fetcher.download() 3218 self.assertTrue(os.path.exists(ud.localpath)) 3219 # Setup the mirror 3220 mirrordir = os.path.join(self.tempdir, 'mirror') 3221 bb.utils.mkdirhier(mirrordir) 3222 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 3223 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 3224 # Fetch again with invalid url 3225 self.assertFalse(os.path.exists(ud.localpath)) 3226 swfile = self.create_shrinkwrap_file({ 3227 'packages': { 3228 'node_modules/array-flatten': { 3229 'version': '1.1.1', 3230 'resolved': 'https://invalid', 3231 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3232 } 3233 } 3234 }) 3235 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3236 fetcher.download() 3237 self.assertTrue(os.path.exists(ud.localpath)) 3238 3239 @skipIfNoNetwork() 3240 def test_npmsw_bundled(self): 3241 swfile = self.create_shrinkwrap_file({ 3242 'packages': { 3243 'node_modules/array-flatten': { 3244 'version': '1.1.1', 3245 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3246 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3247 }, 3248 'node_modules/content-type': { 3249 'version': '1.0.4', 3250 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 3251 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 3252 'inBundle': True 3253 } 3254 } 3255 }) 3256 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3257 fetcher.download() 3258 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 3259 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 3260 3261class GitSharedTest(FetcherTest): 3262 def setUp(self): 3263 super(GitSharedTest, self).setUp() 3264 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 3265 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 3266 self.d.setVar("__BBSRCREV_SEEN", "1") 3267 3268 @skipIfNoNetwork() 3269 def test_shared_unpack(self): 3270 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3271 3272 fetcher.download() 3273 fetcher.unpack(self.unpackdir) 3274 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 3275 self.assertTrue(os.path.exists(alt)) 3276 3277 @skipIfNoNetwork() 3278 def test_noshared_unpack(self): 3279 self.d.setVar('BB_GIT_NOSHARED', '1') 3280 self.unpackdir += '_noshared' 3281 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3282 3283 fetcher.download() 3284 fetcher.unpack(self.unpackdir) 3285 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 3286 self.assertFalse(os.path.exists(alt)) 3287 3288class GitTagVerificationTests(FetcherTest): 3289 3290 @skipIfNoNetwork() 3291 def test_tag_rev_match(self): 3292 # Test a url with rev= and tag= set works 3293 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;rev=aa0e540fc31a1c26839efd2c7785a751ce24ebfb;tag=2.8.7"], self.d) 3294 fetcher.download() 3295 fetcher.unpack(self.unpackdir) 3296 3297 def test_annotated_tag_rev_match(self): 3298 # Test a url with rev= and tag= set works 3299 # rev is the annotated tag revision in this case 3300 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;rev=6d363159e4b7dc566fc40d069b2615e61774a7d8;tag=2.8.7"], self.d) 3301 fetcher.download() 3302 fetcher.unpack(self.unpackdir) 3303 3304 @skipIfNoNetwork() 3305 def test_tag_rev_match2(self): 3306 # Test a url with SRCREV and tag= set works 3307 self.d.setVar('SRCREV', 'aa0e540fc31a1c26839efd2c7785a751ce24ebfb') 3308 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;tag=2.8.7"], self.d) 3309 fetcher.download() 3310 fetcher.unpack(self.unpackdir) 3311 3312 @skipIfNoNetwork() 3313 def test_tag_rev_match3(self): 3314 # Test a url with SRCREV, rev= and tag= set works 3315 self.d.setVar('SRCREV', 'aa0e540fc31a1c26839efd2c7785a751ce24ebfb') 3316 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;rev=aa0e540fc31a1c26839efd2c7785a751ce24ebfb;tag=2.8.7"], self.d) 3317 fetcher.download() 3318 fetcher.unpack(self.unpackdir) 3319 3320 @skipIfNoNetwork() 3321 def test_tag_rev_match4(self): 3322 # Test a url with SRCREV and rev= mismatching errors 3323 self.d.setVar('SRCREV', 'bade540fc31a1c26839efd2c7785a751ce24ebfb') 3324 with self.assertRaises(bb.fetch2.FetchError): 3325 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;rev=aa0e540fc31a1c26839efd2c7785a751ce24ebfb;tag=2.8.7"], self.d) 3326 3327 @skipIfNoNetwork() 3328 def test_tag_rev_match5(self): 3329 # Test a url with SRCREV, rev= and tag= set works when using shallow clones 3330 self.d.setVar('BB_GIT_SHALLOW', '1') 3331 self.d.setVar('SRCREV', 'aa0e540fc31a1c26839efd2c7785a751ce24ebfb') 3332 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;rev=aa0e540fc31a1c26839efd2c7785a751ce24ebfb;tag=2.8.7"], self.d) 3333 fetcher.download() 3334 fetcher.unpack(self.unpackdir) 3335 3336 @skipIfNoNetwork() 3337 def test_tag_rev_match6(self): 3338 # Test a url with SRCREV, rev= and a mismatched tag= when using shallow clones 3339 self.d.setVar('BB_GIT_SHALLOW', '1') 3340 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;rev=aa0e540fc31a1c26839efd2c7785a751ce24ebfb;tag=2.8.6"], self.d) 3341 fetcher.download() 3342 with self.assertRaises(bb.fetch2.FetchError): 3343 fetcher.unpack(self.unpackdir) 3344 3345 @skipIfNoNetwork() 3346 def test_tag_rev_match7(self): 3347 # Test a url with SRCREV, rev= and a mismatched tag= 3348 self.d.setVar('SRCREV', 'aa0e540fc31a1c26839efd2c7785a751ce24ebfb') 3349 fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;rev=aa0e540fc31a1c26839efd2c7785a751ce24ebfb;tag=2.8.6"], self.d) 3350 fetcher.download() 3351 with self.assertRaises(bb.fetch2.FetchError): 3352 fetcher.unpack(self.unpackdir) 3353 3354 3355class FetchPremirroronlyLocalTest(FetcherTest): 3356 3357 def setUp(self): 3358 super(FetchPremirroronlyLocalTest, self).setUp() 3359 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3360 os.mkdir(self.mirrordir) 3361 self.reponame = "bitbake" 3362 self.gitdir = os.path.join(self.tempdir, "git", self.reponame) 3363 self.recipe_url = "git://git.fake.repo/bitbake;branch=master;protocol=https" 3364 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3365 self.d.setVar("BB_NO_NETWORK", "1") 3366 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3367 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" 3368 self.mirrorfile = os.path.join(self.mirrordir, self.mirrorname) 3369 self.testfilename = "bitbake-fetch.test" 3370 3371 def make_git_repo(self): 3372 recipeurl = "git:/git.fake.repo/bitbake" 3373 os.makedirs(self.gitdir) 3374 self.git_init(cwd=self.gitdir) 3375 for i in range(0): 3376 self.git_new_commit() 3377 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3378 3379 def git_new_commit(self): 3380 import random 3381 os.unlink(os.path.join(self.mirrordir, self.mirrorname)) 3382 branch = self.git("branch --show-current", self.gitdir).split() 3383 with open(os.path.join(self.gitdir, self.testfilename), "w") as testfile: 3384 testfile.write("File {} from branch {}; Useless random data {}".format(self.testfilename, branch, random.random())) 3385 self.git("add {}".format(self.testfilename), self.gitdir) 3386 self.git("commit -a -m \"This random commit {} in branch {}. I'm useless.\"".format(random.random(), branch), self.gitdir) 3387 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3388 return self.git("rev-parse HEAD", self.gitdir).strip() 3389 3390 def git_new_branch(self, name): 3391 self.git_new_commit() 3392 head = self.git("rev-parse HEAD", self.gitdir).strip() 3393 self.git("checkout -b {}".format(name), self.gitdir) 3394 newrev = self.git_new_commit() 3395 self.git("checkout {}".format(head), self.gitdir) 3396 return newrev 3397 3398 def test_mirror_multiple_fetches(self): 3399 self.make_git_repo() 3400 self.d.setVar("SRCREV", self.git_new_commit()) 3401 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3402 fetcher.download() 3403 fetcher.unpack(self.unpackdir) 3404 ## New commit in premirror. it's not in the download_dir 3405 self.d.setVar("SRCREV", self.git_new_commit()) 3406 fetcher2 = bb.fetch.Fetch([self.recipe_url], self.d) 3407 fetcher2.download() 3408 fetcher2.unpack(self.unpackdir) 3409 ## New commit in premirror. it's not in the download_dir 3410 self.d.setVar("SRCREV", self.git_new_commit()) 3411 fetcher3 = bb.fetch.Fetch([self.recipe_url], self.d) 3412 fetcher3.download() 3413 fetcher3.unpack(self.unpackdir) 3414 3415 3416 def test_mirror_commit_nonexistent(self): 3417 self.make_git_repo() 3418 self.d.setVar("SRCREV", "0"*40) 3419 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3420 with self.assertRaises(bb.fetch2.NetworkAccess): 3421 fetcher.download() 3422 3423 def test_mirror_commit_exists(self): 3424 self.make_git_repo() 3425 self.d.setVar("SRCREV", self.git_new_commit()) 3426 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3427 fetcher.download() 3428 fetcher.unpack(self.unpackdir) 3429 3430 def test_mirror_tarball_nonexistent(self): 3431 self.d.setVar("SRCREV", "0"*40) 3432 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3433 with self.assertRaises(bb.fetch2.NetworkAccess): 3434 fetcher.download() 3435 3436 3437class FetchPremirroronlyNetworkTest(FetcherTest): 3438 3439 def setUp(self): 3440 super(FetchPremirroronlyNetworkTest, self).setUp() 3441 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3442 os.mkdir(self.mirrordir) 3443 self.reponame = "fstests" 3444 self.clonedir = os.path.join(self.tempdir, "git") 3445 self.gitdir = os.path.join(self.tempdir, "git", "{}.git".format(self.reponame)) 3446 self.recipe_url = "git://git.yoctoproject.org/fstests;protocol=https;branch=master" 3447 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3448 self.d.setVar("BB_NO_NETWORK", "0") 3449 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3450 3451 def make_git_repo(self): 3452 self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz" 3453 os.makedirs(self.clonedir) 3454 self.git("clone --bare {}".format(self.recipe_url), self.clonedir) 3455 self.git("update-ref HEAD 15413486df1f5a5b5af699b6f3ba5f0984e52a9f", self.gitdir) 3456 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3457 shutil.rmtree(self.clonedir) 3458 3459 @skipIfNoNetwork() 3460 def test_mirror_tarball_updated(self): 3461 self.make_git_repo() 3462 ## Upstream commit is in the mirror 3463 self.d.setVar("SRCREV", "15413486df1f5a5b5af699b6f3ba5f0984e52a9f") 3464 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3465 fetcher.download() 3466 3467 @skipIfNoNetwork() 3468 def test_mirror_tarball_outdated(self): 3469 self.make_git_repo() 3470 ## Upstream commit not in the mirror 3471 self.d.setVar("SRCREV", "49d65d53c2bf558ae6e9185af0f3af7b79d255ec") 3472 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3473 with self.assertRaises(bb.fetch2.NetworkAccess): 3474 fetcher.download() 3475 3476class FetchPremirroronlyMercurialTest(FetcherTest): 3477 """ Test for premirrors with mercurial repos 3478 the test covers also basic hg:// clone (see fetch_and_create_tarball 3479 """ 3480 def skipIfNoHg(): 3481 if not shutil.which('hg'): 3482 return unittest.skip('Mercurial not installed') 3483 return lambda f: f 3484 3485 def setUp(self): 3486 super(FetchPremirroronlyMercurialTest, self).setUp() 3487 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3488 os.mkdir(self.mirrordir) 3489 self.reponame = "libgnt" 3490 self.clonedir = os.path.join(self.tempdir, "hg") 3491 self.recipe_url = "hg://keep.imfreedom.org/libgnt;module=libgnt" 3492 self.d.setVar("SRCREV", "53e8b422faaf") 3493 self.mirrorname = "hg_libgnt_keep.imfreedom.org_.libgnt.tar.gz" 3494 3495 def fetch_and_create_tarball(self): 3496 """ 3497 Ask bitbake to download repo and prepare mirror tarball for us 3498 """ 3499 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 3500 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3501 fetcher.download() 3502 mirrorfile = os.path.join(self.d.getVar("DL_DIR"), self.mirrorname) 3503 self.assertTrue(os.path.exists(mirrorfile), "Mirror tarball {} has not been created".format(mirrorfile)) 3504 ## moving tarball to mirror directory 3505 os.rename(mirrorfile, os.path.join(self.mirrordir, self.mirrorname)) 3506 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "0") 3507 3508 3509 @skipIfNoNetwork() 3510 @skipIfNoHg() 3511 def test_premirror_mercurial(self): 3512 self.fetch_and_create_tarball() 3513 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3514 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3515 self.d.setVar("BB_NO_NETWORK", "1") 3516 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3517 fetcher.download() 3518 3519class FetchPremirroronlyBrokenTarball(FetcherTest): 3520 3521 def setUp(self): 3522 super(FetchPremirroronlyBrokenTarball, self).setUp() 3523 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3524 os.mkdir(self.mirrordir) 3525 self.reponame = "bitbake" 3526 self.gitdir = os.path.join(self.tempdir, "git", self.reponame) 3527 self.recipe_url = "git://git.fake.repo/bitbake;protocol=https;branch=master" 3528 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3529 self.d.setVar("BB_NO_NETWORK", "1") 3530 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3531 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" 3532 with open(os.path.join(self.mirrordir, self.mirrorname), 'w') as targz: 3533 targz.write("This is not tar.gz file!") 3534 3535 def test_mirror_broken_download(self): 3536 self.d.setVar("SRCREV", "0"*40) 3537 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3538 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs() as logs: 3539 fetcher.download() 3540 output = "".join(logs.output) 3541 self.assertFalse(" not a git repository (or any parent up to mount point /)" in output) 3542 3543class GoModTest(FetcherTest): 3544 3545 @skipIfNoNetwork() 3546 def test_gomod_url(self): 3547 urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;' 3548 'sha256sum=9bb69aea32f1d59711701f9562d66432c9c0374205e5009d1d1a62f03fb4fdad'] 3549 3550 fetcher = bb.fetch2.Fetch(urls, self.d) 3551 ud = fetcher.ud[urls[0]] 3552 self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.zip') 3553 self.assertEqual(ud.parm['downloadfilename'], 'github.com.Azure.azure-sdk-for-go.sdk.storage.azblob@v1.0.0.zip') 3554 self.assertEqual(ud.parm['name'], 'github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0') 3555 3556 fetcher.download() 3557 fetcher.unpack(self.unpackdir) 3558 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3559 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip'))) 3560 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod'))) 3561 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')), 3562 '7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873') 3563 3564 @skipIfNoNetwork() 3565 def test_gomod_url_go_mod_only(self): 3566 urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;mod=1;' 3567 'sha256sum=7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873'] 3568 3569 fetcher = bb.fetch2.Fetch(urls, self.d) 3570 ud = fetcher.ud[urls[0]] 3571 self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.mod') 3572 self.assertEqual(ud.parm['downloadfilename'], 'github.com.Azure.azure-sdk-for-go.sdk.storage.azblob@v1.0.0.mod') 3573 self.assertEqual(ud.parm['name'], 'github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0') 3574 3575 fetcher.download() 3576 fetcher.unpack(self.unpackdir) 3577 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3578 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod'))) 3579 3580 @skipIfNoNetwork() 3581 def test_gomod_url_sha256sum_varflag(self): 3582 urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0'] 3583 self.d.setVarFlag('SRC_URI', 'gopkg.in/ini.v1@v1.67.0.sha256sum', 'bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6') 3584 3585 fetcher = bb.fetch2.Fetch(urls, self.d) 3586 ud = fetcher.ud[urls[0]] 3587 self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip') 3588 self.assertEqual(ud.parm['downloadfilename'], 'gopkg.in.ini.v1@v1.67.0.zip') 3589 self.assertEqual(ud.parm['name'], 'gopkg.in/ini.v1@v1.67.0') 3590 3591 fetcher.download() 3592 fetcher.unpack(self.unpackdir) 3593 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3594 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3595 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3596 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3597 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3598 3599 @skipIfNoNetwork() 3600 def test_gomod_url_no_go_mod_in_module(self): 3601 urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0;' 3602 'sha256sum=bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6'] 3603 3604 fetcher = bb.fetch2.Fetch(urls, self.d) 3605 ud = fetcher.ud[urls[0]] 3606 self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip') 3607 self.assertEqual(ud.parm['downloadfilename'], 'gopkg.in.ini.v1@v1.67.0.zip') 3608 self.assertEqual(ud.parm['name'], 'gopkg.in/ini.v1@v1.67.0') 3609 3610 fetcher.download() 3611 fetcher.unpack(self.unpackdir) 3612 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3613 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3614 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3615 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3616 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3617 3618 @skipIfNoNetwork() 3619 def test_gomod_url_host_only(self): 3620 urls = ['gomod://go.opencensus.io;version=v0.24.0;' 3621 'sha256sum=203a767d7f8e7c1ebe5588220ad168d1e15b14ae70a636de7ca9a4a88a7e0d0c'] 3622 3623 fetcher = bb.fetch2.Fetch(urls, self.d) 3624 ud = fetcher.ud[urls[0]] 3625 self.assertEqual(ud.url, 'https://proxy.golang.org/go.opencensus.io/%40v/v0.24.0.zip') 3626 self.assertEqual(ud.parm['downloadfilename'], 'go.opencensus.io@v0.24.0.zip') 3627 self.assertEqual(ud.parm['name'], 'go.opencensus.io@v0.24.0') 3628 3629 fetcher.download() 3630 fetcher.unpack(self.unpackdir) 3631 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3632 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.zip'))) 3633 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod'))) 3634 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')), 3635 '0dc9ccc660ad21cebaffd548f2cc6efa27891c68b4fbc1f8a3893b00f1acec96') 3636 3637class GoModGitTest(FetcherTest): 3638 3639 @skipIfNoNetwork() 3640 def test_gomodgit_url_repo(self): 3641 urls = ['gomodgit://golang.org/x/net;version=v0.9.0;' 3642 'repo=go.googlesource.com/net;' 3643 'srcrev=694cff8668bac64e0864b552bffc280cd27f21b1'] 3644 3645 fetcher = bb.fetch2.Fetch(urls, self.d) 3646 ud = fetcher.ud[urls[0]] 3647 self.assertEqual(ud.host, 'go.googlesource.com') 3648 self.assertEqual(ud.path, '/net') 3649 self.assertEqual(ud.name, 'golang.org/x/net@v0.9.0') 3650 self.assertEqual(self.d.getVar('SRCREV_golang.org/x/net@v0.9.0'), '694cff8668bac64e0864b552bffc280cd27f21b1') 3651 3652 fetcher.download() 3653 self.assertTrue(os.path.exists(ud.localpath)) 3654 3655 fetcher.unpack(self.unpackdir) 3656 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3657 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'ed42bd05533fd84ae290a5d33ebd3695a0a2b06131beebd5450825bee8603aca'))) 3658 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3659 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.zip'))) 3660 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod'))) 3661 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod')), 3662 'c5d6851ede50ec1c001afb763040194b68961bf06997e2605e8bf06dcd2aeb2e') 3663 3664 @skipIfNoNetwork() 3665 def test_gomodgit_url_subdir(self): 3666 urls = ['gomodgit://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;' 3667 'repo=github.com/Azure/azure-sdk-for-go;subdir=sdk/storage/azblob;' 3668 'srcrev=ec928e0ed34db682b3f783d3739d1c538142e0c3'] 3669 3670 fetcher = bb.fetch2.Fetch(urls, self.d) 3671 ud = fetcher.ud[urls[0]] 3672 self.assertEqual(ud.host, 'github.com') 3673 self.assertEqual(ud.path, '/Azure/azure-sdk-for-go') 3674 self.assertEqual(ud.parm['subpath'], 'sdk/storage/azblob') 3675 self.assertEqual(ud.name, 'github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0') 3676 self.assertEqual(self.d.getVar('SRCREV_github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'), 'ec928e0ed34db682b3f783d3739d1c538142e0c3') 3677 3678 fetcher.download() 3679 self.assertTrue(os.path.exists(ud.localpath)) 3680 3681 fetcher.unpack(self.unpackdir) 3682 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3683 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'd31d6145676ed3066ce573a8198f326dea5be45a43b3d8f41ce7787fd71d66b3'))) 3684 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3685 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip'))) 3686 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod'))) 3687 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')), 3688 '7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873') 3689 3690 @skipIfNoNetwork() 3691 def test_gomodgit_url_srcrev_var(self): 3692 urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0'] 3693 self.d.setVar('SRCREV_gopkg.in/ini.v1@v1.67.0', 'b2f570e5b5b844226bbefe6fb521d891f529a951') 3694 3695 fetcher = bb.fetch2.Fetch(urls, self.d) 3696 ud = fetcher.ud[urls[0]] 3697 self.assertEqual(ud.host, 'gopkg.in') 3698 self.assertEqual(ud.path, '/ini.v1') 3699 self.assertEqual(ud.name, 'gopkg.in/ini.v1@v1.67.0') 3700 self.assertEqual(ud.parm['srcrev'], 'b2f570e5b5b844226bbefe6fb521d891f529a951') 3701 3702 fetcher.download() 3703 fetcher.unpack(self.unpackdir) 3704 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3705 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3'))) 3706 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3707 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3708 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3709 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3710 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3711 3712 @skipIfNoNetwork() 3713 def test_gomodgit_url_no_go_mod_in_module(self): 3714 urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0;' 3715 'srcrev=b2f570e5b5b844226bbefe6fb521d891f529a951'] 3716 3717 fetcher = bb.fetch2.Fetch(urls, self.d) 3718 ud = fetcher.ud[urls[0]] 3719 self.assertEqual(ud.host, 'gopkg.in') 3720 self.assertEqual(ud.path, '/ini.v1') 3721 self.assertEqual(ud.name, 'gopkg.in/ini.v1@v1.67.0') 3722 self.assertEqual(self.d.getVar('SRCREV_gopkg.in/ini.v1@v1.67.0'), 'b2f570e5b5b844226bbefe6fb521d891f529a951') 3723 3724 fetcher.download() 3725 fetcher.unpack(self.unpackdir) 3726 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3727 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3'))) 3728 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3729 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3730 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3731 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3732 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3733 3734 @skipIfNoNetwork() 3735 def test_gomodgit_url_host_only(self): 3736 urls = ['gomodgit://go.opencensus.io;version=v0.24.0;' 3737 'repo=github.com/census-instrumentation/opencensus-go;' 3738 'srcrev=b1a01ee95db0e690d91d7193d037447816fae4c5'] 3739 3740 fetcher = bb.fetch2.Fetch(urls, self.d) 3741 ud = fetcher.ud[urls[0]] 3742 self.assertEqual(ud.host, 'github.com') 3743 self.assertEqual(ud.path, '/census-instrumentation/opencensus-go') 3744 self.assertEqual(ud.name, 'go.opencensus.io@v0.24.0') 3745 self.assertEqual(self.d.getVar('SRCREV_go.opencensus.io@v0.24.0'), 'b1a01ee95db0e690d91d7193d037447816fae4c5') 3746 3747 fetcher.download() 3748 fetcher.unpack(self.unpackdir) 3749 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3750 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'aae3ac7b2122ed3345654e6327855e9682f4a5350d63e93dbcfc51c4419df0e1'))) 3751 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3752 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.zip'))) 3753 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod'))) 3754 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')), 3755 '0dc9ccc660ad21cebaffd548f2cc6efa27891c68b4fbc1f8a3893b00f1acec96') 3756