1# 2# BitBake Tests for the Fetcher (fetch2/) 3# 4# Copyright (C) 2012 Richard Purdie 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9import contextlib 10import unittest 11import hashlib 12import tempfile 13import collections 14import os 15import signal 16import tarfile 17from bb.fetch2 import URI 18from bb.fetch2 import FetchMethod 19import bb 20from bb.tests.support.httpserver import HTTPService 21 22def skipIfNoNetwork(): 23 if os.environ.get("BB_SKIP_NETTESTS") == "yes": 24 return unittest.skip("network test") 25 return lambda f: f 26 27class TestTimeout(Exception): 28 # Indicate to pytest that this is not a test suite 29 __test__ = False 30 31class Timeout(): 32 33 def __init__(self, seconds): 34 self.seconds = seconds 35 36 def handle_timeout(self, signum, frame): 37 raise TestTimeout("Test failed: timeout reached") 38 39 def __enter__(self): 40 signal.signal(signal.SIGALRM, self.handle_timeout) 41 signal.alarm(self.seconds) 42 43 def __exit__(self, exc_type, exc_val, exc_tb): 44 signal.alarm(0) 45 46class URITest(unittest.TestCase): 47 test_uris = { 48 "http://www.google.com/index.html" : { 49 'uri': 'http://www.google.com/index.html', 50 'scheme': 'http', 51 'hostname': 'www.google.com', 52 'port': None, 53 'hostport': 'www.google.com', 54 'path': '/index.html', 55 'userinfo': '', 56 'username': '', 57 'password': '', 58 'params': {}, 59 'query': {}, 60 'relative': False 61 }, 62 "http://www.google.com/index.html;param1=value1" : { 63 'uri': 'http://www.google.com/index.html;param1=value1', 64 'scheme': 'http', 65 'hostname': 'www.google.com', 66 'port': None, 67 'hostport': 'www.google.com', 68 'path': '/index.html', 69 'userinfo': '', 70 'username': '', 71 'password': '', 72 'params': { 73 'param1': 'value1' 74 }, 75 'query': {}, 76 'relative': False 77 }, 78 "http://www.example.org/index.html?param1=value1" : { 79 'uri': 'http://www.example.org/index.html?param1=value1', 80 'scheme': 'http', 81 'hostname': 'www.example.org', 82 'port': None, 83 'hostport': 'www.example.org', 84 'path': '/index.html', 85 'userinfo': '', 86 'username': '', 87 'password': '', 88 'params': {}, 89 'query': { 90 'param1': 'value1' 91 }, 92 'relative': False 93 }, 94 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : { 95 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 96 'scheme': 'http', 97 'hostname': 'www.example.org', 98 'port': None, 99 'hostport': 'www.example.org', 100 'path': '/index.html', 101 'userinfo': '', 102 'username': '', 103 'password': '', 104 'params': { 105 'param2': 'value2' 106 }, 107 'query': { 108 'qparam1': 'qvalue1' 109 }, 110 'relative': False 111 }, 112 # Check that trailing semicolons are handled correctly 113 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2;" : { 114 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 115 'scheme': 'http', 116 'hostname': 'www.example.org', 117 'port': None, 118 'hostport': 'www.example.org', 119 'path': '/index.html', 120 'userinfo': '', 121 'username': '', 122 'password': '', 123 'params': { 124 'param2': 'value2' 125 }, 126 'query': { 127 'qparam1': 'qvalue1' 128 }, 129 'relative': False 130 }, 131 "http://www.example.com:8080/index.html" : { 132 'uri': 'http://www.example.com:8080/index.html', 133 'scheme': 'http', 134 'hostname': 'www.example.com', 135 'port': 8080, 136 'hostport': 'www.example.com:8080', 137 'path': '/index.html', 138 'userinfo': '', 139 'username': '', 140 'password': '', 141 'params': {}, 142 'query': {}, 143 'relative': False 144 }, 145 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : { 146 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg', 147 'scheme': 'cvs', 148 'hostname': 'cvs.handhelds.org', 149 'port': None, 150 'hostport': 'cvs.handhelds.org', 151 'path': '/cvs', 152 'userinfo': 'anoncvs', 153 'username': 'anoncvs', 154 'password': '', 155 'params': { 156 'module': 'familiar/dist/ipkg' 157 }, 158 'query': {}, 159 'relative': False 160 }, 161 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": { 162 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg', 163 'scheme': 'cvs', 164 'hostname': 'cvs.handhelds.org', 165 'port': None, 166 'hostport': 'cvs.handhelds.org', 167 'path': '/cvs', 168 'userinfo': 'anoncvs:anonymous', 169 'username': 'anoncvs', 170 'password': 'anonymous', 171 'params': collections.OrderedDict([ 172 ('tag', 'V0-99-81'), 173 ('module', 'familiar/dist/ipkg') 174 ]), 175 'query': {}, 176 'relative': False 177 }, 178 "file://example.diff": { # NOTE: Not RFC compliant! 179 'uri': 'file:example.diff', 180 'scheme': 'file', 181 'hostname': '', 182 'port': None, 183 'hostport': '', 184 'path': 'example.diff', 185 'userinfo': '', 186 'username': '', 187 'password': '', 188 'params': {}, 189 'query': {}, 190 'relative': True 191 }, 192 "file:example.diff": { # NOTE: RFC compliant version of the former 193 'uri': 'file:example.diff', 194 'scheme': 'file', 195 'hostname': '', 196 'port': None, 197 'hostport': '', 198 'path': 'example.diff', 199 'userinfo': '', 200 'userinfo': '', 201 'username': '', 202 'password': '', 203 'params': {}, 204 'query': {}, 205 'relative': True 206 }, 207 "file:///tmp/example.diff": { 208 'uri': 'file:///tmp/example.diff', 209 'scheme': 'file', 210 'hostname': '', 211 'port': None, 212 'hostport': '', 213 'path': '/tmp/example.diff', 214 'userinfo': '', 215 'userinfo': '', 216 'username': '', 217 'password': '', 218 'params': {}, 219 'query': {}, 220 'relative': False 221 }, 222 "git:///path/example.git": { 223 'uri': 'git:///path/example.git', 224 'scheme': 'git', 225 'hostname': '', 226 'port': None, 227 'hostport': '', 228 'path': '/path/example.git', 229 'userinfo': '', 230 'userinfo': '', 231 'username': '', 232 'password': '', 233 'params': {}, 234 'query': {}, 235 'relative': False 236 }, 237 "git:path/example.git": { 238 'uri': 'git:path/example.git', 239 'scheme': 'git', 240 'hostname': '', 241 'port': None, 242 'hostport': '', 243 'path': 'path/example.git', 244 'userinfo': '', 245 'userinfo': '', 246 'username': '', 247 'password': '', 248 'params': {}, 249 'query': {}, 250 'relative': True 251 }, 252 "git://example.net/path/example.git": { 253 'uri': 'git://example.net/path/example.git', 254 'scheme': 'git', 255 'hostname': 'example.net', 256 'port': None, 257 'hostport': 'example.net', 258 'path': '/path/example.git', 259 'userinfo': '', 260 'userinfo': '', 261 'username': '', 262 'password': '', 263 'params': {}, 264 'query': {}, 265 'relative': False 266 }, 267 "git://tfs-example.org:22/tfs/example%20path/example.git": { 268 'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git', 269 'scheme': 'git', 270 'hostname': 'tfs-example.org', 271 'port': 22, 272 'hostport': 'tfs-example.org:22', 273 'path': '/tfs/example path/example.git', 274 'userinfo': '', 275 'userinfo': '', 276 'username': '', 277 'password': '', 278 'params': {}, 279 'query': {}, 280 'relative': False 281 }, 282 "http://somesite.net;someparam=1": { 283 'uri': 'http://somesite.net;someparam=1', 284 'scheme': 'http', 285 'hostname': 'somesite.net', 286 'port': None, 287 'hostport': 'somesite.net', 288 'path': '', 289 'userinfo': '', 290 'userinfo': '', 291 'username': '', 292 'password': '', 293 'params': {"someparam" : "1"}, 294 'query': {}, 295 'relative': False 296 }, 297 "file://somelocation;someparam=1": { 298 'uri': 'file:somelocation;someparam=1', 299 'scheme': 'file', 300 'hostname': '', 301 'port': None, 302 'hostport': '', 303 'path': 'somelocation', 304 'userinfo': '', 305 'userinfo': '', 306 'username': '', 307 'password': '', 308 'params': {"someparam" : "1"}, 309 'query': {}, 310 'relative': True 311 }, 312 "https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip": { 313 'uri': 'https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip', 314 'scheme': 'https', 315 'hostname': 'www.innodisk.com', 316 'port': None, 317 'hostport': 'www.innodisk.com', 318 'path': '/Download_file', 319 'userinfo': '', 320 'userinfo': '', 321 'username': '', 322 'password': '', 323 'params': {"downloadfilename" : "EGPL-T101.zip"}, 324 'query': {"9BE0BF6657": None}, 325 'relative': False 326 }, 327 "file://example@.service": { 328 'uri': 'file:example%40.service', 329 'scheme': 'file', 330 'hostname': '', 331 'port': None, 332 'hostport': '', 333 'path': 'example@.service', 334 'userinfo': '', 335 'userinfo': '', 336 'username': '', 337 'password': '', 338 'params': {}, 339 'query': {}, 340 'relative': True 341 } 342 343 } 344 345 def test_uri(self): 346 for test_uri, ref in self.test_uris.items(): 347 uri = URI(test_uri) 348 349 self.assertEqual(str(uri), ref['uri']) 350 351 # expected attributes 352 self.assertEqual(uri.scheme, ref['scheme']) 353 354 self.assertEqual(uri.userinfo, ref['userinfo']) 355 self.assertEqual(uri.username, ref['username']) 356 self.assertEqual(uri.password, ref['password']) 357 358 self.assertEqual(uri.hostname, ref['hostname']) 359 self.assertEqual(uri.port, ref['port']) 360 self.assertEqual(uri.hostport, ref['hostport']) 361 362 self.assertEqual(uri.path, ref['path']) 363 self.assertEqual(uri.params, ref['params']) 364 365 self.assertEqual(uri.relative, ref['relative']) 366 367 def test_dict(self): 368 for test in self.test_uris.values(): 369 uri = URI() 370 371 self.assertEqual(uri.scheme, '') 372 self.assertEqual(uri.userinfo, '') 373 self.assertEqual(uri.username, '') 374 self.assertEqual(uri.password, '') 375 self.assertEqual(uri.hostname, '') 376 self.assertEqual(uri.port, None) 377 self.assertEqual(uri.path, '') 378 self.assertEqual(uri.params, {}) 379 380 381 uri.scheme = test['scheme'] 382 self.assertEqual(uri.scheme, test['scheme']) 383 384 uri.userinfo = test['userinfo'] 385 self.assertEqual(uri.userinfo, test['userinfo']) 386 self.assertEqual(uri.username, test['username']) 387 self.assertEqual(uri.password, test['password']) 388 389 # make sure changing the values doesn't do anything unexpected 390 uri.username = 'changeme' 391 self.assertEqual(uri.username, 'changeme') 392 self.assertEqual(uri.password, test['password']) 393 uri.password = 'insecure' 394 self.assertEqual(uri.username, 'changeme') 395 self.assertEqual(uri.password, 'insecure') 396 397 # reset back after our trickery 398 uri.userinfo = test['userinfo'] 399 self.assertEqual(uri.userinfo, test['userinfo']) 400 self.assertEqual(uri.username, test['username']) 401 self.assertEqual(uri.password, test['password']) 402 403 uri.hostname = test['hostname'] 404 self.assertEqual(uri.hostname, test['hostname']) 405 self.assertEqual(uri.hostport, test['hostname']) 406 407 uri.port = test['port'] 408 self.assertEqual(uri.port, test['port']) 409 self.assertEqual(uri.hostport, test['hostport']) 410 411 uri.path = test['path'] 412 self.assertEqual(uri.path, test['path']) 413 414 uri.params = test['params'] 415 self.assertEqual(uri.params, test['params']) 416 417 uri.query = test['query'] 418 self.assertEqual(uri.query, test['query']) 419 420 self.assertEqual(str(uri), test['uri']) 421 422 uri.params = {} 423 self.assertEqual(uri.params, {}) 424 self.assertEqual(str(uri), (str(uri).split(";"))[0]) 425 426class FetcherTest(unittest.TestCase): 427 428 def setUp(self): 429 self.origdir = os.getcwd() 430 self.d = bb.data.init() 431 self.tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") 432 self.dldir = os.path.join(self.tempdir, "download") 433 os.mkdir(self.dldir) 434 self.d.setVar("DL_DIR", self.dldir) 435 self.unpackdir = os.path.join(self.tempdir, "unpacked") 436 os.mkdir(self.unpackdir) 437 persistdir = os.path.join(self.tempdir, "persistdata") 438 self.d.setVar("PERSISTENT_DIR", persistdir) 439 440 def tearDown(self): 441 os.chdir(self.origdir) 442 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 443 print("Not cleaning up %s. Please remove manually." % self.tempdir) 444 else: 445 bb.process.run('chmod u+rw -R %s' % self.tempdir) 446 bb.utils.prunedir(self.tempdir) 447 448 def git(self, cmd, cwd=None): 449 if isinstance(cmd, str): 450 cmd = 'git -c safe.bareRepository=all ' + cmd 451 else: 452 cmd = ['git', '-c', 'safe.bareRepository=all'] + cmd 453 if cwd is None: 454 cwd = self.gitdir 455 return bb.process.run(cmd, cwd=cwd)[0] 456 457 def git_init(self, cwd=None): 458 self.git('init', cwd=cwd) 459 # Explicitly set initial branch to master as 460 # a common setup is to use other default 461 # branch than master. 462 self.git(['checkout', '-b', 'master'], cwd=cwd) 463 464 try: 465 self.git(['config', 'user.email'], cwd=cwd) 466 except bb.process.ExecutionError: 467 self.git(['config', 'user.email', 'you@example.com'], cwd=cwd) 468 469 try: 470 self.git(['config', 'user.name'], cwd=cwd) 471 except bb.process.ExecutionError: 472 self.git(['config', 'user.name', 'Your Name'], cwd=cwd) 473 474class MirrorUriTest(FetcherTest): 475 476 replaceuris = { 477 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") 478 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz", 479 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 480 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 481 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 482 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 483 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") 484 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 485 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake") 486 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890", 487 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") 488 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 489 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") 490 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 491 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") 492 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 493 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") 494 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 495 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist") 496 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", 497 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/") 498 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2", 499 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 500 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 501 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 502 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 503 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") 504 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 505 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org") 506 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 507 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/") 508 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 509 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http") 510 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 511 ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http") 512 : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 513 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=git;branch=master", "git://someserver.org/bitbake", "git://someotherserver.org/bitbake;protocol=https") 514 : "git://someotherserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=https;branch=master", 515 ("gitsm://git.qemu.org/git/seabios.git/;protocol=https;name=roms/seabios;subpath=roms/seabios;bareclone=1;nobranch=1;rev=1234567890123456789012345678901234567890", "gitsm://.*/.*", "http://petalinux.xilinx.com/sswreleases/rel-v${XILINX_VER_MAIN}/downloads") : "http://petalinux.xilinx.com/sswreleases/rel-v%24%7BXILINX_VER_MAIN%7D/downloads/git2_git.qemu.org.git.seabios.git..tar.gz", 516 ("https://somewhere.org/example/1.0.0/example;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/PATH") 517 : "file:///mirror/example/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 518 ("https://somewhere.org/example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/some-example-1.0.0.tgz") 519 : "file:///mirror/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 520 521 #Renaming files doesn't work 522 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz" 523 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 524 } 525 526 mirrorvar = "http://.*/.* file:///somepath/downloads/ " \ 527 "git://someserver.org/bitbake git://git.openembedded.org/bitbake " \ 528 "https://.*/.* file:///someotherpath/downloads/ " \ 529 "http://.*/.* file:///someotherpath/downloads/ " \ 530 "svn://svn.server1.com/ svn://svn.server2.com/" 531 532 def test_urireplace(self): 533 self.d.setVar("FILESPATH", ".") 534 for k, v in self.replaceuris.items(): 535 ud = bb.fetch.FetchData(k[0], self.d) 536 ud.setup_localpath(self.d) 537 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2])) 538 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d) 539 self.assertEqual([v], newuris) 540 541 def test_urilist1(self): 542 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 543 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 544 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 545 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 546 547 def test_urilist2(self): 548 # Catch https:// -> files:// bug 549 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 550 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 551 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 552 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 553 554 def test_urilistsvn(self): 555 # Catch svn:// -> svn:// bug 556 fetcher = bb.fetch.FetchData("svn://svn.server1.com/isource/svnroot/reponame/tags/tagname;module=path_in_tagnamefolder;protocol=https;rev=2", self.d) 557 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 558 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 559 self.assertEqual(uris, ['svn://svn.server2.com/isource/svnroot/reponame/tags/tagname;module=path_in_tagnamefolder;protocol=https;rev=2']) 560 561 def test_mirror_of_mirror(self): 562 # Test if mirror of a mirror works 563 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/" 564 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/" 565 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 566 mirrors = bb.fetch2.mirror_from_string(mirrorvar) 567 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 568 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 569 'file:///someotherpath/downloads/bitbake-1.0.tar.gz', 570 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', 571 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) 572 573 recmirrorvar = "https://.*/[^/]* http://aaaa/A/A/A/ " \ 574 "https://.*/[^/]* https://bbbb/B/B/B/" 575 576 def test_recursive(self): 577 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 578 mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar) 579 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 580 self.assertEqual(uris, ['http://aaaa/A/A/A/bitbake/bitbake-1.0.tar.gz', 581 'https://bbbb/B/B/B/bitbake/bitbake-1.0.tar.gz', 582 'http://aaaa/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz']) 583 584 585class GitDownloadDirectoryNamingTest(FetcherTest): 586 def setUp(self): 587 super(GitDownloadDirectoryNamingTest, self).setUp() 588 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 589 self.recipe_dir = "git.openembedded.org.bitbake" 590 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 591 self.mirror_dir = "github.com.openembedded.bitbake.git" 592 593 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 594 595 def setup_mirror_rewrite(self): 596 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 597 598 @skipIfNoNetwork() 599 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self): 600 self.setup_mirror_rewrite() 601 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 602 603 fetcher.download() 604 605 dir = os.listdir(self.dldir + "/git2") 606 self.assertIn(self.recipe_dir, dir) 607 608 @skipIfNoNetwork() 609 def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self): 610 self.setup_mirror_rewrite() 611 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 612 613 fetcher.download() 614 615 dir = os.listdir(self.dldir + "/git2") 616 self.assertIn(self.mirror_dir, dir) 617 self.assertIn(self.recipe_dir, dir) 618 619 @skipIfNoNetwork() 620 def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self): 621 self.setup_mirror_rewrite() 622 fetcher = bb.fetch.Fetch([self.mirror_url], self.d) 623 fetcher.download() 624 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 625 626 fetcher.download() 627 628 dir = os.listdir(self.dldir + "/git2") 629 self.assertIn(self.mirror_dir, dir) 630 self.assertIn(self.recipe_dir, dir) 631 632 633class TarballNamingTest(FetcherTest): 634 def setUp(self): 635 super(TarballNamingTest, self).setUp() 636 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 637 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 638 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 639 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz" 640 641 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 642 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 643 644 def setup_mirror_rewrite(self): 645 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 646 647 @skipIfNoNetwork() 648 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self): 649 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 650 651 fetcher.download() 652 653 dir = os.listdir(self.dldir) 654 self.assertIn(self.recipe_tarball, dir) 655 656 @skipIfNoNetwork() 657 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 658 self.setup_mirror_rewrite() 659 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 660 661 fetcher.download() 662 663 dir = os.listdir(self.dldir) 664 self.assertIn(self.mirror_tarball, dir) 665 666 667class GitShallowTarballNamingTest(FetcherTest): 668 def setUp(self): 669 super(GitShallowTarballNamingTest, self).setUp() 670 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 671 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz" 672 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 673 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz" 674 675 self.d.setVar('BB_GIT_SHALLOW', '1') 676 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 677 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 678 679 def setup_mirror_rewrite(self): 680 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 681 682 @skipIfNoNetwork() 683 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self): 684 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 685 686 fetcher.download() 687 688 dir = os.listdir(self.dldir) 689 self.assertIn(self.recipe_tarball, dir) 690 691 @skipIfNoNetwork() 692 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 693 self.setup_mirror_rewrite() 694 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 695 696 fetcher.download() 697 698 dir = os.listdir(self.dldir) 699 self.assertIn(self.mirror_tarball, dir) 700 701 702class CleanTarballTest(FetcherTest): 703 def setUp(self): 704 super(CleanTarballTest, self).setUp() 705 self.recipe_url = "git://git.openembedded.org/bitbake;protocol=https" 706 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 707 708 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 709 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 710 711 @skipIfNoNetwork() 712 def test_that_the_tarball_contents_does_not_leak_info(self): 713 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 714 715 fetcher.download() 716 717 fetcher.unpack(self.unpackdir) 718 mtime = bb.process.run('git log --all -1 --format=%ct', 719 cwd=os.path.join(self.unpackdir, 'git')) 720 self.assertEqual(len(mtime), 2) 721 mtime = int(mtime[0]) 722 723 archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball)) 724 self.assertNotEqual(len(archive.members), 0) 725 for member in archive.members: 726 if member.name == ".": 727 continue 728 self.assertEqual(member.uname, 'oe', "user name for %s differs" % member.name) 729 self.assertEqual(member.uid, 0, "uid for %s differs" % member.name) 730 self.assertEqual(member.gname, 'oe', "group name for %s differs" % member.name) 731 self.assertEqual(member.gid, 0, "gid for %s differs" % member.name) 732 self.assertEqual(member.mtime, mtime, "mtime for %s differs" % member.name) 733 734 735class FetcherLocalTest(FetcherTest): 736 def setUp(self): 737 def touch(fn): 738 with open(fn, 'a'): 739 os.utime(fn, None) 740 741 super(FetcherLocalTest, self).setUp() 742 self.localsrcdir = os.path.join(self.tempdir, 'localsrc') 743 os.makedirs(self.localsrcdir) 744 touch(os.path.join(self.localsrcdir, 'a')) 745 touch(os.path.join(self.localsrcdir, 'b')) 746 touch(os.path.join(self.localsrcdir, 'c@d')) 747 os.makedirs(os.path.join(self.localsrcdir, 'dir')) 748 touch(os.path.join(self.localsrcdir, 'dir', 'c')) 749 touch(os.path.join(self.localsrcdir, 'dir', 'd')) 750 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir')) 751 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e')) 752 touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device')) 753 bb.process.run('tar cf archive.tar -C dir .', cwd=self.localsrcdir) 754 bb.process.run('tar czf archive.tar.gz -C dir .', cwd=self.localsrcdir) 755 bb.process.run('tar cjf archive.tar.bz2 -C dir .', cwd=self.localsrcdir) 756 self.d.setVar("FILESPATH", self.localsrcdir) 757 758 def fetchUnpack(self, uris): 759 fetcher = bb.fetch.Fetch(uris, self.d) 760 fetcher.download() 761 fetcher.unpack(self.unpackdir) 762 flst = [] 763 for root, dirs, files in os.walk(self.unpackdir): 764 for f in files: 765 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir)) 766 flst.sort() 767 return flst 768 769 def test_local_checksum_fails_no_file(self): 770 self.d.setVar("SRC_URI", "file://404") 771 with self.assertRaises(bb.BBHandledException): 772 bb.fetch.get_checksum_file_list(self.d) 773 774 def test_local(self): 775 tree = self.fetchUnpack(['file://a', 'file://dir/c']) 776 self.assertEqual(tree, ['a', 'dir/c']) 777 778 def test_local_at(self): 779 tree = self.fetchUnpack(['file://c@d']) 780 self.assertEqual(tree, ['c@d']) 781 782 def test_local_backslash(self): 783 tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device']) 784 self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device']) 785 786 def test_local_wildcard(self): 787 with self.assertRaises(bb.fetch2.ParameterError): 788 tree = self.fetchUnpack(['file://a', 'file://dir/*']) 789 790 def test_local_dir(self): 791 tree = self.fetchUnpack(['file://a', 'file://dir']) 792 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e']) 793 794 def test_local_subdir(self): 795 tree = self.fetchUnpack(['file://dir/subdir']) 796 self.assertEqual(tree, ['dir/subdir/e']) 797 798 def test_local_subdir_file(self): 799 tree = self.fetchUnpack(['file://dir/subdir/e']) 800 self.assertEqual(tree, ['dir/subdir/e']) 801 802 def test_local_subdirparam(self): 803 tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo']) 804 self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e']) 805 806 def test_local_deepsubdirparam(self): 807 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar']) 808 self.assertEqual(tree, ['bar/dir/subdir/e']) 809 810 def test_local_absolutedir(self): 811 # Unpacking to an absolute path that is a subdirectory of the root 812 # should work 813 tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')]) 814 815 # Unpacking to an absolute path outside of the root should fail 816 with self.assertRaises(bb.fetch2.UnpackError): 817 self.fetchUnpack(['file://a;subdir=/bin/sh']) 818 819 def test_local_striplevel(self): 820 tree = self.fetchUnpack(['file://archive.tar;subdir=bar;striplevel=1']) 821 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 822 823 def test_local_striplevel_gzip(self): 824 tree = self.fetchUnpack(['file://archive.tar.gz;subdir=bar;striplevel=1']) 825 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 826 827 def test_local_striplevel_bzip2(self): 828 tree = self.fetchUnpack(['file://archive.tar.bz2;subdir=bar;striplevel=1']) 829 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 830 831 def dummyGitTest(self, suffix): 832 # Create dummy local Git repo 833 src_dir = tempfile.mkdtemp(dir=self.tempdir, 834 prefix='gitfetch_localusehead_') 835 self.gitdir = os.path.abspath(src_dir) 836 self.git_init() 837 self.git(['commit', '--allow-empty', '-m', 'Dummy commit']) 838 # Use other branch than master 839 self.git(['checkout', '-b', 'my-devel']) 840 self.git(['commit', '--allow-empty', '-m', 'Dummy commit 2']) 841 orig_rev = self.git(['rev-parse', 'HEAD']).strip() 842 843 # Fetch and check revision 844 self.d.setVar("SRCREV", "AUTOINC") 845 self.d.setVar("__BBSRCREV_SEEN", "1") 846 url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix 847 fetcher = bb.fetch.Fetch([url], self.d) 848 fetcher.download() 849 fetcher.unpack(self.unpackdir) 850 unpack_rev = self.git(['rev-parse', 'HEAD'], 851 cwd=os.path.join(self.unpackdir, 'git')).strip() 852 self.assertEqual(orig_rev, unpack_rev) 853 854 def test_local_gitfetch_usehead(self): 855 self.dummyGitTest("usehead=1") 856 857 def test_local_gitfetch_usehead_withname(self): 858 self.dummyGitTest("usehead=1;name=newName") 859 860 def test_local_gitfetch_shared(self): 861 self.dummyGitTest("usehead=1;name=sharedName") 862 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 863 self.assertTrue(os.path.exists(alt)) 864 865 def test_local_gitfetch_noshared(self): 866 self.d.setVar('BB_GIT_NOSHARED', '1') 867 self.unpackdir += '_noshared' 868 self.dummyGitTest("usehead=1;name=noSharedName") 869 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 870 self.assertFalse(os.path.exists(alt)) 871 872class FetcherNoNetworkTest(FetcherTest): 873 def setUp(self): 874 super().setUp() 875 # all test cases are based on not having network 876 self.d.setVar("BB_NO_NETWORK", "1") 877 878 def test_missing(self): 879 string = "this is a test file\n".encode("utf-8") 880 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 881 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 882 883 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 884 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 885 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 886 with self.assertRaises(bb.fetch2.NetworkAccess): 887 fetcher.download() 888 889 def test_valid_missing_donestamp(self): 890 # create the file in the download directory with correct hash 891 string = "this is a test file\n".encode("utf-8") 892 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f: 893 f.write(string) 894 895 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 896 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 897 898 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 899 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 900 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 901 fetcher.download() 902 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 903 904 def test_invalid_missing_donestamp(self): 905 # create an invalid file in the download directory with incorrect hash 906 string = "this is a test file\n".encode("utf-8") 907 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 908 pass 909 910 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 911 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 912 913 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 914 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 915 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 916 with self.assertRaises(bb.fetch2.NetworkAccess): 917 fetcher.download() 918 # the existing file should not exist or should have be moved to "bad-checksum" 919 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 920 921 def test_nochecksums_missing(self): 922 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 923 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 924 # ssh fetch does not support checksums 925 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 926 # attempts to download with missing donestamp 927 with self.assertRaises(bb.fetch2.NetworkAccess): 928 fetcher.download() 929 930 def test_nochecksums_missing_donestamp(self): 931 # create a file in the download directory 932 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 933 pass 934 935 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 936 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 937 # ssh fetch does not support checksums 938 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 939 # attempts to download with missing donestamp 940 with self.assertRaises(bb.fetch2.NetworkAccess): 941 fetcher.download() 942 943 def test_nochecksums_has_donestamp(self): 944 # create a file in the download directory with the donestamp 945 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 946 pass 947 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 948 pass 949 950 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 951 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 952 # ssh fetch does not support checksums 953 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 954 # should not fetch 955 fetcher.download() 956 # both files should still exist 957 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 958 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 959 960 def test_nochecksums_missing_has_donestamp(self): 961 # create a file in the download directory with the donestamp 962 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 963 pass 964 965 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 966 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 967 # ssh fetch does not support checksums 968 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 969 with self.assertRaises(bb.fetch2.NetworkAccess): 970 fetcher.download() 971 # both files should still exist 972 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 973 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 974 975class FetcherNetworkTest(FetcherTest): 976 @skipIfNoNetwork() 977 def test_fetch(self): 978 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 979 fetcher.download() 980 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 981 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892) 982 self.d.setVar("BB_NO_NETWORK", "1") 983 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 984 fetcher.download() 985 fetcher.unpack(self.unpackdir) 986 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9) 987 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9) 988 989 @skipIfNoNetwork() 990 def test_fetch_mirror(self): 991 self.d.setVar("MIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 992 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 993 fetcher.download() 994 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 995 996 @skipIfNoNetwork() 997 def test_fetch_mirror_of_mirror(self): 998 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake") 999 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 1000 fetcher.download() 1001 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1002 1003 @skipIfNoNetwork() 1004 def test_fetch_file_mirror_of_mirror(self): 1005 self.d.setVar("FILESPATH", ".") 1006 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake") 1007 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 1008 os.mkdir(self.dldir + "/some2where") 1009 fetcher.download() 1010 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1011 1012 @skipIfNoNetwork() 1013 def test_fetch_premirror(self): 1014 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 1015 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 1016 fetcher.download() 1017 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1018 1019 @skipIfNoNetwork() 1020 def test_fetch_specify_downloadfilename(self): 1021 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) 1022 fetcher.download() 1023 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) 1024 1025 @skipIfNoNetwork() 1026 def test_fetch_premirror_specify_downloadfilename_regex_uri(self): 1027 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/") 1028 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1029 fetcher.download() 1030 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1031 1032 @skipIfNoNetwork() 1033 # BZ13039 1034 def test_fetch_premirror_specify_downloadfilename_specific_uri(self): 1035 self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake") 1036 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1037 fetcher.download() 1038 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1039 1040 @skipIfNoNetwork() 1041 def test_fetch_premirror_use_downloadfilename_to_fetch(self): 1042 # Ensure downloadfilename is used when fetching from premirror. 1043 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 1044 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1045 fetcher.download() 1046 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1047 1048 @skipIfNoNetwork() 1049 def gitfetcher(self, url1, url2): 1050 def checkrevision(self, fetcher): 1051 fetcher.unpack(self.unpackdir) 1052 revision = self.git(['rev-parse', 'HEAD'], 1053 cwd=os.path.join(self.unpackdir, 'git')).strip() 1054 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 1055 1056 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 1057 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 1058 fetcher = bb.fetch.Fetch([url1], self.d) 1059 fetcher.download() 1060 checkrevision(self, fetcher) 1061 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works 1062 bb.utils.prunedir(self.dldir + "/git2/") 1063 bb.utils.prunedir(self.unpackdir) 1064 self.d.setVar("BB_NO_NETWORK", "1") 1065 fetcher = bb.fetch.Fetch([url2], self.d) 1066 fetcher.download() 1067 checkrevision(self, fetcher) 1068 1069 @skipIfNoNetwork() 1070 def test_gitfetch(self): 1071 url1 = url2 = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 1072 self.gitfetcher(url1, url2) 1073 1074 @skipIfNoNetwork() 1075 def test_gitfetch_goodsrcrev(self): 1076 # SRCREV is set but matches rev= parameter 1077 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https" 1078 self.gitfetcher(url1, url2) 1079 1080 @skipIfNoNetwork() 1081 def test_gitfetch_badsrcrev(self): 1082 # SRCREV is set but does not match rev= parameter 1083 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https" 1084 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1085 1086 @skipIfNoNetwork() 1087 def test_gitfetch_tagandrev(self): 1088 # SRCREV is set but does not match rev= parameter 1089 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5;protocol=https" 1090 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1091 1092 @skipIfNoNetwork() 1093 def test_gitfetch_usehead(self): 1094 # Since self.gitfetcher() sets SRCREV we expect this to override 1095 # `usehead=1' and instead fetch the specified SRCREV. See 1096 # test_local_gitfetch_usehead() for a positive use of the usehead 1097 # feature. 1098 url = "git://git.openembedded.org/bitbake;usehead=1;branch=master;protocol=https" 1099 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1100 1101 @skipIfNoNetwork() 1102 def test_gitfetch_usehead_withname(self): 1103 # Since self.gitfetcher() sets SRCREV we expect this to override 1104 # `usehead=1' and instead fetch the specified SRCREV. See 1105 # test_local_gitfetch_usehead() for a positive use of the usehead 1106 # feature. 1107 url = "git://git.openembedded.org/bitbake;usehead=1;name=newName;branch=master;protocol=https" 1108 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1109 1110 @skipIfNoNetwork() 1111 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self): 1112 recipeurl = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 1113 mirrorurl = "git://someserver.org/bitbake;branch=master;protocol=https" 1114 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1115 self.gitfetcher(recipeurl, mirrorurl) 1116 1117 @skipIfNoNetwork() 1118 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self): 1119 recipeurl = "git://someserver.org/bitbake;branch=master;protocol=https" 1120 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1121 self.gitfetcher(recipeurl, recipeurl) 1122 1123 @skipIfNoNetwork() 1124 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self): 1125 realurl = "https://git.openembedded.org/bitbake" 1126 recipeurl = "git://someserver.org/bitbake;protocol=https" 1127 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git") 1128 os.chdir(self.tempdir) 1129 self.git(['clone', realurl, self.sourcedir], cwd=self.tempdir) 1130 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipeurl, self.sourcedir)) 1131 self.gitfetcher(recipeurl, recipeurl) 1132 1133 @skipIfNoNetwork() 1134 def test_git_submodule(self): 1135 # URL with ssh submodules 1136 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master;protocol=https" 1137 # Original URL (comment this if you have ssh access to git.yoctoproject.org) 1138 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master;protocol=https" 1139 fetcher = bb.fetch.Fetch([url], self.d) 1140 fetcher.download() 1141 # Previous cwd has been deleted 1142 os.chdir(os.path.dirname(self.unpackdir)) 1143 fetcher.unpack(self.unpackdir) 1144 1145 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1146 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing') 1147 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing') 1148 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present') 1149 1150 # Only when we're running the extended test with a submodule's submodule, can we check this. 1151 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')): 1152 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing') 1153 1154 @skipIfNoNetwork() 1155 def test_git_submodule_restricted_network_premirrors(self): 1156 # this test is to ensure that premirrors will be tried in restricted network 1157 # that is, BB_ALLOWED_NETWORKS does not contain the domain the url uses 1158 url = "gitsm://github.com/grpc/grpc.git;protocol=https;name=grpc;branch=v1.60.x;rev=0ef13a7555dbaadd4633399242524129eef5e231" 1159 # create a download directory to be used as premirror later 1160 tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") 1161 dl_premirror = os.path.join(tempdir, "download-premirror") 1162 os.mkdir(dl_premirror) 1163 self.d.setVar("DL_DIR", dl_premirror) 1164 fetcher = bb.fetch.Fetch([url], self.d) 1165 fetcher.download() 1166 # now use the premirror in restricted network 1167 self.d.setVar("DL_DIR", self.dldir) 1168 self.d.setVar("PREMIRRORS", "gitsm://.*/.* gitsm://%s/git2/MIRRORNAME;protocol=file" % dl_premirror) 1169 self.d.setVar("BB_ALLOWED_NETWORKS", "*.some.domain") 1170 fetcher = bb.fetch.Fetch([url], self.d) 1171 fetcher.download() 1172 1173 @skipIfNoNetwork() 1174 def test_git_submodule_dbus_broker(self): 1175 # The following external repositories have show failures in fetch and unpack operations 1176 # We want to avoid regressions! 1177 url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main" 1178 fetcher = bb.fetch.Fetch([url], self.d) 1179 fetcher.download() 1180 # Previous cwd has been deleted 1181 os.chdir(os.path.dirname(self.unpackdir)) 1182 fetcher.unpack(self.unpackdir) 1183 1184 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1185 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"') 1186 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"') 1187 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"') 1188 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"') 1189 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"') 1190 1191 @skipIfNoNetwork() 1192 def test_git_submodule_CLI11(self): 1193 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main" 1194 fetcher = bb.fetch.Fetch([url], self.d) 1195 fetcher.download() 1196 # Previous cwd has been deleted 1197 os.chdir(os.path.dirname(self.unpackdir)) 1198 fetcher.unpack(self.unpackdir) 1199 1200 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1201 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1202 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1203 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1204 1205 @skipIfNoNetwork() 1206 def test_git_submodule_update_CLI11(self): 1207 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """ 1208 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main" 1209 fetcher = bb.fetch.Fetch([url], self.d) 1210 fetcher.download() 1211 1212 # CLI11 that pulls in a newer nlohmann-json 1213 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main" 1214 fetcher = bb.fetch.Fetch([url], self.d) 1215 fetcher.download() 1216 # Previous cwd has been deleted 1217 os.chdir(os.path.dirname(self.unpackdir)) 1218 fetcher.unpack(self.unpackdir) 1219 1220 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1221 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1222 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1223 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1224 1225 @skipIfNoNetwork() 1226 def test_git_submodule_aktualizr(self): 1227 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" 1228 fetcher = bb.fetch.Fetch([url], self.d) 1229 fetcher.download() 1230 # Previous cwd has been deleted 1231 os.chdir(os.path.dirname(self.unpackdir)) 1232 fetcher.unpack(self.unpackdir) 1233 1234 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1235 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"') 1236 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"') 1237 self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly") 1238 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"') 1239 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"') 1240 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"') 1241 1242 @skipIfNoNetwork() 1243 def test_git_submodule_iotedge(self): 1244 """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """ 1245 1246 # This repository also has submodules where the module (name), path and url do not align 1247 url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699;branch=main" 1248 fetcher = bb.fetch.Fetch([url], self.d) 1249 fetcher.download() 1250 # Previous cwd has been deleted 1251 os.chdir(os.path.dirname(self.unpackdir)) 1252 fetcher.unpack(self.unpackdir) 1253 1254 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1255 1256 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/README.md')), msg='Missing submodule checkout') 1257 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/ctest/README.md')), msg='Missing submodule checkout') 1258 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1259 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1260 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1261 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1262 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/README.md')), msg='Missing submodule checkout') 1263 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/README.md')), msg='Missing submodule checkout') 1264 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/ctest/README.md')), msg='Missing submodule checkout') 1265 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1266 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1267 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1268 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1269 1270 @skipIfNoNetwork() 1271 def test_git_submodule_reference_to_parent(self): 1272 self.recipe_url = "gitsm://github.com/gflags/gflags.git;protocol=https;branch=master" 1273 self.d.setVar("SRCREV", "14e1138441bbbb584160cb1c0a0426ec1bac35f1") 1274 with Timeout(60): 1275 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 1276 with self.assertRaises(bb.fetch2.FetchError): 1277 fetcher.download() 1278 1279class SVNTest(FetcherTest): 1280 def skipIfNoSvn(): 1281 import shutil 1282 if not shutil.which("svn"): 1283 return unittest.skip("svn not installed, tests being skipped") 1284 1285 if not shutil.which("svnadmin"): 1286 return unittest.skip("svnadmin not installed, tests being skipped") 1287 1288 return lambda f: f 1289 1290 @skipIfNoSvn() 1291 def setUp(self): 1292 """ Create a local repository """ 1293 1294 super(SVNTest, self).setUp() 1295 1296 # Create something we can fetch 1297 src_dir = tempfile.mkdtemp(dir=self.tempdir, 1298 prefix='svnfetch_srcdir_') 1299 src_dir = os.path.abspath(src_dir) 1300 bb.process.run("echo readme > README.md", cwd=src_dir) 1301 1302 # Store it in a local SVN repository 1303 repo_dir = tempfile.mkdtemp(dir=self.tempdir, 1304 prefix='svnfetch_localrepo_') 1305 repo_dir = os.path.abspath(repo_dir) 1306 bb.process.run("svnadmin create project", cwd=repo_dir) 1307 1308 self.repo_url = "file://%s/project" % repo_dir 1309 bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url), 1310 cwd=repo_dir) 1311 1312 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir) 1313 # Github won't emulate SVN anymore (see https://github.blog/2023-01-20-sunsetting-subversion-support/) 1314 # Use still accessible svn repo (only trunk to avoid longer downloads) 1315 bb.process.run("svn propset svn:externals 'bitbake https://svn.apache.org/repos/asf/serf/trunk' .", 1316 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1317 bb.process.run("svn commit --non-interactive -m 'Add external'", 1318 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1319 1320 self.src_dir = src_dir 1321 self.repo_dir = repo_dir 1322 1323 @skipIfNoSvn() 1324 def tearDown(self): 1325 os.chdir(self.origdir) 1326 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 1327 print("Not cleaning up %s. Please remove manually." % self.tempdir) 1328 else: 1329 bb.utils.prunedir(self.tempdir) 1330 1331 @skipIfNoSvn() 1332 @skipIfNoNetwork() 1333 def test_noexternal_svn(self): 1334 # Always match the rev count from setUp (currently rev 2) 1335 url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '') 1336 fetcher = bb.fetch.Fetch([url], self.d) 1337 fetcher.download() 1338 os.chdir(os.path.dirname(self.unpackdir)) 1339 fetcher.unpack(self.unpackdir) 1340 1341 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1342 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1343 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should NOT exist") 1344 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should NOT exit") 1345 1346 @skipIfNoSvn() 1347 def test_external_svn(self): 1348 # Always match the rev count from setUp (currently rev 2) 1349 url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '') 1350 fetcher = bb.fetch.Fetch([url], self.d) 1351 fetcher.download() 1352 os.chdir(os.path.dirname(self.unpackdir)) 1353 fetcher.unpack(self.unpackdir) 1354 1355 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1356 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1357 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should exist") 1358 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should exit") 1359 1360class TrustedNetworksTest(FetcherTest): 1361 def test_trusted_network(self): 1362 # Ensure trusted_network returns False when the host IS in the list. 1363 url = "git://Someserver.org/foo;rev=1;branch=master" 1364 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org") 1365 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1366 1367 def test_wild_trusted_network(self): 1368 # Ensure trusted_network returns true when the *.host IS in the list. 1369 url = "git://Someserver.org/foo;rev=1;branch=master" 1370 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1371 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1372 1373 def test_prefix_wild_trusted_network(self): 1374 # Ensure trusted_network returns true when the prefix matches *.host. 1375 url = "git://git.Someserver.org/foo;rev=1;branch=master" 1376 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1377 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1378 1379 def test_two_prefix_wild_trusted_network(self): 1380 # Ensure trusted_network returns true when the prefix matches *.host. 1381 url = "git://something.git.Someserver.org/foo;rev=1;branch=master" 1382 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1383 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1384 1385 def test_port_trusted_network(self): 1386 # Ensure trusted_network returns True, even if the url specifies a port. 1387 url = "git://someserver.org:8080/foo;rev=1;branch=master" 1388 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org") 1389 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1390 1391 def test_untrusted_network(self): 1392 # Ensure trusted_network returns False when the host is NOT in the list. 1393 url = "git://someserver.org/foo;rev=1;branch=master" 1394 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1395 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1396 1397 def test_wild_untrusted_network(self): 1398 # Ensure trusted_network returns False when the host is NOT in the list. 1399 url = "git://*.someserver.org/foo;rev=1;branch=master" 1400 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1401 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1402 1403class URLHandle(unittest.TestCase): 1404 import urllib.parse 1405 1406 # Quote password as per RFC3986 1407 password = urllib.parse.quote(r"!#$%^&*()-_={}[]\|:?,.<>~`", r"!$&'/()*+,;=") 1408 datatable = { 1409 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}), 1410 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}), 1411 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])), 1412 "git://git.openembedded.org/bitbake;branch=@foo;protocol=https" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo', 'protocol' : 'https'}), 1413 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}), 1414 "file://example@.service": ('file', '', 'example@.service', '', '', {}), 1415 "https://somesite.com/somerepo.git;user=anyUser:idtoken=1234" : ('https', 'somesite.com', '/somerepo.git', '', '', {'user': 'anyUser:idtoken=1234'}), 1416 'git://s.o-me_ONE:%s@git.openembedded.org/bitbake;branch=main;protocol=https' % password: ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', password, {'branch': 'main', 'protocol' : 'https'}), 1417 } 1418 # we require a pathname to encodeurl but users can still pass such urls to 1419 # decodeurl and we need to handle them 1420 decodedata = datatable.copy() 1421 decodedata.update({ 1422 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}), 1423 "npmsw://some.registry.url;package=@pkg;version=latest": ('npmsw', 'some.registry.url', '/', '', '', {'package': '@pkg', 'version': 'latest'}), 1424 }) 1425 1426 def test_decodeurl(self): 1427 for k, v in self.decodedata.items(): 1428 result = bb.fetch.decodeurl(k) 1429 self.assertEqual(result, v) 1430 1431 def test_encodeurl(self): 1432 import urllib.parse 1433 for k, v in self.datatable.items(): 1434 result = bb.fetch.encodeurl(v) 1435 if result.startswith("file:"): 1436 result = urllib.parse.unquote(result) 1437 self.assertEqual(result, k) 1438 1439class FetchLatestVersionTest(FetcherTest): 1440 1441 test_git_uris = { 1442 # version pattern "X.Y.Z" 1443 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "", "") 1444 : "1.99.4", 1445 # version pattern "vX.Y" 1446 # mirror of git.infradead.org since network issues interfered with testing 1447 ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master;protocol=https", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "", "") 1448 : "1.5.0", 1449 # version pattern "pkg_name-X.Y" 1450 # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing 1451 ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master;protocol=https", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "", "") 1452 : "1.0", 1453 # version pattern "pkg_name-vX.Y.Z" 1454 ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master;protocol=https", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "", "") 1455 : "1.4.0", 1456 # combination version pattern 1457 ("sysprof", "git://git.yoctoproject.org/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "", "") 1458 : "1.2.0", 1459 ("u-boot-mkimage", "git://source.denx.de/u-boot/u-boot.git;branch=master;protocol=https", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "") 1460 : "2014.01", 1461 # version pattern "yyyymmdd" 1462 ("mobile-broadband-provider-info", "git://git.yoctoproject.org/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "", "") 1463 : "20120614", 1464 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX 1465 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing 1466 ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master;protocol=https", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))", "") 1467 : "0.4.3", 1468 ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master;protocol=https", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))", "") 1469 : "11.0.0", 1470 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))", "") 1471 : "1.3.59", 1472 ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))", "") 1473 : "3.82+dbg0.9", 1474 ("sysdig", "git://github.com/draios/sysdig.git;branch=dev;protocol=https", "4fb6288275f567f63515df0ff0a6518043ecfa9b", r"^(?P<pver>\d+(\.\d+)+)", "10.0.0") 1475 : "0.28.0", 1476 } 1477 1478 test_wget_uris = { 1479 # 1480 # packages with versions inside directory name 1481 # 1482 # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2 1483 ("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "") 1484 : "2.24.2", 1485 # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz 1486 ("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "") 1487 : "1.6.0", 1488 # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz 1489 ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "") 1490 : "2.8.12.1", 1491 # https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz 1492 ("libxml2", "/software/libxml2/2.9/libxml2-2.9.14.tar.xz", "", "") 1493 : "2.10.3", 1494 # 1495 # packages with versions only in current directory 1496 # 1497 # https://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2 1498 ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "") 1499 : "2.19", 1500 # https://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2 1501 ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "") 1502 : "20120814", 1503 # 1504 # packages with "99" in the name of possible version 1505 # 1506 # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz 1507 ("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "") 1508 : "5.0", 1509 # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2 1510 ("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "") 1511 : "1.15.1", 1512 # 1513 # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX 1514 # 1515 # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2 1516 # https://github.com/apple/cups/releases 1517 ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", r"(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz") 1518 : "2.0.0", 1519 # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz 1520 # http://ftp.debian.org/debian/pool/main/d/db5.3/ 1521 ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", r"(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz") 1522 : "5.3.10", 1523 # 1524 # packages where the tarball compression changed in the new version 1525 # 1526 # http://ftp.debian.org/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz 1527 ("minicom", "/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz", "", "") 1528 : "2.8", 1529 } 1530 1531 test_crate_uris = { 1532 # basic example; version pattern "A.B.C+cargo-D.E.F" 1533 ("cargo-c", "crate://crates.io/cargo-c/0.9.18+cargo-0.69") 1534 : "0.9.29" 1535 } 1536 1537 @skipIfNoNetwork() 1538 def test_git_latest_versionstring(self): 1539 for k, v in self.test_git_uris.items(): 1540 self.d.setVar("PN", k[0]) 1541 self.d.setVar("SRCREV", k[2]) 1542 self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3]) 1543 ud = bb.fetch2.FetchData(k[1], self.d) 1544 pupver= ud.method.latest_versionstring(ud, self.d) 1545 verstring = pupver[0] 1546 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1547 r = bb.utils.vercmp_string(v, verstring) 1548 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1549 if k[4]: 1550 r = bb.utils.vercmp_string(verstring, k[4]) 1551 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], verstring, k[4])) 1552 1553 def test_wget_latest_versionstring(self): 1554 testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata" 1555 server = HTTPService(testdata, host="127.0.0.1") 1556 server.start() 1557 port = server.port 1558 try: 1559 for k, v in self.test_wget_uris.items(): 1560 self.d.setVar("PN", k[0]) 1561 checkuri = "" 1562 if k[2]: 1563 checkuri = "http://127.0.0.1:%s/" % port + k[2] 1564 self.d.setVar("UPSTREAM_CHECK_URI", checkuri) 1565 self.d.setVar("UPSTREAM_CHECK_REGEX", k[3]) 1566 url = "http://127.0.0.1:%s/" % port + k[1] 1567 ud = bb.fetch2.FetchData(url, self.d) 1568 pupver = ud.method.latest_versionstring(ud, self.d) 1569 verstring = pupver[0] 1570 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1571 r = bb.utils.vercmp_string(v, verstring) 1572 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1573 finally: 1574 server.stop() 1575 1576 @skipIfNoNetwork() 1577 def test_crate_latest_versionstring(self): 1578 for k, v in self.test_crate_uris.items(): 1579 self.d.setVar("PN", k[0]) 1580 ud = bb.fetch2.FetchData(k[1], self.d) 1581 pupver = ud.method.latest_versionstring(ud, self.d) 1582 verstring = pupver[0] 1583 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1584 r = bb.utils.vercmp_string(v, verstring) 1585 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1586 1587class FetchCheckStatusTest(FetcherTest): 1588 test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz", 1589 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz", 1590 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz", 1591 "https://yoctoproject.org/", 1592 "https://docs.yoctoproject.org", 1593 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", 1594 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", 1595 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz", 1596 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD 1597 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz" 1598 ] 1599 1600 @skipIfNoNetwork() 1601 def test_wget_checkstatus(self): 1602 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d) 1603 for u in self.test_wget_uris: 1604 with self.subTest(url=u): 1605 ud = fetch.ud[u] 1606 m = ud.method 1607 ret = m.checkstatus(fetch, ud, self.d) 1608 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1609 1610 @skipIfNoNetwork() 1611 def test_wget_checkstatus_connection_cache(self): 1612 from bb.fetch2 import FetchConnectionCache 1613 1614 connection_cache = FetchConnectionCache() 1615 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d, 1616 connection_cache = connection_cache) 1617 1618 for u in self.test_wget_uris: 1619 with self.subTest(url=u): 1620 ud = fetch.ud[u] 1621 m = ud.method 1622 ret = m.checkstatus(fetch, ud, self.d) 1623 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1624 1625 connection_cache.close_connections() 1626 1627 1628class GitMakeShallowTest(FetcherTest): 1629 def setUp(self): 1630 FetcherTest.setUp(self) 1631 self.gitdir = os.path.join(self.tempdir, 'gitshallow') 1632 bb.utils.mkdirhier(self.gitdir) 1633 self.git_init() 1634 1635 def assertRefs(self, expected_refs): 1636 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines() 1637 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines() 1638 self.assertEqual(sorted(full_expected), sorted(actual_refs)) 1639 1640 def assertRevCount(self, expected_count, args=None): 1641 if args is None: 1642 args = ['HEAD'] 1643 revs = self.git(['rev-list'] + args) 1644 actual_count = len(revs.splitlines()) 1645 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1646 1647 def make_shallow(self, args=None): 1648 if args is None: 1649 args = ['HEAD'] 1650 return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir) 1651 1652 def add_empty_file(self, path, msg=None): 1653 if msg is None: 1654 msg = path 1655 open(os.path.join(self.gitdir, path), 'w').close() 1656 self.git(['add', path]) 1657 self.git(['commit', '-m', msg, path]) 1658 1659 def test_make_shallow_single_branch_no_merge(self): 1660 self.add_empty_file('a') 1661 self.add_empty_file('b') 1662 self.assertRevCount(2) 1663 self.make_shallow() 1664 self.assertRevCount(1) 1665 1666 def test_make_shallow_single_branch_one_merge(self): 1667 self.add_empty_file('a') 1668 self.add_empty_file('b') 1669 self.git('checkout -b a_branch') 1670 self.add_empty_file('c') 1671 self.git('checkout master') 1672 self.add_empty_file('d') 1673 self.git('merge --no-ff --no-edit a_branch') 1674 self.git('branch -d a_branch') 1675 self.add_empty_file('e') 1676 self.assertRevCount(6) 1677 self.make_shallow(['HEAD~2']) 1678 self.assertRevCount(5) 1679 1680 def test_make_shallow_at_merge(self): 1681 self.add_empty_file('a') 1682 self.git('checkout -b a_branch') 1683 self.add_empty_file('b') 1684 self.git('checkout master') 1685 self.git('merge --no-ff --no-edit a_branch') 1686 self.git('branch -d a_branch') 1687 self.assertRevCount(3) 1688 self.make_shallow() 1689 self.assertRevCount(1) 1690 1691 def test_make_shallow_annotated_tag(self): 1692 self.add_empty_file('a') 1693 self.add_empty_file('b') 1694 self.git('tag -a -m a_tag a_tag') 1695 self.assertRevCount(2) 1696 self.make_shallow(['a_tag']) 1697 self.assertRevCount(1) 1698 1699 def test_make_shallow_multi_ref(self): 1700 self.add_empty_file('a') 1701 self.add_empty_file('b') 1702 self.git('checkout -b a_branch') 1703 self.add_empty_file('c') 1704 self.git('checkout master') 1705 self.add_empty_file('d') 1706 self.git('checkout -b a_branch_2') 1707 self.add_empty_file('a_tag') 1708 self.git('tag a_tag') 1709 self.git('checkout master') 1710 self.git('branch -D a_branch_2') 1711 self.add_empty_file('e') 1712 self.assertRevCount(6, ['--all']) 1713 self.make_shallow() 1714 self.assertRevCount(5, ['--all']) 1715 1716 def test_make_shallow_multi_ref_trim(self): 1717 self.add_empty_file('a') 1718 self.git('checkout -b a_branch') 1719 self.add_empty_file('c') 1720 self.git('checkout master') 1721 self.assertRevCount(1) 1722 self.assertRevCount(2, ['--all']) 1723 self.assertRefs(['master', 'a_branch']) 1724 self.make_shallow(['-r', 'master', 'HEAD']) 1725 self.assertRevCount(1, ['--all']) 1726 self.assertRefs(['master']) 1727 1728 def test_make_shallow_noop(self): 1729 self.add_empty_file('a') 1730 self.assertRevCount(1) 1731 self.make_shallow() 1732 self.assertRevCount(1) 1733 1734 @skipIfNoNetwork() 1735 def test_make_shallow_bitbake(self): 1736 self.git('remote add origin https://github.com/openembedded/bitbake') 1737 self.git('fetch --tags origin') 1738 orig_revs = len(self.git('rev-list --all').splitlines()) 1739 self.make_shallow(['refs/tags/1.10.0']) 1740 self.assertRevCount(orig_revs - 1746, ['--all']) 1741 1742class GitShallowTest(FetcherTest): 1743 def setUp(self): 1744 FetcherTest.setUp(self) 1745 self.gitdir = os.path.join(self.tempdir, 'git') 1746 self.srcdir = os.path.join(self.tempdir, 'gitsource') 1747 1748 bb.utils.mkdirhier(self.srcdir) 1749 self.git_init(cwd=self.srcdir) 1750 self.d.setVar('WORKDIR', self.tempdir) 1751 self.d.setVar('S', self.gitdir) 1752 self.d.delVar('PREMIRRORS') 1753 self.d.delVar('MIRRORS') 1754 1755 uri = 'git://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1756 self.d.setVar('SRC_URI', uri) 1757 self.d.setVar('SRCREV', '${AUTOREV}') 1758 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 1759 1760 self.d.setVar('BB_GIT_SHALLOW', '1') 1761 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0') 1762 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 1763 self.d.setVar("__BBSRCREV_SEEN", "1") 1764 1765 def assertRefs(self, expected_refs, cwd=None): 1766 if cwd is None: 1767 cwd = self.gitdir 1768 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines() 1769 # Resolve references into the same format as the comparision (needed by git 2.48 onwards) 1770 actual_refs = self.git(['rev-parse', '--symbolic-full-name'] + actual_refs, cwd=cwd).splitlines() 1771 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines() 1772 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs))) 1773 1774 def assertRevCount(self, expected_count, args=None, cwd=None): 1775 if args is None: 1776 args = ['HEAD'] 1777 if cwd is None: 1778 cwd = self.gitdir 1779 revs = self.git(['rev-list'] + args, cwd=cwd) 1780 actual_count = len(revs.splitlines()) 1781 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1782 1783 def add_empty_file(self, path, cwd=None, msg=None): 1784 if msg is None: 1785 msg = path 1786 if cwd is None: 1787 cwd = self.srcdir 1788 open(os.path.join(cwd, path), 'w').close() 1789 self.git(['add', path], cwd) 1790 self.git(['commit', '-m', msg, path], cwd) 1791 1792 def fetch(self, uri=None): 1793 if uri is None: 1794 uris = self.d.getVar('SRC_URI').split() 1795 uri = uris[0] 1796 d = self.d 1797 else: 1798 d = self.d.createCopy() 1799 d.setVar('SRC_URI', uri) 1800 uri = d.expand(uri) 1801 uris = [uri] 1802 1803 fetcher = bb.fetch2.Fetch(uris, d) 1804 fetcher.download() 1805 ud = fetcher.ud[uri] 1806 return fetcher, ud 1807 1808 def fetch_and_unpack(self, uri=None): 1809 fetcher, ud = self.fetch(uri) 1810 fetcher.unpack(self.d.getVar('WORKDIR')) 1811 assert os.path.exists(self.d.getVar('S')) 1812 return fetcher, ud 1813 1814 def fetch_shallow(self, uri=None, disabled=False, keepclone=False): 1815 """Fetch a uri, generating a shallow tarball, then unpack using it""" 1816 fetcher, ud = self.fetch_and_unpack(uri) 1817 assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri) 1818 1819 # Confirm that the unpacked repo is unshallow 1820 if not disabled: 1821 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0])) 1822 1823 # fetch and unpack, from the shallow tarball 1824 bb.utils.remove(self.gitdir, recurse=True) 1825 bb.process.run('chmod u+w -R "%s"' % ud.clonedir) 1826 bb.utils.remove(ud.clonedir, recurse=True) 1827 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True) 1828 1829 # confirm that the unpacked repo is used when no git clone or git 1830 # mirror tarball is available 1831 fetcher, ud = self.fetch_and_unpack(uri) 1832 if not disabled: 1833 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir 1834 else: 1835 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir 1836 return fetcher, ud 1837 1838 def test_shallow_disabled(self): 1839 self.add_empty_file('a') 1840 self.add_empty_file('b') 1841 self.assertRevCount(2, cwd=self.srcdir) 1842 1843 self.d.setVar('BB_GIT_SHALLOW', '0') 1844 self.fetch_shallow(disabled=True) 1845 self.assertRevCount(2) 1846 1847 def test_shallow_nobranch(self): 1848 self.add_empty_file('a') 1849 self.add_empty_file('b') 1850 self.assertRevCount(2, cwd=self.srcdir) 1851 1852 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip() 1853 self.d.setVar('SRCREV', srcrev) 1854 uri = self.d.getVar('SRC_URI').split()[0] 1855 uri = '%s;nobranch=1;bare=1' % uri 1856 1857 self.fetch_shallow(uri) 1858 self.assertRevCount(1) 1859 1860 # shallow refs are used to ensure the srcrev sticks around when we 1861 # have no other branches referencing it 1862 self.assertRefs(['refs/shallow/default']) 1863 1864 def test_shallow_default_depth_1(self): 1865 # Create initial git repo 1866 self.add_empty_file('a') 1867 self.add_empty_file('b') 1868 self.assertRevCount(2, cwd=self.srcdir) 1869 1870 self.fetch_shallow() 1871 self.assertRevCount(1) 1872 1873 def test_shallow_depth_0_disables(self): 1874 self.add_empty_file('a') 1875 self.add_empty_file('b') 1876 self.assertRevCount(2, cwd=self.srcdir) 1877 1878 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1879 self.fetch_shallow(disabled=True) 1880 self.assertRevCount(2) 1881 1882 def test_shallow_depth_default_override(self): 1883 self.add_empty_file('a') 1884 self.add_empty_file('b') 1885 self.assertRevCount(2, cwd=self.srcdir) 1886 1887 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2') 1888 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1') 1889 self.fetch_shallow() 1890 self.assertRevCount(1) 1891 1892 def test_shallow_depth_default_override_disable(self): 1893 self.add_empty_file('a') 1894 self.add_empty_file('b') 1895 self.add_empty_file('c') 1896 self.assertRevCount(3, cwd=self.srcdir) 1897 1898 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1899 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2') 1900 self.fetch_shallow() 1901 self.assertRevCount(2) 1902 1903 def test_current_shallow_out_of_date_clone(self): 1904 # Create initial git repo 1905 self.add_empty_file('a') 1906 self.add_empty_file('b') 1907 self.add_empty_file('c') 1908 self.assertRevCount(3, cwd=self.srcdir) 1909 1910 # Clone and generate mirror tarball 1911 fetcher, ud = self.fetch() 1912 1913 # Ensure we have a current mirror tarball, but an out of date clone 1914 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir) 1915 self.assertRevCount(2, cwd=ud.clonedir) 1916 1917 # Fetch and unpack, from the current tarball, not the out of date clone 1918 bb.utils.remove(self.gitdir, recurse=True) 1919 fetcher, ud = self.fetch() 1920 fetcher.unpack(self.d.getVar('WORKDIR')) 1921 self.assertRevCount(1) 1922 1923 def test_shallow_single_branch_no_merge(self): 1924 self.add_empty_file('a') 1925 self.add_empty_file('b') 1926 self.assertRevCount(2, cwd=self.srcdir) 1927 1928 self.fetch_shallow() 1929 self.assertRevCount(1) 1930 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1931 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1932 1933 def test_shallow_no_dangling(self): 1934 self.add_empty_file('a') 1935 self.add_empty_file('b') 1936 self.assertRevCount(2, cwd=self.srcdir) 1937 1938 self.fetch_shallow() 1939 self.assertRevCount(1) 1940 assert not self.git('fsck --dangling') 1941 1942 def test_shallow_srcrev_branch_truncation(self): 1943 self.add_empty_file('a') 1944 self.add_empty_file('b') 1945 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip() 1946 self.add_empty_file('c') 1947 self.assertRevCount(3, cwd=self.srcdir) 1948 1949 self.d.setVar('SRCREV', b_commit) 1950 self.fetch_shallow() 1951 1952 # The 'c' commit was removed entirely, and 'a' was removed from history 1953 self.assertRevCount(1, ['--all']) 1954 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit) 1955 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1956 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1957 assert not os.path.exists(os.path.join(self.gitdir, 'c')) 1958 1959 def test_shallow_ref_pruning(self): 1960 self.add_empty_file('a') 1961 self.add_empty_file('b') 1962 self.git('branch a_branch', cwd=self.srcdir) 1963 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 1964 self.assertRevCount(2, cwd=self.srcdir) 1965 1966 self.fetch_shallow() 1967 1968 self.assertRefs(['master', 'origin/master']) 1969 self.assertRevCount(1) 1970 1971 def test_shallow_submodules(self): 1972 self.add_empty_file('a') 1973 self.add_empty_file('b') 1974 1975 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1976 bb.utils.mkdirhier(smdir) 1977 self.git_init(cwd=smdir) 1978 # Make this look like it was cloned from a remote... 1979 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1980 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1981 self.add_empty_file('asub', cwd=smdir) 1982 self.add_empty_file('bsub', cwd=smdir) 1983 1984 self.git('submodule init', cwd=self.srcdir) 1985 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 1986 self.git('submodule update', cwd=self.srcdir) 1987 self.git('commit -m submodule -a', cwd=self.srcdir) 1988 1989 uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1990 fetcher, ud = self.fetch_shallow(uri) 1991 1992 # Verify the main repository is shallow 1993 self.assertRevCount(1) 1994 1995 # Verify the gitsubmodule directory is present 1996 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 1997 1998 # Verify the submodule is also shallow 1999 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 2000 2001 def test_shallow_submodule_mirrors(self): 2002 self.add_empty_file('a') 2003 self.add_empty_file('b') 2004 2005 smdir = os.path.join(self.tempdir, 'gitsubmodule') 2006 bb.utils.mkdirhier(smdir) 2007 self.git_init(cwd=smdir) 2008 # Make this look like it was cloned from a remote... 2009 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 2010 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 2011 self.add_empty_file('asub', cwd=smdir) 2012 self.add_empty_file('bsub', cwd=smdir) 2013 2014 self.git('submodule init', cwd=self.srcdir) 2015 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 2016 self.git('submodule update', cwd=self.srcdir) 2017 self.git('commit -m submodule -a', cwd=self.srcdir) 2018 2019 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir 2020 2021 # Fetch once to generate the shallow tarball 2022 fetcher, ud = self.fetch(uri) 2023 2024 # Set up the mirror 2025 mirrordir = os.path.join(self.tempdir, 'mirror') 2026 bb.utils.rename(self.dldir, mirrordir) 2027 self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/' % mirrordir) 2028 2029 # Fetch from the mirror 2030 bb.utils.remove(self.dldir, recurse=True) 2031 bb.utils.remove(self.gitdir, recurse=True) 2032 self.fetch_and_unpack(uri) 2033 2034 # Verify the main repository is shallow 2035 self.assertRevCount(1) 2036 2037 # Verify the gitsubmodule directory is present 2038 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 2039 2040 # Verify the submodule is also shallow 2041 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 2042 2043 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')): 2044 def test_shallow_annex(self): 2045 self.add_empty_file('a') 2046 self.add_empty_file('b') 2047 self.git('annex init', cwd=self.srcdir) 2048 open(os.path.join(self.srcdir, 'c'), 'w').close() 2049 self.git('annex add c', cwd=self.srcdir) 2050 self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir) 2051 bb.process.run('chmod u+w -R %s' % self.srcdir) 2052 2053 uri = 'gitannex://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 2054 fetcher, ud = self.fetch_shallow(uri) 2055 2056 self.assertRevCount(1) 2057 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0] 2058 assert os.path.exists(os.path.join(self.gitdir, 'c')) 2059 2060 def test_shallow_multi_one_uri(self): 2061 # Create initial git repo 2062 self.add_empty_file('a') 2063 self.add_empty_file('b') 2064 self.git('checkout -b a_branch', cwd=self.srcdir) 2065 self.add_empty_file('c') 2066 self.git('tag v0.0 HEAD', cwd=self.srcdir) 2067 self.add_empty_file('d') 2068 self.git('checkout master', cwd=self.srcdir) 2069 self.add_empty_file('e') 2070 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2071 self.add_empty_file('f') 2072 self.assertRevCount(7, cwd=self.srcdir) 2073 2074 uri = self.d.getVar('SRC_URI').split()[0] 2075 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri 2076 2077 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2078 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2079 self.d.setVar('SRCREV_master', '${AUTOREV}') 2080 self.d.setVar('SRCREV_a_branch', '${AUTOREV}') 2081 2082 self.fetch_shallow(uri) 2083 2084 self.assertRevCount(4) 2085 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2086 2087 def test_shallow_multi_one_uri_depths(self): 2088 # Create initial git repo 2089 self.add_empty_file('a') 2090 self.add_empty_file('b') 2091 self.git('checkout -b a_branch', cwd=self.srcdir) 2092 self.add_empty_file('c') 2093 self.add_empty_file('d') 2094 self.git('checkout master', cwd=self.srcdir) 2095 self.add_empty_file('e') 2096 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2097 self.add_empty_file('f') 2098 self.assertRevCount(7, cwd=self.srcdir) 2099 2100 uri = self.d.getVar('SRC_URI').split()[0] 2101 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri 2102 2103 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2104 self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3') 2105 self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1') 2106 self.d.setVar('SRCREV_master', '${AUTOREV}') 2107 self.d.setVar('SRCREV_a_branch', '${AUTOREV}') 2108 2109 self.fetch_shallow(uri) 2110 2111 self.assertRevCount(4, ['--all']) 2112 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2113 2114 def test_shallow_clone_preferred_over_shallow(self): 2115 self.add_empty_file('a') 2116 self.add_empty_file('b') 2117 2118 # Fetch once to generate the shallow tarball 2119 fetcher, ud = self.fetch() 2120 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0])) 2121 2122 # Fetch and unpack with both the clonedir and shallow tarball available 2123 bb.utils.remove(self.gitdir, recurse=True) 2124 fetcher, ud = self.fetch_and_unpack() 2125 2126 # The unpacked tree should *not* be shallow 2127 self.assertRevCount(2) 2128 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')) 2129 2130 def test_shallow_mirrors(self): 2131 self.add_empty_file('a') 2132 self.add_empty_file('b') 2133 2134 # Fetch once to generate the shallow tarball 2135 fetcher, ud = self.fetch() 2136 mirrortarball = ud.mirrortarballs[0] 2137 assert os.path.exists(os.path.join(self.dldir, mirrortarball)) 2138 2139 # Set up the mirror 2140 mirrordir = os.path.join(self.tempdir, 'mirror') 2141 bb.utils.mkdirhier(mirrordir) 2142 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % mirrordir) 2143 2144 bb.utils.rename(os.path.join(self.dldir, mirrortarball), 2145 os.path.join(mirrordir, mirrortarball)) 2146 2147 # Fetch from the mirror 2148 bb.utils.remove(self.dldir, recurse=True) 2149 bb.utils.remove(self.gitdir, recurse=True) 2150 self.fetch_and_unpack() 2151 self.assertRevCount(1) 2152 2153 def test_shallow_invalid_depth(self): 2154 self.add_empty_file('a') 2155 self.add_empty_file('b') 2156 2157 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12') 2158 with self.assertRaises(bb.fetch2.FetchError): 2159 self.fetch() 2160 2161 def test_shallow_invalid_depth_default(self): 2162 self.add_empty_file('a') 2163 self.add_empty_file('b') 2164 2165 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12') 2166 with self.assertRaises(bb.fetch2.FetchError): 2167 self.fetch() 2168 2169 def test_shallow_extra_refs(self): 2170 self.add_empty_file('a') 2171 self.add_empty_file('b') 2172 self.git('branch a_branch', cwd=self.srcdir) 2173 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 2174 self.assertRevCount(2, cwd=self.srcdir) 2175 2176 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch') 2177 self.fetch_shallow() 2178 2179 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2180 self.assertRevCount(1) 2181 2182 def test_shallow_extra_refs_wildcard(self): 2183 self.add_empty_file('a') 2184 self.add_empty_file('b') 2185 self.git('branch a_branch', cwd=self.srcdir) 2186 self.git('tag v1.0', cwd=self.srcdir) 2187 self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir) 2188 self.assertRevCount(2, cwd=self.srcdir) 2189 2190 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2191 self.fetch_shallow() 2192 2193 self.assertRefs(['master', 'origin/master', 'v1.0']) 2194 self.assertRevCount(1) 2195 2196 def test_shallow_missing_extra_refs(self): 2197 self.add_empty_file('a') 2198 self.add_empty_file('b') 2199 2200 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo') 2201 with self.assertRaises(bb.fetch2.FetchError): 2202 self.fetch() 2203 2204 def test_shallow_missing_extra_refs_wildcard(self): 2205 self.add_empty_file('a') 2206 self.add_empty_file('b') 2207 2208 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2209 self.fetch() 2210 2211 def test_shallow_remove_revs(self): 2212 # Create initial git repo 2213 self.add_empty_file('a') 2214 self.add_empty_file('b') 2215 self.git('checkout -b a_branch', cwd=self.srcdir) 2216 self.add_empty_file('c') 2217 self.add_empty_file('d') 2218 self.git('checkout master', cwd=self.srcdir) 2219 self.git('tag v0.0 a_branch', cwd=self.srcdir) 2220 self.add_empty_file('e') 2221 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2222 self.git('branch -d a_branch', cwd=self.srcdir) 2223 self.add_empty_file('f') 2224 self.assertRevCount(7, cwd=self.srcdir) 2225 2226 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2227 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2228 2229 self.fetch_shallow() 2230 2231 self.assertRevCount(2) 2232 2233 def test_shallow_invalid_revs(self): 2234 self.add_empty_file('a') 2235 self.add_empty_file('b') 2236 2237 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2238 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2239 2240 with self.assertRaises(bb.fetch2.FetchError): 2241 self.fetch() 2242 2243 def test_shallow_fetch_missing_revs(self): 2244 self.add_empty_file('a') 2245 self.add_empty_file('b') 2246 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2247 self.git('tag v0.0 master', cwd=self.srcdir) 2248 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2249 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2250 2251 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm: 2252 self.fetch_shallow() 2253 self.assertIn("fatal: no commits selected for shallow requests", cm.output[0]) 2254 2255 def test_shallow_fetch_missing_revs_fails(self): 2256 self.add_empty_file('a') 2257 self.add_empty_file('b') 2258 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2259 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2260 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2261 2262 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm: 2263 self.fetch_shallow() 2264 self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0]) 2265 2266 @skipIfNoNetwork() 2267 def test_bitbake(self): 2268 self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir) 2269 self.git('config core.bare true', cwd=self.srcdir) 2270 self.git('fetch', cwd=self.srcdir) 2271 2272 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2273 # Note that the 1.10.0 tag is annotated, so this also tests 2274 # reference of an annotated vs unannotated tag 2275 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0') 2276 2277 self.fetch_shallow() 2278 2279 # Confirm that the history of 1.10.0 was removed 2280 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines()) 2281 revs = len(self.git('rev-list master').splitlines()) 2282 self.assertNotEqual(orig_revs, revs) 2283 self.assertRefs(['master', 'origin/master']) 2284 self.assertRevCount(orig_revs - 1760) 2285 2286 def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self): 2287 self.add_empty_file('a') 2288 fetcher, ud = self.fetch() 2289 bb.utils.remove(self.gitdir, recurse=True) 2290 bb.utils.remove(self.dldir, recurse=True) 2291 2292 with self.assertRaises(bb.fetch2.UnpackError) as context: 2293 fetcher.unpack(self.d.getVar('WORKDIR')) 2294 2295 self.assertIn("No up to date source found", context.exception.msg) 2296 self.assertIn("clone directory not available or not up to date", context.exception.msg) 2297 2298 @skipIfNoNetwork() 2299 def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self): 2300 self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0') 2301 self.d.setVar('BB_GIT_SHALLOW', '1') 2302 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 2303 fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests;branch=master;protocol=https"], self.d) 2304 fetcher.download() 2305 2306 bb.utils.remove(self.dldir + "/*.tar.gz") 2307 fetcher.unpack(self.unpackdir) 2308 2309 dir = os.listdir(self.unpackdir + "/git/") 2310 self.assertIn("fstests.doap", dir) 2311 2312class GitLfsTest(FetcherTest): 2313 def skipIfNoGitLFS(): 2314 import shutil 2315 if not shutil.which('git-lfs'): 2316 return unittest.skip('git-lfs not installed') 2317 return lambda f: f 2318 2319 def setUp(self): 2320 FetcherTest.setUp(self) 2321 2322 self.gitdir = os.path.join(self.tempdir, 'git') 2323 self.srcdir = os.path.join(self.tempdir, 'gitsource') 2324 2325 self.d.setVar('WORKDIR', self.tempdir) 2326 self.d.setVar('S', self.gitdir) 2327 self.d.delVar('PREMIRRORS') 2328 self.d.delVar('MIRRORS') 2329 2330 self.d.setVar('SRCREV', '${AUTOREV}') 2331 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 2332 self.d.setVar("__BBSRCREV_SEEN", "1") 2333 2334 bb.utils.mkdirhier(self.srcdir) 2335 self.git_init(cwd=self.srcdir) 2336 self.commit_file('.gitattributes', '*.mp3 filter=lfs -text') 2337 2338 def commit_file(self, filename, content): 2339 with open(os.path.join(self.srcdir, filename), "w") as f: 2340 f.write(content) 2341 self.git(["add", filename], cwd=self.srcdir) 2342 self.git(["commit", "-m", "Change"], cwd=self.srcdir) 2343 return self.git(["rev-parse", "HEAD"], cwd=self.srcdir).strip() 2344 2345 def fetch(self, uri=None, download=True): 2346 uris = self.d.getVar('SRC_URI').split() 2347 uri = uris[0] 2348 d = self.d 2349 2350 fetcher = bb.fetch2.Fetch(uris, d) 2351 if download: 2352 fetcher.download() 2353 ud = fetcher.ud[uri] 2354 return fetcher, ud 2355 2356 def get_real_git_lfs_file(self): 2357 self.d.setVar('PATH', os.environ.get('PATH')) 2358 fetcher, ud = self.fetch() 2359 fetcher.unpack(self.d.getVar('WORKDIR')) 2360 unpacked_lfs_file = os.path.join(self.d.getVar('WORKDIR'), 'git', "Cat_poster_1.jpg") 2361 return unpacked_lfs_file 2362 2363 @skipIfNoGitLFS() 2364 def test_fetch_lfs_on_srcrev_change(self): 2365 """Test if fetch downloads missing LFS objects when a different revision within an existing repository is requested""" 2366 self.git(["lfs", "install", "--local"], cwd=self.srcdir) 2367 2368 @contextlib.contextmanager 2369 def hide_upstream_repository(): 2370 """Hide the upstream repository to make sure that git lfs cannot pull from it""" 2371 temp_name = self.srcdir + ".bak" 2372 os.rename(self.srcdir, temp_name) 2373 try: 2374 yield 2375 finally: 2376 os.rename(temp_name, self.srcdir) 2377 2378 def fetch_and_verify(revision, filename, content): 2379 self.d.setVar('SRCREV', revision) 2380 fetcher, ud = self.fetch() 2381 2382 with hide_upstream_repository(): 2383 workdir = self.d.getVar('WORKDIR') 2384 fetcher.unpack(workdir) 2385 2386 with open(os.path.join(workdir, "git", filename)) as f: 2387 self.assertEqual(f.read(), content) 2388 2389 commit_1 = self.commit_file("a.mp3", "version 1") 2390 commit_2 = self.commit_file("a.mp3", "version 2") 2391 2392 self.d.setVar('SRC_URI', "git://%s;protocol=file;lfs=1;branch=master" % self.srcdir) 2393 2394 # Seed the local download folder by fetching the latest commit and verifying that the LFS contents are 2395 # available even when the upstream repository disappears. 2396 fetch_and_verify(commit_2, "a.mp3", "version 2") 2397 # Verify that even when an older revision is fetched, the needed LFS objects are fetched into the download 2398 # folder. 2399 fetch_and_verify(commit_1, "a.mp3", "version 1") 2400 2401 @skipIfNoGitLFS() 2402 @skipIfNoNetwork() 2403 def test_real_git_lfs_repo_succeeds_without_lfs_param(self): 2404 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master") 2405 f = self.get_real_git_lfs_file() 2406 self.assertTrue(os.path.exists(f)) 2407 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) 2408 2409 @skipIfNoGitLFS() 2410 @skipIfNoNetwork() 2411 def test_real_git_lfs_repo_succeeds(self): 2412 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=1") 2413 f = self.get_real_git_lfs_file() 2414 self.assertTrue(os.path.exists(f)) 2415 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) 2416 2417 @skipIfNoGitLFS() 2418 @skipIfNoNetwork() 2419 def test_real_git_lfs_repo_skips(self): 2420 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=0") 2421 f = self.get_real_git_lfs_file() 2422 # This is the actual non-smudged placeholder file on the repo if git-lfs does not run 2423 lfs_file = ( 2424 'version https://git-lfs.github.com/spec/v1\n' 2425 'oid sha256:34be66b1a39a1955b46a12588df9d5f6fc1da790e05cf01f3c7422f4bbbdc26b\n' 2426 'size 11423554\n' 2427 ) 2428 2429 with open(f) as fh: 2430 self.assertEqual(lfs_file, fh.read()) 2431 2432 @skipIfNoGitLFS() 2433 def test_lfs_enabled(self): 2434 import shutil 2435 2436 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir 2437 self.d.setVar('SRC_URI', uri) 2438 2439 # With git-lfs installed, test that we can fetch and unpack 2440 fetcher, ud = self.fetch() 2441 shutil.rmtree(self.gitdir, ignore_errors=True) 2442 fetcher.unpack(self.d.getVar('WORKDIR')) 2443 2444 @skipIfNoGitLFS() 2445 def test_lfs_disabled(self): 2446 import shutil 2447 2448 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir 2449 self.d.setVar('SRC_URI', uri) 2450 2451 # Verify that the fetcher can survive even if the source 2452 # repository has Git LFS usage configured. 2453 fetcher, ud = self.fetch() 2454 fetcher.unpack(self.d.getVar('WORKDIR')) 2455 2456 def test_lfs_enabled_not_installed(self): 2457 import shutil 2458 2459 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir 2460 self.d.setVar('SRC_URI', uri) 2461 2462 # Careful: suppress initial attempt at downloading 2463 fetcher, ud = self.fetch(uri=None, download=False) 2464 2465 # Artificially assert that git-lfs is not installed, so 2466 # we can verify a failure to unpack in it's absence. 2467 old_find_git_lfs = ud.method._find_git_lfs 2468 try: 2469 # If git-lfs cannot be found, the unpack should throw an error 2470 with self.assertRaises(bb.fetch2.FetchError): 2471 fetcher.download() 2472 ud.method._find_git_lfs = lambda d: False 2473 shutil.rmtree(self.gitdir, ignore_errors=True) 2474 fetcher.unpack(self.d.getVar('WORKDIR')) 2475 finally: 2476 ud.method._find_git_lfs = old_find_git_lfs 2477 2478 def test_lfs_disabled_not_installed(self): 2479 import shutil 2480 2481 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir 2482 self.d.setVar('SRC_URI', uri) 2483 2484 # Careful: suppress initial attempt at downloading 2485 fetcher, ud = self.fetch(uri=None, download=False) 2486 2487 # Artificially assert that git-lfs is not installed, so 2488 # we can verify a failure to unpack in it's absence. 2489 old_find_git_lfs = ud.method._find_git_lfs 2490 try: 2491 # Even if git-lfs cannot be found, the unpack should be successful 2492 fetcher.download() 2493 ud.method._find_git_lfs = lambda d: False 2494 shutil.rmtree(self.gitdir, ignore_errors=True) 2495 fetcher.unpack(self.d.getVar('WORKDIR')) 2496 finally: 2497 ud.method._find_git_lfs = old_find_git_lfs 2498 2499class GitURLWithSpacesTest(FetcherTest): 2500 test_git_urls = { 2501 "git://tfs-example.org:22/tfs/example%20path/example.git;branch=master" : { 2502 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git;branch=master', 2503 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git', 2504 'path': '/tfs/example path/example.git' 2505 }, 2506 "git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master" : { 2507 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master', 2508 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git', 2509 'path': '/tfs/example path/example repo.git' 2510 } 2511 } 2512 2513 def test_urls(self): 2514 2515 # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo 2516 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 2517 2518 for test_git_url, ref in self.test_git_urls.items(): 2519 2520 fetcher = bb.fetch.Fetch([test_git_url], self.d) 2521 ud = fetcher.ud[fetcher.urls[0]] 2522 2523 self.assertEqual(ud.url, ref['url']) 2524 self.assertEqual(ud.path, ref['path']) 2525 self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2526 self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2527 self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock')) 2528 self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2529 self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz')) 2530 2531class CrateTest(FetcherTest): 2532 @skipIfNoNetwork() 2533 def test_crate_url(self): 2534 2535 uri = "crate://crates.io/glob/0.2.11" 2536 self.d.setVar('SRC_URI', uri) 2537 2538 uris = self.d.getVar('SRC_URI').split() 2539 d = self.d 2540 2541 fetcher = bb.fetch2.Fetch(uris, self.d) 2542 ud = fetcher.ud[fetcher.urls[0]] 2543 2544 self.assertIn("name", ud.parm) 2545 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2546 self.assertIn("downloadfilename", ud.parm) 2547 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2548 2549 fetcher.download() 2550 fetcher.unpack(self.tempdir) 2551 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2552 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) 2553 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2554 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2555 2556 @skipIfNoNetwork() 2557 def test_crate_url_matching_recipe(self): 2558 2559 self.d.setVar('BP', 'glob-0.2.11') 2560 2561 uri = "crate://crates.io/glob/0.2.11" 2562 self.d.setVar('SRC_URI', uri) 2563 2564 uris = self.d.getVar('SRC_URI').split() 2565 d = self.d 2566 2567 fetcher = bb.fetch2.Fetch(uris, self.d) 2568 ud = fetcher.ud[fetcher.urls[0]] 2569 2570 self.assertIn("name", ud.parm) 2571 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2572 self.assertIn("downloadfilename", ud.parm) 2573 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2574 2575 fetcher.download() 2576 fetcher.unpack(self.tempdir) 2577 self.assertEqual(sorted(os.listdir(self.tempdir)), ['download', 'glob-0.2.11', 'unpacked']) 2578 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) 2579 self.assertTrue(os.path.exists(self.tempdir + "/glob-0.2.11/src/lib.rs")) 2580 2581 @skipIfNoNetwork() 2582 def test_crate_url_params(self): 2583 2584 uri = "crate://crates.io/aho-corasick/0.7.20;name=aho-corasick-renamed" 2585 self.d.setVar('SRC_URI', uri) 2586 2587 uris = self.d.getVar('SRC_URI').split() 2588 d = self.d 2589 2590 fetcher = bb.fetch2.Fetch(uris, self.d) 2591 ud = fetcher.ud[fetcher.urls[0]] 2592 2593 self.assertIn("name", ud.parm) 2594 self.assertEqual(ud.parm["name"], "aho-corasick-renamed") 2595 self.assertIn("downloadfilename", ud.parm) 2596 self.assertEqual(ud.parm["downloadfilename"], "aho-corasick-0.7.20.crate") 2597 2598 fetcher.download() 2599 fetcher.unpack(self.tempdir) 2600 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2601 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['aho-corasick-0.7.20.crate', 'aho-corasick-0.7.20.crate.done']) 2602 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/.cargo-checksum.json")) 2603 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/src/lib.rs")) 2604 2605 @skipIfNoNetwork() 2606 def test_crate_url_multi(self): 2607 2608 uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35" 2609 self.d.setVar('SRC_URI', uri) 2610 2611 uris = self.d.getVar('SRC_URI').split() 2612 d = self.d 2613 2614 fetcher = bb.fetch2.Fetch(uris, self.d) 2615 ud = fetcher.ud[fetcher.urls[0]] 2616 2617 self.assertIn("name", ud.parm) 2618 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2619 self.assertIn("downloadfilename", ud.parm) 2620 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2621 2622 ud = fetcher.ud[fetcher.urls[1]] 2623 self.assertIn("name", ud.parm) 2624 self.assertEqual(ud.parm["name"], "time-0.1.35") 2625 self.assertIn("downloadfilename", ud.parm) 2626 self.assertEqual(ud.parm["downloadfilename"], "time-0.1.35.crate") 2627 2628 fetcher.download() 2629 fetcher.unpack(self.tempdir) 2630 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2631 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done', 'time-0.1.35.crate', 'time-0.1.35.crate.done']) 2632 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2633 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2634 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json")) 2635 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs")) 2636 2637 @skipIfNoNetwork() 2638 def test_crate_incorrect_cksum(self): 2639 uri = "crate://crates.io/aho-corasick/0.7.20" 2640 self.d.setVar('SRC_URI', uri) 2641 self.d.setVarFlag("SRC_URI", "aho-corasick-0.7.20.sha256sum", hashlib.sha256("Invalid".encode("utf-8")).hexdigest()) 2642 2643 uris = self.d.getVar('SRC_URI').split() 2644 2645 fetcher = bb.fetch2.Fetch(uris, self.d) 2646 with self.assertRaisesRegex(bb.fetch2.FetchError, "Fetcher failure for URL"): 2647 fetcher.download() 2648 2649class NPMTest(FetcherTest): 2650 def skipIfNoNpm(): 2651 import shutil 2652 if not shutil.which('npm'): 2653 return unittest.skip('npm not installed') 2654 return lambda f: f 2655 2656 @skipIfNoNpm() 2657 @skipIfNoNetwork() 2658 def test_npm(self): 2659 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2660 fetcher = bb.fetch.Fetch(urls, self.d) 2661 ud = fetcher.ud[fetcher.urls[0]] 2662 fetcher.download() 2663 self.assertTrue(os.path.exists(ud.localpath)) 2664 self.assertTrue(os.path.exists(ud.localpath + '.done')) 2665 self.assertTrue(os.path.exists(ud.resolvefile)) 2666 fetcher.unpack(self.unpackdir) 2667 unpackdir = os.path.join(self.unpackdir, 'npm') 2668 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2669 2670 @skipIfNoNpm() 2671 @skipIfNoNetwork() 2672 def test_npm_bad_checksum(self): 2673 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2674 # Fetch once to get a tarball 2675 fetcher = bb.fetch.Fetch(urls, self.d) 2676 ud = fetcher.ud[fetcher.urls[0]] 2677 fetcher.download() 2678 self.assertTrue(os.path.exists(ud.localpath)) 2679 # Modify the tarball 2680 bad = b'bad checksum' 2681 with open(ud.localpath, 'wb') as f: 2682 f.write(bad) 2683 # Verify that the tarball is fetched again 2684 fetcher.download() 2685 badsum = hashlib.sha512(bad).hexdigest() 2686 self.assertTrue(os.path.exists(ud.localpath + '_bad-checksum_' + badsum)) 2687 self.assertTrue(os.path.exists(ud.localpath)) 2688 2689 @skipIfNoNpm() 2690 @skipIfNoNetwork() 2691 def test_npm_premirrors(self): 2692 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2693 # Fetch once to get a tarball 2694 fetcher = bb.fetch.Fetch(urls, self.d) 2695 ud = fetcher.ud[fetcher.urls[0]] 2696 fetcher.download() 2697 self.assertTrue(os.path.exists(ud.localpath)) 2698 2699 # Setup the mirror by renaming the download directory 2700 mirrordir = os.path.join(self.tempdir, 'mirror') 2701 bb.utils.rename(self.dldir, mirrordir) 2702 os.mkdir(self.dldir) 2703 2704 # Configure the premirror to be used 2705 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/npm2' % mirrordir) 2706 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2707 2708 # Fetch again 2709 self.assertFalse(os.path.exists(ud.localpath)) 2710 # The npm fetcher doesn't handle that the .resolved file disappears 2711 # while the fetcher object exists, which it does when we rename the 2712 # download directory to "mirror" above. Thus we need a new fetcher to go 2713 # with the now empty download directory. 2714 fetcher = bb.fetch.Fetch(urls, self.d) 2715 ud = fetcher.ud[fetcher.urls[0]] 2716 fetcher.download() 2717 self.assertTrue(os.path.exists(ud.localpath)) 2718 2719 @skipIfNoNpm() 2720 @skipIfNoNetwork() 2721 def test_npm_premirrors_with_specified_filename(self): 2722 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2723 # Fetch once to get a tarball 2724 fetcher = bb.fetch.Fetch(urls, self.d) 2725 ud = fetcher.ud[fetcher.urls[0]] 2726 fetcher.download() 2727 self.assertTrue(os.path.exists(ud.localpath)) 2728 # Setup the mirror 2729 mirrordir = os.path.join(self.tempdir, 'mirror') 2730 bb.utils.mkdirhier(mirrordir) 2731 mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath)) 2732 os.replace(ud.localpath, mirrorfilename) 2733 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s' % mirrorfilename) 2734 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2735 # Fetch again 2736 self.assertFalse(os.path.exists(ud.localpath)) 2737 fetcher.download() 2738 self.assertTrue(os.path.exists(ud.localpath)) 2739 2740 @skipIfNoNpm() 2741 @skipIfNoNetwork() 2742 def test_npm_mirrors(self): 2743 # Fetch once to get a tarball 2744 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2745 fetcher = bb.fetch.Fetch(urls, self.d) 2746 ud = fetcher.ud[fetcher.urls[0]] 2747 fetcher.download() 2748 self.assertTrue(os.path.exists(ud.localpath)) 2749 # Setup the mirror 2750 mirrordir = os.path.join(self.tempdir, 'mirror') 2751 bb.utils.mkdirhier(mirrordir) 2752 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 2753 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 2754 # Update the resolved url to an invalid url 2755 with open(ud.resolvefile, 'r') as f: 2756 url = f.read() 2757 uri = URI(url) 2758 uri.path = '/invalid' 2759 with open(ud.resolvefile, 'w') as f: 2760 f.write(str(uri)) 2761 # Fetch again 2762 self.assertFalse(os.path.exists(ud.localpath)) 2763 fetcher.download() 2764 self.assertTrue(os.path.exists(ud.localpath)) 2765 2766 @skipIfNoNpm() 2767 @skipIfNoNetwork() 2768 def test_npm_destsuffix_downloadfilename(self): 2769 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz'] 2770 fetcher = bb.fetch.Fetch(urls, self.d) 2771 fetcher.download() 2772 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'foo-bar.tgz'))) 2773 fetcher.unpack(self.unpackdir) 2774 unpackdir = os.path.join(self.unpackdir, 'foo', 'bar') 2775 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2776 2777 def test_npm_no_network_no_tarball(self): 2778 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2779 self.d.setVar('BB_NO_NETWORK', '1') 2780 fetcher = bb.fetch.Fetch(urls, self.d) 2781 with self.assertRaises(bb.fetch2.NetworkAccess): 2782 fetcher.download() 2783 2784 @skipIfNoNpm() 2785 @skipIfNoNetwork() 2786 def test_npm_no_network_with_tarball(self): 2787 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2788 # Fetch once to get a tarball 2789 fetcher = bb.fetch.Fetch(urls, self.d) 2790 fetcher.download() 2791 # Disable network access 2792 self.d.setVar('BB_NO_NETWORK', '1') 2793 # Fetch again 2794 fetcher.download() 2795 fetcher.unpack(self.unpackdir) 2796 unpackdir = os.path.join(self.unpackdir, 'npm') 2797 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2798 2799 @skipIfNoNpm() 2800 @skipIfNoNetwork() 2801 def test_npm_registry_alternate(self): 2802 urls = ['npm://skimdb.npmjs.com;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2803 fetcher = bb.fetch.Fetch(urls, self.d) 2804 fetcher.download() 2805 fetcher.unpack(self.unpackdir) 2806 unpackdir = os.path.join(self.unpackdir, 'npm') 2807 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2808 2809 @skipIfNoNpm() 2810 @skipIfNoNetwork() 2811 def test_npm_version_latest(self): 2812 url = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest'] 2813 fetcher = bb.fetch.Fetch(urls, self.d) 2814 fetcher.download() 2815 fetcher.unpack(self.unpackdir) 2816 unpackdir = os.path.join(self.unpackdir, 'npm') 2817 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2818 2819 @skipIfNoNpm() 2820 @skipIfNoNetwork() 2821 def test_npm_registry_invalid(self): 2822 urls = ['npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2823 fetcher = bb.fetch.Fetch(urls, self.d) 2824 with self.assertRaises(bb.fetch2.FetchError): 2825 fetcher.download() 2826 2827 @skipIfNoNpm() 2828 @skipIfNoNetwork() 2829 def test_npm_package_invalid(self): 2830 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0'] 2831 fetcher = bb.fetch.Fetch(urls, self.d) 2832 with self.assertRaises(bb.fetch2.FetchError): 2833 fetcher.download() 2834 2835 @skipIfNoNpm() 2836 @skipIfNoNetwork() 2837 def test_npm_version_invalid(self): 2838 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid'] 2839 with self.assertRaises(bb.fetch2.ParameterError): 2840 fetcher = bb.fetch.Fetch(urls, self.d) 2841 2842 @skipIfNoNpm() 2843 @skipIfNoNetwork() 2844 def test_npm_registry_none(self): 2845 urls = ['npm://;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2846 with self.assertRaises(bb.fetch2.MalformedUrl): 2847 fetcher = bb.fetch.Fetch(urls, self.d) 2848 2849 @skipIfNoNpm() 2850 @skipIfNoNetwork() 2851 def test_npm_package_none(self): 2852 urls = ['npm://registry.npmjs.org;version=1.0.0'] 2853 with self.assertRaises(bb.fetch2.MissingParameterError): 2854 fetcher = bb.fetch.Fetch(urls, self.d) 2855 2856 @skipIfNoNpm() 2857 @skipIfNoNetwork() 2858 def test_npm_version_none(self): 2859 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example'] 2860 with self.assertRaises(bb.fetch2.MissingParameterError): 2861 fetcher = bb.fetch.Fetch(urls, self.d) 2862 2863 def create_shrinkwrap_file(self, data): 2864 import json 2865 datadir = os.path.join(self.tempdir, 'data') 2866 swfile = os.path.join(datadir, 'npm-shrinkwrap.json') 2867 bb.utils.mkdirhier(datadir) 2868 with open(swfile, 'w') as f: 2869 json.dump(data, f) 2870 return swfile 2871 2872 @skipIfNoNetwork() 2873 def test_npmsw(self): 2874 swfile = self.create_shrinkwrap_file({ 2875 'packages': { 2876 'node_modules/array-flatten': { 2877 'version': '1.1.1', 2878 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2879 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=', 2880 'dependencies': { 2881 'content-type': "1.0.4" 2882 } 2883 }, 2884 'node_modules/array-flatten/node_modules/content-type': { 2885 'version': '1.0.4', 2886 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 2887 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 2888 'dependencies': { 2889 'cookie': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09' 2890 } 2891 }, 2892 'node_modules/array-flatten/node_modules/content-type/node_modules/cookie': { 2893 'resolved': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09' 2894 } 2895 } 2896 }) 2897 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2898 fetcher.download() 2899 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2900 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2901 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2902 fetcher.unpack(self.unpackdir) 2903 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm-shrinkwrap.json'))) 2904 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'package.json'))) 2905 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json'))) 2906 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json'))) 2907 2908 @skipIfNoNetwork() 2909 def test_npmsw_git(self): 2910 swfile = self.create_shrinkwrap_file({ 2911 'packages': { 2912 'node_modules/cookie': { 2913 'resolved': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09' 2914 } 2915 } 2916 }) 2917 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2918 fetcher.download() 2919 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2920 2921 @skipIfNoNetwork() 2922 def test_npmsw_dev(self): 2923 swfile = self.create_shrinkwrap_file({ 2924 'packages': { 2925 'node_modules/array-flatten': { 2926 'version': '1.1.1', 2927 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2928 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2929 }, 2930 'node_modules/content-type': { 2931 'version': '1.0.4', 2932 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 2933 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 2934 'dev': True 2935 } 2936 } 2937 }) 2938 # Fetch with dev disabled 2939 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2940 fetcher.download() 2941 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2942 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2943 # Fetch with dev enabled 2944 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d) 2945 fetcher.download() 2946 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2947 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2948 2949 @skipIfNoNetwork() 2950 def test_npmsw_destsuffix(self): 2951 swfile = self.create_shrinkwrap_file({ 2952 'packages': { 2953 'node_modules/array-flatten': { 2954 'version': '1.1.1', 2955 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2956 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2957 } 2958 } 2959 }) 2960 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d) 2961 fetcher.download() 2962 fetcher.unpack(self.unpackdir) 2963 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json'))) 2964 2965 def test_npmsw_no_network_no_tarball(self): 2966 swfile = self.create_shrinkwrap_file({ 2967 'packages': { 2968 'node_modules/array-flatten': { 2969 'version': '1.1.1', 2970 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2971 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2972 } 2973 } 2974 }) 2975 self.d.setVar('BB_NO_NETWORK', '1') 2976 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2977 with self.assertRaises(bb.fetch2.NetworkAccess): 2978 fetcher.download() 2979 2980 @skipIfNoNpm() 2981 @skipIfNoNetwork() 2982 def test_npmsw_no_network_with_tarball(self): 2983 # Fetch once to get a tarball 2984 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2985 fetcher.download() 2986 # Disable network access 2987 self.d.setVar('BB_NO_NETWORK', '1') 2988 # Fetch again 2989 swfile = self.create_shrinkwrap_file({ 2990 'packages': { 2991 'node_modules/array-flatten': { 2992 'version': '1.1.1', 2993 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2994 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2995 } 2996 } 2997 }) 2998 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2999 fetcher.download() 3000 fetcher.unpack(self.unpackdir) 3001 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'package.json'))) 3002 3003 @skipIfNoNetwork() 3004 def test_npmsw_npm_reusability(self): 3005 # Fetch once with npmsw 3006 swfile = self.create_shrinkwrap_file({ 3007 'packages': { 3008 'node_modules/array-flatten': { 3009 'version': '1.1.1', 3010 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3011 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3012 } 3013 } 3014 }) 3015 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3016 fetcher.download() 3017 # Disable network access 3018 self.d.setVar('BB_NO_NETWORK', '1') 3019 # Fetch again with npm 3020 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 3021 fetcher.download() 3022 fetcher.unpack(self.unpackdir) 3023 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json'))) 3024 3025 @skipIfNoNetwork() 3026 def test_npmsw_bad_checksum(self): 3027 # Try to fetch with bad checksum 3028 swfile = self.create_shrinkwrap_file({ 3029 'packages': { 3030 'node_modules/array-flatten': { 3031 'version': '1.1.1', 3032 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3033 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg=' 3034 } 3035 } 3036 }) 3037 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3038 with self.assertRaises(bb.fetch2.FetchError): 3039 fetcher.download() 3040 # Fetch correctly to get a tarball 3041 swfile = self.create_shrinkwrap_file({ 3042 'packages': { 3043 'node_modules/array-flatten': { 3044 'version': '1.1.1', 3045 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3046 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3047 } 3048 } 3049 }) 3050 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3051 fetcher.download() 3052 localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz') 3053 self.assertTrue(os.path.exists(localpath)) 3054 # Modify the tarball 3055 bad = b'bad checksum' 3056 with open(localpath, 'wb') as f: 3057 f.write(bad) 3058 # Verify that the tarball is fetched again 3059 fetcher.download() 3060 badsum = hashlib.sha1(bad).hexdigest() 3061 self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum)) 3062 self.assertTrue(os.path.exists(localpath)) 3063 3064 @skipIfNoNpm() 3065 @skipIfNoNetwork() 3066 def test_npmsw_premirrors(self): 3067 # Fetch once to get a tarball 3068 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 3069 ud = fetcher.ud[fetcher.urls[0]] 3070 fetcher.download() 3071 self.assertTrue(os.path.exists(ud.localpath)) 3072 # Setup the mirror 3073 mirrordir = os.path.join(self.tempdir, 'mirror') 3074 bb.utils.mkdirhier(mirrordir) 3075 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 3076 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 3077 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 3078 # Fetch again 3079 self.assertFalse(os.path.exists(ud.localpath)) 3080 swfile = self.create_shrinkwrap_file({ 3081 'packages': { 3082 'node_modules/array-flatten': { 3083 'version': '1.1.1', 3084 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3085 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3086 } 3087 } 3088 }) 3089 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3090 fetcher.download() 3091 self.assertTrue(os.path.exists(ud.localpath)) 3092 3093 @skipIfNoNpm() 3094 @skipIfNoNetwork() 3095 def test_npmsw_mirrors(self): 3096 # Fetch once to get a tarball 3097 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 3098 ud = fetcher.ud[fetcher.urls[0]] 3099 fetcher.download() 3100 self.assertTrue(os.path.exists(ud.localpath)) 3101 # Setup the mirror 3102 mirrordir = os.path.join(self.tempdir, 'mirror') 3103 bb.utils.mkdirhier(mirrordir) 3104 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 3105 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 3106 # Fetch again with invalid url 3107 self.assertFalse(os.path.exists(ud.localpath)) 3108 swfile = self.create_shrinkwrap_file({ 3109 'packages': { 3110 'node_modules/array-flatten': { 3111 'version': '1.1.1', 3112 'resolved': 'https://invalid', 3113 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3114 } 3115 } 3116 }) 3117 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3118 fetcher.download() 3119 self.assertTrue(os.path.exists(ud.localpath)) 3120 3121 @skipIfNoNetwork() 3122 def test_npmsw_bundled(self): 3123 swfile = self.create_shrinkwrap_file({ 3124 'packages': { 3125 'node_modules/array-flatten': { 3126 'version': '1.1.1', 3127 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3128 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3129 }, 3130 'node_modules/content-type': { 3131 'version': '1.0.4', 3132 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 3133 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 3134 'inBundle': True 3135 } 3136 } 3137 }) 3138 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3139 fetcher.download() 3140 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 3141 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 3142 3143class GitSharedTest(FetcherTest): 3144 def setUp(self): 3145 super(GitSharedTest, self).setUp() 3146 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 3147 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 3148 self.d.setVar("__BBSRCREV_SEEN", "1") 3149 3150 @skipIfNoNetwork() 3151 def test_shared_unpack(self): 3152 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3153 3154 fetcher.download() 3155 fetcher.unpack(self.unpackdir) 3156 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 3157 self.assertTrue(os.path.exists(alt)) 3158 3159 @skipIfNoNetwork() 3160 def test_noshared_unpack(self): 3161 self.d.setVar('BB_GIT_NOSHARED', '1') 3162 self.unpackdir += '_noshared' 3163 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3164 3165 fetcher.download() 3166 fetcher.unpack(self.unpackdir) 3167 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 3168 self.assertFalse(os.path.exists(alt)) 3169 3170 3171class FetchPremirroronlyLocalTest(FetcherTest): 3172 3173 def setUp(self): 3174 super(FetchPremirroronlyLocalTest, self).setUp() 3175 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3176 os.mkdir(self.mirrordir) 3177 self.reponame = "bitbake" 3178 self.gitdir = os.path.join(self.tempdir, "git", self.reponame) 3179 self.recipe_url = "git://git.fake.repo/bitbake;branch=master;protocol=https" 3180 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3181 self.d.setVar("BB_NO_NETWORK", "1") 3182 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3183 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" 3184 self.mirrorfile = os.path.join(self.mirrordir, self.mirrorname) 3185 self.testfilename = "bitbake-fetch.test" 3186 3187 def make_git_repo(self): 3188 recipeurl = "git:/git.fake.repo/bitbake" 3189 os.makedirs(self.gitdir) 3190 self.git_init(cwd=self.gitdir) 3191 for i in range(0): 3192 self.git_new_commit() 3193 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3194 3195 def git_new_commit(self): 3196 import random 3197 os.unlink(os.path.join(self.mirrordir, self.mirrorname)) 3198 branch = self.git("branch --show-current", self.gitdir).split() 3199 with open(os.path.join(self.gitdir, self.testfilename), "w") as testfile: 3200 testfile.write("File {} from branch {}; Useless random data {}".format(self.testfilename, branch, random.random())) 3201 self.git("add {}".format(self.testfilename), self.gitdir) 3202 self.git("commit -a -m \"This random commit {} in branch {}. I'm useless.\"".format(random.random(), branch), self.gitdir) 3203 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3204 return self.git("rev-parse HEAD", self.gitdir).strip() 3205 3206 def git_new_branch(self, name): 3207 self.git_new_commit() 3208 head = self.git("rev-parse HEAD", self.gitdir).strip() 3209 self.git("checkout -b {}".format(name), self.gitdir) 3210 newrev = self.git_new_commit() 3211 self.git("checkout {}".format(head), self.gitdir) 3212 return newrev 3213 3214 def test_mirror_multiple_fetches(self): 3215 self.make_git_repo() 3216 self.d.setVar("SRCREV", self.git_new_commit()) 3217 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3218 fetcher.download() 3219 fetcher.unpack(self.unpackdir) 3220 ## New commit in premirror. it's not in the download_dir 3221 self.d.setVar("SRCREV", self.git_new_commit()) 3222 fetcher2 = bb.fetch.Fetch([self.recipe_url], self.d) 3223 fetcher2.download() 3224 fetcher2.unpack(self.unpackdir) 3225 ## New commit in premirror. it's not in the download_dir 3226 self.d.setVar("SRCREV", self.git_new_commit()) 3227 fetcher3 = bb.fetch.Fetch([self.recipe_url], self.d) 3228 fetcher3.download() 3229 fetcher3.unpack(self.unpackdir) 3230 3231 3232 def test_mirror_commit_nonexistent(self): 3233 self.make_git_repo() 3234 self.d.setVar("SRCREV", "0"*40) 3235 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3236 with self.assertRaises(bb.fetch2.NetworkAccess): 3237 fetcher.download() 3238 3239 def test_mirror_commit_exists(self): 3240 self.make_git_repo() 3241 self.d.setVar("SRCREV", self.git_new_commit()) 3242 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3243 fetcher.download() 3244 fetcher.unpack(self.unpackdir) 3245 3246 def test_mirror_tarball_nonexistent(self): 3247 self.d.setVar("SRCREV", "0"*40) 3248 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3249 with self.assertRaises(bb.fetch2.NetworkAccess): 3250 fetcher.download() 3251 3252 def test_mirror_tarball_multiple_branches(self): 3253 """ 3254 test if PREMIRRORS can handle multiple name/branches correctly 3255 both branches have required revisions 3256 """ 3257 self.make_git_repo() 3258 branch1rev = self.git_new_branch("testbranch1") 3259 branch2rev = self.git_new_branch("testbranch2") 3260 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2" 3261 self.d.setVar("SRCREV_branch1", branch1rev) 3262 self.d.setVar("SRCREV_branch2", branch2rev) 3263 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3264 self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist") 3265 fetcher.download() 3266 fetcher.unpack(os.path.join(self.tempdir, "unpacked")) 3267 unpacked = os.path.join(self.tempdir, "unpacked", "git", self.testfilename) 3268 self.assertTrue(os.path.exists(unpacked), "Repo has not been unpackaged properly!") 3269 with open(unpacked, 'r') as f: 3270 content = f.read() 3271 ## We expect to see testbranch1 in the file, not master, not testbranch2 3272 self.assertTrue(content.find("testbranch1") != -1, "Wrong branch has been checked out!") 3273 3274 def test_mirror_tarball_multiple_branches_nobranch(self): 3275 """ 3276 test if PREMIRRORS can handle multiple name/branches correctly 3277 Unbalanced name/branches raises ParameterError 3278 """ 3279 self.make_git_repo() 3280 branch1rev = self.git_new_branch("testbranch1") 3281 branch2rev = self.git_new_branch("testbranch2") 3282 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1;protocol=https;name=branch1,branch2" 3283 self.d.setVar("SRCREV_branch1", branch1rev) 3284 self.d.setVar("SRCREV_branch2", branch2rev) 3285 with self.assertRaises(bb.fetch2.ParameterError): 3286 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3287 3288 def test_mirror_tarball_multiple_branches_norev(self): 3289 """ 3290 test if PREMIRRORS can handle multiple name/branches correctly 3291 one of the branches specifies non existing SRCREV 3292 """ 3293 self.make_git_repo() 3294 branch1rev = self.git_new_branch("testbranch1") 3295 branch2rev = self.git_new_branch("testbranch2") 3296 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2" 3297 self.d.setVar("SRCREV_branch1", branch1rev) 3298 self.d.setVar("SRCREV_branch2", "0"*40) 3299 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3300 self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist") 3301 with self.assertRaises(bb.fetch2.NetworkAccess): 3302 fetcher.download() 3303 3304 3305class FetchPremirroronlyNetworkTest(FetcherTest): 3306 3307 def setUp(self): 3308 super(FetchPremirroronlyNetworkTest, self).setUp() 3309 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3310 os.mkdir(self.mirrordir) 3311 self.reponame = "fstests" 3312 self.clonedir = os.path.join(self.tempdir, "git") 3313 self.gitdir = os.path.join(self.tempdir, "git", "{}.git".format(self.reponame)) 3314 self.recipe_url = "git://git.yoctoproject.org/fstests;protocol=https" 3315 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3316 self.d.setVar("BB_NO_NETWORK", "0") 3317 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3318 3319 def make_git_repo(self): 3320 import shutil 3321 self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz" 3322 os.makedirs(self.clonedir) 3323 self.git("clone --bare --shallow-since=\"01.01.2013\" {}".format(self.recipe_url), self.clonedir) 3324 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3325 shutil.rmtree(self.clonedir) 3326 3327 @skipIfNoNetwork() 3328 def test_mirror_tarball_updated(self): 3329 self.make_git_repo() 3330 ## Upstream commit is in the mirror 3331 self.d.setVar("SRCREV", "49d65d53c2bf558ae6e9185af0f3af7b79d255ec") 3332 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3333 fetcher.download() 3334 3335 @skipIfNoNetwork() 3336 def test_mirror_tarball_outdated(self): 3337 self.make_git_repo() 3338 ## Upstream commit not in the mirror 3339 self.d.setVar("SRCREV", "15413486df1f5a5b5af699b6f3ba5f0984e52a9f") 3340 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3341 with self.assertRaises(bb.fetch2.NetworkAccess): 3342 fetcher.download() 3343 3344class FetchPremirroronlyMercurialTest(FetcherTest): 3345 """ Test for premirrors with mercurial repos 3346 the test covers also basic hg:// clone (see fetch_and_create_tarball 3347 """ 3348 def skipIfNoHg(): 3349 import shutil 3350 if not shutil.which('hg'): 3351 return unittest.skip('Mercurial not installed') 3352 return lambda f: f 3353 3354 def setUp(self): 3355 super(FetchPremirroronlyMercurialTest, self).setUp() 3356 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3357 os.mkdir(self.mirrordir) 3358 self.reponame = "libgnt" 3359 self.clonedir = os.path.join(self.tempdir, "hg") 3360 self.recipe_url = "hg://keep.imfreedom.org/libgnt;module=libgnt" 3361 self.d.setVar("SRCREV", "53e8b422faaf") 3362 self.mirrorname = "hg_libgnt_keep.imfreedom.org_.libgnt.tar.gz" 3363 3364 def fetch_and_create_tarball(self): 3365 """ 3366 Ask bitbake to download repo and prepare mirror tarball for us 3367 """ 3368 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 3369 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3370 fetcher.download() 3371 mirrorfile = os.path.join(self.d.getVar("DL_DIR"), self.mirrorname) 3372 self.assertTrue(os.path.exists(mirrorfile), "Mirror tarball {} has not been created".format(mirrorfile)) 3373 ## moving tarball to mirror directory 3374 os.rename(mirrorfile, os.path.join(self.mirrordir, self.mirrorname)) 3375 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "0") 3376 3377 3378 @skipIfNoNetwork() 3379 @skipIfNoHg() 3380 def test_premirror_mercurial(self): 3381 self.fetch_and_create_tarball() 3382 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3383 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3384 self.d.setVar("BB_NO_NETWORK", "1") 3385 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3386 fetcher.download() 3387 3388class FetchPremirroronlyBrokenTarball(FetcherTest): 3389 3390 def setUp(self): 3391 super(FetchPremirroronlyBrokenTarball, self).setUp() 3392 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3393 os.mkdir(self.mirrordir) 3394 self.reponame = "bitbake" 3395 self.gitdir = os.path.join(self.tempdir, "git", self.reponame) 3396 self.recipe_url = "git://git.fake.repo/bitbake;protocol=https" 3397 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3398 self.d.setVar("BB_NO_NETWORK", "1") 3399 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3400 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" 3401 with open(os.path.join(self.mirrordir, self.mirrorname), 'w') as targz: 3402 targz.write("This is not tar.gz file!") 3403 3404 def test_mirror_broken_download(self): 3405 import sys 3406 self.d.setVar("SRCREV", "0"*40) 3407 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3408 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs() as logs: 3409 fetcher.download() 3410 output = "".join(logs.output) 3411 self.assertFalse(" not a git repository (or any parent up to mount point /)" in output) 3412 3413class GoModTest(FetcherTest): 3414 3415 @skipIfNoNetwork() 3416 def test_gomod_url(self): 3417 urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;' 3418 'sha256sum=9bb69aea32f1d59711701f9562d66432c9c0374205e5009d1d1a62f03fb4fdad'] 3419 3420 fetcher = bb.fetch2.Fetch(urls, self.d) 3421 ud = fetcher.ud[urls[0]] 3422 self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.zip') 3423 3424 fetcher.download() 3425 fetcher.unpack(self.unpackdir) 3426 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3427 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip'))) 3428 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod'))) 3429 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')), 3430 '7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873') 3431 3432 @skipIfNoNetwork() 3433 def test_gomod_url_go_mod_only(self): 3434 urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;mod=1;' 3435 'sha256sum=7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873'] 3436 3437 fetcher = bb.fetch2.Fetch(urls, self.d) 3438 ud = fetcher.ud[urls[0]] 3439 self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.mod') 3440 3441 fetcher.download() 3442 fetcher.unpack(self.unpackdir) 3443 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3444 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod'))) 3445 3446 @skipIfNoNetwork() 3447 def test_gomod_url_sha256sum_varflag(self): 3448 urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0'] 3449 self.d.setVarFlag('SRC_URI', 'gopkg.in/ini.v1@v1.67.0.sha256sum', 'bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6') 3450 3451 fetcher = bb.fetch2.Fetch(urls, self.d) 3452 ud = fetcher.ud[urls[0]] 3453 self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip') 3454 self.assertEqual(ud.parm['name'], 'gopkg.in/ini.v1@v1.67.0') 3455 3456 fetcher.download() 3457 fetcher.unpack(self.unpackdir) 3458 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3459 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3460 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3461 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3462 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3463 3464 @skipIfNoNetwork() 3465 def test_gomod_url_no_go_mod_in_module(self): 3466 urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0;' 3467 'sha256sum=bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6'] 3468 3469 fetcher = bb.fetch2.Fetch(urls, self.d) 3470 ud = fetcher.ud[urls[0]] 3471 self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip') 3472 3473 fetcher.download() 3474 fetcher.unpack(self.unpackdir) 3475 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3476 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3477 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3478 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3479 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3480 3481 @skipIfNoNetwork() 3482 def test_gomod_url_host_only(self): 3483 urls = ['gomod://go.opencensus.io;version=v0.24.0;' 3484 'sha256sum=203a767d7f8e7c1ebe5588220ad168d1e15b14ae70a636de7ca9a4a88a7e0d0c'] 3485 3486 fetcher = bb.fetch2.Fetch(urls, self.d) 3487 ud = fetcher.ud[urls[0]] 3488 self.assertEqual(ud.url, 'https://proxy.golang.org/go.opencensus.io/%40v/v0.24.0.zip') 3489 3490 fetcher.download() 3491 fetcher.unpack(self.unpackdir) 3492 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3493 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.zip'))) 3494 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod'))) 3495 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')), 3496 '0dc9ccc660ad21cebaffd548f2cc6efa27891c68b4fbc1f8a3893b00f1acec96') 3497 3498class GoModGitTest(FetcherTest): 3499 3500 @skipIfNoNetwork() 3501 def test_gomodgit_url_repo(self): 3502 urls = ['gomodgit://golang.org/x/net;version=v0.9.0;' 3503 'repo=go.googlesource.com/net;' 3504 'srcrev=694cff8668bac64e0864b552bffc280cd27f21b1'] 3505 3506 fetcher = bb.fetch2.Fetch(urls, self.d) 3507 ud = fetcher.ud[urls[0]] 3508 self.assertEqual(ud.host, 'go.googlesource.com') 3509 self.assertEqual(ud.path, '/net') 3510 self.assertEqual(ud.names, ['golang.org/x/net@v0.9.0']) 3511 self.assertEqual(self.d.getVar('SRCREV_golang.org/x/net@v0.9.0'), '694cff8668bac64e0864b552bffc280cd27f21b1') 3512 3513 fetcher.download() 3514 self.assertTrue(os.path.exists(ud.localpath)) 3515 3516 fetcher.unpack(self.unpackdir) 3517 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3518 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'ed42bd05533fd84ae290a5d33ebd3695a0a2b06131beebd5450825bee8603aca'))) 3519 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3520 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.zip'))) 3521 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod'))) 3522 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod')), 3523 'c5d6851ede50ec1c001afb763040194b68961bf06997e2605e8bf06dcd2aeb2e') 3524 3525 @skipIfNoNetwork() 3526 def test_gomodgit_url_subdir(self): 3527 urls = ['gomodgit://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;' 3528 'repo=github.com/Azure/azure-sdk-for-go;subdir=sdk/storage/azblob;' 3529 'srcrev=ec928e0ed34db682b3f783d3739d1c538142e0c3'] 3530 3531 fetcher = bb.fetch2.Fetch(urls, self.d) 3532 ud = fetcher.ud[urls[0]] 3533 self.assertEqual(ud.host, 'github.com') 3534 self.assertEqual(ud.path, '/Azure/azure-sdk-for-go') 3535 self.assertEqual(ud.parm['subpath'], 'sdk/storage/azblob') 3536 self.assertEqual(ud.names, ['github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0']) 3537 self.assertEqual(self.d.getVar('SRCREV_github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'), 'ec928e0ed34db682b3f783d3739d1c538142e0c3') 3538 3539 fetcher.download() 3540 self.assertTrue(os.path.exists(ud.localpath)) 3541 3542 fetcher.unpack(self.unpackdir) 3543 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3544 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'd31d6145676ed3066ce573a8198f326dea5be45a43b3d8f41ce7787fd71d66b3'))) 3545 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3546 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip'))) 3547 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod'))) 3548 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')), 3549 '7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873') 3550 3551 @skipIfNoNetwork() 3552 def test_gomodgit_url_srcrev_var(self): 3553 urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0'] 3554 self.d.setVar('SRCREV_gopkg.in/ini.v1@v1.67.0', 'b2f570e5b5b844226bbefe6fb521d891f529a951') 3555 3556 fetcher = bb.fetch2.Fetch(urls, self.d) 3557 ud = fetcher.ud[urls[0]] 3558 self.assertEqual(ud.host, 'gopkg.in') 3559 self.assertEqual(ud.path, '/ini.v1') 3560 self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0']) 3561 self.assertEqual(ud.parm['srcrev'], 'b2f570e5b5b844226bbefe6fb521d891f529a951') 3562 3563 fetcher.download() 3564 fetcher.unpack(self.unpackdir) 3565 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3566 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3'))) 3567 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3568 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3569 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3570 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3571 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3572 3573 @skipIfNoNetwork() 3574 def test_gomodgit_url_no_go_mod_in_module(self): 3575 urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0;' 3576 'srcrev=b2f570e5b5b844226bbefe6fb521d891f529a951'] 3577 3578 fetcher = bb.fetch2.Fetch(urls, self.d) 3579 ud = fetcher.ud[urls[0]] 3580 self.assertEqual(ud.host, 'gopkg.in') 3581 self.assertEqual(ud.path, '/ini.v1') 3582 self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0']) 3583 self.assertEqual(self.d.getVar('SRCREV_gopkg.in/ini.v1@v1.67.0'), 'b2f570e5b5b844226bbefe6fb521d891f529a951') 3584 3585 fetcher.download() 3586 fetcher.unpack(self.unpackdir) 3587 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3588 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3'))) 3589 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3590 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3591 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3592 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3593 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3594 3595 @skipIfNoNetwork() 3596 def test_gomodgit_url_host_only(self): 3597 urls = ['gomodgit://go.opencensus.io;version=v0.24.0;' 3598 'repo=github.com/census-instrumentation/opencensus-go;' 3599 'srcrev=b1a01ee95db0e690d91d7193d037447816fae4c5'] 3600 3601 fetcher = bb.fetch2.Fetch(urls, self.d) 3602 ud = fetcher.ud[urls[0]] 3603 self.assertEqual(ud.host, 'github.com') 3604 self.assertEqual(ud.path, '/census-instrumentation/opencensus-go') 3605 self.assertEqual(ud.names, ['go.opencensus.io@v0.24.0']) 3606 self.assertEqual(self.d.getVar('SRCREV_go.opencensus.io@v0.24.0'), 'b1a01ee95db0e690d91d7193d037447816fae4c5') 3607 3608 fetcher.download() 3609 fetcher.unpack(self.unpackdir) 3610 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3611 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'aae3ac7b2122ed3345654e6327855e9682f4a5350d63e93dbcfc51c4419df0e1'))) 3612 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3613 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.zip'))) 3614 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod'))) 3615 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')), 3616 '0dc9ccc660ad21cebaffd548f2cc6efa27891c68b4fbc1f8a3893b00f1acec96') 3617