1# 2# BitBake Tests for the Fetcher (fetch2/) 3# 4# Copyright (C) 2012 Richard Purdie 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9import contextlib 10import unittest 11import hashlib 12import tempfile 13import collections 14import os 15import signal 16import tarfile 17from bb.fetch2 import URI 18from bb.fetch2 import FetchMethod 19import bb 20from bb.tests.support.httpserver import HTTPService 21 22def skipIfNoNetwork(): 23 if os.environ.get("BB_SKIP_NETTESTS") == "yes": 24 return unittest.skip("network test") 25 return lambda f: f 26 27class TestTimeout(Exception): 28 # Indicate to pytest that this is not a test suite 29 __test__ = False 30 31class Timeout(): 32 33 def __init__(self, seconds): 34 self.seconds = seconds 35 36 def handle_timeout(self, signum, frame): 37 raise TestTimeout("Test failed: timeout reached") 38 39 def __enter__(self): 40 signal.signal(signal.SIGALRM, self.handle_timeout) 41 signal.alarm(self.seconds) 42 43 def __exit__(self, exc_type, exc_val, exc_tb): 44 signal.alarm(0) 45 46class URITest(unittest.TestCase): 47 test_uris = { 48 "http://www.google.com/index.html" : { 49 'uri': 'http://www.google.com/index.html', 50 'scheme': 'http', 51 'hostname': 'www.google.com', 52 'port': None, 53 'hostport': 'www.google.com', 54 'path': '/index.html', 55 'userinfo': '', 56 'username': '', 57 'password': '', 58 'params': {}, 59 'query': {}, 60 'relative': False 61 }, 62 "http://www.google.com/index.html;param1=value1" : { 63 'uri': 'http://www.google.com/index.html;param1=value1', 64 'scheme': 'http', 65 'hostname': 'www.google.com', 66 'port': None, 67 'hostport': 'www.google.com', 68 'path': '/index.html', 69 'userinfo': '', 70 'username': '', 71 'password': '', 72 'params': { 73 'param1': 'value1' 74 }, 75 'query': {}, 76 'relative': False 77 }, 78 "http://www.example.org/index.html?param1=value1" : { 79 'uri': 'http://www.example.org/index.html?param1=value1', 80 'scheme': 'http', 81 'hostname': 'www.example.org', 82 'port': None, 83 'hostport': 'www.example.org', 84 'path': '/index.html', 85 'userinfo': '', 86 'username': '', 87 'password': '', 88 'params': {}, 89 'query': { 90 'param1': 'value1' 91 }, 92 'relative': False 93 }, 94 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : { 95 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 96 'scheme': 'http', 97 'hostname': 'www.example.org', 98 'port': None, 99 'hostport': 'www.example.org', 100 'path': '/index.html', 101 'userinfo': '', 102 'username': '', 103 'password': '', 104 'params': { 105 'param2': 'value2' 106 }, 107 'query': { 108 'qparam1': 'qvalue1' 109 }, 110 'relative': False 111 }, 112 # Check that trailing semicolons are handled correctly 113 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2;" : { 114 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 115 'scheme': 'http', 116 'hostname': 'www.example.org', 117 'port': None, 118 'hostport': 'www.example.org', 119 'path': '/index.html', 120 'userinfo': '', 121 'username': '', 122 'password': '', 123 'params': { 124 'param2': 'value2' 125 }, 126 'query': { 127 'qparam1': 'qvalue1' 128 }, 129 'relative': False 130 }, 131 "http://www.example.com:8080/index.html" : { 132 'uri': 'http://www.example.com:8080/index.html', 133 'scheme': 'http', 134 'hostname': 'www.example.com', 135 'port': 8080, 136 'hostport': 'www.example.com:8080', 137 'path': '/index.html', 138 'userinfo': '', 139 'username': '', 140 'password': '', 141 'params': {}, 142 'query': {}, 143 'relative': False 144 }, 145 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : { 146 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg', 147 'scheme': 'cvs', 148 'hostname': 'cvs.handhelds.org', 149 'port': None, 150 'hostport': 'cvs.handhelds.org', 151 'path': '/cvs', 152 'userinfo': 'anoncvs', 153 'username': 'anoncvs', 154 'password': '', 155 'params': { 156 'module': 'familiar/dist/ipkg' 157 }, 158 'query': {}, 159 'relative': False 160 }, 161 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": { 162 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg', 163 'scheme': 'cvs', 164 'hostname': 'cvs.handhelds.org', 165 'port': None, 166 'hostport': 'cvs.handhelds.org', 167 'path': '/cvs', 168 'userinfo': 'anoncvs:anonymous', 169 'username': 'anoncvs', 170 'password': 'anonymous', 171 'params': collections.OrderedDict([ 172 ('tag', 'V0-99-81'), 173 ('module', 'familiar/dist/ipkg') 174 ]), 175 'query': {}, 176 'relative': False 177 }, 178 "file://example.diff": { # NOTE: Not RFC compliant! 179 'uri': 'file:example.diff', 180 'scheme': 'file', 181 'hostname': '', 182 'port': None, 183 'hostport': '', 184 'path': 'example.diff', 185 'userinfo': '', 186 'username': '', 187 'password': '', 188 'params': {}, 189 'query': {}, 190 'relative': True 191 }, 192 "file:example.diff": { # NOTE: RFC compliant version of the former 193 'uri': 'file:example.diff', 194 'scheme': 'file', 195 'hostname': '', 196 'port': None, 197 'hostport': '', 198 'path': 'example.diff', 199 'userinfo': '', 200 'userinfo': '', 201 'username': '', 202 'password': '', 203 'params': {}, 204 'query': {}, 205 'relative': True 206 }, 207 "file:///tmp/example.diff": { 208 'uri': 'file:///tmp/example.diff', 209 'scheme': 'file', 210 'hostname': '', 211 'port': None, 212 'hostport': '', 213 'path': '/tmp/example.diff', 214 'userinfo': '', 215 'userinfo': '', 216 'username': '', 217 'password': '', 218 'params': {}, 219 'query': {}, 220 'relative': False 221 }, 222 "git:///path/example.git": { 223 'uri': 'git:///path/example.git', 224 'scheme': 'git', 225 'hostname': '', 226 'port': None, 227 'hostport': '', 228 'path': '/path/example.git', 229 'userinfo': '', 230 'userinfo': '', 231 'username': '', 232 'password': '', 233 'params': {}, 234 'query': {}, 235 'relative': False 236 }, 237 "git:path/example.git": { 238 'uri': 'git:path/example.git', 239 'scheme': 'git', 240 'hostname': '', 241 'port': None, 242 'hostport': '', 243 'path': 'path/example.git', 244 'userinfo': '', 245 'userinfo': '', 246 'username': '', 247 'password': '', 248 'params': {}, 249 'query': {}, 250 'relative': True 251 }, 252 "git://example.net/path/example.git": { 253 'uri': 'git://example.net/path/example.git', 254 'scheme': 'git', 255 'hostname': 'example.net', 256 'port': None, 257 'hostport': 'example.net', 258 'path': '/path/example.git', 259 'userinfo': '', 260 'userinfo': '', 261 'username': '', 262 'password': '', 263 'params': {}, 264 'query': {}, 265 'relative': False 266 }, 267 "git://tfs-example.org:22/tfs/example%20path/example.git": { 268 'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git', 269 'scheme': 'git', 270 'hostname': 'tfs-example.org', 271 'port': 22, 272 'hostport': 'tfs-example.org:22', 273 'path': '/tfs/example path/example.git', 274 'userinfo': '', 275 'userinfo': '', 276 'username': '', 277 'password': '', 278 'params': {}, 279 'query': {}, 280 'relative': False 281 }, 282 "http://somesite.net;someparam=1": { 283 'uri': 'http://somesite.net;someparam=1', 284 'scheme': 'http', 285 'hostname': 'somesite.net', 286 'port': None, 287 'hostport': 'somesite.net', 288 'path': '', 289 'userinfo': '', 290 'userinfo': '', 291 'username': '', 292 'password': '', 293 'params': {"someparam" : "1"}, 294 'query': {}, 295 'relative': False 296 }, 297 "file://somelocation;someparam=1": { 298 'uri': 'file:somelocation;someparam=1', 299 'scheme': 'file', 300 'hostname': '', 301 'port': None, 302 'hostport': '', 303 'path': 'somelocation', 304 'userinfo': '', 305 'userinfo': '', 306 'username': '', 307 'password': '', 308 'params': {"someparam" : "1"}, 309 'query': {}, 310 'relative': True 311 }, 312 "https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip": { 313 'uri': 'https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip', 314 'scheme': 'https', 315 'hostname': 'www.innodisk.com', 316 'port': None, 317 'hostport': 'www.innodisk.com', 318 'path': '/Download_file', 319 'userinfo': '', 320 'userinfo': '', 321 'username': '', 322 'password': '', 323 'params': {"downloadfilename" : "EGPL-T101.zip"}, 324 'query': {"9BE0BF6657": None}, 325 'relative': False 326 } 327 328 } 329 330 def test_uri(self): 331 for test_uri, ref in self.test_uris.items(): 332 uri = URI(test_uri) 333 334 self.assertEqual(str(uri), ref['uri']) 335 336 # expected attributes 337 self.assertEqual(uri.scheme, ref['scheme']) 338 339 self.assertEqual(uri.userinfo, ref['userinfo']) 340 self.assertEqual(uri.username, ref['username']) 341 self.assertEqual(uri.password, ref['password']) 342 343 self.assertEqual(uri.hostname, ref['hostname']) 344 self.assertEqual(uri.port, ref['port']) 345 self.assertEqual(uri.hostport, ref['hostport']) 346 347 self.assertEqual(uri.path, ref['path']) 348 self.assertEqual(uri.params, ref['params']) 349 350 self.assertEqual(uri.relative, ref['relative']) 351 352 def test_dict(self): 353 for test in self.test_uris.values(): 354 uri = URI() 355 356 self.assertEqual(uri.scheme, '') 357 self.assertEqual(uri.userinfo, '') 358 self.assertEqual(uri.username, '') 359 self.assertEqual(uri.password, '') 360 self.assertEqual(uri.hostname, '') 361 self.assertEqual(uri.port, None) 362 self.assertEqual(uri.path, '') 363 self.assertEqual(uri.params, {}) 364 365 366 uri.scheme = test['scheme'] 367 self.assertEqual(uri.scheme, test['scheme']) 368 369 uri.userinfo = test['userinfo'] 370 self.assertEqual(uri.userinfo, test['userinfo']) 371 self.assertEqual(uri.username, test['username']) 372 self.assertEqual(uri.password, test['password']) 373 374 # make sure changing the values doesn't do anything unexpected 375 uri.username = 'changeme' 376 self.assertEqual(uri.username, 'changeme') 377 self.assertEqual(uri.password, test['password']) 378 uri.password = 'insecure' 379 self.assertEqual(uri.username, 'changeme') 380 self.assertEqual(uri.password, 'insecure') 381 382 # reset back after our trickery 383 uri.userinfo = test['userinfo'] 384 self.assertEqual(uri.userinfo, test['userinfo']) 385 self.assertEqual(uri.username, test['username']) 386 self.assertEqual(uri.password, test['password']) 387 388 uri.hostname = test['hostname'] 389 self.assertEqual(uri.hostname, test['hostname']) 390 self.assertEqual(uri.hostport, test['hostname']) 391 392 uri.port = test['port'] 393 self.assertEqual(uri.port, test['port']) 394 self.assertEqual(uri.hostport, test['hostport']) 395 396 uri.path = test['path'] 397 self.assertEqual(uri.path, test['path']) 398 399 uri.params = test['params'] 400 self.assertEqual(uri.params, test['params']) 401 402 uri.query = test['query'] 403 self.assertEqual(uri.query, test['query']) 404 405 self.assertEqual(str(uri), test['uri']) 406 407 uri.params = {} 408 self.assertEqual(uri.params, {}) 409 self.assertEqual(str(uri), (str(uri).split(";"))[0]) 410 411class FetcherTest(unittest.TestCase): 412 413 def setUp(self): 414 self.origdir = os.getcwd() 415 self.d = bb.data.init() 416 self.tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") 417 self.dldir = os.path.join(self.tempdir, "download") 418 os.mkdir(self.dldir) 419 self.d.setVar("DL_DIR", self.dldir) 420 self.unpackdir = os.path.join(self.tempdir, "unpacked") 421 os.mkdir(self.unpackdir) 422 persistdir = os.path.join(self.tempdir, "persistdata") 423 self.d.setVar("PERSISTENT_DIR", persistdir) 424 425 def tearDown(self): 426 os.chdir(self.origdir) 427 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 428 print("Not cleaning up %s. Please remove manually." % self.tempdir) 429 else: 430 bb.process.run('chmod u+rw -R %s' % self.tempdir) 431 bb.utils.prunedir(self.tempdir) 432 433 def git(self, cmd, cwd=None): 434 if isinstance(cmd, str): 435 cmd = 'git -c safe.bareRepository=all ' + cmd 436 else: 437 cmd = ['git', '-c', 'safe.bareRepository=all'] + cmd 438 if cwd is None: 439 cwd = self.gitdir 440 return bb.process.run(cmd, cwd=cwd)[0] 441 442 def git_init(self, cwd=None): 443 self.git('init', cwd=cwd) 444 # Explicitly set initial branch to master as 445 # a common setup is to use other default 446 # branch than master. 447 self.git(['checkout', '-b', 'master'], cwd=cwd) 448 449 try: 450 self.git(['config', 'user.email'], cwd=cwd) 451 except bb.process.ExecutionError: 452 self.git(['config', 'user.email', 'you@example.com'], cwd=cwd) 453 454 try: 455 self.git(['config', 'user.name'], cwd=cwd) 456 except bb.process.ExecutionError: 457 self.git(['config', 'user.name', 'Your Name'], cwd=cwd) 458 459class MirrorUriTest(FetcherTest): 460 461 replaceuris = { 462 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") 463 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz", 464 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 465 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 466 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 467 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 468 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") 469 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 470 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake") 471 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890", 472 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") 473 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 474 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") 475 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 476 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") 477 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 478 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") 479 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 480 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist") 481 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", 482 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/") 483 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2", 484 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 485 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 486 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 487 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 488 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") 489 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 490 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org") 491 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 492 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/") 493 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 494 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http") 495 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 496 ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http") 497 : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 498 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=git;branch=master", "git://someserver.org/bitbake", "git://someotherserver.org/bitbake;protocol=https") 499 : "git://someotherserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=https;branch=master", 500 ("gitsm://git.qemu.org/git/seabios.git/;protocol=https;name=roms/seabios;subpath=roms/seabios;bareclone=1;nobranch=1;rev=1234567890123456789012345678901234567890", "gitsm://.*/.*", "http://petalinux.xilinx.com/sswreleases/rel-v${XILINX_VER_MAIN}/downloads") : "http://petalinux.xilinx.com/sswreleases/rel-v%24%7BXILINX_VER_MAIN%7D/downloads/git2_git.qemu.org.git.seabios.git..tar.gz", 501 ("https://somewhere.org/example/1.0.0/example;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/PATH") 502 : "file:///mirror/example/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 503 ("https://somewhere.org/example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/some-example-1.0.0.tgz") 504 : "file:///mirror/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 505 506 #Renaming files doesn't work 507 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz" 508 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 509 } 510 511 mirrorvar = "http://.*/.* file:///somepath/downloads/ " \ 512 "git://someserver.org/bitbake git://git.openembedded.org/bitbake " \ 513 "https://.*/.* file:///someotherpath/downloads/ " \ 514 "http://.*/.* file:///someotherpath/downloads/ " \ 515 "svn://svn.server1.com/ svn://svn.server2.com/" 516 517 def test_urireplace(self): 518 self.d.setVar("FILESPATH", ".") 519 for k, v in self.replaceuris.items(): 520 ud = bb.fetch.FetchData(k[0], self.d) 521 ud.setup_localpath(self.d) 522 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2])) 523 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d) 524 self.assertEqual([v], newuris) 525 526 def test_urilist1(self): 527 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 528 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 529 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 530 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 531 532 def test_urilist2(self): 533 # Catch https:// -> files:// bug 534 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 535 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 536 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 537 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 538 539 def test_urilistsvn(self): 540 # Catch svn:// -> svn:// bug 541 fetcher = bb.fetch.FetchData("svn://svn.server1.com/isource/svnroot/reponame/tags/tagname;module=path_in_tagnamefolder;protocol=https;rev=2", self.d) 542 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 543 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 544 self.assertEqual(uris, ['svn://svn.server2.com/isource/svnroot/reponame/tags/tagname;module=path_in_tagnamefolder;protocol=https;rev=2']) 545 546 def test_mirror_of_mirror(self): 547 # Test if mirror of a mirror works 548 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/" 549 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/" 550 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 551 mirrors = bb.fetch2.mirror_from_string(mirrorvar) 552 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 553 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 554 'file:///someotherpath/downloads/bitbake-1.0.tar.gz', 555 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', 556 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) 557 558 recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ " \ 559 "https://.*/[^/]* https://BBBB/B/B/B/" 560 561 def test_recursive(self): 562 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 563 mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar) 564 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 565 self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz', 566 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz', 567 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz']) 568 569 570class GitDownloadDirectoryNamingTest(FetcherTest): 571 def setUp(self): 572 super(GitDownloadDirectoryNamingTest, self).setUp() 573 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 574 self.recipe_dir = "git.openembedded.org.bitbake" 575 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 576 self.mirror_dir = "github.com.openembedded.bitbake.git" 577 578 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 579 580 def setup_mirror_rewrite(self): 581 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 582 583 @skipIfNoNetwork() 584 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self): 585 self.setup_mirror_rewrite() 586 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 587 588 fetcher.download() 589 590 dir = os.listdir(self.dldir + "/git2") 591 self.assertIn(self.recipe_dir, dir) 592 593 @skipIfNoNetwork() 594 def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self): 595 self.setup_mirror_rewrite() 596 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 597 598 fetcher.download() 599 600 dir = os.listdir(self.dldir + "/git2") 601 self.assertIn(self.mirror_dir, dir) 602 self.assertIn(self.recipe_dir, dir) 603 604 @skipIfNoNetwork() 605 def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self): 606 self.setup_mirror_rewrite() 607 fetcher = bb.fetch.Fetch([self.mirror_url], self.d) 608 fetcher.download() 609 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 610 611 fetcher.download() 612 613 dir = os.listdir(self.dldir + "/git2") 614 self.assertIn(self.mirror_dir, dir) 615 self.assertIn(self.recipe_dir, dir) 616 617 618class TarballNamingTest(FetcherTest): 619 def setUp(self): 620 super(TarballNamingTest, self).setUp() 621 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 622 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 623 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 624 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz" 625 626 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 627 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 628 629 def setup_mirror_rewrite(self): 630 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 631 632 @skipIfNoNetwork() 633 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self): 634 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 635 636 fetcher.download() 637 638 dir = os.listdir(self.dldir) 639 self.assertIn(self.recipe_tarball, dir) 640 641 @skipIfNoNetwork() 642 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 643 self.setup_mirror_rewrite() 644 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 645 646 fetcher.download() 647 648 dir = os.listdir(self.dldir) 649 self.assertIn(self.mirror_tarball, dir) 650 651 652class GitShallowTarballNamingTest(FetcherTest): 653 def setUp(self): 654 super(GitShallowTarballNamingTest, self).setUp() 655 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 656 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz" 657 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 658 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz" 659 660 self.d.setVar('BB_GIT_SHALLOW', '1') 661 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 662 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 663 664 def setup_mirror_rewrite(self): 665 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 666 667 @skipIfNoNetwork() 668 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self): 669 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 670 671 fetcher.download() 672 673 dir = os.listdir(self.dldir) 674 self.assertIn(self.recipe_tarball, dir) 675 676 @skipIfNoNetwork() 677 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 678 self.setup_mirror_rewrite() 679 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 680 681 fetcher.download() 682 683 dir = os.listdir(self.dldir) 684 self.assertIn(self.mirror_tarball, dir) 685 686 687class CleanTarballTest(FetcherTest): 688 def setUp(self): 689 super(CleanTarballTest, self).setUp() 690 self.recipe_url = "git://git.openembedded.org/bitbake;protocol=https" 691 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 692 693 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 694 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 695 696 @skipIfNoNetwork() 697 def test_that_the_tarball_contents_does_not_leak_info(self): 698 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 699 700 fetcher.download() 701 702 fetcher.unpack(self.unpackdir) 703 mtime = bb.process.run('git log --all -1 --format=%ct', 704 cwd=os.path.join(self.unpackdir, 'git')) 705 self.assertEqual(len(mtime), 2) 706 mtime = int(mtime[0]) 707 708 archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball)) 709 self.assertNotEqual(len(archive.members), 0) 710 for member in archive.members: 711 if member.name == ".": 712 continue 713 self.assertEqual(member.uname, 'oe', "user name for %s differs" % member.name) 714 self.assertEqual(member.uid, 0, "uid for %s differs" % member.name) 715 self.assertEqual(member.gname, 'oe', "group name for %s differs" % member.name) 716 self.assertEqual(member.gid, 0, "gid for %s differs" % member.name) 717 self.assertEqual(member.mtime, mtime, "mtime for %s differs" % member.name) 718 719 720class FetcherLocalTest(FetcherTest): 721 def setUp(self): 722 def touch(fn): 723 with open(fn, 'a'): 724 os.utime(fn, None) 725 726 super(FetcherLocalTest, self).setUp() 727 self.localsrcdir = os.path.join(self.tempdir, 'localsrc') 728 os.makedirs(self.localsrcdir) 729 touch(os.path.join(self.localsrcdir, 'a')) 730 touch(os.path.join(self.localsrcdir, 'b')) 731 os.makedirs(os.path.join(self.localsrcdir, 'dir')) 732 touch(os.path.join(self.localsrcdir, 'dir', 'c')) 733 touch(os.path.join(self.localsrcdir, 'dir', 'd')) 734 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir')) 735 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e')) 736 touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device')) 737 bb.process.run('tar cf archive.tar -C dir .', cwd=self.localsrcdir) 738 bb.process.run('tar czf archive.tar.gz -C dir .', cwd=self.localsrcdir) 739 bb.process.run('tar cjf archive.tar.bz2 -C dir .', cwd=self.localsrcdir) 740 self.d.setVar("FILESPATH", self.localsrcdir) 741 742 def fetchUnpack(self, uris): 743 fetcher = bb.fetch.Fetch(uris, self.d) 744 fetcher.download() 745 fetcher.unpack(self.unpackdir) 746 flst = [] 747 for root, dirs, files in os.walk(self.unpackdir): 748 for f in files: 749 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir)) 750 flst.sort() 751 return flst 752 753 def test_local_checksum_fails_no_file(self): 754 self.d.setVar("SRC_URI", "file://404") 755 with self.assertRaises(bb.BBHandledException): 756 bb.fetch.get_checksum_file_list(self.d) 757 758 def test_local(self): 759 tree = self.fetchUnpack(['file://a', 'file://dir/c']) 760 self.assertEqual(tree, ['a', 'dir/c']) 761 762 def test_local_backslash(self): 763 tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device']) 764 self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device']) 765 766 def test_local_wildcard(self): 767 with self.assertRaises(bb.fetch2.ParameterError): 768 tree = self.fetchUnpack(['file://a', 'file://dir/*']) 769 770 def test_local_dir(self): 771 tree = self.fetchUnpack(['file://a', 'file://dir']) 772 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e']) 773 774 def test_local_subdir(self): 775 tree = self.fetchUnpack(['file://dir/subdir']) 776 self.assertEqual(tree, ['dir/subdir/e']) 777 778 def test_local_subdir_file(self): 779 tree = self.fetchUnpack(['file://dir/subdir/e']) 780 self.assertEqual(tree, ['dir/subdir/e']) 781 782 def test_local_subdirparam(self): 783 tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo']) 784 self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e']) 785 786 def test_local_deepsubdirparam(self): 787 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar']) 788 self.assertEqual(tree, ['bar/dir/subdir/e']) 789 790 def test_local_absolutedir(self): 791 # Unpacking to an absolute path that is a subdirectory of the root 792 # should work 793 tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')]) 794 795 # Unpacking to an absolute path outside of the root should fail 796 with self.assertRaises(bb.fetch2.UnpackError): 797 self.fetchUnpack(['file://a;subdir=/bin/sh']) 798 799 def test_local_striplevel(self): 800 tree = self.fetchUnpack(['file://archive.tar;subdir=bar;striplevel=1']) 801 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 802 803 def test_local_striplevel_gzip(self): 804 tree = self.fetchUnpack(['file://archive.tar.gz;subdir=bar;striplevel=1']) 805 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 806 807 def test_local_striplevel_bzip2(self): 808 tree = self.fetchUnpack(['file://archive.tar.bz2;subdir=bar;striplevel=1']) 809 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 810 811 def dummyGitTest(self, suffix): 812 # Create dummy local Git repo 813 src_dir = tempfile.mkdtemp(dir=self.tempdir, 814 prefix='gitfetch_localusehead_') 815 self.gitdir = os.path.abspath(src_dir) 816 self.git_init() 817 self.git(['commit', '--allow-empty', '-m', 'Dummy commit']) 818 # Use other branch than master 819 self.git(['checkout', '-b', 'my-devel']) 820 self.git(['commit', '--allow-empty', '-m', 'Dummy commit 2']) 821 orig_rev = self.git(['rev-parse', 'HEAD']).strip() 822 823 # Fetch and check revision 824 self.d.setVar("SRCREV", "AUTOINC") 825 self.d.setVar("__BBSRCREV_SEEN", "1") 826 url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix 827 fetcher = bb.fetch.Fetch([url], self.d) 828 fetcher.download() 829 fetcher.unpack(self.unpackdir) 830 unpack_rev = self.git(['rev-parse', 'HEAD'], 831 cwd=os.path.join(self.unpackdir, 'git')).strip() 832 self.assertEqual(orig_rev, unpack_rev) 833 834 def test_local_gitfetch_usehead(self): 835 self.dummyGitTest("usehead=1") 836 837 def test_local_gitfetch_usehead_withname(self): 838 self.dummyGitTest("usehead=1;name=newName") 839 840 def test_local_gitfetch_shared(self): 841 self.dummyGitTest("usehead=1;name=sharedName") 842 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 843 self.assertTrue(os.path.exists(alt)) 844 845 def test_local_gitfetch_noshared(self): 846 self.d.setVar('BB_GIT_NOSHARED', '1') 847 self.unpackdir += '_noshared' 848 self.dummyGitTest("usehead=1;name=noSharedName") 849 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 850 self.assertFalse(os.path.exists(alt)) 851 852class FetcherNoNetworkTest(FetcherTest): 853 def setUp(self): 854 super().setUp() 855 # all test cases are based on not having network 856 self.d.setVar("BB_NO_NETWORK", "1") 857 858 def test_missing(self): 859 string = "this is a test file\n".encode("utf-8") 860 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 861 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 862 863 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 864 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 865 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 866 with self.assertRaises(bb.fetch2.NetworkAccess): 867 fetcher.download() 868 869 def test_valid_missing_donestamp(self): 870 # create the file in the download directory with correct hash 871 string = "this is a test file\n".encode("utf-8") 872 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f: 873 f.write(string) 874 875 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 876 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 877 878 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 879 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 880 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 881 fetcher.download() 882 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 883 884 def test_invalid_missing_donestamp(self): 885 # create an invalid file in the download directory with incorrect hash 886 string = "this is a test file\n".encode("utf-8") 887 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 888 pass 889 890 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 891 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 892 893 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 894 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 895 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 896 with self.assertRaises(bb.fetch2.NetworkAccess): 897 fetcher.download() 898 # the existing file should not exist or should have be moved to "bad-checksum" 899 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 900 901 def test_nochecksums_missing(self): 902 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 903 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 904 # ssh fetch does not support checksums 905 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 906 # attempts to download with missing donestamp 907 with self.assertRaises(bb.fetch2.NetworkAccess): 908 fetcher.download() 909 910 def test_nochecksums_missing_donestamp(self): 911 # create a file in the download directory 912 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 913 pass 914 915 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 916 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 917 # ssh fetch does not support checksums 918 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 919 # attempts to download with missing donestamp 920 with self.assertRaises(bb.fetch2.NetworkAccess): 921 fetcher.download() 922 923 def test_nochecksums_has_donestamp(self): 924 # create a file in the download directory with the donestamp 925 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 926 pass 927 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 928 pass 929 930 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 931 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 932 # ssh fetch does not support checksums 933 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 934 # should not fetch 935 fetcher.download() 936 # both files should still exist 937 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 938 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 939 940 def test_nochecksums_missing_has_donestamp(self): 941 # create a file in the download directory with the donestamp 942 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 943 pass 944 945 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 946 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 947 # ssh fetch does not support checksums 948 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 949 with self.assertRaises(bb.fetch2.NetworkAccess): 950 fetcher.download() 951 # both files should still exist 952 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 953 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 954 955class FetcherNetworkTest(FetcherTest): 956 @skipIfNoNetwork() 957 def test_fetch(self): 958 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 959 fetcher.download() 960 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 961 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892) 962 self.d.setVar("BB_NO_NETWORK", "1") 963 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 964 fetcher.download() 965 fetcher.unpack(self.unpackdir) 966 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9) 967 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9) 968 969 @skipIfNoNetwork() 970 def test_fetch_mirror(self): 971 self.d.setVar("MIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 972 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 973 fetcher.download() 974 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 975 976 @skipIfNoNetwork() 977 def test_fetch_mirror_of_mirror(self): 978 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake") 979 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 980 fetcher.download() 981 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 982 983 @skipIfNoNetwork() 984 def test_fetch_file_mirror_of_mirror(self): 985 self.d.setVar("FILESPATH", ".") 986 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake") 987 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 988 os.mkdir(self.dldir + "/some2where") 989 fetcher.download() 990 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 991 992 @skipIfNoNetwork() 993 def test_fetch_premirror(self): 994 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 995 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 996 fetcher.download() 997 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 998 999 @skipIfNoNetwork() 1000 def test_fetch_specify_downloadfilename(self): 1001 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) 1002 fetcher.download() 1003 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) 1004 1005 @skipIfNoNetwork() 1006 def test_fetch_premirror_specify_downloadfilename_regex_uri(self): 1007 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/") 1008 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1009 fetcher.download() 1010 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1011 1012 @skipIfNoNetwork() 1013 # BZ13039 1014 def test_fetch_premirror_specify_downloadfilename_specific_uri(self): 1015 self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake") 1016 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1017 fetcher.download() 1018 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1019 1020 @skipIfNoNetwork() 1021 def test_fetch_premirror_use_downloadfilename_to_fetch(self): 1022 # Ensure downloadfilename is used when fetching from premirror. 1023 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 1024 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1025 fetcher.download() 1026 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1027 1028 @skipIfNoNetwork() 1029 def gitfetcher(self, url1, url2): 1030 def checkrevision(self, fetcher): 1031 fetcher.unpack(self.unpackdir) 1032 revision = self.git(['rev-parse', 'HEAD'], 1033 cwd=os.path.join(self.unpackdir, 'git')).strip() 1034 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 1035 1036 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 1037 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 1038 fetcher = bb.fetch.Fetch([url1], self.d) 1039 fetcher.download() 1040 checkrevision(self, fetcher) 1041 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works 1042 bb.utils.prunedir(self.dldir + "/git2/") 1043 bb.utils.prunedir(self.unpackdir) 1044 self.d.setVar("BB_NO_NETWORK", "1") 1045 fetcher = bb.fetch.Fetch([url2], self.d) 1046 fetcher.download() 1047 checkrevision(self, fetcher) 1048 1049 @skipIfNoNetwork() 1050 def test_gitfetch(self): 1051 url1 = url2 = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 1052 self.gitfetcher(url1, url2) 1053 1054 @skipIfNoNetwork() 1055 def test_gitfetch_goodsrcrev(self): 1056 # SRCREV is set but matches rev= parameter 1057 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https" 1058 self.gitfetcher(url1, url2) 1059 1060 @skipIfNoNetwork() 1061 def test_gitfetch_badsrcrev(self): 1062 # SRCREV is set but does not match rev= parameter 1063 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https" 1064 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1065 1066 @skipIfNoNetwork() 1067 def test_gitfetch_tagandrev(self): 1068 # SRCREV is set but does not match rev= parameter 1069 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5;protocol=https" 1070 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1071 1072 @skipIfNoNetwork() 1073 def test_gitfetch_usehead(self): 1074 # Since self.gitfetcher() sets SRCREV we expect this to override 1075 # `usehead=1' and instead fetch the specified SRCREV. See 1076 # test_local_gitfetch_usehead() for a positive use of the usehead 1077 # feature. 1078 url = "git://git.openembedded.org/bitbake;usehead=1;branch=master;protocol=https" 1079 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1080 1081 @skipIfNoNetwork() 1082 def test_gitfetch_usehead_withname(self): 1083 # Since self.gitfetcher() sets SRCREV we expect this to override 1084 # `usehead=1' and instead fetch the specified SRCREV. See 1085 # test_local_gitfetch_usehead() for a positive use of the usehead 1086 # feature. 1087 url = "git://git.openembedded.org/bitbake;usehead=1;name=newName;branch=master;protocol=https" 1088 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1089 1090 @skipIfNoNetwork() 1091 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self): 1092 recipeurl = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 1093 mirrorurl = "git://someserver.org/bitbake;branch=master;protocol=https" 1094 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1095 self.gitfetcher(recipeurl, mirrorurl) 1096 1097 @skipIfNoNetwork() 1098 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self): 1099 recipeurl = "git://someserver.org/bitbake;branch=master;protocol=https" 1100 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1101 self.gitfetcher(recipeurl, recipeurl) 1102 1103 @skipIfNoNetwork() 1104 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self): 1105 realurl = "https://git.openembedded.org/bitbake" 1106 recipeurl = "git://someserver.org/bitbake;protocol=https" 1107 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git") 1108 os.chdir(self.tempdir) 1109 self.git(['clone', realurl, self.sourcedir], cwd=self.tempdir) 1110 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipeurl, self.sourcedir)) 1111 self.gitfetcher(recipeurl, recipeurl) 1112 1113 @skipIfNoNetwork() 1114 def test_git_submodule(self): 1115 # URL with ssh submodules 1116 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master;protocol=https" 1117 # Original URL (comment this if you have ssh access to git.yoctoproject.org) 1118 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master;protocol=https" 1119 fetcher = bb.fetch.Fetch([url], self.d) 1120 fetcher.download() 1121 # Previous cwd has been deleted 1122 os.chdir(os.path.dirname(self.unpackdir)) 1123 fetcher.unpack(self.unpackdir) 1124 1125 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1126 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing') 1127 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing') 1128 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present') 1129 1130 # Only when we're running the extended test with a submodule's submodule, can we check this. 1131 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')): 1132 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing') 1133 1134 @skipIfNoNetwork() 1135 def test_git_submodule_restricted_network_premirrors(self): 1136 # this test is to ensure that premirrors will be tried in restricted network 1137 # that is, BB_ALLOWED_NETWORKS does not contain the domain the url uses 1138 url = "gitsm://github.com/grpc/grpc.git;protocol=https;name=grpc;branch=v1.60.x;rev=0ef13a7555dbaadd4633399242524129eef5e231" 1139 # create a download directory to be used as premirror later 1140 tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") 1141 dl_premirror = os.path.join(tempdir, "download-premirror") 1142 os.mkdir(dl_premirror) 1143 self.d.setVar("DL_DIR", dl_premirror) 1144 fetcher = bb.fetch.Fetch([url], self.d) 1145 fetcher.download() 1146 # now use the premirror in restricted network 1147 self.d.setVar("DL_DIR", self.dldir) 1148 self.d.setVar("PREMIRRORS", "gitsm://.*/.* gitsm://%s/git2/MIRRORNAME;protocol=file" % dl_premirror) 1149 self.d.setVar("BB_ALLOWED_NETWORKS", "*.some.domain") 1150 fetcher = bb.fetch.Fetch([url], self.d) 1151 fetcher.download() 1152 1153 @skipIfNoNetwork() 1154 def test_git_submodule_dbus_broker(self): 1155 # The following external repositories have show failures in fetch and unpack operations 1156 # We want to avoid regressions! 1157 url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main" 1158 fetcher = bb.fetch.Fetch([url], self.d) 1159 fetcher.download() 1160 # Previous cwd has been deleted 1161 os.chdir(os.path.dirname(self.unpackdir)) 1162 fetcher.unpack(self.unpackdir) 1163 1164 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1165 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"') 1166 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"') 1167 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"') 1168 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"') 1169 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"') 1170 1171 @skipIfNoNetwork() 1172 def test_git_submodule_CLI11(self): 1173 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main" 1174 fetcher = bb.fetch.Fetch([url], self.d) 1175 fetcher.download() 1176 # Previous cwd has been deleted 1177 os.chdir(os.path.dirname(self.unpackdir)) 1178 fetcher.unpack(self.unpackdir) 1179 1180 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1181 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1182 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1183 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1184 1185 @skipIfNoNetwork() 1186 def test_git_submodule_update_CLI11(self): 1187 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """ 1188 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main" 1189 fetcher = bb.fetch.Fetch([url], self.d) 1190 fetcher.download() 1191 1192 # CLI11 that pulls in a newer nlohmann-json 1193 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main" 1194 fetcher = bb.fetch.Fetch([url], self.d) 1195 fetcher.download() 1196 # Previous cwd has been deleted 1197 os.chdir(os.path.dirname(self.unpackdir)) 1198 fetcher.unpack(self.unpackdir) 1199 1200 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1201 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1202 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1203 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1204 1205 @skipIfNoNetwork() 1206 def test_git_submodule_aktualizr(self): 1207 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" 1208 fetcher = bb.fetch.Fetch([url], self.d) 1209 fetcher.download() 1210 # Previous cwd has been deleted 1211 os.chdir(os.path.dirname(self.unpackdir)) 1212 fetcher.unpack(self.unpackdir) 1213 1214 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1215 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"') 1216 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"') 1217 self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly") 1218 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"') 1219 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"') 1220 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"') 1221 1222 @skipIfNoNetwork() 1223 def test_git_submodule_iotedge(self): 1224 """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """ 1225 1226 # This repository also has submodules where the module (name), path and url do not align 1227 url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699;branch=main" 1228 fetcher = bb.fetch.Fetch([url], self.d) 1229 fetcher.download() 1230 # Previous cwd has been deleted 1231 os.chdir(os.path.dirname(self.unpackdir)) 1232 fetcher.unpack(self.unpackdir) 1233 1234 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1235 1236 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/README.md')), msg='Missing submodule checkout') 1237 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/ctest/README.md')), msg='Missing submodule checkout') 1238 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1239 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1240 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1241 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1242 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/README.md')), msg='Missing submodule checkout') 1243 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/README.md')), msg='Missing submodule checkout') 1244 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/ctest/README.md')), msg='Missing submodule checkout') 1245 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1246 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1247 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1248 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1249 1250 @skipIfNoNetwork() 1251 def test_git_submodule_reference_to_parent(self): 1252 self.recipe_url = "gitsm://github.com/gflags/gflags.git;protocol=https;branch=master" 1253 self.d.setVar("SRCREV", "14e1138441bbbb584160cb1c0a0426ec1bac35f1") 1254 with Timeout(60): 1255 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 1256 with self.assertRaises(bb.fetch2.FetchError): 1257 fetcher.download() 1258 1259class SVNTest(FetcherTest): 1260 def skipIfNoSvn(): 1261 import shutil 1262 if not shutil.which("svn"): 1263 return unittest.skip("svn not installed, tests being skipped") 1264 1265 if not shutil.which("svnadmin"): 1266 return unittest.skip("svnadmin not installed, tests being skipped") 1267 1268 return lambda f: f 1269 1270 @skipIfNoSvn() 1271 def setUp(self): 1272 """ Create a local repository """ 1273 1274 super(SVNTest, self).setUp() 1275 1276 # Create something we can fetch 1277 src_dir = tempfile.mkdtemp(dir=self.tempdir, 1278 prefix='svnfetch_srcdir_') 1279 src_dir = os.path.abspath(src_dir) 1280 bb.process.run("echo readme > README.md", cwd=src_dir) 1281 1282 # Store it in a local SVN repository 1283 repo_dir = tempfile.mkdtemp(dir=self.tempdir, 1284 prefix='svnfetch_localrepo_') 1285 repo_dir = os.path.abspath(repo_dir) 1286 bb.process.run("svnadmin create project", cwd=repo_dir) 1287 1288 self.repo_url = "file://%s/project" % repo_dir 1289 bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url), 1290 cwd=repo_dir) 1291 1292 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir) 1293 # Github won't emulate SVN anymore (see https://github.blog/2023-01-20-sunsetting-subversion-support/) 1294 # Use still accessible svn repo (only trunk to avoid longer downloads) 1295 bb.process.run("svn propset svn:externals 'bitbake https://svn.apache.org/repos/asf/serf/trunk' .", 1296 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1297 bb.process.run("svn commit --non-interactive -m 'Add external'", 1298 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1299 1300 self.src_dir = src_dir 1301 self.repo_dir = repo_dir 1302 1303 @skipIfNoSvn() 1304 def tearDown(self): 1305 os.chdir(self.origdir) 1306 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 1307 print("Not cleaning up %s. Please remove manually." % self.tempdir) 1308 else: 1309 bb.utils.prunedir(self.tempdir) 1310 1311 @skipIfNoSvn() 1312 @skipIfNoNetwork() 1313 def test_noexternal_svn(self): 1314 # Always match the rev count from setUp (currently rev 2) 1315 url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '') 1316 fetcher = bb.fetch.Fetch([url], self.d) 1317 fetcher.download() 1318 os.chdir(os.path.dirname(self.unpackdir)) 1319 fetcher.unpack(self.unpackdir) 1320 1321 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1322 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1323 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should NOT exist") 1324 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should NOT exit") 1325 1326 @skipIfNoSvn() 1327 def test_external_svn(self): 1328 # Always match the rev count from setUp (currently rev 2) 1329 url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '') 1330 fetcher = bb.fetch.Fetch([url], self.d) 1331 fetcher.download() 1332 os.chdir(os.path.dirname(self.unpackdir)) 1333 fetcher.unpack(self.unpackdir) 1334 1335 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1336 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1337 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should exist") 1338 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should exit") 1339 1340class TrustedNetworksTest(FetcherTest): 1341 def test_trusted_network(self): 1342 # Ensure trusted_network returns False when the host IS in the list. 1343 url = "git://Someserver.org/foo;rev=1;branch=master" 1344 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org") 1345 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1346 1347 def test_wild_trusted_network(self): 1348 # Ensure trusted_network returns true when the *.host IS in the list. 1349 url = "git://Someserver.org/foo;rev=1;branch=master" 1350 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1351 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1352 1353 def test_prefix_wild_trusted_network(self): 1354 # Ensure trusted_network returns true when the prefix matches *.host. 1355 url = "git://git.Someserver.org/foo;rev=1;branch=master" 1356 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1357 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1358 1359 def test_two_prefix_wild_trusted_network(self): 1360 # Ensure trusted_network returns true when the prefix matches *.host. 1361 url = "git://something.git.Someserver.org/foo;rev=1;branch=master" 1362 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1363 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1364 1365 def test_port_trusted_network(self): 1366 # Ensure trusted_network returns True, even if the url specifies a port. 1367 url = "git://someserver.org:8080/foo;rev=1;branch=master" 1368 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org") 1369 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1370 1371 def test_untrusted_network(self): 1372 # Ensure trusted_network returns False when the host is NOT in the list. 1373 url = "git://someserver.org/foo;rev=1;branch=master" 1374 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1375 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1376 1377 def test_wild_untrusted_network(self): 1378 # Ensure trusted_network returns False when the host is NOT in the list. 1379 url = "git://*.someserver.org/foo;rev=1;branch=master" 1380 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1381 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1382 1383class URLHandle(unittest.TestCase): 1384 1385 datatable = { 1386 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}), 1387 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}), 1388 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])), 1389 "git://git.openembedded.org/bitbake;branch=@foo;protocol=https" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo', 'protocol' : 'https'}), 1390 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}), 1391 "https://somesite.com/somerepo.git;user=anyUser:idtoken=1234" : ('https', 'somesite.com', '/somerepo.git', '', '', {'user': 'anyUser:idtoken=1234'}), 1392 r'git://s.o-me_ONE:!#$%^&*()-_={}[]\|:?,.<>~`@git.openembedded.org/bitbake;branch=main;protocol=https': ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', r'!#$%^&*()-_={}[]\|:?,.<>~`', {'branch': 'main', 'protocol' : 'https'}), 1393 } 1394 # we require a pathname to encodeurl but users can still pass such urls to 1395 # decodeurl and we need to handle them 1396 decodedata = datatable.copy() 1397 decodedata.update({ 1398 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}), 1399 "npmsw://some.registry.url;package=@pkg;version=latest": ('npmsw', 'some.registry.url', '/', '', '', {'package': '@pkg', 'version': 'latest'}), 1400 }) 1401 1402 def test_decodeurl(self): 1403 for k, v in self.decodedata.items(): 1404 result = bb.fetch.decodeurl(k) 1405 self.assertEqual(result, v) 1406 1407 def test_encodeurl(self): 1408 for k, v in self.datatable.items(): 1409 result = bb.fetch.encodeurl(v) 1410 self.assertEqual(result, k) 1411 1412class FetchLatestVersionTest(FetcherTest): 1413 1414 test_git_uris = { 1415 # version pattern "X.Y.Z" 1416 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "", "") 1417 : "1.99.4", 1418 # version pattern "vX.Y" 1419 # mirror of git.infradead.org since network issues interfered with testing 1420 ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master;protocol=https", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "", "") 1421 : "1.5.0", 1422 # version pattern "pkg_name-X.Y" 1423 # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing 1424 ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master;protocol=https", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "", "") 1425 : "1.0", 1426 # version pattern "pkg_name-vX.Y.Z" 1427 ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master;protocol=https", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "", "") 1428 : "1.4.0", 1429 # combination version pattern 1430 ("sysprof", "git://git.yoctoproject.org/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "", "") 1431 : "1.2.0", 1432 ("u-boot-mkimage", "git://source.denx.de/u-boot/u-boot.git;branch=master;protocol=https", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "") 1433 : "2014.01", 1434 # version pattern "yyyymmdd" 1435 ("mobile-broadband-provider-info", "git://git.yoctoproject.org/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "", "") 1436 : "20120614", 1437 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX 1438 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing 1439 ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master;protocol=https", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))", "") 1440 : "0.4.3", 1441 ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master;protocol=https", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))", "") 1442 : "11.0.0", 1443 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))", "") 1444 : "1.3.59", 1445 ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))", "") 1446 : "3.82+dbg0.9", 1447 ("sysdig", "git://github.com/draios/sysdig.git;branch=dev;protocol=https", "4fb6288275f567f63515df0ff0a6518043ecfa9b", r"^(?P<pver>\d+(\.\d+)+)", "10.0.0") 1448 : "0.28.0", 1449 } 1450 1451 test_wget_uris = { 1452 # 1453 # packages with versions inside directory name 1454 # 1455 # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2 1456 ("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "") 1457 : "2.24.2", 1458 # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz 1459 ("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "") 1460 : "1.6.0", 1461 # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz 1462 ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "") 1463 : "2.8.12.1", 1464 # https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz 1465 ("libxml2", "/software/libxml2/2.9/libxml2-2.9.14.tar.xz", "", "") 1466 : "2.10.3", 1467 # 1468 # packages with versions only in current directory 1469 # 1470 # https://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2 1471 ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "") 1472 : "2.19", 1473 # https://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2 1474 ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "") 1475 : "20120814", 1476 # 1477 # packages with "99" in the name of possible version 1478 # 1479 # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz 1480 ("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "") 1481 : "5.0", 1482 # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2 1483 ("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "") 1484 : "1.15.1", 1485 # 1486 # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX 1487 # 1488 # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2 1489 # https://github.com/apple/cups/releases 1490 ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", r"(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz") 1491 : "2.0.0", 1492 # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz 1493 # http://ftp.debian.org/debian/pool/main/d/db5.3/ 1494 ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", r"(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz") 1495 : "5.3.10", 1496 # 1497 # packages where the tarball compression changed in the new version 1498 # 1499 # http://ftp.debian.org/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz 1500 ("minicom", "/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz", "", "") 1501 : "2.8", 1502 } 1503 1504 test_crate_uris = { 1505 # basic example; version pattern "A.B.C+cargo-D.E.F" 1506 ("cargo-c", "crate://crates.io/cargo-c/0.9.18+cargo-0.69") 1507 : "0.9.29" 1508 } 1509 1510 @skipIfNoNetwork() 1511 def test_git_latest_versionstring(self): 1512 for k, v in self.test_git_uris.items(): 1513 self.d.setVar("PN", k[0]) 1514 self.d.setVar("SRCREV", k[2]) 1515 self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3]) 1516 ud = bb.fetch2.FetchData(k[1], self.d) 1517 pupver= ud.method.latest_versionstring(ud, self.d) 1518 verstring = pupver[0] 1519 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1520 r = bb.utils.vercmp_string(v, verstring) 1521 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1522 if k[4]: 1523 r = bb.utils.vercmp_string(verstring, k[4]) 1524 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], verstring, k[4])) 1525 1526 def test_wget_latest_versionstring(self): 1527 testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata" 1528 server = HTTPService(testdata, host="127.0.0.1") 1529 server.start() 1530 port = server.port 1531 try: 1532 for k, v in self.test_wget_uris.items(): 1533 self.d.setVar("PN", k[0]) 1534 checkuri = "" 1535 if k[2]: 1536 checkuri = "http://127.0.0.1:%s/" % port + k[2] 1537 self.d.setVar("UPSTREAM_CHECK_URI", checkuri) 1538 self.d.setVar("UPSTREAM_CHECK_REGEX", k[3]) 1539 url = "http://127.0.0.1:%s/" % port + k[1] 1540 ud = bb.fetch2.FetchData(url, self.d) 1541 pupver = ud.method.latest_versionstring(ud, self.d) 1542 verstring = pupver[0] 1543 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1544 r = bb.utils.vercmp_string(v, verstring) 1545 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1546 finally: 1547 server.stop() 1548 1549 @skipIfNoNetwork() 1550 def test_crate_latest_versionstring(self): 1551 for k, v in self.test_crate_uris.items(): 1552 self.d.setVar("PN", k[0]) 1553 ud = bb.fetch2.FetchData(k[1], self.d) 1554 pupver = ud.method.latest_versionstring(ud, self.d) 1555 verstring = pupver[0] 1556 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1557 r = bb.utils.vercmp_string(v, verstring) 1558 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1559 1560class FetchCheckStatusTest(FetcherTest): 1561 test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz", 1562 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz", 1563 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz", 1564 "https://yoctoproject.org/", 1565 "https://docs.yoctoproject.org", 1566 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", 1567 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", 1568 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz", 1569 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD 1570 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz" 1571 ] 1572 1573 @skipIfNoNetwork() 1574 def test_wget_checkstatus(self): 1575 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d) 1576 for u in self.test_wget_uris: 1577 with self.subTest(url=u): 1578 ud = fetch.ud[u] 1579 m = ud.method 1580 ret = m.checkstatus(fetch, ud, self.d) 1581 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1582 1583 @skipIfNoNetwork() 1584 def test_wget_checkstatus_connection_cache(self): 1585 from bb.fetch2 import FetchConnectionCache 1586 1587 connection_cache = FetchConnectionCache() 1588 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d, 1589 connection_cache = connection_cache) 1590 1591 for u in self.test_wget_uris: 1592 with self.subTest(url=u): 1593 ud = fetch.ud[u] 1594 m = ud.method 1595 ret = m.checkstatus(fetch, ud, self.d) 1596 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1597 1598 connection_cache.close_connections() 1599 1600 1601class GitMakeShallowTest(FetcherTest): 1602 def setUp(self): 1603 FetcherTest.setUp(self) 1604 self.gitdir = os.path.join(self.tempdir, 'gitshallow') 1605 bb.utils.mkdirhier(self.gitdir) 1606 self.git_init() 1607 1608 def assertRefs(self, expected_refs): 1609 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines() 1610 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines() 1611 self.assertEqual(sorted(full_expected), sorted(actual_refs)) 1612 1613 def assertRevCount(self, expected_count, args=None): 1614 if args is None: 1615 args = ['HEAD'] 1616 revs = self.git(['rev-list'] + args) 1617 actual_count = len(revs.splitlines()) 1618 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1619 1620 def make_shallow(self, args=None): 1621 if args is None: 1622 args = ['HEAD'] 1623 return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir) 1624 1625 def add_empty_file(self, path, msg=None): 1626 if msg is None: 1627 msg = path 1628 open(os.path.join(self.gitdir, path), 'w').close() 1629 self.git(['add', path]) 1630 self.git(['commit', '-m', msg, path]) 1631 1632 def test_make_shallow_single_branch_no_merge(self): 1633 self.add_empty_file('a') 1634 self.add_empty_file('b') 1635 self.assertRevCount(2) 1636 self.make_shallow() 1637 self.assertRevCount(1) 1638 1639 def test_make_shallow_single_branch_one_merge(self): 1640 self.add_empty_file('a') 1641 self.add_empty_file('b') 1642 self.git('checkout -b a_branch') 1643 self.add_empty_file('c') 1644 self.git('checkout master') 1645 self.add_empty_file('d') 1646 self.git('merge --no-ff --no-edit a_branch') 1647 self.git('branch -d a_branch') 1648 self.add_empty_file('e') 1649 self.assertRevCount(6) 1650 self.make_shallow(['HEAD~2']) 1651 self.assertRevCount(5) 1652 1653 def test_make_shallow_at_merge(self): 1654 self.add_empty_file('a') 1655 self.git('checkout -b a_branch') 1656 self.add_empty_file('b') 1657 self.git('checkout master') 1658 self.git('merge --no-ff --no-edit a_branch') 1659 self.git('branch -d a_branch') 1660 self.assertRevCount(3) 1661 self.make_shallow() 1662 self.assertRevCount(1) 1663 1664 def test_make_shallow_annotated_tag(self): 1665 self.add_empty_file('a') 1666 self.add_empty_file('b') 1667 self.git('tag -a -m a_tag a_tag') 1668 self.assertRevCount(2) 1669 self.make_shallow(['a_tag']) 1670 self.assertRevCount(1) 1671 1672 def test_make_shallow_multi_ref(self): 1673 self.add_empty_file('a') 1674 self.add_empty_file('b') 1675 self.git('checkout -b a_branch') 1676 self.add_empty_file('c') 1677 self.git('checkout master') 1678 self.add_empty_file('d') 1679 self.git('checkout -b a_branch_2') 1680 self.add_empty_file('a_tag') 1681 self.git('tag a_tag') 1682 self.git('checkout master') 1683 self.git('branch -D a_branch_2') 1684 self.add_empty_file('e') 1685 self.assertRevCount(6, ['--all']) 1686 self.make_shallow() 1687 self.assertRevCount(5, ['--all']) 1688 1689 def test_make_shallow_multi_ref_trim(self): 1690 self.add_empty_file('a') 1691 self.git('checkout -b a_branch') 1692 self.add_empty_file('c') 1693 self.git('checkout master') 1694 self.assertRevCount(1) 1695 self.assertRevCount(2, ['--all']) 1696 self.assertRefs(['master', 'a_branch']) 1697 self.make_shallow(['-r', 'master', 'HEAD']) 1698 self.assertRevCount(1, ['--all']) 1699 self.assertRefs(['master']) 1700 1701 def test_make_shallow_noop(self): 1702 self.add_empty_file('a') 1703 self.assertRevCount(1) 1704 self.make_shallow() 1705 self.assertRevCount(1) 1706 1707 @skipIfNoNetwork() 1708 def test_make_shallow_bitbake(self): 1709 self.git('remote add origin https://github.com/openembedded/bitbake') 1710 self.git('fetch --tags origin') 1711 orig_revs = len(self.git('rev-list --all').splitlines()) 1712 self.make_shallow(['refs/tags/1.10.0']) 1713 self.assertRevCount(orig_revs - 1746, ['--all']) 1714 1715class GitShallowTest(FetcherTest): 1716 def setUp(self): 1717 FetcherTest.setUp(self) 1718 self.gitdir = os.path.join(self.tempdir, 'git') 1719 self.srcdir = os.path.join(self.tempdir, 'gitsource') 1720 1721 bb.utils.mkdirhier(self.srcdir) 1722 self.git_init(cwd=self.srcdir) 1723 self.d.setVar('WORKDIR', self.tempdir) 1724 self.d.setVar('S', self.gitdir) 1725 self.d.delVar('PREMIRRORS') 1726 self.d.delVar('MIRRORS') 1727 1728 uri = 'git://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1729 self.d.setVar('SRC_URI', uri) 1730 self.d.setVar('SRCREV', '${AUTOREV}') 1731 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 1732 1733 self.d.setVar('BB_GIT_SHALLOW', '1') 1734 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0') 1735 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 1736 self.d.setVar("__BBSRCREV_SEEN", "1") 1737 1738 def assertRefs(self, expected_refs, cwd=None): 1739 if cwd is None: 1740 cwd = self.gitdir 1741 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines() 1742 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines() 1743 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs))) 1744 1745 def assertRevCount(self, expected_count, args=None, cwd=None): 1746 if args is None: 1747 args = ['HEAD'] 1748 if cwd is None: 1749 cwd = self.gitdir 1750 revs = self.git(['rev-list'] + args, cwd=cwd) 1751 actual_count = len(revs.splitlines()) 1752 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1753 1754 def add_empty_file(self, path, cwd=None, msg=None): 1755 if msg is None: 1756 msg = path 1757 if cwd is None: 1758 cwd = self.srcdir 1759 open(os.path.join(cwd, path), 'w').close() 1760 self.git(['add', path], cwd) 1761 self.git(['commit', '-m', msg, path], cwd) 1762 1763 def fetch(self, uri=None): 1764 if uri is None: 1765 uris = self.d.getVar('SRC_URI').split() 1766 uri = uris[0] 1767 d = self.d 1768 else: 1769 d = self.d.createCopy() 1770 d.setVar('SRC_URI', uri) 1771 uri = d.expand(uri) 1772 uris = [uri] 1773 1774 fetcher = bb.fetch2.Fetch(uris, d) 1775 fetcher.download() 1776 ud = fetcher.ud[uri] 1777 return fetcher, ud 1778 1779 def fetch_and_unpack(self, uri=None): 1780 fetcher, ud = self.fetch(uri) 1781 fetcher.unpack(self.d.getVar('WORKDIR')) 1782 assert os.path.exists(self.d.getVar('S')) 1783 return fetcher, ud 1784 1785 def fetch_shallow(self, uri=None, disabled=False, keepclone=False): 1786 """Fetch a uri, generating a shallow tarball, then unpack using it""" 1787 fetcher, ud = self.fetch_and_unpack(uri) 1788 assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri) 1789 1790 # Confirm that the unpacked repo is unshallow 1791 if not disabled: 1792 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0])) 1793 1794 # fetch and unpack, from the shallow tarball 1795 bb.utils.remove(self.gitdir, recurse=True) 1796 bb.process.run('chmod u+w -R "%s"' % ud.clonedir) 1797 bb.utils.remove(ud.clonedir, recurse=True) 1798 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True) 1799 1800 # confirm that the unpacked repo is used when no git clone or git 1801 # mirror tarball is available 1802 fetcher, ud = self.fetch_and_unpack(uri) 1803 if not disabled: 1804 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir 1805 else: 1806 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir 1807 return fetcher, ud 1808 1809 def test_shallow_disabled(self): 1810 self.add_empty_file('a') 1811 self.add_empty_file('b') 1812 self.assertRevCount(2, cwd=self.srcdir) 1813 1814 self.d.setVar('BB_GIT_SHALLOW', '0') 1815 self.fetch_shallow(disabled=True) 1816 self.assertRevCount(2) 1817 1818 def test_shallow_nobranch(self): 1819 self.add_empty_file('a') 1820 self.add_empty_file('b') 1821 self.assertRevCount(2, cwd=self.srcdir) 1822 1823 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip() 1824 self.d.setVar('SRCREV', srcrev) 1825 uri = self.d.getVar('SRC_URI').split()[0] 1826 uri = '%s;nobranch=1;bare=1' % uri 1827 1828 self.fetch_shallow(uri) 1829 self.assertRevCount(1) 1830 1831 # shallow refs are used to ensure the srcrev sticks around when we 1832 # have no other branches referencing it 1833 self.assertRefs(['refs/shallow/default']) 1834 1835 def test_shallow_default_depth_1(self): 1836 # Create initial git repo 1837 self.add_empty_file('a') 1838 self.add_empty_file('b') 1839 self.assertRevCount(2, cwd=self.srcdir) 1840 1841 self.fetch_shallow() 1842 self.assertRevCount(1) 1843 1844 def test_shallow_depth_0_disables(self): 1845 self.add_empty_file('a') 1846 self.add_empty_file('b') 1847 self.assertRevCount(2, cwd=self.srcdir) 1848 1849 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1850 self.fetch_shallow(disabled=True) 1851 self.assertRevCount(2) 1852 1853 def test_shallow_depth_default_override(self): 1854 self.add_empty_file('a') 1855 self.add_empty_file('b') 1856 self.assertRevCount(2, cwd=self.srcdir) 1857 1858 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2') 1859 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1') 1860 self.fetch_shallow() 1861 self.assertRevCount(1) 1862 1863 def test_shallow_depth_default_override_disable(self): 1864 self.add_empty_file('a') 1865 self.add_empty_file('b') 1866 self.add_empty_file('c') 1867 self.assertRevCount(3, cwd=self.srcdir) 1868 1869 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1870 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2') 1871 self.fetch_shallow() 1872 self.assertRevCount(2) 1873 1874 def test_current_shallow_out_of_date_clone(self): 1875 # Create initial git repo 1876 self.add_empty_file('a') 1877 self.add_empty_file('b') 1878 self.add_empty_file('c') 1879 self.assertRevCount(3, cwd=self.srcdir) 1880 1881 # Clone and generate mirror tarball 1882 fetcher, ud = self.fetch() 1883 1884 # Ensure we have a current mirror tarball, but an out of date clone 1885 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir) 1886 self.assertRevCount(2, cwd=ud.clonedir) 1887 1888 # Fetch and unpack, from the current tarball, not the out of date clone 1889 bb.utils.remove(self.gitdir, recurse=True) 1890 fetcher, ud = self.fetch() 1891 fetcher.unpack(self.d.getVar('WORKDIR')) 1892 self.assertRevCount(1) 1893 1894 def test_shallow_single_branch_no_merge(self): 1895 self.add_empty_file('a') 1896 self.add_empty_file('b') 1897 self.assertRevCount(2, cwd=self.srcdir) 1898 1899 self.fetch_shallow() 1900 self.assertRevCount(1) 1901 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1902 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1903 1904 def test_shallow_no_dangling(self): 1905 self.add_empty_file('a') 1906 self.add_empty_file('b') 1907 self.assertRevCount(2, cwd=self.srcdir) 1908 1909 self.fetch_shallow() 1910 self.assertRevCount(1) 1911 assert not self.git('fsck --dangling') 1912 1913 def test_shallow_srcrev_branch_truncation(self): 1914 self.add_empty_file('a') 1915 self.add_empty_file('b') 1916 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip() 1917 self.add_empty_file('c') 1918 self.assertRevCount(3, cwd=self.srcdir) 1919 1920 self.d.setVar('SRCREV', b_commit) 1921 self.fetch_shallow() 1922 1923 # The 'c' commit was removed entirely, and 'a' was removed from history 1924 self.assertRevCount(1, ['--all']) 1925 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit) 1926 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1927 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1928 assert not os.path.exists(os.path.join(self.gitdir, 'c')) 1929 1930 def test_shallow_ref_pruning(self): 1931 self.add_empty_file('a') 1932 self.add_empty_file('b') 1933 self.git('branch a_branch', cwd=self.srcdir) 1934 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 1935 self.assertRevCount(2, cwd=self.srcdir) 1936 1937 self.fetch_shallow() 1938 1939 self.assertRefs(['master', 'origin/master']) 1940 self.assertRevCount(1) 1941 1942 def test_shallow_submodules(self): 1943 self.add_empty_file('a') 1944 self.add_empty_file('b') 1945 1946 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1947 bb.utils.mkdirhier(smdir) 1948 self.git_init(cwd=smdir) 1949 # Make this look like it was cloned from a remote... 1950 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1951 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1952 self.add_empty_file('asub', cwd=smdir) 1953 self.add_empty_file('bsub', cwd=smdir) 1954 1955 self.git('submodule init', cwd=self.srcdir) 1956 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 1957 self.git('submodule update', cwd=self.srcdir) 1958 self.git('commit -m submodule -a', cwd=self.srcdir) 1959 1960 uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1961 fetcher, ud = self.fetch_shallow(uri) 1962 1963 # Verify the main repository is shallow 1964 self.assertRevCount(1) 1965 1966 # Verify the gitsubmodule directory is present 1967 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 1968 1969 # Verify the submodule is also shallow 1970 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 1971 1972 def test_shallow_submodule_mirrors(self): 1973 self.add_empty_file('a') 1974 self.add_empty_file('b') 1975 1976 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1977 bb.utils.mkdirhier(smdir) 1978 self.git_init(cwd=smdir) 1979 # Make this look like it was cloned from a remote... 1980 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1981 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1982 self.add_empty_file('asub', cwd=smdir) 1983 self.add_empty_file('bsub', cwd=smdir) 1984 1985 self.git('submodule init', cwd=self.srcdir) 1986 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 1987 self.git('submodule update', cwd=self.srcdir) 1988 self.git('commit -m submodule -a', cwd=self.srcdir) 1989 1990 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir 1991 1992 # Fetch once to generate the shallow tarball 1993 fetcher, ud = self.fetch(uri) 1994 1995 # Set up the mirror 1996 mirrordir = os.path.join(self.tempdir, 'mirror') 1997 bb.utils.rename(self.dldir, mirrordir) 1998 self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/' % mirrordir) 1999 2000 # Fetch from the mirror 2001 bb.utils.remove(self.dldir, recurse=True) 2002 bb.utils.remove(self.gitdir, recurse=True) 2003 self.fetch_and_unpack(uri) 2004 2005 # Verify the main repository is shallow 2006 self.assertRevCount(1) 2007 2008 # Verify the gitsubmodule directory is present 2009 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 2010 2011 # Verify the submodule is also shallow 2012 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 2013 2014 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')): 2015 def test_shallow_annex(self): 2016 self.add_empty_file('a') 2017 self.add_empty_file('b') 2018 self.git('annex init', cwd=self.srcdir) 2019 open(os.path.join(self.srcdir, 'c'), 'w').close() 2020 self.git('annex add c', cwd=self.srcdir) 2021 self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir) 2022 bb.process.run('chmod u+w -R %s' % self.srcdir) 2023 2024 uri = 'gitannex://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 2025 fetcher, ud = self.fetch_shallow(uri) 2026 2027 self.assertRevCount(1) 2028 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0] 2029 assert os.path.exists(os.path.join(self.gitdir, 'c')) 2030 2031 def test_shallow_multi_one_uri(self): 2032 # Create initial git repo 2033 self.add_empty_file('a') 2034 self.add_empty_file('b') 2035 self.git('checkout -b a_branch', cwd=self.srcdir) 2036 self.add_empty_file('c') 2037 self.git('tag v0.0 HEAD', cwd=self.srcdir) 2038 self.add_empty_file('d') 2039 self.git('checkout master', cwd=self.srcdir) 2040 self.add_empty_file('e') 2041 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2042 self.add_empty_file('f') 2043 self.assertRevCount(7, cwd=self.srcdir) 2044 2045 uri = self.d.getVar('SRC_URI').split()[0] 2046 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri 2047 2048 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2049 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2050 self.d.setVar('SRCREV_master', '${AUTOREV}') 2051 self.d.setVar('SRCREV_a_branch', '${AUTOREV}') 2052 2053 self.fetch_shallow(uri) 2054 2055 self.assertRevCount(4) 2056 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2057 2058 def test_shallow_multi_one_uri_depths(self): 2059 # Create initial git repo 2060 self.add_empty_file('a') 2061 self.add_empty_file('b') 2062 self.git('checkout -b a_branch', cwd=self.srcdir) 2063 self.add_empty_file('c') 2064 self.add_empty_file('d') 2065 self.git('checkout master', cwd=self.srcdir) 2066 self.add_empty_file('e') 2067 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2068 self.add_empty_file('f') 2069 self.assertRevCount(7, cwd=self.srcdir) 2070 2071 uri = self.d.getVar('SRC_URI').split()[0] 2072 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri 2073 2074 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2075 self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3') 2076 self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1') 2077 self.d.setVar('SRCREV_master', '${AUTOREV}') 2078 self.d.setVar('SRCREV_a_branch', '${AUTOREV}') 2079 2080 self.fetch_shallow(uri) 2081 2082 self.assertRevCount(4, ['--all']) 2083 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2084 2085 def test_shallow_clone_preferred_over_shallow(self): 2086 self.add_empty_file('a') 2087 self.add_empty_file('b') 2088 2089 # Fetch once to generate the shallow tarball 2090 fetcher, ud = self.fetch() 2091 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0])) 2092 2093 # Fetch and unpack with both the clonedir and shallow tarball available 2094 bb.utils.remove(self.gitdir, recurse=True) 2095 fetcher, ud = self.fetch_and_unpack() 2096 2097 # The unpacked tree should *not* be shallow 2098 self.assertRevCount(2) 2099 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')) 2100 2101 def test_shallow_mirrors(self): 2102 self.add_empty_file('a') 2103 self.add_empty_file('b') 2104 2105 # Fetch once to generate the shallow tarball 2106 fetcher, ud = self.fetch() 2107 mirrortarball = ud.mirrortarballs[0] 2108 assert os.path.exists(os.path.join(self.dldir, mirrortarball)) 2109 2110 # Set up the mirror 2111 mirrordir = os.path.join(self.tempdir, 'mirror') 2112 bb.utils.mkdirhier(mirrordir) 2113 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % mirrordir) 2114 2115 bb.utils.rename(os.path.join(self.dldir, mirrortarball), 2116 os.path.join(mirrordir, mirrortarball)) 2117 2118 # Fetch from the mirror 2119 bb.utils.remove(self.dldir, recurse=True) 2120 bb.utils.remove(self.gitdir, recurse=True) 2121 self.fetch_and_unpack() 2122 self.assertRevCount(1) 2123 2124 def test_shallow_invalid_depth(self): 2125 self.add_empty_file('a') 2126 self.add_empty_file('b') 2127 2128 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12') 2129 with self.assertRaises(bb.fetch2.FetchError): 2130 self.fetch() 2131 2132 def test_shallow_invalid_depth_default(self): 2133 self.add_empty_file('a') 2134 self.add_empty_file('b') 2135 2136 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12') 2137 with self.assertRaises(bb.fetch2.FetchError): 2138 self.fetch() 2139 2140 def test_shallow_extra_refs(self): 2141 self.add_empty_file('a') 2142 self.add_empty_file('b') 2143 self.git('branch a_branch', cwd=self.srcdir) 2144 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 2145 self.assertRevCount(2, cwd=self.srcdir) 2146 2147 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch') 2148 self.fetch_shallow() 2149 2150 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2151 self.assertRevCount(1) 2152 2153 def test_shallow_extra_refs_wildcard(self): 2154 self.add_empty_file('a') 2155 self.add_empty_file('b') 2156 self.git('branch a_branch', cwd=self.srcdir) 2157 self.git('tag v1.0', cwd=self.srcdir) 2158 self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir) 2159 self.assertRevCount(2, cwd=self.srcdir) 2160 2161 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2162 self.fetch_shallow() 2163 2164 self.assertRefs(['master', 'origin/master', 'v1.0']) 2165 self.assertRevCount(1) 2166 2167 def test_shallow_missing_extra_refs(self): 2168 self.add_empty_file('a') 2169 self.add_empty_file('b') 2170 2171 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo') 2172 with self.assertRaises(bb.fetch2.FetchError): 2173 self.fetch() 2174 2175 def test_shallow_missing_extra_refs_wildcard(self): 2176 self.add_empty_file('a') 2177 self.add_empty_file('b') 2178 2179 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2180 self.fetch() 2181 2182 def test_shallow_remove_revs(self): 2183 # Create initial git repo 2184 self.add_empty_file('a') 2185 self.add_empty_file('b') 2186 self.git('checkout -b a_branch', cwd=self.srcdir) 2187 self.add_empty_file('c') 2188 self.add_empty_file('d') 2189 self.git('checkout master', cwd=self.srcdir) 2190 self.git('tag v0.0 a_branch', cwd=self.srcdir) 2191 self.add_empty_file('e') 2192 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2193 self.git('branch -d a_branch', cwd=self.srcdir) 2194 self.add_empty_file('f') 2195 self.assertRevCount(7, cwd=self.srcdir) 2196 2197 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2198 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2199 2200 self.fetch_shallow() 2201 2202 self.assertRevCount(2) 2203 2204 def test_shallow_invalid_revs(self): 2205 self.add_empty_file('a') 2206 self.add_empty_file('b') 2207 2208 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2209 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2210 2211 with self.assertRaises(bb.fetch2.FetchError): 2212 self.fetch() 2213 2214 def test_shallow_fetch_missing_revs(self): 2215 self.add_empty_file('a') 2216 self.add_empty_file('b') 2217 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2218 self.git('tag v0.0 master', cwd=self.srcdir) 2219 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2220 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2221 2222 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm: 2223 self.fetch_shallow() 2224 self.assertIn("fatal: no commits selected for shallow requests", cm.output[0]) 2225 2226 def test_shallow_fetch_missing_revs_fails(self): 2227 self.add_empty_file('a') 2228 self.add_empty_file('b') 2229 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2230 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2231 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2232 2233 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm: 2234 self.fetch_shallow() 2235 self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0]) 2236 2237 @skipIfNoNetwork() 2238 def test_bitbake(self): 2239 self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir) 2240 self.git('config core.bare true', cwd=self.srcdir) 2241 self.git('fetch', cwd=self.srcdir) 2242 2243 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2244 # Note that the 1.10.0 tag is annotated, so this also tests 2245 # reference of an annotated vs unannotated tag 2246 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0') 2247 2248 self.fetch_shallow() 2249 2250 # Confirm that the history of 1.10.0 was removed 2251 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines()) 2252 revs = len(self.git('rev-list master').splitlines()) 2253 self.assertNotEqual(orig_revs, revs) 2254 self.assertRefs(['master', 'origin/master']) 2255 self.assertRevCount(orig_revs - 1760) 2256 2257 def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self): 2258 self.add_empty_file('a') 2259 fetcher, ud = self.fetch() 2260 bb.utils.remove(self.gitdir, recurse=True) 2261 bb.utils.remove(self.dldir, recurse=True) 2262 2263 with self.assertRaises(bb.fetch2.UnpackError) as context: 2264 fetcher.unpack(self.d.getVar('WORKDIR')) 2265 2266 self.assertIn("No up to date source found", context.exception.msg) 2267 self.assertIn("clone directory not available or not up to date", context.exception.msg) 2268 2269 @skipIfNoNetwork() 2270 def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self): 2271 self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0') 2272 self.d.setVar('BB_GIT_SHALLOW', '1') 2273 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 2274 fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests;branch=master;protocol=https"], self.d) 2275 fetcher.download() 2276 2277 bb.utils.remove(self.dldir + "/*.tar.gz") 2278 fetcher.unpack(self.unpackdir) 2279 2280 dir = os.listdir(self.unpackdir + "/git/") 2281 self.assertIn("fstests.doap", dir) 2282 2283class GitLfsTest(FetcherTest): 2284 def skipIfNoGitLFS(): 2285 import shutil 2286 if not shutil.which('git-lfs'): 2287 return unittest.skip('git-lfs not installed') 2288 return lambda f: f 2289 2290 def setUp(self): 2291 FetcherTest.setUp(self) 2292 2293 self.gitdir = os.path.join(self.tempdir, 'git') 2294 self.srcdir = os.path.join(self.tempdir, 'gitsource') 2295 2296 self.d.setVar('WORKDIR', self.tempdir) 2297 self.d.setVar('S', self.gitdir) 2298 self.d.delVar('PREMIRRORS') 2299 self.d.delVar('MIRRORS') 2300 2301 self.d.setVar('SRCREV', '${AUTOREV}') 2302 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 2303 self.d.setVar("__BBSRCREV_SEEN", "1") 2304 2305 bb.utils.mkdirhier(self.srcdir) 2306 self.git_init(cwd=self.srcdir) 2307 self.commit_file('.gitattributes', '*.mp3 filter=lfs -text') 2308 2309 def commit_file(self, filename, content): 2310 with open(os.path.join(self.srcdir, filename), "w") as f: 2311 f.write(content) 2312 self.git(["add", filename], cwd=self.srcdir) 2313 self.git(["commit", "-m", "Change"], cwd=self.srcdir) 2314 return self.git(["rev-parse", "HEAD"], cwd=self.srcdir).strip() 2315 2316 def fetch(self, uri=None, download=True): 2317 uris = self.d.getVar('SRC_URI').split() 2318 uri = uris[0] 2319 d = self.d 2320 2321 fetcher = bb.fetch2.Fetch(uris, d) 2322 if download: 2323 fetcher.download() 2324 ud = fetcher.ud[uri] 2325 return fetcher, ud 2326 2327 def get_real_git_lfs_file(self): 2328 self.d.setVar('PATH', os.environ.get('PATH')) 2329 fetcher, ud = self.fetch() 2330 fetcher.unpack(self.d.getVar('WORKDIR')) 2331 unpacked_lfs_file = os.path.join(self.d.getVar('WORKDIR'), 'git', "Cat_poster_1.jpg") 2332 return unpacked_lfs_file 2333 2334 @skipIfNoGitLFS() 2335 def test_fetch_lfs_on_srcrev_change(self): 2336 """Test if fetch downloads missing LFS objects when a different revision within an existing repository is requested""" 2337 self.git(["lfs", "install", "--local"], cwd=self.srcdir) 2338 2339 @contextlib.contextmanager 2340 def hide_upstream_repository(): 2341 """Hide the upstream repository to make sure that git lfs cannot pull from it""" 2342 temp_name = self.srcdir + ".bak" 2343 os.rename(self.srcdir, temp_name) 2344 try: 2345 yield 2346 finally: 2347 os.rename(temp_name, self.srcdir) 2348 2349 def fetch_and_verify(revision, filename, content): 2350 self.d.setVar('SRCREV', revision) 2351 fetcher, ud = self.fetch() 2352 2353 with hide_upstream_repository(): 2354 workdir = self.d.getVar('WORKDIR') 2355 fetcher.unpack(workdir) 2356 2357 with open(os.path.join(workdir, "git", filename)) as f: 2358 self.assertEqual(f.read(), content) 2359 2360 commit_1 = self.commit_file("a.mp3", "version 1") 2361 commit_2 = self.commit_file("a.mp3", "version 2") 2362 2363 self.d.setVar('SRC_URI', "git://%s;protocol=file;lfs=1;branch=master" % self.srcdir) 2364 2365 # Seed the local download folder by fetching the latest commit and verifying that the LFS contents are 2366 # available even when the upstream repository disappears. 2367 fetch_and_verify(commit_2, "a.mp3", "version 2") 2368 # Verify that even when an older revision is fetched, the needed LFS objects are fetched into the download 2369 # folder. 2370 fetch_and_verify(commit_1, "a.mp3", "version 1") 2371 2372 @skipIfNoGitLFS() 2373 @skipIfNoNetwork() 2374 def test_real_git_lfs_repo_succeeds_without_lfs_param(self): 2375 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master") 2376 f = self.get_real_git_lfs_file() 2377 self.assertTrue(os.path.exists(f)) 2378 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) 2379 2380 @skipIfNoGitLFS() 2381 @skipIfNoNetwork() 2382 def test_real_git_lfs_repo_succeeds(self): 2383 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=1") 2384 f = self.get_real_git_lfs_file() 2385 self.assertTrue(os.path.exists(f)) 2386 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) 2387 2388 @skipIfNoGitLFS() 2389 @skipIfNoNetwork() 2390 def test_real_git_lfs_repo_skips(self): 2391 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=0") 2392 f = self.get_real_git_lfs_file() 2393 # This is the actual non-smudged placeholder file on the repo if git-lfs does not run 2394 lfs_file = ( 2395 'version https://git-lfs.github.com/spec/v1\n' 2396 'oid sha256:34be66b1a39a1955b46a12588df9d5f6fc1da790e05cf01f3c7422f4bbbdc26b\n' 2397 'size 11423554\n' 2398 ) 2399 2400 with open(f) as fh: 2401 self.assertEqual(lfs_file, fh.read()) 2402 2403 @skipIfNoGitLFS() 2404 def test_lfs_enabled(self): 2405 import shutil 2406 2407 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir 2408 self.d.setVar('SRC_URI', uri) 2409 2410 # With git-lfs installed, test that we can fetch and unpack 2411 fetcher, ud = self.fetch() 2412 shutil.rmtree(self.gitdir, ignore_errors=True) 2413 fetcher.unpack(self.d.getVar('WORKDIR')) 2414 2415 @skipIfNoGitLFS() 2416 def test_lfs_disabled(self): 2417 import shutil 2418 2419 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir 2420 self.d.setVar('SRC_URI', uri) 2421 2422 # Verify that the fetcher can survive even if the source 2423 # repository has Git LFS usage configured. 2424 fetcher, ud = self.fetch() 2425 fetcher.unpack(self.d.getVar('WORKDIR')) 2426 2427 def test_lfs_enabled_not_installed(self): 2428 import shutil 2429 2430 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir 2431 self.d.setVar('SRC_URI', uri) 2432 2433 # Careful: suppress initial attempt at downloading 2434 fetcher, ud = self.fetch(uri=None, download=False) 2435 2436 # Artificially assert that git-lfs is not installed, so 2437 # we can verify a failure to unpack in it's absence. 2438 old_find_git_lfs = ud.method._find_git_lfs 2439 try: 2440 # If git-lfs cannot be found, the unpack should throw an error 2441 with self.assertRaises(bb.fetch2.FetchError): 2442 fetcher.download() 2443 ud.method._find_git_lfs = lambda d: False 2444 shutil.rmtree(self.gitdir, ignore_errors=True) 2445 fetcher.unpack(self.d.getVar('WORKDIR')) 2446 finally: 2447 ud.method._find_git_lfs = old_find_git_lfs 2448 2449 def test_lfs_disabled_not_installed(self): 2450 import shutil 2451 2452 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir 2453 self.d.setVar('SRC_URI', uri) 2454 2455 # Careful: suppress initial attempt at downloading 2456 fetcher, ud = self.fetch(uri=None, download=False) 2457 2458 # Artificially assert that git-lfs is not installed, so 2459 # we can verify a failure to unpack in it's absence. 2460 old_find_git_lfs = ud.method._find_git_lfs 2461 try: 2462 # Even if git-lfs cannot be found, the unpack should be successful 2463 fetcher.download() 2464 ud.method._find_git_lfs = lambda d: False 2465 shutil.rmtree(self.gitdir, ignore_errors=True) 2466 fetcher.unpack(self.d.getVar('WORKDIR')) 2467 finally: 2468 ud.method._find_git_lfs = old_find_git_lfs 2469 2470class GitURLWithSpacesTest(FetcherTest): 2471 test_git_urls = { 2472 "git://tfs-example.org:22/tfs/example%20path/example.git;branch=master" : { 2473 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git;branch=master', 2474 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git', 2475 'path': '/tfs/example path/example.git' 2476 }, 2477 "git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master" : { 2478 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master', 2479 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git', 2480 'path': '/tfs/example path/example repo.git' 2481 } 2482 } 2483 2484 def test_urls(self): 2485 2486 # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo 2487 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 2488 2489 for test_git_url, ref in self.test_git_urls.items(): 2490 2491 fetcher = bb.fetch.Fetch([test_git_url], self.d) 2492 ud = fetcher.ud[fetcher.urls[0]] 2493 2494 self.assertEqual(ud.url, ref['url']) 2495 self.assertEqual(ud.path, ref['path']) 2496 self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2497 self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2498 self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock')) 2499 self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2500 self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz')) 2501 2502class CrateTest(FetcherTest): 2503 @skipIfNoNetwork() 2504 def test_crate_url(self): 2505 2506 uri = "crate://crates.io/glob/0.2.11" 2507 self.d.setVar('SRC_URI', uri) 2508 2509 uris = self.d.getVar('SRC_URI').split() 2510 d = self.d 2511 2512 fetcher = bb.fetch2.Fetch(uris, self.d) 2513 ud = fetcher.ud[fetcher.urls[0]] 2514 2515 self.assertIn("name", ud.parm) 2516 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2517 self.assertIn("downloadfilename", ud.parm) 2518 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2519 2520 fetcher.download() 2521 fetcher.unpack(self.tempdir) 2522 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2523 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) 2524 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2525 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2526 2527 @skipIfNoNetwork() 2528 def test_crate_url_matching_recipe(self): 2529 2530 self.d.setVar('BP', 'glob-0.2.11') 2531 2532 uri = "crate://crates.io/glob/0.2.11" 2533 self.d.setVar('SRC_URI', uri) 2534 2535 uris = self.d.getVar('SRC_URI').split() 2536 d = self.d 2537 2538 fetcher = bb.fetch2.Fetch(uris, self.d) 2539 ud = fetcher.ud[fetcher.urls[0]] 2540 2541 self.assertIn("name", ud.parm) 2542 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2543 self.assertIn("downloadfilename", ud.parm) 2544 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2545 2546 fetcher.download() 2547 fetcher.unpack(self.tempdir) 2548 self.assertEqual(sorted(os.listdir(self.tempdir)), ['download', 'glob-0.2.11', 'unpacked']) 2549 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) 2550 self.assertTrue(os.path.exists(self.tempdir + "/glob-0.2.11/src/lib.rs")) 2551 2552 @skipIfNoNetwork() 2553 def test_crate_url_params(self): 2554 2555 uri = "crate://crates.io/aho-corasick/0.7.20;name=aho-corasick-renamed" 2556 self.d.setVar('SRC_URI', uri) 2557 2558 uris = self.d.getVar('SRC_URI').split() 2559 d = self.d 2560 2561 fetcher = bb.fetch2.Fetch(uris, self.d) 2562 ud = fetcher.ud[fetcher.urls[0]] 2563 2564 self.assertIn("name", ud.parm) 2565 self.assertEqual(ud.parm["name"], "aho-corasick-renamed") 2566 self.assertIn("downloadfilename", ud.parm) 2567 self.assertEqual(ud.parm["downloadfilename"], "aho-corasick-0.7.20.crate") 2568 2569 fetcher.download() 2570 fetcher.unpack(self.tempdir) 2571 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2572 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['aho-corasick-0.7.20.crate', 'aho-corasick-0.7.20.crate.done']) 2573 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/.cargo-checksum.json")) 2574 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/src/lib.rs")) 2575 2576 @skipIfNoNetwork() 2577 def test_crate_url_multi(self): 2578 2579 uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35" 2580 self.d.setVar('SRC_URI', uri) 2581 2582 uris = self.d.getVar('SRC_URI').split() 2583 d = self.d 2584 2585 fetcher = bb.fetch2.Fetch(uris, self.d) 2586 ud = fetcher.ud[fetcher.urls[0]] 2587 2588 self.assertIn("name", ud.parm) 2589 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2590 self.assertIn("downloadfilename", ud.parm) 2591 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2592 2593 ud = fetcher.ud[fetcher.urls[1]] 2594 self.assertIn("name", ud.parm) 2595 self.assertEqual(ud.parm["name"], "time-0.1.35") 2596 self.assertIn("downloadfilename", ud.parm) 2597 self.assertEqual(ud.parm["downloadfilename"], "time-0.1.35.crate") 2598 2599 fetcher.download() 2600 fetcher.unpack(self.tempdir) 2601 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2602 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done', 'time-0.1.35.crate', 'time-0.1.35.crate.done']) 2603 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2604 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2605 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json")) 2606 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs")) 2607 2608 @skipIfNoNetwork() 2609 def test_crate_incorrect_cksum(self): 2610 uri = "crate://crates.io/aho-corasick/0.7.20" 2611 self.d.setVar('SRC_URI', uri) 2612 self.d.setVarFlag("SRC_URI", "aho-corasick-0.7.20.sha256sum", hashlib.sha256("Invalid".encode("utf-8")).hexdigest()) 2613 2614 uris = self.d.getVar('SRC_URI').split() 2615 2616 fetcher = bb.fetch2.Fetch(uris, self.d) 2617 with self.assertRaisesRegex(bb.fetch2.FetchError, "Fetcher failure for URL"): 2618 fetcher.download() 2619 2620class NPMTest(FetcherTest): 2621 def skipIfNoNpm(): 2622 import shutil 2623 if not shutil.which('npm'): 2624 return unittest.skip('npm not installed') 2625 return lambda f: f 2626 2627 @skipIfNoNpm() 2628 @skipIfNoNetwork() 2629 def test_npm(self): 2630 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2631 fetcher = bb.fetch.Fetch([url], self.d) 2632 ud = fetcher.ud[fetcher.urls[0]] 2633 fetcher.download() 2634 self.assertTrue(os.path.exists(ud.localpath)) 2635 self.assertTrue(os.path.exists(ud.localpath + '.done')) 2636 self.assertTrue(os.path.exists(ud.resolvefile)) 2637 fetcher.unpack(self.unpackdir) 2638 unpackdir = os.path.join(self.unpackdir, 'npm') 2639 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2640 2641 @skipIfNoNpm() 2642 @skipIfNoNetwork() 2643 def test_npm_bad_checksum(self): 2644 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2645 # Fetch once to get a tarball 2646 fetcher = bb.fetch.Fetch([url], self.d) 2647 ud = fetcher.ud[fetcher.urls[0]] 2648 fetcher.download() 2649 self.assertTrue(os.path.exists(ud.localpath)) 2650 # Modify the tarball 2651 bad = b'bad checksum' 2652 with open(ud.localpath, 'wb') as f: 2653 f.write(bad) 2654 # Verify that the tarball is fetched again 2655 fetcher.download() 2656 badsum = hashlib.sha512(bad).hexdigest() 2657 self.assertTrue(os.path.exists(ud.localpath + '_bad-checksum_' + badsum)) 2658 self.assertTrue(os.path.exists(ud.localpath)) 2659 2660 @skipIfNoNpm() 2661 @skipIfNoNetwork() 2662 def test_npm_premirrors(self): 2663 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2664 # Fetch once to get a tarball 2665 fetcher = bb.fetch.Fetch([url], self.d) 2666 ud = fetcher.ud[fetcher.urls[0]] 2667 fetcher.download() 2668 self.assertTrue(os.path.exists(ud.localpath)) 2669 2670 # Setup the mirror by renaming the download directory 2671 mirrordir = os.path.join(self.tempdir, 'mirror') 2672 bb.utils.rename(self.dldir, mirrordir) 2673 os.mkdir(self.dldir) 2674 2675 # Configure the premirror to be used 2676 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/npm2' % mirrordir) 2677 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2678 2679 # Fetch again 2680 self.assertFalse(os.path.exists(ud.localpath)) 2681 # The npm fetcher doesn't handle that the .resolved file disappears 2682 # while the fetcher object exists, which it does when we rename the 2683 # download directory to "mirror" above. Thus we need a new fetcher to go 2684 # with the now empty download directory. 2685 fetcher = bb.fetch.Fetch([url], self.d) 2686 ud = fetcher.ud[fetcher.urls[0]] 2687 fetcher.download() 2688 self.assertTrue(os.path.exists(ud.localpath)) 2689 2690 @skipIfNoNpm() 2691 @skipIfNoNetwork() 2692 def test_npm_premirrors_with_specified_filename(self): 2693 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2694 # Fetch once to get a tarball 2695 fetcher = bb.fetch.Fetch([url], self.d) 2696 ud = fetcher.ud[fetcher.urls[0]] 2697 fetcher.download() 2698 self.assertTrue(os.path.exists(ud.localpath)) 2699 # Setup the mirror 2700 mirrordir = os.path.join(self.tempdir, 'mirror') 2701 bb.utils.mkdirhier(mirrordir) 2702 mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath)) 2703 os.replace(ud.localpath, mirrorfilename) 2704 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s' % mirrorfilename) 2705 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2706 # Fetch again 2707 self.assertFalse(os.path.exists(ud.localpath)) 2708 fetcher.download() 2709 self.assertTrue(os.path.exists(ud.localpath)) 2710 2711 @skipIfNoNpm() 2712 @skipIfNoNetwork() 2713 def test_npm_mirrors(self): 2714 # Fetch once to get a tarball 2715 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2716 fetcher = bb.fetch.Fetch([url], self.d) 2717 ud = fetcher.ud[fetcher.urls[0]] 2718 fetcher.download() 2719 self.assertTrue(os.path.exists(ud.localpath)) 2720 # Setup the mirror 2721 mirrordir = os.path.join(self.tempdir, 'mirror') 2722 bb.utils.mkdirhier(mirrordir) 2723 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 2724 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 2725 # Update the resolved url to an invalid url 2726 with open(ud.resolvefile, 'r') as f: 2727 url = f.read() 2728 uri = URI(url) 2729 uri.path = '/invalid' 2730 with open(ud.resolvefile, 'w') as f: 2731 f.write(str(uri)) 2732 # Fetch again 2733 self.assertFalse(os.path.exists(ud.localpath)) 2734 fetcher.download() 2735 self.assertTrue(os.path.exists(ud.localpath)) 2736 2737 @skipIfNoNpm() 2738 @skipIfNoNetwork() 2739 def test_npm_destsuffix_downloadfilename(self): 2740 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz' 2741 fetcher = bb.fetch.Fetch([url], self.d) 2742 fetcher.download() 2743 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'foo-bar.tgz'))) 2744 fetcher.unpack(self.unpackdir) 2745 unpackdir = os.path.join(self.unpackdir, 'foo', 'bar') 2746 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2747 2748 def test_npm_no_network_no_tarball(self): 2749 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2750 self.d.setVar('BB_NO_NETWORK', '1') 2751 fetcher = bb.fetch.Fetch([url], self.d) 2752 with self.assertRaises(bb.fetch2.NetworkAccess): 2753 fetcher.download() 2754 2755 @skipIfNoNpm() 2756 @skipIfNoNetwork() 2757 def test_npm_no_network_with_tarball(self): 2758 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2759 # Fetch once to get a tarball 2760 fetcher = bb.fetch.Fetch([url], self.d) 2761 fetcher.download() 2762 # Disable network access 2763 self.d.setVar('BB_NO_NETWORK', '1') 2764 # Fetch again 2765 fetcher.download() 2766 fetcher.unpack(self.unpackdir) 2767 unpackdir = os.path.join(self.unpackdir, 'npm') 2768 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2769 2770 @skipIfNoNpm() 2771 @skipIfNoNetwork() 2772 def test_npm_registry_alternate(self): 2773 url = 'npm://skimdb.npmjs.com;package=@savoirfairelinux/node-server-example;version=1.0.0' 2774 fetcher = bb.fetch.Fetch([url], self.d) 2775 fetcher.download() 2776 fetcher.unpack(self.unpackdir) 2777 unpackdir = os.path.join(self.unpackdir, 'npm') 2778 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2779 2780 @skipIfNoNpm() 2781 @skipIfNoNetwork() 2782 def test_npm_version_latest(self): 2783 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest' 2784 fetcher = bb.fetch.Fetch([url], self.d) 2785 fetcher.download() 2786 fetcher.unpack(self.unpackdir) 2787 unpackdir = os.path.join(self.unpackdir, 'npm') 2788 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2789 2790 @skipIfNoNpm() 2791 @skipIfNoNetwork() 2792 def test_npm_registry_invalid(self): 2793 url = 'npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2794 fetcher = bb.fetch.Fetch([url], self.d) 2795 with self.assertRaises(bb.fetch2.FetchError): 2796 fetcher.download() 2797 2798 @skipIfNoNpm() 2799 @skipIfNoNetwork() 2800 def test_npm_package_invalid(self): 2801 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0' 2802 fetcher = bb.fetch.Fetch([url], self.d) 2803 with self.assertRaises(bb.fetch2.FetchError): 2804 fetcher.download() 2805 2806 @skipIfNoNpm() 2807 @skipIfNoNetwork() 2808 def test_npm_version_invalid(self): 2809 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid' 2810 with self.assertRaises(bb.fetch2.ParameterError): 2811 fetcher = bb.fetch.Fetch([url], self.d) 2812 2813 @skipIfNoNpm() 2814 @skipIfNoNetwork() 2815 def test_npm_registry_none(self): 2816 url = 'npm://;package=@savoirfairelinux/node-server-example;version=1.0.0' 2817 with self.assertRaises(bb.fetch2.MalformedUrl): 2818 fetcher = bb.fetch.Fetch([url], self.d) 2819 2820 @skipIfNoNpm() 2821 @skipIfNoNetwork() 2822 def test_npm_package_none(self): 2823 url = 'npm://registry.npmjs.org;version=1.0.0' 2824 with self.assertRaises(bb.fetch2.MissingParameterError): 2825 fetcher = bb.fetch.Fetch([url], self.d) 2826 2827 @skipIfNoNpm() 2828 @skipIfNoNetwork() 2829 def test_npm_version_none(self): 2830 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example' 2831 with self.assertRaises(bb.fetch2.MissingParameterError): 2832 fetcher = bb.fetch.Fetch([url], self.d) 2833 2834 def create_shrinkwrap_file(self, data): 2835 import json 2836 datadir = os.path.join(self.tempdir, 'data') 2837 swfile = os.path.join(datadir, 'npm-shrinkwrap.json') 2838 bb.utils.mkdirhier(datadir) 2839 with open(swfile, 'w') as f: 2840 json.dump(data, f) 2841 # Also configure the S directory 2842 self.sdir = os.path.join(self.unpackdir, 'S') 2843 self.d.setVar('S', self.sdir) 2844 return swfile 2845 2846 @skipIfNoNpm() 2847 @skipIfNoNetwork() 2848 def test_npmsw(self): 2849 swfile = self.create_shrinkwrap_file({ 2850 'dependencies': { 2851 'array-flatten': { 2852 'version': '1.1.1', 2853 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2854 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=', 2855 'dependencies': { 2856 'content-type': { 2857 'version': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 2858 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 2859 'dependencies': { 2860 'cookie': { 2861 'version': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', 2862 'from': 'git+https://github.com/jshttp/cookie.git' 2863 } 2864 } 2865 } 2866 } 2867 } 2868 } 2869 }) 2870 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2871 fetcher.download() 2872 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2873 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2874 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2875 fetcher.unpack(self.unpackdir) 2876 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'npm-shrinkwrap.json'))) 2877 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json'))) 2878 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json'))) 2879 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json'))) 2880 2881 @skipIfNoNpm() 2882 @skipIfNoNetwork() 2883 def test_npmsw_git(self): 2884 swfile = self.create_shrinkwrap_file({ 2885 'dependencies': { 2886 'cookie': { 2887 'version': 'github:jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', 2888 'from': 'github:jshttp/cookie.git' 2889 } 2890 } 2891 }) 2892 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2893 fetcher.download() 2894 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2895 2896 swfile = self.create_shrinkwrap_file({ 2897 'dependencies': { 2898 'cookie': { 2899 'version': 'jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', 2900 'from': 'jshttp/cookie.git' 2901 } 2902 } 2903 }) 2904 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2905 fetcher.download() 2906 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2907 2908 swfile = self.create_shrinkwrap_file({ 2909 'dependencies': { 2910 'nodejs': { 2911 'version': 'gitlab:gitlab-examples/nodejs.git#892a1f16725e56cc3a2cb0d677be42935c8fc262', 2912 'from': 'gitlab:gitlab-examples/nodejs' 2913 } 2914 } 2915 }) 2916 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2917 fetcher.download() 2918 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'gitlab.com.gitlab-examples.nodejs.git'))) 2919 2920 @skipIfNoNpm() 2921 @skipIfNoNetwork() 2922 def test_npmsw_dev(self): 2923 swfile = self.create_shrinkwrap_file({ 2924 'dependencies': { 2925 'array-flatten': { 2926 'version': '1.1.1', 2927 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2928 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2929 }, 2930 'content-type': { 2931 'version': '1.0.4', 2932 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 2933 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 2934 'dev': True 2935 } 2936 } 2937 }) 2938 # Fetch with dev disabled 2939 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2940 fetcher.download() 2941 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2942 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2943 # Fetch with dev enabled 2944 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d) 2945 fetcher.download() 2946 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2947 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2948 2949 @skipIfNoNpm() 2950 @skipIfNoNetwork() 2951 def test_npmsw_destsuffix(self): 2952 swfile = self.create_shrinkwrap_file({ 2953 'dependencies': { 2954 'array-flatten': { 2955 'version': '1.1.1', 2956 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2957 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2958 } 2959 } 2960 }) 2961 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d) 2962 fetcher.download() 2963 fetcher.unpack(self.unpackdir) 2964 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json'))) 2965 2966 def test_npmsw_no_network_no_tarball(self): 2967 swfile = self.create_shrinkwrap_file({ 2968 'dependencies': { 2969 'array-flatten': { 2970 'version': '1.1.1', 2971 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2972 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2973 } 2974 } 2975 }) 2976 self.d.setVar('BB_NO_NETWORK', '1') 2977 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2978 with self.assertRaises(bb.fetch2.NetworkAccess): 2979 fetcher.download() 2980 2981 @skipIfNoNpm() 2982 @skipIfNoNetwork() 2983 def test_npmsw_no_network_with_tarball(self): 2984 # Fetch once to get a tarball 2985 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2986 fetcher.download() 2987 # Disable network access 2988 self.d.setVar('BB_NO_NETWORK', '1') 2989 # Fetch again 2990 swfile = self.create_shrinkwrap_file({ 2991 'dependencies': { 2992 'array-flatten': { 2993 'version': '1.1.1', 2994 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2995 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2996 } 2997 } 2998 }) 2999 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3000 fetcher.download() 3001 fetcher.unpack(self.unpackdir) 3002 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json'))) 3003 3004 @skipIfNoNpm() 3005 @skipIfNoNetwork() 3006 def test_npmsw_npm_reusability(self): 3007 # Fetch once with npmsw 3008 swfile = self.create_shrinkwrap_file({ 3009 'dependencies': { 3010 'array-flatten': { 3011 'version': '1.1.1', 3012 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3013 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3014 } 3015 } 3016 }) 3017 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3018 fetcher.download() 3019 # Disable network access 3020 self.d.setVar('BB_NO_NETWORK', '1') 3021 # Fetch again with npm 3022 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 3023 fetcher.download() 3024 fetcher.unpack(self.unpackdir) 3025 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json'))) 3026 3027 @skipIfNoNpm() 3028 @skipIfNoNetwork() 3029 def test_npmsw_bad_checksum(self): 3030 # Try to fetch with bad checksum 3031 swfile = self.create_shrinkwrap_file({ 3032 'dependencies': { 3033 'array-flatten': { 3034 'version': '1.1.1', 3035 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3036 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg=' 3037 } 3038 } 3039 }) 3040 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3041 with self.assertRaises(bb.fetch2.FetchError): 3042 fetcher.download() 3043 # Fetch correctly to get a tarball 3044 swfile = self.create_shrinkwrap_file({ 3045 'dependencies': { 3046 'array-flatten': { 3047 'version': '1.1.1', 3048 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3049 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3050 } 3051 } 3052 }) 3053 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3054 fetcher.download() 3055 localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz') 3056 self.assertTrue(os.path.exists(localpath)) 3057 # Modify the tarball 3058 bad = b'bad checksum' 3059 with open(localpath, 'wb') as f: 3060 f.write(bad) 3061 # Verify that the tarball is fetched again 3062 fetcher.download() 3063 badsum = hashlib.sha1(bad).hexdigest() 3064 self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum)) 3065 self.assertTrue(os.path.exists(localpath)) 3066 3067 @skipIfNoNpm() 3068 @skipIfNoNetwork() 3069 def test_npmsw_premirrors(self): 3070 # Fetch once to get a tarball 3071 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 3072 ud = fetcher.ud[fetcher.urls[0]] 3073 fetcher.download() 3074 self.assertTrue(os.path.exists(ud.localpath)) 3075 # Setup the mirror 3076 mirrordir = os.path.join(self.tempdir, 'mirror') 3077 bb.utils.mkdirhier(mirrordir) 3078 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 3079 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 3080 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 3081 # Fetch again 3082 self.assertFalse(os.path.exists(ud.localpath)) 3083 swfile = self.create_shrinkwrap_file({ 3084 'dependencies': { 3085 'array-flatten': { 3086 'version': '1.1.1', 3087 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3088 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3089 } 3090 } 3091 }) 3092 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3093 fetcher.download() 3094 self.assertTrue(os.path.exists(ud.localpath)) 3095 3096 @skipIfNoNpm() 3097 @skipIfNoNetwork() 3098 def test_npmsw_mirrors(self): 3099 # Fetch once to get a tarball 3100 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 3101 ud = fetcher.ud[fetcher.urls[0]] 3102 fetcher.download() 3103 self.assertTrue(os.path.exists(ud.localpath)) 3104 # Setup the mirror 3105 mirrordir = os.path.join(self.tempdir, 'mirror') 3106 bb.utils.mkdirhier(mirrordir) 3107 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 3108 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 3109 # Fetch again with invalid url 3110 self.assertFalse(os.path.exists(ud.localpath)) 3111 swfile = self.create_shrinkwrap_file({ 3112 'dependencies': { 3113 'array-flatten': { 3114 'version': '1.1.1', 3115 'resolved': 'https://invalid', 3116 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3117 } 3118 } 3119 }) 3120 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3121 fetcher.download() 3122 self.assertTrue(os.path.exists(ud.localpath)) 3123 3124class GitSharedTest(FetcherTest): 3125 def setUp(self): 3126 super(GitSharedTest, self).setUp() 3127 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 3128 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 3129 self.d.setVar("__BBSRCREV_SEEN", "1") 3130 3131 @skipIfNoNetwork() 3132 def test_shared_unpack(self): 3133 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3134 3135 fetcher.download() 3136 fetcher.unpack(self.unpackdir) 3137 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 3138 self.assertTrue(os.path.exists(alt)) 3139 3140 @skipIfNoNetwork() 3141 def test_noshared_unpack(self): 3142 self.d.setVar('BB_GIT_NOSHARED', '1') 3143 self.unpackdir += '_noshared' 3144 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3145 3146 fetcher.download() 3147 fetcher.unpack(self.unpackdir) 3148 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 3149 self.assertFalse(os.path.exists(alt)) 3150 3151 3152class FetchPremirroronlyLocalTest(FetcherTest): 3153 3154 def setUp(self): 3155 super(FetchPremirroronlyLocalTest, self).setUp() 3156 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3157 os.mkdir(self.mirrordir) 3158 self.reponame = "bitbake" 3159 self.gitdir = os.path.join(self.tempdir, "git", self.reponame) 3160 self.recipe_url = "git://git.fake.repo/bitbake;branch=master;protocol=https" 3161 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3162 self.d.setVar("BB_NO_NETWORK", "1") 3163 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3164 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" 3165 self.mirrorfile = os.path.join(self.mirrordir, self.mirrorname) 3166 self.testfilename = "bitbake-fetch.test" 3167 3168 def make_git_repo(self): 3169 recipeurl = "git:/git.fake.repo/bitbake" 3170 os.makedirs(self.gitdir) 3171 self.git_init(cwd=self.gitdir) 3172 for i in range(0): 3173 self.git_new_commit() 3174 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3175 3176 def git_new_commit(self): 3177 import random 3178 os.unlink(os.path.join(self.mirrordir, self.mirrorname)) 3179 branch = self.git("branch --show-current", self.gitdir).split() 3180 with open(os.path.join(self.gitdir, self.testfilename), "w") as testfile: 3181 testfile.write("File {} from branch {}; Useless random data {}".format(self.testfilename, branch, random.random())) 3182 self.git("add {}".format(self.testfilename), self.gitdir) 3183 self.git("commit -a -m \"This random commit {} in branch {}. I'm useless.\"".format(random.random(), branch), self.gitdir) 3184 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3185 return self.git("rev-parse HEAD", self.gitdir).strip() 3186 3187 def git_new_branch(self, name): 3188 self.git_new_commit() 3189 head = self.git("rev-parse HEAD", self.gitdir).strip() 3190 self.git("checkout -b {}".format(name), self.gitdir) 3191 newrev = self.git_new_commit() 3192 self.git("checkout {}".format(head), self.gitdir) 3193 return newrev 3194 3195 def test_mirror_multiple_fetches(self): 3196 self.make_git_repo() 3197 self.d.setVar("SRCREV", self.git_new_commit()) 3198 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3199 fetcher.download() 3200 fetcher.unpack(self.unpackdir) 3201 ## New commit in premirror. it's not in the download_dir 3202 self.d.setVar("SRCREV", self.git_new_commit()) 3203 fetcher2 = bb.fetch.Fetch([self.recipe_url], self.d) 3204 fetcher2.download() 3205 fetcher2.unpack(self.unpackdir) 3206 ## New commit in premirror. it's not in the download_dir 3207 self.d.setVar("SRCREV", self.git_new_commit()) 3208 fetcher3 = bb.fetch.Fetch([self.recipe_url], self.d) 3209 fetcher3.download() 3210 fetcher3.unpack(self.unpackdir) 3211 3212 3213 def test_mirror_commit_nonexistent(self): 3214 self.make_git_repo() 3215 self.d.setVar("SRCREV", "0"*40) 3216 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3217 with self.assertRaises(bb.fetch2.NetworkAccess): 3218 fetcher.download() 3219 3220 def test_mirror_commit_exists(self): 3221 self.make_git_repo() 3222 self.d.setVar("SRCREV", self.git_new_commit()) 3223 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3224 fetcher.download() 3225 fetcher.unpack(self.unpackdir) 3226 3227 def test_mirror_tarball_nonexistent(self): 3228 self.d.setVar("SRCREV", "0"*40) 3229 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3230 with self.assertRaises(bb.fetch2.NetworkAccess): 3231 fetcher.download() 3232 3233 def test_mirror_tarball_multiple_branches(self): 3234 """ 3235 test if PREMIRRORS can handle multiple name/branches correctly 3236 both branches have required revisions 3237 """ 3238 self.make_git_repo() 3239 branch1rev = self.git_new_branch("testbranch1") 3240 branch2rev = self.git_new_branch("testbranch2") 3241 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2" 3242 self.d.setVar("SRCREV_branch1", branch1rev) 3243 self.d.setVar("SRCREV_branch2", branch2rev) 3244 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3245 self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist") 3246 fetcher.download() 3247 fetcher.unpack(os.path.join(self.tempdir, "unpacked")) 3248 unpacked = os.path.join(self.tempdir, "unpacked", "git", self.testfilename) 3249 self.assertTrue(os.path.exists(unpacked), "Repo has not been unpackaged properly!") 3250 with open(unpacked, 'r') as f: 3251 content = f.read() 3252 ## We expect to see testbranch1 in the file, not master, not testbranch2 3253 self.assertTrue(content.find("testbranch1") != -1, "Wrong branch has been checked out!") 3254 3255 def test_mirror_tarball_multiple_branches_nobranch(self): 3256 """ 3257 test if PREMIRRORS can handle multiple name/branches correctly 3258 Unbalanced name/branches raises ParameterError 3259 """ 3260 self.make_git_repo() 3261 branch1rev = self.git_new_branch("testbranch1") 3262 branch2rev = self.git_new_branch("testbranch2") 3263 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1;protocol=https;name=branch1,branch2" 3264 self.d.setVar("SRCREV_branch1", branch1rev) 3265 self.d.setVar("SRCREV_branch2", branch2rev) 3266 with self.assertRaises(bb.fetch2.ParameterError): 3267 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3268 3269 def test_mirror_tarball_multiple_branches_norev(self): 3270 """ 3271 test if PREMIRRORS can handle multiple name/branches correctly 3272 one of the branches specifies non existing SRCREV 3273 """ 3274 self.make_git_repo() 3275 branch1rev = self.git_new_branch("testbranch1") 3276 branch2rev = self.git_new_branch("testbranch2") 3277 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2" 3278 self.d.setVar("SRCREV_branch1", branch1rev) 3279 self.d.setVar("SRCREV_branch2", "0"*40) 3280 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3281 self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist") 3282 with self.assertRaises(bb.fetch2.NetworkAccess): 3283 fetcher.download() 3284 3285 3286class FetchPremirroronlyNetworkTest(FetcherTest): 3287 3288 def setUp(self): 3289 super(FetchPremirroronlyNetworkTest, self).setUp() 3290 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3291 os.mkdir(self.mirrordir) 3292 self.reponame = "fstests" 3293 self.clonedir = os.path.join(self.tempdir, "git") 3294 self.gitdir = os.path.join(self.tempdir, "git", "{}.git".format(self.reponame)) 3295 self.recipe_url = "git://git.yoctoproject.org/fstests;protocol=https" 3296 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3297 self.d.setVar("BB_NO_NETWORK", "0") 3298 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3299 3300 def make_git_repo(self): 3301 import shutil 3302 self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz" 3303 os.makedirs(self.clonedir) 3304 self.git("clone --bare --shallow-since=\"01.01.2013\" {}".format(self.recipe_url), self.clonedir) 3305 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3306 shutil.rmtree(self.clonedir) 3307 3308 @skipIfNoNetwork() 3309 def test_mirror_tarball_updated(self): 3310 self.make_git_repo() 3311 ## Upstream commit is in the mirror 3312 self.d.setVar("SRCREV", "49d65d53c2bf558ae6e9185af0f3af7b79d255ec") 3313 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3314 fetcher.download() 3315 3316 @skipIfNoNetwork() 3317 def test_mirror_tarball_outdated(self): 3318 self.make_git_repo() 3319 ## Upstream commit not in the mirror 3320 self.d.setVar("SRCREV", "15413486df1f5a5b5af699b6f3ba5f0984e52a9f") 3321 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3322 with self.assertRaises(bb.fetch2.NetworkAccess): 3323 fetcher.download() 3324 3325class FetchPremirroronlyMercurialTest(FetcherTest): 3326 """ Test for premirrors with mercurial repos 3327 the test covers also basic hg:// clone (see fetch_and_create_tarball 3328 """ 3329 def skipIfNoHg(): 3330 import shutil 3331 if not shutil.which('hg'): 3332 return unittest.skip('Mercurial not installed') 3333 return lambda f: f 3334 3335 def setUp(self): 3336 super(FetchPremirroronlyMercurialTest, self).setUp() 3337 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3338 os.mkdir(self.mirrordir) 3339 self.reponame = "libgnt" 3340 self.clonedir = os.path.join(self.tempdir, "hg") 3341 self.recipe_url = "hg://keep.imfreedom.org/libgnt;module=libgnt" 3342 self.d.setVar("SRCREV", "53e8b422faaf") 3343 self.mirrorname = "hg_libgnt_keep.imfreedom.org_.libgnt.tar.gz" 3344 3345 def fetch_and_create_tarball(self): 3346 """ 3347 Ask bitbake to download repo and prepare mirror tarball for us 3348 """ 3349 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 3350 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3351 fetcher.download() 3352 mirrorfile = os.path.join(self.d.getVar("DL_DIR"), self.mirrorname) 3353 self.assertTrue(os.path.exists(mirrorfile), "Mirror tarball {} has not been created".format(mirrorfile)) 3354 ## moving tarball to mirror directory 3355 os.rename(mirrorfile, os.path.join(self.mirrordir, self.mirrorname)) 3356 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "0") 3357 3358 3359 @skipIfNoNetwork() 3360 @skipIfNoHg() 3361 def test_premirror_mercurial(self): 3362 self.fetch_and_create_tarball() 3363 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3364 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3365 self.d.setVar("BB_NO_NETWORK", "1") 3366 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3367 fetcher.download() 3368 3369class FetchPremirroronlyBrokenTarball(FetcherTest): 3370 3371 def setUp(self): 3372 super(FetchPremirroronlyBrokenTarball, self).setUp() 3373 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3374 os.mkdir(self.mirrordir) 3375 self.reponame = "bitbake" 3376 self.gitdir = os.path.join(self.tempdir, "git", self.reponame) 3377 self.recipe_url = "git://git.fake.repo/bitbake;protocol=https" 3378 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3379 self.d.setVar("BB_NO_NETWORK", "1") 3380 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3381 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" 3382 with open(os.path.join(self.mirrordir, self.mirrorname), 'w') as targz: 3383 targz.write("This is not tar.gz file!") 3384 3385 def test_mirror_broken_download(self): 3386 import sys 3387 self.d.setVar("SRCREV", "0"*40) 3388 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3389 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs() as logs: 3390 fetcher.download() 3391 output = "".join(logs.output) 3392 self.assertFalse(" not a git repository (or any parent up to mount point /)" in output) 3393 3394class GoModTest(FetcherTest): 3395 3396 @skipIfNoNetwork() 3397 def test_gomod_url(self): 3398 urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;' 3399 'sha256sum=9bb69aea32f1d59711701f9562d66432c9c0374205e5009d1d1a62f03fb4fdad'] 3400 3401 fetcher = bb.fetch2.Fetch(urls, self.d) 3402 ud = fetcher.ud[urls[0]] 3403 self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.zip') 3404 self.assertNotIn('name', ud.parm) 3405 3406 fetcher.download() 3407 fetcher.unpack(self.unpackdir) 3408 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3409 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip'))) 3410 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod'))) 3411 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')), 3412 '7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873') 3413 3414 @skipIfNoNetwork() 3415 def test_gomod_url_go_mod_only(self): 3416 urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;mod=1;' 3417 'sha256sum=7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873'] 3418 3419 fetcher = bb.fetch2.Fetch(urls, self.d) 3420 ud = fetcher.ud[urls[0]] 3421 self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.mod') 3422 self.assertNotIn('name', ud.parm) 3423 3424 fetcher.download() 3425 fetcher.unpack(self.unpackdir) 3426 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3427 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod'))) 3428 3429 @skipIfNoNetwork() 3430 def test_gomod_url_sha256sum_varflag(self): 3431 urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0'] 3432 self.d.setVarFlag('SRC_URI', 'gopkg.in/ini.v1@v1.67.0.sha256sum', 'bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6') 3433 3434 fetcher = bb.fetch2.Fetch(urls, self.d) 3435 ud = fetcher.ud[urls[0]] 3436 self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip') 3437 self.assertEqual(ud.parm['name'], 'gopkg.in/ini.v1@v1.67.0') 3438 3439 fetcher.download() 3440 fetcher.unpack(self.unpackdir) 3441 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3442 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3443 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3444 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3445 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3446 3447 @skipIfNoNetwork() 3448 def test_gomod_url_no_go_mod_in_module(self): 3449 urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0;' 3450 'sha256sum=bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6'] 3451 3452 fetcher = bb.fetch2.Fetch(urls, self.d) 3453 ud = fetcher.ud[urls[0]] 3454 self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip') 3455 self.assertNotIn('name', ud.parm) 3456 3457 fetcher.download() 3458 fetcher.unpack(self.unpackdir) 3459 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3460 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3461 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3462 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3463 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3464 3465 @skipIfNoNetwork() 3466 def test_gomod_url_host_only(self): 3467 urls = ['gomod://go.opencensus.io;version=v0.24.0;' 3468 'sha256sum=203a767d7f8e7c1ebe5588220ad168d1e15b14ae70a636de7ca9a4a88a7e0d0c'] 3469 3470 fetcher = bb.fetch2.Fetch(urls, self.d) 3471 ud = fetcher.ud[urls[0]] 3472 self.assertEqual(ud.url, 'https://proxy.golang.org/go.opencensus.io/%40v/v0.24.0.zip') 3473 self.assertNotIn('name', ud.parm) 3474 3475 fetcher.download() 3476 fetcher.unpack(self.unpackdir) 3477 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3478 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.zip'))) 3479 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod'))) 3480 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')), 3481 '0dc9ccc660ad21cebaffd548f2cc6efa27891c68b4fbc1f8a3893b00f1acec96') 3482 3483class GoModGitTest(FetcherTest): 3484 3485 @skipIfNoNetwork() 3486 def test_gomodgit_url_repo(self): 3487 urls = ['gomodgit://golang.org/x/net;version=v0.9.0;' 3488 'repo=go.googlesource.com/net;' 3489 'srcrev=694cff8668bac64e0864b552bffc280cd27f21b1'] 3490 3491 fetcher = bb.fetch2.Fetch(urls, self.d) 3492 ud = fetcher.ud[urls[0]] 3493 self.assertEqual(ud.host, 'go.googlesource.com') 3494 self.assertEqual(ud.path, '/net') 3495 self.assertEqual(ud.names, ['golang.org/x/net@v0.9.0']) 3496 self.assertEqual(self.d.getVar('SRCREV_golang.org/x/net@v0.9.0'), '694cff8668bac64e0864b552bffc280cd27f21b1') 3497 3498 fetcher.download() 3499 self.assertTrue(os.path.exists(ud.localpath)) 3500 3501 fetcher.unpack(self.unpackdir) 3502 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3503 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'ed42bd05533fd84ae290a5d33ebd3695a0a2b06131beebd5450825bee8603aca'))) 3504 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3505 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.zip'))) 3506 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod'))) 3507 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod')), 3508 'c5d6851ede50ec1c001afb763040194b68961bf06997e2605e8bf06dcd2aeb2e') 3509 3510 @skipIfNoNetwork() 3511 def test_gomodgit_url_subdir(self): 3512 urls = ['gomodgit://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;' 3513 'repo=github.com/Azure/azure-sdk-for-go;subdir=sdk/storage/azblob;' 3514 'srcrev=ec928e0ed34db682b3f783d3739d1c538142e0c3'] 3515 3516 fetcher = bb.fetch2.Fetch(urls, self.d) 3517 ud = fetcher.ud[urls[0]] 3518 self.assertEqual(ud.host, 'github.com') 3519 self.assertEqual(ud.path, '/Azure/azure-sdk-for-go') 3520 self.assertEqual(ud.parm['subpath'], 'sdk/storage/azblob') 3521 self.assertEqual(ud.names, ['github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0']) 3522 self.assertEqual(self.d.getVar('SRCREV_github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'), 'ec928e0ed34db682b3f783d3739d1c538142e0c3') 3523 3524 fetcher.download() 3525 self.assertTrue(os.path.exists(ud.localpath)) 3526 3527 fetcher.unpack(self.unpackdir) 3528 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3529 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'd31d6145676ed3066ce573a8198f326dea5be45a43b3d8f41ce7787fd71d66b3'))) 3530 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3531 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip'))) 3532 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod'))) 3533 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')), 3534 '7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873') 3535 3536 @skipIfNoNetwork() 3537 def test_gomodgit_url_srcrev_var(self): 3538 urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0'] 3539 self.d.setVar('SRCREV_gopkg.in/ini.v1@v1.67.0', 'b2f570e5b5b844226bbefe6fb521d891f529a951') 3540 3541 fetcher = bb.fetch2.Fetch(urls, self.d) 3542 ud = fetcher.ud[urls[0]] 3543 self.assertEqual(ud.host, 'gopkg.in') 3544 self.assertEqual(ud.path, '/ini.v1') 3545 self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0']) 3546 self.assertEqual(ud.parm['srcrev'], 'b2f570e5b5b844226bbefe6fb521d891f529a951') 3547 3548 fetcher.download() 3549 fetcher.unpack(self.unpackdir) 3550 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3551 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3'))) 3552 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3553 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3554 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3555 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3556 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3557 3558 @skipIfNoNetwork() 3559 def test_gomodgit_url_no_go_mod_in_module(self): 3560 urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0;' 3561 'srcrev=b2f570e5b5b844226bbefe6fb521d891f529a951'] 3562 3563 fetcher = bb.fetch2.Fetch(urls, self.d) 3564 ud = fetcher.ud[urls[0]] 3565 self.assertEqual(ud.host, 'gopkg.in') 3566 self.assertEqual(ud.path, '/ini.v1') 3567 self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0']) 3568 self.assertEqual(self.d.getVar('SRCREV_gopkg.in/ini.v1@v1.67.0'), 'b2f570e5b5b844226bbefe6fb521d891f529a951') 3569 3570 fetcher.download() 3571 fetcher.unpack(self.unpackdir) 3572 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3573 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3'))) 3574 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3575 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3576 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3577 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3578 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3579 3580 @skipIfNoNetwork() 3581 def test_gomodgit_url_host_only(self): 3582 urls = ['gomodgit://go.opencensus.io;version=v0.24.0;' 3583 'repo=github.com/census-instrumentation/opencensus-go;' 3584 'srcrev=b1a01ee95db0e690d91d7193d037447816fae4c5'] 3585 3586 fetcher = bb.fetch2.Fetch(urls, self.d) 3587 ud = fetcher.ud[urls[0]] 3588 self.assertEqual(ud.host, 'github.com') 3589 self.assertEqual(ud.path, '/census-instrumentation/opencensus-go') 3590 self.assertEqual(ud.names, ['go.opencensus.io@v0.24.0']) 3591 self.assertEqual(self.d.getVar('SRCREV_go.opencensus.io@v0.24.0'), 'b1a01ee95db0e690d91d7193d037447816fae4c5') 3592 3593 fetcher.download() 3594 fetcher.unpack(self.unpackdir) 3595 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3596 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'aae3ac7b2122ed3345654e6327855e9682f4a5350d63e93dbcfc51c4419df0e1'))) 3597 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3598 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.zip'))) 3599 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod'))) 3600 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')), 3601 '0dc9ccc660ad21cebaffd548f2cc6efa27891c68b4fbc1f8a3893b00f1acec96') 3602