1# 2# BitBake Toaster Implementation 3# 4# Copyright (C) 2014 Intel Corporation 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9import os 10import re 11import shutil 12import time 13from bldcontrol.models import BuildEnvironment, BuildRequest, Build 14from orm.models import CustomImageRecipe, Layer, Layer_Version, Project, ToasterSetting 15from orm.models import signal_runbuilds 16import subprocess 17 18from toastermain import settings 19 20from bldcontrol.bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException 21 22import logging 23logger = logging.getLogger("toaster") 24 25install_dir = os.environ.get('TOASTER_DIR') 26 27from pprint import pformat 28 29class LocalhostBEController(BuildEnvironmentController): 30 """ Implementation of the BuildEnvironmentController for the localhost; 31 this controller manages the default build directory, 32 the server setup and system start and stop for the localhost-type build environment 33 34 """ 35 36 def __init__(self, be): 37 super(LocalhostBEController, self).__init__(be) 38 self.pokydirname = None 39 self.islayerset = False 40 41 def _shellcmd(self, command, cwd=None, nowait=False,env=None): 42 if cwd is None: 43 cwd = self.be.sourcedir 44 if env is None: 45 env=os.environ.copy() 46 47 logger.debug("lbc_shellcmd: (%s) %s" % (cwd, command)) 48 p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) 49 if nowait: 50 return 51 (out,err) = p.communicate() 52 p.wait() 53 if p.returncode: 54 if len(err) == 0: 55 err = "command: %s \n%s" % (command, out) 56 else: 57 err = "command: %s \n%s" % (command, err) 58 logger.warning("localhostbecontroller: shellcmd error %s" % err) 59 raise ShellCmdException(err) 60 else: 61 logger.debug("localhostbecontroller: shellcmd success") 62 return out.decode('utf-8') 63 64 def getGitCloneDirectory(self, url, branch): 65 """Construct unique clone directory name out of url and branch.""" 66 if branch != "HEAD": 67 return "_toaster_clones/_%s_%s" % (re.sub('[:/@+%]', '_', url), branch) 68 69 # word of attention; this is a localhost-specific issue; only on the localhost we expect to have "HEAD" releases 70 # which _ALWAYS_ means the current poky checkout 71 from os.path import dirname as DN 72 local_checkout_path = DN(DN(DN(DN(DN(os.path.abspath(__file__)))))) 73 #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path) 74 return local_checkout_path 75 76 def setCloneStatus(self,bitbake,status,total,current,repo_name): 77 bitbake.req.build.repos_cloned=current 78 bitbake.req.build.repos_to_clone=total 79 bitbake.req.build.progress_item=repo_name 80 bitbake.req.build.save() 81 82 def setLayers(self, bitbake, layers, targets): 83 """ a word of attention: by convention, the first layer for any build will be poky! """ 84 85 assert self.be.sourcedir is not None 86 87 layerlist = [] 88 nongitlayerlist = [] 89 layer_index = 0 90 git_env = os.environ.copy() 91 # (note: add custom environment settings here) 92 93 # set layers in the layersource 94 95 # 1. get a list of repos with branches, and map dirpaths for each layer 96 gitrepos = {} 97 98 # if we're using a remotely fetched version of bitbake add its git 99 # details to the list of repos to clone 100 if bitbake.giturl and bitbake.commit: 101 gitrepos[(bitbake.giturl, bitbake.commit)] = [] 102 gitrepos[(bitbake.giturl, bitbake.commit)].append( 103 ("bitbake", bitbake.dirpath, 0)) 104 105 for layer in layers: 106 # We don't need to git clone the layer for the CustomImageRecipe 107 # as it's generated by us layer on if needed 108 if CustomImageRecipe.LAYER_NAME in layer.name: 109 continue 110 111 # If we have local layers then we don't need clone them 112 # For local layers giturl will be empty 113 if not layer.giturl: 114 nongitlayerlist.append( "%03d:%s" % (layer_index,layer.local_source_dir) ) 115 continue 116 117 if not (layer.giturl, layer.commit) in gitrepos: 118 gitrepos[(layer.giturl, layer.commit)] = [] 119 gitrepos[(layer.giturl, layer.commit)].append( (layer.name,layer.dirpath,layer_index) ) 120 layer_index += 1 121 122 123 logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) 124 125 126 # 2. Note for future use if the current source directory is a 127 # checked-out git repos that could match a layer's vcs_url and therefore 128 # be used to speed up cloning (rather than fetching it again). 129 130 cached_layers = {} 131 132 try: 133 for remotes in self._shellcmd("git remote -v", self.be.sourcedir,env=git_env).split("\n"): 134 try: 135 remote = remotes.split("\t")[1].split(" ")[0] 136 if remote not in cached_layers: 137 cached_layers[remote] = self.be.sourcedir 138 except IndexError: 139 pass 140 except ShellCmdException: 141 # ignore any errors in collecting git remotes this is an optional 142 # step 143 pass 144 145 logger.info("Using pre-checked out source for layer %s", cached_layers) 146 147 # 3. checkout the repositories 148 clone_count=0 149 clone_total=len(gitrepos.keys()) 150 self.setCloneStatus(bitbake,'Started',clone_total,clone_count,'') 151 for giturl, commit in gitrepos.keys(): 152 self.setCloneStatus(bitbake,'progress',clone_total,clone_count,gitrepos[(giturl, commit)][0][0]) 153 clone_count += 1 154 155 localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) 156 logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) 157 158 # see if our directory is a git repository 159 if os.path.exists(localdirname): 160 try: 161 localremotes = self._shellcmd("git remote -v", 162 localdirname,env=git_env) 163 # NOTE: this nice-to-have check breaks when using git remaping to get past firewall 164 # Re-enable later with .gitconfig remapping checks 165 #if not giturl in localremotes and commit != 'HEAD': 166 # raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl)) 167 pass 168 except ShellCmdException: 169 # our localdirname might not be a git repository 170 #- that's fine 171 pass 172 else: 173 if giturl in cached_layers: 174 logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) 175 self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname),env=git_env) 176 self._shellcmd("git remote remove origin", localdirname,env=git_env) 177 self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname,env=git_env) 178 else: 179 logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname)) 180 self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname),env=git_env) 181 182 # branch magic name "HEAD" will inhibit checkout 183 if commit != "HEAD": 184 logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) 185 ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit 186 self._shellcmd('git fetch && git reset --hard "%s"' % ref, localdirname,env=git_env) 187 188 # take the localdirname as poky dir if we can find the oe-init-build-env 189 if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): 190 logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname) 191 self.pokydirname = localdirname 192 193 # make sure we have a working bitbake 194 if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): 195 logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname) 196 self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')),env=git_env) 197 198 # verify our repositories 199 for name, dirpath, index in gitrepos[(giturl, commit)]: 200 localdirpath = os.path.join(localdirname, dirpath) 201 logger.debug("localhostbecontroller: localdirpath expects '%s'" % localdirpath) 202 if not os.path.exists(localdirpath): 203 raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Exiting." % (localdirpath, giturl, commit)) 204 205 if name != "bitbake": 206 layerlist.append("%03d:%s" % (index,localdirpath.rstrip("/"))) 207 208 self.setCloneStatus(bitbake,'complete',clone_total,clone_count,'') 209 logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) 210 211 # Resolve self.pokydirname if not resolved yet, consider the scenario 212 # where all layers are local, that's the else clause 213 if self.pokydirname is None: 214 if os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")): 215 logger.debug("localhostbecontroller: selected poky dir name %s" % self.be.sourcedir) 216 self.pokydirname = self.be.sourcedir 217 else: 218 # Alternatively, scan local layers for relative "oe-init-build-env" location 219 for layer in layers: 220 if os.path.exists(os.path.join(layer.layer_version.layer.local_source_dir,"..","oe-init-build-env")): 221 logger.debug("localhostbecontroller, setting pokydirname to %s" % (layer.layer_version.layer.local_source_dir)) 222 self.pokydirname = os.path.join(layer.layer_version.layer.local_source_dir,"..") 223 break 224 else: 225 logger.error("pokydirname is not set, you will run into trouble!") 226 227 # 5. create custom layer and add custom recipes to it 228 for target in targets: 229 try: 230 customrecipe = CustomImageRecipe.objects.get( 231 name=target.target, 232 project=bitbake.req.project) 233 234 custom_layer_path = self.setup_custom_image_recipe( 235 customrecipe, layers) 236 237 if os.path.isdir(custom_layer_path): 238 layerlist.append("%03d:%s" % (layer_index,custom_layer_path)) 239 240 except CustomImageRecipe.DoesNotExist: 241 continue # not a custom recipe, skip 242 243 layerlist.extend(nongitlayerlist) 244 logger.debug("\n\nset layers gives this list %s" % pformat(layerlist)) 245 self.islayerset = True 246 247 # restore the order of layer list for bblayers.conf 248 layerlist.sort() 249 sorted_layerlist = [l[4:] for l in layerlist] 250 return sorted_layerlist 251 252 def setup_custom_image_recipe(self, customrecipe, layers): 253 """ Set up toaster-custom-images layer and recipe files """ 254 layerpath = os.path.join(self.be.builddir, 255 CustomImageRecipe.LAYER_NAME) 256 257 # create directory structure 258 for name in ("conf", "recipes"): 259 path = os.path.join(layerpath, name) 260 if not os.path.isdir(path): 261 os.makedirs(path) 262 263 # create layer.conf 264 config = os.path.join(layerpath, "conf", "layer.conf") 265 if not os.path.isfile(config): 266 with open(config, "w") as conf: 267 conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n') 268 269 # Update the Layer_Version dirpath that has our base_recipe in 270 # to be able to read the base recipe to then generate the 271 # custom recipe. 272 br_layer_base_recipe = layers.get( 273 layer_version=customrecipe.base_recipe.layer_version) 274 275 # If the layer is one that we've cloned we know where it lives 276 if br_layer_base_recipe.giturl and br_layer_base_recipe.commit: 277 layer_path = self.getGitCloneDirectory( 278 br_layer_base_recipe.giturl, 279 br_layer_base_recipe.commit) 280 # Otherwise it's a local layer 281 elif br_layer_base_recipe.local_source_dir: 282 layer_path = br_layer_base_recipe.local_source_dir 283 else: 284 logger.error("Unable to workout the dir path for the custom" 285 " image recipe") 286 287 br_layer_base_dirpath = os.path.join( 288 self.be.sourcedir, 289 layer_path, 290 customrecipe.base_recipe.layer_version.dirpath) 291 292 customrecipe.base_recipe.layer_version.dirpath = br_layer_base_dirpath 293 294 customrecipe.base_recipe.layer_version.save() 295 296 # create recipe 297 recipe_path = os.path.join(layerpath, "recipes", "%s.bb" % 298 customrecipe.name) 299 with open(recipe_path, "w") as recipef: 300 recipef.write(customrecipe.generate_recipe_file_contents()) 301 302 # Update the layer and recipe objects 303 customrecipe.layer_version.dirpath = layerpath 304 customrecipe.layer_version.layer.local_source_dir = layerpath 305 customrecipe.layer_version.layer.save() 306 customrecipe.layer_version.save() 307 308 customrecipe.file_path = recipe_path 309 customrecipe.save() 310 311 return layerpath 312 313 314 def readServerLogFile(self): 315 return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read() 316 317 318 def triggerBuild(self, bitbake, layers, variables, targets, brbe): 319 layers = self.setLayers(bitbake, layers, targets) 320 is_merged_attr = bitbake.req.project.merged_attr 321 322 git_env = os.environ.copy() 323 # (note: add custom environment settings here) 324 try: 325 # insure that the project init/build uses the selected bitbake, and not Toaster's 326 del git_env['TEMPLATECONF'] 327 del git_env['BBBASEDIR'] 328 del git_env['BUILDDIR'] 329 except KeyError: 330 pass 331 332 # init build environment from the clone 333 if bitbake.req.project.builddir: 334 builddir = bitbake.req.project.builddir 335 else: 336 builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id) 337 oe_init = os.path.join(self.pokydirname, 'oe-init-build-env') 338 # init build environment 339 try: 340 custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value 341 custom_script = custom_script.replace("%BUILDDIR%" ,builddir) 342 self._shellcmd("bash -c 'source %s'" % (custom_script),env=git_env) 343 except ToasterSetting.DoesNotExist: 344 self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir), 345 self.be.sourcedir,env=git_env) 346 347 # update bblayers.conf 348 if not is_merged_attr: 349 bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf") 350 with open(bblconfpath, 'w') as bblayers: 351 bblayers.write('# line added by toaster build control\n' 352 'BBLAYERS = "%s"' % ' '.join(layers)) 353 354 # write configuration file 355 confpath = os.path.join(builddir, 'conf/toaster.conf') 356 with open(confpath, 'w') as conf: 357 for var in variables: 358 conf.write('%s="%s"\n' % (var.name, var.value)) 359 conf.write('INHERIT+="toaster buildhistory"') 360 else: 361 # Append the Toaster-specific values directly to the bblayers.conf 362 bblconfpath = os.path.join(builddir, "conf/bblayers.conf") 363 bblconfpath_save = os.path.join(builddir, "conf/bblayers.conf.save") 364 shutil.copyfile(bblconfpath, bblconfpath_save) 365 with open(bblconfpath) as bblayers: 366 content = bblayers.readlines() 367 do_write = True 368 was_toaster = False 369 with open(bblconfpath,'w') as bblayers: 370 for line in content: 371 #line = line.strip('\n') 372 if 'TOASTER_CONFIG_PROLOG' in line: 373 do_write = False 374 was_toaster = True 375 elif 'TOASTER_CONFIG_EPILOG' in line: 376 do_write = True 377 elif do_write: 378 bblayers.write(line) 379 if not was_toaster: 380 bblayers.write('\n') 381 bblayers.write('#=== TOASTER_CONFIG_PROLOG ===\n') 382 bblayers.write('BBLAYERS = "\\\n') 383 for layer in layers: 384 bblayers.write(' %s \\\n' % layer) 385 bblayers.write(' "\n') 386 bblayers.write('#=== TOASTER_CONFIG_EPILOG ===\n') 387 # Append the Toaster-specific values directly to the local.conf 388 bbconfpath = os.path.join(builddir, "conf/local.conf") 389 bbconfpath_save = os.path.join(builddir, "conf/local.conf.save") 390 shutil.copyfile(bbconfpath, bbconfpath_save) 391 with open(bbconfpath) as f: 392 content = f.readlines() 393 do_write = True 394 was_toaster = False 395 with open(bbconfpath,'w') as conf: 396 for line in content: 397 #line = line.strip('\n') 398 if 'TOASTER_CONFIG_PROLOG' in line: 399 do_write = False 400 was_toaster = True 401 elif 'TOASTER_CONFIG_EPILOG' in line: 402 do_write = True 403 elif do_write: 404 conf.write(line) 405 if not was_toaster: 406 conf.write('\n') 407 conf.write('#=== TOASTER_CONFIG_PROLOG ===\n') 408 for var in variables: 409 if (not var.name.startswith("INTERNAL_")) and (not var.name == "BBLAYERS"): 410 conf.write('%s="%s"\n' % (var.name, var.value)) 411 conf.write('#=== TOASTER_CONFIG_EPILOG ===\n') 412 413 # If 'target' is just the project preparation target, then we are done 414 for target in targets: 415 if "_PROJECT_PREPARE_" == target.target: 416 logger.debug('localhostbecontroller: Project has been prepared. Done.') 417 # Update the Build Request and release the build environment 418 bitbake.req.state = BuildRequest.REQ_COMPLETED 419 bitbake.req.save() 420 self.be.lock = BuildEnvironment.LOCK_FREE 421 self.be.save() 422 # Close the project build and progress bar 423 bitbake.req.build.outcome = Build.SUCCEEDED 424 bitbake.req.build.save() 425 # Update the project status 426 bitbake.req.project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_CLONING_SUCCESS) 427 signal_runbuilds() 428 return 429 430 # clean the Toaster to build environment 431 env_clean = 'unset BBPATH;' # clean BBPATH for <= YP-2.4.0 432 433 # run bitbake server from the clone if available 434 # otherwise pick it from the PATH 435 bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake') 436 if not os.path.exists(bitbake): 437 logger.info("Bitbake not available under %s, will try to use it from PATH" % 438 self.pokydirname) 439 for path in os.environ["PATH"].split(os.pathsep): 440 if os.path.exists(os.path.join(path, 'bitbake')): 441 bitbake = os.path.join(path, 'bitbake') 442 break 443 else: 444 logger.error("Looks like Bitbake is not available, please fix your environment") 445 446 toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf") 447 if not is_merged_attr: 448 self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s ' 449 '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, 450 builddir, bitbake, confpath, toasterlayers), self.be.sourcedir) 451 else: 452 self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s ' 453 '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, 454 builddir, bitbake), self.be.sourcedir) 455 456 # read port number from bitbake.lock 457 self.be.bbport = -1 458 bblock = os.path.join(builddir, 'bitbake.lock') 459 # allow 10 seconds for bb lock file to appear but also be populated 460 for lock_check in range(10): 461 if not os.path.exists(bblock): 462 logger.debug("localhostbecontroller: waiting for bblock file to appear") 463 time.sleep(1) 464 continue 465 if 10 < os.stat(bblock).st_size: 466 break 467 logger.debug("localhostbecontroller: waiting for bblock content to appear") 468 time.sleep(1) 469 else: 470 raise BuildSetupException("Cannot find bitbake server lock file '%s'. Exiting." % bblock) 471 472 with open(bblock) as fplock: 473 for line in fplock: 474 if ":" in line: 475 self.be.bbport = line.split(":")[-1].strip() 476 logger.debug("localhostbecontroller: bitbake port %s", self.be.bbport) 477 break 478 479 if -1 == self.be.bbport: 480 raise BuildSetupException("localhostbecontroller: can't read bitbake port from %s" % bblock) 481 482 self.be.bbaddress = "localhost" 483 self.be.bbstate = BuildEnvironment.SERVER_STARTED 484 self.be.lock = BuildEnvironment.LOCK_RUNNING 485 self.be.save() 486 487 bbtargets = '' 488 for target in targets: 489 task = target.task 490 if task: 491 if not task.startswith('do_'): 492 task = 'do_' + task 493 task = ':%s' % task 494 bbtargets += '%s%s ' % (target.target, task) 495 496 # run build with local bitbake. stop the server after the build. 497 log = os.path.join(builddir, 'toaster_ui.log') 498 local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')), 499 'bitbake') 500 if not is_merged_attr: 501 self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' 502 '%s %s -u toasterui --read %s --read %s --token="" >>%s 2>&1;' 503 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \ 504 % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, confpath, toasterlayers, log, 505 self.be.bbport, bitbake,)], 506 builddir, nowait=True) 507 else: 508 self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' 509 '%s %s -u toasterui --token="" >>%s 2>&1;' 510 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \ 511 % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, log, 512 self.be.bbport, bitbake,)], 513 builddir, nowait=True) 514 515 logger.debug('localhostbecontroller: Build launched, exiting. ' 516 'Follow build logs at %s' % log) 517