1# 2# BitBake Toaster Implementation 3# 4# Copyright (C) 2018 Wind River Systems 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9# buildimport: import a project for project specific configuration 10# 11# Usage: 12# (a) Set up Toaster environent 13# 14# (b) Call buildimport 15# $ /path/to/bitbake/lib/toaster/manage.py buildimport \ 16# --name=$PROJECTNAME \ 17# --path=$BUILD_DIRECTORY \ 18# --callback="$CALLBACK_SCRIPT" \ 19# --command="configure|reconfigure|import" 20# 21# (c) Return is "|Default_image=%s|Project_id=%d" 22# 23# (d) Open Toaster to this project using for example: 24# $ xdg-open http://localhost:$toaster_port/toastergui/project_specific/$project_id 25# 26# (e) To delete a project: 27# $ /path/to/bitbake/lib/toaster/manage.py buildimport \ 28# --name=$PROJECTNAME --delete-project 29# 30 31 32# ../bitbake/lib/toaster/manage.py buildimport --name=test --path=`pwd` --callback="" --command=import 33 34from django.core.management.base import BaseCommand 35from orm.models import Project, Release, ProjectVariable 36from orm.models import Layer, Layer_Version, LayerSource, ProjectLayer 37from toastergui.api import scan_layer_content 38 39import os 40import re 41import os.path 42import subprocess 43import shutil 44 45# Toaster variable section delimiters 46TOASTER_PROLOG = '#=== TOASTER_CONFIG_PROLOG ===' 47TOASTER_EPILOG = '#=== TOASTER_CONFIG_EPILOG ===' 48 49# quick development/debugging support 50verbose = 2 51def _log(msg): 52 if 1 == verbose: 53 print(msg) 54 elif 2 == verbose: 55 f1=open('/tmp/toaster.log', 'a') 56 f1.write("|" + msg + "|\n" ) 57 f1.close() 58 59 60__config_regexp__ = re.compile( r""" 61 ^ 62 (?P<exp>export\s+)? 63 (?P<var>[a-zA-Z0-9\-_+.${}/~]+?) 64 (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])? 65 66 \s* ( 67 (?P<colon>:=) | 68 (?P<lazyques>\?\?=) | 69 (?P<ques>\?=) | 70 (?P<append>\+=) | 71 (?P<prepend>=\+) | 72 (?P<predot>=\.) | 73 (?P<postdot>\.=) | 74 = 75 ) \s* 76 77 (?!'[^']*'[^']*'$) 78 (?!\"[^\"]*\"[^\"]*\"$) 79 (?P<apo>['\"]) 80 (?P<value>.*) 81 (?P=apo) 82 $ 83 """, re.X) 84 85class Command(BaseCommand): 86 args = "<name> <path> <release>" 87 help = "Import a command line build directory" 88 vars = {} 89 toaster_vars = {} 90 91 def add_arguments(self, parser): 92 parser.add_argument( 93 '--name', dest='name', required=True, 94 help='name of the project', 95 ) 96 parser.add_argument( 97 '--path', dest='path', required=True, 98 help='path to the project', 99 ) 100 parser.add_argument( 101 '--release', dest='release', required=False, 102 help='release for the project', 103 ) 104 parser.add_argument( 105 '--callback', dest='callback', required=False, 106 help='callback for project config update', 107 ) 108 parser.add_argument( 109 '--delete-project', dest='delete_project', required=False, 110 help='delete this project from the database', 111 ) 112 parser.add_argument( 113 '--command', dest='command', required=False, 114 help='command (configure,reconfigure,import)', 115 ) 116 117 def get_var(self, varname): 118 value = self.vars.get(varname, '') 119 if value: 120 varrefs = re.findall('\${([^}]*)}', value) 121 for ref in varrefs: 122 if ref in self.vars: 123 value = value.replace('${%s}' % ref, self.vars[ref]) 124 return value 125 126 # Extract the bb variables from a conf file 127 def scan_conf(self,fn): 128 vars = self.vars 129 toaster_vars = self.toaster_vars 130 131 #_log("scan_conf:%s" % fn) 132 if not os.path.isfile(fn): 133 return 134 f = open(fn, 'r') 135 136 #statements = ast.StatementGroup() 137 lineno = 0 138 is_toaster_section = False 139 while True: 140 lineno = lineno + 1 141 s = f.readline() 142 if not s: 143 break 144 w = s.strip() 145 # skip empty lines 146 if not w: 147 continue 148 # evaluate Toaster sections 149 if w.startswith(TOASTER_PROLOG): 150 is_toaster_section = True 151 continue 152 if w.startswith(TOASTER_EPILOG): 153 is_toaster_section = False 154 continue 155 s = s.rstrip() 156 while s[-1] == '\\': 157 s2 = f.readline().strip() 158 lineno = lineno + 1 159 if (not s2 or s2 and s2[0] != "#") and s[0] == "#" : 160 echo("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s)) 161 s = s[:-1] + s2 162 # skip comments 163 if s[0] == '#': 164 continue 165 # process the line for just assignments 166 m = __config_regexp__.match(s) 167 if m: 168 groupd = m.groupdict() 169 var = groupd['var'] 170 value = groupd['value'] 171 172 if groupd['lazyques']: 173 if not var in vars: 174 vars[var] = value 175 continue 176 if groupd['ques']: 177 if not var in vars: 178 vars[var] = value 179 continue 180 # preset empty blank for remaining operators 181 if not var in vars: 182 vars[var] = '' 183 if groupd['append']: 184 vars[var] += value 185 elif groupd['prepend']: 186 vars[var] = "%s%s" % (value,vars[var]) 187 elif groupd['predot']: 188 vars[var] = "%s %s" % (value,vars[var]) 189 elif groupd['postdot']: 190 vars[var] = "%s %s" % (vars[var],value) 191 else: 192 vars[var] = "%s" % (value) 193 # capture vars in a Toaster section 194 if is_toaster_section: 195 toaster_vars[var] = vars[var] 196 197 # DONE WITH PARSING 198 f.close() 199 self.vars = vars 200 self.toaster_vars = toaster_vars 201 202 # Update the scanned project variables 203 def update_project_vars(self,project,name): 204 pv, create = ProjectVariable.objects.get_or_create(project = project, name = name) 205 if (not name in self.vars.keys()) or (not self.vars[name]): 206 self.vars[name] = pv.value 207 else: 208 if pv.value != self.vars[name]: 209 pv.value = self.vars[name] 210 pv.save() 211 212 # Find the git version of the installation 213 def find_layer_dir_version(self,path): 214 # * rocko ... 215 216 install_version = '' 217 cwd = os.getcwd() 218 os.chdir(path) 219 p = subprocess.Popen(['git', 'branch', '-av'], stdout=subprocess.PIPE, 220 stderr=subprocess.PIPE) 221 out, err = p.communicate() 222 out = out.decode("utf-8") 223 for branch in out.split('\n'): 224 if ('*' == branch[0:1]) and ('no branch' not in branch): 225 install_version = re.sub(' .*','',branch[2:]) 226 break 227 if 'remotes/m/master' in branch: 228 install_version = re.sub('.*base/','',branch) 229 break 230 os.chdir(cwd) 231 return install_version 232 233 # Compute table of the installation's registered layer versions (branch or commit) 234 def find_layer_dir_versions(self,INSTALL_URL_PREFIX): 235 lv_dict = {} 236 layer_versions = Layer_Version.objects.all() 237 for lv in layer_versions: 238 layer = Layer.objects.filter(pk=lv.layer.pk)[0] 239 if layer.vcs_url: 240 url_short = layer.vcs_url.replace(INSTALL_URL_PREFIX,'') 241 else: 242 url_short = '' 243 # register the core, branch, and the version variations 244 lv_dict["%s,%s,%s" % (url_short,lv.dirpath,'')] = (lv.id,layer.name) 245 lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.branch)] = (lv.id,layer.name) 246 lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.commit)] = (lv.id,layer.name) 247 #_log(" (%s,%s,%s|%s) = (%s,%s)" % (url_short,lv.dirpath,lv.branch,lv.commit,lv.id,layer.name)) 248 return lv_dict 249 250 # Apply table of all layer versions 251 def extract_bblayers(self): 252 # set up the constants 253 bblayer_str = self.get_var('BBLAYERS') 254 TOASTER_DIR = os.environ.get('TOASTER_DIR') 255 INSTALL_CLONE_PREFIX = os.path.dirname(TOASTER_DIR) + "/" 256 TOASTER_CLONE_PREFIX = TOASTER_DIR + "/_toaster_clones/" 257 INSTALL_URL_PREFIX = '' 258 layers = Layer.objects.filter(name='openembedded-core') 259 for layer in layers: 260 if layer.vcs_url: 261 INSTALL_URL_PREFIX = layer.vcs_url 262 break 263 INSTALL_URL_PREFIX = INSTALL_URL_PREFIX.replace("/poky","/") 264 INSTALL_VERSION_DIR = TOASTER_DIR 265 INSTALL_URL_POSTFIX = INSTALL_URL_PREFIX.replace(':','_') 266 INSTALL_URL_POSTFIX = INSTALL_URL_POSTFIX.replace('/','_') 267 INSTALL_URL_POSTFIX = "%s_%s" % (TOASTER_CLONE_PREFIX,INSTALL_URL_POSTFIX) 268 269 # get the set of available layer:layer_versions 270 lv_dict = self.find_layer_dir_versions(INSTALL_URL_PREFIX) 271 272 # compute the layer matches 273 layers_list = [] 274 for line in bblayer_str.split(' '): 275 if not line: 276 continue 277 if line.endswith('/local'): 278 continue 279 280 # isolate the repo 281 layer_path = line 282 line = line.replace(INSTALL_URL_POSTFIX,'').replace(INSTALL_CLONE_PREFIX,'').replace('/layers/','/').replace('/poky/','/') 283 284 # isolate the sub-path 285 path_index = line.rfind('/') 286 if path_index > 0: 287 sub_path = line[path_index+1:] 288 line = line[0:path_index] 289 else: 290 sub_path = '' 291 292 # isolate the version 293 if TOASTER_CLONE_PREFIX in layer_path: 294 is_toaster_clone = True 295 # extract version from name syntax 296 version_index = line.find('_') 297 if version_index > 0: 298 version = line[version_index+1:] 299 line = line[0:version_index] 300 else: 301 version = '' 302 _log("TOASTER_CLONE(%s/%s), version=%s" % (line,sub_path,version)) 303 else: 304 is_toaster_clone = False 305 # version is from the installation 306 version = self.find_layer_dir_version(layer_path) 307 _log("LOCAL_CLONE(%s/%s), version=%s" % (line,sub_path,version)) 308 309 # capture the layer information into layers_list 310 layers_list.append( (line,sub_path,version,layer_path,is_toaster_clone) ) 311 return layers_list,lv_dict 312 313 # 314 def find_import_release(self,layers_list,lv_dict,default_release): 315 # poky,meta,rocko => 4;openembedded-core 316 release = default_release 317 for line,path,version,layer_path,is_toaster_clone in layers_list: 318 key = "%s,%s,%s" % (line,path,version) 319 if key in lv_dict: 320 lv_id = lv_dict[key] 321 if 'openembedded-core' == lv_id[1]: 322 _log("Find_import_release(%s):version=%s,Toaster=%s" % (lv_id[1],version,is_toaster_clone)) 323 # only versions in Toaster managed layers are accepted 324 if not is_toaster_clone: 325 break 326 try: 327 release = Release.objects.get(name=version) 328 except: 329 pass 330 break 331 _log("Find_import_release:RELEASE=%s" % release.name) 332 return release 333 334 # Apply the found conf layers 335 def apply_conf_bblayers(self,layers_list,lv_dict,project,release=None): 336 for line,path,version,layer_path,is_toaster_clone in layers_list: 337 # Assert release promote if present 338 if release: 339 version = release 340 # try to match the key to a layer_version 341 key = "%s,%s,%s" % (line,path,version) 342 key_short = "%s,%s,%s" % (line,path,'') 343 lv_id = '' 344 if key in lv_dict: 345 lv_id = lv_dict[key] 346 lv = Layer_Version.objects.get(pk=int(lv_id[0])) 347 pl,created = ProjectLayer.objects.get_or_create(project=project, 348 layercommit=lv) 349 pl.optional=False 350 pl.save() 351 _log(" %s => %s;%s" % (key,lv_id[0],lv_id[1])) 352 elif key_short in lv_dict: 353 lv_id = lv_dict[key_short] 354 lv = Layer_Version.objects.get(pk=int(lv_id[0])) 355 pl,created = ProjectLayer.objects.get_or_create(project=project, 356 layercommit=lv) 357 pl.optional=False 358 pl.save() 359 _log(" %s ?> %s" % (key,lv_dict[key_short])) 360 else: 361 _log("%s <= %s" % (key,layer_path)) 362 found = False 363 # does local layer already exist in this project? 364 try: 365 for pl in ProjectLayer.objects.filter(project=project): 366 if pl.layercommit.layer.local_source_dir == layer_path: 367 found = True 368 _log(" Project Local Layer found!") 369 except Exception as e: 370 _log("ERROR: Local Layer '%s'" % e) 371 pass 372 373 if not found: 374 # Does Layer name+path already exist? 375 try: 376 layer_name_base = os.path.basename(layer_path) 377 _log("Layer_lookup: try '%s','%s'" % (layer_name_base,layer_path)) 378 layer = Layer.objects.get(name=layer_name_base,local_source_dir = layer_path) 379 # Found! Attach layer_version and ProjectLayer 380 layer_version = Layer_Version.objects.create( 381 layer=layer, 382 project=project, 383 layer_source=LayerSource.TYPE_IMPORTED) 384 layer_version.save() 385 pl,created = ProjectLayer.objects.get_or_create(project=project, 386 layercommit=layer_version) 387 pl.optional=False 388 pl.save() 389 found = True 390 # add layer contents to this layer version 391 scan_layer_content(layer,layer_version) 392 _log(" Parent Local Layer found in db!") 393 except Exception as e: 394 _log("Layer_exists_test_failed: Local Layer '%s'" % e) 395 pass 396 397 if not found: 398 # Insure that layer path exists, in case of user typo 399 if not os.path.isdir(layer_path): 400 _log("ERROR:Layer path '%s' not found" % layer_path) 401 continue 402 # Add layer to db and attach project to it 403 layer_name_base = os.path.basename(layer_path) 404 # generate a unique layer name 405 layer_name_matches = {} 406 for layer in Layer.objects.filter(name__contains=layer_name_base): 407 layer_name_matches[layer.name] = '1' 408 layer_name_idx = 0 409 layer_name_test = layer_name_base 410 while layer_name_test in layer_name_matches.keys(): 411 layer_name_idx += 1 412 layer_name_test = "%s_%d" % (layer_name_base,layer_name_idx) 413 # create the layer and layer_verion objects 414 layer = Layer.objects.create(name=layer_name_test) 415 layer.local_source_dir = layer_path 416 layer_version = Layer_Version.objects.create( 417 layer=layer, 418 project=project, 419 layer_source=LayerSource.TYPE_IMPORTED) 420 layer.save() 421 layer_version.save() 422 pl,created = ProjectLayer.objects.get_or_create(project=project, 423 layercommit=layer_version) 424 pl.optional=False 425 pl.save() 426 # register the layer's content 427 _log(" Local Layer Add content") 428 scan_layer_content(layer,layer_version) 429 _log(" Local Layer Added '%s'!" % layer_name_test) 430 431 # Scan the project's conf files (if any) 432 def scan_conf_variables(self,project_path): 433 self.vars['TOPDIR'] = project_path 434 # scan the project's settings, add any new layers or variables 435 if os.path.isfile("%s/conf/local.conf" % project_path): 436 self.scan_conf("%s/conf/local.conf" % project_path) 437 self.scan_conf("%s/conf/bblayers.conf" % project_path) 438 # Import then disable old style Toaster conf files (before 'merged_attr') 439 old_toaster_local = "%s/conf/toaster.conf" % project_path 440 if os.path.isfile(old_toaster_local): 441 self.scan_conf(old_toaster_local) 442 shutil.move(old_toaster_local, old_toaster_local+"_old") 443 old_toaster_layer = "%s/conf/toaster-bblayers.conf" % project_path 444 if os.path.isfile(old_toaster_layer): 445 self.scan_conf(old_toaster_layer) 446 shutil.move(old_toaster_layer, old_toaster_layer+"_old") 447 448 # Scan the found conf variables (if any) 449 def apply_conf_variables(self,project,layers_list,lv_dict,release=None): 450 if self.vars: 451 # Catch vars relevant to Toaster (in case no Toaster section) 452 self.update_project_vars(project,'DISTRO') 453 self.update_project_vars(project,'MACHINE') 454 self.update_project_vars(project,'IMAGE_INSTALL:append') 455 self.update_project_vars(project,'IMAGE_FSTYPES') 456 self.update_project_vars(project,'PACKAGE_CLASSES') 457 # These vars are typically only assigned by Toaster 458 #self.update_project_vars(project,'DL_DIR') 459 #self.update_project_vars(project,'SSTATE_DIR') 460 461 # Assert found Toaster vars 462 for var in self.toaster_vars.keys(): 463 pv, create = ProjectVariable.objects.get_or_create(project = project, name = var) 464 pv.value = self.toaster_vars[var] 465 _log("* Add/update Toaster var '%s' = '%s'" % (pv.name,pv.value)) 466 pv.save() 467 468 # Assert found BBLAYERS 469 if 0 < verbose: 470 for pl in ProjectLayer.objects.filter(project=project): 471 release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name 472 print(" BEFORE:ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit)) 473 self.apply_conf_bblayers(layers_list,lv_dict,project,release) 474 if 0 < verbose: 475 for pl in ProjectLayer.objects.filter(project=project): 476 release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name 477 print(" AFTER :ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit)) 478 479 def handle(self, *args, **options): 480 project_name = options['name'] 481 project_path = options['path'] 482 project_callback = options['callback'] if options['callback'] else '' 483 release_name = options['release'] if options['release'] else '' 484 485 # 486 # Delete project 487 # 488 489 if options['delete_project']: 490 try: 491 print("Project '%s' delete from Toaster database" % (project_name)) 492 project = Project.objects.get(name=project_name) 493 # TODO: deep project delete 494 project.delete() 495 print("Project '%s' Deleted" % (project_name)) 496 return 497 except Exception as e: 498 print("Project '%s' not found, not deleted (%s)" % (project_name,e)) 499 return 500 501 # 502 # Create/Update/Import project 503 # 504 505 # See if project (by name) exists 506 project = None 507 try: 508 # Project already exists 509 project = Project.objects.get(name=project_name) 510 except Exception as e: 511 pass 512 513 # Find the installation's default release 514 default_release = Release.objects.get(id=1) 515 516 # SANITY: if 'reconfig' but project does not exist (deleted externally), switch to 'import' 517 if ("reconfigure" == options['command']) and project is None: 518 options['command'] = 'import' 519 520 # 'Configure': 521 if "configure" == options['command']: 522 # Note: ignore any existing conf files 523 # create project, SANITY: reuse any project of same name 524 project = Project.objects.create_project(project_name,default_release,project) 525 526 # 'Re-configure': 527 if "reconfigure" == options['command']: 528 # Scan the directory's conf files 529 self.scan_conf_variables(project_path) 530 # Scan the layer list 531 layers_list,lv_dict = self.extract_bblayers() 532 # Apply any new layers or variables 533 self.apply_conf_variables(project,layers_list,lv_dict) 534 535 # 'Import': 536 if "import" == options['command']: 537 # Scan the directory's conf files 538 self.scan_conf_variables(project_path) 539 # Remove these Toaster controlled variables 540 for var in ('DL_DIR','SSTATE_DIR'): 541 self.vars.pop(var, None) 542 self.toaster_vars.pop(var, None) 543 # Scan the layer list 544 layers_list,lv_dict = self.extract_bblayers() 545 # Find the directory's release, and promote to default_release if local paths 546 release = self.find_import_release(layers_list,lv_dict,default_release) 547 # create project, SANITY: reuse any project of same name 548 project = Project.objects.create_project(project_name,release,project, imported=True) 549 # Apply any new layers or variables 550 self.apply_conf_variables(project,layers_list,lv_dict,release) 551 # WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific' 552 project.set_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE','1') 553 554 # Set up the project's meta data 555 project.builddir = project_path 556 project.merged_attr = True 557 project.set_variable(Project.PROJECT_SPECIFIC_CALLBACK,project_callback) 558 project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_EDIT) 559 if ("configure" == options['command']) or ("import" == options['command']): 560 # preset the mode and default image recipe 561 project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NEW) 562 project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,"core-image-minimal") 563 564 # Assert any extended/custom actions or variables for new non-Toaster projects 565 if not len(self.toaster_vars): 566 pass 567 else: 568 project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NONE) 569 570 # Save the updated Project 571 project.save() 572 573 _log("Buildimport:project='%s' at '%d'" % (project_name,project.id)) 574 575 if ('DEFAULT_IMAGE' in self.vars) and (self.vars['DEFAULT_IMAGE']): 576 print("|Default_image=%s|Project_id=%d" % (self.vars['DEFAULT_IMAGE'],project.id)) 577 else: 578 print("|Project_id=%d" % (project.id)) 579 580