1 # 2 # BitBake Toaster Implementation 3 # 4 # Copyright (C) 2013 Intel Corporation 5 # 6 # SPDX-License-Identifier: GPL-2.0-only 7 # 8 9 from __future__ import unicode_literals 10 11 from django.db import models, IntegrityError, DataError 12 from django.db.models import F, Q, Sum, Count 13 from django.utils import timezone 14 from django.utils.encoding import force_bytes 15 16 from django.urls import reverse 17 18 from django.core import validators 19 from django.conf import settings 20 import django.db.models.signals 21 22 import sys 23 import os 24 import re 25 import itertools 26 from signal import SIGUSR1 27 28 29 import logging 30 logger = logging.getLogger("toaster") 31 32 if 'sqlite' in settings.DATABASES['default']['ENGINE']: 33 from django.db import transaction, OperationalError 34 from time import sleep 35 36 _base_save = models.Model.save 37 def save(self, *args, **kwargs): 38 while True: 39 try: 40 with transaction.atomic(): 41 return _base_save(self, *args, **kwargs) 42 except OperationalError as err: 43 if 'database is locked' in str(err): 44 logger.warning("%s, model: %s, args: %s, kwargs: %s", 45 err, self.__class__, args, kwargs) 46 sleep(0.5) 47 continue 48 raise 49 50 models.Model.save = save 51 52 # HACK: Monkey patch Django to fix 'database is locked' issue 53 54 from django.db.models.query import QuerySet 55 _base_insert = QuerySet._insert 56 def _insert(self, *args, **kwargs): 57 with transaction.atomic(using=self.db, savepoint=False): 58 return _base_insert(self, *args, **kwargs) 59 QuerySet._insert = _insert 60 61 def _create_object_from_params(self, lookup, params): 62 """ 63 Tries to create an object using passed params. 64 Used by get_or_create and update_or_create 65 """ 66 try: 67 obj = self.create(**params) 68 return obj, True 69 except (IntegrityError, DataError): 70 exc_info = sys.exc_info() 71 try: 72 return self.get(**lookup), False 73 except self.model.DoesNotExist: 74 pass 75 six.reraise(*exc_info) 76 77 QuerySet._create_object_from_params = _create_object_from_params 78 79 # end of HACK 80 81 class GitURLValidator(validators.URLValidator): 82 import re 83 regex = re.compile( 84 r'^(?:ssh|git|http|ftp)s?://' # http:// or https:// 85 r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... 86 r'localhost|' # localhost... 87 r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4 88 r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6 89 r'(?::\d+)?' # optional port 90 r'(?:/?|[/?]\S+)$', re.IGNORECASE) 91 92 def GitURLField(**kwargs): 93 r = models.URLField(**kwargs) 94 for i in range(len(r.validators)): 95 if isinstance(r.validators[i], validators.URLValidator): 96 r.validators[i] = GitURLValidator() 97 return r 98 99 100 class ToasterSetting(models.Model): 101 name = models.CharField(max_length=63) 102 helptext = models.TextField() 103 value = models.CharField(max_length=255) 104 105 def __unicode__(self): 106 return "Setting %s = %s" % (self.name, self.value) 107 108 109 class ProjectManager(models.Manager): 110 def create_project(self, name, release, existing_project=None, imported=False): 111 if existing_project and (release is not None): 112 prj = existing_project 113 prj.bitbake_version = release.bitbake_version 114 prj.release = release 115 # Delete the previous ProjectLayer mappings 116 for pl in ProjectLayer.objects.filter(project=prj): 117 pl.delete() 118 elif release is not None: 119 prj = self.model(name=name, 120 bitbake_version=release.bitbake_version, 121 release=release) 122 else: 123 prj = self.model(name=name, 124 bitbake_version=None, 125 release=None) 126 prj.save() 127 128 for defaultconf in ToasterSetting.objects.filter( 129 name__startswith="DEFCONF_"): 130 name = defaultconf.name[8:] 131 pv,create = ProjectVariable.objects.get_or_create(project=prj,name=name) 132 pv.value = defaultconf.value 133 pv.save() 134 135 if release is None: 136 return prj 137 if not imported: 138 for rdl in release.releasedefaultlayer_set.all(): 139 lv = Layer_Version.objects.filter( 140 layer__name=rdl.layer_name, 141 release=release).first() 142 143 if lv: 144 ProjectLayer.objects.create(project=prj, 145 layercommit=lv, 146 optional=False) 147 else: 148 logger.warning("Default project layer %s not found" % 149 rdl.layer_name) 150 151 return prj 152 153 # return single object with is_default = True 154 def get_or_create_default_project(self): 155 projects = super(ProjectManager, self).filter(is_default=True) 156 157 if len(projects) > 1: 158 raise Exception('Inconsistent project data: multiple ' + 159 'default projects (i.e. with is_default=True)') 160 elif len(projects) < 1: 161 options = { 162 'name': 'Command line builds', 163 'short_description': 164 'Project for builds started outside Toaster', 165 'is_default': True 166 } 167 project = Project.objects.create(**options) 168 project.save() 169 170 return project 171 else: 172 return projects[0] 173 174 175 class Project(models.Model): 176 search_allowed_fields = ['name', 'short_description', 'release__name', 177 'release__branch_name'] 178 name = models.CharField(max_length=100) 179 short_description = models.CharField(max_length=50, blank=True) 180 bitbake_version = models.ForeignKey('BitbakeVersion', on_delete=models.CASCADE, null=True) 181 release = models.ForeignKey("Release", on_delete=models.CASCADE, null=True) 182 created = models.DateTimeField(auto_now_add=True) 183 updated = models.DateTimeField(auto_now=True) 184 # This is a horrible hack; since Toaster has no "User" model available when 185 # running in interactive mode, we can't reference the field here directly 186 # Instead, we keep a possible null reference to the User id, 187 # as not to force 188 # hard links to possibly missing models 189 user_id = models.IntegerField(null=True) 190 objects = ProjectManager() 191 192 # build directory override (e.g. imported) 193 builddir = models.TextField() 194 # merge the Toaster configure attributes directly into the standard conf files 195 merged_attr = models.BooleanField(default=False) 196 197 # set to True for the project which is the default container 198 # for builds initiated by the command line etc. 199 is_default= models.BooleanField(default=False) 200 201 def __unicode__(self): 202 return "%s (Release %s, BBV %s)" % (self.name, self.release, self.bitbake_version) 203 204 def get_current_machine_name(self): 205 try: 206 return self.projectvariable_set.get(name="MACHINE").value 207 except (ProjectVariable.DoesNotExist,IndexError): 208 return None; 209 210 def get_number_of_builds(self): 211 """Return the number of builds which have ended""" 212 213 return self.build_set.exclude( 214 Q(outcome=Build.IN_PROGRESS) | 215 Q(outcome=Build.CANCELLED) 216 ).count() 217 218 def get_last_build_id(self): 219 try: 220 return Build.objects.filter( project = self.id ).order_by('-completed_on')[0].id 221 except (Build.DoesNotExist,IndexError): 222 return( -1 ) 223 224 def get_last_outcome(self): 225 build_id = self.get_last_build_id() 226 if (-1 == build_id): 227 return( "" ) 228 try: 229 return Build.objects.filter( id = build_id )[ 0 ].outcome 230 except (Build.DoesNotExist,IndexError): 231 return( "not_found" ) 232 233 def get_last_target(self): 234 build_id = self.get_last_build_id() 235 if (-1 == build_id): 236 return( "" ) 237 try: 238 return Target.objects.filter(build = build_id)[0].target 239 except (Target.DoesNotExist,IndexError): 240 return( "not_found" ) 241 242 def get_last_errors(self): 243 build_id = self.get_last_build_id() 244 if (-1 == build_id): 245 return( 0 ) 246 try: 247 return Build.objects.filter(id = build_id)[ 0 ].errors.count() 248 except (Build.DoesNotExist,IndexError): 249 return( "not_found" ) 250 251 def get_last_warnings(self): 252 build_id = self.get_last_build_id() 253 if (-1 == build_id): 254 return( 0 ) 255 try: 256 return Build.objects.filter(id = build_id)[ 0 ].warnings.count() 257 except (Build.DoesNotExist,IndexError): 258 return( "not_found" ) 259 260 def get_last_build_extensions(self): 261 """ 262 Get list of file name extensions for images produced by the most 263 recent build 264 """ 265 last_build = Build.objects.get(pk = self.get_last_build_id()) 266 return last_build.get_image_file_extensions() 267 268 def get_last_imgfiles(self): 269 build_id = self.get_last_build_id() 270 if (-1 == build_id): 271 return( "" ) 272 try: 273 return Variable.objects.filter(build = build_id, variable_name = "IMAGE_FSTYPES")[ 0 ].variable_value 274 except (Variable.DoesNotExist,IndexError): 275 return( "not_found" ) 276 277 def get_all_compatible_layer_versions(self): 278 """ Returns Queryset of all Layer_Versions which are compatible with 279 this project""" 280 queryset = None 281 282 # guard on release, as it can be null 283 if self.release: 284 queryset = Layer_Version.objects.filter( 285 (Q(release=self.release) & 286 Q(build=None) & 287 Q(project=None)) | 288 Q(project=self)) 289 else: 290 queryset = Layer_Version.objects.none() 291 292 return queryset 293 294 def get_project_layer_versions(self, pk=False): 295 """ Returns the Layer_Versions currently added to this project """ 296 layer_versions = self.projectlayer_set.all().values_list('layercommit', 297 flat=True) 298 299 if pk is False: 300 return Layer_Version.objects.filter(pk__in=layer_versions) 301 else: 302 return layer_versions 303 304 305 def get_default_image_recipe(self): 306 try: 307 return self.projectvariable_set.get(name="DEFAULT_IMAGE").value 308 except (ProjectVariable.DoesNotExist,IndexError): 309 return None; 310 311 def get_is_new(self): 312 return self.get_variable(Project.PROJECT_SPECIFIC_ISNEW) 313 314 def get_available_machines(self): 315 """ Returns QuerySet of all Machines which are provided by the 316 Layers currently added to the Project """ 317 queryset = Machine.objects.filter( 318 layer_version__in=self.get_project_layer_versions()) 319 320 return queryset 321 322 def get_all_compatible_machines(self): 323 """ Returns QuerySet of all the compatible machines available to the 324 project including ones from Layers not currently added """ 325 queryset = Machine.objects.filter( 326 layer_version__in=self.get_all_compatible_layer_versions()) 327 328 return queryset 329 330 def get_available_distros(self): 331 """ Returns QuerySet of all Distros which are provided by the 332 Layers currently added to the Project """ 333 queryset = Distro.objects.filter( 334 layer_version__in=self.get_project_layer_versions()) 335 336 return queryset 337 338 def get_all_compatible_distros(self): 339 """ Returns QuerySet of all the compatible Wind River distros available to the 340 project including ones from Layers not currently added """ 341 queryset = Distro.objects.filter( 342 layer_version__in=self.get_all_compatible_layer_versions()) 343 344 return queryset 345 346 def get_available_recipes(self): 347 """ Returns QuerySet of all the recipes that are provided by layers 348 added to this project """ 349 queryset = Recipe.objects.filter( 350 layer_version__in=self.get_project_layer_versions()) 351 352 return queryset 353 354 def get_all_compatible_recipes(self): 355 """ Returns QuerySet of all the compatible Recipes available to the 356 project including ones from Layers not currently added """ 357 queryset = Recipe.objects.filter( 358 layer_version__in=self.get_all_compatible_layer_versions()).exclude(name__exact='') 359 360 return queryset 361 362 # Project Specific status management 363 PROJECT_SPECIFIC_STATUS = 'INTERNAL_PROJECT_SPECIFIC_STATUS' 364 PROJECT_SPECIFIC_CALLBACK = 'INTERNAL_PROJECT_SPECIFIC_CALLBACK' 365 PROJECT_SPECIFIC_ISNEW = 'INTERNAL_PROJECT_SPECIFIC_ISNEW' 366 PROJECT_SPECIFIC_DEFAULTIMAGE = 'PROJECT_SPECIFIC_DEFAULTIMAGE' 367 PROJECT_SPECIFIC_NONE = '' 368 PROJECT_SPECIFIC_NEW = '1' 369 PROJECT_SPECIFIC_EDIT = '2' 370 PROJECT_SPECIFIC_CLONING = '3' 371 PROJECT_SPECIFIC_CLONING_SUCCESS = '4' 372 PROJECT_SPECIFIC_CLONING_FAIL = '5' 373 374 def get_variable(self,variable,default_value = ''): 375 try: 376 return self.projectvariable_set.get(name=variable).value 377 except (ProjectVariable.DoesNotExist,IndexError): 378 return default_value 379 380 def set_variable(self,variable,value): 381 pv,create = ProjectVariable.objects.get_or_create(project = self, name = variable) 382 pv.value = value 383 pv.save() 384 385 def get_default_image(self): 386 return self.get_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE) 387 388 def schedule_build(self): 389 390 from bldcontrol.models import BuildRequest, BRTarget, BRLayer 391 from bldcontrol.models import BRBitbake, BRVariable 392 393 try: 394 now = timezone.now() 395 build = Build.objects.create(project=self, 396 completed_on=now, 397 started_on=now) 398 399 br = BuildRequest.objects.create(project=self, 400 state=BuildRequest.REQ_QUEUED, 401 build=build) 402 BRBitbake.objects.create(req=br, 403 giturl=self.bitbake_version.giturl, 404 commit=self.bitbake_version.branch, 405 dirpath=self.bitbake_version.dirpath) 406 407 for t in self.projecttarget_set.all(): 408 BRTarget.objects.create(req=br, target=t.target, task=t.task) 409 Target.objects.create(build=br.build, target=t.target, 410 task=t.task) 411 # If we're about to build a custom image recipe make sure 412 # that layer is currently in the project before we create the 413 # BRLayer objects 414 customrecipe = CustomImageRecipe.objects.filter( 415 name=t.target, 416 project=self).first() 417 if customrecipe: 418 ProjectLayer.objects.get_or_create( 419 project=self, 420 layercommit=customrecipe.layer_version, 421 optional=False) 422 423 for l in self.projectlayer_set.all().order_by("pk"): 424 commit = l.layercommit.get_vcs_reference() 425 logger.debug("Adding layer to build %s" % 426 l.layercommit.layer.name) 427 BRLayer.objects.create( 428 req=br, 429 name=l.layercommit.layer.name, 430 giturl=l.layercommit.layer.vcs_url, 431 commit=commit, 432 dirpath=l.layercommit.dirpath, 433 layer_version=l.layercommit, 434 local_source_dir=l.layercommit.layer.local_source_dir 435 ) 436 437 for v in self.projectvariable_set.all(): 438 BRVariable.objects.create(req=br, name=v.name, value=v.value) 439 440 try: 441 br.build.machine = self.projectvariable_set.get( 442 name='MACHINE').value 443 br.build.save() 444 except ProjectVariable.DoesNotExist: 445 pass 446 447 br.save() 448 signal_runbuilds() 449 450 except Exception: 451 # revert the build request creation since we're not done cleanly 452 br.delete() 453 raise 454 return br 455 456 class Build(models.Model): 457 SUCCEEDED = 0 458 FAILED = 1 459 IN_PROGRESS = 2 460 CANCELLED = 3 461 462 BUILD_OUTCOME = ( 463 (SUCCEEDED, 'Succeeded'), 464 (FAILED, 'Failed'), 465 (IN_PROGRESS, 'In Progress'), 466 (CANCELLED, 'Cancelled'), 467 ) 468 469 search_allowed_fields = ['machine', 'cooker_log_path', "target__target", "target__target_image_file__file_name"] 470 471 project = models.ForeignKey(Project, on_delete=models.CASCADE) # must have a project 472 machine = models.CharField(max_length=100) 473 distro = models.CharField(max_length=100) 474 distro_version = models.CharField(max_length=100) 475 started_on = models.DateTimeField() 476 completed_on = models.DateTimeField() 477 outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS) 478 cooker_log_path = models.CharField(max_length=500) 479 build_name = models.CharField(max_length=100, default='') 480 bitbake_version = models.CharField(max_length=50) 481 482 # number of recipes to parse for this build 483 recipes_to_parse = models.IntegerField(default=1) 484 485 # number of recipes parsed so far for this build 486 recipes_parsed = models.IntegerField(default=1) 487 488 # number of repos to clone for this build 489 repos_to_clone = models.IntegerField(default=1) 490 491 # number of repos cloned so far for this build (default off) 492 repos_cloned = models.IntegerField(default=1) 493 494 # Hint on current progress item 495 progress_item = models.CharField(max_length=40) 496 497 @staticmethod 498 def get_recent(project=None): 499 """ 500 Return recent builds as a list; if project is set, only return 501 builds for that project 502 """ 503 504 builds = Build.objects.all() 505 506 if project: 507 builds = builds.filter(project=project) 508 509 finished_criteria = \ 510 Q(outcome=Build.SUCCEEDED) | \ 511 Q(outcome=Build.FAILED) | \ 512 Q(outcome=Build.CANCELLED) 513 514 recent_builds = list(itertools.chain( 515 builds.filter(outcome=Build.IN_PROGRESS).order_by("-started_on"), 516 builds.filter(finished_criteria).order_by("-completed_on")[:3] 517 )) 518 519 # add percentage done property to each build; this is used 520 # to show build progress in mrb_section.html 521 for build in recent_builds: 522 build.percentDone = build.completeper() 523 build.outcomeText = build.get_outcome_text() 524 525 return recent_builds 526 527 def started(self): 528 """ 529 As build variables are only added for a build when its BuildStarted event 530 is received, a build with no build variables is counted as 531 "in preparation" and not properly started yet. This method 532 will return False if a build has no build variables (it never properly 533 started), or True otherwise. 534 535 Note that this is a temporary workaround for the fact that we don't 536 have a fine-grained state variable on a build which would allow us 537 to record "in progress" (BuildStarted received) vs. "in preparation". 538 """ 539 variables = Variable.objects.filter(build=self) 540 return len(variables) > 0 541 542 def completeper(self): 543 tf = Task.objects.filter(build = self) 544 tfc = tf.count() 545 if tfc > 0: 546 completeper = tf.exclude(outcome=Task.OUTCOME_NA).count()*100 // tfc 547 else: 548 completeper = 0 549 return completeper 550 551 def eta(self): 552 eta = timezone.now() 553 completeper = self.completeper() 554 if self.completeper() > 0: 555 eta += ((eta - self.started_on)*(100-completeper))/completeper 556 return eta 557 558 def has_images(self): 559 """ 560 Returns True if at least one of the targets for this build has an 561 image file associated with it, False otherwise 562 """ 563 targets = Target.objects.filter(build_id=self.id) 564 has_images = False 565 for target in targets: 566 if target.has_images(): 567 has_images = True 568 break 569 return has_images 570 571 def has_image_recipes(self): 572 """ 573 Returns True if a build has any targets which were built from 574 image recipes. 575 """ 576 image_recipes = self.get_image_recipes() 577 return len(image_recipes) > 0 578 579 def get_image_file_extensions(self): 580 """ 581 Get string of file name extensions for images produced by this build; 582 note that this is the actual list of extensions stored on Target objects 583 for this build, and not the value of IMAGE_FSTYPES. 584 585 Returns comma-separated string, e.g. "vmdk, ext4" 586 """ 587 extensions = [] 588 589 targets = Target.objects.filter(build_id = self.id) 590 for target in targets: 591 if not target.is_image: 592 continue 593 594 target_image_files = Target_Image_File.objects.filter( 595 target_id=target.id) 596 597 for target_image_file in target_image_files: 598 extensions.append(target_image_file.suffix) 599 600 extensions = list(set(extensions)) 601 extensions.sort() 602 603 return ', '.join(extensions) 604 605 def get_image_fstypes(self): 606 """ 607 Get the IMAGE_FSTYPES variable value for this build as a de-duplicated 608 list of image file suffixes. 609 """ 610 image_fstypes = Variable.objects.get( 611 build=self, variable_name='IMAGE_FSTYPES').variable_value 612 return list(set(re.split(r' {1,}', image_fstypes))) 613 614 def get_sorted_target_list(self): 615 tgts = Target.objects.filter(build_id = self.id).order_by( 'target' ); 616 return( tgts ); 617 618 def get_recipes(self): 619 """ 620 Get the recipes related to this build; 621 note that the related layer versions and layers are also prefetched 622 by this query, as this queryset can be sorted by these objects in the 623 build recipes view; prefetching them here removes the need 624 for another query in that view 625 """ 626 layer_versions = Layer_Version.objects.filter(build=self) 627 criteria = Q(layer_version__id__in=layer_versions) 628 return Recipe.objects.filter(criteria) \ 629 .select_related('layer_version', 'layer_version__layer') 630 631 def get_image_recipes(self): 632 """ 633 Returns a list of image Recipes (custom and built-in) related to this 634 build, sorted by name; note that this has to be done in two steps, as 635 there's no way to get all the custom image recipes and image recipes 636 in one query 637 """ 638 custom_image_recipes = self.get_custom_image_recipes() 639 custom_image_recipe_names = custom_image_recipes.values_list('name', flat=True) 640 641 not_custom_image_recipes = ~Q(name__in=custom_image_recipe_names) & \ 642 Q(is_image=True) 643 644 built_image_recipes = self.get_recipes().filter(not_custom_image_recipes) 645 646 # append to the custom image recipes and sort 647 customisable_image_recipes = list( 648 itertools.chain(custom_image_recipes, built_image_recipes) 649 ) 650 651 return sorted(customisable_image_recipes, key=lambda recipe: recipe.name) 652 653 def get_custom_image_recipes(self): 654 """ 655 Returns a queryset of CustomImageRecipes related to this build, 656 sorted by name 657 """ 658 built_recipe_names = self.get_recipes().values_list('name', flat=True) 659 criteria = Q(name__in=built_recipe_names) & Q(project=self.project) 660 queryset = CustomImageRecipe.objects.filter(criteria).order_by('name') 661 return queryset 662 663 def get_outcome_text(self): 664 return Build.BUILD_OUTCOME[int(self.outcome)][1] 665 666 @property 667 def failed_tasks(self): 668 """ Get failed tasks for the build """ 669 tasks = self.task_build.all() 670 return tasks.filter(order__gt=0, outcome=Task.OUTCOME_FAILED) 671 672 @property 673 def errors(self): 674 return (self.logmessage_set.filter(level=LogMessage.ERROR) | 675 self.logmessage_set.filter(level=LogMessage.EXCEPTION) | 676 self.logmessage_set.filter(level=LogMessage.CRITICAL)) 677 678 @property 679 def warnings(self): 680 return self.logmessage_set.filter(level=LogMessage.WARNING) 681 682 @property 683 def timespent(self): 684 return self.completed_on - self.started_on 685 686 @property 687 def timespent_seconds(self): 688 return self.timespent.total_seconds() 689 690 @property 691 def target_labels(self): 692 """ 693 Sorted (a-z) "target1:task, target2, target3" etc. string for all 694 targets in this build 695 """ 696 targets = self.target_set.all() 697 target_labels = [target.target + 698 (':' + target.task if target.task else '') 699 for target in targets] 700 target_labels.sort() 701 702 return target_labels 703 704 def get_buildrequest(self): 705 buildrequest = None 706 if hasattr(self, 'buildrequest'): 707 buildrequest = self.buildrequest 708 return buildrequest 709 710 def is_queued(self): 711 from bldcontrol.models import BuildRequest 712 buildrequest = self.get_buildrequest() 713 if buildrequest: 714 return buildrequest.state == BuildRequest.REQ_QUEUED 715 else: 716 return False 717 718 def is_cancelling(self): 719 from bldcontrol.models import BuildRequest 720 buildrequest = self.get_buildrequest() 721 if buildrequest: 722 return self.outcome == Build.IN_PROGRESS and \ 723 buildrequest.state == BuildRequest.REQ_CANCELLING 724 else: 725 return False 726 727 def is_cloning(self): 728 """ 729 True if the build is still cloning repos 730 """ 731 return self.outcome == Build.IN_PROGRESS and \ 732 self.repos_cloned < self.repos_to_clone 733 734 def is_parsing(self): 735 """ 736 True if the build is still parsing recipes 737 """ 738 return self.outcome == Build.IN_PROGRESS and \ 739 self.recipes_parsed < self.recipes_to_parse 740 741 def is_starting(self): 742 """ 743 True if the build has no completed tasks yet and is still just starting 744 tasks. 745 746 Note that the mechanism for testing whether a Task is "done" is whether 747 its outcome field is set, as per the completeper() method. 748 """ 749 return self.outcome == Build.IN_PROGRESS and \ 750 self.task_build.exclude(outcome=Task.OUTCOME_NA).count() == 0 751 752 753 def get_state(self): 754 """ 755 Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress', 756 'Cancelled' (Build outcomes); or 'Queued', 'Cancelling' (states 757 dependent on the BuildRequest state). 758 759 This works around the fact that we have BuildRequest states as well 760 as Build states, but really we just want to know the state of the build. 761 """ 762 if self.is_cancelling(): 763 return 'Cancelling'; 764 elif self.is_queued(): 765 return 'Queued' 766 elif self.is_cloning(): 767 return 'Cloning' 768 elif self.is_parsing(): 769 return 'Parsing' 770 elif self.is_starting(): 771 return 'Starting' 772 else: 773 return self.get_outcome_text() 774 775 def __str__(self): 776 return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()])) 777 778 class ProjectTarget(models.Model): 779 project = models.ForeignKey(Project, on_delete=models.CASCADE) 780 target = models.CharField(max_length=100) 781 task = models.CharField(max_length=100, null=True) 782 783 class Target(models.Model): 784 search_allowed_fields = ['target', 'file_name'] 785 build = models.ForeignKey(Build, on_delete=models.CASCADE) 786 target = models.CharField(max_length=100) 787 task = models.CharField(max_length=100, null=True) 788 is_image = models.BooleanField(default = False) 789 image_size = models.IntegerField(default=0) 790 license_manifest_path = models.CharField(max_length=500, null=True) 791 package_manifest_path = models.CharField(max_length=500, null=True) 792 793 def package_count(self): 794 return Target_Installed_Package.objects.filter(target_id__exact=self.id).count() 795 796 def __unicode__(self): 797 return self.target 798 799 def get_similar_targets(self): 800 """ 801 Get target sfor the same machine, task and target name 802 (e.g. 'core-image-minimal') from a successful build for this project 803 (but excluding this target). 804 805 Note that we only look for targets built by this project because 806 projects can have different configurations from each other, and put 807 their artifacts in different directories. 808 809 The possibility of error when retrieving candidate targets 810 is minimised by the fact that bitbake will rebuild artifacts if MACHINE 811 (or various other variables) change. In this case, there is no need to 812 clone artifacts from another target, as those artifacts will have 813 been re-generated for this target anyway. 814 """ 815 query = ~Q(pk=self.pk) & \ 816 Q(target=self.target) & \ 817 Q(build__machine=self.build.machine) & \ 818 Q(build__outcome=Build.SUCCEEDED) & \ 819 Q(build__project=self.build.project) 820 821 return Target.objects.filter(query) 822 823 def get_similar_target_with_image_files(self): 824 """ 825 Get the most recent similar target with Target_Image_Files associated 826 with it, for the purpose of cloning those files onto this target. 827 """ 828 similar_target = None 829 830 candidates = self.get_similar_targets() 831 if candidates.count() == 0: 832 return similar_target 833 834 task_subquery = Q(task=self.task) 835 836 # we can look for a 'build' task if this task is a 'populate_sdk_ext' 837 # task, as the latter also creates images; and vice versa; note that 838 # 'build' targets can have their task set to ''; 839 # also note that 'populate_sdk' does not produce image files 840 image_tasks = [ 841 '', # aka 'build' 842 'build', 843 'image', 844 'populate_sdk_ext' 845 ] 846 if self.task in image_tasks: 847 task_subquery = Q(task__in=image_tasks) 848 849 # annotate with the count of files, to exclude any targets which 850 # don't have associated files 851 candidates = candidates.annotate(num_files=Count('target_image_file')) 852 853 query = task_subquery & Q(num_files__gt=0) 854 855 candidates = candidates.filter(query) 856 857 if candidates.count() > 0: 858 candidates.order_by('build__completed_on') 859 similar_target = candidates.last() 860 861 return similar_target 862 863 def get_similar_target_with_sdk_files(self): 864 """ 865 Get the most recent similar target with TargetSDKFiles associated 866 with it, for the purpose of cloning those files onto this target. 867 """ 868 similar_target = None 869 870 candidates = self.get_similar_targets() 871 if candidates.count() == 0: 872 return similar_target 873 874 # annotate with the count of files, to exclude any targets which 875 # don't have associated files 876 candidates = candidates.annotate(num_files=Count('targetsdkfile')) 877 878 query = Q(task=self.task) & Q(num_files__gt=0) 879 880 candidates = candidates.filter(query) 881 882 if candidates.count() > 0: 883 candidates.order_by('build__completed_on') 884 similar_target = candidates.last() 885 886 return similar_target 887 888 def clone_image_artifacts_from(self, target): 889 """ 890 Make clones of the Target_Image_Files and TargetKernelFile objects 891 associated with Target target, then associate them with this target. 892 893 Note that for Target_Image_Files, we only want files from the previous 894 build whose suffix matches one of the suffixes defined in this 895 target's build's IMAGE_FSTYPES configuration variable. This prevents the 896 Target_Image_File object for an ext4 image being associated with a 897 target for a project which didn't produce an ext4 image (for example). 898 899 Also sets the license_manifest_path and package_manifest_path 900 of this target to the same path as that of target being cloned from, as 901 the manifests are also build artifacts but are treated differently. 902 """ 903 904 image_fstypes = self.build.get_image_fstypes() 905 906 # filter out any image files whose suffixes aren't in the 907 # IMAGE_FSTYPES suffixes variable for this target's build 908 image_files = [target_image_file \ 909 for target_image_file in target.target_image_file_set.all() \ 910 if target_image_file.suffix in image_fstypes] 911 912 for image_file in image_files: 913 image_file.pk = None 914 image_file.target = self 915 image_file.save() 916 917 kernel_files = target.targetkernelfile_set.all() 918 for kernel_file in kernel_files: 919 kernel_file.pk = None 920 kernel_file.target = self 921 kernel_file.save() 922 923 self.license_manifest_path = target.license_manifest_path 924 self.package_manifest_path = target.package_manifest_path 925 self.save() 926 927 def clone_sdk_artifacts_from(self, target): 928 """ 929 Clone TargetSDKFile objects from target and associate them with this 930 target. 931 """ 932 sdk_files = target.targetsdkfile_set.all() 933 for sdk_file in sdk_files: 934 sdk_file.pk = None 935 sdk_file.target = self 936 sdk_file.save() 937 938 def has_images(self): 939 """ 940 Returns True if this target has one or more image files attached to it. 941 """ 942 return self.target_image_file_set.all().count() > 0 943 944 # kernel artifacts for a target: bzImage and modules* 945 class TargetKernelFile(models.Model): 946 target = models.ForeignKey(Target, on_delete=models.CASCADE) 947 file_name = models.FilePathField() 948 file_size = models.IntegerField() 949 950 @property 951 def basename(self): 952 return os.path.basename(self.file_name) 953 954 # SDK artifacts for a target: sh and manifest files 955 class TargetSDKFile(models.Model): 956 target = models.ForeignKey(Target, on_delete=models.CASCADE) 957 file_name = models.FilePathField() 958 file_size = models.IntegerField() 959 960 @property 961 def basename(self): 962 return os.path.basename(self.file_name) 963 964 class Target_Image_File(models.Model): 965 # valid suffixes for image files produced by a build 966 SUFFIXES = { 967 'btrfs', 'container', 'cpio', 'cpio.gz', 'cpio.lz4', 'cpio.lzma', 968 'cpio.xz', 'cramfs', 'ext2', 'ext2.bz2', 'ext2.gz', 'ext2.lzma', 969 'ext3', 'ext3.gz', 'ext4', 'ext4.gz', 'f2fs', 'hddimg', 'iso', 'jffs2', 970 'jffs2.sum', 'multiubi', 'squashfs', 'squashfs-lz4', 'squashfs-lzo', 971 'squashfs-xz', 'tar', 'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi', 972 'ubifs', 'wic', 'wic.bz2', 'wic.gz', 'wic.lzma' 973 } 974 975 target = models.ForeignKey(Target, on_delete=models.CASCADE) 976 file_name = models.FilePathField(max_length=254) 977 file_size = models.IntegerField() 978 979 @property 980 def suffix(self): 981 """ 982 Suffix for image file, minus leading "." 983 """ 984 for suffix in Target_Image_File.SUFFIXES: 985 if self.file_name.endswith(suffix): 986 return suffix 987 988 filename, suffix = os.path.splitext(self.file_name) 989 suffix = suffix.lstrip('.') 990 return suffix 991 992 class Target_File(models.Model): 993 ITYPE_REGULAR = 1 994 ITYPE_DIRECTORY = 2 995 ITYPE_SYMLINK = 3 996 ITYPE_SOCKET = 4 997 ITYPE_FIFO = 5 998 ITYPE_CHARACTER = 6 999 ITYPE_BLOCK = 7 1000 ITYPES = ( (ITYPE_REGULAR ,'regular'), 1001 ( ITYPE_DIRECTORY ,'directory'), 1002 ( ITYPE_SYMLINK ,'symlink'), 1003 ( ITYPE_SOCKET ,'socket'), 1004 ( ITYPE_FIFO ,'fifo'), 1005 ( ITYPE_CHARACTER ,'character'), 1006 ( ITYPE_BLOCK ,'block'), 1007 ) 1008 1009 target = models.ForeignKey(Target, on_delete=models.CASCADE) 1010 path = models.FilePathField() 1011 size = models.IntegerField() 1012 inodetype = models.IntegerField(choices = ITYPES) 1013 permission = models.CharField(max_length=16) 1014 owner = models.CharField(max_length=128) 1015 group = models.CharField(max_length=128) 1016 directory = models.ForeignKey('Target_File', on_delete=models.CASCADE, related_name="directory_set", null=True) 1017 sym_target = models.ForeignKey('Target_File', on_delete=models.CASCADE, related_name="symlink_set", null=True) 1018 1019 1020 class Task(models.Model): 1021 1022 SSTATE_NA = 0 1023 SSTATE_MISS = 1 1024 SSTATE_FAILED = 2 1025 SSTATE_RESTORED = 3 1026 1027 SSTATE_RESULT = ( 1028 (SSTATE_NA, 'Not Applicable'), # For rest of tasks, but they still need checking. 1029 (SSTATE_MISS, 'File not in cache'), # the sstate object was not found 1030 (SSTATE_FAILED, 'Failed'), # there was a pkg, but the script failed 1031 (SSTATE_RESTORED, 'Succeeded'), # successfully restored 1032 ) 1033 1034 CODING_NA = 0 1035 CODING_PYTHON = 2 1036 CODING_SHELL = 3 1037 1038 TASK_CODING = ( 1039 (CODING_NA, 'N/A'), 1040 (CODING_PYTHON, 'Python'), 1041 (CODING_SHELL, 'Shell'), 1042 ) 1043 1044 OUTCOME_NA = -1 1045 OUTCOME_SUCCESS = 0 1046 OUTCOME_COVERED = 1 1047 OUTCOME_CACHED = 2 1048 OUTCOME_PREBUILT = 3 1049 OUTCOME_FAILED = 4 1050 OUTCOME_EMPTY = 5 1051 1052 TASK_OUTCOME = ( 1053 (OUTCOME_NA, 'Not Available'), 1054 (OUTCOME_SUCCESS, 'Succeeded'), 1055 (OUTCOME_COVERED, 'Covered'), 1056 (OUTCOME_CACHED, 'Cached'), 1057 (OUTCOME_PREBUILT, 'Prebuilt'), 1058 (OUTCOME_FAILED, 'Failed'), 1059 (OUTCOME_EMPTY, 'Empty'), 1060 ) 1061 1062 TASK_OUTCOME_HELP = ( 1063 (OUTCOME_SUCCESS, 'This task successfully completed'), 1064 (OUTCOME_COVERED, 'This task did not run because its output is provided by another task'), 1065 (OUTCOME_CACHED, 'This task restored output from the sstate-cache directory or mirrors'), 1066 (OUTCOME_PREBUILT, 'This task did not run because its outcome was reused from a previous build'), 1067 (OUTCOME_FAILED, 'This task did not complete'), 1068 (OUTCOME_EMPTY, 'This task has no executable content'), 1069 (OUTCOME_NA, ''), 1070 ) 1071 1072 search_allowed_fields = [ "recipe__name", "recipe__version", "task_name", "logfile" ] 1073 1074 def __init__(self, *args, **kwargs): 1075 super(Task, self).__init__(*args, **kwargs) 1076 try: 1077 self._helptext = HelpText.objects.get(key=self.task_name, area=HelpText.VARIABLE, build=self.build).text 1078 except HelpText.DoesNotExist: 1079 self._helptext = None 1080 1081 def get_related_setscene(self): 1082 return Task.objects.filter(task_executed=True, build = self.build, recipe = self.recipe, task_name=self.task_name+"_setscene") 1083 1084 def get_outcome_text(self): 1085 return Task.TASK_OUTCOME[int(self.outcome) + 1][1] 1086 1087 def get_outcome_help(self): 1088 return Task.TASK_OUTCOME_HELP[int(self.outcome)][1] 1089 1090 def get_sstate_text(self): 1091 if self.sstate_result==Task.SSTATE_NA: 1092 return '' 1093 else: 1094 return Task.SSTATE_RESULT[int(self.sstate_result)][1] 1095 1096 def get_executed_display(self): 1097 if self.task_executed: 1098 return "Executed" 1099 return "Not Executed" 1100 1101 def get_description(self): 1102 return self._helptext 1103 1104 build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='task_build') 1105 order = models.IntegerField(null=True) 1106 task_executed = models.BooleanField(default=False) # True means Executed, False means Not/Executed 1107 outcome = models.IntegerField(choices=TASK_OUTCOME, default=OUTCOME_NA) 1108 sstate_checksum = models.CharField(max_length=100, blank=True) 1109 path_to_sstate_obj = models.FilePathField(max_length=500, blank=True) 1110 recipe = models.ForeignKey('Recipe', on_delete=models.CASCADE, related_name='tasks') 1111 task_name = models.CharField(max_length=100) 1112 source_url = models.FilePathField(max_length=255, blank=True) 1113 work_directory = models.FilePathField(max_length=255, blank=True) 1114 script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NA) 1115 line_number = models.IntegerField(default=0) 1116 1117 # start/end times 1118 started = models.DateTimeField(null=True) 1119 ended = models.DateTimeField(null=True) 1120 1121 # in seconds; this is stored to enable sorting 1122 elapsed_time = models.DecimalField(max_digits=8, decimal_places=2, null=True) 1123 1124 # in bytes; note that disk_io is stored to enable sorting 1125 disk_io = models.IntegerField(null=True) 1126 disk_io_read = models.IntegerField(null=True) 1127 disk_io_write = models.IntegerField(null=True) 1128 1129 # in seconds 1130 cpu_time_user = models.DecimalField(max_digits=8, decimal_places=2, null=True) 1131 cpu_time_system = models.DecimalField(max_digits=8, decimal_places=2, null=True) 1132 1133 sstate_result = models.IntegerField(choices=SSTATE_RESULT, default=SSTATE_NA) 1134 message = models.CharField(max_length=240) 1135 logfile = models.FilePathField(max_length=255, blank=True) 1136 1137 outcome_text = property(get_outcome_text) 1138 sstate_text = property(get_sstate_text) 1139 1140 def __unicode__(self): 1141 return "%d(%d) %s:%s" % (self.pk, self.build.pk, self.recipe.name, self.task_name) 1142 1143 class Meta: 1144 ordering = ('order', 'recipe' ,) 1145 unique_together = ('build', 'recipe', 'task_name', ) 1146 1147 1148 class Task_Dependency(models.Model): 1149 task = models.ForeignKey(Task, on_delete=models.CASCADE, related_name='task_dependencies_task') 1150 depends_on = models.ForeignKey(Task, on_delete=models.CASCADE, related_name='task_dependencies_depends') 1151 1152 class Package(models.Model): 1153 search_allowed_fields = ['name', 'version', 'revision', 'recipe__name', 'recipe__version', 'recipe__license', 'recipe__layer_version__layer__name', 'recipe__layer_version__branch', 'recipe__layer_version__commit', 'recipe__layer_version__local_path', 'installed_name'] 1154 build = models.ForeignKey('Build', on_delete=models.CASCADE, null=True) 1155 recipe = models.ForeignKey('Recipe', on_delete=models.CASCADE, null=True) 1156 name = models.CharField(max_length=100) 1157 installed_name = models.CharField(max_length=100, default='') 1158 version = models.CharField(max_length=100, blank=True) 1159 revision = models.CharField(max_length=32, blank=True) 1160 summary = models.TextField(blank=True) 1161 description = models.TextField(blank=True) 1162 size = models.IntegerField(default=0) 1163 installed_size = models.IntegerField(default=0) 1164 section = models.CharField(max_length=80, blank=True) 1165 license = models.CharField(max_length=80, blank=True) 1166 1167 @property 1168 def is_locale_package(self): 1169 """ Returns True if this package is identifiable as a locale package """ 1170 if self.name.find('locale') != -1: 1171 return True 1172 return False 1173 1174 @property 1175 def is_packagegroup(self): 1176 """ Returns True is this package is identifiable as a packagegroup """ 1177 if self.name.find('packagegroup') != -1: 1178 return True 1179 return False 1180 1181 class CustomImagePackage(Package): 1182 # CustomImageRecipe fields to track pacakges appended, 1183 # included and excluded from a CustomImageRecipe 1184 recipe_includes = models.ManyToManyField('CustomImageRecipe', 1185 related_name='includes_set') 1186 recipe_excludes = models.ManyToManyField('CustomImageRecipe', 1187 related_name='excludes_set') 1188 recipe_appends = models.ManyToManyField('CustomImageRecipe', 1189 related_name='appends_set') 1190 1191 1192 class Package_DependencyManager(models.Manager): 1193 use_for_related_fields = True 1194 TARGET_LATEST = "use-latest-target-for-target" 1195 1196 def get_queryset(self): 1197 return super(Package_DependencyManager, self).get_queryset().exclude(package_id = F('depends_on__id')) 1198 1199 def for_target_or_none(self, target): 1200 """ filter the dependencies to be displayed by the supplied target 1201 if no dependences are found for the target then try None as the target 1202 which will return the dependences calculated without the context of a 1203 target e.g. non image recipes. 1204 1205 returns: { size, packages } 1206 """ 1207 package_dependencies = self.all_depends().order_by('depends_on__name') 1208 1209 if target is self.TARGET_LATEST: 1210 installed_deps =\ 1211 package_dependencies.filter(~Q(target__target=None)) 1212 else: 1213 installed_deps =\ 1214 package_dependencies.filter(Q(target__target=target)) 1215 1216 packages_list = None 1217 total_size = 0 1218 1219 # If we have installed depdencies for this package and target then use 1220 # these to display 1221 if installed_deps.count() > 0: 1222 packages_list = installed_deps 1223 total_size = installed_deps.aggregate( 1224 Sum('depends_on__size'))['depends_on__size__sum'] 1225 else: 1226 new_list = [] 1227 package_names = [] 1228 1229 # Find dependencies for the package that we know about even if 1230 # it's not installed on a target e.g. from a non-image recipe 1231 for p in package_dependencies.filter(Q(target=None)): 1232 if p.depends_on.name in package_names: 1233 continue 1234 else: 1235 package_names.append(p.depends_on.name) 1236 new_list.append(p.pk) 1237 # while we're here we may as well total up the size to 1238 # avoid iterating again 1239 total_size += p.depends_on.size 1240 1241 # We want to return a queryset here for consistency so pick the 1242 # deps from the new_list 1243 packages_list = package_dependencies.filter(Q(pk__in=new_list)) 1244 1245 return {'packages': packages_list, 1246 'size': total_size} 1247 1248 def all_depends(self): 1249 """ Returns just the depends packages and not any other dep_type 1250 Note that this is for any target 1251 """ 1252 return self.filter(Q(dep_type=Package_Dependency.TYPE_RDEPENDS) | 1253 Q(dep_type=Package_Dependency.TYPE_TRDEPENDS)) 1254 1255 1256 class Package_Dependency(models.Model): 1257 TYPE_RDEPENDS = 0 1258 TYPE_TRDEPENDS = 1 1259 TYPE_RRECOMMENDS = 2 1260 TYPE_TRECOMMENDS = 3 1261 TYPE_RSUGGESTS = 4 1262 TYPE_RPROVIDES = 5 1263 TYPE_RREPLACES = 6 1264 TYPE_RCONFLICTS = 7 1265 ' TODO: bpackage should be changed to remove the DEPENDS_TYPE access ' 1266 DEPENDS_TYPE = ( 1267 (TYPE_RDEPENDS, "depends"), 1268 (TYPE_TRDEPENDS, "depends"), 1269 (TYPE_TRECOMMENDS, "recommends"), 1270 (TYPE_RRECOMMENDS, "recommends"), 1271 (TYPE_RSUGGESTS, "suggests"), 1272 (TYPE_RPROVIDES, "provides"), 1273 (TYPE_RREPLACES, "replaces"), 1274 (TYPE_RCONFLICTS, "conflicts"), 1275 ) 1276 """ Indexed by dep_type, in view order, key for short name and help 1277 description which when viewed will be printf'd with the 1278 package name. 1279 """ 1280 DEPENDS_DICT = { 1281 TYPE_RDEPENDS : ("depends", "%s is required to run %s"), 1282 TYPE_TRDEPENDS : ("depends", "%s is required to run %s"), 1283 TYPE_TRECOMMENDS : ("recommends", "%s extends the usability of %s"), 1284 TYPE_RRECOMMENDS : ("recommends", "%s extends the usability of %s"), 1285 TYPE_RSUGGESTS : ("suggests", "%s is suggested for installation with %s"), 1286 TYPE_RPROVIDES : ("provides", "%s is provided by %s"), 1287 TYPE_RREPLACES : ("replaces", "%s is replaced by %s"), 1288 TYPE_RCONFLICTS : ("conflicts", "%s conflicts with %s, which will not be installed if this package is not first removed"), 1289 } 1290 1291 package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='package_dependencies_source') 1292 depends_on = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='package_dependencies_target') # soft dependency 1293 dep_type = models.IntegerField(choices=DEPENDS_TYPE) 1294 target = models.ForeignKey(Target, on_delete=models.CASCADE, null=True) 1295 objects = Package_DependencyManager() 1296 1297 class Target_Installed_Package(models.Model): 1298 target = models.ForeignKey(Target, on_delete=models.CASCADE) 1299 package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='buildtargetlist_package') 1300 1301 1302 class Package_File(models.Model): 1303 package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='buildfilelist_package') 1304 path = models.FilePathField(max_length=255, blank=True) 1305 size = models.IntegerField() 1306 1307 1308 class Recipe(models.Model): 1309 search_allowed_fields = ['name', 'version', 'file_path', 'section', 1310 'summary', 'description', 'license', 1311 'layer_version__layer__name', 1312 'layer_version__branch', 'layer_version__commit', 1313 'layer_version__local_path', 1314 'layer_version__layer_source'] 1315 1316 up_date = models.DateTimeField(null=True, default=None) 1317 1318 name = models.CharField(max_length=100, blank=True) 1319 version = models.CharField(max_length=100, blank=True) 1320 layer_version = models.ForeignKey('Layer_Version', on_delete=models.CASCADE, 1321 related_name='recipe_layer_version') 1322 summary = models.TextField(blank=True) 1323 description = models.TextField(blank=True) 1324 section = models.CharField(max_length=100, blank=True) 1325 license = models.CharField(max_length=200, blank=True) 1326 homepage = models.URLField(blank=True) 1327 bugtracker = models.URLField(blank=True) 1328 file_path = models.FilePathField(max_length=255) 1329 pathflags = models.CharField(max_length=200, blank=True) 1330 is_image = models.BooleanField(default=False) 1331 1332 def __unicode__(self): 1333 return "Recipe " + self.name + ":" + self.version 1334 1335 def get_vcs_recipe_file_link_url(self): 1336 return self.layer_version.get_vcs_file_link_url(self.file_path) 1337 1338 def get_description_or_summary(self): 1339 if self.description: 1340 return self.description 1341 elif self.summary: 1342 return self.summary 1343 else: 1344 return "" 1345 1346 class Meta: 1347 unique_together = (("layer_version", "file_path", "pathflags"), ) 1348 1349 1350 class Recipe_DependencyManager(models.Manager): 1351 use_for_related_fields = True 1352 1353 def get_queryset(self): 1354 return super(Recipe_DependencyManager, self).get_queryset().exclude(recipe_id = F('depends_on__id')) 1355 1356 class Provides(models.Model): 1357 name = models.CharField(max_length=100) 1358 recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE) 1359 1360 class Recipe_Dependency(models.Model): 1361 TYPE_DEPENDS = 0 1362 TYPE_RDEPENDS = 1 1363 1364 DEPENDS_TYPE = ( 1365 (TYPE_DEPENDS, "depends"), 1366 (TYPE_RDEPENDS, "rdepends"), 1367 ) 1368 recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE, related_name='r_dependencies_recipe') 1369 depends_on = models.ForeignKey(Recipe, on_delete=models.CASCADE, related_name='r_dependencies_depends') 1370 via = models.ForeignKey(Provides, on_delete=models.CASCADE, null=True, default=None) 1371 dep_type = models.IntegerField(choices=DEPENDS_TYPE) 1372 objects = Recipe_DependencyManager() 1373 1374 1375 class Machine(models.Model): 1376 search_allowed_fields = ["name", "description", "layer_version__layer__name"] 1377 up_date = models.DateTimeField(null = True, default = None) 1378 1379 layer_version = models.ForeignKey('Layer_Version', on_delete=models.CASCADE) 1380 name = models.CharField(max_length=255) 1381 description = models.CharField(max_length=255) 1382 1383 def get_vcs_machine_file_link_url(self): 1384 path = 'conf/machine/'+self.name+'.conf' 1385 1386 return self.layer_version.get_vcs_file_link_url(path) 1387 1388 def __unicode__(self): 1389 return "Machine " + self.name + "(" + self.description + ")" 1390 1391 1392 class BitbakeVersion(models.Model): 1393 1394 name = models.CharField(max_length=32, unique = True) 1395 giturl = GitURLField() 1396 branch = models.CharField(max_length=32) 1397 dirpath = models.CharField(max_length=255) 1398 1399 def __unicode__(self): 1400 return "%s (Branch: %s)" % (self.name, self.branch) 1401 1402 1403 class Release(models.Model): 1404 """ A release is a project template, used to pre-populate Project settings with a configuration set """ 1405 name = models.CharField(max_length=32, unique = True) 1406 description = models.CharField(max_length=255) 1407 bitbake_version = models.ForeignKey(BitbakeVersion, on_delete=models.CASCADE) 1408 branch_name = models.CharField(max_length=50, default = "") 1409 helptext = models.TextField(null=True) 1410 1411 def __unicode__(self): 1412 return "%s (%s)" % (self.name, self.branch_name) 1413 1414 def __str__(self): 1415 return self.name 1416 1417 class ReleaseDefaultLayer(models.Model): 1418 release = models.ForeignKey(Release, on_delete=models.CASCADE) 1419 layer_name = models.CharField(max_length=100, default="") 1420 1421 1422 class LayerSource(object): 1423 """ Where the layer metadata came from """ 1424 TYPE_LOCAL = 0 1425 TYPE_LAYERINDEX = 1 1426 TYPE_IMPORTED = 2 1427 TYPE_BUILD = 3 1428 1429 SOURCE_TYPE = ( 1430 (TYPE_LOCAL, "local"), 1431 (TYPE_LAYERINDEX, "layerindex"), 1432 (TYPE_IMPORTED, "imported"), 1433 (TYPE_BUILD, "build"), 1434 ) 1435 1436 def types_dict(): 1437 """ Turn the TYPES enums into a simple dictionary """ 1438 dictionary = {} 1439 for key in LayerSource.__dict__: 1440 if "TYPE" in key: 1441 dictionary[key] = getattr(LayerSource, key) 1442 return dictionary 1443 1444 1445 class Layer(models.Model): 1446 1447 up_date = models.DateTimeField(null=True, default=timezone.now) 1448 1449 name = models.CharField(max_length=100) 1450 layer_index_url = models.URLField() 1451 vcs_url = GitURLField(default=None, null=True) 1452 local_source_dir = models.TextField(null=True, default=None) 1453 vcs_web_url = models.URLField(null=True, default=None) 1454 vcs_web_tree_base_url = models.URLField(null=True, default=None) 1455 vcs_web_file_base_url = models.URLField(null=True, default=None) 1456 1457 summary = models.TextField(help_text='One-line description of the layer', 1458 null=True, default=None) 1459 description = models.TextField(null=True, default=None) 1460 1461 def __unicode__(self): 1462 return "%s / %s " % (self.name, self.summary) 1463 1464 1465 class Layer_Version(models.Model): 1466 """ 1467 A Layer_Version either belongs to a single project or no project 1468 """ 1469 search_allowed_fields = ["layer__name", "layer__summary", 1470 "layer__description", "layer__vcs_url", 1471 "dirpath", "release__name", "commit", "branch"] 1472 1473 build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='layer_version_build', 1474 default=None, null=True) 1475 1476 layer = models.ForeignKey(Layer, on_delete=models.CASCADE, related_name='layer_version_layer') 1477 1478 layer_source = models.IntegerField(choices=LayerSource.SOURCE_TYPE, 1479 default=0) 1480 1481 up_date = models.DateTimeField(null=True, default=timezone.now) 1482 1483 # To which metadata release does this layer version belong to 1484 release = models.ForeignKey(Release, on_delete=models.CASCADE, null=True, default=None) 1485 1486 branch = models.CharField(max_length=80) 1487 commit = models.CharField(max_length=100) 1488 # If the layer is in a subdir 1489 dirpath = models.CharField(max_length=255, null=True, default=None) 1490 1491 # if -1, this is a default layer 1492 priority = models.IntegerField(default=0) 1493 1494 # where this layer exists on the filesystem 1495 local_path = models.FilePathField(max_length=1024, default="/") 1496 1497 # Set if this layer is restricted to a particular project 1498 project = models.ForeignKey('Project', on_delete=models.CASCADE, null=True, default=None) 1499 1500 # code lifted, with adaptations, from the layerindex-web application 1501 # https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/ 1502 def _handle_url_path(self, base_url, path): 1503 import re, posixpath 1504 if base_url: 1505 if self.dirpath: 1506 if path: 1507 extra_path = self.dirpath + '/' + path 1508 # Normalise out ../ in path for usage URL 1509 extra_path = posixpath.normpath(extra_path) 1510 # Minor workaround to handle case where subdirectory has been added between branches 1511 # (should probably support usage URL per branch to handle this... sigh...) 1512 if extra_path.startswith('../'): 1513 extra_path = extra_path[3:] 1514 else: 1515 extra_path = self.dirpath 1516 else: 1517 extra_path = path 1518 branchname = self.release.name 1519 url = base_url.replace('%branch%', branchname) 1520 1521 # If there's a % in the path (e.g. a wildcard bbappend) we need to encode it 1522 if extra_path: 1523 extra_path = extra_path.replace('%', '%25') 1524 1525 if '%path%' in base_url: 1526 if extra_path: 1527 url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '\\1', url) 1528 else: 1529 url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '', url) 1530 return url.replace('%path%', extra_path) 1531 else: 1532 return url + extra_path 1533 return None 1534 1535 def get_vcs_link_url(self): 1536 if self.layer.vcs_web_url is None: 1537 return None 1538 return self.layer.vcs_web_url 1539 1540 def get_vcs_file_link_url(self, file_path=""): 1541 if self.layer.vcs_web_file_base_url is None: 1542 return None 1543 return self._handle_url_path(self.layer.vcs_web_file_base_url, 1544 file_path) 1545 1546 def get_vcs_dirpath_link_url(self): 1547 if self.layer.vcs_web_tree_base_url is None: 1548 return None 1549 return self._handle_url_path(self.layer.vcs_web_tree_base_url, '') 1550 1551 def get_vcs_reference(self): 1552 if self.commit is not None and len(self.commit) > 0: 1553 return self.commit 1554 if self.branch is not None and len(self.branch) > 0: 1555 return self.branch 1556 if self.release is not None: 1557 return self.release.name 1558 return 'N/A' 1559 1560 def get_detailspage_url(self, project_id=None): 1561 """ returns the url to the layer details page uses own project 1562 field if project_id is not specified """ 1563 1564 if project_id is None: 1565 project_id = self.project.pk 1566 1567 return reverse('layerdetails', args=(project_id, self.pk)) 1568 1569 def get_alldeps(self, project_id): 1570 """Get full list of unique layer dependencies.""" 1571 def gen_layerdeps(lver, project, depth): 1572 if depth == 0: 1573 return 1574 for ldep in lver.dependencies.all(): 1575 yield ldep.depends_on 1576 # get next level of deps recursively calling gen_layerdeps 1577 for subdep in gen_layerdeps(ldep.depends_on, project, depth-1): 1578 yield subdep 1579 1580 project = Project.objects.get(pk=project_id) 1581 result = [] 1582 projectlvers = [player.layercommit for player in 1583 project.projectlayer_set.all()] 1584 # protect against infinite layer dependency loops 1585 maxdepth = 20 1586 for dep in gen_layerdeps(self, project, maxdepth): 1587 # filter out duplicates and layers already belonging to the project 1588 if dep not in result + projectlvers: 1589 result.append(dep) 1590 1591 return sorted(result, key=lambda x: x.layer.name) 1592 1593 def __unicode__(self): 1594 return ("id %d belongs to layer: %s" % (self.pk, self.layer.name)) 1595 1596 def __str__(self): 1597 if self.release: 1598 release = self.release.name 1599 else: 1600 release = "No release set" 1601 1602 return "%d %s (%s)" % (self.pk, self.layer.name, release) 1603 1604 1605 class LayerVersionDependency(models.Model): 1606 1607 layer_version = models.ForeignKey(Layer_Version, on_delete=models.CASCADE, 1608 related_name="dependencies") 1609 depends_on = models.ForeignKey(Layer_Version, on_delete=models.CASCADE, 1610 related_name="dependees") 1611 1612 class ProjectLayer(models.Model): 1613 project = models.ForeignKey(Project, on_delete=models.CASCADE) 1614 layercommit = models.ForeignKey(Layer_Version, on_delete=models.CASCADE, null=True) 1615 optional = models.BooleanField(default = True) 1616 1617 def __unicode__(self): 1618 return "%s, %s" % (self.project.name, self.layercommit) 1619 1620 class Meta: 1621 unique_together = (("project", "layercommit"),) 1622 1623 class CustomImageRecipe(Recipe): 1624 1625 # CustomImageRecipe's belong to layers called: 1626 LAYER_NAME = "toaster-custom-images" 1627 1628 search_allowed_fields = ['name'] 1629 base_recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE, related_name='based_on_recipe') 1630 project = models.ForeignKey(Project, on_delete=models.CASCADE) 1631 last_updated = models.DateTimeField(null=True, default=None) 1632 1633 def get_last_successful_built_target(self): 1634 """ Return the last successful built target object if one exists 1635 otherwise return None """ 1636 return Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) & 1637 Q(build__project=self.project) & 1638 Q(target=self.name)).last() 1639 1640 def update_package_list(self): 1641 """ Update the package list from the last good build of this 1642 CustomImageRecipe 1643 """ 1644 # Check if we're aldready up-to-date or not 1645 target = self.get_last_successful_built_target() 1646 if target is None: 1647 # So we've never actually built this Custom recipe but what about 1648 # the recipe it's based on? 1649 target = \ 1650 Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) & 1651 Q(build__project=self.project) & 1652 Q(target=self.base_recipe.name)).last() 1653 if target is None: 1654 return 1655 1656 if target.build.completed_on == self.last_updated: 1657 return 1658 1659 self.includes_set.clear() 1660 1661 excludes_list = self.excludes_set.values_list('name', flat=True) 1662 appends_list = self.appends_set.values_list('name', flat=True) 1663 1664 built_packages_list = \ 1665 target.target_installed_package_set.values_list('package__name', 1666 flat=True) 1667 for built_package in built_packages_list: 1668 # Is the built package in the custom packages list? 1669 if built_package in excludes_list: 1670 continue 1671 1672 if built_package in appends_list: 1673 continue 1674 1675 cust_img_p = \ 1676 CustomImagePackage.objects.get(name=built_package) 1677 self.includes_set.add(cust_img_p) 1678 1679 1680 self.last_updated = target.build.completed_on 1681 self.save() 1682 1683 def get_all_packages(self): 1684 """Get the included packages and any appended packages""" 1685 self.update_package_list() 1686 1687 return CustomImagePackage.objects.filter((Q(recipe_appends=self) | 1688 Q(recipe_includes=self)) & 1689 ~Q(recipe_excludes=self)) 1690 1691 def get_base_recipe_file(self): 1692 """Get the base recipe file path if it exists on the file system""" 1693 path_schema_one = "%s/%s" % (self.base_recipe.layer_version.local_path, 1694 self.base_recipe.file_path) 1695 1696 path_schema_two = self.base_recipe.file_path 1697 1698 path_schema_three = "%s/%s" % (self.base_recipe.layer_version.layer.local_source_dir, 1699 self.base_recipe.file_path) 1700 1701 if os.path.exists(path_schema_one): 1702 return path_schema_one 1703 1704 # The path may now be the full path if the recipe has been built 1705 if os.path.exists(path_schema_two): 1706 return path_schema_two 1707 1708 # Or a local path if all layers are local 1709 if os.path.exists(path_schema_three): 1710 return path_schema_three 1711 1712 return None 1713 1714 def generate_recipe_file_contents(self): 1715 """Generate the contents for the recipe file.""" 1716 # If we have no excluded packages we only need to :append 1717 if self.excludes_set.count() == 0: 1718 packages_conf = "IMAGE_INSTALL:append = \" " 1719 1720 for pkg in self.appends_set.all(): 1721 packages_conf += pkg.name+' ' 1722 else: 1723 packages_conf = "IMAGE_FEATURES =\"\"\nIMAGE_INSTALL = \"" 1724 # We add all the known packages to be built by this recipe apart 1725 # from locale packages which are are controlled with IMAGE_LINGUAS. 1726 for pkg in self.get_all_packages().exclude( 1727 name__icontains="locale"): 1728 packages_conf += pkg.name+' ' 1729 1730 packages_conf += "\"" 1731 1732 base_recipe_path = self.get_base_recipe_file() 1733 if base_recipe_path and os.path.isfile(base_recipe_path): 1734 base_recipe = open(base_recipe_path, 'r').read() 1735 else: 1736 # Pass back None to trigger error message to user 1737 return None 1738 1739 # Add a special case for when the recipe we have based a custom image 1740 # recipe on requires another recipe. 1741 # For example: 1742 # "require core-image-minimal.bb" is changed to: 1743 # "require recipes-core/images/core-image-minimal.bb" 1744 1745 req_search = re.search(r'(require\s+)(.+\.bb\s*$)', 1746 base_recipe, 1747 re.MULTILINE) 1748 if req_search: 1749 require_filename = req_search.group(2).strip() 1750 1751 corrected_location = Recipe.objects.filter( 1752 Q(layer_version=self.base_recipe.layer_version) & 1753 Q(file_path__icontains=require_filename)).last().file_path 1754 1755 new_require_line = "require %s" % corrected_location 1756 1757 base_recipe = base_recipe.replace(req_search.group(0), 1758 new_require_line) 1759 1760 info = { 1761 "date": timezone.now().strftime("%Y-%m-%d %H:%M:%S"), 1762 "base_recipe": base_recipe, 1763 "recipe_name": self.name, 1764 "base_recipe_name": self.base_recipe.name, 1765 "license": self.license, 1766 "summary": self.summary, 1767 "description": self.description, 1768 "packages_conf": packages_conf.strip() 1769 } 1770 1771 recipe_contents = ("# Original recipe %(base_recipe_name)s \n" 1772 "%(base_recipe)s\n\n" 1773 "# Recipe %(recipe_name)s \n" 1774 "# Customisation Generated by Toaster on %(date)s\n" 1775 "SUMMARY = \"%(summary)s\"\n" 1776 "DESCRIPTION = \"%(description)s\"\n" 1777 "LICENSE = \"%(license)s\"\n" 1778 "%(packages_conf)s") % info 1779 1780 return recipe_contents 1781 1782 class ProjectVariable(models.Model): 1783 project = models.ForeignKey(Project, on_delete=models.CASCADE) 1784 name = models.CharField(max_length=100) 1785 value = models.TextField(blank = True) 1786 1787 class Variable(models.Model): 1788 search_allowed_fields = ['variable_name', 'variable_value', 1789 'vhistory__file_name', "description"] 1790 build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='variable_build') 1791 variable_name = models.CharField(max_length=100) 1792 variable_value = models.TextField(blank=True) 1793 changed = models.BooleanField(default=False) 1794 human_readable_name = models.CharField(max_length=200) 1795 description = models.TextField(blank=True) 1796 1797 class VariableHistory(models.Model): 1798 variable = models.ForeignKey(Variable, on_delete=models.CASCADE, related_name='vhistory') 1799 value = models.TextField(blank=True) 1800 file_name = models.FilePathField(max_length=255) 1801 line_number = models.IntegerField(null=True) 1802 operation = models.CharField(max_length=64) 1803 1804 class HelpText(models.Model): 1805 VARIABLE = 0 1806 HELPTEXT_AREA = ((VARIABLE, 'variable'), ) 1807 1808 build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='helptext_build') 1809 area = models.IntegerField(choices=HELPTEXT_AREA) 1810 key = models.CharField(max_length=100) 1811 text = models.TextField() 1812 1813 class LogMessage(models.Model): 1814 EXCEPTION = -1 # used to signal self-toaster-exceptions 1815 INFO = 0 1816 WARNING = 1 1817 ERROR = 2 1818 CRITICAL = 3 1819 1820 LOG_LEVEL = ( 1821 (INFO, "info"), 1822 (WARNING, "warn"), 1823 (ERROR, "error"), 1824 (CRITICAL, "critical"), 1825 (EXCEPTION, "toaster exception") 1826 ) 1827 1828 build = models.ForeignKey(Build, on_delete=models.CASCADE) 1829 task = models.ForeignKey(Task, on_delete=models.CASCADE, blank = True, null=True) 1830 level = models.IntegerField(choices=LOG_LEVEL, default=INFO) 1831 message = models.TextField(blank=True, null=True) 1832 pathname = models.FilePathField(max_length=255, blank=True) 1833 lineno = models.IntegerField(null=True) 1834 1835 def __str__(self): 1836 return force_bytes('%s %s %s' % (self.get_level_display(), self.message, self.build)) 1837 1838 def invalidate_cache(**kwargs): 1839 from django.core.cache import cache 1840 try: 1841 cache.clear() 1842 except Exception as e: 1843 logger.warning("Problem with cache backend: Failed to clear cache: %s" % e) 1844 1845 def signal_runbuilds(): 1846 """Send SIGUSR1 to runbuilds process""" 1847 try: 1848 with open(os.path.join(os.getenv('BUILDDIR', '.'), 1849 '.runbuilds.pid')) as pidf: 1850 os.kill(int(pidf.read()), SIGUSR1) 1851 except FileNotFoundError: 1852 logger.info("Stopping existing runbuilds: no current process found") 1853 except ProcessLookupError: 1854 logger.warning("Stopping existing runbuilds: process lookup not found") 1855 1856 class Distro(models.Model): 1857 search_allowed_fields = ["name", "description", "layer_version__layer__name"] 1858 up_date = models.DateTimeField(null = True, default = None) 1859 1860 layer_version = models.ForeignKey('Layer_Version', on_delete=models.CASCADE) 1861 name = models.CharField(max_length=255) 1862 description = models.CharField(max_length=255) 1863 1864 def get_vcs_distro_file_link_url(self): 1865 path = 'conf/distro/%s.conf' % self.name 1866 return self.layer_version.get_vcs_file_link_url(path) 1867 1868 def __unicode__(self): 1869 return "Distro " + self.name + "(" + self.description + ")" 1870 1871 class EventLogsImports(models.Model): 1872 name = models.CharField(max_length=255) 1873 imported = models.BooleanField(default=False) 1874 build_id = models.IntegerField(blank=True, null=True) 1875 1876 def __str__(self): 1877 return self.name 1878 1879 1880 django.db.models.signals.post_save.connect(invalidate_cache) 1881 django.db.models.signals.post_delete.connect(invalidate_cache) 1882 django.db.models.signals.m2m_changed.connect(invalidate_cache) 1883