1# 2# BitBake Toaster Implementation 3# 4# Copyright (C) 2013 Intel Corporation 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9from __future__ import unicode_literals 10 11from django.db import models, IntegrityError, DataError 12from django.db.models import F, Q, Sum, Count 13from django.utils import timezone 14from django.utils.encoding import force_bytes 15 16from django.urls import reverse 17 18from django.core import validators 19from django.conf import settings 20import django.db.models.signals 21 22import sys 23import os 24import re 25import itertools 26from signal import SIGUSR1 27 28 29import logging 30logger = logging.getLogger("toaster") 31 32if 'sqlite' in settings.DATABASES['default']['ENGINE']: 33 from django.db import transaction, OperationalError 34 from time import sleep 35 36 _base_save = models.Model.save 37 def save(self, *args, **kwargs): 38 while True: 39 try: 40 with transaction.atomic(): 41 return _base_save(self, *args, **kwargs) 42 except OperationalError as err: 43 if 'database is locked' in str(err): 44 logger.warning("%s, model: %s, args: %s, kwargs: %s", 45 err, self.__class__, args, kwargs) 46 sleep(0.5) 47 continue 48 raise 49 50 models.Model.save = save 51 52 # HACK: Monkey patch Django to fix 'database is locked' issue 53 54 from django.db.models.query import QuerySet 55 _base_insert = QuerySet._insert 56 def _insert(self, *args, **kwargs): 57 with transaction.atomic(using=self.db, savepoint=False): 58 return _base_insert(self, *args, **kwargs) 59 QuerySet._insert = _insert 60 61 def _create_object_from_params(self, lookup, params): 62 """ 63 Tries to create an object using passed params. 64 Used by get_or_create and update_or_create 65 """ 66 try: 67 obj = self.create(**params) 68 return obj, True 69 except (IntegrityError, DataError): 70 exc_info = sys.exc_info() 71 try: 72 return self.get(**lookup), False 73 except self.model.DoesNotExist: 74 pass 75 six.reraise(*exc_info) 76 77 QuerySet._create_object_from_params = _create_object_from_params 78 79 # end of HACK 80 81class GitURLValidator(validators.URLValidator): 82 regex = re.compile( 83 r'^(?:ssh|git|http|ftp)s?://' # http:// or https:// 84 r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... 85 r'localhost|' # localhost... 86 r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4 87 r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6 88 r'(?::\d+)?' # optional port 89 r'(?:/?|[/?]\S+)$', re.IGNORECASE) 90 91def GitURLField(**kwargs): 92 r = models.URLField(**kwargs) 93 for i in range(len(r.validators)): 94 if isinstance(r.validators[i], validators.URLValidator): 95 r.validators[i] = GitURLValidator() 96 return r 97 98 99class ToasterSetting(models.Model): 100 name = models.CharField(max_length=63) 101 helptext = models.TextField() 102 value = models.CharField(max_length=255) 103 104 def __unicode__(self): 105 return "Setting %s = %s" % (self.name, self.value) 106 107 108class ProjectManager(models.Manager): 109 def create_project(self, name, release, existing_project=None, imported=False): 110 if existing_project and (release is not None): 111 prj = existing_project 112 prj.bitbake_version = release.bitbake_version 113 prj.release = release 114 # Delete the previous ProjectLayer mappings 115 for pl in ProjectLayer.objects.filter(project=prj): 116 pl.delete() 117 elif release is not None: 118 prj = self.model(name=name, 119 bitbake_version=release.bitbake_version, 120 release=release) 121 else: 122 prj = self.model(name=name, 123 bitbake_version=None, 124 release=None) 125 prj.save() 126 127 for defaultconf in ToasterSetting.objects.filter( 128 name__startswith="DEFCONF_"): 129 name = defaultconf.name[8:] 130 pv,create = ProjectVariable.objects.get_or_create(project=prj,name=name) 131 pv.value = defaultconf.value 132 pv.save() 133 134 if release is None: 135 return prj 136 if not imported: 137 for rdl in release.releasedefaultlayer_set.all(): 138 lv = Layer_Version.objects.filter( 139 layer__name=rdl.layer_name, 140 release=release).first() 141 142 if lv: 143 ProjectLayer.objects.create(project=prj, 144 layercommit=lv, 145 optional=False) 146 else: 147 logger.warning("Default project layer %s not found" % 148 rdl.layer_name) 149 150 return prj 151 152 # return single object with is_default = True 153 def get_or_create_default_project(self): 154 projects = super(ProjectManager, self).filter(is_default=True) 155 156 if len(projects) > 1: 157 raise Exception('Inconsistent project data: multiple ' + 158 'default projects (i.e. with is_default=True)') 159 elif len(projects) < 1: 160 options = { 161 'name': 'Command line builds', 162 'short_description': 163 'Project for builds started outside Toaster', 164 'is_default': True 165 } 166 project = Project.objects.create(**options) 167 project.save() 168 169 return project 170 else: 171 return projects[0] 172 173 174class Project(models.Model): 175 search_allowed_fields = ['name', 'short_description', 'release__name', 176 'release__branch_name'] 177 name = models.CharField(max_length=100) 178 short_description = models.CharField(max_length=50, blank=True) 179 bitbake_version = models.ForeignKey('BitbakeVersion', on_delete=models.CASCADE, null=True) 180 release = models.ForeignKey("Release", on_delete=models.CASCADE, null=True) 181 created = models.DateTimeField(auto_now_add=True) 182 updated = models.DateTimeField(auto_now=True) 183 # This is a horrible hack; since Toaster has no "User" model available when 184 # running in interactive mode, we can't reference the field here directly 185 # Instead, we keep a possible null reference to the User id, 186 # as not to force 187 # hard links to possibly missing models 188 user_id = models.IntegerField(null=True) 189 objects = ProjectManager() 190 191 # build directory override (e.g. imported) 192 builddir = models.TextField() 193 # merge the Toaster configure attributes directly into the standard conf files 194 merged_attr = models.BooleanField(default=False) 195 196 # set to True for the project which is the default container 197 # for builds initiated by the command line etc. 198 is_default= models.BooleanField(default=False) 199 200 def __unicode__(self): 201 return "%s (Release %s, BBV %s)" % (self.name, self.release, self.bitbake_version) 202 203 def get_current_machine_name(self): 204 try: 205 return self.projectvariable_set.get(name="MACHINE").value 206 except (ProjectVariable.DoesNotExist,IndexError): 207 return None; 208 209 def get_number_of_builds(self): 210 """Return the number of builds which have ended""" 211 212 return self.build_set.exclude( 213 Q(outcome=Build.IN_PROGRESS) | 214 Q(outcome=Build.CANCELLED) 215 ).count() 216 217 def get_last_build_id(self): 218 try: 219 return Build.objects.filter( project = self.id ).order_by('-completed_on')[0].id 220 except (Build.DoesNotExist,IndexError): 221 return( -1 ) 222 223 def get_last_outcome(self): 224 build_id = self.get_last_build_id() 225 if (-1 == build_id): 226 return( "" ) 227 try: 228 return Build.objects.filter( id = build_id )[ 0 ].outcome 229 except (Build.DoesNotExist,IndexError): 230 return( "not_found" ) 231 232 def get_last_target(self): 233 build_id = self.get_last_build_id() 234 if (-1 == build_id): 235 return( "" ) 236 try: 237 return Target.objects.filter(build = build_id)[0].target 238 except (Target.DoesNotExist,IndexError): 239 return( "not_found" ) 240 241 def get_last_errors(self): 242 build_id = self.get_last_build_id() 243 if (-1 == build_id): 244 return( 0 ) 245 try: 246 return Build.objects.filter(id = build_id)[ 0 ].errors.count() 247 except (Build.DoesNotExist,IndexError): 248 return( "not_found" ) 249 250 def get_last_warnings(self): 251 build_id = self.get_last_build_id() 252 if (-1 == build_id): 253 return( 0 ) 254 try: 255 return Build.objects.filter(id = build_id)[ 0 ].warnings.count() 256 except (Build.DoesNotExist,IndexError): 257 return( "not_found" ) 258 259 def get_last_build_extensions(self): 260 """ 261 Get list of file name extensions for images produced by the most 262 recent build 263 """ 264 last_build = Build.objects.get(pk = self.get_last_build_id()) 265 return last_build.get_image_file_extensions() 266 267 def get_last_imgfiles(self): 268 build_id = self.get_last_build_id() 269 if (-1 == build_id): 270 return( "" ) 271 try: 272 return Variable.objects.filter(build = build_id, variable_name = "IMAGE_FSTYPES")[ 0 ].variable_value 273 except (Variable.DoesNotExist,IndexError): 274 return( "not_found" ) 275 276 def get_all_compatible_layer_versions(self): 277 """ Returns Queryset of all Layer_Versions which are compatible with 278 this project""" 279 queryset = None 280 281 # guard on release, as it can be null 282 if self.release: 283 queryset = Layer_Version.objects.filter( 284 (Q(release=self.release) & 285 Q(build=None) & 286 Q(project=None)) | 287 Q(project=self)) 288 else: 289 queryset = Layer_Version.objects.none() 290 291 return queryset 292 293 def get_project_layer_versions(self, pk=False): 294 """ Returns the Layer_Versions currently added to this project """ 295 layer_versions = self.projectlayer_set.all().values_list('layercommit', 296 flat=True) 297 298 if pk is False: 299 return Layer_Version.objects.filter(pk__in=layer_versions) 300 else: 301 return layer_versions 302 303 304 def get_default_image_recipe(self): 305 try: 306 return self.projectvariable_set.get(name="DEFAULT_IMAGE").value 307 except (ProjectVariable.DoesNotExist,IndexError): 308 return None; 309 310 def get_is_new(self): 311 return self.get_variable(Project.PROJECT_SPECIFIC_ISNEW) 312 313 def get_available_machines(self): 314 """ Returns QuerySet of all Machines which are provided by the 315 Layers currently added to the Project """ 316 queryset = Machine.objects.filter( 317 layer_version__in=self.get_project_layer_versions()) 318 319 return queryset 320 321 def get_all_compatible_machines(self): 322 """ Returns QuerySet of all the compatible machines available to the 323 project including ones from Layers not currently added """ 324 queryset = Machine.objects.filter( 325 layer_version__in=self.get_all_compatible_layer_versions()) 326 327 return queryset 328 329 def get_available_distros(self): 330 """ Returns QuerySet of all Distros which are provided by the 331 Layers currently added to the Project """ 332 queryset = Distro.objects.filter( 333 layer_version__in=self.get_project_layer_versions()) 334 335 return queryset 336 337 def get_all_compatible_distros(self): 338 """ Returns QuerySet of all the compatible Wind River distros available to the 339 project including ones from Layers not currently added """ 340 queryset = Distro.objects.filter( 341 layer_version__in=self.get_all_compatible_layer_versions()) 342 343 return queryset 344 345 def get_available_recipes(self): 346 """ Returns QuerySet of all the recipes that are provided by layers 347 added to this project """ 348 queryset = Recipe.objects.filter( 349 layer_version__in=self.get_project_layer_versions()) 350 351 return queryset 352 353 def get_all_compatible_recipes(self): 354 """ Returns QuerySet of all the compatible Recipes available to the 355 project including ones from Layers not currently added """ 356 queryset = Recipe.objects.filter( 357 layer_version__in=self.get_all_compatible_layer_versions()).exclude(name__exact='') 358 359 return queryset 360 361 # Project Specific status management 362 PROJECT_SPECIFIC_STATUS = 'INTERNAL_PROJECT_SPECIFIC_STATUS' 363 PROJECT_SPECIFIC_CALLBACK = 'INTERNAL_PROJECT_SPECIFIC_CALLBACK' 364 PROJECT_SPECIFIC_ISNEW = 'INTERNAL_PROJECT_SPECIFIC_ISNEW' 365 PROJECT_SPECIFIC_DEFAULTIMAGE = 'PROJECT_SPECIFIC_DEFAULTIMAGE' 366 PROJECT_SPECIFIC_NONE = '' 367 PROJECT_SPECIFIC_NEW = '1' 368 PROJECT_SPECIFIC_EDIT = '2' 369 PROJECT_SPECIFIC_CLONING = '3' 370 PROJECT_SPECIFIC_CLONING_SUCCESS = '4' 371 PROJECT_SPECIFIC_CLONING_FAIL = '5' 372 373 def get_variable(self,variable,default_value = ''): 374 try: 375 return self.projectvariable_set.get(name=variable).value 376 except (ProjectVariable.DoesNotExist,IndexError): 377 return default_value 378 379 def set_variable(self,variable,value): 380 pv,create = ProjectVariable.objects.get_or_create(project = self, name = variable) 381 pv.value = value 382 pv.save() 383 384 def get_default_image(self): 385 return self.get_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE) 386 387 def schedule_build(self): 388 389 from bldcontrol.models import BuildRequest, BRTarget, BRLayer 390 from bldcontrol.models import BRBitbake, BRVariable 391 392 try: 393 now = timezone.now() 394 build = Build.objects.create(project=self, 395 completed_on=now, 396 started_on=now) 397 398 br = BuildRequest.objects.create(project=self, 399 state=BuildRequest.REQ_QUEUED, 400 build=build) 401 BRBitbake.objects.create(req=br, 402 giturl=self.bitbake_version.giturl, 403 commit=self.bitbake_version.branch, 404 dirpath=self.bitbake_version.dirpath) 405 406 for t in self.projecttarget_set.all(): 407 BRTarget.objects.create(req=br, target=t.target, task=t.task) 408 Target.objects.create(build=br.build, target=t.target, 409 task=t.task) 410 # If we're about to build a custom image recipe make sure 411 # that layer is currently in the project before we create the 412 # BRLayer objects 413 customrecipe = CustomImageRecipe.objects.filter( 414 name=t.target, 415 project=self).first() 416 if customrecipe: 417 ProjectLayer.objects.get_or_create( 418 project=self, 419 layercommit=customrecipe.layer_version, 420 optional=False) 421 422 for l in self.projectlayer_set.all().order_by("pk"): 423 commit = l.layercommit.get_vcs_reference() 424 logger.debug("Adding layer to build %s" % 425 l.layercommit.layer.name) 426 BRLayer.objects.create( 427 req=br, 428 name=l.layercommit.layer.name, 429 giturl=l.layercommit.layer.vcs_url, 430 commit=commit, 431 dirpath=l.layercommit.dirpath, 432 layer_version=l.layercommit, 433 local_source_dir=l.layercommit.layer.local_source_dir 434 ) 435 436 for v in self.projectvariable_set.all(): 437 BRVariable.objects.create(req=br, name=v.name, value=v.value) 438 439 try: 440 br.build.machine = self.projectvariable_set.get( 441 name='MACHINE').value 442 br.build.save() 443 except ProjectVariable.DoesNotExist: 444 pass 445 446 br.save() 447 signal_runbuilds() 448 449 except Exception: 450 # revert the build request creation since we're not done cleanly 451 br.delete() 452 raise 453 return br 454 455class Build(models.Model): 456 SUCCEEDED = 0 457 FAILED = 1 458 IN_PROGRESS = 2 459 CANCELLED = 3 460 461 BUILD_OUTCOME = ( 462 (SUCCEEDED, 'Succeeded'), 463 (FAILED, 'Failed'), 464 (IN_PROGRESS, 'In Progress'), 465 (CANCELLED, 'Cancelled'), 466 ) 467 468 search_allowed_fields = ['machine', 'cooker_log_path', "target__target", "target__target_image_file__file_name"] 469 470 project = models.ForeignKey(Project, on_delete=models.CASCADE) # must have a project 471 machine = models.CharField(max_length=100) 472 distro = models.CharField(max_length=100) 473 distro_version = models.CharField(max_length=100) 474 started_on = models.DateTimeField() 475 completed_on = models.DateTimeField() 476 outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS) 477 cooker_log_path = models.CharField(max_length=500) 478 build_name = models.CharField(max_length=100, default='') 479 bitbake_version = models.CharField(max_length=50) 480 481 # number of recipes to parse for this build 482 recipes_to_parse = models.IntegerField(default=1) 483 484 # number of recipes parsed so far for this build 485 recipes_parsed = models.IntegerField(default=1) 486 487 # number of repos to clone for this build 488 repos_to_clone = models.IntegerField(default=1) 489 490 # number of repos cloned so far for this build (default off) 491 repos_cloned = models.IntegerField(default=1) 492 493 # Hint on current progress item 494 progress_item = models.CharField(max_length=40) 495 496 @staticmethod 497 def get_recent(project=None): 498 """ 499 Return recent builds as a list; if project is set, only return 500 builds for that project 501 """ 502 503 builds = Build.objects.all() 504 505 if project: 506 builds = builds.filter(project=project) 507 508 finished_criteria = \ 509 Q(outcome=Build.SUCCEEDED) | \ 510 Q(outcome=Build.FAILED) | \ 511 Q(outcome=Build.CANCELLED) 512 513 recent_builds = list(itertools.chain( 514 builds.filter(outcome=Build.IN_PROGRESS).order_by("-started_on"), 515 builds.filter(finished_criteria).order_by("-completed_on")[:3] 516 )) 517 518 # add percentage done property to each build; this is used 519 # to show build progress in mrb_section.html 520 for build in recent_builds: 521 build.percentDone = build.completeper() 522 build.outcomeText = build.get_outcome_text() 523 524 return recent_builds 525 526 def started(self): 527 """ 528 As build variables are only added for a build when its BuildStarted event 529 is received, a build with no build variables is counted as 530 "in preparation" and not properly started yet. This method 531 will return False if a build has no build variables (it never properly 532 started), or True otherwise. 533 534 Note that this is a temporary workaround for the fact that we don't 535 have a fine-grained state variable on a build which would allow us 536 to record "in progress" (BuildStarted received) vs. "in preparation". 537 """ 538 variables = Variable.objects.filter(build=self) 539 return len(variables) > 0 540 541 def completeper(self): 542 tf = Task.objects.filter(build = self) 543 tfc = tf.count() 544 if tfc > 0: 545 completeper = tf.exclude(outcome=Task.OUTCOME_NA).count()*100 // tfc 546 else: 547 completeper = 0 548 return completeper 549 550 def eta(self): 551 eta = timezone.now() 552 completeper = self.completeper() 553 if self.completeper() > 0: 554 eta += ((eta - self.started_on)*(100-completeper))/completeper 555 return eta 556 557 def has_images(self): 558 """ 559 Returns True if at least one of the targets for this build has an 560 image file associated with it, False otherwise 561 """ 562 targets = Target.objects.filter(build_id=self.id) 563 has_images = False 564 for target in targets: 565 if target.has_images(): 566 has_images = True 567 break 568 return has_images 569 570 def has_image_recipes(self): 571 """ 572 Returns True if a build has any targets which were built from 573 image recipes. 574 """ 575 image_recipes = self.get_image_recipes() 576 return len(image_recipes) > 0 577 578 def get_image_file_extensions(self): 579 """ 580 Get string of file name extensions for images produced by this build; 581 note that this is the actual list of extensions stored on Target objects 582 for this build, and not the value of IMAGE_FSTYPES. 583 584 Returns comma-separated string, e.g. "vmdk, ext4" 585 """ 586 extensions = [] 587 588 targets = Target.objects.filter(build_id = self.id) 589 for target in targets: 590 if not target.is_image: 591 continue 592 593 target_image_files = Target_Image_File.objects.filter( 594 target_id=target.id) 595 596 for target_image_file in target_image_files: 597 extensions.append(target_image_file.suffix) 598 599 extensions = list(set(extensions)) 600 extensions.sort() 601 602 return ', '.join(extensions) 603 604 def get_image_fstypes(self): 605 """ 606 Get the IMAGE_FSTYPES variable value for this build as a de-duplicated 607 list of image file suffixes. 608 """ 609 image_fstypes = Variable.objects.get( 610 build=self, variable_name='IMAGE_FSTYPES').variable_value 611 return list(set(re.split(r' {1,}', image_fstypes))) 612 613 def get_sorted_target_list(self): 614 tgts = Target.objects.filter(build_id = self.id).order_by( 'target' ); 615 return( tgts ); 616 617 def get_recipes(self): 618 """ 619 Get the recipes related to this build; 620 note that the related layer versions and layers are also prefetched 621 by this query, as this queryset can be sorted by these objects in the 622 build recipes view; prefetching them here removes the need 623 for another query in that view 624 """ 625 layer_versions = Layer_Version.objects.filter(build=self) 626 criteria = Q(layer_version__id__in=layer_versions) 627 return Recipe.objects.filter(criteria) \ 628 .select_related('layer_version', 'layer_version__layer') 629 630 def get_image_recipes(self): 631 """ 632 Returns a list of image Recipes (custom and built-in) related to this 633 build, sorted by name; note that this has to be done in two steps, as 634 there's no way to get all the custom image recipes and image recipes 635 in one query 636 """ 637 custom_image_recipes = self.get_custom_image_recipes() 638 custom_image_recipe_names = custom_image_recipes.values_list('name', flat=True) 639 640 not_custom_image_recipes = ~Q(name__in=custom_image_recipe_names) & \ 641 Q(is_image=True) 642 643 built_image_recipes = self.get_recipes().filter(not_custom_image_recipes) 644 645 # append to the custom image recipes and sort 646 customisable_image_recipes = list( 647 itertools.chain(custom_image_recipes, built_image_recipes) 648 ) 649 650 return sorted(customisable_image_recipes, key=lambda recipe: recipe.name) 651 652 def get_custom_image_recipes(self): 653 """ 654 Returns a queryset of CustomImageRecipes related to this build, 655 sorted by name 656 """ 657 built_recipe_names = self.get_recipes().values_list('name', flat=True) 658 criteria = Q(name__in=built_recipe_names) & Q(project=self.project) 659 queryset = CustomImageRecipe.objects.filter(criteria).order_by('name') 660 return queryset 661 662 def get_outcome_text(self): 663 return Build.BUILD_OUTCOME[int(self.outcome)][1] 664 665 @property 666 def failed_tasks(self): 667 """ Get failed tasks for the build """ 668 tasks = self.task_build.all() 669 return tasks.filter(order__gt=0, outcome=Task.OUTCOME_FAILED) 670 671 @property 672 def errors(self): 673 return (self.logmessage_set.filter(level=LogMessage.ERROR) | 674 self.logmessage_set.filter(level=LogMessage.EXCEPTION) | 675 self.logmessage_set.filter(level=LogMessage.CRITICAL)) 676 677 @property 678 def warnings(self): 679 return self.logmessage_set.filter(level=LogMessage.WARNING) 680 681 @property 682 def timespent(self): 683 return self.completed_on - self.started_on 684 685 @property 686 def timespent_seconds(self): 687 return self.timespent.total_seconds() 688 689 @property 690 def target_labels(self): 691 """ 692 Sorted (a-z) "target1:task, target2, target3" etc. string for all 693 targets in this build 694 """ 695 targets = self.target_set.all() 696 target_labels = [target.target + 697 (':' + target.task if target.task else '') 698 for target in targets] 699 target_labels.sort() 700 701 return target_labels 702 703 def get_buildrequest(self): 704 buildrequest = None 705 if hasattr(self, 'buildrequest'): 706 buildrequest = self.buildrequest 707 return buildrequest 708 709 def is_queued(self): 710 from bldcontrol.models import BuildRequest 711 buildrequest = self.get_buildrequest() 712 if buildrequest: 713 return buildrequest.state == BuildRequest.REQ_QUEUED 714 else: 715 return False 716 717 def is_cancelling(self): 718 from bldcontrol.models import BuildRequest 719 buildrequest = self.get_buildrequest() 720 if buildrequest: 721 return self.outcome == Build.IN_PROGRESS and \ 722 buildrequest.state == BuildRequest.REQ_CANCELLING 723 else: 724 return False 725 726 def is_cloning(self): 727 """ 728 True if the build is still cloning repos 729 """ 730 return self.outcome == Build.IN_PROGRESS and \ 731 self.repos_cloned < self.repos_to_clone 732 733 def is_parsing(self): 734 """ 735 True if the build is still parsing recipes 736 """ 737 return self.outcome == Build.IN_PROGRESS and \ 738 self.recipes_parsed < self.recipes_to_parse 739 740 def is_starting(self): 741 """ 742 True if the build has no completed tasks yet and is still just starting 743 tasks. 744 745 Note that the mechanism for testing whether a Task is "done" is whether 746 its outcome field is set, as per the completeper() method. 747 """ 748 return self.outcome == Build.IN_PROGRESS and \ 749 self.task_build.exclude(outcome=Task.OUTCOME_NA).count() == 0 750 751 752 def get_state(self): 753 """ 754 Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress', 755 'Cancelled' (Build outcomes); or 'Queued', 'Cancelling' (states 756 dependent on the BuildRequest state). 757 758 This works around the fact that we have BuildRequest states as well 759 as Build states, but really we just want to know the state of the build. 760 """ 761 if self.is_cancelling(): 762 return 'Cancelling'; 763 elif self.is_queued(): 764 return 'Queued' 765 elif self.is_cloning(): 766 return 'Cloning' 767 elif self.is_parsing(): 768 return 'Parsing' 769 elif self.is_starting(): 770 return 'Starting' 771 else: 772 return self.get_outcome_text() 773 774 def __str__(self): 775 return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()])) 776 777class ProjectTarget(models.Model): 778 project = models.ForeignKey(Project, on_delete=models.CASCADE) 779 target = models.CharField(max_length=100) 780 task = models.CharField(max_length=100, null=True) 781 782class Target(models.Model): 783 search_allowed_fields = ['target', 'file_name'] 784 build = models.ForeignKey(Build, on_delete=models.CASCADE) 785 target = models.CharField(max_length=100) 786 task = models.CharField(max_length=100, null=True) 787 is_image = models.BooleanField(default = False) 788 image_size = models.IntegerField(default=0) 789 license_manifest_path = models.CharField(max_length=500, null=True) 790 package_manifest_path = models.CharField(max_length=500, null=True) 791 792 def package_count(self): 793 return Target_Installed_Package.objects.filter(target_id__exact=self.id).count() 794 795 def __unicode__(self): 796 return self.target 797 798 def get_similar_targets(self): 799 """ 800 Get target sfor the same machine, task and target name 801 (e.g. 'core-image-minimal') from a successful build for this project 802 (but excluding this target). 803 804 Note that we only look for targets built by this project because 805 projects can have different configurations from each other, and put 806 their artifacts in different directories. 807 808 The possibility of error when retrieving candidate targets 809 is minimised by the fact that bitbake will rebuild artifacts if MACHINE 810 (or various other variables) change. In this case, there is no need to 811 clone artifacts from another target, as those artifacts will have 812 been re-generated for this target anyway. 813 """ 814 query = ~Q(pk=self.pk) & \ 815 Q(target=self.target) & \ 816 Q(build__machine=self.build.machine) & \ 817 Q(build__outcome=Build.SUCCEEDED) & \ 818 Q(build__project=self.build.project) 819 820 return Target.objects.filter(query) 821 822 def get_similar_target_with_image_files(self): 823 """ 824 Get the most recent similar target with Target_Image_Files associated 825 with it, for the purpose of cloning those files onto this target. 826 """ 827 similar_target = None 828 829 candidates = self.get_similar_targets() 830 if candidates.count() == 0: 831 return similar_target 832 833 task_subquery = Q(task=self.task) 834 835 # we can look for a 'build' task if this task is a 'populate_sdk_ext' 836 # task, as the latter also creates images; and vice versa; note that 837 # 'build' targets can have their task set to ''; 838 # also note that 'populate_sdk' does not produce image files 839 image_tasks = [ 840 '', # aka 'build' 841 'build', 842 'image', 843 'populate_sdk_ext' 844 ] 845 if self.task in image_tasks: 846 task_subquery = Q(task__in=image_tasks) 847 848 # annotate with the count of files, to exclude any targets which 849 # don't have associated files 850 candidates = candidates.annotate(num_files=Count('target_image_file')) 851 852 query = task_subquery & Q(num_files__gt=0) 853 854 candidates = candidates.filter(query) 855 856 if candidates.count() > 0: 857 candidates.order_by('build__completed_on') 858 similar_target = candidates.last() 859 860 return similar_target 861 862 def get_similar_target_with_sdk_files(self): 863 """ 864 Get the most recent similar target with TargetSDKFiles associated 865 with it, for the purpose of cloning those files onto this target. 866 """ 867 similar_target = None 868 869 candidates = self.get_similar_targets() 870 if candidates.count() == 0: 871 return similar_target 872 873 # annotate with the count of files, to exclude any targets which 874 # don't have associated files 875 candidates = candidates.annotate(num_files=Count('targetsdkfile')) 876 877 query = Q(task=self.task) & Q(num_files__gt=0) 878 879 candidates = candidates.filter(query) 880 881 if candidates.count() > 0: 882 candidates.order_by('build__completed_on') 883 similar_target = candidates.last() 884 885 return similar_target 886 887 def clone_image_artifacts_from(self, target): 888 """ 889 Make clones of the Target_Image_Files and TargetKernelFile objects 890 associated with Target target, then associate them with this target. 891 892 Note that for Target_Image_Files, we only want files from the previous 893 build whose suffix matches one of the suffixes defined in this 894 target's build's IMAGE_FSTYPES configuration variable. This prevents the 895 Target_Image_File object for an ext4 image being associated with a 896 target for a project which didn't produce an ext4 image (for example). 897 898 Also sets the license_manifest_path and package_manifest_path 899 of this target to the same path as that of target being cloned from, as 900 the manifests are also build artifacts but are treated differently. 901 """ 902 903 image_fstypes = self.build.get_image_fstypes() 904 905 # filter out any image files whose suffixes aren't in the 906 # IMAGE_FSTYPES suffixes variable for this target's build 907 image_files = [target_image_file \ 908 for target_image_file in target.target_image_file_set.all() \ 909 if target_image_file.suffix in image_fstypes] 910 911 for image_file in image_files: 912 image_file.pk = None 913 image_file.target = self 914 image_file.save() 915 916 kernel_files = target.targetkernelfile_set.all() 917 for kernel_file in kernel_files: 918 kernel_file.pk = None 919 kernel_file.target = self 920 kernel_file.save() 921 922 self.license_manifest_path = target.license_manifest_path 923 self.package_manifest_path = target.package_manifest_path 924 self.save() 925 926 def clone_sdk_artifacts_from(self, target): 927 """ 928 Clone TargetSDKFile objects from target and associate them with this 929 target. 930 """ 931 sdk_files = target.targetsdkfile_set.all() 932 for sdk_file in sdk_files: 933 sdk_file.pk = None 934 sdk_file.target = self 935 sdk_file.save() 936 937 def has_images(self): 938 """ 939 Returns True if this target has one or more image files attached to it. 940 """ 941 return self.target_image_file_set.all().count() > 0 942 943# kernel artifacts for a target: bzImage and modules* 944class TargetKernelFile(models.Model): 945 target = models.ForeignKey(Target, on_delete=models.CASCADE) 946 file_name = models.FilePathField() 947 file_size = models.IntegerField() 948 949 @property 950 def basename(self): 951 return os.path.basename(self.file_name) 952 953# SDK artifacts for a target: sh and manifest files 954class TargetSDKFile(models.Model): 955 target = models.ForeignKey(Target, on_delete=models.CASCADE) 956 file_name = models.FilePathField() 957 file_size = models.IntegerField() 958 959 @property 960 def basename(self): 961 return os.path.basename(self.file_name) 962 963class Target_Image_File(models.Model): 964 # valid suffixes for image files produced by a build 965 SUFFIXES = { 966 'btrfs', 'container', 'cpio', 'cpio.gz', 'cpio.lz4', 'cpio.lzma', 967 'cpio.xz', 'cramfs', 'ext2', 'ext2.bz2', 'ext2.gz', 'ext2.lzma', 968 'ext3', 'ext3.gz', 'ext4', 'ext4.gz', 'f2fs', 'hddimg', 'iso', 'jffs2', 969 'jffs2.sum', 'multiubi', 'squashfs', 'squashfs-lz4', 'squashfs-lzo', 970 'squashfs-xz', 'tar', 'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi', 971 'ubifs', 'wic', 'wic.bz2', 'wic.gz', 'wic.lzma' 972 } 973 974 target = models.ForeignKey(Target, on_delete=models.CASCADE) 975 file_name = models.FilePathField(max_length=254) 976 file_size = models.IntegerField() 977 978 @property 979 def suffix(self): 980 """ 981 Suffix for image file, minus leading "." 982 """ 983 for suffix in Target_Image_File.SUFFIXES: 984 if self.file_name.endswith(suffix): 985 return suffix 986 987 filename, suffix = os.path.splitext(self.file_name) 988 suffix = suffix.lstrip('.') 989 return suffix 990 991class Target_File(models.Model): 992 ITYPE_REGULAR = 1 993 ITYPE_DIRECTORY = 2 994 ITYPE_SYMLINK = 3 995 ITYPE_SOCKET = 4 996 ITYPE_FIFO = 5 997 ITYPE_CHARACTER = 6 998 ITYPE_BLOCK = 7 999 ITYPES = ( (ITYPE_REGULAR ,'regular'), 1000 ( ITYPE_DIRECTORY ,'directory'), 1001 ( ITYPE_SYMLINK ,'symlink'), 1002 ( ITYPE_SOCKET ,'socket'), 1003 ( ITYPE_FIFO ,'fifo'), 1004 ( ITYPE_CHARACTER ,'character'), 1005 ( ITYPE_BLOCK ,'block'), 1006 ) 1007 1008 target = models.ForeignKey(Target, on_delete=models.CASCADE) 1009 path = models.FilePathField() 1010 size = models.IntegerField() 1011 inodetype = models.IntegerField(choices = ITYPES) 1012 permission = models.CharField(max_length=16) 1013 owner = models.CharField(max_length=128) 1014 group = models.CharField(max_length=128) 1015 directory = models.ForeignKey('Target_File', on_delete=models.CASCADE, related_name="directory_set", null=True) 1016 sym_target = models.ForeignKey('Target_File', on_delete=models.CASCADE, related_name="symlink_set", null=True) 1017 1018 1019class Task(models.Model): 1020 1021 SSTATE_NA = 0 1022 SSTATE_MISS = 1 1023 SSTATE_FAILED = 2 1024 SSTATE_RESTORED = 3 1025 1026 SSTATE_RESULT = ( 1027 (SSTATE_NA, 'Not Applicable'), # For rest of tasks, but they still need checking. 1028 (SSTATE_MISS, 'File not in cache'), # the sstate object was not found 1029 (SSTATE_FAILED, 'Failed'), # there was a pkg, but the script failed 1030 (SSTATE_RESTORED, 'Succeeded'), # successfully restored 1031 ) 1032 1033 CODING_NA = 0 1034 CODING_PYTHON = 2 1035 CODING_SHELL = 3 1036 1037 TASK_CODING = ( 1038 (CODING_NA, 'N/A'), 1039 (CODING_PYTHON, 'Python'), 1040 (CODING_SHELL, 'Shell'), 1041 ) 1042 1043 OUTCOME_NA = -1 1044 OUTCOME_SUCCESS = 0 1045 OUTCOME_COVERED = 1 1046 OUTCOME_CACHED = 2 1047 OUTCOME_PREBUILT = 3 1048 OUTCOME_FAILED = 4 1049 OUTCOME_EMPTY = 5 1050 1051 TASK_OUTCOME = ( 1052 (OUTCOME_NA, 'Not Available'), 1053 (OUTCOME_SUCCESS, 'Succeeded'), 1054 (OUTCOME_COVERED, 'Covered'), 1055 (OUTCOME_CACHED, 'Cached'), 1056 (OUTCOME_PREBUILT, 'Prebuilt'), 1057 (OUTCOME_FAILED, 'Failed'), 1058 (OUTCOME_EMPTY, 'Empty'), 1059 ) 1060 1061 TASK_OUTCOME_HELP = ( 1062 (OUTCOME_SUCCESS, 'This task successfully completed'), 1063 (OUTCOME_COVERED, 'This task did not run because its output is provided by another task'), 1064 (OUTCOME_CACHED, 'This task restored output from the sstate-cache directory or mirrors'), 1065 (OUTCOME_PREBUILT, 'This task did not run because its outcome was reused from a previous build'), 1066 (OUTCOME_FAILED, 'This task did not complete'), 1067 (OUTCOME_EMPTY, 'This task has no executable content'), 1068 (OUTCOME_NA, ''), 1069 ) 1070 1071 search_allowed_fields = [ "recipe__name", "recipe__version", "task_name", "logfile" ] 1072 1073 def __init__(self, *args, **kwargs): 1074 super(Task, self).__init__(*args, **kwargs) 1075 try: 1076 self._helptext = HelpText.objects.get(key=self.task_name, area=HelpText.VARIABLE, build=self.build).text 1077 except HelpText.DoesNotExist: 1078 self._helptext = None 1079 1080 def get_related_setscene(self): 1081 return Task.objects.filter(task_executed=True, build = self.build, recipe = self.recipe, task_name=self.task_name+"_setscene") 1082 1083 def get_outcome_text(self): 1084 return Task.TASK_OUTCOME[int(self.outcome) + 1][1] 1085 1086 def get_outcome_help(self): 1087 return Task.TASK_OUTCOME_HELP[int(self.outcome)][1] 1088 1089 def get_sstate_text(self): 1090 if self.sstate_result==Task.SSTATE_NA: 1091 return '' 1092 else: 1093 return Task.SSTATE_RESULT[int(self.sstate_result)][1] 1094 1095 def get_executed_display(self): 1096 if self.task_executed: 1097 return "Executed" 1098 return "Not Executed" 1099 1100 def get_description(self): 1101 return self._helptext 1102 1103 build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='task_build') 1104 order = models.IntegerField(null=True) 1105 task_executed = models.BooleanField(default=False) # True means Executed, False means Not/Executed 1106 outcome = models.IntegerField(choices=TASK_OUTCOME, default=OUTCOME_NA) 1107 sstate_checksum = models.CharField(max_length=100, blank=True) 1108 path_to_sstate_obj = models.FilePathField(max_length=500, blank=True) 1109 recipe = models.ForeignKey('Recipe', on_delete=models.CASCADE, related_name='tasks') 1110 task_name = models.CharField(max_length=100) 1111 source_url = models.FilePathField(max_length=255, blank=True) 1112 work_directory = models.FilePathField(max_length=255, blank=True) 1113 script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NA) 1114 line_number = models.IntegerField(default=0) 1115 1116 # start/end times 1117 started = models.DateTimeField(null=True) 1118 ended = models.DateTimeField(null=True) 1119 1120 # in seconds; this is stored to enable sorting 1121 elapsed_time = models.DecimalField(max_digits=8, decimal_places=2, null=True) 1122 1123 # in bytes; note that disk_io is stored to enable sorting 1124 disk_io = models.IntegerField(null=True) 1125 disk_io_read = models.IntegerField(null=True) 1126 disk_io_write = models.IntegerField(null=True) 1127 1128 # in seconds 1129 cpu_time_user = models.DecimalField(max_digits=8, decimal_places=2, null=True) 1130 cpu_time_system = models.DecimalField(max_digits=8, decimal_places=2, null=True) 1131 1132 sstate_result = models.IntegerField(choices=SSTATE_RESULT, default=SSTATE_NA) 1133 message = models.CharField(max_length=240) 1134 logfile = models.FilePathField(max_length=255, blank=True) 1135 1136 outcome_text = property(get_outcome_text) 1137 sstate_text = property(get_sstate_text) 1138 1139 def __unicode__(self): 1140 return "%d(%d) %s:%s" % (self.pk, self.build.pk, self.recipe.name, self.task_name) 1141 1142 class Meta: 1143 ordering = ('order', 'recipe' ,) 1144 unique_together = ('build', 'recipe', 'task_name', ) 1145 1146 1147class Task_Dependency(models.Model): 1148 task = models.ForeignKey(Task, on_delete=models.CASCADE, related_name='task_dependencies_task') 1149 depends_on = models.ForeignKey(Task, on_delete=models.CASCADE, related_name='task_dependencies_depends') 1150 1151class Package(models.Model): 1152 search_allowed_fields = ['name', 'version', 'revision', 'recipe__name', 'recipe__version', 'recipe__license', 'recipe__layer_version__layer__name', 'recipe__layer_version__branch', 'recipe__layer_version__commit', 'recipe__layer_version__local_path', 'installed_name'] 1153 build = models.ForeignKey('Build', on_delete=models.CASCADE, null=True) 1154 recipe = models.ForeignKey('Recipe', on_delete=models.CASCADE, null=True) 1155 name = models.CharField(max_length=100) 1156 installed_name = models.CharField(max_length=100, default='') 1157 version = models.CharField(max_length=100, blank=True) 1158 revision = models.CharField(max_length=32, blank=True) 1159 summary = models.TextField(blank=True) 1160 description = models.TextField(blank=True) 1161 size = models.IntegerField(default=0) 1162 installed_size = models.IntegerField(default=0) 1163 section = models.CharField(max_length=80, blank=True) 1164 license = models.CharField(max_length=80, blank=True) 1165 1166 @property 1167 def is_locale_package(self): 1168 """ Returns True if this package is identifiable as a locale package """ 1169 if self.name.find('locale') != -1: 1170 return True 1171 return False 1172 1173 @property 1174 def is_packagegroup(self): 1175 """ Returns True is this package is identifiable as a packagegroup """ 1176 if self.name.find('packagegroup') != -1: 1177 return True 1178 return False 1179 1180class CustomImagePackage(Package): 1181 # CustomImageRecipe fields to track pacakges appended, 1182 # included and excluded from a CustomImageRecipe 1183 recipe_includes = models.ManyToManyField('CustomImageRecipe', 1184 related_name='includes_set') 1185 recipe_excludes = models.ManyToManyField('CustomImageRecipe', 1186 related_name='excludes_set') 1187 recipe_appends = models.ManyToManyField('CustomImageRecipe', 1188 related_name='appends_set') 1189 1190 1191class Package_DependencyManager(models.Manager): 1192 use_for_related_fields = True 1193 TARGET_LATEST = "use-latest-target-for-target" 1194 1195 def get_queryset(self): 1196 return super(Package_DependencyManager, self).get_queryset().exclude(package_id = F('depends_on__id')) 1197 1198 def for_target_or_none(self, target): 1199 """ filter the dependencies to be displayed by the supplied target 1200 if no dependences are found for the target then try None as the target 1201 which will return the dependences calculated without the context of a 1202 target e.g. non image recipes. 1203 1204 returns: { size, packages } 1205 """ 1206 package_dependencies = self.all_depends().order_by('depends_on__name') 1207 1208 if target is self.TARGET_LATEST: 1209 installed_deps =\ 1210 package_dependencies.filter(~Q(target__target=None)) 1211 else: 1212 installed_deps =\ 1213 package_dependencies.filter(Q(target__target=target)) 1214 1215 packages_list = None 1216 total_size = 0 1217 1218 # If we have installed depdencies for this package and target then use 1219 # these to display 1220 if installed_deps.count() > 0: 1221 packages_list = installed_deps 1222 total_size = installed_deps.aggregate( 1223 Sum('depends_on__size'))['depends_on__size__sum'] 1224 else: 1225 new_list = [] 1226 package_names = [] 1227 1228 # Find dependencies for the package that we know about even if 1229 # it's not installed on a target e.g. from a non-image recipe 1230 for p in package_dependencies.filter(Q(target=None)): 1231 if p.depends_on.name in package_names: 1232 continue 1233 else: 1234 package_names.append(p.depends_on.name) 1235 new_list.append(p.pk) 1236 # while we're here we may as well total up the size to 1237 # avoid iterating again 1238 total_size += p.depends_on.size 1239 1240 # We want to return a queryset here for consistency so pick the 1241 # deps from the new_list 1242 packages_list = package_dependencies.filter(Q(pk__in=new_list)) 1243 1244 return {'packages': packages_list, 1245 'size': total_size} 1246 1247 def all_depends(self): 1248 """ Returns just the depends packages and not any other dep_type 1249 Note that this is for any target 1250 """ 1251 return self.filter(Q(dep_type=Package_Dependency.TYPE_RDEPENDS) | 1252 Q(dep_type=Package_Dependency.TYPE_TRDEPENDS)) 1253 1254 1255class Package_Dependency(models.Model): 1256 TYPE_RDEPENDS = 0 1257 TYPE_TRDEPENDS = 1 1258 TYPE_RRECOMMENDS = 2 1259 TYPE_TRECOMMENDS = 3 1260 TYPE_RSUGGESTS = 4 1261 TYPE_RPROVIDES = 5 1262 TYPE_RREPLACES = 6 1263 TYPE_RCONFLICTS = 7 1264 ' TODO: bpackage should be changed to remove the DEPENDS_TYPE access ' 1265 DEPENDS_TYPE = ( 1266 (TYPE_RDEPENDS, "depends"), 1267 (TYPE_TRDEPENDS, "depends"), 1268 (TYPE_TRECOMMENDS, "recommends"), 1269 (TYPE_RRECOMMENDS, "recommends"), 1270 (TYPE_RSUGGESTS, "suggests"), 1271 (TYPE_RPROVIDES, "provides"), 1272 (TYPE_RREPLACES, "replaces"), 1273 (TYPE_RCONFLICTS, "conflicts"), 1274 ) 1275 """ Indexed by dep_type, in view order, key for short name and help 1276 description which when viewed will be printf'd with the 1277 package name. 1278 """ 1279 DEPENDS_DICT = { 1280 TYPE_RDEPENDS : ("depends", "%s is required to run %s"), 1281 TYPE_TRDEPENDS : ("depends", "%s is required to run %s"), 1282 TYPE_TRECOMMENDS : ("recommends", "%s extends the usability of %s"), 1283 TYPE_RRECOMMENDS : ("recommends", "%s extends the usability of %s"), 1284 TYPE_RSUGGESTS : ("suggests", "%s is suggested for installation with %s"), 1285 TYPE_RPROVIDES : ("provides", "%s is provided by %s"), 1286 TYPE_RREPLACES : ("replaces", "%s is replaced by %s"), 1287 TYPE_RCONFLICTS : ("conflicts", "%s conflicts with %s, which will not be installed if this package is not first removed"), 1288 } 1289 1290 package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='package_dependencies_source') 1291 depends_on = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='package_dependencies_target') # soft dependency 1292 dep_type = models.IntegerField(choices=DEPENDS_TYPE) 1293 target = models.ForeignKey(Target, on_delete=models.CASCADE, null=True) 1294 objects = Package_DependencyManager() 1295 1296class Target_Installed_Package(models.Model): 1297 target = models.ForeignKey(Target, on_delete=models.CASCADE) 1298 package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='buildtargetlist_package') 1299 1300 1301class Package_File(models.Model): 1302 package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='buildfilelist_package') 1303 path = models.FilePathField(max_length=255, blank=True) 1304 size = models.IntegerField() 1305 1306 1307class Recipe(models.Model): 1308 search_allowed_fields = ['name', 'version', 'file_path', 'section', 1309 'summary', 'description', 'license', 1310 'layer_version__layer__name', 1311 'layer_version__branch', 'layer_version__commit', 1312 'layer_version__local_path', 1313 'layer_version__layer_source'] 1314 1315 up_date = models.DateTimeField(null=True, default=None) 1316 1317 name = models.CharField(max_length=100, blank=True) 1318 version = models.CharField(max_length=100, blank=True) 1319 layer_version = models.ForeignKey('Layer_Version', on_delete=models.CASCADE, 1320 related_name='recipe_layer_version') 1321 summary = models.TextField(blank=True) 1322 description = models.TextField(blank=True) 1323 section = models.CharField(max_length=100, blank=True) 1324 license = models.CharField(max_length=200, blank=True) 1325 homepage = models.URLField(blank=True) 1326 bugtracker = models.URLField(blank=True) 1327 file_path = models.FilePathField(max_length=255) 1328 pathflags = models.CharField(max_length=200, blank=True) 1329 is_image = models.BooleanField(default=False) 1330 1331 def __unicode__(self): 1332 return "Recipe " + self.name + ":" + self.version 1333 1334 def get_vcs_recipe_file_link_url(self): 1335 return self.layer_version.get_vcs_file_link_url(self.file_path) 1336 1337 def get_description_or_summary(self): 1338 if self.description: 1339 return self.description 1340 elif self.summary: 1341 return self.summary 1342 else: 1343 return "" 1344 1345 class Meta: 1346 unique_together = (("layer_version", "file_path", "pathflags"), ) 1347 1348 1349class Recipe_DependencyManager(models.Manager): 1350 use_for_related_fields = True 1351 1352 def get_queryset(self): 1353 return super(Recipe_DependencyManager, self).get_queryset().exclude(recipe_id = F('depends_on__id')) 1354 1355class Provides(models.Model): 1356 name = models.CharField(max_length=100) 1357 recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE) 1358 1359class Recipe_Dependency(models.Model): 1360 TYPE_DEPENDS = 0 1361 TYPE_RDEPENDS = 1 1362 1363 DEPENDS_TYPE = ( 1364 (TYPE_DEPENDS, "depends"), 1365 (TYPE_RDEPENDS, "rdepends"), 1366 ) 1367 recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE, related_name='r_dependencies_recipe') 1368 depends_on = models.ForeignKey(Recipe, on_delete=models.CASCADE, related_name='r_dependencies_depends') 1369 via = models.ForeignKey(Provides, on_delete=models.CASCADE, null=True, default=None) 1370 dep_type = models.IntegerField(choices=DEPENDS_TYPE) 1371 objects = Recipe_DependencyManager() 1372 1373 1374class Machine(models.Model): 1375 search_allowed_fields = ["name", "description", "layer_version__layer__name"] 1376 up_date = models.DateTimeField(null = True, default = None) 1377 1378 layer_version = models.ForeignKey('Layer_Version', on_delete=models.CASCADE) 1379 name = models.CharField(max_length=255) 1380 description = models.CharField(max_length=255) 1381 1382 def get_vcs_machine_file_link_url(self): 1383 path = 'conf/machine/'+self.name+'.conf' 1384 1385 return self.layer_version.get_vcs_file_link_url(path) 1386 1387 def __unicode__(self): 1388 return "Machine " + self.name + "(" + self.description + ")" 1389 1390 1391class BitbakeVersion(models.Model): 1392 1393 name = models.CharField(max_length=32, unique = True) 1394 giturl = GitURLField() 1395 branch = models.CharField(max_length=32) 1396 dirpath = models.CharField(max_length=255) 1397 1398 def __unicode__(self): 1399 return "%s (Branch: %s)" % (self.name, self.branch) 1400 1401 1402class Release(models.Model): 1403 """ A release is a project template, used to pre-populate Project settings with a configuration set """ 1404 name = models.CharField(max_length=32, unique = True) 1405 description = models.CharField(max_length=255) 1406 bitbake_version = models.ForeignKey(BitbakeVersion, on_delete=models.CASCADE) 1407 branch_name = models.CharField(max_length=50, default = "") 1408 helptext = models.TextField(null=True) 1409 1410 def __unicode__(self): 1411 return "%s (%s)" % (self.name, self.branch_name) 1412 1413 def __str__(self): 1414 return self.name 1415 1416class ReleaseDefaultLayer(models.Model): 1417 release = models.ForeignKey(Release, on_delete=models.CASCADE) 1418 layer_name = models.CharField(max_length=100, default="") 1419 1420 1421class LayerSource(object): 1422 """ Where the layer metadata came from """ 1423 TYPE_LOCAL = 0 1424 TYPE_LAYERINDEX = 1 1425 TYPE_IMPORTED = 2 1426 TYPE_BUILD = 3 1427 1428 SOURCE_TYPE = ( 1429 (TYPE_LOCAL, "local"), 1430 (TYPE_LAYERINDEX, "layerindex"), 1431 (TYPE_IMPORTED, "imported"), 1432 (TYPE_BUILD, "build"), 1433 ) 1434 1435 def types_dict(): 1436 """ Turn the TYPES enums into a simple dictionary """ 1437 dictionary = {} 1438 for key in LayerSource.__dict__: 1439 if "TYPE" in key: 1440 dictionary[key] = getattr(LayerSource, key) 1441 return dictionary 1442 1443 1444class Layer(models.Model): 1445 1446 up_date = models.DateTimeField(null=True, default=timezone.now) 1447 1448 name = models.CharField(max_length=100) 1449 layer_index_url = models.URLField() 1450 vcs_url = GitURLField(default=None, null=True) 1451 local_source_dir = models.TextField(null=True, default=None) 1452 vcs_web_url = models.URLField(null=True, default=None) 1453 vcs_web_tree_base_url = models.URLField(null=True, default=None) 1454 vcs_web_file_base_url = models.URLField(null=True, default=None) 1455 1456 summary = models.TextField(help_text='One-line description of the layer', 1457 null=True, default=None) 1458 description = models.TextField(null=True, default=None) 1459 1460 def __unicode__(self): 1461 return "%s / %s " % (self.name, self.summary) 1462 1463 1464class Layer_Version(models.Model): 1465 """ 1466 A Layer_Version either belongs to a single project or no project 1467 """ 1468 search_allowed_fields = ["layer__name", "layer__summary", 1469 "layer__description", "layer__vcs_url", 1470 "dirpath", "release__name", "commit", "branch"] 1471 1472 build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='layer_version_build', 1473 default=None, null=True) 1474 1475 layer = models.ForeignKey(Layer, on_delete=models.CASCADE, related_name='layer_version_layer') 1476 1477 layer_source = models.IntegerField(choices=LayerSource.SOURCE_TYPE, 1478 default=0) 1479 1480 up_date = models.DateTimeField(null=True, default=timezone.now) 1481 1482 # To which metadata release does this layer version belong to 1483 release = models.ForeignKey(Release, on_delete=models.CASCADE, null=True, default=None) 1484 1485 branch = models.CharField(max_length=80) 1486 commit = models.CharField(max_length=100) 1487 # If the layer is in a subdir 1488 dirpath = models.CharField(max_length=255, null=True, default=None) 1489 1490 # if -1, this is a default layer 1491 priority = models.IntegerField(default=0) 1492 1493 # where this layer exists on the filesystem 1494 local_path = models.FilePathField(max_length=1024, default="/") 1495 1496 # Set if this layer is restricted to a particular project 1497 project = models.ForeignKey('Project', on_delete=models.CASCADE, null=True, default=None) 1498 1499 # code lifted, with adaptations, from the layerindex-web application 1500 # https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/ 1501 def _handle_url_path(self, base_url, path): 1502 import posixpath 1503 if base_url: 1504 if self.dirpath: 1505 if path: 1506 extra_path = self.dirpath + '/' + path 1507 # Normalise out ../ in path for usage URL 1508 extra_path = posixpath.normpath(extra_path) 1509 # Minor workaround to handle case where subdirectory has been added between branches 1510 # (should probably support usage URL per branch to handle this... sigh...) 1511 if extra_path.startswith('../'): 1512 extra_path = extra_path[3:] 1513 else: 1514 extra_path = self.dirpath 1515 else: 1516 extra_path = path 1517 branchname = self.release.name 1518 url = base_url.replace('%branch%', branchname) 1519 1520 # If there's a % in the path (e.g. a wildcard bbappend) we need to encode it 1521 if extra_path: 1522 extra_path = extra_path.replace('%', '%25') 1523 1524 if '%path%' in base_url: 1525 if extra_path: 1526 url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '\\1', url) 1527 else: 1528 url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '', url) 1529 return url.replace('%path%', extra_path) 1530 else: 1531 return url + extra_path 1532 return None 1533 1534 def get_vcs_link_url(self): 1535 if self.layer.vcs_web_url is None: 1536 return None 1537 return self.layer.vcs_web_url 1538 1539 def get_vcs_file_link_url(self, file_path=""): 1540 if self.layer.vcs_web_file_base_url is None: 1541 return None 1542 return self._handle_url_path(self.layer.vcs_web_file_base_url, 1543 file_path) 1544 1545 def get_vcs_dirpath_link_url(self): 1546 if self.layer.vcs_web_tree_base_url is None: 1547 return None 1548 return self._handle_url_path(self.layer.vcs_web_tree_base_url, '') 1549 1550 def get_vcs_reference(self): 1551 if self.commit is not None and len(self.commit) > 0: 1552 return self.commit 1553 if self.branch is not None and len(self.branch) > 0: 1554 return self.branch 1555 if self.release is not None: 1556 return self.release.name 1557 return 'N/A' 1558 1559 def get_detailspage_url(self, project_id=None): 1560 """ returns the url to the layer details page uses own project 1561 field if project_id is not specified """ 1562 1563 if project_id is None: 1564 project_id = self.project.pk 1565 1566 return reverse('layerdetails', args=(project_id, self.pk)) 1567 1568 def get_alldeps(self, project_id): 1569 """Get full list of unique layer dependencies.""" 1570 def gen_layerdeps(lver, project, depth): 1571 if depth == 0: 1572 return 1573 for ldep in lver.dependencies.all(): 1574 yield ldep.depends_on 1575 # get next level of deps recursively calling gen_layerdeps 1576 for subdep in gen_layerdeps(ldep.depends_on, project, depth-1): 1577 yield subdep 1578 1579 project = Project.objects.get(pk=project_id) 1580 result = [] 1581 projectlvers = [player.layercommit for player in 1582 project.projectlayer_set.all()] 1583 # protect against infinite layer dependency loops 1584 maxdepth = 20 1585 for dep in gen_layerdeps(self, project, maxdepth): 1586 # filter out duplicates and layers already belonging to the project 1587 if dep not in result + projectlvers: 1588 result.append(dep) 1589 1590 return sorted(result, key=lambda x: x.layer.name) 1591 1592 def __unicode__(self): 1593 return ("id %d belongs to layer: %s" % (self.pk, self.layer.name)) 1594 1595 def __str__(self): 1596 if self.release: 1597 release = self.release.name 1598 else: 1599 release = "No release set" 1600 1601 return "%d %s (%s)" % (self.pk, self.layer.name, release) 1602 1603 1604class LayerVersionDependency(models.Model): 1605 1606 layer_version = models.ForeignKey(Layer_Version, on_delete=models.CASCADE, 1607 related_name="dependencies") 1608 depends_on = models.ForeignKey(Layer_Version, on_delete=models.CASCADE, 1609 related_name="dependees") 1610 1611class ProjectLayer(models.Model): 1612 project = models.ForeignKey(Project, on_delete=models.CASCADE) 1613 layercommit = models.ForeignKey(Layer_Version, on_delete=models.CASCADE, null=True) 1614 optional = models.BooleanField(default = True) 1615 1616 def __unicode__(self): 1617 return "%s, %s" % (self.project.name, self.layercommit) 1618 1619 class Meta: 1620 unique_together = (("project", "layercommit"),) 1621 1622class CustomImageRecipe(Recipe): 1623 1624 # CustomImageRecipe's belong to layers called: 1625 LAYER_NAME = "toaster-custom-images" 1626 1627 search_allowed_fields = ['name'] 1628 base_recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE, related_name='based_on_recipe') 1629 project = models.ForeignKey(Project, on_delete=models.CASCADE) 1630 last_updated = models.DateTimeField(null=True, default=None) 1631 1632 def get_last_successful_built_target(self): 1633 """ Return the last successful built target object if one exists 1634 otherwise return None """ 1635 return Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) & 1636 Q(build__project=self.project) & 1637 Q(target=self.name)).last() 1638 1639 def update_package_list(self): 1640 """ Update the package list from the last good build of this 1641 CustomImageRecipe 1642 """ 1643 # Check if we're aldready up-to-date or not 1644 target = self.get_last_successful_built_target() 1645 if target is None: 1646 # So we've never actually built this Custom recipe but what about 1647 # the recipe it's based on? 1648 target = \ 1649 Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) & 1650 Q(build__project=self.project) & 1651 Q(target=self.base_recipe.name)).last() 1652 if target is None: 1653 return 1654 1655 if target.build.completed_on == self.last_updated: 1656 return 1657 1658 self.includes_set.clear() 1659 1660 excludes_list = self.excludes_set.values_list('name', flat=True) 1661 appends_list = self.appends_set.values_list('name', flat=True) 1662 1663 built_packages_list = \ 1664 target.target_installed_package_set.values_list('package__name', 1665 flat=True) 1666 for built_package in built_packages_list: 1667 # Is the built package in the custom packages list? 1668 if built_package in excludes_list: 1669 continue 1670 1671 if built_package in appends_list: 1672 continue 1673 1674 cust_img_p = \ 1675 CustomImagePackage.objects.get(name=built_package) 1676 self.includes_set.add(cust_img_p) 1677 1678 1679 self.last_updated = target.build.completed_on 1680 self.save() 1681 1682 def get_all_packages(self): 1683 """Get the included packages and any appended packages""" 1684 self.update_package_list() 1685 1686 return CustomImagePackage.objects.filter((Q(recipe_appends=self) | 1687 Q(recipe_includes=self)) & 1688 ~Q(recipe_excludes=self)) 1689 1690 def get_base_recipe_file(self): 1691 """Get the base recipe file path if it exists on the file system""" 1692 path_schema_one = "%s/%s" % (self.base_recipe.layer_version.local_path, 1693 self.base_recipe.file_path) 1694 1695 path_schema_two = self.base_recipe.file_path 1696 1697 path_schema_three = "%s/%s" % (self.base_recipe.layer_version.layer.local_source_dir, 1698 self.base_recipe.file_path) 1699 1700 if os.path.exists(path_schema_one): 1701 return path_schema_one 1702 1703 # The path may now be the full path if the recipe has been built 1704 if os.path.exists(path_schema_two): 1705 return path_schema_two 1706 1707 # Or a local path if all layers are local 1708 if os.path.exists(path_schema_three): 1709 return path_schema_three 1710 1711 return None 1712 1713 def generate_recipe_file_contents(self): 1714 """Generate the contents for the recipe file.""" 1715 # If we have no excluded packages we only need to :append 1716 if self.excludes_set.count() == 0: 1717 packages_conf = "IMAGE_INSTALL:append = \" " 1718 1719 for pkg in self.appends_set.all(): 1720 packages_conf += pkg.name+' ' 1721 else: 1722 packages_conf = "IMAGE_FEATURES =\"\"\nIMAGE_INSTALL = \"" 1723 # We add all the known packages to be built by this recipe apart 1724 # from locale packages which are are controlled with IMAGE_LINGUAS. 1725 for pkg in self.get_all_packages().exclude( 1726 name__icontains="locale"): 1727 packages_conf += pkg.name+' ' 1728 1729 packages_conf += "\"" 1730 1731 base_recipe_path = self.get_base_recipe_file() 1732 if base_recipe_path and os.path.isfile(base_recipe_path): 1733 base_recipe = open(base_recipe_path, 'r').read() 1734 else: 1735 # Pass back None to trigger error message to user 1736 return None 1737 1738 # Add a special case for when the recipe we have based a custom image 1739 # recipe on requires another recipe. 1740 # For example: 1741 # "require core-image-minimal.bb" is changed to: 1742 # "require recipes-core/images/core-image-minimal.bb" 1743 1744 req_search = re.search(r'(require\s+)(.+\.bb\s*$)', 1745 base_recipe, 1746 re.MULTILINE) 1747 if req_search: 1748 require_filename = req_search.group(2).strip() 1749 1750 corrected_location = Recipe.objects.filter( 1751 Q(layer_version=self.base_recipe.layer_version) & 1752 Q(file_path__icontains=require_filename)).last().file_path 1753 1754 new_require_line = "require %s" % corrected_location 1755 1756 base_recipe = base_recipe.replace(req_search.group(0), 1757 new_require_line) 1758 1759 info = { 1760 "date": timezone.now().strftime("%Y-%m-%d %H:%M:%S"), 1761 "base_recipe": base_recipe, 1762 "recipe_name": self.name, 1763 "base_recipe_name": self.base_recipe.name, 1764 "license": self.license, 1765 "summary": self.summary, 1766 "description": self.description, 1767 "packages_conf": packages_conf.strip() 1768 } 1769 1770 recipe_contents = ("# Original recipe %(base_recipe_name)s \n" 1771 "%(base_recipe)s\n\n" 1772 "# Recipe %(recipe_name)s \n" 1773 "# Customisation Generated by Toaster on %(date)s\n" 1774 "SUMMARY = \"%(summary)s\"\n" 1775 "DESCRIPTION = \"%(description)s\"\n" 1776 "LICENSE = \"%(license)s\"\n" 1777 "%(packages_conf)s") % info 1778 1779 return recipe_contents 1780 1781class ProjectVariable(models.Model): 1782 project = models.ForeignKey(Project, on_delete=models.CASCADE) 1783 name = models.CharField(max_length=100) 1784 value = models.TextField(blank = True) 1785 1786class Variable(models.Model): 1787 search_allowed_fields = ['variable_name', 'variable_value', 1788 'vhistory__file_name', "description"] 1789 build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='variable_build') 1790 variable_name = models.CharField(max_length=100) 1791 variable_value = models.TextField(blank=True) 1792 changed = models.BooleanField(default=False) 1793 human_readable_name = models.CharField(max_length=200) 1794 description = models.TextField(blank=True) 1795 1796class VariableHistory(models.Model): 1797 variable = models.ForeignKey(Variable, on_delete=models.CASCADE, related_name='vhistory') 1798 value = models.TextField(blank=True) 1799 file_name = models.FilePathField(max_length=255) 1800 line_number = models.IntegerField(null=True) 1801 operation = models.CharField(max_length=64) 1802 1803class HelpText(models.Model): 1804 VARIABLE = 0 1805 HELPTEXT_AREA = ((VARIABLE, 'variable'), ) 1806 1807 build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='helptext_build') 1808 area = models.IntegerField(choices=HELPTEXT_AREA) 1809 key = models.CharField(max_length=100) 1810 text = models.TextField() 1811 1812class LogMessage(models.Model): 1813 EXCEPTION = -1 # used to signal self-toaster-exceptions 1814 INFO = 0 1815 WARNING = 1 1816 ERROR = 2 1817 CRITICAL = 3 1818 1819 LOG_LEVEL = ( 1820 (INFO, "info"), 1821 (WARNING, "warn"), 1822 (ERROR, "error"), 1823 (CRITICAL, "critical"), 1824 (EXCEPTION, "toaster exception") 1825 ) 1826 1827 build = models.ForeignKey(Build, on_delete=models.CASCADE) 1828 task = models.ForeignKey(Task, on_delete=models.CASCADE, blank = True, null=True) 1829 level = models.IntegerField(choices=LOG_LEVEL, default=INFO) 1830 message = models.TextField(blank=True, null=True) 1831 pathname = models.FilePathField(max_length=255, blank=True) 1832 lineno = models.IntegerField(null=True) 1833 1834 def __str__(self): 1835 return force_bytes('%s %s %s' % (self.get_level_display(), self.message, self.build)) 1836 1837def invalidate_cache(**kwargs): 1838 from django.core.cache import cache 1839 try: 1840 cache.clear() 1841 except Exception as e: 1842 logger.warning("Problem with cache backend: Failed to clear cache: %s" % e) 1843 1844def signal_runbuilds(): 1845 """Send SIGUSR1 to runbuilds process""" 1846 try: 1847 with open(os.path.join(os.getenv('BUILDDIR', '.'), 1848 '.runbuilds.pid')) as pidf: 1849 os.kill(int(pidf.read()), SIGUSR1) 1850 except FileNotFoundError: 1851 logger.info("Stopping existing runbuilds: no current process found") 1852 except ProcessLookupError: 1853 logger.warning("Stopping existing runbuilds: process lookup not found") 1854 1855class Distro(models.Model): 1856 search_allowed_fields = ["name", "description", "layer_version__layer__name"] 1857 up_date = models.DateTimeField(null = True, default = None) 1858 1859 layer_version = models.ForeignKey('Layer_Version', on_delete=models.CASCADE) 1860 name = models.CharField(max_length=255) 1861 description = models.CharField(max_length=255) 1862 1863 def get_vcs_distro_file_link_url(self): 1864 path = 'conf/distro/%s.conf' % self.name 1865 return self.layer_version.get_vcs_file_link_url(path) 1866 1867 def __unicode__(self): 1868 return "Distro " + self.name + "(" + self.description + ")" 1869 1870class EventLogsImports(models.Model): 1871 name = models.CharField(max_length=255) 1872 imported = models.BooleanField(default=False) 1873 build_id = models.IntegerField(blank=True, null=True) 1874 1875 def __str__(self): 1876 return self.name 1877 1878 1879django.db.models.signals.post_save.connect(invalidate_cache) 1880django.db.models.signals.post_delete.connect(invalidate_cache) 1881django.db.models.signals.m2m_changed.connect(invalidate_cache) 1882