1# 2# BitBake Toaster Implementation 3# 4# Copyright (C) 2013 Intel Corporation 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9from __future__ import unicode_literals 10 11from django.db import models, IntegrityError, DataError 12from django.db.models import F, Q, Sum, Count 13from django.utils import timezone 14from django.utils.encoding import force_bytes 15 16from django.urls import reverse 17 18from django.core import validators 19from django.conf import settings 20import django.db.models.signals 21 22import sys 23import os 24import re 25import itertools 26from signal import SIGUSR1 27 28 29import logging 30logger = logging.getLogger("toaster") 31 32if 'sqlite' in settings.DATABASES['default']['ENGINE']: 33 from django.db import transaction, OperationalError 34 from time import sleep 35 36 _base_save = models.Model.save 37 def save(self, *args, **kwargs): 38 while True: 39 try: 40 with transaction.atomic(): 41 return _base_save(self, *args, **kwargs) 42 except OperationalError as err: 43 if 'database is locked' in str(err): 44 logger.warning("%s, model: %s, args: %s, kwargs: %s", 45 err, self.__class__, args, kwargs) 46 sleep(0.5) 47 continue 48 raise 49 50 models.Model.save = save 51 52 # HACK: Monkey patch Django to fix 'database is locked' issue 53 54 from django.db.models.query import QuerySet 55 _base_insert = QuerySet._insert 56 def _insert(self, *args, **kwargs): 57 with transaction.atomic(using=self.db, savepoint=False): 58 return _base_insert(self, *args, **kwargs) 59 QuerySet._insert = _insert 60 61 def _create_object_from_params(self, lookup, params): 62 """ 63 Tries to create an object using passed params. 64 Used by get_or_create and update_or_create 65 """ 66 try: 67 obj = self.create(**params) 68 return obj, True 69 except (IntegrityError, DataError): 70 exc_info = sys.exc_info() 71 try: 72 return self.get(**lookup), False 73 except self.model.DoesNotExist: 74 pass 75 six.reraise(*exc_info) 76 77 QuerySet._create_object_from_params = _create_object_from_params 78 79 # end of HACK 80 81class GitURLValidator(validators.URLValidator): 82 import re 83 regex = re.compile( 84 r'^(?:ssh|git|http|ftp)s?://' # http:// or https:// 85 r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... 86 r'localhost|' # localhost... 87 r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4 88 r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6 89 r'(?::\d+)?' # optional port 90 r'(?:/?|[/?]\S+)$', re.IGNORECASE) 91 92def GitURLField(**kwargs): 93 r = models.URLField(**kwargs) 94 for i in range(len(r.validators)): 95 if isinstance(r.validators[i], validators.URLValidator): 96 r.validators[i] = GitURLValidator() 97 return r 98 99 100class ToasterSetting(models.Model): 101 name = models.CharField(max_length=63) 102 helptext = models.TextField() 103 value = models.CharField(max_length=255) 104 105 def __unicode__(self): 106 return "Setting %s = %s" % (self.name, self.value) 107 108 109class ProjectManager(models.Manager): 110 def create_project(self, name, release, existing_project=None): 111 if existing_project and (release is not None): 112 prj = existing_project 113 prj.bitbake_version = release.bitbake_version 114 prj.release = release 115 # Delete the previous ProjectLayer mappings 116 for pl in ProjectLayer.objects.filter(project=prj): 117 pl.delete() 118 elif release is not None: 119 prj = self.model(name=name, 120 bitbake_version=release.bitbake_version, 121 release=release) 122 else: 123 prj = self.model(name=name, 124 bitbake_version=None, 125 release=None) 126 prj.save() 127 128 for defaultconf in ToasterSetting.objects.filter( 129 name__startswith="DEFCONF_"): 130 name = defaultconf.name[8:] 131 pv,create = ProjectVariable.objects.get_or_create(project=prj,name=name) 132 pv.value = defaultconf.value 133 pv.save() 134 135 if release is None: 136 return prj 137 138 for rdl in release.releasedefaultlayer_set.all(): 139 lv = Layer_Version.objects.filter( 140 layer__name=rdl.layer_name, 141 release=release).first() 142 143 if lv: 144 ProjectLayer.objects.create(project=prj, 145 layercommit=lv, 146 optional=False) 147 else: 148 logger.warning("Default project layer %s not found" % 149 rdl.layer_name) 150 151 return prj 152 153 # return single object with is_default = True 154 def get_or_create_default_project(self): 155 projects = super(ProjectManager, self).filter(is_default=True) 156 157 if len(projects) > 1: 158 raise Exception('Inconsistent project data: multiple ' + 159 'default projects (i.e. with is_default=True)') 160 elif len(projects) < 1: 161 options = { 162 'name': 'Command line builds', 163 'short_description': 164 'Project for builds started outside Toaster', 165 'is_default': True 166 } 167 project = Project.objects.create(**options) 168 project.save() 169 170 return project 171 else: 172 return projects[0] 173 174 175class Project(models.Model): 176 search_allowed_fields = ['name', 'short_description', 'release__name', 177 'release__branch_name'] 178 name = models.CharField(max_length=100) 179 short_description = models.CharField(max_length=50, blank=True) 180 bitbake_version = models.ForeignKey('BitbakeVersion', on_delete=models.CASCADE, null=True) 181 release = models.ForeignKey("Release", on_delete=models.CASCADE, null=True) 182 created = models.DateTimeField(auto_now_add=True) 183 updated = models.DateTimeField(auto_now=True) 184 # This is a horrible hack; since Toaster has no "User" model available when 185 # running in interactive mode, we can't reference the field here directly 186 # Instead, we keep a possible null reference to the User id, 187 # as not to force 188 # hard links to possibly missing models 189 user_id = models.IntegerField(null=True) 190 objects = ProjectManager() 191 192 # build directory override (e.g. imported) 193 builddir = models.TextField() 194 # merge the Toaster configure attributes directly into the standard conf files 195 merged_attr = models.BooleanField(default=False) 196 197 # set to True for the project which is the default container 198 # for builds initiated by the command line etc. 199 is_default= models.BooleanField(default=False) 200 201 def __unicode__(self): 202 return "%s (Release %s, BBV %s)" % (self.name, self.release, self.bitbake_version) 203 204 def get_current_machine_name(self): 205 try: 206 return self.projectvariable_set.get(name="MACHINE").value 207 except (ProjectVariable.DoesNotExist,IndexError): 208 return None; 209 210 def get_number_of_builds(self): 211 """Return the number of builds which have ended""" 212 213 return self.build_set.exclude( 214 Q(outcome=Build.IN_PROGRESS) | 215 Q(outcome=Build.CANCELLED) 216 ).count() 217 218 def get_last_build_id(self): 219 try: 220 return Build.objects.filter( project = self.id ).order_by('-completed_on')[0].id 221 except (Build.DoesNotExist,IndexError): 222 return( -1 ) 223 224 def get_last_outcome(self): 225 build_id = self.get_last_build_id() 226 if (-1 == build_id): 227 return( "" ) 228 try: 229 return Build.objects.filter( id = build_id )[ 0 ].outcome 230 except (Build.DoesNotExist,IndexError): 231 return( "not_found" ) 232 233 def get_last_target(self): 234 build_id = self.get_last_build_id() 235 if (-1 == build_id): 236 return( "" ) 237 try: 238 return Target.objects.filter(build = build_id)[0].target 239 except (Target.DoesNotExist,IndexError): 240 return( "not_found" ) 241 242 def get_last_errors(self): 243 build_id = self.get_last_build_id() 244 if (-1 == build_id): 245 return( 0 ) 246 try: 247 return Build.objects.filter(id = build_id)[ 0 ].errors.count() 248 except (Build.DoesNotExist,IndexError): 249 return( "not_found" ) 250 251 def get_last_warnings(self): 252 build_id = self.get_last_build_id() 253 if (-1 == build_id): 254 return( 0 ) 255 try: 256 return Build.objects.filter(id = build_id)[ 0 ].warnings.count() 257 except (Build.DoesNotExist,IndexError): 258 return( "not_found" ) 259 260 def get_last_build_extensions(self): 261 """ 262 Get list of file name extensions for images produced by the most 263 recent build 264 """ 265 last_build = Build.objects.get(pk = self.get_last_build_id()) 266 return last_build.get_image_file_extensions() 267 268 def get_last_imgfiles(self): 269 build_id = self.get_last_build_id() 270 if (-1 == build_id): 271 return( "" ) 272 try: 273 return Variable.objects.filter(build = build_id, variable_name = "IMAGE_FSTYPES")[ 0 ].variable_value 274 except (Variable.DoesNotExist,IndexError): 275 return( "not_found" ) 276 277 def get_all_compatible_layer_versions(self): 278 """ Returns Queryset of all Layer_Versions which are compatible with 279 this project""" 280 queryset = None 281 282 # guard on release, as it can be null 283 if self.release: 284 queryset = Layer_Version.objects.filter( 285 (Q(release=self.release) & 286 Q(build=None) & 287 Q(project=None)) | 288 Q(project=self)) 289 else: 290 queryset = Layer_Version.objects.none() 291 292 return queryset 293 294 def get_project_layer_versions(self, pk=False): 295 """ Returns the Layer_Versions currently added to this project """ 296 layer_versions = self.projectlayer_set.all().values_list('layercommit', 297 flat=True) 298 299 if pk is False: 300 return Layer_Version.objects.filter(pk__in=layer_versions) 301 else: 302 return layer_versions 303 304 305 def get_default_image_recipe(self): 306 try: 307 return self.projectvariable_set.get(name="DEFAULT_IMAGE").value 308 except (ProjectVariable.DoesNotExist,IndexError): 309 return None; 310 311 def get_is_new(self): 312 return self.get_variable(Project.PROJECT_SPECIFIC_ISNEW) 313 314 def get_available_machines(self): 315 """ Returns QuerySet of all Machines which are provided by the 316 Layers currently added to the Project """ 317 queryset = Machine.objects.filter( 318 layer_version__in=self.get_project_layer_versions()) 319 320 return queryset 321 322 def get_all_compatible_machines(self): 323 """ Returns QuerySet of all the compatible machines available to the 324 project including ones from Layers not currently added """ 325 queryset = Machine.objects.filter( 326 layer_version__in=self.get_all_compatible_layer_versions()) 327 328 return queryset 329 330 def get_available_distros(self): 331 """ Returns QuerySet of all Distros which are provided by the 332 Layers currently added to the Project """ 333 queryset = Distro.objects.filter( 334 layer_version__in=self.get_project_layer_versions()) 335 336 return queryset 337 338 def get_all_compatible_distros(self): 339 """ Returns QuerySet of all the compatible Wind River distros available to the 340 project including ones from Layers not currently added """ 341 queryset = Distro.objects.filter( 342 layer_version__in=self.get_all_compatible_layer_versions()) 343 344 return queryset 345 346 def get_available_recipes(self): 347 """ Returns QuerySet of all the recipes that are provided by layers 348 added to this project """ 349 queryset = Recipe.objects.filter( 350 layer_version__in=self.get_project_layer_versions()) 351 352 return queryset 353 354 def get_all_compatible_recipes(self): 355 """ Returns QuerySet of all the compatible Recipes available to the 356 project including ones from Layers not currently added """ 357 queryset = Recipe.objects.filter( 358 layer_version__in=self.get_all_compatible_layer_versions()).exclude(name__exact='') 359 360 return queryset 361 362 # Project Specific status management 363 PROJECT_SPECIFIC_STATUS = 'INTERNAL_PROJECT_SPECIFIC_STATUS' 364 PROJECT_SPECIFIC_CALLBACK = 'INTERNAL_PROJECT_SPECIFIC_CALLBACK' 365 PROJECT_SPECIFIC_ISNEW = 'INTERNAL_PROJECT_SPECIFIC_ISNEW' 366 PROJECT_SPECIFIC_DEFAULTIMAGE = 'PROJECT_SPECIFIC_DEFAULTIMAGE' 367 PROJECT_SPECIFIC_NONE = '' 368 PROJECT_SPECIFIC_NEW = '1' 369 PROJECT_SPECIFIC_EDIT = '2' 370 PROJECT_SPECIFIC_CLONING = '3' 371 PROJECT_SPECIFIC_CLONING_SUCCESS = '4' 372 PROJECT_SPECIFIC_CLONING_FAIL = '5' 373 374 def get_variable(self,variable,default_value = ''): 375 try: 376 return self.projectvariable_set.get(name=variable).value 377 except (ProjectVariable.DoesNotExist,IndexError): 378 return default_value 379 380 def set_variable(self,variable,value): 381 pv,create = ProjectVariable.objects.get_or_create(project = self, name = variable) 382 pv.value = value 383 pv.save() 384 385 def get_default_image(self): 386 return self.get_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE) 387 388 def schedule_build(self): 389 390 from bldcontrol.models import BuildRequest, BRTarget, BRLayer 391 from bldcontrol.models import BRBitbake, BRVariable 392 393 try: 394 now = timezone.now() 395 build = Build.objects.create(project=self, 396 completed_on=now, 397 started_on=now) 398 399 br = BuildRequest.objects.create(project=self, 400 state=BuildRequest.REQ_QUEUED, 401 build=build) 402 BRBitbake.objects.create(req=br, 403 giturl=self.bitbake_version.giturl, 404 commit=self.bitbake_version.branch, 405 dirpath=self.bitbake_version.dirpath) 406 407 for t in self.projecttarget_set.all(): 408 BRTarget.objects.create(req=br, target=t.target, task=t.task) 409 Target.objects.create(build=br.build, target=t.target, 410 task=t.task) 411 # If we're about to build a custom image recipe make sure 412 # that layer is currently in the project before we create the 413 # BRLayer objects 414 customrecipe = CustomImageRecipe.objects.filter( 415 name=t.target, 416 project=self).first() 417 if customrecipe: 418 ProjectLayer.objects.get_or_create( 419 project=self, 420 layercommit=customrecipe.layer_version, 421 optional=False) 422 423 for l in self.projectlayer_set.all().order_by("pk"): 424 commit = l.layercommit.get_vcs_reference() 425 logger.debug("Adding layer to build %s" % 426 l.layercommit.layer.name) 427 BRLayer.objects.create( 428 req=br, 429 name=l.layercommit.layer.name, 430 giturl=l.layercommit.layer.vcs_url, 431 commit=commit, 432 dirpath=l.layercommit.dirpath, 433 layer_version=l.layercommit, 434 local_source_dir=l.layercommit.layer.local_source_dir 435 ) 436 437 for v in self.projectvariable_set.all(): 438 BRVariable.objects.create(req=br, name=v.name, value=v.value) 439 440 try: 441 br.build.machine = self.projectvariable_set.get( 442 name='MACHINE').value 443 br.build.save() 444 except ProjectVariable.DoesNotExist: 445 pass 446 447 br.save() 448 signal_runbuilds() 449 450 except Exception: 451 # revert the build request creation since we're not done cleanly 452 br.delete() 453 raise 454 return br 455 456class Build(models.Model): 457 SUCCEEDED = 0 458 FAILED = 1 459 IN_PROGRESS = 2 460 CANCELLED = 3 461 462 BUILD_OUTCOME = ( 463 (SUCCEEDED, 'Succeeded'), 464 (FAILED, 'Failed'), 465 (IN_PROGRESS, 'In Progress'), 466 (CANCELLED, 'Cancelled'), 467 ) 468 469 search_allowed_fields = ['machine', 'cooker_log_path', "target__target", "target__target_image_file__file_name"] 470 471 project = models.ForeignKey(Project, on_delete=models.CASCADE) # must have a project 472 machine = models.CharField(max_length=100) 473 distro = models.CharField(max_length=100) 474 distro_version = models.CharField(max_length=100) 475 started_on = models.DateTimeField() 476 completed_on = models.DateTimeField() 477 outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS) 478 cooker_log_path = models.CharField(max_length=500) 479 build_name = models.CharField(max_length=100, default='') 480 bitbake_version = models.CharField(max_length=50) 481 482 # number of recipes to parse for this build 483 recipes_to_parse = models.IntegerField(default=1) 484 485 # number of recipes parsed so far for this build 486 recipes_parsed = models.IntegerField(default=1) 487 488 # number of repos to clone for this build 489 repos_to_clone = models.IntegerField(default=1) 490 491 # number of repos cloned so far for this build (default off) 492 repos_cloned = models.IntegerField(default=1) 493 494 # Hint on current progress item 495 progress_item = models.CharField(max_length=40) 496 497 @staticmethod 498 def get_recent(project=None): 499 """ 500 Return recent builds as a list; if project is set, only return 501 builds for that project 502 """ 503 504 builds = Build.objects.all() 505 506 if project: 507 builds = builds.filter(project=project) 508 509 finished_criteria = \ 510 Q(outcome=Build.SUCCEEDED) | \ 511 Q(outcome=Build.FAILED) | \ 512 Q(outcome=Build.CANCELLED) 513 514 recent_builds = list(itertools.chain( 515 builds.filter(outcome=Build.IN_PROGRESS).order_by("-started_on"), 516 builds.filter(finished_criteria).order_by("-completed_on")[:3] 517 )) 518 519 # add percentage done property to each build; this is used 520 # to show build progress in mrb_section.html 521 for build in recent_builds: 522 build.percentDone = build.completeper() 523 build.outcomeText = build.get_outcome_text() 524 525 return recent_builds 526 527 def started(self): 528 """ 529 As build variables are only added for a build when its BuildStarted event 530 is received, a build with no build variables is counted as 531 "in preparation" and not properly started yet. This method 532 will return False if a build has no build variables (it never properly 533 started), or True otherwise. 534 535 Note that this is a temporary workaround for the fact that we don't 536 have a fine-grained state variable on a build which would allow us 537 to record "in progress" (BuildStarted received) vs. "in preparation". 538 """ 539 variables = Variable.objects.filter(build=self) 540 return len(variables) > 0 541 542 def completeper(self): 543 tf = Task.objects.filter(build = self) 544 tfc = tf.count() 545 if tfc > 0: 546 completeper = tf.exclude(outcome=Task.OUTCOME_NA).count()*100 // tfc 547 else: 548 completeper = 0 549 return completeper 550 551 def eta(self): 552 eta = timezone.now() 553 completeper = self.completeper() 554 if self.completeper() > 0: 555 eta += ((eta - self.started_on)*(100-completeper))/completeper 556 return eta 557 558 def has_images(self): 559 """ 560 Returns True if at least one of the targets for this build has an 561 image file associated with it, False otherwise 562 """ 563 targets = Target.objects.filter(build_id=self.id) 564 has_images = False 565 for target in targets: 566 if target.has_images(): 567 has_images = True 568 break 569 return has_images 570 571 def has_image_recipes(self): 572 """ 573 Returns True if a build has any targets which were built from 574 image recipes. 575 """ 576 image_recipes = self.get_image_recipes() 577 return len(image_recipes) > 0 578 579 def get_image_file_extensions(self): 580 """ 581 Get string of file name extensions for images produced by this build; 582 note that this is the actual list of extensions stored on Target objects 583 for this build, and not the value of IMAGE_FSTYPES. 584 585 Returns comma-separated string, e.g. "vmdk, ext4" 586 """ 587 extensions = [] 588 589 targets = Target.objects.filter(build_id = self.id) 590 for target in targets: 591 if not target.is_image: 592 continue 593 594 target_image_files = Target_Image_File.objects.filter( 595 target_id=target.id) 596 597 for target_image_file in target_image_files: 598 extensions.append(target_image_file.suffix) 599 600 extensions = list(set(extensions)) 601 extensions.sort() 602 603 return ', '.join(extensions) 604 605 def get_image_fstypes(self): 606 """ 607 Get the IMAGE_FSTYPES variable value for this build as a de-duplicated 608 list of image file suffixes. 609 """ 610 image_fstypes = Variable.objects.get( 611 build=self, variable_name='IMAGE_FSTYPES').variable_value 612 return list(set(re.split(r' {1,}', image_fstypes))) 613 614 def get_sorted_target_list(self): 615 tgts = Target.objects.filter(build_id = self.id).order_by( 'target' ); 616 return( tgts ); 617 618 def get_recipes(self): 619 """ 620 Get the recipes related to this build; 621 note that the related layer versions and layers are also prefetched 622 by this query, as this queryset can be sorted by these objects in the 623 build recipes view; prefetching them here removes the need 624 for another query in that view 625 """ 626 layer_versions = Layer_Version.objects.filter(build=self) 627 criteria = Q(layer_version__id__in=layer_versions) 628 return Recipe.objects.filter(criteria) \ 629 .select_related('layer_version', 'layer_version__layer') 630 631 def get_image_recipes(self): 632 """ 633 Returns a list of image Recipes (custom and built-in) related to this 634 build, sorted by name; note that this has to be done in two steps, as 635 there's no way to get all the custom image recipes and image recipes 636 in one query 637 """ 638 custom_image_recipes = self.get_custom_image_recipes() 639 custom_image_recipe_names = custom_image_recipes.values_list('name', flat=True) 640 641 not_custom_image_recipes = ~Q(name__in=custom_image_recipe_names) & \ 642 Q(is_image=True) 643 644 built_image_recipes = self.get_recipes().filter(not_custom_image_recipes) 645 646 # append to the custom image recipes and sort 647 customisable_image_recipes = list( 648 itertools.chain(custom_image_recipes, built_image_recipes) 649 ) 650 651 return sorted(customisable_image_recipes, key=lambda recipe: recipe.name) 652 653 def get_custom_image_recipes(self): 654 """ 655 Returns a queryset of CustomImageRecipes related to this build, 656 sorted by name 657 """ 658 built_recipe_names = self.get_recipes().values_list('name', flat=True) 659 criteria = Q(name__in=built_recipe_names) & Q(project=self.project) 660 queryset = CustomImageRecipe.objects.filter(criteria).order_by('name') 661 return queryset 662 663 def get_outcome_text(self): 664 return Build.BUILD_OUTCOME[int(self.outcome)][1] 665 666 @property 667 def failed_tasks(self): 668 """ Get failed tasks for the build """ 669 tasks = self.task_build.all() 670 return tasks.filter(order__gt=0, outcome=Task.OUTCOME_FAILED) 671 672 @property 673 def errors(self): 674 return (self.logmessage_set.filter(level=LogMessage.ERROR) | 675 self.logmessage_set.filter(level=LogMessage.EXCEPTION) | 676 self.logmessage_set.filter(level=LogMessage.CRITICAL)) 677 678 @property 679 def warnings(self): 680 return self.logmessage_set.filter(level=LogMessage.WARNING) 681 682 @property 683 def timespent(self): 684 return self.completed_on - self.started_on 685 686 @property 687 def timespent_seconds(self): 688 return self.timespent.total_seconds() 689 690 @property 691 def target_labels(self): 692 """ 693 Sorted (a-z) "target1:task, target2, target3" etc. string for all 694 targets in this build 695 """ 696 targets = self.target_set.all() 697 target_labels = [target.target + 698 (':' + target.task if target.task else '') 699 for target in targets] 700 target_labels.sort() 701 702 return target_labels 703 704 def get_buildrequest(self): 705 buildrequest = None 706 if hasattr(self, 'buildrequest'): 707 buildrequest = self.buildrequest 708 return buildrequest 709 710 def is_queued(self): 711 from bldcontrol.models import BuildRequest 712 buildrequest = self.get_buildrequest() 713 if buildrequest: 714 return buildrequest.state == BuildRequest.REQ_QUEUED 715 else: 716 return False 717 718 def is_cancelling(self): 719 from bldcontrol.models import BuildRequest 720 buildrequest = self.get_buildrequest() 721 if buildrequest: 722 return self.outcome == Build.IN_PROGRESS and \ 723 buildrequest.state == BuildRequest.REQ_CANCELLING 724 else: 725 return False 726 727 def is_cloning(self): 728 """ 729 True if the build is still cloning repos 730 """ 731 return self.outcome == Build.IN_PROGRESS and \ 732 self.repos_cloned < self.repos_to_clone 733 734 def is_parsing(self): 735 """ 736 True if the build is still parsing recipes 737 """ 738 return self.outcome == Build.IN_PROGRESS and \ 739 self.recipes_parsed < self.recipes_to_parse 740 741 def is_starting(self): 742 """ 743 True if the build has no completed tasks yet and is still just starting 744 tasks. 745 746 Note that the mechanism for testing whether a Task is "done" is whether 747 its outcome field is set, as per the completeper() method. 748 """ 749 return self.outcome == Build.IN_PROGRESS and \ 750 self.task_build.exclude(outcome=Task.OUTCOME_NA).count() == 0 751 752 753 def get_state(self): 754 """ 755 Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress', 756 'Cancelled' (Build outcomes); or 'Queued', 'Cancelling' (states 757 dependent on the BuildRequest state). 758 759 This works around the fact that we have BuildRequest states as well 760 as Build states, but really we just want to know the state of the build. 761 """ 762 if self.is_cancelling(): 763 return 'Cancelling'; 764 elif self.is_queued(): 765 return 'Queued' 766 elif self.is_cloning(): 767 return 'Cloning' 768 elif self.is_parsing(): 769 return 'Parsing' 770 elif self.is_starting(): 771 return 'Starting' 772 else: 773 return self.get_outcome_text() 774 775 def __str__(self): 776 return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()])) 777 778class ProjectTarget(models.Model): 779 project = models.ForeignKey(Project, on_delete=models.CASCADE) 780 target = models.CharField(max_length=100) 781 task = models.CharField(max_length=100, null=True) 782 783class Target(models.Model): 784 search_allowed_fields = ['target', 'file_name'] 785 build = models.ForeignKey(Build, on_delete=models.CASCADE) 786 target = models.CharField(max_length=100) 787 task = models.CharField(max_length=100, null=True) 788 is_image = models.BooleanField(default = False) 789 image_size = models.IntegerField(default=0) 790 license_manifest_path = models.CharField(max_length=500, null=True) 791 package_manifest_path = models.CharField(max_length=500, null=True) 792 793 def package_count(self): 794 return Target_Installed_Package.objects.filter(target_id__exact=self.id).count() 795 796 def __unicode__(self): 797 return self.target 798 799 def get_similar_targets(self): 800 """ 801 Get target sfor the same machine, task and target name 802 (e.g. 'core-image-minimal') from a successful build for this project 803 (but excluding this target). 804 805 Note that we only look for targets built by this project because 806 projects can have different configurations from each other, and put 807 their artifacts in different directories. 808 809 The possibility of error when retrieving candidate targets 810 is minimised by the fact that bitbake will rebuild artifacts if MACHINE 811 (or various other variables) change. In this case, there is no need to 812 clone artifacts from another target, as those artifacts will have 813 been re-generated for this target anyway. 814 """ 815 query = ~Q(pk=self.pk) & \ 816 Q(target=self.target) & \ 817 Q(build__machine=self.build.machine) & \ 818 Q(build__outcome=Build.SUCCEEDED) & \ 819 Q(build__project=self.build.project) 820 821 return Target.objects.filter(query) 822 823 def get_similar_target_with_image_files(self): 824 """ 825 Get the most recent similar target with Target_Image_Files associated 826 with it, for the purpose of cloning those files onto this target. 827 """ 828 similar_target = None 829 830 candidates = self.get_similar_targets() 831 if candidates.count() == 0: 832 return similar_target 833 834 task_subquery = Q(task=self.task) 835 836 # we can look for a 'build' task if this task is a 'populate_sdk_ext' 837 # task, as the latter also creates images; and vice versa; note that 838 # 'build' targets can have their task set to ''; 839 # also note that 'populate_sdk' does not produce image files 840 image_tasks = [ 841 '', # aka 'build' 842 'build', 843 'image', 844 'populate_sdk_ext' 845 ] 846 if self.task in image_tasks: 847 task_subquery = Q(task__in=image_tasks) 848 849 # annotate with the count of files, to exclude any targets which 850 # don't have associated files 851 candidates = candidates.annotate(num_files=Count('target_image_file')) 852 853 query = task_subquery & Q(num_files__gt=0) 854 855 candidates = candidates.filter(query) 856 857 if candidates.count() > 0: 858 candidates.order_by('build__completed_on') 859 similar_target = candidates.last() 860 861 return similar_target 862 863 def get_similar_target_with_sdk_files(self): 864 """ 865 Get the most recent similar target with TargetSDKFiles associated 866 with it, for the purpose of cloning those files onto this target. 867 """ 868 similar_target = None 869 870 candidates = self.get_similar_targets() 871 if candidates.count() == 0: 872 return similar_target 873 874 # annotate with the count of files, to exclude any targets which 875 # don't have associated files 876 candidates = candidates.annotate(num_files=Count('targetsdkfile')) 877 878 query = Q(task=self.task) & Q(num_files__gt=0) 879 880 candidates = candidates.filter(query) 881 882 if candidates.count() > 0: 883 candidates.order_by('build__completed_on') 884 similar_target = candidates.last() 885 886 return similar_target 887 888 def clone_image_artifacts_from(self, target): 889 """ 890 Make clones of the Target_Image_Files and TargetKernelFile objects 891 associated with Target target, then associate them with this target. 892 893 Note that for Target_Image_Files, we only want files from the previous 894 build whose suffix matches one of the suffixes defined in this 895 target's build's IMAGE_FSTYPES configuration variable. This prevents the 896 Target_Image_File object for an ext4 image being associated with a 897 target for a project which didn't produce an ext4 image (for example). 898 899 Also sets the license_manifest_path and package_manifest_path 900 of this target to the same path as that of target being cloned from, as 901 the manifests are also build artifacts but are treated differently. 902 """ 903 904 image_fstypes = self.build.get_image_fstypes() 905 906 # filter out any image files whose suffixes aren't in the 907 # IMAGE_FSTYPES suffixes variable for this target's build 908 image_files = [target_image_file \ 909 for target_image_file in target.target_image_file_set.all() \ 910 if target_image_file.suffix in image_fstypes] 911 912 for image_file in image_files: 913 image_file.pk = None 914 image_file.target = self 915 image_file.save() 916 917 kernel_files = target.targetkernelfile_set.all() 918 for kernel_file in kernel_files: 919 kernel_file.pk = None 920 kernel_file.target = self 921 kernel_file.save() 922 923 self.license_manifest_path = target.license_manifest_path 924 self.package_manifest_path = target.package_manifest_path 925 self.save() 926 927 def clone_sdk_artifacts_from(self, target): 928 """ 929 Clone TargetSDKFile objects from target and associate them with this 930 target. 931 """ 932 sdk_files = target.targetsdkfile_set.all() 933 for sdk_file in sdk_files: 934 sdk_file.pk = None 935 sdk_file.target = self 936 sdk_file.save() 937 938 def has_images(self): 939 """ 940 Returns True if this target has one or more image files attached to it. 941 """ 942 return self.target_image_file_set.all().count() > 0 943 944# kernel artifacts for a target: bzImage and modules* 945class TargetKernelFile(models.Model): 946 target = models.ForeignKey(Target, on_delete=models.CASCADE) 947 file_name = models.FilePathField() 948 file_size = models.IntegerField() 949 950 @property 951 def basename(self): 952 return os.path.basename(self.file_name) 953 954# SDK artifacts for a target: sh and manifest files 955class TargetSDKFile(models.Model): 956 target = models.ForeignKey(Target, on_delete=models.CASCADE) 957 file_name = models.FilePathField() 958 file_size = models.IntegerField() 959 960 @property 961 def basename(self): 962 return os.path.basename(self.file_name) 963 964class Target_Image_File(models.Model): 965 # valid suffixes for image files produced by a build 966 SUFFIXES = { 967 'btrfs', 'container', 'cpio', 'cpio.gz', 'cpio.lz4', 'cpio.lzma', 968 'cpio.xz', 'cramfs', 'ext2', 'ext2.bz2', 'ext2.gz', 'ext2.lzma', 969 'ext3', 'ext3.gz', 'ext4', 'ext4.gz', 'f2fs', 'hddimg', 'iso', 'jffs2', 970 'jffs2.sum', 'multiubi', 'squashfs', 'squashfs-lz4', 'squashfs-lzo', 971 'squashfs-xz', 'tar', 'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi', 972 'ubifs', 'wic', 'wic.bz2', 'wic.gz', 'wic.lzma' 973 } 974 975 target = models.ForeignKey(Target, on_delete=models.CASCADE) 976 file_name = models.FilePathField(max_length=254) 977 file_size = models.IntegerField() 978 979 @property 980 def suffix(self): 981 """ 982 Suffix for image file, minus leading "." 983 """ 984 for suffix in Target_Image_File.SUFFIXES: 985 if self.file_name.endswith(suffix): 986 return suffix 987 988 filename, suffix = os.path.splitext(self.file_name) 989 suffix = suffix.lstrip('.') 990 return suffix 991 992class Target_File(models.Model): 993 ITYPE_REGULAR = 1 994 ITYPE_DIRECTORY = 2 995 ITYPE_SYMLINK = 3 996 ITYPE_SOCKET = 4 997 ITYPE_FIFO = 5 998 ITYPE_CHARACTER = 6 999 ITYPE_BLOCK = 7 1000 ITYPES = ( (ITYPE_REGULAR ,'regular'), 1001 ( ITYPE_DIRECTORY ,'directory'), 1002 ( ITYPE_SYMLINK ,'symlink'), 1003 ( ITYPE_SOCKET ,'socket'), 1004 ( ITYPE_FIFO ,'fifo'), 1005 ( ITYPE_CHARACTER ,'character'), 1006 ( ITYPE_BLOCK ,'block'), 1007 ) 1008 1009 target = models.ForeignKey(Target, on_delete=models.CASCADE) 1010 path = models.FilePathField() 1011 size = models.IntegerField() 1012 inodetype = models.IntegerField(choices = ITYPES) 1013 permission = models.CharField(max_length=16) 1014 owner = models.CharField(max_length=128) 1015 group = models.CharField(max_length=128) 1016 directory = models.ForeignKey('Target_File', on_delete=models.CASCADE, related_name="directory_set", null=True) 1017 sym_target = models.ForeignKey('Target_File', on_delete=models.CASCADE, related_name="symlink_set", null=True) 1018 1019 1020class Task(models.Model): 1021 1022 SSTATE_NA = 0 1023 SSTATE_MISS = 1 1024 SSTATE_FAILED = 2 1025 SSTATE_RESTORED = 3 1026 1027 SSTATE_RESULT = ( 1028 (SSTATE_NA, 'Not Applicable'), # For rest of tasks, but they still need checking. 1029 (SSTATE_MISS, 'File not in cache'), # the sstate object was not found 1030 (SSTATE_FAILED, 'Failed'), # there was a pkg, but the script failed 1031 (SSTATE_RESTORED, 'Succeeded'), # successfully restored 1032 ) 1033 1034 CODING_NA = 0 1035 CODING_PYTHON = 2 1036 CODING_SHELL = 3 1037 1038 TASK_CODING = ( 1039 (CODING_NA, 'N/A'), 1040 (CODING_PYTHON, 'Python'), 1041 (CODING_SHELL, 'Shell'), 1042 ) 1043 1044 OUTCOME_NA = -1 1045 OUTCOME_SUCCESS = 0 1046 OUTCOME_COVERED = 1 1047 OUTCOME_CACHED = 2 1048 OUTCOME_PREBUILT = 3 1049 OUTCOME_FAILED = 4 1050 OUTCOME_EMPTY = 5 1051 1052 TASK_OUTCOME = ( 1053 (OUTCOME_NA, 'Not Available'), 1054 (OUTCOME_SUCCESS, 'Succeeded'), 1055 (OUTCOME_COVERED, 'Covered'), 1056 (OUTCOME_CACHED, 'Cached'), 1057 (OUTCOME_PREBUILT, 'Prebuilt'), 1058 (OUTCOME_FAILED, 'Failed'), 1059 (OUTCOME_EMPTY, 'Empty'), 1060 ) 1061 1062 TASK_OUTCOME_HELP = ( 1063 (OUTCOME_SUCCESS, 'This task successfully completed'), 1064 (OUTCOME_COVERED, 'This task did not run because its output is provided by another task'), 1065 (OUTCOME_CACHED, 'This task restored output from the sstate-cache directory or mirrors'), 1066 (OUTCOME_PREBUILT, 'This task did not run because its outcome was reused from a previous build'), 1067 (OUTCOME_FAILED, 'This task did not complete'), 1068 (OUTCOME_EMPTY, 'This task has no executable content'), 1069 (OUTCOME_NA, ''), 1070 ) 1071 1072 search_allowed_fields = [ "recipe__name", "recipe__version", "task_name", "logfile" ] 1073 1074 def __init__(self, *args, **kwargs): 1075 super(Task, self).__init__(*args, **kwargs) 1076 try: 1077 self._helptext = HelpText.objects.get(key=self.task_name, area=HelpText.VARIABLE, build=self.build).text 1078 except HelpText.DoesNotExist: 1079 self._helptext = None 1080 1081 def get_related_setscene(self): 1082 return Task.objects.filter(task_executed=True, build = self.build, recipe = self.recipe, task_name=self.task_name+"_setscene") 1083 1084 def get_outcome_text(self): 1085 return Task.TASK_OUTCOME[int(self.outcome) + 1][1] 1086 1087 def get_outcome_help(self): 1088 return Task.TASK_OUTCOME_HELP[int(self.outcome)][1] 1089 1090 def get_sstate_text(self): 1091 if self.sstate_result==Task.SSTATE_NA: 1092 return '' 1093 else: 1094 return Task.SSTATE_RESULT[int(self.sstate_result)][1] 1095 1096 def get_executed_display(self): 1097 if self.task_executed: 1098 return "Executed" 1099 return "Not Executed" 1100 1101 def get_description(self): 1102 return self._helptext 1103 1104 build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='task_build') 1105 order = models.IntegerField(null=True) 1106 task_executed = models.BooleanField(default=False) # True means Executed, False means Not/Executed 1107 outcome = models.IntegerField(choices=TASK_OUTCOME, default=OUTCOME_NA) 1108 sstate_checksum = models.CharField(max_length=100, blank=True) 1109 path_to_sstate_obj = models.FilePathField(max_length=500, blank=True) 1110 recipe = models.ForeignKey('Recipe', on_delete=models.CASCADE, related_name='tasks') 1111 task_name = models.CharField(max_length=100) 1112 source_url = models.FilePathField(max_length=255, blank=True) 1113 work_directory = models.FilePathField(max_length=255, blank=True) 1114 script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NA) 1115 line_number = models.IntegerField(default=0) 1116 1117 # start/end times 1118 started = models.DateTimeField(null=True) 1119 ended = models.DateTimeField(null=True) 1120 1121 # in seconds; this is stored to enable sorting 1122 elapsed_time = models.DecimalField(max_digits=8, decimal_places=2, null=True) 1123 1124 # in bytes; note that disk_io is stored to enable sorting 1125 disk_io = models.IntegerField(null=True) 1126 disk_io_read = models.IntegerField(null=True) 1127 disk_io_write = models.IntegerField(null=True) 1128 1129 # in seconds 1130 cpu_time_user = models.DecimalField(max_digits=8, decimal_places=2, null=True) 1131 cpu_time_system = models.DecimalField(max_digits=8, decimal_places=2, null=True) 1132 1133 sstate_result = models.IntegerField(choices=SSTATE_RESULT, default=SSTATE_NA) 1134 message = models.CharField(max_length=240) 1135 logfile = models.FilePathField(max_length=255, blank=True) 1136 1137 outcome_text = property(get_outcome_text) 1138 sstate_text = property(get_sstate_text) 1139 1140 def __unicode__(self): 1141 return "%d(%d) %s:%s" % (self.pk, self.build.pk, self.recipe.name, self.task_name) 1142 1143 class Meta: 1144 ordering = ('order', 'recipe' ,) 1145 unique_together = ('build', 'recipe', 'task_name', ) 1146 1147 1148class Task_Dependency(models.Model): 1149 task = models.ForeignKey(Task, on_delete=models.CASCADE, related_name='task_dependencies_task') 1150 depends_on = models.ForeignKey(Task, on_delete=models.CASCADE, related_name='task_dependencies_depends') 1151 1152class Package(models.Model): 1153 search_allowed_fields = ['name', 'version', 'revision', 'recipe__name', 'recipe__version', 'recipe__license', 'recipe__layer_version__layer__name', 'recipe__layer_version__branch', 'recipe__layer_version__commit', 'recipe__layer_version__local_path', 'installed_name'] 1154 build = models.ForeignKey('Build', on_delete=models.CASCADE, null=True) 1155 recipe = models.ForeignKey('Recipe', on_delete=models.CASCADE, null=True) 1156 name = models.CharField(max_length=100) 1157 installed_name = models.CharField(max_length=100, default='') 1158 version = models.CharField(max_length=100, blank=True) 1159 revision = models.CharField(max_length=32, blank=True) 1160 summary = models.TextField(blank=True) 1161 description = models.TextField(blank=True) 1162 size = models.IntegerField(default=0) 1163 installed_size = models.IntegerField(default=0) 1164 section = models.CharField(max_length=80, blank=True) 1165 license = models.CharField(max_length=80, blank=True) 1166 1167 @property 1168 def is_locale_package(self): 1169 """ Returns True if this package is identifiable as a locale package """ 1170 if self.name.find('locale') != -1: 1171 return True 1172 return False 1173 1174 @property 1175 def is_packagegroup(self): 1176 """ Returns True is this package is identifiable as a packagegroup """ 1177 if self.name.find('packagegroup') != -1: 1178 return True 1179 return False 1180 1181class CustomImagePackage(Package): 1182 # CustomImageRecipe fields to track pacakges appended, 1183 # included and excluded from a CustomImageRecipe 1184 recipe_includes = models.ManyToManyField('CustomImageRecipe', 1185 related_name='includes_set') 1186 recipe_excludes = models.ManyToManyField('CustomImageRecipe', 1187 related_name='excludes_set') 1188 recipe_appends = models.ManyToManyField('CustomImageRecipe', 1189 related_name='appends_set') 1190 1191 1192class Package_DependencyManager(models.Manager): 1193 use_for_related_fields = True 1194 TARGET_LATEST = "use-latest-target-for-target" 1195 1196 def get_queryset(self): 1197 return super(Package_DependencyManager, self).get_queryset().exclude(package_id = F('depends_on__id')) 1198 1199 def for_target_or_none(self, target): 1200 """ filter the dependencies to be displayed by the supplied target 1201 if no dependences are found for the target then try None as the target 1202 which will return the dependences calculated without the context of a 1203 target e.g. non image recipes. 1204 1205 returns: { size, packages } 1206 """ 1207 package_dependencies = self.all_depends().order_by('depends_on__name') 1208 1209 if target is self.TARGET_LATEST: 1210 installed_deps =\ 1211 package_dependencies.filter(~Q(target__target=None)) 1212 else: 1213 installed_deps =\ 1214 package_dependencies.filter(Q(target__target=target)) 1215 1216 packages_list = None 1217 total_size = 0 1218 1219 # If we have installed depdencies for this package and target then use 1220 # these to display 1221 if installed_deps.count() > 0: 1222 packages_list = installed_deps 1223 total_size = installed_deps.aggregate( 1224 Sum('depends_on__size'))['depends_on__size__sum'] 1225 else: 1226 new_list = [] 1227 package_names = [] 1228 1229 # Find dependencies for the package that we know about even if 1230 # it's not installed on a target e.g. from a non-image recipe 1231 for p in package_dependencies.filter(Q(target=None)): 1232 if p.depends_on.name in package_names: 1233 continue 1234 else: 1235 package_names.append(p.depends_on.name) 1236 new_list.append(p.pk) 1237 # while we're here we may as well total up the size to 1238 # avoid iterating again 1239 total_size += p.depends_on.size 1240 1241 # We want to return a queryset here for consistency so pick the 1242 # deps from the new_list 1243 packages_list = package_dependencies.filter(Q(pk__in=new_list)) 1244 1245 return {'packages': packages_list, 1246 'size': total_size} 1247 1248 def all_depends(self): 1249 """ Returns just the depends packages and not any other dep_type 1250 Note that this is for any target 1251 """ 1252 return self.filter(Q(dep_type=Package_Dependency.TYPE_RDEPENDS) | 1253 Q(dep_type=Package_Dependency.TYPE_TRDEPENDS)) 1254 1255 1256class Package_Dependency(models.Model): 1257 TYPE_RDEPENDS = 0 1258 TYPE_TRDEPENDS = 1 1259 TYPE_RRECOMMENDS = 2 1260 TYPE_TRECOMMENDS = 3 1261 TYPE_RSUGGESTS = 4 1262 TYPE_RPROVIDES = 5 1263 TYPE_RREPLACES = 6 1264 TYPE_RCONFLICTS = 7 1265 ' TODO: bpackage should be changed to remove the DEPENDS_TYPE access ' 1266 DEPENDS_TYPE = ( 1267 (TYPE_RDEPENDS, "depends"), 1268 (TYPE_TRDEPENDS, "depends"), 1269 (TYPE_TRECOMMENDS, "recommends"), 1270 (TYPE_RRECOMMENDS, "recommends"), 1271 (TYPE_RSUGGESTS, "suggests"), 1272 (TYPE_RPROVIDES, "provides"), 1273 (TYPE_RREPLACES, "replaces"), 1274 (TYPE_RCONFLICTS, "conflicts"), 1275 ) 1276 """ Indexed by dep_type, in view order, key for short name and help 1277 description which when viewed will be printf'd with the 1278 package name. 1279 """ 1280 DEPENDS_DICT = { 1281 TYPE_RDEPENDS : ("depends", "%s is required to run %s"), 1282 TYPE_TRDEPENDS : ("depends", "%s is required to run %s"), 1283 TYPE_TRECOMMENDS : ("recommends", "%s extends the usability of %s"), 1284 TYPE_RRECOMMENDS : ("recommends", "%s extends the usability of %s"), 1285 TYPE_RSUGGESTS : ("suggests", "%s is suggested for installation with %s"), 1286 TYPE_RPROVIDES : ("provides", "%s is provided by %s"), 1287 TYPE_RREPLACES : ("replaces", "%s is replaced by %s"), 1288 TYPE_RCONFLICTS : ("conflicts", "%s conflicts with %s, which will not be installed if this package is not first removed"), 1289 } 1290 1291 package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='package_dependencies_source') 1292 depends_on = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='package_dependencies_target') # soft dependency 1293 dep_type = models.IntegerField(choices=DEPENDS_TYPE) 1294 target = models.ForeignKey(Target, on_delete=models.CASCADE, null=True) 1295 objects = Package_DependencyManager() 1296 1297class Target_Installed_Package(models.Model): 1298 target = models.ForeignKey(Target, on_delete=models.CASCADE) 1299 package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='buildtargetlist_package') 1300 1301 1302class Package_File(models.Model): 1303 package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='buildfilelist_package') 1304 path = models.FilePathField(max_length=255, blank=True) 1305 size = models.IntegerField() 1306 1307 1308class Recipe(models.Model): 1309 search_allowed_fields = ['name', 'version', 'file_path', 'section', 1310 'summary', 'description', 'license', 1311 'layer_version__layer__name', 1312 'layer_version__branch', 'layer_version__commit', 1313 'layer_version__local_path', 1314 'layer_version__layer_source'] 1315 1316 up_date = models.DateTimeField(null=True, default=None) 1317 1318 name = models.CharField(max_length=100, blank=True) 1319 version = models.CharField(max_length=100, blank=True) 1320 layer_version = models.ForeignKey('Layer_Version', on_delete=models.CASCADE, 1321 related_name='recipe_layer_version') 1322 summary = models.TextField(blank=True) 1323 description = models.TextField(blank=True) 1324 section = models.CharField(max_length=100, blank=True) 1325 license = models.CharField(max_length=200, blank=True) 1326 homepage = models.URLField(blank=True) 1327 bugtracker = models.URLField(blank=True) 1328 file_path = models.FilePathField(max_length=255) 1329 pathflags = models.CharField(max_length=200, blank=True) 1330 is_image = models.BooleanField(default=False) 1331 1332 def __unicode__(self): 1333 return "Recipe " + self.name + ":" + self.version 1334 1335 def get_vcs_recipe_file_link_url(self): 1336 return self.layer_version.get_vcs_file_link_url(self.file_path) 1337 1338 def get_description_or_summary(self): 1339 if self.description: 1340 return self.description 1341 elif self.summary: 1342 return self.summary 1343 else: 1344 return "" 1345 1346 class Meta: 1347 unique_together = (("layer_version", "file_path", "pathflags"), ) 1348 1349 1350class Recipe_DependencyManager(models.Manager): 1351 use_for_related_fields = True 1352 1353 def get_queryset(self): 1354 return super(Recipe_DependencyManager, self).get_queryset().exclude(recipe_id = F('depends_on__id')) 1355 1356class Provides(models.Model): 1357 name = models.CharField(max_length=100) 1358 recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE) 1359 1360class Recipe_Dependency(models.Model): 1361 TYPE_DEPENDS = 0 1362 TYPE_RDEPENDS = 1 1363 1364 DEPENDS_TYPE = ( 1365 (TYPE_DEPENDS, "depends"), 1366 (TYPE_RDEPENDS, "rdepends"), 1367 ) 1368 recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE, related_name='r_dependencies_recipe') 1369 depends_on = models.ForeignKey(Recipe, on_delete=models.CASCADE, related_name='r_dependencies_depends') 1370 via = models.ForeignKey(Provides, on_delete=models.CASCADE, null=True, default=None) 1371 dep_type = models.IntegerField(choices=DEPENDS_TYPE) 1372 objects = Recipe_DependencyManager() 1373 1374 1375class Machine(models.Model): 1376 search_allowed_fields = ["name", "description", "layer_version__layer__name"] 1377 up_date = models.DateTimeField(null = True, default = None) 1378 1379 layer_version = models.ForeignKey('Layer_Version', on_delete=models.CASCADE) 1380 name = models.CharField(max_length=255) 1381 description = models.CharField(max_length=255) 1382 1383 def get_vcs_machine_file_link_url(self): 1384 path = 'conf/machine/'+self.name+'.conf' 1385 1386 return self.layer_version.get_vcs_file_link_url(path) 1387 1388 def __unicode__(self): 1389 return "Machine " + self.name + "(" + self.description + ")" 1390 1391 1392 1393 1394 1395class BitbakeVersion(models.Model): 1396 1397 name = models.CharField(max_length=32, unique = True) 1398 giturl = GitURLField() 1399 branch = models.CharField(max_length=32) 1400 dirpath = models.CharField(max_length=255) 1401 1402 def __unicode__(self): 1403 return "%s (Branch: %s)" % (self.name, self.branch) 1404 1405 1406class Release(models.Model): 1407 """ A release is a project template, used to pre-populate Project settings with a configuration set """ 1408 name = models.CharField(max_length=32, unique = True) 1409 description = models.CharField(max_length=255) 1410 bitbake_version = models.ForeignKey(BitbakeVersion, on_delete=models.CASCADE) 1411 branch_name = models.CharField(max_length=50, default = "") 1412 helptext = models.TextField(null=True) 1413 1414 def __unicode__(self): 1415 return "%s (%s)" % (self.name, self.branch_name) 1416 1417 def __str__(self): 1418 return self.name 1419 1420class ReleaseDefaultLayer(models.Model): 1421 release = models.ForeignKey(Release, on_delete=models.CASCADE) 1422 layer_name = models.CharField(max_length=100, default="") 1423 1424 1425class LayerSource(object): 1426 """ Where the layer metadata came from """ 1427 TYPE_LOCAL = 0 1428 TYPE_LAYERINDEX = 1 1429 TYPE_IMPORTED = 2 1430 TYPE_BUILD = 3 1431 1432 SOURCE_TYPE = ( 1433 (TYPE_LOCAL, "local"), 1434 (TYPE_LAYERINDEX, "layerindex"), 1435 (TYPE_IMPORTED, "imported"), 1436 (TYPE_BUILD, "build"), 1437 ) 1438 1439 def types_dict(): 1440 """ Turn the TYPES enums into a simple dictionary """ 1441 dictionary = {} 1442 for key in LayerSource.__dict__: 1443 if "TYPE" in key: 1444 dictionary[key] = getattr(LayerSource, key) 1445 return dictionary 1446 1447 1448class Layer(models.Model): 1449 1450 up_date = models.DateTimeField(null=True, default=timezone.now) 1451 1452 name = models.CharField(max_length=100) 1453 layer_index_url = models.URLField() 1454 vcs_url = GitURLField(default=None, null=True) 1455 local_source_dir = models.TextField(null=True, default=None) 1456 vcs_web_url = models.URLField(null=True, default=None) 1457 vcs_web_tree_base_url = models.URLField(null=True, default=None) 1458 vcs_web_file_base_url = models.URLField(null=True, default=None) 1459 1460 summary = models.TextField(help_text='One-line description of the layer', 1461 null=True, default=None) 1462 description = models.TextField(null=True, default=None) 1463 1464 def __unicode__(self): 1465 return "%s / %s " % (self.name, self.summary) 1466 1467 1468class Layer_Version(models.Model): 1469 """ 1470 A Layer_Version either belongs to a single project or no project 1471 """ 1472 search_allowed_fields = ["layer__name", "layer__summary", 1473 "layer__description", "layer__vcs_url", 1474 "dirpath", "release__name", "commit", "branch"] 1475 1476 build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='layer_version_build', 1477 default=None, null=True) 1478 1479 layer = models.ForeignKey(Layer, on_delete=models.CASCADE, related_name='layer_version_layer') 1480 1481 layer_source = models.IntegerField(choices=LayerSource.SOURCE_TYPE, 1482 default=0) 1483 1484 up_date = models.DateTimeField(null=True, default=timezone.now) 1485 1486 # To which metadata release does this layer version belong to 1487 release = models.ForeignKey(Release, on_delete=models.CASCADE, null=True, default=None) 1488 1489 branch = models.CharField(max_length=80) 1490 commit = models.CharField(max_length=100) 1491 # If the layer is in a subdir 1492 dirpath = models.CharField(max_length=255, null=True, default=None) 1493 1494 # if -1, this is a default layer 1495 priority = models.IntegerField(default=0) 1496 1497 # where this layer exists on the filesystem 1498 local_path = models.FilePathField(max_length=1024, default="/") 1499 1500 # Set if this layer is restricted to a particular project 1501 project = models.ForeignKey('Project', on_delete=models.CASCADE, null=True, default=None) 1502 1503 # code lifted, with adaptations, from the layerindex-web application 1504 # https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/ 1505 def _handle_url_path(self, base_url, path): 1506 import re, posixpath 1507 if base_url: 1508 if self.dirpath: 1509 if path: 1510 extra_path = self.dirpath + '/' + path 1511 # Normalise out ../ in path for usage URL 1512 extra_path = posixpath.normpath(extra_path) 1513 # Minor workaround to handle case where subdirectory has been added between branches 1514 # (should probably support usage URL per branch to handle this... sigh...) 1515 if extra_path.startswith('../'): 1516 extra_path = extra_path[3:] 1517 else: 1518 extra_path = self.dirpath 1519 else: 1520 extra_path = path 1521 branchname = self.release.name 1522 url = base_url.replace('%branch%', branchname) 1523 1524 # If there's a % in the path (e.g. a wildcard bbappend) we need to encode it 1525 if extra_path: 1526 extra_path = extra_path.replace('%', '%25') 1527 1528 if '%path%' in base_url: 1529 if extra_path: 1530 url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '\\1', url) 1531 else: 1532 url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '', url) 1533 return url.replace('%path%', extra_path) 1534 else: 1535 return url + extra_path 1536 return None 1537 1538 def get_vcs_link_url(self): 1539 if self.layer.vcs_web_url is None: 1540 return None 1541 return self.layer.vcs_web_url 1542 1543 def get_vcs_file_link_url(self, file_path=""): 1544 if self.layer.vcs_web_file_base_url is None: 1545 return None 1546 return self._handle_url_path(self.layer.vcs_web_file_base_url, 1547 file_path) 1548 1549 def get_vcs_dirpath_link_url(self): 1550 if self.layer.vcs_web_tree_base_url is None: 1551 return None 1552 return self._handle_url_path(self.layer.vcs_web_tree_base_url, '') 1553 1554 def get_vcs_reference(self): 1555 if self.commit is not None and len(self.commit) > 0: 1556 return self.commit 1557 if self.branch is not None and len(self.branch) > 0: 1558 return self.branch 1559 if self.release is not None: 1560 return self.release.name 1561 return 'N/A' 1562 1563 def get_detailspage_url(self, project_id=None): 1564 """ returns the url to the layer details page uses own project 1565 field if project_id is not specified """ 1566 1567 if project_id is None: 1568 project_id = self.project.pk 1569 1570 return reverse('layerdetails', args=(project_id, self.pk)) 1571 1572 def get_alldeps(self, project_id): 1573 """Get full list of unique layer dependencies.""" 1574 def gen_layerdeps(lver, project, depth): 1575 if depth == 0: 1576 return 1577 for ldep in lver.dependencies.all(): 1578 yield ldep.depends_on 1579 # get next level of deps recursively calling gen_layerdeps 1580 for subdep in gen_layerdeps(ldep.depends_on, project, depth-1): 1581 yield subdep 1582 1583 project = Project.objects.get(pk=project_id) 1584 result = [] 1585 projectlvers = [player.layercommit for player in 1586 project.projectlayer_set.all()] 1587 # protect against infinite layer dependency loops 1588 maxdepth = 20 1589 for dep in gen_layerdeps(self, project, maxdepth): 1590 # filter out duplicates and layers already belonging to the project 1591 if dep not in result + projectlvers: 1592 result.append(dep) 1593 1594 return sorted(result, key=lambda x: x.layer.name) 1595 1596 def __unicode__(self): 1597 return ("id %d belongs to layer: %s" % (self.pk, self.layer.name)) 1598 1599 def __str__(self): 1600 if self.release: 1601 release = self.release.name 1602 else: 1603 release = "No release set" 1604 1605 return "%d %s (%s)" % (self.pk, self.layer.name, release) 1606 1607 1608class LayerVersionDependency(models.Model): 1609 1610 layer_version = models.ForeignKey(Layer_Version, on_delete=models.CASCADE, 1611 related_name="dependencies") 1612 depends_on = models.ForeignKey(Layer_Version, on_delete=models.CASCADE, 1613 related_name="dependees") 1614 1615class ProjectLayer(models.Model): 1616 project = models.ForeignKey(Project, on_delete=models.CASCADE) 1617 layercommit = models.ForeignKey(Layer_Version, on_delete=models.CASCADE, null=True) 1618 optional = models.BooleanField(default = True) 1619 1620 def __unicode__(self): 1621 return "%s, %s" % (self.project.name, self.layercommit) 1622 1623 class Meta: 1624 unique_together = (("project", "layercommit"),) 1625 1626class CustomImageRecipe(Recipe): 1627 1628 # CustomImageRecipe's belong to layers called: 1629 LAYER_NAME = "toaster-custom-images" 1630 1631 search_allowed_fields = ['name'] 1632 base_recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE, related_name='based_on_recipe') 1633 project = models.ForeignKey(Project, on_delete=models.CASCADE) 1634 last_updated = models.DateTimeField(null=True, default=None) 1635 1636 def get_last_successful_built_target(self): 1637 """ Return the last successful built target object if one exists 1638 otherwise return None """ 1639 return Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) & 1640 Q(build__project=self.project) & 1641 Q(target=self.name)).last() 1642 1643 def update_package_list(self): 1644 """ Update the package list from the last good build of this 1645 CustomImageRecipe 1646 """ 1647 # Check if we're aldready up-to-date or not 1648 target = self.get_last_successful_built_target() 1649 if target is None: 1650 # So we've never actually built this Custom recipe but what about 1651 # the recipe it's based on? 1652 target = \ 1653 Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) & 1654 Q(build__project=self.project) & 1655 Q(target=self.base_recipe.name)).last() 1656 if target is None: 1657 return 1658 1659 if target.build.completed_on == self.last_updated: 1660 return 1661 1662 self.includes_set.clear() 1663 1664 excludes_list = self.excludes_set.values_list('name', flat=True) 1665 appends_list = self.appends_set.values_list('name', flat=True) 1666 1667 built_packages_list = \ 1668 target.target_installed_package_set.values_list('package__name', 1669 flat=True) 1670 for built_package in built_packages_list: 1671 # Is the built package in the custom packages list? 1672 if built_package in excludes_list: 1673 continue 1674 1675 if built_package in appends_list: 1676 continue 1677 1678 cust_img_p = \ 1679 CustomImagePackage.objects.get(name=built_package) 1680 self.includes_set.add(cust_img_p) 1681 1682 1683 self.last_updated = target.build.completed_on 1684 self.save() 1685 1686 def get_all_packages(self): 1687 """Get the included packages and any appended packages""" 1688 self.update_package_list() 1689 1690 return CustomImagePackage.objects.filter((Q(recipe_appends=self) | 1691 Q(recipe_includes=self)) & 1692 ~Q(recipe_excludes=self)) 1693 1694 def get_base_recipe_file(self): 1695 """Get the base recipe file path if it exists on the file system""" 1696 path_schema_one = "%s/%s" % (self.base_recipe.layer_version.local_path, 1697 self.base_recipe.file_path) 1698 1699 path_schema_two = self.base_recipe.file_path 1700 1701 path_schema_three = "%s/%s" % (self.base_recipe.layer_version.layer.local_source_dir, 1702 self.base_recipe.file_path) 1703 1704 if os.path.exists(path_schema_one): 1705 return path_schema_one 1706 1707 # The path may now be the full path if the recipe has been built 1708 if os.path.exists(path_schema_two): 1709 return path_schema_two 1710 1711 # Or a local path if all layers are local 1712 if os.path.exists(path_schema_three): 1713 return path_schema_three 1714 1715 return None 1716 1717 def generate_recipe_file_contents(self): 1718 """Generate the contents for the recipe file.""" 1719 # If we have no excluded packages we only need to :append 1720 if self.excludes_set.count() == 0: 1721 packages_conf = "IMAGE_INSTALL:append = \" " 1722 1723 for pkg in self.appends_set.all(): 1724 packages_conf += pkg.name+' ' 1725 else: 1726 packages_conf = "IMAGE_FEATURES =\"\"\nIMAGE_INSTALL = \"" 1727 # We add all the known packages to be built by this recipe apart 1728 # from locale packages which are are controlled with IMAGE_LINGUAS. 1729 for pkg in self.get_all_packages().exclude( 1730 name__icontains="locale"): 1731 packages_conf += pkg.name+' ' 1732 1733 packages_conf += "\"" 1734 1735 base_recipe_path = self.get_base_recipe_file() 1736 if base_recipe_path: 1737 base_recipe = open(base_recipe_path, 'r').read() 1738 else: 1739 # Pass back None to trigger error message to user 1740 return None 1741 1742 # Add a special case for when the recipe we have based a custom image 1743 # recipe on requires another recipe. 1744 # For example: 1745 # "require core-image-minimal.bb" is changed to: 1746 # "require recipes-core/images/core-image-minimal.bb" 1747 1748 req_search = re.search(r'(require\s+)(.+\.bb\s*$)', 1749 base_recipe, 1750 re.MULTILINE) 1751 if req_search: 1752 require_filename = req_search.group(2).strip() 1753 1754 corrected_location = Recipe.objects.filter( 1755 Q(layer_version=self.base_recipe.layer_version) & 1756 Q(file_path__icontains=require_filename)).last().file_path 1757 1758 new_require_line = "require %s" % corrected_location 1759 1760 base_recipe = base_recipe.replace(req_search.group(0), 1761 new_require_line) 1762 1763 info = { 1764 "date": timezone.now().strftime("%Y-%m-%d %H:%M:%S"), 1765 "base_recipe": base_recipe, 1766 "recipe_name": self.name, 1767 "base_recipe_name": self.base_recipe.name, 1768 "license": self.license, 1769 "summary": self.summary, 1770 "description": self.description, 1771 "packages_conf": packages_conf.strip() 1772 } 1773 1774 recipe_contents = ("# Original recipe %(base_recipe_name)s \n" 1775 "%(base_recipe)s\n\n" 1776 "# Recipe %(recipe_name)s \n" 1777 "# Customisation Generated by Toaster on %(date)s\n" 1778 "SUMMARY = \"%(summary)s\"\n" 1779 "DESCRIPTION = \"%(description)s\"\n" 1780 "LICENSE = \"%(license)s\"\n" 1781 "%(packages_conf)s") % info 1782 1783 return recipe_contents 1784 1785class ProjectVariable(models.Model): 1786 project = models.ForeignKey(Project, on_delete=models.CASCADE) 1787 name = models.CharField(max_length=100) 1788 value = models.TextField(blank = True) 1789 1790class Variable(models.Model): 1791 search_allowed_fields = ['variable_name', 'variable_value', 1792 'vhistory__file_name', "description"] 1793 build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='variable_build') 1794 variable_name = models.CharField(max_length=100) 1795 variable_value = models.TextField(blank=True) 1796 changed = models.BooleanField(default=False) 1797 human_readable_name = models.CharField(max_length=200) 1798 description = models.TextField(blank=True) 1799 1800class VariableHistory(models.Model): 1801 variable = models.ForeignKey(Variable, on_delete=models.CASCADE, related_name='vhistory') 1802 value = models.TextField(blank=True) 1803 file_name = models.FilePathField(max_length=255) 1804 line_number = models.IntegerField(null=True) 1805 operation = models.CharField(max_length=64) 1806 1807class HelpText(models.Model): 1808 VARIABLE = 0 1809 HELPTEXT_AREA = ((VARIABLE, 'variable'), ) 1810 1811 build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='helptext_build') 1812 area = models.IntegerField(choices=HELPTEXT_AREA) 1813 key = models.CharField(max_length=100) 1814 text = models.TextField() 1815 1816class LogMessage(models.Model): 1817 EXCEPTION = -1 # used to signal self-toaster-exceptions 1818 INFO = 0 1819 WARNING = 1 1820 ERROR = 2 1821 CRITICAL = 3 1822 1823 LOG_LEVEL = ( 1824 (INFO, "info"), 1825 (WARNING, "warn"), 1826 (ERROR, "error"), 1827 (CRITICAL, "critical"), 1828 (EXCEPTION, "toaster exception") 1829 ) 1830 1831 build = models.ForeignKey(Build, on_delete=models.CASCADE) 1832 task = models.ForeignKey(Task, on_delete=models.CASCADE, blank = True, null=True) 1833 level = models.IntegerField(choices=LOG_LEVEL, default=INFO) 1834 message = models.TextField(blank=True, null=True) 1835 pathname = models.FilePathField(max_length=255, blank=True) 1836 lineno = models.IntegerField(null=True) 1837 1838 def __str__(self): 1839 return force_bytes('%s %s %s' % (self.get_level_display(), self.message, self.build)) 1840 1841def invalidate_cache(**kwargs): 1842 from django.core.cache import cache 1843 try: 1844 cache.clear() 1845 except Exception as e: 1846 logger.warning("Problem with cache backend: Failed to clear cache: %s" % e) 1847 1848def signal_runbuilds(): 1849 """Send SIGUSR1 to runbuilds process""" 1850 try: 1851 with open(os.path.join(os.getenv('BUILDDIR', '.'), 1852 '.runbuilds.pid')) as pidf: 1853 os.kill(int(pidf.read()), SIGUSR1) 1854 except FileNotFoundError: 1855 logger.info("Stopping existing runbuilds: no current process found") 1856 1857class Distro(models.Model): 1858 search_allowed_fields = ["name", "description", "layer_version__layer__name"] 1859 up_date = models.DateTimeField(null = True, default = None) 1860 1861 layer_version = models.ForeignKey('Layer_Version', on_delete=models.CASCADE) 1862 name = models.CharField(max_length=255) 1863 description = models.CharField(max_length=255) 1864 1865 def get_vcs_distro_file_link_url(self): 1866 path = 'conf/distro/%s.conf' % self.name 1867 return self.layer_version.get_vcs_file_link_url(path) 1868 1869 def __unicode__(self): 1870 return "Distro " + self.name + "(" + self.description + ")" 1871 1872django.db.models.signals.post_save.connect(invalidate_cache) 1873django.db.models.signals.post_delete.connect(invalidate_cache) 1874django.db.models.signals.m2m_changed.connect(invalidate_cache) 1875