1#
2# BitBake Toaster Implementation
3#
4# Copyright (C) 2013        Intel Corporation
5#
6# SPDX-License-Identifier: GPL-2.0-only
7#
8
9from __future__ import unicode_literals
10
11from django.db import models, IntegrityError, DataError
12from django.db.models import F, Q, Sum, Count
13from django.utils import timezone
14from django.utils.encoding import force_bytes
15
16from django.urls import reverse
17
18from django.core import validators
19from django.conf import settings
20import django.db.models.signals
21
22import sys
23import os
24import re
25import itertools
26from signal import SIGUSR1
27
28
29import logging
30logger = logging.getLogger("toaster")
31
32if 'sqlite' in settings.DATABASES['default']['ENGINE']:
33    from django.db import transaction, OperationalError
34    from time import sleep
35
36    _base_save = models.Model.save
37    def save(self, *args, **kwargs):
38        while True:
39            try:
40                with transaction.atomic():
41                    return _base_save(self, *args, **kwargs)
42            except OperationalError as err:
43                if 'database is locked' in str(err):
44                    logger.warning("%s, model: %s, args: %s, kwargs: %s",
45                                   err, self.__class__, args, kwargs)
46                    sleep(0.5)
47                    continue
48                raise
49
50    models.Model.save = save
51
52    # HACK: Monkey patch Django to fix 'database is locked' issue
53
54    from django.db.models.query import QuerySet
55    _base_insert = QuerySet._insert
56    def _insert(self,  *args, **kwargs):
57        with transaction.atomic(using=self.db, savepoint=False):
58            return _base_insert(self, *args, **kwargs)
59    QuerySet._insert = _insert
60
61    def _create_object_from_params(self, lookup, params):
62        """
63        Tries to create an object using passed params.
64        Used by get_or_create and update_or_create
65        """
66        try:
67            obj = self.create(**params)
68            return obj, True
69        except (IntegrityError, DataError):
70            exc_info = sys.exc_info()
71            try:
72                return self.get(**lookup), False
73            except self.model.DoesNotExist:
74                pass
75            six.reraise(*exc_info)
76
77    QuerySet._create_object_from_params = _create_object_from_params
78
79    # end of HACK
80
81class GitURLValidator(validators.URLValidator):
82    import re
83    regex = re.compile(
84        r'^(?:ssh|git|http|ftp)s?://'  # http:// or https://
85        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # domain...
86        r'localhost|'  # localhost...
87        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|'  # ...or ipv4
88        r'\[?[A-F0-9]*:[A-F0-9:]+\]?)'  # ...or ipv6
89        r'(?::\d+)?'  # optional port
90        r'(?:/?|[/?]\S+)$', re.IGNORECASE)
91
92def GitURLField(**kwargs):
93    r = models.URLField(**kwargs)
94    for i in range(len(r.validators)):
95        if isinstance(r.validators[i], validators.URLValidator):
96            r.validators[i] = GitURLValidator()
97    return r
98
99
100class ToasterSetting(models.Model):
101    name = models.CharField(max_length=63)
102    helptext = models.TextField()
103    value = models.CharField(max_length=255)
104
105    def __unicode__(self):
106        return "Setting %s = %s" % (self.name, self.value)
107
108
109class ProjectManager(models.Manager):
110    def create_project(self, name, release, existing_project=None):
111        if existing_project and (release is not None):
112            prj = existing_project
113            prj.bitbake_version = release.bitbake_version
114            prj.release = release
115            # Delete the previous ProjectLayer mappings
116            for pl in ProjectLayer.objects.filter(project=prj):
117                pl.delete()
118        elif release is not None:
119            prj = self.model(name=name,
120                             bitbake_version=release.bitbake_version,
121                             release=release)
122        else:
123            prj = self.model(name=name,
124                             bitbake_version=None,
125                             release=None)
126        prj.save()
127
128        for defaultconf in ToasterSetting.objects.filter(
129                name__startswith="DEFCONF_"):
130            name = defaultconf.name[8:]
131            pv,create = ProjectVariable.objects.get_or_create(project=prj,name=name)
132            pv.value = defaultconf.value
133            pv.save()
134
135        if release is None:
136            return prj
137
138        for rdl in release.releasedefaultlayer_set.all():
139            lv = Layer_Version.objects.filter(
140                layer__name=rdl.layer_name,
141                release=release).first()
142
143            if lv:
144                ProjectLayer.objects.create(project=prj,
145                                            layercommit=lv,
146                                            optional=False)
147            else:
148                logger.warning("Default project layer %s not found" %
149                               rdl.layer_name)
150
151        return prj
152
153    # return single object with is_default = True
154    def get_or_create_default_project(self):
155        projects = super(ProjectManager, self).filter(is_default=True)
156
157        if len(projects) > 1:
158            raise Exception('Inconsistent project data: multiple ' +
159                            'default projects (i.e. with is_default=True)')
160        elif len(projects) < 1:
161            options = {
162                'name': 'Command line builds',
163                'short_description':
164                'Project for builds started outside Toaster',
165                'is_default': True
166            }
167            project = Project.objects.create(**options)
168            project.save()
169
170            return project
171        else:
172            return projects[0]
173
174
175class Project(models.Model):
176    search_allowed_fields = ['name', 'short_description', 'release__name',
177                             'release__branch_name']
178    name = models.CharField(max_length=100)
179    short_description = models.CharField(max_length=50, blank=True)
180    bitbake_version = models.ForeignKey('BitbakeVersion', on_delete=models.CASCADE, null=True)
181    release = models.ForeignKey("Release", on_delete=models.CASCADE, null=True)
182    created = models.DateTimeField(auto_now_add=True)
183    updated = models.DateTimeField(auto_now=True)
184    # This is a horrible hack; since Toaster has no "User" model available when
185    # running in interactive mode, we can't reference the field here directly
186    # Instead, we keep a possible null reference to the User id,
187    # as not to force
188    # hard links to possibly missing models
189    user_id = models.IntegerField(null=True)
190    objects = ProjectManager()
191
192    # build directory override (e.g. imported)
193    builddir = models.TextField()
194    # merge the Toaster configure attributes directly into the standard conf files
195    merged_attr = models.BooleanField(default=False)
196
197    # set to True for the project which is the default container
198    # for builds initiated by the command line etc.
199    is_default= models.BooleanField(default=False)
200
201    def __unicode__(self):
202        return "%s (Release %s, BBV %s)" % (self.name, self.release, self.bitbake_version)
203
204    def get_current_machine_name(self):
205        try:
206            return self.projectvariable_set.get(name="MACHINE").value
207        except (ProjectVariable.DoesNotExist,IndexError):
208            return None;
209
210    def get_number_of_builds(self):
211        """Return the number of builds which have ended"""
212
213        return self.build_set.exclude(
214            Q(outcome=Build.IN_PROGRESS) |
215            Q(outcome=Build.CANCELLED)
216        ).count()
217
218    def get_last_build_id(self):
219        try:
220            return Build.objects.filter( project = self.id ).order_by('-completed_on')[0].id
221        except (Build.DoesNotExist,IndexError):
222            return( -1 )
223
224    def get_last_outcome(self):
225        build_id = self.get_last_build_id()
226        if (-1 == build_id):
227            return( "" )
228        try:
229            return Build.objects.filter( id = build_id )[ 0 ].outcome
230        except (Build.DoesNotExist,IndexError):
231            return( "not_found" )
232
233    def get_last_target(self):
234        build_id = self.get_last_build_id()
235        if (-1 == build_id):
236            return( "" )
237        try:
238            return Target.objects.filter(build = build_id)[0].target
239        except (Target.DoesNotExist,IndexError):
240            return( "not_found" )
241
242    def get_last_errors(self):
243        build_id = self.get_last_build_id()
244        if (-1 == build_id):
245            return( 0 )
246        try:
247            return Build.objects.filter(id = build_id)[ 0 ].errors.count()
248        except (Build.DoesNotExist,IndexError):
249            return( "not_found" )
250
251    def get_last_warnings(self):
252        build_id = self.get_last_build_id()
253        if (-1 == build_id):
254            return( 0 )
255        try:
256            return Build.objects.filter(id = build_id)[ 0 ].warnings.count()
257        except (Build.DoesNotExist,IndexError):
258            return( "not_found" )
259
260    def get_last_build_extensions(self):
261        """
262        Get list of file name extensions for images produced by the most
263        recent build
264        """
265        last_build = Build.objects.get(pk = self.get_last_build_id())
266        return last_build.get_image_file_extensions()
267
268    def get_last_imgfiles(self):
269        build_id = self.get_last_build_id()
270        if (-1 == build_id):
271            return( "" )
272        try:
273            return Variable.objects.filter(build = build_id, variable_name = "IMAGE_FSTYPES")[ 0 ].variable_value
274        except (Variable.DoesNotExist,IndexError):
275            return( "not_found" )
276
277    def get_all_compatible_layer_versions(self):
278        """ Returns Queryset of all Layer_Versions which are compatible with
279        this project"""
280        queryset = None
281
282        # guard on release, as it can be null
283        if self.release:
284            queryset = Layer_Version.objects.filter(
285                (Q(release=self.release) &
286                 Q(build=None) &
287                 Q(project=None)) |
288                 Q(project=self))
289        else:
290            queryset = Layer_Version.objects.none()
291
292        return queryset
293
294    def get_project_layer_versions(self, pk=False):
295        """ Returns the Layer_Versions currently added to this project """
296        layer_versions = self.projectlayer_set.all().values_list('layercommit',
297                                                                 flat=True)
298
299        if pk is False:
300            return Layer_Version.objects.filter(pk__in=layer_versions)
301        else:
302            return layer_versions
303
304
305    def get_default_image_recipe(self):
306        try:
307            return self.projectvariable_set.get(name="DEFAULT_IMAGE").value
308        except (ProjectVariable.DoesNotExist,IndexError):
309            return None;
310
311    def get_is_new(self):
312        return self.get_variable(Project.PROJECT_SPECIFIC_ISNEW)
313
314    def get_available_machines(self):
315        """ Returns QuerySet of all Machines which are provided by the
316        Layers currently added to the Project """
317        queryset = Machine.objects.filter(
318            layer_version__in=self.get_project_layer_versions())
319
320        return queryset
321
322    def get_all_compatible_machines(self):
323        """ Returns QuerySet of all the compatible machines available to the
324        project including ones from Layers not currently added """
325        queryset = Machine.objects.filter(
326            layer_version__in=self.get_all_compatible_layer_versions())
327
328        return queryset
329
330    def get_available_distros(self):
331        """ Returns QuerySet of all Distros which are provided by the
332        Layers currently added to the Project """
333        queryset = Distro.objects.filter(
334            layer_version__in=self.get_project_layer_versions())
335
336        return queryset
337
338    def get_all_compatible_distros(self):
339        """ Returns QuerySet of all the compatible Wind River distros available to the
340        project including ones from Layers not currently added """
341        queryset = Distro.objects.filter(
342            layer_version__in=self.get_all_compatible_layer_versions())
343
344        return queryset
345
346    def get_available_recipes(self):
347        """ Returns QuerySet of all the recipes that are provided by layers
348        added to this project """
349        queryset = Recipe.objects.filter(
350            layer_version__in=self.get_project_layer_versions())
351
352        return queryset
353
354    def get_all_compatible_recipes(self):
355        """ Returns QuerySet of all the compatible Recipes available to the
356        project including ones from Layers not currently added """
357        queryset = Recipe.objects.filter(
358            layer_version__in=self.get_all_compatible_layer_versions()).exclude(name__exact='')
359
360        return queryset
361
362    # Project Specific status management
363    PROJECT_SPECIFIC_STATUS = 'INTERNAL_PROJECT_SPECIFIC_STATUS'
364    PROJECT_SPECIFIC_CALLBACK = 'INTERNAL_PROJECT_SPECIFIC_CALLBACK'
365    PROJECT_SPECIFIC_ISNEW = 'INTERNAL_PROJECT_SPECIFIC_ISNEW'
366    PROJECT_SPECIFIC_DEFAULTIMAGE = 'PROJECT_SPECIFIC_DEFAULTIMAGE'
367    PROJECT_SPECIFIC_NONE = ''
368    PROJECT_SPECIFIC_NEW = '1'
369    PROJECT_SPECIFIC_EDIT = '2'
370    PROJECT_SPECIFIC_CLONING = '3'
371    PROJECT_SPECIFIC_CLONING_SUCCESS = '4'
372    PROJECT_SPECIFIC_CLONING_FAIL = '5'
373
374    def get_variable(self,variable,default_value = ''):
375        try:
376            return self.projectvariable_set.get(name=variable).value
377        except (ProjectVariable.DoesNotExist,IndexError):
378            return default_value
379
380    def set_variable(self,variable,value):
381        pv,create = ProjectVariable.objects.get_or_create(project = self, name = variable)
382        pv.value = value
383        pv.save()
384
385    def get_default_image(self):
386        return self.get_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE)
387
388    def schedule_build(self):
389
390        from bldcontrol.models import BuildRequest, BRTarget, BRLayer
391        from bldcontrol.models import BRBitbake, BRVariable
392
393        try:
394            now = timezone.now()
395            build = Build.objects.create(project=self,
396                                         completed_on=now,
397                                         started_on=now)
398
399            br = BuildRequest.objects.create(project=self,
400                                             state=BuildRequest.REQ_QUEUED,
401                                             build=build)
402            BRBitbake.objects.create(req=br,
403                                     giturl=self.bitbake_version.giturl,
404                                     commit=self.bitbake_version.branch,
405                                     dirpath=self.bitbake_version.dirpath)
406
407            for t in self.projecttarget_set.all():
408                BRTarget.objects.create(req=br, target=t.target, task=t.task)
409                Target.objects.create(build=br.build, target=t.target,
410                                      task=t.task)
411                # If we're about to build a custom image recipe make sure
412                # that layer is currently in the project before we create the
413                # BRLayer objects
414                customrecipe = CustomImageRecipe.objects.filter(
415                    name=t.target,
416                    project=self).first()
417                if customrecipe:
418                    ProjectLayer.objects.get_or_create(
419                        project=self,
420                        layercommit=customrecipe.layer_version,
421                        optional=False)
422
423            for l in self.projectlayer_set.all().order_by("pk"):
424                commit = l.layercommit.get_vcs_reference()
425                logger.debug("Adding layer to build %s" %
426                             l.layercommit.layer.name)
427                BRLayer.objects.create(
428                    req=br,
429                    name=l.layercommit.layer.name,
430                    giturl=l.layercommit.layer.vcs_url,
431                    commit=commit,
432                    dirpath=l.layercommit.dirpath,
433                    layer_version=l.layercommit,
434                    local_source_dir=l.layercommit.layer.local_source_dir
435                )
436
437            for v in self.projectvariable_set.all():
438                BRVariable.objects.create(req=br, name=v.name, value=v.value)
439
440            try:
441                br.build.machine = self.projectvariable_set.get(
442                    name='MACHINE').value
443                br.build.save()
444            except ProjectVariable.DoesNotExist:
445                pass
446
447            br.save()
448            signal_runbuilds()
449
450        except Exception:
451            # revert the build request creation since we're not done cleanly
452            br.delete()
453            raise
454        return br
455
456class Build(models.Model):
457    SUCCEEDED = 0
458    FAILED = 1
459    IN_PROGRESS = 2
460    CANCELLED = 3
461
462    BUILD_OUTCOME = (
463        (SUCCEEDED, 'Succeeded'),
464        (FAILED, 'Failed'),
465        (IN_PROGRESS, 'In Progress'),
466        (CANCELLED, 'Cancelled'),
467    )
468
469    search_allowed_fields = ['machine', 'cooker_log_path', "target__target", "target__target_image_file__file_name"]
470
471    project = models.ForeignKey(Project, on_delete=models.CASCADE)            # must have a project
472    machine = models.CharField(max_length=100)
473    distro = models.CharField(max_length=100)
474    distro_version = models.CharField(max_length=100)
475    started_on = models.DateTimeField()
476    completed_on = models.DateTimeField()
477    outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS)
478    cooker_log_path = models.CharField(max_length=500)
479    build_name = models.CharField(max_length=100, default='')
480    bitbake_version = models.CharField(max_length=50)
481
482    # number of recipes to parse for this build
483    recipes_to_parse = models.IntegerField(default=1)
484
485    # number of recipes parsed so far for this build
486    recipes_parsed = models.IntegerField(default=1)
487
488    # number of repos to clone for this build
489    repos_to_clone = models.IntegerField(default=1)
490
491    # number of repos cloned so far for this build (default off)
492    repos_cloned = models.IntegerField(default=1)
493
494    # Hint on current progress item
495    progress_item = models.CharField(max_length=40)
496
497    @staticmethod
498    def get_recent(project=None):
499        """
500        Return recent builds as a list; if project is set, only return
501        builds for that project
502        """
503
504        builds = Build.objects.all()
505
506        if project:
507            builds = builds.filter(project=project)
508
509        finished_criteria = \
510                Q(outcome=Build.SUCCEEDED) | \
511                Q(outcome=Build.FAILED) | \
512                Q(outcome=Build.CANCELLED)
513
514        recent_builds = list(itertools.chain(
515            builds.filter(outcome=Build.IN_PROGRESS).order_by("-started_on"),
516            builds.filter(finished_criteria).order_by("-completed_on")[:3]
517        ))
518
519        # add percentage done property to each build; this is used
520        # to show build progress in mrb_section.html
521        for build in recent_builds:
522            build.percentDone = build.completeper()
523            build.outcomeText = build.get_outcome_text()
524
525        return recent_builds
526
527    def started(self):
528        """
529        As build variables are only added for a build when its BuildStarted event
530        is received, a build with no build variables is counted as
531        "in preparation" and not properly started yet. This method
532        will return False if a build has no build variables (it never properly
533        started), or True otherwise.
534
535        Note that this is a temporary workaround for the fact that we don't
536        have a fine-grained state variable on a build which would allow us
537        to record "in progress" (BuildStarted received) vs. "in preparation".
538        """
539        variables = Variable.objects.filter(build=self)
540        return len(variables) > 0
541
542    def completeper(self):
543        tf = Task.objects.filter(build = self)
544        tfc = tf.count()
545        if tfc > 0:
546            completeper = tf.exclude(outcome=Task.OUTCOME_NA).count()*100 // tfc
547        else:
548            completeper = 0
549        return completeper
550
551    def eta(self):
552        eta = timezone.now()
553        completeper = self.completeper()
554        if self.completeper() > 0:
555            eta += ((eta - self.started_on)*(100-completeper))/completeper
556        return eta
557
558    def has_images(self):
559        """
560        Returns True if at least one of the targets for this build has an
561        image file associated with it, False otherwise
562        """
563        targets = Target.objects.filter(build_id=self.id)
564        has_images = False
565        for target in targets:
566            if target.has_images():
567                has_images = True
568                break
569        return has_images
570
571    def has_image_recipes(self):
572        """
573        Returns True if a build has any targets which were built from
574        image recipes.
575        """
576        image_recipes = self.get_image_recipes()
577        return len(image_recipes) > 0
578
579    def get_image_file_extensions(self):
580        """
581        Get string of file name extensions for images produced by this build;
582        note that this is the actual list of extensions stored on Target objects
583        for this build, and not the value of IMAGE_FSTYPES.
584
585        Returns comma-separated string, e.g. "vmdk, ext4"
586        """
587        extensions = []
588
589        targets = Target.objects.filter(build_id = self.id)
590        for target in targets:
591            if not target.is_image:
592                continue
593
594            target_image_files = Target_Image_File.objects.filter(
595                target_id=target.id)
596
597            for target_image_file in target_image_files:
598                extensions.append(target_image_file.suffix)
599
600        extensions = list(set(extensions))
601        extensions.sort()
602
603        return ', '.join(extensions)
604
605    def get_image_fstypes(self):
606        """
607        Get the IMAGE_FSTYPES variable value for this build as a de-duplicated
608        list of image file suffixes.
609        """
610        image_fstypes = Variable.objects.get(
611            build=self, variable_name='IMAGE_FSTYPES').variable_value
612        return list(set(re.split(r' {1,}', image_fstypes)))
613
614    def get_sorted_target_list(self):
615        tgts = Target.objects.filter(build_id = self.id).order_by( 'target' );
616        return( tgts );
617
618    def get_recipes(self):
619        """
620        Get the recipes related to this build;
621        note that the related layer versions and layers are also prefetched
622        by this query, as this queryset can be sorted by these objects in the
623        build recipes view; prefetching them here removes the need
624        for another query in that view
625        """
626        layer_versions = Layer_Version.objects.filter(build=self)
627        criteria = Q(layer_version__id__in=layer_versions)
628        return Recipe.objects.filter(criteria) \
629                             .select_related('layer_version', 'layer_version__layer')
630
631    def get_image_recipes(self):
632        """
633        Returns a list of image Recipes (custom and built-in) related to this
634        build, sorted by name; note that this has to be done in two steps, as
635        there's no way to get all the custom image recipes and image recipes
636        in one query
637        """
638        custom_image_recipes = self.get_custom_image_recipes()
639        custom_image_recipe_names = custom_image_recipes.values_list('name', flat=True)
640
641        not_custom_image_recipes = ~Q(name__in=custom_image_recipe_names) & \
642                                   Q(is_image=True)
643
644        built_image_recipes = self.get_recipes().filter(not_custom_image_recipes)
645
646        # append to the custom image recipes and sort
647        customisable_image_recipes = list(
648            itertools.chain(custom_image_recipes, built_image_recipes)
649        )
650
651        return sorted(customisable_image_recipes, key=lambda recipe: recipe.name)
652
653    def get_custom_image_recipes(self):
654        """
655        Returns a queryset of CustomImageRecipes related to this build,
656        sorted by name
657        """
658        built_recipe_names = self.get_recipes().values_list('name', flat=True)
659        criteria = Q(name__in=built_recipe_names) & Q(project=self.project)
660        queryset = CustomImageRecipe.objects.filter(criteria).order_by('name')
661        return queryset
662
663    def get_outcome_text(self):
664        return Build.BUILD_OUTCOME[int(self.outcome)][1]
665
666    @property
667    def failed_tasks(self):
668        """ Get failed tasks for the build """
669        tasks = self.task_build.all()
670        return tasks.filter(order__gt=0, outcome=Task.OUTCOME_FAILED)
671
672    @property
673    def errors(self):
674        return (self.logmessage_set.filter(level=LogMessage.ERROR) |
675                self.logmessage_set.filter(level=LogMessage.EXCEPTION) |
676                self.logmessage_set.filter(level=LogMessage.CRITICAL))
677
678    @property
679    def warnings(self):
680        return self.logmessage_set.filter(level=LogMessage.WARNING)
681
682    @property
683    def timespent(self):
684        return self.completed_on - self.started_on
685
686    @property
687    def timespent_seconds(self):
688        return self.timespent.total_seconds()
689
690    @property
691    def target_labels(self):
692        """
693        Sorted (a-z) "target1:task, target2, target3" etc. string for all
694        targets in this build
695        """
696        targets = self.target_set.all()
697        target_labels = [target.target +
698                         (':' + target.task if target.task else '')
699                         for target in targets]
700        target_labels.sort()
701
702        return target_labels
703
704    def get_buildrequest(self):
705        buildrequest = None
706        if hasattr(self, 'buildrequest'):
707            buildrequest = self.buildrequest
708        return buildrequest
709
710    def is_queued(self):
711        from bldcontrol.models import BuildRequest
712        buildrequest = self.get_buildrequest()
713        if buildrequest:
714            return buildrequest.state == BuildRequest.REQ_QUEUED
715        else:
716            return False
717
718    def is_cancelling(self):
719        from bldcontrol.models import BuildRequest
720        buildrequest = self.get_buildrequest()
721        if buildrequest:
722            return self.outcome == Build.IN_PROGRESS and \
723                buildrequest.state == BuildRequest.REQ_CANCELLING
724        else:
725            return False
726
727    def is_cloning(self):
728        """
729        True if the build is still cloning repos
730        """
731        return self.outcome == Build.IN_PROGRESS and \
732            self.repos_cloned < self.repos_to_clone
733
734    def is_parsing(self):
735        """
736        True if the build is still parsing recipes
737        """
738        return self.outcome == Build.IN_PROGRESS and \
739            self.recipes_parsed < self.recipes_to_parse
740
741    def is_starting(self):
742        """
743        True if the build has no completed tasks yet and is still just starting
744        tasks.
745
746        Note that the mechanism for testing whether a Task is "done" is whether
747        its outcome field is set, as per the completeper() method.
748        """
749        return self.outcome == Build.IN_PROGRESS and \
750            self.task_build.exclude(outcome=Task.OUTCOME_NA).count() == 0
751
752
753    def get_state(self):
754        """
755        Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress',
756        'Cancelled' (Build outcomes); or 'Queued', 'Cancelling' (states
757        dependent on the BuildRequest state).
758
759        This works around the fact that we have BuildRequest states as well
760        as Build states, but really we just want to know the state of the build.
761        """
762        if self.is_cancelling():
763            return 'Cancelling';
764        elif self.is_queued():
765            return 'Queued'
766        elif self.is_cloning():
767            return 'Cloning'
768        elif self.is_parsing():
769            return 'Parsing'
770        elif self.is_starting():
771            return 'Starting'
772        else:
773            return self.get_outcome_text()
774
775    def __str__(self):
776        return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()]))
777
778class ProjectTarget(models.Model):
779    project = models.ForeignKey(Project, on_delete=models.CASCADE)
780    target = models.CharField(max_length=100)
781    task = models.CharField(max_length=100, null=True)
782
783class Target(models.Model):
784    search_allowed_fields = ['target', 'file_name']
785    build = models.ForeignKey(Build, on_delete=models.CASCADE)
786    target = models.CharField(max_length=100)
787    task = models.CharField(max_length=100, null=True)
788    is_image = models.BooleanField(default = False)
789    image_size = models.IntegerField(default=0)
790    license_manifest_path = models.CharField(max_length=500, null=True)
791    package_manifest_path = models.CharField(max_length=500, null=True)
792
793    def package_count(self):
794        return Target_Installed_Package.objects.filter(target_id__exact=self.id).count()
795
796    def __unicode__(self):
797        return self.target
798
799    def get_similar_targets(self):
800        """
801        Get target sfor the same machine, task and target name
802        (e.g. 'core-image-minimal') from a successful build for this project
803        (but excluding this target).
804
805        Note that we only look for targets built by this project because
806        projects can have different configurations from each other, and put
807        their artifacts in different directories.
808
809        The possibility of error when retrieving candidate targets
810        is minimised by the fact that bitbake will rebuild artifacts if MACHINE
811        (or various other variables) change. In this case, there is no need to
812        clone artifacts from another target, as those artifacts will have
813        been re-generated for this target anyway.
814        """
815        query = ~Q(pk=self.pk) & \
816            Q(target=self.target) & \
817            Q(build__machine=self.build.machine) & \
818            Q(build__outcome=Build.SUCCEEDED) & \
819            Q(build__project=self.build.project)
820
821        return Target.objects.filter(query)
822
823    def get_similar_target_with_image_files(self):
824        """
825        Get the most recent similar target with Target_Image_Files associated
826        with it, for the purpose of cloning those files onto this target.
827        """
828        similar_target = None
829
830        candidates = self.get_similar_targets()
831        if candidates.count() == 0:
832            return similar_target
833
834        task_subquery = Q(task=self.task)
835
836        # we can look for a 'build' task if this task is a 'populate_sdk_ext'
837        # task, as the latter also creates images; and vice versa; note that
838        # 'build' targets can have their task set to '';
839        # also note that 'populate_sdk' does not produce image files
840        image_tasks = [
841            '', # aka 'build'
842            'build',
843            'image',
844            'populate_sdk_ext'
845        ]
846        if self.task in image_tasks:
847            task_subquery = Q(task__in=image_tasks)
848
849        # annotate with the count of files, to exclude any targets which
850        # don't have associated files
851        candidates = candidates.annotate(num_files=Count('target_image_file'))
852
853        query = task_subquery & Q(num_files__gt=0)
854
855        candidates = candidates.filter(query)
856
857        if candidates.count() > 0:
858            candidates.order_by('build__completed_on')
859            similar_target = candidates.last()
860
861        return similar_target
862
863    def get_similar_target_with_sdk_files(self):
864        """
865        Get the most recent similar target with TargetSDKFiles associated
866        with it, for the purpose of cloning those files onto this target.
867        """
868        similar_target = None
869
870        candidates = self.get_similar_targets()
871        if candidates.count() == 0:
872            return similar_target
873
874        # annotate with the count of files, to exclude any targets which
875        # don't have associated files
876        candidates = candidates.annotate(num_files=Count('targetsdkfile'))
877
878        query = Q(task=self.task) & Q(num_files__gt=0)
879
880        candidates = candidates.filter(query)
881
882        if candidates.count() > 0:
883            candidates.order_by('build__completed_on')
884            similar_target = candidates.last()
885
886        return similar_target
887
888    def clone_image_artifacts_from(self, target):
889        """
890        Make clones of the Target_Image_Files and TargetKernelFile objects
891        associated with Target target, then associate them with this target.
892
893        Note that for Target_Image_Files, we only want files from the previous
894        build whose suffix matches one of the suffixes defined in this
895        target's build's IMAGE_FSTYPES configuration variable. This prevents the
896        Target_Image_File object for an ext4 image being associated with a
897        target for a project which didn't produce an ext4 image (for example).
898
899        Also sets the license_manifest_path and package_manifest_path
900        of this target to the same path as that of target being cloned from, as
901        the manifests are also build artifacts but are treated differently.
902        """
903
904        image_fstypes = self.build.get_image_fstypes()
905
906        # filter out any image files whose suffixes aren't in the
907        # IMAGE_FSTYPES suffixes variable for this target's build
908        image_files = [target_image_file \
909            for target_image_file in target.target_image_file_set.all() \
910            if target_image_file.suffix in image_fstypes]
911
912        for image_file in image_files:
913            image_file.pk = None
914            image_file.target = self
915            image_file.save()
916
917        kernel_files = target.targetkernelfile_set.all()
918        for kernel_file in kernel_files:
919            kernel_file.pk = None
920            kernel_file.target = self
921            kernel_file.save()
922
923        self.license_manifest_path = target.license_manifest_path
924        self.package_manifest_path = target.package_manifest_path
925        self.save()
926
927    def clone_sdk_artifacts_from(self, target):
928        """
929        Clone TargetSDKFile objects from target and associate them with this
930        target.
931        """
932        sdk_files = target.targetsdkfile_set.all()
933        for sdk_file in sdk_files:
934            sdk_file.pk = None
935            sdk_file.target = self
936            sdk_file.save()
937
938    def has_images(self):
939        """
940        Returns True if this target has one or more image files attached to it.
941        """
942        return self.target_image_file_set.all().count() > 0
943
944# kernel artifacts for a target: bzImage and modules*
945class TargetKernelFile(models.Model):
946    target = models.ForeignKey(Target, on_delete=models.CASCADE)
947    file_name = models.FilePathField()
948    file_size = models.IntegerField()
949
950    @property
951    def basename(self):
952        return os.path.basename(self.file_name)
953
954# SDK artifacts for a target: sh and manifest files
955class TargetSDKFile(models.Model):
956    target = models.ForeignKey(Target, on_delete=models.CASCADE)
957    file_name = models.FilePathField()
958    file_size = models.IntegerField()
959
960    @property
961    def basename(self):
962        return os.path.basename(self.file_name)
963
964class Target_Image_File(models.Model):
965    # valid suffixes for image files produced by a build
966    SUFFIXES = {
967        'btrfs', 'container', 'cpio', 'cpio.gz', 'cpio.lz4', 'cpio.lzma',
968        'cpio.xz', 'cramfs', 'ext2', 'ext2.bz2', 'ext2.gz', 'ext2.lzma',
969        'ext3', 'ext3.gz', 'ext4', 'ext4.gz', 'f2fs', 'hddimg', 'iso', 'jffs2',
970        'jffs2.sum', 'multiubi', 'squashfs', 'squashfs-lz4', 'squashfs-lzo',
971        'squashfs-xz', 'tar', 'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi',
972        'ubifs', 'wic', 'wic.bz2', 'wic.gz', 'wic.lzma'
973    }
974
975    target = models.ForeignKey(Target, on_delete=models.CASCADE)
976    file_name = models.FilePathField(max_length=254)
977    file_size = models.IntegerField()
978
979    @property
980    def suffix(self):
981        """
982        Suffix for image file, minus leading "."
983        """
984        for suffix in Target_Image_File.SUFFIXES:
985            if self.file_name.endswith(suffix):
986                return suffix
987
988        filename, suffix = os.path.splitext(self.file_name)
989        suffix = suffix.lstrip('.')
990        return suffix
991
992class Target_File(models.Model):
993    ITYPE_REGULAR = 1
994    ITYPE_DIRECTORY = 2
995    ITYPE_SYMLINK = 3
996    ITYPE_SOCKET = 4
997    ITYPE_FIFO = 5
998    ITYPE_CHARACTER = 6
999    ITYPE_BLOCK = 7
1000    ITYPES = ( (ITYPE_REGULAR ,'regular'),
1001        ( ITYPE_DIRECTORY ,'directory'),
1002        ( ITYPE_SYMLINK ,'symlink'),
1003        ( ITYPE_SOCKET ,'socket'),
1004        ( ITYPE_FIFO ,'fifo'),
1005        ( ITYPE_CHARACTER ,'character'),
1006        ( ITYPE_BLOCK ,'block'),
1007        )
1008
1009    target = models.ForeignKey(Target, on_delete=models.CASCADE)
1010    path = models.FilePathField()
1011    size = models.IntegerField()
1012    inodetype = models.IntegerField(choices = ITYPES)
1013    permission = models.CharField(max_length=16)
1014    owner = models.CharField(max_length=128)
1015    group = models.CharField(max_length=128)
1016    directory = models.ForeignKey('Target_File', on_delete=models.CASCADE, related_name="directory_set", null=True)
1017    sym_target = models.ForeignKey('Target_File', on_delete=models.CASCADE, related_name="symlink_set", null=True)
1018
1019
1020class Task(models.Model):
1021
1022    SSTATE_NA = 0
1023    SSTATE_MISS = 1
1024    SSTATE_FAILED = 2
1025    SSTATE_RESTORED = 3
1026
1027    SSTATE_RESULT = (
1028        (SSTATE_NA, 'Not Applicable'), # For rest of tasks, but they still need checking.
1029        (SSTATE_MISS, 'File not in cache'), # the sstate object was not found
1030        (SSTATE_FAILED, 'Failed'), # there was a pkg, but the script failed
1031        (SSTATE_RESTORED, 'Succeeded'), # successfully restored
1032    )
1033
1034    CODING_NA = 0
1035    CODING_PYTHON = 2
1036    CODING_SHELL = 3
1037
1038    TASK_CODING = (
1039        (CODING_NA, 'N/A'),
1040        (CODING_PYTHON, 'Python'),
1041        (CODING_SHELL, 'Shell'),
1042    )
1043
1044    OUTCOME_NA = -1
1045    OUTCOME_SUCCESS = 0
1046    OUTCOME_COVERED = 1
1047    OUTCOME_CACHED = 2
1048    OUTCOME_PREBUILT = 3
1049    OUTCOME_FAILED = 4
1050    OUTCOME_EMPTY = 5
1051
1052    TASK_OUTCOME = (
1053        (OUTCOME_NA, 'Not Available'),
1054        (OUTCOME_SUCCESS, 'Succeeded'),
1055        (OUTCOME_COVERED, 'Covered'),
1056        (OUTCOME_CACHED, 'Cached'),
1057        (OUTCOME_PREBUILT, 'Prebuilt'),
1058        (OUTCOME_FAILED, 'Failed'),
1059        (OUTCOME_EMPTY, 'Empty'),
1060    )
1061
1062    TASK_OUTCOME_HELP = (
1063        (OUTCOME_SUCCESS, 'This task successfully completed'),
1064        (OUTCOME_COVERED, 'This task did not run because its output is provided by another task'),
1065        (OUTCOME_CACHED, 'This task restored output from the sstate-cache directory or mirrors'),
1066        (OUTCOME_PREBUILT, 'This task did not run because its outcome was reused from a previous build'),
1067        (OUTCOME_FAILED, 'This task did not complete'),
1068        (OUTCOME_EMPTY, 'This task has no executable content'),
1069        (OUTCOME_NA, ''),
1070    )
1071
1072    search_allowed_fields = [ "recipe__name", "recipe__version", "task_name", "logfile" ]
1073
1074    def __init__(self, *args, **kwargs):
1075        super(Task, self).__init__(*args, **kwargs)
1076        try:
1077            self._helptext = HelpText.objects.get(key=self.task_name, area=HelpText.VARIABLE, build=self.build).text
1078        except HelpText.DoesNotExist:
1079            self._helptext = None
1080
1081    def get_related_setscene(self):
1082        return Task.objects.filter(task_executed=True, build = self.build, recipe = self.recipe, task_name=self.task_name+"_setscene")
1083
1084    def get_outcome_text(self):
1085        return Task.TASK_OUTCOME[int(self.outcome) + 1][1]
1086
1087    def get_outcome_help(self):
1088        return Task.TASK_OUTCOME_HELP[int(self.outcome)][1]
1089
1090    def get_sstate_text(self):
1091        if self.sstate_result==Task.SSTATE_NA:
1092            return ''
1093        else:
1094            return Task.SSTATE_RESULT[int(self.sstate_result)][1]
1095
1096    def get_executed_display(self):
1097        if self.task_executed:
1098            return "Executed"
1099        return "Not Executed"
1100
1101    def get_description(self):
1102        return self._helptext
1103
1104    build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='task_build')
1105    order = models.IntegerField(null=True)
1106    task_executed = models.BooleanField(default=False) # True means Executed, False means Not/Executed
1107    outcome = models.IntegerField(choices=TASK_OUTCOME, default=OUTCOME_NA)
1108    sstate_checksum = models.CharField(max_length=100, blank=True)
1109    path_to_sstate_obj = models.FilePathField(max_length=500, blank=True)
1110    recipe = models.ForeignKey('Recipe', on_delete=models.CASCADE, related_name='tasks')
1111    task_name = models.CharField(max_length=100)
1112    source_url = models.FilePathField(max_length=255, blank=True)
1113    work_directory = models.FilePathField(max_length=255, blank=True)
1114    script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NA)
1115    line_number = models.IntegerField(default=0)
1116
1117    # start/end times
1118    started = models.DateTimeField(null=True)
1119    ended = models.DateTimeField(null=True)
1120
1121    # in seconds; this is stored to enable sorting
1122    elapsed_time = models.DecimalField(max_digits=8, decimal_places=2, null=True)
1123
1124    # in bytes; note that disk_io is stored to enable sorting
1125    disk_io = models.IntegerField(null=True)
1126    disk_io_read = models.IntegerField(null=True)
1127    disk_io_write = models.IntegerField(null=True)
1128
1129    # in seconds
1130    cpu_time_user = models.DecimalField(max_digits=8, decimal_places=2, null=True)
1131    cpu_time_system = models.DecimalField(max_digits=8, decimal_places=2, null=True)
1132
1133    sstate_result = models.IntegerField(choices=SSTATE_RESULT, default=SSTATE_NA)
1134    message = models.CharField(max_length=240)
1135    logfile = models.FilePathField(max_length=255, blank=True)
1136
1137    outcome_text = property(get_outcome_text)
1138    sstate_text  = property(get_sstate_text)
1139
1140    def __unicode__(self):
1141        return "%d(%d) %s:%s" % (self.pk, self.build.pk, self.recipe.name, self.task_name)
1142
1143    class Meta:
1144        ordering = ('order', 'recipe' ,)
1145        unique_together = ('build', 'recipe', 'task_name', )
1146
1147
1148class Task_Dependency(models.Model):
1149    task = models.ForeignKey(Task, on_delete=models.CASCADE, related_name='task_dependencies_task')
1150    depends_on = models.ForeignKey(Task, on_delete=models.CASCADE, related_name='task_dependencies_depends')
1151
1152class Package(models.Model):
1153    search_allowed_fields = ['name', 'version', 'revision', 'recipe__name', 'recipe__version', 'recipe__license', 'recipe__layer_version__layer__name', 'recipe__layer_version__branch', 'recipe__layer_version__commit', 'recipe__layer_version__local_path', 'installed_name']
1154    build = models.ForeignKey('Build', on_delete=models.CASCADE, null=True)
1155    recipe = models.ForeignKey('Recipe', on_delete=models.CASCADE, null=True)
1156    name = models.CharField(max_length=100)
1157    installed_name = models.CharField(max_length=100, default='')
1158    version = models.CharField(max_length=100, blank=True)
1159    revision = models.CharField(max_length=32, blank=True)
1160    summary = models.TextField(blank=True)
1161    description = models.TextField(blank=True)
1162    size = models.IntegerField(default=0)
1163    installed_size = models.IntegerField(default=0)
1164    section = models.CharField(max_length=80, blank=True)
1165    license = models.CharField(max_length=80, blank=True)
1166
1167    @property
1168    def is_locale_package(self):
1169        """ Returns True if this package is identifiable as a locale package """
1170        if self.name.find('locale') != -1:
1171            return True
1172        return False
1173
1174    @property
1175    def is_packagegroup(self):
1176        """ Returns True is this package is identifiable as a packagegroup """
1177        if self.name.find('packagegroup') != -1:
1178            return True
1179        return False
1180
1181class CustomImagePackage(Package):
1182    # CustomImageRecipe fields to track pacakges appended,
1183    # included and excluded from a CustomImageRecipe
1184    recipe_includes = models.ManyToManyField('CustomImageRecipe',
1185                                             related_name='includes_set')
1186    recipe_excludes = models.ManyToManyField('CustomImageRecipe',
1187                                             related_name='excludes_set')
1188    recipe_appends = models.ManyToManyField('CustomImageRecipe',
1189                                            related_name='appends_set')
1190
1191
1192class Package_DependencyManager(models.Manager):
1193    use_for_related_fields = True
1194    TARGET_LATEST = "use-latest-target-for-target"
1195
1196    def get_queryset(self):
1197        return super(Package_DependencyManager, self).get_queryset().exclude(package_id = F('depends_on__id'))
1198
1199    def for_target_or_none(self, target):
1200        """ filter the dependencies to be displayed by the supplied target
1201        if no dependences are found for the target then try None as the target
1202        which will return the dependences calculated without the context of a
1203        target e.g. non image recipes.
1204
1205        returns: { size, packages }
1206        """
1207        package_dependencies = self.all_depends().order_by('depends_on__name')
1208
1209        if target is self.TARGET_LATEST:
1210            installed_deps =\
1211                    package_dependencies.filter(~Q(target__target=None))
1212        else:
1213            installed_deps =\
1214                    package_dependencies.filter(Q(target__target=target))
1215
1216        packages_list = None
1217        total_size = 0
1218
1219        # If we have installed depdencies for this package and target then use
1220        # these to display
1221        if installed_deps.count() > 0:
1222            packages_list = installed_deps
1223            total_size = installed_deps.aggregate(
1224                Sum('depends_on__size'))['depends_on__size__sum']
1225        else:
1226            new_list = []
1227            package_names = []
1228
1229            # Find dependencies for the package that we know about even if
1230            # it's not installed on a target e.g. from a non-image recipe
1231            for p in package_dependencies.filter(Q(target=None)):
1232                if p.depends_on.name in package_names:
1233                    continue
1234                else:
1235                    package_names.append(p.depends_on.name)
1236                    new_list.append(p.pk)
1237                    # while we're here we may as well total up the size to
1238                    # avoid iterating again
1239                    total_size += p.depends_on.size
1240
1241            # We want to return a queryset here for consistency so pick the
1242            # deps from the new_list
1243            packages_list = package_dependencies.filter(Q(pk__in=new_list))
1244
1245        return {'packages': packages_list,
1246                'size': total_size}
1247
1248    def all_depends(self):
1249        """ Returns just the depends packages and not any other dep_type
1250        Note that this is for any target
1251        """
1252        return self.filter(Q(dep_type=Package_Dependency.TYPE_RDEPENDS) |
1253                           Q(dep_type=Package_Dependency.TYPE_TRDEPENDS))
1254
1255
1256class Package_Dependency(models.Model):
1257    TYPE_RDEPENDS = 0
1258    TYPE_TRDEPENDS = 1
1259    TYPE_RRECOMMENDS = 2
1260    TYPE_TRECOMMENDS = 3
1261    TYPE_RSUGGESTS = 4
1262    TYPE_RPROVIDES = 5
1263    TYPE_RREPLACES = 6
1264    TYPE_RCONFLICTS = 7
1265    ' TODO: bpackage should be changed to remove the DEPENDS_TYPE access '
1266    DEPENDS_TYPE = (
1267        (TYPE_RDEPENDS, "depends"),
1268        (TYPE_TRDEPENDS, "depends"),
1269        (TYPE_TRECOMMENDS, "recommends"),
1270        (TYPE_RRECOMMENDS, "recommends"),
1271        (TYPE_RSUGGESTS, "suggests"),
1272        (TYPE_RPROVIDES, "provides"),
1273        (TYPE_RREPLACES, "replaces"),
1274        (TYPE_RCONFLICTS, "conflicts"),
1275    )
1276    """ Indexed by dep_type, in view order, key for short name and help
1277        description which when viewed will be printf'd with the
1278        package name.
1279    """
1280    DEPENDS_DICT = {
1281        TYPE_RDEPENDS :     ("depends", "%s is required to run %s"),
1282        TYPE_TRDEPENDS :    ("depends", "%s is required to run %s"),
1283        TYPE_TRECOMMENDS :  ("recommends", "%s extends the usability of %s"),
1284        TYPE_RRECOMMENDS :  ("recommends", "%s extends the usability of %s"),
1285        TYPE_RSUGGESTS :    ("suggests", "%s is suggested for installation with %s"),
1286        TYPE_RPROVIDES :    ("provides", "%s is provided by %s"),
1287        TYPE_RREPLACES :    ("replaces", "%s is replaced by %s"),
1288        TYPE_RCONFLICTS :   ("conflicts", "%s conflicts with %s, which will not be installed if this package is not first removed"),
1289    }
1290
1291    package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='package_dependencies_source')
1292    depends_on = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='package_dependencies_target')   # soft dependency
1293    dep_type = models.IntegerField(choices=DEPENDS_TYPE)
1294    target = models.ForeignKey(Target, on_delete=models.CASCADE, null=True)
1295    objects = Package_DependencyManager()
1296
1297class Target_Installed_Package(models.Model):
1298    target = models.ForeignKey(Target, on_delete=models.CASCADE)
1299    package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='buildtargetlist_package')
1300
1301
1302class Package_File(models.Model):
1303    package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='buildfilelist_package')
1304    path = models.FilePathField(max_length=255, blank=True)
1305    size = models.IntegerField()
1306
1307
1308class Recipe(models.Model):
1309    search_allowed_fields = ['name', 'version', 'file_path', 'section',
1310                             'summary', 'description', 'license',
1311                             'layer_version__layer__name',
1312                             'layer_version__branch', 'layer_version__commit',
1313                             'layer_version__local_path',
1314                             'layer_version__layer_source']
1315
1316    up_date = models.DateTimeField(null=True, default=None)
1317
1318    name = models.CharField(max_length=100, blank=True)
1319    version = models.CharField(max_length=100, blank=True)
1320    layer_version = models.ForeignKey('Layer_Version', on_delete=models.CASCADE,
1321                                      related_name='recipe_layer_version')
1322    summary = models.TextField(blank=True)
1323    description = models.TextField(blank=True)
1324    section = models.CharField(max_length=100, blank=True)
1325    license = models.CharField(max_length=200, blank=True)
1326    homepage = models.URLField(blank=True)
1327    bugtracker = models.URLField(blank=True)
1328    file_path = models.FilePathField(max_length=255)
1329    pathflags = models.CharField(max_length=200, blank=True)
1330    is_image = models.BooleanField(default=False)
1331
1332    def __unicode__(self):
1333        return "Recipe " + self.name + ":" + self.version
1334
1335    def get_vcs_recipe_file_link_url(self):
1336        return self.layer_version.get_vcs_file_link_url(self.file_path)
1337
1338    def get_description_or_summary(self):
1339        if self.description:
1340            return self.description
1341        elif self.summary:
1342            return self.summary
1343        else:
1344            return ""
1345
1346    class Meta:
1347        unique_together = (("layer_version", "file_path", "pathflags"), )
1348
1349
1350class Recipe_DependencyManager(models.Manager):
1351    use_for_related_fields = True
1352
1353    def get_queryset(self):
1354        return super(Recipe_DependencyManager, self).get_queryset().exclude(recipe_id = F('depends_on__id'))
1355
1356class Provides(models.Model):
1357    name = models.CharField(max_length=100)
1358    recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE)
1359
1360class Recipe_Dependency(models.Model):
1361    TYPE_DEPENDS = 0
1362    TYPE_RDEPENDS = 1
1363
1364    DEPENDS_TYPE = (
1365        (TYPE_DEPENDS, "depends"),
1366        (TYPE_RDEPENDS, "rdepends"),
1367    )
1368    recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE, related_name='r_dependencies_recipe')
1369    depends_on = models.ForeignKey(Recipe, on_delete=models.CASCADE, related_name='r_dependencies_depends')
1370    via = models.ForeignKey(Provides, on_delete=models.CASCADE, null=True, default=None)
1371    dep_type = models.IntegerField(choices=DEPENDS_TYPE)
1372    objects = Recipe_DependencyManager()
1373
1374
1375class Machine(models.Model):
1376    search_allowed_fields = ["name", "description", "layer_version__layer__name"]
1377    up_date = models.DateTimeField(null = True, default = None)
1378
1379    layer_version = models.ForeignKey('Layer_Version', on_delete=models.CASCADE)
1380    name = models.CharField(max_length=255)
1381    description = models.CharField(max_length=255)
1382
1383    def get_vcs_machine_file_link_url(self):
1384        path = 'conf/machine/'+self.name+'.conf'
1385
1386        return self.layer_version.get_vcs_file_link_url(path)
1387
1388    def __unicode__(self):
1389        return "Machine " + self.name + "(" + self.description + ")"
1390
1391
1392
1393
1394
1395class BitbakeVersion(models.Model):
1396
1397    name = models.CharField(max_length=32, unique = True)
1398    giturl = GitURLField()
1399    branch = models.CharField(max_length=32)
1400    dirpath = models.CharField(max_length=255)
1401
1402    def __unicode__(self):
1403        return "%s (Branch: %s)" % (self.name, self.branch)
1404
1405
1406class Release(models.Model):
1407    """ A release is a project template, used to pre-populate Project settings with a configuration set """
1408    name = models.CharField(max_length=32, unique = True)
1409    description = models.CharField(max_length=255)
1410    bitbake_version = models.ForeignKey(BitbakeVersion, on_delete=models.CASCADE)
1411    branch_name = models.CharField(max_length=50, default = "")
1412    helptext = models.TextField(null=True)
1413
1414    def __unicode__(self):
1415        return "%s (%s)" % (self.name, self.branch_name)
1416
1417    def __str__(self):
1418        return self.name
1419
1420class ReleaseDefaultLayer(models.Model):
1421    release = models.ForeignKey(Release, on_delete=models.CASCADE)
1422    layer_name = models.CharField(max_length=100, default="")
1423
1424
1425class LayerSource(object):
1426    """ Where the layer metadata came from """
1427    TYPE_LOCAL = 0
1428    TYPE_LAYERINDEX = 1
1429    TYPE_IMPORTED = 2
1430    TYPE_BUILD = 3
1431
1432    SOURCE_TYPE = (
1433        (TYPE_LOCAL, "local"),
1434        (TYPE_LAYERINDEX, "layerindex"),
1435        (TYPE_IMPORTED, "imported"),
1436        (TYPE_BUILD, "build"),
1437    )
1438
1439    def types_dict():
1440        """ Turn the TYPES enums into a simple dictionary """
1441        dictionary = {}
1442        for key in LayerSource.__dict__:
1443            if "TYPE" in key:
1444                dictionary[key] = getattr(LayerSource, key)
1445        return dictionary
1446
1447
1448class Layer(models.Model):
1449
1450    up_date = models.DateTimeField(null=True, default=timezone.now)
1451
1452    name = models.CharField(max_length=100)
1453    layer_index_url = models.URLField()
1454    vcs_url = GitURLField(default=None, null=True)
1455    local_source_dir = models.TextField(null=True, default=None)
1456    vcs_web_url = models.URLField(null=True, default=None)
1457    vcs_web_tree_base_url = models.URLField(null=True, default=None)
1458    vcs_web_file_base_url = models.URLField(null=True, default=None)
1459
1460    summary = models.TextField(help_text='One-line description of the layer',
1461                               null=True, default=None)
1462    description = models.TextField(null=True, default=None)
1463
1464    def __unicode__(self):
1465        return "%s / %s " % (self.name, self.summary)
1466
1467
1468class Layer_Version(models.Model):
1469    """
1470    A Layer_Version either belongs to a single project or no project
1471    """
1472    search_allowed_fields = ["layer__name", "layer__summary",
1473                             "layer__description", "layer__vcs_url",
1474                             "dirpath", "release__name", "commit", "branch"]
1475
1476    build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='layer_version_build',
1477                              default=None, null=True)
1478
1479    layer = models.ForeignKey(Layer, on_delete=models.CASCADE, related_name='layer_version_layer')
1480
1481    layer_source = models.IntegerField(choices=LayerSource.SOURCE_TYPE,
1482                                       default=0)
1483
1484    up_date = models.DateTimeField(null=True, default=timezone.now)
1485
1486    # To which metadata release does this layer version belong to
1487    release = models.ForeignKey(Release, on_delete=models.CASCADE, null=True, default=None)
1488
1489    branch = models.CharField(max_length=80)
1490    commit = models.CharField(max_length=100)
1491    # If the layer is in a subdir
1492    dirpath = models.CharField(max_length=255, null=True, default=None)
1493
1494    # if -1, this is a default layer
1495    priority = models.IntegerField(default=0)
1496
1497    # where this layer exists on the filesystem
1498    local_path = models.FilePathField(max_length=1024, default="/")
1499
1500    # Set if this layer is restricted to a particular project
1501    project = models.ForeignKey('Project', on_delete=models.CASCADE, null=True, default=None)
1502
1503    # code lifted, with adaptations, from the layerindex-web application
1504    # https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/
1505    def _handle_url_path(self, base_url, path):
1506        import re, posixpath
1507        if base_url:
1508            if self.dirpath:
1509                if path:
1510                    extra_path = self.dirpath + '/' + path
1511                    # Normalise out ../ in path for usage URL
1512                    extra_path = posixpath.normpath(extra_path)
1513                    # Minor workaround to handle case where subdirectory has been added between branches
1514                    # (should probably support usage URL per branch to handle this... sigh...)
1515                    if extra_path.startswith('../'):
1516                        extra_path = extra_path[3:]
1517                else:
1518                    extra_path = self.dirpath
1519            else:
1520                extra_path = path
1521            branchname = self.release.name
1522            url = base_url.replace('%branch%', branchname)
1523
1524            # If there's a % in the path (e.g. a wildcard bbappend) we need to encode it
1525            if extra_path:
1526                extra_path = extra_path.replace('%', '%25')
1527
1528            if '%path%' in base_url:
1529                if extra_path:
1530                    url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '\\1', url)
1531                else:
1532                    url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '', url)
1533                return url.replace('%path%', extra_path)
1534            else:
1535                return url + extra_path
1536        return None
1537
1538    def get_vcs_link_url(self):
1539        if self.layer.vcs_web_url is None:
1540            return None
1541        return self.layer.vcs_web_url
1542
1543    def get_vcs_file_link_url(self, file_path=""):
1544        if self.layer.vcs_web_file_base_url is None:
1545            return None
1546        return self._handle_url_path(self.layer.vcs_web_file_base_url,
1547                                     file_path)
1548
1549    def get_vcs_dirpath_link_url(self):
1550        if self.layer.vcs_web_tree_base_url is None:
1551            return None
1552        return self._handle_url_path(self.layer.vcs_web_tree_base_url, '')
1553
1554    def get_vcs_reference(self):
1555        if self.commit is not None and len(self.commit) > 0:
1556            return self.commit
1557        if self.branch is not None and len(self.branch) > 0:
1558            return self.branch
1559        if self.release is not None:
1560            return self.release.name
1561        return 'N/A'
1562
1563    def get_detailspage_url(self, project_id=None):
1564        """ returns the url to the layer details page uses own project
1565        field if project_id is not specified """
1566
1567        if project_id is None:
1568            project_id = self.project.pk
1569
1570        return reverse('layerdetails', args=(project_id, self.pk))
1571
1572    def get_alldeps(self, project_id):
1573        """Get full list of unique layer dependencies."""
1574        def gen_layerdeps(lver, project, depth):
1575            if depth == 0:
1576                return
1577            for ldep in lver.dependencies.all():
1578                yield ldep.depends_on
1579                # get next level of deps recursively calling gen_layerdeps
1580                for subdep in gen_layerdeps(ldep.depends_on, project, depth-1):
1581                    yield subdep
1582
1583        project = Project.objects.get(pk=project_id)
1584        result = []
1585        projectlvers = [player.layercommit for player in
1586                        project.projectlayer_set.all()]
1587        # protect against infinite layer dependency loops
1588        maxdepth = 20
1589        for dep in gen_layerdeps(self, project, maxdepth):
1590            # filter out duplicates and layers already belonging to the project
1591            if dep not in result + projectlvers:
1592                result.append(dep)
1593
1594        return sorted(result, key=lambda x: x.layer.name)
1595
1596    def __unicode__(self):
1597        return ("id %d belongs to layer: %s" % (self.pk, self.layer.name))
1598
1599    def __str__(self):
1600        if self.release:
1601            release = self.release.name
1602        else:
1603            release = "No release set"
1604
1605        return "%d %s (%s)" % (self.pk, self.layer.name, release)
1606
1607
1608class LayerVersionDependency(models.Model):
1609
1610    layer_version = models.ForeignKey(Layer_Version, on_delete=models.CASCADE,
1611                                      related_name="dependencies")
1612    depends_on = models.ForeignKey(Layer_Version, on_delete=models.CASCADE,
1613                                   related_name="dependees")
1614
1615class ProjectLayer(models.Model):
1616    project = models.ForeignKey(Project, on_delete=models.CASCADE)
1617    layercommit = models.ForeignKey(Layer_Version, on_delete=models.CASCADE, null=True)
1618    optional = models.BooleanField(default = True)
1619
1620    def __unicode__(self):
1621        return "%s, %s" % (self.project.name, self.layercommit)
1622
1623    class Meta:
1624        unique_together = (("project", "layercommit"),)
1625
1626class CustomImageRecipe(Recipe):
1627
1628    # CustomImageRecipe's belong to layers called:
1629    LAYER_NAME = "toaster-custom-images"
1630
1631    search_allowed_fields = ['name']
1632    base_recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE, related_name='based_on_recipe')
1633    project = models.ForeignKey(Project, on_delete=models.CASCADE)
1634    last_updated = models.DateTimeField(null=True, default=None)
1635
1636    def get_last_successful_built_target(self):
1637        """ Return the last successful built target object if one exists
1638        otherwise return None """
1639        return Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
1640                                     Q(build__project=self.project) &
1641                                     Q(target=self.name)).last()
1642
1643    def update_package_list(self):
1644        """ Update the package list from the last good build of this
1645        CustomImageRecipe
1646        """
1647        # Check if we're aldready up-to-date or not
1648        target = self.get_last_successful_built_target()
1649        if target is None:
1650            # So we've never actually built this Custom recipe but what about
1651            # the recipe it's based on?
1652            target = \
1653                Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
1654                                      Q(build__project=self.project) &
1655                                      Q(target=self.base_recipe.name)).last()
1656            if target is None:
1657                return
1658
1659        if target.build.completed_on == self.last_updated:
1660            return
1661
1662        self.includes_set.clear()
1663
1664        excludes_list = self.excludes_set.values_list('name', flat=True)
1665        appends_list = self.appends_set.values_list('name', flat=True)
1666
1667        built_packages_list = \
1668            target.target_installed_package_set.values_list('package__name',
1669                                                            flat=True)
1670        for built_package in built_packages_list:
1671            # Is the built package in the custom packages list?
1672            if built_package in excludes_list:
1673                continue
1674
1675            if built_package in appends_list:
1676                continue
1677
1678            cust_img_p = \
1679                    CustomImagePackage.objects.get(name=built_package)
1680            self.includes_set.add(cust_img_p)
1681
1682
1683        self.last_updated = target.build.completed_on
1684        self.save()
1685
1686    def get_all_packages(self):
1687        """Get the included packages and any appended packages"""
1688        self.update_package_list()
1689
1690        return CustomImagePackage.objects.filter((Q(recipe_appends=self) |
1691                                                  Q(recipe_includes=self)) &
1692                                                 ~Q(recipe_excludes=self))
1693
1694    def get_base_recipe_file(self):
1695        """Get the base recipe file path if it exists on the file system"""
1696        path_schema_one = "%s/%s" % (self.base_recipe.layer_version.local_path,
1697                                     self.base_recipe.file_path)
1698
1699        path_schema_two = self.base_recipe.file_path
1700
1701        path_schema_three = "%s/%s" % (self.base_recipe.layer_version.layer.local_source_dir,
1702                                     self.base_recipe.file_path)
1703
1704        if os.path.exists(path_schema_one):
1705            return path_schema_one
1706
1707        # The path may now be the full path if the recipe has been built
1708        if os.path.exists(path_schema_two):
1709            return path_schema_two
1710
1711        # Or a local path if all layers are local
1712        if os.path.exists(path_schema_three):
1713            return path_schema_three
1714
1715        return None
1716
1717    def generate_recipe_file_contents(self):
1718        """Generate the contents for the recipe file."""
1719        # If we have no excluded packages we only need to :append
1720        if self.excludes_set.count() == 0:
1721            packages_conf = "IMAGE_INSTALL:append = \" "
1722
1723            for pkg in self.appends_set.all():
1724                packages_conf += pkg.name+' '
1725        else:
1726            packages_conf = "IMAGE_FEATURES =\"\"\nIMAGE_INSTALL = \""
1727            # We add all the known packages to be built by this recipe apart
1728            # from locale packages which are are controlled with IMAGE_LINGUAS.
1729            for pkg in self.get_all_packages().exclude(
1730                    name__icontains="locale"):
1731                packages_conf += pkg.name+' '
1732
1733        packages_conf += "\""
1734
1735        base_recipe_path = self.get_base_recipe_file()
1736        if base_recipe_path:
1737            base_recipe = open(base_recipe_path, 'r').read()
1738        else:
1739            # Pass back None to trigger error message to user
1740            return None
1741
1742        # Add a special case for when the recipe we have based a custom image
1743        # recipe on requires another recipe.
1744        # For example:
1745        # "require core-image-minimal.bb" is changed to:
1746        # "require recipes-core/images/core-image-minimal.bb"
1747
1748        req_search = re.search(r'(require\s+)(.+\.bb\s*$)',
1749                               base_recipe,
1750                               re.MULTILINE)
1751        if req_search:
1752            require_filename = req_search.group(2).strip()
1753
1754            corrected_location = Recipe.objects.filter(
1755                Q(layer_version=self.base_recipe.layer_version) &
1756                Q(file_path__icontains=require_filename)).last().file_path
1757
1758            new_require_line = "require %s" % corrected_location
1759
1760            base_recipe = base_recipe.replace(req_search.group(0),
1761                                              new_require_line)
1762
1763        info = {
1764            "date": timezone.now().strftime("%Y-%m-%d %H:%M:%S"),
1765            "base_recipe": base_recipe,
1766            "recipe_name": self.name,
1767            "base_recipe_name": self.base_recipe.name,
1768            "license": self.license,
1769            "summary": self.summary,
1770            "description": self.description,
1771            "packages_conf": packages_conf.strip()
1772        }
1773
1774        recipe_contents = ("# Original recipe %(base_recipe_name)s \n"
1775                           "%(base_recipe)s\n\n"
1776                           "# Recipe %(recipe_name)s \n"
1777                           "# Customisation Generated by Toaster on %(date)s\n"
1778                           "SUMMARY = \"%(summary)s\"\n"
1779                           "DESCRIPTION = \"%(description)s\"\n"
1780                           "LICENSE = \"%(license)s\"\n"
1781                           "%(packages_conf)s") % info
1782
1783        return recipe_contents
1784
1785class ProjectVariable(models.Model):
1786    project = models.ForeignKey(Project, on_delete=models.CASCADE)
1787    name = models.CharField(max_length=100)
1788    value = models.TextField(blank = True)
1789
1790class Variable(models.Model):
1791    search_allowed_fields = ['variable_name', 'variable_value',
1792                             'vhistory__file_name', "description"]
1793    build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='variable_build')
1794    variable_name = models.CharField(max_length=100)
1795    variable_value = models.TextField(blank=True)
1796    changed = models.BooleanField(default=False)
1797    human_readable_name = models.CharField(max_length=200)
1798    description = models.TextField(blank=True)
1799
1800class VariableHistory(models.Model):
1801    variable = models.ForeignKey(Variable, on_delete=models.CASCADE, related_name='vhistory')
1802    value   = models.TextField(blank=True)
1803    file_name = models.FilePathField(max_length=255)
1804    line_number = models.IntegerField(null=True)
1805    operation = models.CharField(max_length=64)
1806
1807class HelpText(models.Model):
1808    VARIABLE = 0
1809    HELPTEXT_AREA = ((VARIABLE, 'variable'), )
1810
1811    build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='helptext_build')
1812    area = models.IntegerField(choices=HELPTEXT_AREA)
1813    key = models.CharField(max_length=100)
1814    text = models.TextField()
1815
1816class LogMessage(models.Model):
1817    EXCEPTION = -1      # used to signal self-toaster-exceptions
1818    INFO = 0
1819    WARNING = 1
1820    ERROR = 2
1821    CRITICAL = 3
1822
1823    LOG_LEVEL = (
1824        (INFO, "info"),
1825        (WARNING, "warn"),
1826        (ERROR, "error"),
1827        (CRITICAL, "critical"),
1828        (EXCEPTION, "toaster exception")
1829    )
1830
1831    build = models.ForeignKey(Build, on_delete=models.CASCADE)
1832    task  = models.ForeignKey(Task, on_delete=models.CASCADE, blank = True, null=True)
1833    level = models.IntegerField(choices=LOG_LEVEL, default=INFO)
1834    message = models.TextField(blank=True, null=True)
1835    pathname = models.FilePathField(max_length=255, blank=True)
1836    lineno = models.IntegerField(null=True)
1837
1838    def __str__(self):
1839        return force_bytes('%s %s %s' % (self.get_level_display(), self.message, self.build))
1840
1841def invalidate_cache(**kwargs):
1842    from django.core.cache import cache
1843    try:
1844      cache.clear()
1845    except Exception as e:
1846      logger.warning("Problem with cache backend: Failed to clear cache: %s" % e)
1847
1848def signal_runbuilds():
1849    """Send SIGUSR1 to runbuilds process"""
1850    try:
1851        with open(os.path.join(os.getenv('BUILDDIR', '.'),
1852                               '.runbuilds.pid')) as pidf:
1853            os.kill(int(pidf.read()), SIGUSR1)
1854    except FileNotFoundError:
1855        logger.info("Stopping existing runbuilds: no current process found")
1856
1857class Distro(models.Model):
1858    search_allowed_fields = ["name", "description", "layer_version__layer__name"]
1859    up_date = models.DateTimeField(null = True, default = None)
1860
1861    layer_version = models.ForeignKey('Layer_Version', on_delete=models.CASCADE)
1862    name = models.CharField(max_length=255)
1863    description = models.CharField(max_length=255)
1864
1865    def get_vcs_distro_file_link_url(self):
1866        path = 'conf/distro/%s.conf' % self.name
1867        return self.layer_version.get_vcs_file_link_url(path)
1868
1869    def __unicode__(self):
1870        return "Distro " + self.name + "(" + self.description + ")"
1871
1872django.db.models.signals.post_save.connect(invalidate_cache)
1873django.db.models.signals.post_delete.connect(invalidate_cache)
1874django.db.models.signals.m2m_changed.connect(invalidate_cache)
1875