xref: /openbmc/openbmc/poky/bitbake/lib/toaster/orm/models.py (revision c9537f57ab488bf5d90132917b0184e2527970a5)
1#
2# BitBake Toaster Implementation
3#
4# Copyright (C) 2013        Intel Corporation
5#
6# SPDX-License-Identifier: GPL-2.0-only
7#
8
9from __future__ import unicode_literals
10
11from django.db import models, IntegrityError, DataError
12from django.db.models import F, Q, Sum, Count
13from django.utils import timezone
14from django.utils.encoding import force_bytes
15
16from django.urls import reverse
17
18from django.core import validators
19from django.conf import settings
20import django.db.models.signals
21
22import sys
23import os
24import re
25import itertools
26from signal import SIGUSR1
27
28
29import logging
30logger = logging.getLogger("toaster")
31
32if 'sqlite' in settings.DATABASES['default']['ENGINE']:
33    from django.db import transaction, OperationalError
34    from time import sleep
35
36    _base_save = models.Model.save
37    def save(self, *args, **kwargs):
38        while True:
39            try:
40                with transaction.atomic():
41                    return _base_save(self, *args, **kwargs)
42            except OperationalError as err:
43                if 'database is locked' in str(err):
44                    logger.warning("%s, model: %s, args: %s, kwargs: %s",
45                                   err, self.__class__, args, kwargs)
46                    sleep(0.5)
47                    continue
48                raise
49
50    models.Model.save = save
51
52    # HACK: Monkey patch Django to fix 'database is locked' issue
53
54    from django.db.models.query import QuerySet
55    _base_insert = QuerySet._insert
56    def _insert(self,  *args, **kwargs):
57        with transaction.atomic(using=self.db, savepoint=False):
58            return _base_insert(self, *args, **kwargs)
59    QuerySet._insert = _insert
60
61    def _create_object_from_params(self, lookup, params):
62        """
63        Tries to create an object using passed params.
64        Used by get_or_create and update_or_create
65        """
66        try:
67            obj = self.create(**params)
68            return obj, True
69        except (IntegrityError, DataError):
70            exc_info = sys.exc_info()
71            try:
72                return self.get(**lookup), False
73            except self.model.DoesNotExist:
74                pass
75            six.reraise(*exc_info)
76
77    QuerySet._create_object_from_params = _create_object_from_params
78
79    # end of HACK
80
81class GitURLValidator(validators.URLValidator):
82    regex = re.compile(
83        r'^(?:ssh|git|http|ftp)s?://'  # http:// or https://
84        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # domain...
85        r'localhost|'  # localhost...
86        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|'  # ...or ipv4
87        r'\[?[A-F0-9]*:[A-F0-9:]+\]?)'  # ...or ipv6
88        r'(?::\d+)?'  # optional port
89        r'(?:/?|[/?]\S+)$', re.IGNORECASE)
90
91def GitURLField(**kwargs):
92    r = models.URLField(**kwargs)
93    for i in range(len(r.validators)):
94        if isinstance(r.validators[i], validators.URLValidator):
95            r.validators[i] = GitURLValidator()
96    return r
97
98
99class ToasterSetting(models.Model):
100    name = models.CharField(max_length=63)
101    helptext = models.TextField()
102    value = models.CharField(max_length=255)
103
104    def __unicode__(self):
105        return "Setting %s = %s" % (self.name, self.value)
106
107
108class ProjectManager(models.Manager):
109    def create_project(self, name, release, existing_project=None, imported=False):
110        if existing_project and (release is not None):
111            prj = existing_project
112            prj.bitbake_version = release.bitbake_version
113            prj.release = release
114            # Delete the previous ProjectLayer mappings
115            for pl in ProjectLayer.objects.filter(project=prj):
116                pl.delete()
117        elif release is not None:
118            prj = self.model(name=name,
119                             bitbake_version=release.bitbake_version,
120                             release=release)
121        else:
122            prj = self.model(name=name,
123                             bitbake_version=None,
124                             release=None)
125        prj.save()
126
127        for defaultconf in ToasterSetting.objects.filter(
128                name__startswith="DEFCONF_"):
129            name = defaultconf.name[8:]
130            pv,create = ProjectVariable.objects.get_or_create(project=prj,name=name)
131            pv.value = defaultconf.value
132            pv.save()
133
134        if release is None:
135            return prj
136        if not imported:
137            for rdl in release.releasedefaultlayer_set.all():
138                lv = Layer_Version.objects.filter(
139                    layer__name=rdl.layer_name,
140                    release=release).first()
141
142                if lv:
143                    ProjectLayer.objects.create(project=prj,
144                                                layercommit=lv,
145                                                optional=False)
146                else:
147                    logger.warning("Default project layer %s not found" %
148                                rdl.layer_name)
149
150        return prj
151
152    # return single object with is_default = True
153    def get_or_create_default_project(self):
154        projects = super(ProjectManager, self).filter(is_default=True)
155
156        if len(projects) > 1:
157            raise Exception('Inconsistent project data: multiple ' +
158                            'default projects (i.e. with is_default=True)')
159        elif len(projects) < 1:
160            options = {
161                'name': 'Command line builds',
162                'short_description':
163                'Project for builds started outside Toaster',
164                'is_default': True
165            }
166            project = Project.objects.create(**options)
167            project.save()
168
169            return project
170        else:
171            return projects[0]
172
173
174class Project(models.Model):
175    search_allowed_fields = ['name', 'short_description', 'release__name',
176                             'release__branch_name']
177    name = models.CharField(max_length=100)
178    short_description = models.CharField(max_length=50, blank=True)
179    bitbake_version = models.ForeignKey('BitbakeVersion', on_delete=models.CASCADE, null=True)
180    release = models.ForeignKey("Release", on_delete=models.CASCADE, null=True)
181    created = models.DateTimeField(auto_now_add=True)
182    updated = models.DateTimeField(auto_now=True)
183    # This is a horrible hack; since Toaster has no "User" model available when
184    # running in interactive mode, we can't reference the field here directly
185    # Instead, we keep a possible null reference to the User id,
186    # as not to force
187    # hard links to possibly missing models
188    user_id = models.IntegerField(null=True)
189    objects = ProjectManager()
190
191    # build directory override (e.g. imported)
192    builddir = models.TextField()
193    # merge the Toaster configure attributes directly into the standard conf files
194    merged_attr = models.BooleanField(default=False)
195
196    # set to True for the project which is the default container
197    # for builds initiated by the command line etc.
198    is_default= models.BooleanField(default=False)
199
200    def __unicode__(self):
201        return "%s (Release %s, BBV %s)" % (self.name, self.release, self.bitbake_version)
202
203    def get_current_machine_name(self):
204        try:
205            return self.projectvariable_set.get(name="MACHINE").value
206        except (ProjectVariable.DoesNotExist,IndexError):
207            return None;
208
209    def get_number_of_builds(self):
210        """Return the number of builds which have ended"""
211
212        return self.build_set.exclude(
213            Q(outcome=Build.IN_PROGRESS) |
214            Q(outcome=Build.CANCELLED)
215        ).count()
216
217    def get_last_build_id(self):
218        try:
219            return Build.objects.filter( project = self.id ).order_by('-completed_on')[0].id
220        except (Build.DoesNotExist,IndexError):
221            return( -1 )
222
223    def get_last_outcome(self):
224        build_id = self.get_last_build_id()
225        if (-1 == build_id):
226            return( "" )
227        try:
228            return Build.objects.filter( id = build_id )[ 0 ].outcome
229        except (Build.DoesNotExist,IndexError):
230            return( "not_found" )
231
232    def get_last_target(self):
233        build_id = self.get_last_build_id()
234        if (-1 == build_id):
235            return( "" )
236        try:
237            return Target.objects.filter(build = build_id)[0].target
238        except (Target.DoesNotExist,IndexError):
239            return( "not_found" )
240
241    def get_last_errors(self):
242        build_id = self.get_last_build_id()
243        if (-1 == build_id):
244            return( 0 )
245        try:
246            return Build.objects.filter(id = build_id)[ 0 ].errors.count()
247        except (Build.DoesNotExist,IndexError):
248            return( "not_found" )
249
250    def get_last_warnings(self):
251        build_id = self.get_last_build_id()
252        if (-1 == build_id):
253            return( 0 )
254        try:
255            return Build.objects.filter(id = build_id)[ 0 ].warnings.count()
256        except (Build.DoesNotExist,IndexError):
257            return( "not_found" )
258
259    def get_last_build_extensions(self):
260        """
261        Get list of file name extensions for images produced by the most
262        recent build
263        """
264        last_build = Build.objects.get(pk = self.get_last_build_id())
265        return last_build.get_image_file_extensions()
266
267    def get_last_imgfiles(self):
268        build_id = self.get_last_build_id()
269        if (-1 == build_id):
270            return( "" )
271        try:
272            return Variable.objects.filter(build = build_id, variable_name = "IMAGE_FSTYPES")[ 0 ].variable_value
273        except (Variable.DoesNotExist,IndexError):
274            return( "not_found" )
275
276    def get_all_compatible_layer_versions(self):
277        """ Returns Queryset of all Layer_Versions which are compatible with
278        this project"""
279        queryset = None
280
281        # guard on release, as it can be null
282        if self.release:
283            queryset = Layer_Version.objects.filter(
284                (Q(release=self.release) &
285                 Q(build=None) &
286                 Q(project=None)) |
287                 Q(project=self))
288        else:
289            queryset = Layer_Version.objects.none()
290
291        return queryset
292
293    def get_project_layer_versions(self, pk=False):
294        """ Returns the Layer_Versions currently added to this project """
295        layer_versions = self.projectlayer_set.all().values_list('layercommit',
296                                                                 flat=True)
297
298        if pk is False:
299            return Layer_Version.objects.filter(pk__in=layer_versions)
300        else:
301            return layer_versions
302
303
304    def get_default_image_recipe(self):
305        try:
306            return self.projectvariable_set.get(name="DEFAULT_IMAGE").value
307        except (ProjectVariable.DoesNotExist,IndexError):
308            return None;
309
310    def get_is_new(self):
311        return self.get_variable(Project.PROJECT_SPECIFIC_ISNEW)
312
313    def get_available_machines(self):
314        """ Returns QuerySet of all Machines which are provided by the
315        Layers currently added to the Project """
316        queryset = Machine.objects.filter(
317            layer_version__in=self.get_project_layer_versions())
318
319        return queryset
320
321    def get_all_compatible_machines(self):
322        """ Returns QuerySet of all the compatible machines available to the
323        project including ones from Layers not currently added """
324        queryset = Machine.objects.filter(
325            layer_version__in=self.get_all_compatible_layer_versions())
326
327        return queryset
328
329    def get_available_distros(self):
330        """ Returns QuerySet of all Distros which are provided by the
331        Layers currently added to the Project """
332        queryset = Distro.objects.filter(
333            layer_version__in=self.get_project_layer_versions())
334
335        return queryset
336
337    def get_all_compatible_distros(self):
338        """ Returns QuerySet of all the compatible Wind River distros available to the
339        project including ones from Layers not currently added """
340        queryset = Distro.objects.filter(
341            layer_version__in=self.get_all_compatible_layer_versions())
342
343        return queryset
344
345    def get_available_recipes(self):
346        """ Returns QuerySet of all the recipes that are provided by layers
347        added to this project """
348        queryset = Recipe.objects.filter(
349            layer_version__in=self.get_project_layer_versions())
350
351        return queryset
352
353    def get_all_compatible_recipes(self):
354        """ Returns QuerySet of all the compatible Recipes available to the
355        project including ones from Layers not currently added """
356        queryset = Recipe.objects.filter(
357            layer_version__in=self.get_all_compatible_layer_versions()).exclude(name__exact='')
358
359        return queryset
360
361    # Project Specific status management
362    PROJECT_SPECIFIC_STATUS = 'INTERNAL_PROJECT_SPECIFIC_STATUS'
363    PROJECT_SPECIFIC_CALLBACK = 'INTERNAL_PROJECT_SPECIFIC_CALLBACK'
364    PROJECT_SPECIFIC_ISNEW = 'INTERNAL_PROJECT_SPECIFIC_ISNEW'
365    PROJECT_SPECIFIC_DEFAULTIMAGE = 'PROJECT_SPECIFIC_DEFAULTIMAGE'
366    PROJECT_SPECIFIC_NONE = ''
367    PROJECT_SPECIFIC_NEW = '1'
368    PROJECT_SPECIFIC_EDIT = '2'
369    PROJECT_SPECIFIC_CLONING = '3'
370    PROJECT_SPECIFIC_CLONING_SUCCESS = '4'
371    PROJECT_SPECIFIC_CLONING_FAIL = '5'
372
373    def get_variable(self,variable,default_value = ''):
374        try:
375            return self.projectvariable_set.get(name=variable).value
376        except (ProjectVariable.DoesNotExist,IndexError):
377            return default_value
378
379    def set_variable(self,variable,value):
380        pv,create = ProjectVariable.objects.get_or_create(project = self, name = variable)
381        pv.value = value
382        pv.save()
383
384    def get_default_image(self):
385        return self.get_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE)
386
387    def schedule_build(self):
388
389        from bldcontrol.models import BuildRequest, BRTarget, BRLayer
390        from bldcontrol.models import BRBitbake, BRVariable
391
392        try:
393            now = timezone.now()
394            build = Build.objects.create(project=self,
395                                         completed_on=now,
396                                         started_on=now)
397
398            br = BuildRequest.objects.create(project=self,
399                                             state=BuildRequest.REQ_QUEUED,
400                                             build=build)
401            BRBitbake.objects.create(req=br,
402                                     giturl=self.bitbake_version.giturl,
403                                     commit=self.bitbake_version.branch,
404                                     dirpath=self.bitbake_version.dirpath)
405
406            for t in self.projecttarget_set.all():
407                BRTarget.objects.create(req=br, target=t.target, task=t.task)
408                Target.objects.create(build=br.build, target=t.target,
409                                      task=t.task)
410                # If we're about to build a custom image recipe make sure
411                # that layer is currently in the project before we create the
412                # BRLayer objects
413                customrecipe = CustomImageRecipe.objects.filter(
414                    name=t.target,
415                    project=self).first()
416                if customrecipe:
417                    ProjectLayer.objects.get_or_create(
418                        project=self,
419                        layercommit=customrecipe.layer_version,
420                        optional=False)
421
422            for l in self.projectlayer_set.all().order_by("pk"):
423                commit = l.layercommit.get_vcs_reference()
424                logger.debug("Adding layer to build %s" %
425                             l.layercommit.layer.name)
426                BRLayer.objects.create(
427                    req=br,
428                    name=l.layercommit.layer.name,
429                    giturl=l.layercommit.layer.vcs_url,
430                    commit=commit,
431                    dirpath=l.layercommit.dirpath,
432                    layer_version=l.layercommit,
433                    local_source_dir=l.layercommit.layer.local_source_dir
434                )
435
436            for v in self.projectvariable_set.all():
437                BRVariable.objects.create(req=br, name=v.name, value=v.value)
438
439            try:
440                br.build.machine = self.projectvariable_set.get(
441                    name='MACHINE').value
442                br.build.save()
443            except ProjectVariable.DoesNotExist:
444                pass
445
446            br.save()
447            signal_runbuilds()
448
449        except Exception:
450            # revert the build request creation since we're not done cleanly
451            br.delete()
452            raise
453        return br
454
455class Build(models.Model):
456    SUCCEEDED = 0
457    FAILED = 1
458    IN_PROGRESS = 2
459    CANCELLED = 3
460
461    BUILD_OUTCOME = (
462        (SUCCEEDED, 'Succeeded'),
463        (FAILED, 'Failed'),
464        (IN_PROGRESS, 'In Progress'),
465        (CANCELLED, 'Cancelled'),
466    )
467
468    search_allowed_fields = ['machine', 'cooker_log_path', "target__target", "target__target_image_file__file_name"]
469
470    project = models.ForeignKey(Project, on_delete=models.CASCADE)            # must have a project
471    machine = models.CharField(max_length=100)
472    distro = models.CharField(max_length=100)
473    distro_version = models.CharField(max_length=100)
474    started_on = models.DateTimeField()
475    completed_on = models.DateTimeField()
476    outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS)
477    cooker_log_path = models.CharField(max_length=500)
478    build_name = models.CharField(max_length=100, default='')
479    bitbake_version = models.CharField(max_length=50)
480
481    # number of recipes to parse for this build
482    recipes_to_parse = models.IntegerField(default=1)
483
484    # number of recipes parsed so far for this build
485    recipes_parsed = models.IntegerField(default=1)
486
487    # number of repos to clone for this build
488    repos_to_clone = models.IntegerField(default=1)
489
490    # number of repos cloned so far for this build (default off)
491    repos_cloned = models.IntegerField(default=1)
492
493    # Hint on current progress item
494    progress_item = models.CharField(max_length=40)
495
496    @staticmethod
497    def get_recent(project=None):
498        """
499        Return recent builds as a list; if project is set, only return
500        builds for that project
501        """
502
503        builds = Build.objects.all()
504
505        if project:
506            builds = builds.filter(project=project)
507
508        finished_criteria = \
509                Q(outcome=Build.SUCCEEDED) | \
510                Q(outcome=Build.FAILED) | \
511                Q(outcome=Build.CANCELLED)
512
513        recent_builds = list(itertools.chain(
514            builds.filter(outcome=Build.IN_PROGRESS).order_by("-started_on"),
515            builds.filter(finished_criteria).order_by("-completed_on")[:3]
516        ))
517
518        # add percentage done property to each build; this is used
519        # to show build progress in mrb_section.html
520        for build in recent_builds:
521            build.percentDone = build.completeper()
522            build.outcomeText = build.get_outcome_text()
523
524        return recent_builds
525
526    def started(self):
527        """
528        As build variables are only added for a build when its BuildStarted event
529        is received, a build with no build variables is counted as
530        "in preparation" and not properly started yet. This method
531        will return False if a build has no build variables (it never properly
532        started), or True otherwise.
533
534        Note that this is a temporary workaround for the fact that we don't
535        have a fine-grained state variable on a build which would allow us
536        to record "in progress" (BuildStarted received) vs. "in preparation".
537        """
538        variables = Variable.objects.filter(build=self)
539        return len(variables) > 0
540
541    def completeper(self):
542        tf = Task.objects.filter(build = self)
543        tfc = tf.count()
544        if tfc > 0:
545            completeper = tf.exclude(outcome=Task.OUTCOME_NA).count()*100 // tfc
546        else:
547            completeper = 0
548        return completeper
549
550    def eta(self):
551        eta = timezone.now()
552        completeper = self.completeper()
553        if self.completeper() > 0:
554            eta += ((eta - self.started_on)*(100-completeper))/completeper
555        return eta
556
557    def has_images(self):
558        """
559        Returns True if at least one of the targets for this build has an
560        image file associated with it, False otherwise
561        """
562        targets = Target.objects.filter(build_id=self.id)
563        has_images = False
564        for target in targets:
565            if target.has_images():
566                has_images = True
567                break
568        return has_images
569
570    def has_image_recipes(self):
571        """
572        Returns True if a build has any targets which were built from
573        image recipes.
574        """
575        image_recipes = self.get_image_recipes()
576        return len(image_recipes) > 0
577
578    def get_image_file_extensions(self):
579        """
580        Get string of file name extensions for images produced by this build;
581        note that this is the actual list of extensions stored on Target objects
582        for this build, and not the value of IMAGE_FSTYPES.
583
584        Returns comma-separated string, e.g. "vmdk, ext4"
585        """
586        extensions = []
587
588        targets = Target.objects.filter(build_id = self.id)
589        for target in targets:
590            if not target.is_image:
591                continue
592
593            target_image_files = Target_Image_File.objects.filter(
594                target_id=target.id)
595
596            for target_image_file in target_image_files:
597                extensions.append(target_image_file.suffix)
598
599        extensions = list(set(extensions))
600        extensions.sort()
601
602        return ', '.join(extensions)
603
604    def get_image_fstypes(self):
605        """
606        Get the IMAGE_FSTYPES variable value for this build as a de-duplicated
607        list of image file suffixes.
608        """
609        image_fstypes = Variable.objects.get(
610            build=self, variable_name='IMAGE_FSTYPES').variable_value
611        return list(set(re.split(r' {1,}', image_fstypes)))
612
613    def get_sorted_target_list(self):
614        tgts = Target.objects.filter(build_id = self.id).order_by( 'target' );
615        return( tgts );
616
617    def get_recipes(self):
618        """
619        Get the recipes related to this build;
620        note that the related layer versions and layers are also prefetched
621        by this query, as this queryset can be sorted by these objects in the
622        build recipes view; prefetching them here removes the need
623        for another query in that view
624        """
625        layer_versions = Layer_Version.objects.filter(build=self)
626        criteria = Q(layer_version__id__in=layer_versions)
627        return Recipe.objects.filter(criteria) \
628                             .select_related('layer_version', 'layer_version__layer')
629
630    def get_image_recipes(self):
631        """
632        Returns a list of image Recipes (custom and built-in) related to this
633        build, sorted by name; note that this has to be done in two steps, as
634        there's no way to get all the custom image recipes and image recipes
635        in one query
636        """
637        custom_image_recipes = self.get_custom_image_recipes()
638        custom_image_recipe_names = custom_image_recipes.values_list('name', flat=True)
639
640        not_custom_image_recipes = ~Q(name__in=custom_image_recipe_names) & \
641                                   Q(is_image=True)
642
643        built_image_recipes = self.get_recipes().filter(not_custom_image_recipes)
644
645        # append to the custom image recipes and sort
646        customisable_image_recipes = list(
647            itertools.chain(custom_image_recipes, built_image_recipes)
648        )
649
650        return sorted(customisable_image_recipes, key=lambda recipe: recipe.name)
651
652    def get_custom_image_recipes(self):
653        """
654        Returns a queryset of CustomImageRecipes related to this build,
655        sorted by name
656        """
657        built_recipe_names = self.get_recipes().values_list('name', flat=True)
658        criteria = Q(name__in=built_recipe_names) & Q(project=self.project)
659        queryset = CustomImageRecipe.objects.filter(criteria).order_by('name')
660        return queryset
661
662    def get_outcome_text(self):
663        return Build.BUILD_OUTCOME[int(self.outcome)][1]
664
665    @property
666    def failed_tasks(self):
667        """ Get failed tasks for the build """
668        tasks = self.task_build.all()
669        return tasks.filter(order__gt=0, outcome=Task.OUTCOME_FAILED)
670
671    @property
672    def errors(self):
673        return (self.logmessage_set.filter(level=LogMessage.ERROR) |
674                self.logmessage_set.filter(level=LogMessage.EXCEPTION) |
675                self.logmessage_set.filter(level=LogMessage.CRITICAL))
676
677    @property
678    def warnings(self):
679        return self.logmessage_set.filter(level=LogMessage.WARNING)
680
681    @property
682    def timespent(self):
683        return self.completed_on - self.started_on
684
685    @property
686    def timespent_seconds(self):
687        return self.timespent.total_seconds()
688
689    @property
690    def target_labels(self):
691        """
692        Sorted (a-z) "target1:task, target2, target3" etc. string for all
693        targets in this build
694        """
695        targets = self.target_set.all()
696        target_labels = [target.target +
697                         (':' + target.task if target.task else '')
698                         for target in targets]
699        target_labels.sort()
700
701        return target_labels
702
703    def get_buildrequest(self):
704        buildrequest = None
705        if hasattr(self, 'buildrequest'):
706            buildrequest = self.buildrequest
707        return buildrequest
708
709    def is_queued(self):
710        from bldcontrol.models import BuildRequest
711        buildrequest = self.get_buildrequest()
712        if buildrequest:
713            return buildrequest.state == BuildRequest.REQ_QUEUED
714        else:
715            return False
716
717    def is_cancelling(self):
718        from bldcontrol.models import BuildRequest
719        buildrequest = self.get_buildrequest()
720        if buildrequest:
721            return self.outcome == Build.IN_PROGRESS and \
722                buildrequest.state == BuildRequest.REQ_CANCELLING
723        else:
724            return False
725
726    def is_cloning(self):
727        """
728        True if the build is still cloning repos
729        """
730        return self.outcome == Build.IN_PROGRESS and \
731            self.repos_cloned < self.repos_to_clone
732
733    def is_parsing(self):
734        """
735        True if the build is still parsing recipes
736        """
737        return self.outcome == Build.IN_PROGRESS and \
738            self.recipes_parsed < self.recipes_to_parse
739
740    def is_starting(self):
741        """
742        True if the build has no completed tasks yet and is still just starting
743        tasks.
744
745        Note that the mechanism for testing whether a Task is "done" is whether
746        its outcome field is set, as per the completeper() method.
747        """
748        return self.outcome == Build.IN_PROGRESS and \
749            self.task_build.exclude(outcome=Task.OUTCOME_NA).count() == 0
750
751
752    def get_state(self):
753        """
754        Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress',
755        'Cancelled' (Build outcomes); or 'Queued', 'Cancelling' (states
756        dependent on the BuildRequest state).
757
758        This works around the fact that we have BuildRequest states as well
759        as Build states, but really we just want to know the state of the build.
760        """
761        if self.is_cancelling():
762            return 'Cancelling';
763        elif self.is_queued():
764            return 'Queued'
765        elif self.is_cloning():
766            return 'Cloning'
767        elif self.is_parsing():
768            return 'Parsing'
769        elif self.is_starting():
770            return 'Starting'
771        else:
772            return self.get_outcome_text()
773
774    def __str__(self):
775        return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()]))
776
777class ProjectTarget(models.Model):
778    project = models.ForeignKey(Project, on_delete=models.CASCADE)
779    target = models.CharField(max_length=100)
780    task = models.CharField(max_length=100, null=True)
781
782class Target(models.Model):
783    search_allowed_fields = ['target', 'file_name']
784    build = models.ForeignKey(Build, on_delete=models.CASCADE)
785    target = models.CharField(max_length=100)
786    task = models.CharField(max_length=100, null=True)
787    is_image = models.BooleanField(default = False)
788    image_size = models.IntegerField(default=0)
789    license_manifest_path = models.CharField(max_length=500, null=True)
790    package_manifest_path = models.CharField(max_length=500, null=True)
791
792    def package_count(self):
793        return Target_Installed_Package.objects.filter(target_id__exact=self.id).count()
794
795    def __unicode__(self):
796        return self.target
797
798    def get_similar_targets(self):
799        """
800        Get target sfor the same machine, task and target name
801        (e.g. 'core-image-minimal') from a successful build for this project
802        (but excluding this target).
803
804        Note that we only look for targets built by this project because
805        projects can have different configurations from each other, and put
806        their artifacts in different directories.
807
808        The possibility of error when retrieving candidate targets
809        is minimised by the fact that bitbake will rebuild artifacts if MACHINE
810        (or various other variables) change. In this case, there is no need to
811        clone artifacts from another target, as those artifacts will have
812        been re-generated for this target anyway.
813        """
814        query = ~Q(pk=self.pk) & \
815            Q(target=self.target) & \
816            Q(build__machine=self.build.machine) & \
817            Q(build__outcome=Build.SUCCEEDED) & \
818            Q(build__project=self.build.project)
819
820        return Target.objects.filter(query)
821
822    def get_similar_target_with_image_files(self):
823        """
824        Get the most recent similar target with Target_Image_Files associated
825        with it, for the purpose of cloning those files onto this target.
826        """
827        similar_target = None
828
829        candidates = self.get_similar_targets()
830        if candidates.count() == 0:
831            return similar_target
832
833        task_subquery = Q(task=self.task)
834
835        # we can look for a 'build' task if this task is a 'populate_sdk_ext'
836        # task, as the latter also creates images; and vice versa; note that
837        # 'build' targets can have their task set to '';
838        # also note that 'populate_sdk' does not produce image files
839        image_tasks = [
840            '', # aka 'build'
841            'build',
842            'image',
843            'populate_sdk_ext'
844        ]
845        if self.task in image_tasks:
846            task_subquery = Q(task__in=image_tasks)
847
848        # annotate with the count of files, to exclude any targets which
849        # don't have associated files
850        candidates = candidates.annotate(num_files=Count('target_image_file'))
851
852        query = task_subquery & Q(num_files__gt=0)
853
854        candidates = candidates.filter(query)
855
856        if candidates.count() > 0:
857            candidates.order_by('build__completed_on')
858            similar_target = candidates.last()
859
860        return similar_target
861
862    def get_similar_target_with_sdk_files(self):
863        """
864        Get the most recent similar target with TargetSDKFiles associated
865        with it, for the purpose of cloning those files onto this target.
866        """
867        similar_target = None
868
869        candidates = self.get_similar_targets()
870        if candidates.count() == 0:
871            return similar_target
872
873        # annotate with the count of files, to exclude any targets which
874        # don't have associated files
875        candidates = candidates.annotate(num_files=Count('targetsdkfile'))
876
877        query = Q(task=self.task) & Q(num_files__gt=0)
878
879        candidates = candidates.filter(query)
880
881        if candidates.count() > 0:
882            candidates.order_by('build__completed_on')
883            similar_target = candidates.last()
884
885        return similar_target
886
887    def clone_image_artifacts_from(self, target):
888        """
889        Make clones of the Target_Image_Files and TargetKernelFile objects
890        associated with Target target, then associate them with this target.
891
892        Note that for Target_Image_Files, we only want files from the previous
893        build whose suffix matches one of the suffixes defined in this
894        target's build's IMAGE_FSTYPES configuration variable. This prevents the
895        Target_Image_File object for an ext4 image being associated with a
896        target for a project which didn't produce an ext4 image (for example).
897
898        Also sets the license_manifest_path and package_manifest_path
899        of this target to the same path as that of target being cloned from, as
900        the manifests are also build artifacts but are treated differently.
901        """
902
903        image_fstypes = self.build.get_image_fstypes()
904
905        # filter out any image files whose suffixes aren't in the
906        # IMAGE_FSTYPES suffixes variable for this target's build
907        image_files = [target_image_file \
908            for target_image_file in target.target_image_file_set.all() \
909            if target_image_file.suffix in image_fstypes]
910
911        for image_file in image_files:
912            image_file.pk = None
913            image_file.target = self
914            image_file.save()
915
916        kernel_files = target.targetkernelfile_set.all()
917        for kernel_file in kernel_files:
918            kernel_file.pk = None
919            kernel_file.target = self
920            kernel_file.save()
921
922        self.license_manifest_path = target.license_manifest_path
923        self.package_manifest_path = target.package_manifest_path
924        self.save()
925
926    def clone_sdk_artifacts_from(self, target):
927        """
928        Clone TargetSDKFile objects from target and associate them with this
929        target.
930        """
931        sdk_files = target.targetsdkfile_set.all()
932        for sdk_file in sdk_files:
933            sdk_file.pk = None
934            sdk_file.target = self
935            sdk_file.save()
936
937    def has_images(self):
938        """
939        Returns True if this target has one or more image files attached to it.
940        """
941        return self.target_image_file_set.all().count() > 0
942
943# kernel artifacts for a target: bzImage and modules*
944class TargetKernelFile(models.Model):
945    target = models.ForeignKey(Target, on_delete=models.CASCADE)
946    file_name = models.FilePathField()
947    file_size = models.IntegerField()
948
949    @property
950    def basename(self):
951        return os.path.basename(self.file_name)
952
953# SDK artifacts for a target: sh and manifest files
954class TargetSDKFile(models.Model):
955    target = models.ForeignKey(Target, on_delete=models.CASCADE)
956    file_name = models.FilePathField()
957    file_size = models.IntegerField()
958
959    @property
960    def basename(self):
961        return os.path.basename(self.file_name)
962
963class Target_Image_File(models.Model):
964    # valid suffixes for image files produced by a build
965    SUFFIXES = {
966        'btrfs', 'container', 'cpio', 'cpio.gz', 'cpio.lz4', 'cpio.lzma',
967        'cpio.xz', 'cramfs', 'ext2', 'ext2.bz2', 'ext2.gz', 'ext2.lzma',
968        'ext3', 'ext3.gz', 'ext4', 'ext4.gz', 'f2fs', 'hddimg', 'iso', 'jffs2',
969        'jffs2.sum', 'multiubi', 'squashfs', 'squashfs-lz4', 'squashfs-lzo',
970        'squashfs-xz', 'tar', 'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi',
971        'ubifs', 'wic', 'wic.bz2', 'wic.gz', 'wic.lzma'
972    }
973
974    target = models.ForeignKey(Target, on_delete=models.CASCADE)
975    file_name = models.FilePathField(max_length=254)
976    file_size = models.IntegerField()
977
978    @property
979    def suffix(self):
980        """
981        Suffix for image file, minus leading "."
982        """
983        for suffix in Target_Image_File.SUFFIXES:
984            if self.file_name.endswith(suffix):
985                return suffix
986
987        filename, suffix = os.path.splitext(self.file_name)
988        suffix = suffix.lstrip('.')
989        return suffix
990
991class Target_File(models.Model):
992    ITYPE_REGULAR = 1
993    ITYPE_DIRECTORY = 2
994    ITYPE_SYMLINK = 3
995    ITYPE_SOCKET = 4
996    ITYPE_FIFO = 5
997    ITYPE_CHARACTER = 6
998    ITYPE_BLOCK = 7
999    ITYPES = ( (ITYPE_REGULAR ,'regular'),
1000        ( ITYPE_DIRECTORY ,'directory'),
1001        ( ITYPE_SYMLINK ,'symlink'),
1002        ( ITYPE_SOCKET ,'socket'),
1003        ( ITYPE_FIFO ,'fifo'),
1004        ( ITYPE_CHARACTER ,'character'),
1005        ( ITYPE_BLOCK ,'block'),
1006        )
1007
1008    target = models.ForeignKey(Target, on_delete=models.CASCADE)
1009    path = models.FilePathField()
1010    size = models.IntegerField()
1011    inodetype = models.IntegerField(choices = ITYPES)
1012    permission = models.CharField(max_length=16)
1013    owner = models.CharField(max_length=128)
1014    group = models.CharField(max_length=128)
1015    directory = models.ForeignKey('Target_File', on_delete=models.CASCADE, related_name="directory_set", null=True)
1016    sym_target = models.ForeignKey('Target_File', on_delete=models.CASCADE, related_name="symlink_set", null=True)
1017
1018
1019class Task(models.Model):
1020
1021    SSTATE_NA = 0
1022    SSTATE_MISS = 1
1023    SSTATE_FAILED = 2
1024    SSTATE_RESTORED = 3
1025
1026    SSTATE_RESULT = (
1027        (SSTATE_NA, 'Not Applicable'), # For rest of tasks, but they still need checking.
1028        (SSTATE_MISS, 'File not in cache'), # the sstate object was not found
1029        (SSTATE_FAILED, 'Failed'), # there was a pkg, but the script failed
1030        (SSTATE_RESTORED, 'Succeeded'), # successfully restored
1031    )
1032
1033    CODING_NA = 0
1034    CODING_PYTHON = 2
1035    CODING_SHELL = 3
1036
1037    TASK_CODING = (
1038        (CODING_NA, 'N/A'),
1039        (CODING_PYTHON, 'Python'),
1040        (CODING_SHELL, 'Shell'),
1041    )
1042
1043    OUTCOME_NA = -1
1044    OUTCOME_SUCCESS = 0
1045    OUTCOME_COVERED = 1
1046    OUTCOME_CACHED = 2
1047    OUTCOME_PREBUILT = 3
1048    OUTCOME_FAILED = 4
1049    OUTCOME_EMPTY = 5
1050
1051    TASK_OUTCOME = (
1052        (OUTCOME_NA, 'Not Available'),
1053        (OUTCOME_SUCCESS, 'Succeeded'),
1054        (OUTCOME_COVERED, 'Covered'),
1055        (OUTCOME_CACHED, 'Cached'),
1056        (OUTCOME_PREBUILT, 'Prebuilt'),
1057        (OUTCOME_FAILED, 'Failed'),
1058        (OUTCOME_EMPTY, 'Empty'),
1059    )
1060
1061    TASK_OUTCOME_HELP = (
1062        (OUTCOME_SUCCESS, 'This task successfully completed'),
1063        (OUTCOME_COVERED, 'This task did not run because its output is provided by another task'),
1064        (OUTCOME_CACHED, 'This task restored output from the sstate-cache directory or mirrors'),
1065        (OUTCOME_PREBUILT, 'This task did not run because its outcome was reused from a previous build'),
1066        (OUTCOME_FAILED, 'This task did not complete'),
1067        (OUTCOME_EMPTY, 'This task has no executable content'),
1068        (OUTCOME_NA, ''),
1069    )
1070
1071    search_allowed_fields = [ "recipe__name", "recipe__version", "task_name", "logfile" ]
1072
1073    def __init__(self, *args, **kwargs):
1074        super(Task, self).__init__(*args, **kwargs)
1075        try:
1076            self._helptext = HelpText.objects.get(key=self.task_name, area=HelpText.VARIABLE, build=self.build).text
1077        except HelpText.DoesNotExist:
1078            self._helptext = None
1079
1080    def get_related_setscene(self):
1081        return Task.objects.filter(task_executed=True, build = self.build, recipe = self.recipe, task_name=self.task_name+"_setscene")
1082
1083    def get_outcome_text(self):
1084        return Task.TASK_OUTCOME[int(self.outcome) + 1][1]
1085
1086    def get_outcome_help(self):
1087        return Task.TASK_OUTCOME_HELP[int(self.outcome)][1]
1088
1089    def get_sstate_text(self):
1090        if self.sstate_result==Task.SSTATE_NA:
1091            return ''
1092        else:
1093            return Task.SSTATE_RESULT[int(self.sstate_result)][1]
1094
1095    def get_executed_display(self):
1096        if self.task_executed:
1097            return "Executed"
1098        return "Not Executed"
1099
1100    def get_description(self):
1101        return self._helptext
1102
1103    build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='task_build')
1104    order = models.IntegerField(null=True)
1105    task_executed = models.BooleanField(default=False) # True means Executed, False means Not/Executed
1106    outcome = models.IntegerField(choices=TASK_OUTCOME, default=OUTCOME_NA)
1107    sstate_checksum = models.CharField(max_length=100, blank=True)
1108    path_to_sstate_obj = models.FilePathField(max_length=500, blank=True)
1109    recipe = models.ForeignKey('Recipe', on_delete=models.CASCADE, related_name='tasks')
1110    task_name = models.CharField(max_length=100)
1111    source_url = models.FilePathField(max_length=255, blank=True)
1112    work_directory = models.FilePathField(max_length=255, blank=True)
1113    script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NA)
1114    line_number = models.IntegerField(default=0)
1115
1116    # start/end times
1117    started = models.DateTimeField(null=True)
1118    ended = models.DateTimeField(null=True)
1119
1120    # in seconds; this is stored to enable sorting
1121    elapsed_time = models.DecimalField(max_digits=8, decimal_places=2, null=True)
1122
1123    # in bytes; note that disk_io is stored to enable sorting
1124    disk_io = models.IntegerField(null=True)
1125    disk_io_read = models.IntegerField(null=True)
1126    disk_io_write = models.IntegerField(null=True)
1127
1128    # in seconds
1129    cpu_time_user = models.DecimalField(max_digits=8, decimal_places=2, null=True)
1130    cpu_time_system = models.DecimalField(max_digits=8, decimal_places=2, null=True)
1131
1132    sstate_result = models.IntegerField(choices=SSTATE_RESULT, default=SSTATE_NA)
1133    message = models.CharField(max_length=240)
1134    logfile = models.FilePathField(max_length=255, blank=True)
1135
1136    outcome_text = property(get_outcome_text)
1137    sstate_text  = property(get_sstate_text)
1138
1139    def __unicode__(self):
1140        return "%d(%d) %s:%s" % (self.pk, self.build.pk, self.recipe.name, self.task_name)
1141
1142    class Meta:
1143        ordering = ('order', 'recipe' ,)
1144        unique_together = ('build', 'recipe', 'task_name', )
1145
1146
1147class Task_Dependency(models.Model):
1148    task = models.ForeignKey(Task, on_delete=models.CASCADE, related_name='task_dependencies_task')
1149    depends_on = models.ForeignKey(Task, on_delete=models.CASCADE, related_name='task_dependencies_depends')
1150
1151class Package(models.Model):
1152    search_allowed_fields = ['name', 'version', 'revision', 'recipe__name', 'recipe__version', 'recipe__license', 'recipe__layer_version__layer__name', 'recipe__layer_version__branch', 'recipe__layer_version__commit', 'recipe__layer_version__local_path', 'installed_name']
1153    build = models.ForeignKey('Build', on_delete=models.CASCADE, null=True)
1154    recipe = models.ForeignKey('Recipe', on_delete=models.CASCADE, null=True)
1155    name = models.CharField(max_length=100)
1156    installed_name = models.CharField(max_length=100, default='')
1157    version = models.CharField(max_length=100, blank=True)
1158    revision = models.CharField(max_length=32, blank=True)
1159    summary = models.TextField(blank=True)
1160    description = models.TextField(blank=True)
1161    size = models.IntegerField(default=0)
1162    installed_size = models.IntegerField(default=0)
1163    section = models.CharField(max_length=80, blank=True)
1164    license = models.CharField(max_length=80, blank=True)
1165
1166    @property
1167    def is_locale_package(self):
1168        """ Returns True if this package is identifiable as a locale package """
1169        if self.name.find('locale') != -1:
1170            return True
1171        return False
1172
1173    @property
1174    def is_packagegroup(self):
1175        """ Returns True is this package is identifiable as a packagegroup """
1176        if self.name.find('packagegroup') != -1:
1177            return True
1178        return False
1179
1180class CustomImagePackage(Package):
1181    # CustomImageRecipe fields to track pacakges appended,
1182    # included and excluded from a CustomImageRecipe
1183    recipe_includes = models.ManyToManyField('CustomImageRecipe',
1184                                             related_name='includes_set')
1185    recipe_excludes = models.ManyToManyField('CustomImageRecipe',
1186                                             related_name='excludes_set')
1187    recipe_appends = models.ManyToManyField('CustomImageRecipe',
1188                                            related_name='appends_set')
1189
1190
1191class Package_DependencyManager(models.Manager):
1192    use_for_related_fields = True
1193    TARGET_LATEST = "use-latest-target-for-target"
1194
1195    def get_queryset(self):
1196        return super(Package_DependencyManager, self).get_queryset().exclude(package_id = F('depends_on__id'))
1197
1198    def for_target_or_none(self, target):
1199        """ filter the dependencies to be displayed by the supplied target
1200        if no dependences are found for the target then try None as the target
1201        which will return the dependences calculated without the context of a
1202        target e.g. non image recipes.
1203
1204        returns: { size, packages }
1205        """
1206        package_dependencies = self.all_depends().order_by('depends_on__name')
1207
1208        if target is self.TARGET_LATEST:
1209            installed_deps =\
1210                    package_dependencies.filter(~Q(target__target=None))
1211        else:
1212            installed_deps =\
1213                    package_dependencies.filter(Q(target__target=target))
1214
1215        packages_list = None
1216        total_size = 0
1217
1218        # If we have installed depdencies for this package and target then use
1219        # these to display
1220        if installed_deps.count() > 0:
1221            packages_list = installed_deps
1222            total_size = installed_deps.aggregate(
1223                Sum('depends_on__size'))['depends_on__size__sum']
1224        else:
1225            new_list = []
1226            package_names = []
1227
1228            # Find dependencies for the package that we know about even if
1229            # it's not installed on a target e.g. from a non-image recipe
1230            for p in package_dependencies.filter(Q(target=None)):
1231                if p.depends_on.name in package_names:
1232                    continue
1233                else:
1234                    package_names.append(p.depends_on.name)
1235                    new_list.append(p.pk)
1236                    # while we're here we may as well total up the size to
1237                    # avoid iterating again
1238                    total_size += p.depends_on.size
1239
1240            # We want to return a queryset here for consistency so pick the
1241            # deps from the new_list
1242            packages_list = package_dependencies.filter(Q(pk__in=new_list))
1243
1244        return {'packages': packages_list,
1245                'size': total_size}
1246
1247    def all_depends(self):
1248        """ Returns just the depends packages and not any other dep_type
1249        Note that this is for any target
1250        """
1251        return self.filter(Q(dep_type=Package_Dependency.TYPE_RDEPENDS) |
1252                           Q(dep_type=Package_Dependency.TYPE_TRDEPENDS))
1253
1254
1255class Package_Dependency(models.Model):
1256    TYPE_RDEPENDS = 0
1257    TYPE_TRDEPENDS = 1
1258    TYPE_RRECOMMENDS = 2
1259    TYPE_TRECOMMENDS = 3
1260    TYPE_RSUGGESTS = 4
1261    TYPE_RPROVIDES = 5
1262    TYPE_RREPLACES = 6
1263    TYPE_RCONFLICTS = 7
1264    ' TODO: bpackage should be changed to remove the DEPENDS_TYPE access '
1265    DEPENDS_TYPE = (
1266        (TYPE_RDEPENDS, "depends"),
1267        (TYPE_TRDEPENDS, "depends"),
1268        (TYPE_TRECOMMENDS, "recommends"),
1269        (TYPE_RRECOMMENDS, "recommends"),
1270        (TYPE_RSUGGESTS, "suggests"),
1271        (TYPE_RPROVIDES, "provides"),
1272        (TYPE_RREPLACES, "replaces"),
1273        (TYPE_RCONFLICTS, "conflicts"),
1274    )
1275    """ Indexed by dep_type, in view order, key for short name and help
1276        description which when viewed will be printf'd with the
1277        package name.
1278    """
1279    DEPENDS_DICT = {
1280        TYPE_RDEPENDS :     ("depends", "%s is required to run %s"),
1281        TYPE_TRDEPENDS :    ("depends", "%s is required to run %s"),
1282        TYPE_TRECOMMENDS :  ("recommends", "%s extends the usability of %s"),
1283        TYPE_RRECOMMENDS :  ("recommends", "%s extends the usability of %s"),
1284        TYPE_RSUGGESTS :    ("suggests", "%s is suggested for installation with %s"),
1285        TYPE_RPROVIDES :    ("provides", "%s is provided by %s"),
1286        TYPE_RREPLACES :    ("replaces", "%s is replaced by %s"),
1287        TYPE_RCONFLICTS :   ("conflicts", "%s conflicts with %s, which will not be installed if this package is not first removed"),
1288    }
1289
1290    package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='package_dependencies_source')
1291    depends_on = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='package_dependencies_target')   # soft dependency
1292    dep_type = models.IntegerField(choices=DEPENDS_TYPE)
1293    target = models.ForeignKey(Target, on_delete=models.CASCADE, null=True)
1294    objects = Package_DependencyManager()
1295
1296class Target_Installed_Package(models.Model):
1297    target = models.ForeignKey(Target, on_delete=models.CASCADE)
1298    package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='buildtargetlist_package')
1299
1300
1301class Package_File(models.Model):
1302    package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='buildfilelist_package')
1303    path = models.FilePathField(max_length=255, blank=True)
1304    size = models.IntegerField()
1305
1306
1307class Recipe(models.Model):
1308    search_allowed_fields = ['name', 'version', 'file_path', 'section',
1309                             'summary', 'description', 'license',
1310                             'layer_version__layer__name',
1311                             'layer_version__branch', 'layer_version__commit',
1312                             'layer_version__local_path',
1313                             'layer_version__layer_source']
1314
1315    up_date = models.DateTimeField(null=True, default=None)
1316
1317    name = models.CharField(max_length=100, blank=True)
1318    version = models.CharField(max_length=100, blank=True)
1319    layer_version = models.ForeignKey('Layer_Version', on_delete=models.CASCADE,
1320                                      related_name='recipe_layer_version')
1321    summary = models.TextField(blank=True)
1322    description = models.TextField(blank=True)
1323    section = models.CharField(max_length=100, blank=True)
1324    license = models.CharField(max_length=200, blank=True)
1325    homepage = models.URLField(blank=True)
1326    bugtracker = models.URLField(blank=True)
1327    file_path = models.FilePathField(max_length=255)
1328    pathflags = models.CharField(max_length=200, blank=True)
1329    is_image = models.BooleanField(default=False)
1330
1331    def __unicode__(self):
1332        return "Recipe " + self.name + ":" + self.version
1333
1334    def get_vcs_recipe_file_link_url(self):
1335        return self.layer_version.get_vcs_file_link_url(self.file_path)
1336
1337    def get_description_or_summary(self):
1338        if self.description:
1339            return self.description
1340        elif self.summary:
1341            return self.summary
1342        else:
1343            return ""
1344
1345    class Meta:
1346        unique_together = (("layer_version", "file_path", "pathflags"), )
1347
1348
1349class Recipe_DependencyManager(models.Manager):
1350    use_for_related_fields = True
1351
1352    def get_queryset(self):
1353        return super(Recipe_DependencyManager, self).get_queryset().exclude(recipe_id = F('depends_on__id'))
1354
1355class Provides(models.Model):
1356    name = models.CharField(max_length=100)
1357    recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE)
1358
1359class Recipe_Dependency(models.Model):
1360    TYPE_DEPENDS = 0
1361    TYPE_RDEPENDS = 1
1362
1363    DEPENDS_TYPE = (
1364        (TYPE_DEPENDS, "depends"),
1365        (TYPE_RDEPENDS, "rdepends"),
1366    )
1367    recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE, related_name='r_dependencies_recipe')
1368    depends_on = models.ForeignKey(Recipe, on_delete=models.CASCADE, related_name='r_dependencies_depends')
1369    via = models.ForeignKey(Provides, on_delete=models.CASCADE, null=True, default=None)
1370    dep_type = models.IntegerField(choices=DEPENDS_TYPE)
1371    objects = Recipe_DependencyManager()
1372
1373
1374class Machine(models.Model):
1375    search_allowed_fields = ["name", "description", "layer_version__layer__name"]
1376    up_date = models.DateTimeField(null = True, default = None)
1377
1378    layer_version = models.ForeignKey('Layer_Version', on_delete=models.CASCADE)
1379    name = models.CharField(max_length=255)
1380    description = models.CharField(max_length=255)
1381
1382    def get_vcs_machine_file_link_url(self):
1383        path = 'conf/machine/'+self.name+'.conf'
1384
1385        return self.layer_version.get_vcs_file_link_url(path)
1386
1387    def __unicode__(self):
1388        return "Machine " + self.name + "(" + self.description + ")"
1389
1390
1391class BitbakeVersion(models.Model):
1392
1393    name = models.CharField(max_length=32, unique = True)
1394    giturl = GitURLField()
1395    branch = models.CharField(max_length=32)
1396    dirpath = models.CharField(max_length=255)
1397
1398    def __unicode__(self):
1399        return "%s (Branch: %s)" % (self.name, self.branch)
1400
1401
1402class Release(models.Model):
1403    """ A release is a project template, used to pre-populate Project settings with a configuration set """
1404    name = models.CharField(max_length=32, unique = True)
1405    description = models.CharField(max_length=255)
1406    bitbake_version = models.ForeignKey(BitbakeVersion, on_delete=models.CASCADE)
1407    branch_name = models.CharField(max_length=50, default = "")
1408    helptext = models.TextField(null=True)
1409
1410    def __unicode__(self):
1411        return "%s (%s)" % (self.name, self.branch_name)
1412
1413    def __str__(self):
1414        return self.name
1415
1416class ReleaseDefaultLayer(models.Model):
1417    release = models.ForeignKey(Release, on_delete=models.CASCADE)
1418    layer_name = models.CharField(max_length=100, default="")
1419
1420
1421class LayerSource(object):
1422    """ Where the layer metadata came from """
1423    TYPE_LOCAL = 0
1424    TYPE_LAYERINDEX = 1
1425    TYPE_IMPORTED = 2
1426    TYPE_BUILD = 3
1427
1428    SOURCE_TYPE = (
1429        (TYPE_LOCAL, "local"),
1430        (TYPE_LAYERINDEX, "layerindex"),
1431        (TYPE_IMPORTED, "imported"),
1432        (TYPE_BUILD, "build"),
1433    )
1434
1435    def types_dict():
1436        """ Turn the TYPES enums into a simple dictionary """
1437        dictionary = {}
1438        for key in LayerSource.__dict__:
1439            if "TYPE" in key:
1440                dictionary[key] = getattr(LayerSource, key)
1441        return dictionary
1442
1443
1444class Layer(models.Model):
1445
1446    up_date = models.DateTimeField(null=True, default=timezone.now)
1447
1448    name = models.CharField(max_length=100)
1449    layer_index_url = models.URLField()
1450    vcs_url = GitURLField(default=None, null=True)
1451    local_source_dir = models.TextField(null=True, default=None)
1452    vcs_web_url = models.URLField(null=True, default=None)
1453    vcs_web_tree_base_url = models.URLField(null=True, default=None)
1454    vcs_web_file_base_url = models.URLField(null=True, default=None)
1455
1456    summary = models.TextField(help_text='One-line description of the layer',
1457                               null=True, default=None)
1458    description = models.TextField(null=True, default=None)
1459
1460    def __unicode__(self):
1461        return "%s / %s " % (self.name, self.summary)
1462
1463
1464class Layer_Version(models.Model):
1465    """
1466    A Layer_Version either belongs to a single project or no project
1467    """
1468    search_allowed_fields = ["layer__name", "layer__summary",
1469                             "layer__description", "layer__vcs_url",
1470                             "dirpath", "release__name", "commit", "branch"]
1471
1472    build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='layer_version_build',
1473                              default=None, null=True)
1474
1475    layer = models.ForeignKey(Layer, on_delete=models.CASCADE, related_name='layer_version_layer')
1476
1477    layer_source = models.IntegerField(choices=LayerSource.SOURCE_TYPE,
1478                                       default=0)
1479
1480    up_date = models.DateTimeField(null=True, default=timezone.now)
1481
1482    # To which metadata release does this layer version belong to
1483    release = models.ForeignKey(Release, on_delete=models.CASCADE, null=True, default=None)
1484
1485    branch = models.CharField(max_length=80)
1486    commit = models.CharField(max_length=100)
1487    # If the layer is in a subdir
1488    dirpath = models.CharField(max_length=255, null=True, default=None)
1489
1490    # if -1, this is a default layer
1491    priority = models.IntegerField(default=0)
1492
1493    # where this layer exists on the filesystem
1494    local_path = models.FilePathField(max_length=1024, default="/")
1495
1496    # Set if this layer is restricted to a particular project
1497    project = models.ForeignKey('Project', on_delete=models.CASCADE, null=True, default=None)
1498
1499    # code lifted, with adaptations, from the layerindex-web application
1500    # https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/
1501    def _handle_url_path(self, base_url, path):
1502        import posixpath
1503        if base_url:
1504            if self.dirpath:
1505                if path:
1506                    extra_path = self.dirpath + '/' + path
1507                    # Normalise out ../ in path for usage URL
1508                    extra_path = posixpath.normpath(extra_path)
1509                    # Minor workaround to handle case where subdirectory has been added between branches
1510                    # (should probably support usage URL per branch to handle this... sigh...)
1511                    if extra_path.startswith('../'):
1512                        extra_path = extra_path[3:]
1513                else:
1514                    extra_path = self.dirpath
1515            else:
1516                extra_path = path
1517            branchname = self.release.name
1518            url = base_url.replace('%branch%', branchname)
1519
1520            # If there's a % in the path (e.g. a wildcard bbappend) we need to encode it
1521            if extra_path:
1522                extra_path = extra_path.replace('%', '%25')
1523
1524            if '%path%' in base_url:
1525                if extra_path:
1526                    url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '\\1', url)
1527                else:
1528                    url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '', url)
1529                return url.replace('%path%', extra_path)
1530            else:
1531                return url + extra_path
1532        return None
1533
1534    def get_vcs_link_url(self):
1535        if self.layer.vcs_web_url is None:
1536            return None
1537        return self.layer.vcs_web_url
1538
1539    def get_vcs_file_link_url(self, file_path=""):
1540        if self.layer.vcs_web_file_base_url is None:
1541            return None
1542        return self._handle_url_path(self.layer.vcs_web_file_base_url,
1543                                     file_path)
1544
1545    def get_vcs_dirpath_link_url(self):
1546        if self.layer.vcs_web_tree_base_url is None:
1547            return None
1548        return self._handle_url_path(self.layer.vcs_web_tree_base_url, '')
1549
1550    def get_vcs_reference(self):
1551        if self.commit is not None and len(self.commit) > 0:
1552            return self.commit
1553        if self.branch is not None and len(self.branch) > 0:
1554            return self.branch
1555        if self.release is not None:
1556            return self.release.name
1557        return 'N/A'
1558
1559    def get_detailspage_url(self, project_id=None):
1560        """ returns the url to the layer details page uses own project
1561        field if project_id is not specified """
1562
1563        if project_id is None:
1564            project_id = self.project.pk
1565
1566        return reverse('layerdetails', args=(project_id, self.pk))
1567
1568    def get_alldeps(self, project_id):
1569        """Get full list of unique layer dependencies."""
1570        def gen_layerdeps(lver, project, depth):
1571            if depth == 0:
1572                return
1573            for ldep in lver.dependencies.all():
1574                yield ldep.depends_on
1575                # get next level of deps recursively calling gen_layerdeps
1576                for subdep in gen_layerdeps(ldep.depends_on, project, depth-1):
1577                    yield subdep
1578
1579        project = Project.objects.get(pk=project_id)
1580        result = []
1581        projectlvers = [player.layercommit for player in
1582                        project.projectlayer_set.all()]
1583        # protect against infinite layer dependency loops
1584        maxdepth = 20
1585        for dep in gen_layerdeps(self, project, maxdepth):
1586            # filter out duplicates and layers already belonging to the project
1587            if dep not in result + projectlvers:
1588                result.append(dep)
1589
1590        return sorted(result, key=lambda x: x.layer.name)
1591
1592    def __unicode__(self):
1593        return ("id %d belongs to layer: %s" % (self.pk, self.layer.name))
1594
1595    def __str__(self):
1596        if self.release:
1597            release = self.release.name
1598        else:
1599            release = "No release set"
1600
1601        return "%d %s (%s)" % (self.pk, self.layer.name, release)
1602
1603
1604class LayerVersionDependency(models.Model):
1605
1606    layer_version = models.ForeignKey(Layer_Version, on_delete=models.CASCADE,
1607                                      related_name="dependencies")
1608    depends_on = models.ForeignKey(Layer_Version, on_delete=models.CASCADE,
1609                                   related_name="dependees")
1610
1611class ProjectLayer(models.Model):
1612    project = models.ForeignKey(Project, on_delete=models.CASCADE)
1613    layercommit = models.ForeignKey(Layer_Version, on_delete=models.CASCADE, null=True)
1614    optional = models.BooleanField(default = True)
1615
1616    def __unicode__(self):
1617        return "%s, %s" % (self.project.name, self.layercommit)
1618
1619    class Meta:
1620        unique_together = (("project", "layercommit"),)
1621
1622class CustomImageRecipe(Recipe):
1623
1624    # CustomImageRecipe's belong to layers called:
1625    LAYER_NAME = "toaster-custom-images"
1626
1627    search_allowed_fields = ['name']
1628    base_recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE, related_name='based_on_recipe')
1629    project = models.ForeignKey(Project, on_delete=models.CASCADE)
1630    last_updated = models.DateTimeField(null=True, default=None)
1631
1632    def get_last_successful_built_target(self):
1633        """ Return the last successful built target object if one exists
1634        otherwise return None """
1635        return Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
1636                                     Q(build__project=self.project) &
1637                                     Q(target=self.name)).last()
1638
1639    def update_package_list(self):
1640        """ Update the package list from the last good build of this
1641        CustomImageRecipe
1642        """
1643        # Check if we're aldready up-to-date or not
1644        target = self.get_last_successful_built_target()
1645        if target is None:
1646            # So we've never actually built this Custom recipe but what about
1647            # the recipe it's based on?
1648            target = \
1649                Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
1650                                      Q(build__project=self.project) &
1651                                      Q(target=self.base_recipe.name)).last()
1652            if target is None:
1653                return
1654
1655        if target.build.completed_on == self.last_updated:
1656            return
1657
1658        self.includes_set.clear()
1659
1660        excludes_list = self.excludes_set.values_list('name', flat=True)
1661        appends_list = self.appends_set.values_list('name', flat=True)
1662
1663        built_packages_list = \
1664            target.target_installed_package_set.values_list('package__name',
1665                                                            flat=True)
1666        for built_package in built_packages_list:
1667            # Is the built package in the custom packages list?
1668            if built_package in excludes_list:
1669                continue
1670
1671            if built_package in appends_list:
1672                continue
1673
1674            cust_img_p = \
1675                    CustomImagePackage.objects.get(name=built_package)
1676            self.includes_set.add(cust_img_p)
1677
1678
1679        self.last_updated = target.build.completed_on
1680        self.save()
1681
1682    def get_all_packages(self):
1683        """Get the included packages and any appended packages"""
1684        self.update_package_list()
1685
1686        return CustomImagePackage.objects.filter((Q(recipe_appends=self) |
1687                                                  Q(recipe_includes=self)) &
1688                                                 ~Q(recipe_excludes=self))
1689
1690    def get_base_recipe_file(self):
1691        """Get the base recipe file path if it exists on the file system"""
1692        path_schema_one = "%s/%s" % (self.base_recipe.layer_version.local_path,
1693                                     self.base_recipe.file_path)
1694
1695        path_schema_two = self.base_recipe.file_path
1696
1697        path_schema_three = "%s/%s" % (self.base_recipe.layer_version.layer.local_source_dir,
1698                                     self.base_recipe.file_path)
1699
1700        if os.path.exists(path_schema_one):
1701            return path_schema_one
1702
1703        # The path may now be the full path if the recipe has been built
1704        if os.path.exists(path_schema_two):
1705            return path_schema_two
1706
1707        # Or a local path if all layers are local
1708        if os.path.exists(path_schema_three):
1709            return path_schema_three
1710
1711        return None
1712
1713    def generate_recipe_file_contents(self):
1714        """Generate the contents for the recipe file."""
1715        # If we have no excluded packages we only need to :append
1716        if self.excludes_set.count() == 0:
1717            packages_conf = "IMAGE_INSTALL:append = \" "
1718
1719            for pkg in self.appends_set.all():
1720                packages_conf += pkg.name+' '
1721        else:
1722            packages_conf = "IMAGE_FEATURES =\"\"\nIMAGE_INSTALL = \""
1723            # We add all the known packages to be built by this recipe apart
1724            # from locale packages which are are controlled with IMAGE_LINGUAS.
1725            for pkg in self.get_all_packages().exclude(
1726                    name__icontains="locale"):
1727                packages_conf += pkg.name+' '
1728
1729        packages_conf += "\""
1730
1731        base_recipe_path = self.get_base_recipe_file()
1732        if base_recipe_path and os.path.isfile(base_recipe_path):
1733            base_recipe = open(base_recipe_path, 'r').read()
1734        else:
1735            # Pass back None to trigger error message to user
1736            return None
1737
1738        # Add a special case for when the recipe we have based a custom image
1739        # recipe on requires another recipe.
1740        # For example:
1741        # "require core-image-minimal.bb" is changed to:
1742        # "require recipes-core/images/core-image-minimal.bb"
1743
1744        req_search = re.search(r'(require\s+)(.+\.bb\s*$)',
1745                               base_recipe,
1746                               re.MULTILINE)
1747        if req_search:
1748            require_filename = req_search.group(2).strip()
1749
1750            corrected_location = Recipe.objects.filter(
1751                Q(layer_version=self.base_recipe.layer_version) &
1752                Q(file_path__icontains=require_filename)).last().file_path
1753
1754            new_require_line = "require %s" % corrected_location
1755
1756            base_recipe = base_recipe.replace(req_search.group(0),
1757                                              new_require_line)
1758
1759        info = {
1760            "date": timezone.now().strftime("%Y-%m-%d %H:%M:%S"),
1761            "base_recipe": base_recipe,
1762            "recipe_name": self.name,
1763            "base_recipe_name": self.base_recipe.name,
1764            "license": self.license,
1765            "summary": self.summary,
1766            "description": self.description,
1767            "packages_conf": packages_conf.strip()
1768        }
1769
1770        recipe_contents = ("# Original recipe %(base_recipe_name)s \n"
1771                           "%(base_recipe)s\n\n"
1772                           "# Recipe %(recipe_name)s \n"
1773                           "# Customisation Generated by Toaster on %(date)s\n"
1774                           "SUMMARY = \"%(summary)s\"\n"
1775                           "DESCRIPTION = \"%(description)s\"\n"
1776                           "LICENSE = \"%(license)s\"\n"
1777                           "%(packages_conf)s") % info
1778
1779        return recipe_contents
1780
1781class ProjectVariable(models.Model):
1782    project = models.ForeignKey(Project, on_delete=models.CASCADE)
1783    name = models.CharField(max_length=100)
1784    value = models.TextField(blank = True)
1785
1786class Variable(models.Model):
1787    search_allowed_fields = ['variable_name', 'variable_value',
1788                             'vhistory__file_name', "description"]
1789    build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='variable_build')
1790    variable_name = models.CharField(max_length=100)
1791    variable_value = models.TextField(blank=True)
1792    changed = models.BooleanField(default=False)
1793    human_readable_name = models.CharField(max_length=200)
1794    description = models.TextField(blank=True)
1795
1796class VariableHistory(models.Model):
1797    variable = models.ForeignKey(Variable, on_delete=models.CASCADE, related_name='vhistory')
1798    value   = models.TextField(blank=True)
1799    file_name = models.FilePathField(max_length=255)
1800    line_number = models.IntegerField(null=True)
1801    operation = models.CharField(max_length=64)
1802
1803class HelpText(models.Model):
1804    VARIABLE = 0
1805    HELPTEXT_AREA = ((VARIABLE, 'variable'), )
1806
1807    build = models.ForeignKey(Build, on_delete=models.CASCADE, related_name='helptext_build')
1808    area = models.IntegerField(choices=HELPTEXT_AREA)
1809    key = models.CharField(max_length=100)
1810    text = models.TextField()
1811
1812class LogMessage(models.Model):
1813    EXCEPTION = -1      # used to signal self-toaster-exceptions
1814    INFO = 0
1815    WARNING = 1
1816    ERROR = 2
1817    CRITICAL = 3
1818
1819    LOG_LEVEL = (
1820        (INFO, "info"),
1821        (WARNING, "warn"),
1822        (ERROR, "error"),
1823        (CRITICAL, "critical"),
1824        (EXCEPTION, "toaster exception")
1825    )
1826
1827    build = models.ForeignKey(Build, on_delete=models.CASCADE)
1828    task  = models.ForeignKey(Task, on_delete=models.CASCADE, blank = True, null=True)
1829    level = models.IntegerField(choices=LOG_LEVEL, default=INFO)
1830    message = models.TextField(blank=True, null=True)
1831    pathname = models.FilePathField(max_length=255, blank=True)
1832    lineno = models.IntegerField(null=True)
1833
1834    def __str__(self):
1835        return force_bytes('%s %s %s' % (self.get_level_display(), self.message, self.build))
1836
1837def invalidate_cache(**kwargs):
1838    from django.core.cache import cache
1839    try:
1840      cache.clear()
1841    except Exception as e:
1842      logger.warning("Problem with cache backend: Failed to clear cache: %s" % e)
1843
1844def signal_runbuilds():
1845    """Send SIGUSR1 to runbuilds process"""
1846    try:
1847        with open(os.path.join(os.getenv('BUILDDIR', '.'),
1848                               '.runbuilds.pid')) as pidf:
1849            os.kill(int(pidf.read()), SIGUSR1)
1850    except FileNotFoundError:
1851        logger.info("Stopping existing runbuilds: no current process found")
1852    except ProcessLookupError:
1853        logger.warning("Stopping existing runbuilds: process lookup not found")
1854
1855class Distro(models.Model):
1856    search_allowed_fields = ["name", "description", "layer_version__layer__name"]
1857    up_date = models.DateTimeField(null = True, default = None)
1858
1859    layer_version = models.ForeignKey('Layer_Version', on_delete=models.CASCADE)
1860    name = models.CharField(max_length=255)
1861    description = models.CharField(max_length=255)
1862
1863    def get_vcs_distro_file_link_url(self):
1864        path = 'conf/distro/%s.conf' % self.name
1865        return self.layer_version.get_vcs_file_link_url(path)
1866
1867    def __unicode__(self):
1868        return "Distro " + self.name + "(" + self.description + ")"
1869
1870class EventLogsImports(models.Model):
1871    name = models.CharField(max_length=255)
1872    imported = models.BooleanField(default=False)
1873    build_id = models.IntegerField(blank=True, null=True)
1874
1875    def __str__(self):
1876        return self.name
1877
1878
1879django.db.models.signals.post_save.connect(invalidate_cache)
1880django.db.models.signals.post_delete.connect(invalidate_cache)
1881django.db.models.signals.m2m_changed.connect(invalidate_cache)
1882