1#
2# SPDX-License-Identifier: GPL-2.0-only
3#
4
5from django.core.management.base import BaseCommand
6from django.db import transaction
7from django.db.models import Q
8
9from bldcontrol.bbcontroller import getBuildEnvironmentController
10from bldcontrol.models import BuildRequest, BuildEnvironment
11from bldcontrol.models import BRError, BRVariable
12
13from orm.models import Build, LogMessage, Target
14
15import logging
16import traceback
17import signal
18import os
19
20logger = logging.getLogger("toaster")
21
22
23class Command(BaseCommand):
24    args = ""
25    help = "Schedules and executes build requests as possible. "\
26           "Does not return (interrupt with Ctrl-C)"
27
28    @transaction.atomic
29    def _selectBuildEnvironment(self):
30        bec = getBuildEnvironmentController(lock=BuildEnvironment.LOCK_FREE)
31        bec.be.lock = BuildEnvironment.LOCK_LOCK
32        bec.be.save()
33        return bec
34
35    @transaction.atomic
36    def _selectBuildRequest(self):
37        br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first()
38        return br
39
40    def schedule(self):
41        try:
42            # select the build environment and the request to build
43            br = self._selectBuildRequest()
44            if br:
45                br.state = BuildRequest.REQ_INPROGRESS
46                br.save()
47            else:
48                return
49
50            try:
51                bec = self._selectBuildEnvironment()
52            except IndexError as e:
53                # we could not find a BEC; postpone the BR
54                br.state = BuildRequest.REQ_QUEUED
55                br.save()
56                logger.debug("runbuilds: No build env (%s)" % e)
57                return
58
59            logger.info("runbuilds: starting build %s, environment %s" %
60                        (br, bec.be))
61
62            # let the build request know where it is being executed
63            br.environment = bec.be
64            br.save()
65
66            # this triggers an async build
67            bec.triggerBuild(br.brbitbake, br.brlayer_set.all(),
68                             br.brvariable_set.all(), br.brtarget_set.all(),
69                             "%d:%d" % (br.pk, bec.be.pk))
70
71        except Exception as e:
72            logger.error("runbuilds: Error launching build %s" % e)
73            traceback.print_exc()
74            if "[Errno 111] Connection refused" in str(e):
75                # Connection refused, read toaster_server.out
76                errmsg = bec.readServerLogFile()
77            else:
78                errmsg = str(e)
79
80            BRError.objects.create(req=br, errtype=str(type(e)), errmsg=errmsg,
81                                   traceback=traceback.format_exc())
82            br.state = BuildRequest.REQ_FAILED
83            br.save()
84            bec.be.lock = BuildEnvironment.LOCK_FREE
85            bec.be.save()
86            # Cancel the pending build and report the exception to the UI
87            log_object = LogMessage.objects.create(
88                            build = br.build,
89                            level = LogMessage.EXCEPTION,
90                            message = errmsg)
91            log_object.save()
92            br.build.outcome = Build.FAILED
93            br.build.save()
94
95    def archive(self):
96        for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE):
97            if br.build is None:
98                br.state = BuildRequest.REQ_FAILED
99            else:
100                br.state = BuildRequest.REQ_COMPLETED
101            br.save()
102
103    def cleanup(self):
104        from django.utils import timezone
105        from datetime import timedelta
106        # environments locked for more than 30 seconds
107        # they should be unlocked
108        BuildEnvironment.objects.filter(
109            Q(buildrequest__state__in=[BuildRequest.REQ_FAILED,
110                                       BuildRequest.REQ_COMPLETED,
111                                       BuildRequest.REQ_CANCELLING]) &
112            Q(lock=BuildEnvironment.LOCK_LOCK) &
113            Q(updated__lt=timezone.now() - timedelta(seconds=30))
114        ).update(lock=BuildEnvironment.LOCK_FREE)
115
116        # update all Builds that were in progress and failed to start
117        for br in BuildRequest.objects.filter(
118                state=BuildRequest.REQ_FAILED,
119                build__outcome=Build.IN_PROGRESS):
120            # transpose the launch errors in ToasterExceptions
121            br.build.outcome = Build.FAILED
122            for brerror in br.brerror_set.all():
123                logger.debug("Saving error %s" % brerror)
124                LogMessage.objects.create(build=br.build,
125                                          level=LogMessage.EXCEPTION,
126                                          message=brerror.errmsg)
127            br.build.save()
128
129            # we don't have a true build object here; hence, toasterui
130            # didn't have a change to release the BE lock
131            br.environment.lock = BuildEnvironment.LOCK_FREE
132            br.environment.save()
133
134        # update all BuildRequests without a build created
135        for br in BuildRequest.objects.filter(build=None):
136            br.build = Build.objects.create(project=br.project,
137                                            completed_on=br.updated,
138                                            started_on=br.created)
139            br.build.outcome = Build.FAILED
140            try:
141                br.build.machine = br.brvariable_set.get(name='MACHINE').value
142            except BRVariable.DoesNotExist:
143                pass
144            br.save()
145            # transpose target information
146            for brtarget in br.brtarget_set.all():
147                Target.objects.create(build=br.build,
148                                      target=brtarget.target,
149                                      task=brtarget.task)
150            # transpose the launch errors in ToasterExceptions
151            for brerror in br.brerror_set.all():
152                LogMessage.objects.create(build=br.build,
153                                          level=LogMessage.EXCEPTION,
154                                          message=brerror.errmsg)
155
156            br.build.save()
157
158        # Make sure the LOCK is removed for builds which have been fully
159        # cancelled
160        for br in BuildRequest.objects.filter(
161                      Q(build__outcome=Build.CANCELLED) &
162                      Q(state=BuildRequest.REQ_CANCELLING) &
163                      ~Q(environment=None)):
164            br.environment.lock = BuildEnvironment.LOCK_FREE
165            br.environment.save()
166
167    def runbuild(self):
168        try:
169            self.cleanup()
170        except Exception as e:
171            logger.warning("runbuilds: cleanup exception %s" % str(e))
172
173        try:
174            self.archive()
175        except Exception as e:
176            logger.warning("runbuilds: archive exception %s" % str(e))
177
178        try:
179            self.schedule()
180        except Exception as e:
181            logger.warning("runbuilds: schedule exception %s" % str(e))
182
183    # Test to see if a build pre-maturely died due to a bitbake crash
184    def check_dead_builds(self):
185        do_cleanup = False
186        try:
187            for br in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS):
188                # Get the build directory
189                if br.project.builddir:
190                    builddir =  br.project.builddir
191                else:
192                    builddir = '%s-toaster-%d' % (br.environment.builddir,br.project.id)
193                # Check log to see if there is a recent traceback
194                toaster_ui_log = os.path.join(builddir, 'toaster_ui.log')
195                test_file = os.path.join(builddir, '._toaster_check.txt')
196                os.system("tail -n 50 %s > %s" % (os.path.join(builddir, 'toaster_ui.log'),test_file))
197                traceback_text = ''
198                is_traceback = False
199                with open(test_file,'r') as test_file_fd:
200                    test_file_tail = test_file_fd.readlines()
201                    for line in test_file_tail:
202                        if line.startswith('Traceback (most recent call last):'):
203                            traceback_text = line
204                            is_traceback = True
205                        elif line.startswith('NOTE: ToasterUI waiting for events'):
206                            # Ignore any traceback before new build start
207                            traceback_text = ''
208                            is_traceback = False
209                        elif line.startswith('Note: Toaster traceback auto-stop'):
210                            # Ignore any traceback before this previous traceback catch
211                            traceback_text = ''
212                            is_traceback = False
213                        elif is_traceback:
214                            traceback_text += line
215                # Test the results
216                is_stop = False
217                if is_traceback:
218                    # Found a traceback
219                    errtype = 'Bitbake crash'
220                    errmsg = 'Bitbake crash\n' + traceback_text
221                    state = BuildRequest.REQ_FAILED
222                    # Clean up bitbake files
223                    bitbake_lock = os.path.join(builddir, 'bitbake.lock')
224                    if os.path.isfile(bitbake_lock):
225                        os.remove(bitbake_lock)
226                    bitbake_sock = os.path.join(builddir, 'bitbake.sock')
227                    if os.path.isfile(bitbake_sock):
228                        os.remove(bitbake_sock)
229                    if os.path.isfile(test_file):
230                        os.remove(test_file)
231                    # Add note to ignore this traceback on next check
232                    os.system('echo "Note: Toaster traceback auto-stop" >> %s' % toaster_ui_log)
233                    is_stop = True
234                # Add more tests here
235                #elif ...
236                # Stop the build request?
237                if is_stop:
238                    brerror = BRError(
239                        req = br,
240                        errtype = errtype,
241                        errmsg = errmsg,
242                        traceback = traceback_text,
243                        )
244                    brerror.save()
245                    br.state = state
246                    br.save()
247                    do_cleanup = True
248            # Do cleanup
249            if do_cleanup:
250                self.cleanup()
251        except Exception as e:
252            logger.error("runbuilds: Error in check_dead_builds %s" % e)
253
254    def handle(self, **options):
255        pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
256                                    ".runbuilds.pid")
257
258        with open(pidfile_path, 'w') as pidfile:
259            pidfile.write("%s" % os.getpid())
260
261        # Clean up any stale/failed builds from previous Toaster run
262        self.runbuild()
263
264        signal.signal(signal.SIGUSR1, lambda sig, frame: None)
265
266        while True:
267            sigset = signal.sigtimedwait([signal.SIGUSR1], 5)
268            if sigset:
269                for sig in sigset:
270                    # Consume each captured pending event
271                    self.runbuild()
272            else:
273                # Check for build exceptions
274                self.check_dead_builds()
275
276