1# 2# SPDX-License-Identifier: GPL-2.0-only 3# 4 5from django.core.management.base import BaseCommand 6from django.db import transaction 7from django.db.models import Q 8 9from bldcontrol.bbcontroller import getBuildEnvironmentController 10from bldcontrol.models import BuildRequest, BuildEnvironment 11from bldcontrol.models import BRError, BRVariable 12 13from orm.models import Build, LogMessage, Target 14 15import logging 16import traceback 17import signal 18import os 19 20logger = logging.getLogger("toaster") 21 22 23class Command(BaseCommand): 24 args = "" 25 help = "Schedules and executes build requests as possible. "\ 26 "Does not return (interrupt with Ctrl-C)" 27 28 @transaction.atomic 29 def _selectBuildEnvironment(self): 30 bec = getBuildEnvironmentController(lock=BuildEnvironment.LOCK_FREE) 31 bec.be.lock = BuildEnvironment.LOCK_LOCK 32 bec.be.save() 33 return bec 34 35 @transaction.atomic 36 def _selectBuildRequest(self): 37 br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first() 38 return br 39 40 def schedule(self): 41 try: 42 # select the build environment and the request to build 43 br = self._selectBuildRequest() 44 if br: 45 br.state = BuildRequest.REQ_INPROGRESS 46 br.save() 47 else: 48 return 49 50 try: 51 bec = self._selectBuildEnvironment() 52 except IndexError as e: 53 # we could not find a BEC; postpone the BR 54 br.state = BuildRequest.REQ_QUEUED 55 br.save() 56 logger.debug("runbuilds: No build env (%s)" % e) 57 return 58 59 logger.info("runbuilds: starting build %s, environment %s" % 60 (br, bec.be)) 61 62 # let the build request know where it is being executed 63 br.environment = bec.be 64 br.save() 65 66 # this triggers an async build 67 bec.triggerBuild(br.brbitbake, br.brlayer_set.all(), 68 br.brvariable_set.all(), br.brtarget_set.all(), 69 "%d:%d" % (br.pk, bec.be.pk)) 70 71 except Exception as e: 72 logger.error("runbuilds: Error launching build %s" % e) 73 traceback.print_exc() 74 if "[Errno 111] Connection refused" in str(e): 75 # Connection refused, read toaster_server.out 76 errmsg = bec.readServerLogFile() 77 else: 78 errmsg = str(e) 79 80 BRError.objects.create(req=br, errtype=str(type(e)), errmsg=errmsg, 81 traceback=traceback.format_exc()) 82 br.state = BuildRequest.REQ_FAILED 83 br.save() 84 bec.be.lock = BuildEnvironment.LOCK_FREE 85 bec.be.save() 86 # Cancel the pending build and report the exception to the UI 87 log_object = LogMessage.objects.create( 88 build = br.build, 89 level = LogMessage.EXCEPTION, 90 message = errmsg) 91 log_object.save() 92 br.build.outcome = Build.FAILED 93 br.build.save() 94 95 def archive(self): 96 for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE): 97 if br.build is None: 98 br.state = BuildRequest.REQ_FAILED 99 else: 100 br.state = BuildRequest.REQ_COMPLETED 101 br.save() 102 103 def cleanup(self): 104 from django.utils import timezone 105 from datetime import timedelta 106 # environments locked for more than 30 seconds 107 # they should be unlocked 108 BuildEnvironment.objects.filter( 109 Q(buildrequest__state__in=[BuildRequest.REQ_FAILED, 110 BuildRequest.REQ_COMPLETED, 111 BuildRequest.REQ_CANCELLING]) & 112 Q(lock=BuildEnvironment.LOCK_LOCK) & 113 Q(updated__lt=timezone.now() - timedelta(seconds=30)) 114 ).update(lock=BuildEnvironment.LOCK_FREE) 115 116 # update all Builds that were in progress and failed to start 117 for br in BuildRequest.objects.filter( 118 state=BuildRequest.REQ_FAILED, 119 build__outcome=Build.IN_PROGRESS): 120 # transpose the launch errors in ToasterExceptions 121 br.build.outcome = Build.FAILED 122 for brerror in br.brerror_set.all(): 123 logger.debug("Saving error %s" % brerror) 124 LogMessage.objects.create(build=br.build, 125 level=LogMessage.EXCEPTION, 126 message=brerror.errmsg) 127 br.build.save() 128 129 # we don't have a true build object here; hence, toasterui 130 # didn't have a change to release the BE lock 131 br.environment.lock = BuildEnvironment.LOCK_FREE 132 br.environment.save() 133 134 # update all BuildRequests without a build created 135 for br in BuildRequest.objects.filter(build=None): 136 br.build = Build.objects.create(project=br.project, 137 completed_on=br.updated, 138 started_on=br.created) 139 br.build.outcome = Build.FAILED 140 try: 141 br.build.machine = br.brvariable_set.get(name='MACHINE').value 142 except BRVariable.DoesNotExist: 143 pass 144 br.save() 145 # transpose target information 146 for brtarget in br.brtarget_set.all(): 147 Target.objects.create(build=br.build, 148 target=brtarget.target, 149 task=brtarget.task) 150 # transpose the launch errors in ToasterExceptions 151 for brerror in br.brerror_set.all(): 152 LogMessage.objects.create(build=br.build, 153 level=LogMessage.EXCEPTION, 154 message=brerror.errmsg) 155 156 br.build.save() 157 158 # Make sure the LOCK is removed for builds which have been fully 159 # cancelled 160 for br in BuildRequest.objects.filter( 161 Q(build__outcome=Build.CANCELLED) & 162 Q(state=BuildRequest.REQ_CANCELLING) & 163 ~Q(environment=None)): 164 br.environment.lock = BuildEnvironment.LOCK_FREE 165 br.environment.save() 166 167 def runbuild(self): 168 try: 169 self.cleanup() 170 except Exception as e: 171 logger.warning("runbuilds: cleanup exception %s" % str(e)) 172 173 try: 174 self.archive() 175 except Exception as e: 176 logger.warning("runbuilds: archive exception %s" % str(e)) 177 178 try: 179 self.schedule() 180 except Exception as e: 181 logger.warning("runbuilds: schedule exception %s" % str(e)) 182 183 # Test to see if a build pre-maturely died due to a bitbake crash 184 def check_dead_builds(self): 185 do_cleanup = False 186 try: 187 for br in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS): 188 # Get the build directory 189 if br.project.builddir: 190 builddir = br.project.builddir 191 else: 192 builddir = '%s-toaster-%d' % (br.environment.builddir,br.project.id) 193 # Check log to see if there is a recent traceback 194 toaster_ui_log = os.path.join(builddir, 'toaster_ui.log') 195 test_file = os.path.join(builddir, '._toaster_check.txt') 196 os.system("tail -n 50 %s > %s" % (os.path.join(builddir, 'toaster_ui.log'),test_file)) 197 traceback_text = '' 198 is_traceback = False 199 with open(test_file,'r') as test_file_fd: 200 test_file_tail = test_file_fd.readlines() 201 for line in test_file_tail: 202 if line.startswith('Traceback (most recent call last):'): 203 traceback_text = line 204 is_traceback = True 205 elif line.startswith('NOTE: ToasterUI waiting for events'): 206 # Ignore any traceback before new build start 207 traceback_text = '' 208 is_traceback = False 209 elif line.startswith('Note: Toaster traceback auto-stop'): 210 # Ignore any traceback before this previous traceback catch 211 traceback_text = '' 212 is_traceback = False 213 elif is_traceback: 214 traceback_text += line 215 # Test the results 216 is_stop = False 217 if is_traceback: 218 # Found a traceback 219 errtype = 'Bitbake crash' 220 errmsg = 'Bitbake crash\n' + traceback_text 221 state = BuildRequest.REQ_FAILED 222 # Clean up bitbake files 223 bitbake_lock = os.path.join(builddir, 'bitbake.lock') 224 if os.path.isfile(bitbake_lock): 225 os.remove(bitbake_lock) 226 bitbake_sock = os.path.join(builddir, 'bitbake.sock') 227 if os.path.isfile(bitbake_sock): 228 os.remove(bitbake_sock) 229 if os.path.isfile(test_file): 230 os.remove(test_file) 231 # Add note to ignore this traceback on next check 232 os.system('echo "Note: Toaster traceback auto-stop" >> %s' % toaster_ui_log) 233 is_stop = True 234 # Add more tests here 235 #elif ... 236 # Stop the build request? 237 if is_stop: 238 brerror = BRError( 239 req = br, 240 errtype = errtype, 241 errmsg = errmsg, 242 traceback = traceback_text, 243 ) 244 brerror.save() 245 br.state = state 246 br.save() 247 do_cleanup = True 248 # Do cleanup 249 if do_cleanup: 250 self.cleanup() 251 except Exception as e: 252 logger.error("runbuilds: Error in check_dead_builds %s" % e) 253 254 def handle(self, **options): 255 pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."), 256 ".runbuilds.pid") 257 258 with open(pidfile_path, 'w') as pidfile: 259 pidfile.write("%s" % os.getpid()) 260 261 # Clean up any stale/failed builds from previous Toaster run 262 self.runbuild() 263 264 signal.signal(signal.SIGUSR1, lambda sig, frame: None) 265 266 while True: 267 sigset = signal.sigtimedwait([signal.SIGUSR1], 5) 268 if sigset: 269 for sig in sigset: 270 # Consume each captured pending event 271 self.runbuild() 272 else: 273 # Check for build exceptions 274 self.check_dead_builds() 275 276