1from django.core.management.base import BaseCommand 2from django.db import transaction 3from django.db.models import Q 4 5from bldcontrol.bbcontroller import getBuildEnvironmentController 6from bldcontrol.models import BuildRequest, BuildEnvironment 7from bldcontrol.models import BRError, BRVariable 8 9from orm.models import Build, LogMessage, Target 10 11import logging 12import traceback 13import signal 14import os 15 16logger = logging.getLogger("toaster") 17 18 19class Command(BaseCommand): 20 args = "" 21 help = "Schedules and executes build requests as possible. "\ 22 "Does not return (interrupt with Ctrl-C)" 23 24 @transaction.atomic 25 def _selectBuildEnvironment(self): 26 bec = getBuildEnvironmentController(lock=BuildEnvironment.LOCK_FREE) 27 bec.be.lock = BuildEnvironment.LOCK_LOCK 28 bec.be.save() 29 return bec 30 31 @transaction.atomic 32 def _selectBuildRequest(self): 33 br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first() 34 return br 35 36 def schedule(self): 37 try: 38 # select the build environment and the request to build 39 br = self._selectBuildRequest() 40 if br: 41 br.state = BuildRequest.REQ_INPROGRESS 42 br.save() 43 else: 44 return 45 46 try: 47 bec = self._selectBuildEnvironment() 48 except IndexError as e: 49 # we could not find a BEC; postpone the BR 50 br.state = BuildRequest.REQ_QUEUED 51 br.save() 52 logger.debug("runbuilds: No build env") 53 return 54 55 logger.info("runbuilds: starting build %s, environment %s" % 56 (br, bec.be)) 57 58 # let the build request know where it is being executed 59 br.environment = bec.be 60 br.save() 61 62 # this triggers an async build 63 bec.triggerBuild(br.brbitbake, br.brlayer_set.all(), 64 br.brvariable_set.all(), br.brtarget_set.all(), 65 "%d:%d" % (br.pk, bec.be.pk)) 66 67 except Exception as e: 68 logger.error("runbuilds: Error launching build %s" % e) 69 traceback.print_exc() 70 if "[Errno 111] Connection refused" in str(e): 71 # Connection refused, read toaster_server.out 72 errmsg = bec.readServerLogFile() 73 else: 74 errmsg = str(e) 75 76 BRError.objects.create(req=br, errtype=str(type(e)), errmsg=errmsg, 77 traceback=traceback.format_exc()) 78 br.state = BuildRequest.REQ_FAILED 79 br.save() 80 bec.be.lock = BuildEnvironment.LOCK_FREE 81 bec.be.save() 82 # Cancel the pending build and report the exception to the UI 83 log_object = LogMessage.objects.create( 84 build = br.build, 85 level = LogMessage.EXCEPTION, 86 message = errmsg) 87 log_object.save() 88 br.build.outcome = Build.FAILED 89 br.build.save() 90 91 def archive(self): 92 for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE): 93 if br.build is None: 94 br.state = BuildRequest.REQ_FAILED 95 else: 96 br.state = BuildRequest.REQ_COMPLETED 97 br.save() 98 99 def cleanup(self): 100 from django.utils import timezone 101 from datetime import timedelta 102 # environments locked for more than 30 seconds 103 # they should be unlocked 104 BuildEnvironment.objects.filter( 105 Q(buildrequest__state__in=[BuildRequest.REQ_FAILED, 106 BuildRequest.REQ_COMPLETED, 107 BuildRequest.REQ_CANCELLING]) & 108 Q(lock=BuildEnvironment.LOCK_LOCK) & 109 Q(updated__lt=timezone.now() - timedelta(seconds=30)) 110 ).update(lock=BuildEnvironment.LOCK_FREE) 111 112 # update all Builds that were in progress and failed to start 113 for br in BuildRequest.objects.filter( 114 state=BuildRequest.REQ_FAILED, 115 build__outcome=Build.IN_PROGRESS): 116 # transpose the launch errors in ToasterExceptions 117 br.build.outcome = Build.FAILED 118 for brerror in br.brerror_set.all(): 119 logger.debug("Saving error %s" % brerror) 120 LogMessage.objects.create(build=br.build, 121 level=LogMessage.EXCEPTION, 122 message=brerror.errmsg) 123 br.build.save() 124 125 # we don't have a true build object here; hence, toasterui 126 # didn't have a change to release the BE lock 127 br.environment.lock = BuildEnvironment.LOCK_FREE 128 br.environment.save() 129 130 # update all BuildRequests without a build created 131 for br in BuildRequest.objects.filter(build=None): 132 br.build = Build.objects.create(project=br.project, 133 completed_on=br.updated, 134 started_on=br.created) 135 br.build.outcome = Build.FAILED 136 try: 137 br.build.machine = br.brvariable_set.get(name='MACHINE').value 138 except BRVariable.DoesNotExist: 139 pass 140 br.save() 141 # transpose target information 142 for brtarget in br.brtarget_set.all(): 143 Target.objects.create(build=br.build, 144 target=brtarget.target, 145 task=brtarget.task) 146 # transpose the launch errors in ToasterExceptions 147 for brerror in br.brerror_set.all(): 148 LogMessage.objects.create(build=br.build, 149 level=LogMessage.EXCEPTION, 150 message=brerror.errmsg) 151 152 br.build.save() 153 154 # Make sure the LOCK is removed for builds which have been fully 155 # cancelled 156 for br in BuildRequest.objects.filter( 157 Q(build__outcome=Build.CANCELLED) & 158 Q(state=BuildRequest.REQ_CANCELLING) & 159 ~Q(environment=None)): 160 br.environment.lock = BuildEnvironment.LOCK_FREE 161 br.environment.save() 162 163 def runbuild(self): 164 try: 165 self.cleanup() 166 except Exception as e: 167 logger.warn("runbuilds: cleanup exception %s" % str(e)) 168 169 try: 170 self.archive() 171 except Exception as e: 172 logger.warn("runbuilds: archive exception %s" % str(e)) 173 174 try: 175 self.schedule() 176 except Exception as e: 177 logger.warn("runbuilds: schedule exception %s" % str(e)) 178 179 def handle(self, **options): 180 pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."), 181 ".runbuilds.pid") 182 183 with open(pidfile_path, 'w') as pidfile: 184 pidfile.write("%s" % os.getpid()) 185 186 self.runbuild() 187 188 signal.signal(signal.SIGUSR1, lambda sig, frame: None) 189 190 while True: 191 signal.pause() 192 self.runbuild() 193