1#
2# BitBake Toaster Implementation
3#
4# Copyright (C) 2018        Wind River Systems
5#
6# SPDX-License-Identifier: GPL-2.0-only
7#
8
9# buildimport: import a project for project specific configuration
10#
11# Usage:
12#  (a) Set up Toaster environent
13#
14#  (b) Call buildimport
15#      $ /path/to/bitbake/lib/toaster/manage.py buildimport \
16#        --name=$PROJECTNAME \
17#        --path=$BUILD_DIRECTORY \
18#        --callback="$CALLBACK_SCRIPT" \
19#        --command="configure|reconfigure|import"
20#
21#  (c) Return is "|Default_image=%s|Project_id=%d"
22#
23#  (d) Open Toaster to this project using for example:
24#      $ xdg-open http://localhost:$toaster_port/toastergui/project_specific/$project_id
25#
26#  (e) To delete a project:
27#      $ /path/to/bitbake/lib/toaster/manage.py buildimport \
28#        --name=$PROJECTNAME --delete-project
29#
30
31
32# ../bitbake/lib/toaster/manage.py buildimport --name=test --path=`pwd` --callback="" --command=import
33
34from django.core.management.base import BaseCommand
35from orm.models import Project, Release, ProjectVariable
36from orm.models import Layer, Layer_Version, LayerSource, ProjectLayer
37from toastergui.api import scan_layer_content
38
39import os
40import re
41import os.path
42import subprocess
43import shutil
44
45# Toaster variable section delimiters
46TOASTER_PROLOG = '#=== TOASTER_CONFIG_PROLOG ==='
47TOASTER_EPILOG = '#=== TOASTER_CONFIG_EPILOG ==='
48
49# quick development/debugging support
50verbose = 2
51def _log(msg):
52    if 1 == verbose:
53        print(msg)
54    elif 2 == verbose:
55        f1=open('/tmp/toaster.log', 'a')
56        f1.write("|" + msg + "|\n" )
57        f1.close()
58
59
60__config_regexp__  = re.compile( r"""
61    ^
62    (?P<exp>export\s+)?
63    (?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
64    (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
65
66    \s* (
67        (?P<colon>:=) |
68        (?P<lazyques>\?\?=) |
69        (?P<ques>\?=) |
70        (?P<append>\+=) |
71        (?P<prepend>=\+) |
72        (?P<predot>=\.) |
73        (?P<postdot>\.=) |
74        =
75    ) \s*
76
77    (?!'[^']*'[^']*'$)
78    (?!\"[^\"]*\"[^\"]*\"$)
79    (?P<apo>['\"])
80    (?P<value>.*)
81    (?P=apo)
82    $
83    """, re.X)
84
85class Command(BaseCommand):
86    args    = "<name> <path> <release>"
87    help    = "Import a command line build directory"
88    vars    = {}
89    toaster_vars = {}
90
91    def add_arguments(self, parser):
92        parser.add_argument(
93            '--name', dest='name', required=True,
94            help='name of the project',
95            )
96        parser.add_argument(
97            '--path', dest='path', required=True,
98            help='path to the project',
99            )
100        parser.add_argument(
101            '--release', dest='release', required=False,
102            help='release for the project',
103            )
104        parser.add_argument(
105            '--callback', dest='callback', required=False,
106            help='callback for project config update',
107            )
108        parser.add_argument(
109            '--delete-project', dest='delete_project', required=False,
110            help='delete this project from the database',
111            )
112        parser.add_argument(
113            '--command', dest='command', required=False,
114            help='command (configure,reconfigure,import)',
115            )
116
117    def get_var(self, varname):
118        value = self.vars.get(varname, '')
119        if value:
120            varrefs = re.findall('\${([^}]*)}', value)
121            for ref in varrefs:
122                if ref in self.vars:
123                    value = value.replace('${%s}' % ref, self.vars[ref])
124        return value
125
126    # Extract the bb variables from a conf file
127    def scan_conf(self,fn):
128        vars = self.vars
129        toaster_vars = self.toaster_vars
130
131        #_log("scan_conf:%s" % fn)
132        if not os.path.isfile(fn):
133            return
134        f = open(fn, 'r')
135
136        #statements = ast.StatementGroup()
137        lineno = 0
138        is_toaster_section = False
139        while True:
140            lineno = lineno + 1
141            s = f.readline()
142            if not s:
143                break
144            w = s.strip()
145            # skip empty lines
146            if not w:
147                continue
148            # evaluate Toaster sections
149            if w.startswith(TOASTER_PROLOG):
150                is_toaster_section = True
151                continue
152            if w.startswith(TOASTER_EPILOG):
153                is_toaster_section = False
154                continue
155            s = s.rstrip()
156            while s[-1] == '\\':
157                s2 = f.readline().strip()
158                lineno = lineno + 1
159                if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
160                    echo("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
161                s = s[:-1] + s2
162            # skip comments
163            if s[0] == '#':
164                continue
165            # process the line for just assignments
166            m = __config_regexp__.match(s)
167            if m:
168                groupd = m.groupdict()
169                var = groupd['var']
170                value = groupd['value']
171
172                if groupd['lazyques']:
173                    if not var in vars:
174                        vars[var] = value
175                    continue
176                if groupd['ques']:
177                    if not var in vars:
178                        vars[var] = value
179                    continue
180                # preset empty blank for remaining operators
181                if not var in vars:
182                    vars[var] = ''
183                if groupd['append']:
184                    vars[var] += value
185                elif groupd['prepend']:
186                    vars[var] = "%s%s" % (value,vars[var])
187                elif groupd['predot']:
188                    vars[var] = "%s %s" % (value,vars[var])
189                elif groupd['postdot']:
190                    vars[var] = "%s %s" % (vars[var],value)
191                else:
192                    vars[var] = "%s" % (value)
193                # capture vars in a Toaster section
194                if is_toaster_section:
195                    toaster_vars[var] = vars[var]
196
197        # DONE WITH PARSING
198        f.close()
199        self.vars = vars
200        self.toaster_vars = toaster_vars
201
202    # Update the scanned project variables
203    def update_project_vars(self,project,name):
204        pv, create = ProjectVariable.objects.get_or_create(project = project, name = name)
205        if (not name in self.vars.keys()) or (not self.vars[name]):
206            self.vars[name] = pv.value
207        else:
208            if pv.value != self.vars[name]:
209                pv.value = self.vars[name]
210        pv.save()
211
212    # Find the git version of the installation
213    def find_layer_dir_version(self,path):
214        #  * rocko               ...
215
216        install_version = ''
217        cwd = os.getcwd()
218        os.chdir(path)
219        p = subprocess.Popen(['git', 'branch', '-av'], stdout=subprocess.PIPE,
220                                                stderr=subprocess.PIPE)
221        out, err = p.communicate()
222        out = out.decode("utf-8")
223        for branch in out.split('\n'):
224            if ('*' == branch[0:1]) and ('no branch' not in branch):
225                install_version = re.sub(' .*','',branch[2:])
226                break
227            if 'remotes/m/master' in branch:
228                install_version = re.sub('.*base/','',branch)
229                break
230        os.chdir(cwd)
231        return install_version
232
233    # Compute table of the installation's registered layer versions (branch or commit)
234    def find_layer_dir_versions(self,INSTALL_URL_PREFIX):
235        lv_dict = {}
236        layer_versions = Layer_Version.objects.all()
237        for lv in layer_versions:
238            layer = Layer.objects.filter(pk=lv.layer.pk)[0]
239            if layer.vcs_url:
240                url_short = layer.vcs_url.replace(INSTALL_URL_PREFIX,'')
241            else:
242                url_short = ''
243            # register the core, branch, and the version variations
244            lv_dict["%s,%s,%s" % (url_short,lv.dirpath,'')] = (lv.id,layer.name)
245            lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.branch)] = (lv.id,layer.name)
246            lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.commit)] = (lv.id,layer.name)
247            #_log("  (%s,%s,%s|%s) = (%s,%s)" % (url_short,lv.dirpath,lv.branch,lv.commit,lv.id,layer.name))
248        return lv_dict
249
250    # Apply table of all layer versions
251    def extract_bblayers(self):
252        # set up the constants
253        bblayer_str = self.get_var('BBLAYERS')
254        TOASTER_DIR = os.environ.get('TOASTER_DIR')
255        INSTALL_CLONE_PREFIX = os.path.dirname(TOASTER_DIR) + "/"
256        TOASTER_CLONE_PREFIX = TOASTER_DIR + "/_toaster_clones/"
257        INSTALL_URL_PREFIX = ''
258        layers = Layer.objects.filter(name='openembedded-core')
259        for layer in layers:
260            if layer.vcs_url:
261                INSTALL_URL_PREFIX = layer.vcs_url
262                break
263        INSTALL_URL_PREFIX = INSTALL_URL_PREFIX.replace("/poky","/")
264        INSTALL_VERSION_DIR = TOASTER_DIR
265        INSTALL_URL_POSTFIX = INSTALL_URL_PREFIX.replace(':','_')
266        INSTALL_URL_POSTFIX = INSTALL_URL_POSTFIX.replace('/','_')
267        INSTALL_URL_POSTFIX = "%s_%s" % (TOASTER_CLONE_PREFIX,INSTALL_URL_POSTFIX)
268
269        # get the set of available layer:layer_versions
270        lv_dict = self.find_layer_dir_versions(INSTALL_URL_PREFIX)
271
272        # compute the layer matches
273        layers_list = []
274        for line in bblayer_str.split(' '):
275            if not line:
276                continue
277            if line.endswith('/local'):
278                continue
279
280            # isolate the repo
281            layer_path = line
282            line = line.replace(INSTALL_URL_POSTFIX,'').replace(INSTALL_CLONE_PREFIX,'').replace('/layers/','/').replace('/poky/','/')
283
284            # isolate the sub-path
285            path_index = line.rfind('/')
286            if path_index > 0:
287                sub_path = line[path_index+1:]
288                line = line[0:path_index]
289            else:
290                sub_path = ''
291
292            # isolate the version
293            if TOASTER_CLONE_PREFIX in layer_path:
294                is_toaster_clone = True
295                # extract version from name syntax
296                version_index = line.find('_')
297                if version_index > 0:
298                    version = line[version_index+1:]
299                    line = line[0:version_index]
300                else:
301                    version = ''
302                _log("TOASTER_CLONE(%s/%s), version=%s" % (line,sub_path,version))
303            else:
304                is_toaster_clone = False
305                # version is from the installation
306                version = self.find_layer_dir_version(layer_path)
307                _log("LOCAL_CLONE(%s/%s), version=%s" % (line,sub_path,version))
308
309            # capture the layer information into layers_list
310            layers_list.append( (line,sub_path,version,layer_path,is_toaster_clone) )
311        return layers_list,lv_dict
312
313    #
314    def find_import_release(self,layers_list,lv_dict,default_release):
315        #   poky,meta,rocko => 4;openembedded-core
316        release = default_release
317        for line,path,version,layer_path,is_toaster_clone in layers_list:
318            key = "%s,%s,%s" % (line,path,version)
319            if key in lv_dict:
320                lv_id = lv_dict[key]
321                if 'openembedded-core' == lv_id[1]:
322                    _log("Find_import_release(%s):version=%s,Toaster=%s" % (lv_id[1],version,is_toaster_clone))
323                    # only versions in Toaster managed layers are accepted
324                    if not is_toaster_clone:
325                        break
326                    try:
327                        release = Release.objects.get(name=version)
328                    except:
329                        pass
330                    break
331        _log("Find_import_release:RELEASE=%s" % release.name)
332        return release
333
334    # Apply the found conf layers
335    def apply_conf_bblayers(self,layers_list,lv_dict,project,release=None):
336        for line,path,version,layer_path,is_toaster_clone in layers_list:
337            # Assert release promote if present
338            if release:
339                version = release
340            # try to match the key to a layer_version
341            key = "%s,%s,%s" % (line,path,version)
342            key_short = "%s,%s,%s" % (line,path,'')
343            lv_id = ''
344            if key in lv_dict:
345                lv_id = lv_dict[key]
346                lv = Layer_Version.objects.get(pk=int(lv_id[0]))
347                pl,created = ProjectLayer.objects.get_or_create(project=project,
348                                                   layercommit=lv)
349                pl.optional=False
350                pl.save()
351                _log("  %s => %s;%s" % (key,lv_id[0],lv_id[1]))
352            elif key_short in lv_dict:
353                lv_id = lv_dict[key_short]
354                lv = Layer_Version.objects.get(pk=int(lv_id[0]))
355                pl,created = ProjectLayer.objects.get_or_create(project=project,
356                                                   layercommit=lv)
357                pl.optional=False
358                pl.save()
359                _log("  %s ?> %s" % (key,lv_dict[key_short]))
360            else:
361                _log("%s <= %s" % (key,layer_path))
362                found = False
363                # does local layer already exist in this project?
364                try:
365                    for pl in ProjectLayer.objects.filter(project=project):
366                        if pl.layercommit.layer.local_source_dir == layer_path:
367                            found = True
368                            _log("  Project Local Layer found!")
369                except Exception as e:
370                    _log("ERROR: Local Layer '%s'" % e)
371                    pass
372
373                if not found:
374                    # Does Layer name+path already exist?
375                    try:
376                        layer_name_base = os.path.basename(layer_path)
377                        _log("Layer_lookup: try '%s','%s'" % (layer_name_base,layer_path))
378                        layer = Layer.objects.get(name=layer_name_base,local_source_dir = layer_path)
379                        # Found! Attach layer_version and ProjectLayer
380                        layer_version = Layer_Version.objects.create(
381                            layer=layer,
382                            project=project,
383                            layer_source=LayerSource.TYPE_IMPORTED)
384                        layer_version.save()
385                        pl,created = ProjectLayer.objects.get_or_create(project=project,
386                                                           layercommit=layer_version)
387                        pl.optional=False
388                        pl.save()
389                        found = True
390                        # add layer contents to this layer version
391                        scan_layer_content(layer,layer_version)
392                        _log("  Parent Local Layer found in db!")
393                    except Exception as e:
394                        _log("Layer_exists_test_failed: Local Layer '%s'" % e)
395                        pass
396
397                if not found:
398                    # Insure that layer path exists, in case of user typo
399                    if not os.path.isdir(layer_path):
400                        _log("ERROR:Layer path '%s' not found" % layer_path)
401                        continue
402                    # Add layer to db and attach project to it
403                    layer_name_base = os.path.basename(layer_path)
404                    # generate a unique layer name
405                    layer_name_matches = {}
406                    for layer in Layer.objects.filter(name__contains=layer_name_base):
407                        layer_name_matches[layer.name] = '1'
408                    layer_name_idx = 0
409                    layer_name_test = layer_name_base
410                    while layer_name_test in layer_name_matches.keys():
411                        layer_name_idx += 1
412                        layer_name_test = "%s_%d" % (layer_name_base,layer_name_idx)
413                    # create the layer and layer_verion objects
414                    layer = Layer.objects.create(name=layer_name_test)
415                    layer.local_source_dir = layer_path
416                    layer_version = Layer_Version.objects.create(
417                        layer=layer,
418                        project=project,
419                        layer_source=LayerSource.TYPE_IMPORTED)
420                    layer.save()
421                    layer_version.save()
422                    pl,created = ProjectLayer.objects.get_or_create(project=project,
423                                                       layercommit=layer_version)
424                    pl.optional=False
425                    pl.save()
426                    # register the layer's content
427                    _log("  Local Layer Add content")
428                    scan_layer_content(layer,layer_version)
429                    _log("  Local Layer Added '%s'!" % layer_name_test)
430
431    # Scan the project's conf files (if any)
432    def scan_conf_variables(self,project_path):
433        self.vars['TOPDIR'] = project_path
434        # scan the project's settings, add any new layers or variables
435        if os.path.isfile("%s/conf/local.conf" % project_path):
436            self.scan_conf("%s/conf/local.conf" % project_path)
437            self.scan_conf("%s/conf/bblayers.conf" % project_path)
438            # Import then disable old style Toaster conf files (before 'merged_attr')
439            old_toaster_local = "%s/conf/toaster.conf" % project_path
440            if os.path.isfile(old_toaster_local):
441                self.scan_conf(old_toaster_local)
442                shutil.move(old_toaster_local, old_toaster_local+"_old")
443            old_toaster_layer = "%s/conf/toaster-bblayers.conf" % project_path
444            if os.path.isfile(old_toaster_layer):
445                self.scan_conf(old_toaster_layer)
446                shutil.move(old_toaster_layer, old_toaster_layer+"_old")
447
448    # Scan the found conf variables (if any)
449    def apply_conf_variables(self,project,layers_list,lv_dict,release=None):
450        if self.vars:
451            # Catch vars relevant to Toaster (in case no Toaster section)
452            self.update_project_vars(project,'DISTRO')
453            self.update_project_vars(project,'MACHINE')
454            self.update_project_vars(project,'IMAGE_INSTALL_append')
455            self.update_project_vars(project,'IMAGE_FSTYPES')
456            self.update_project_vars(project,'PACKAGE_CLASSES')
457            # These vars are typically only assigned by Toaster
458            #self.update_project_vars(project,'DL_DIR')
459            #self.update_project_vars(project,'SSTATE_DIR')
460
461            # Assert found Toaster vars
462            for var in self.toaster_vars.keys():
463                pv, create = ProjectVariable.objects.get_or_create(project = project, name = var)
464                pv.value = self.toaster_vars[var]
465                _log("* Add/update Toaster var '%s' = '%s'" % (pv.name,pv.value))
466                pv.save()
467
468            # Assert found BBLAYERS
469            if 0 < verbose:
470                for pl in ProjectLayer.objects.filter(project=project):
471                    release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
472                    print(" BEFORE:ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
473            self.apply_conf_bblayers(layers_list,lv_dict,project,release)
474            if 0 < verbose:
475                for pl in ProjectLayer.objects.filter(project=project):
476                    release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
477                    print(" AFTER :ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
478
479    def handle(self, *args, **options):
480        project_name = options['name']
481        project_path = options['path']
482        project_callback = options['callback'] if options['callback'] else ''
483        release_name = options['release'] if options['release'] else ''
484
485        #
486        # Delete project
487        #
488
489        if options['delete_project']:
490            try:
491                print("Project '%s' delete from Toaster database" % (project_name))
492                project = Project.objects.get(name=project_name)
493                # TODO: deep project delete
494                project.delete()
495                print("Project '%s' Deleted" % (project_name))
496                return
497            except Exception as e:
498                print("Project '%s' not found, not deleted (%s)" % (project_name,e))
499                return
500
501        #
502        # Create/Update/Import project
503        #
504
505        # See if project (by name) exists
506        project = None
507        try:
508            # Project already exists
509            project = Project.objects.get(name=project_name)
510        except Exception as e:
511            pass
512
513        # Find the installation's default release
514        default_release = Release.objects.get(id=1)
515
516        # SANITY: if 'reconfig' but project does not exist (deleted externally), switch to 'import'
517        if ("reconfigure" == options['command']) and project is None:
518            options['command'] = 'import'
519
520        # 'Configure':
521        if "configure" == options['command']:
522            # Note: ignore any existing conf files
523            # create project, SANITY: reuse any project of same name
524            project = Project.objects.create_project(project_name,default_release,project)
525
526        # 'Re-configure':
527        if "reconfigure" == options['command']:
528            # Scan the directory's conf files
529            self.scan_conf_variables(project_path)
530            # Scan the layer list
531            layers_list,lv_dict = self.extract_bblayers()
532            # Apply any new layers or variables
533            self.apply_conf_variables(project,layers_list,lv_dict)
534
535        # 'Import':
536        if "import" == options['command']:
537            # Scan the directory's conf files
538            self.scan_conf_variables(project_path)
539            # Remove these Toaster controlled variables
540            for var in ('DL_DIR','SSTATE_DIR'):
541                self.vars.pop(var, None)
542                self.toaster_vars.pop(var, None)
543            # Scan the layer list
544            layers_list,lv_dict = self.extract_bblayers()
545            # Find the directory's release, and promote to default_release if local paths
546            release = self.find_import_release(layers_list,lv_dict,default_release)
547            # create project, SANITY: reuse any project of same name
548            project = Project.objects.create_project(project_name,release,project)
549            # Apply any new layers or variables
550            self.apply_conf_variables(project,layers_list,lv_dict,release)
551            # WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific'
552            project.set_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE','1')
553
554        # Set up the project's meta data
555        project.builddir = project_path
556        project.merged_attr = True
557        project.set_variable(Project.PROJECT_SPECIFIC_CALLBACK,project_callback)
558        project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_EDIT)
559        if ("configure" == options['command']) or ("import" == options['command']):
560            # preset the mode and default image recipe
561            project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NEW)
562            project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,"core-image-minimal")
563
564            # Assert any extended/custom actions or variables for new non-Toaster projects
565            if not len(self.toaster_vars):
566                pass
567        else:
568            project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NONE)
569
570        # Save the updated Project
571        project.save()
572
573        _log("Buildimport:project='%s' at '%d'" % (project_name,project.id))
574
575        if ('DEFAULT_IMAGE' in self.vars) and (self.vars['DEFAULT_IMAGE']):
576            print("|Default_image=%s|Project_id=%d" % (self.vars['DEFAULT_IMAGE'],project.id))
577        else:
578            print("|Project_id=%d" % (project.id))
579
580