1#
2# BitBake Toaster Implementation
3#
4# Copyright (C) 2015        Intel Corporation
5#
6# SPDX-License-Identifier: GPL-2.0-only
7#
8
9from django.views.generic import View, TemplateView
10from django.views.decorators.cache import cache_control
11from django.shortcuts import HttpResponse
12from django.core.cache import cache
13from django.core.paginator import Paginator, EmptyPage
14from django.db.models import Q
15from orm.models import Project, Build
16from django.template import Context, Template
17from django.template import VariableDoesNotExist
18from django.template import TemplateSyntaxError
19from django.core.serializers.json import DjangoJSONEncoder
20from django.core.exceptions import FieldError
21from django.utils import timezone
22from toastergui.templatetags.projecttags import sectohms, get_tasks
23from toastergui.templatetags.projecttags import json as template_json
24from django.http import JsonResponse
25from django.urls import reverse
26
27import types
28import json
29import collections
30import re
31import os
32
33from toastergui.tablefilter import TableFilterMap
34
35try:
36    from urllib import unquote_plus
37except ImportError:
38    from urllib.parse import unquote_plus
39
40import logging
41logger = logging.getLogger("toaster")
42
43
44class NoFieldOrDataName(Exception):
45    pass
46
47
48class ToasterTable(TemplateView):
49    def __init__(self, *args, **kwargs):
50        super(ToasterTable, self).__init__()
51        if 'template_name' in kwargs:
52            self.template_name = kwargs['template_name']
53        self.title = "Table"
54        self.queryset = None
55        self.columns = []
56
57        # map from field names to Filter instances
58        self.filter_map = TableFilterMap()
59
60        self.total_count = 0
61        self.static_context_extra = {}
62        self.empty_state = "Sorry - no data found"
63        self.default_orderby = ""
64
65    # prevent HTTP caching of table data
66    @cache_control(must_revalidate=True,
67                   max_age=0, no_store=True, no_cache=True)
68    def dispatch(self, *args, **kwargs):
69        return super(ToasterTable, self).dispatch(*args, **kwargs)
70
71    def get_context_data(self, **kwargs):
72        context = super(ToasterTable, self).get_context_data(**kwargs)
73        context['title'] = self.title
74        context['table_name'] = type(self).__name__.lower()
75        context['empty_state'] = self.empty_state
76
77        # global variables
78        context['project_enable'] = ('1' == os.environ.get('TOASTER_BUILDSERVER'))
79        try:
80            context['project_specific'] = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC'))
81        except:
82            context['project_specific'] = ''
83
84        return context
85
86    def get(self, request, *args, **kwargs):
87        if request.GET.get('format', None) == 'json':
88
89            self.setup_queryset(*args, **kwargs)
90            # Put the project id into the context for the static_data_template
91            if 'pid' in kwargs:
92                self.static_context_extra['pid'] = kwargs['pid']
93
94            cmd = request.GET.get('cmd', None)
95            if cmd and 'filterinfo' in cmd:
96                data = self.get_filter_info(request, **kwargs)
97            else:
98                # If no cmd is specified we give you the table data
99                data = self.get_data(request, **kwargs)
100
101            return HttpResponse(data, content_type="application/json")
102
103        return super(ToasterTable, self).get(request, *args, **kwargs)
104
105    def get_filter_info(self, request, **kwargs):
106        self.setup_filters(**kwargs)
107
108        search = request.GET.get("search", None)
109        if search:
110            self.apply_search(search)
111
112        name = request.GET.get("name", None)
113        table_filter = self.filter_map.get_filter(name)
114        return json.dumps(table_filter.to_json(self.queryset),
115                          indent=2,
116                          cls=DjangoJSONEncoder)
117
118    def setup_columns(self, *args, **kwargs):
119        """ function to implement in the subclass which sets up
120        the columns """
121        pass
122
123    def setup_filters(self, *args, **kwargs):
124        """ function to implement in the subclass which sets up the
125        filters """
126        pass
127
128    def setup_queryset(self, *args, **kwargs):
129        """ function to implement in the subclass which sets up the
130        queryset"""
131        pass
132
133    def add_filter(self, table_filter):
134        """Add a filter to the table.
135
136        Args:
137            table_filter: Filter instance
138        """
139        self.filter_map.add_filter(table_filter.name, table_filter)
140
141    def add_column(self, title="", help_text="",
142                   orderable=False, hideable=True, hidden=False,
143                   field_name="", filter_name=None, static_data_name=None,
144                   static_data_template=None):
145        """Add a column to the table.
146
147        Args:
148            title (str): Title for the table header
149            help_text (str): Optional help text to describe the column
150            orderable (bool): Whether the column can be ordered.
151                We order on the field_name.
152            hideable (bool): Whether the user can hide the column
153            hidden (bool): Whether the column is default hidden
154            field_name (str or list): field(s) required for this column's data
155            static_data_name (str, optional): The column's main identifier
156                which will replace the field_name.
157            static_data_template(str, optional): The template to be rendered
158                as data
159        """
160
161        self.columns.append({'title': title,
162                             'help_text': help_text,
163                             'orderable': orderable,
164                             'hideable': hideable,
165                             'hidden': hidden,
166                             'field_name': field_name,
167                             'filter_name': filter_name,
168                             'static_data_name': static_data_name,
169                             'static_data_template': static_data_template})
170
171    def set_column_hidden(self, title, hidden):
172        """
173        Set the hidden state of the column to the value of hidden
174        """
175        for col in self.columns:
176            if col['title'] == title:
177                col['hidden'] = hidden
178                break
179
180    def set_column_hideable(self, title, hideable):
181        """
182        Set the hideable state of the column to the value of hideable
183        """
184        for col in self.columns:
185            if col['title'] == title:
186                col['hideable'] = hideable
187                break
188
189    def render_static_data(self, template, row):
190        """Utility function to render the static data template"""
191
192        context = {
193          'extra': self.static_context_extra,
194          'data': row,
195        }
196
197        context = Context(context)
198        template = Template(template)
199
200        return template.render(context)
201
202    def apply_filter(self, filters, filter_value, **kwargs):
203        """
204        Apply a filter submitted in the querystring to the ToasterTable
205
206        filters: (str) in the format:
207          '<filter name>:<action name>'
208        filter_value: (str) parameters to pass to the named filter
209
210        <filter name> and <action name> are used to look up the correct filter
211        in the ToasterTable's filter map; the <action params> are set on
212        TableFilterAction* before its filter is applied and may modify the
213        queryset returned by the filter
214        """
215        self.setup_filters(**kwargs)
216
217        try:
218            filter_name, action_name = filters.split(':')
219            action_params = unquote_plus(filter_value)
220        except ValueError:
221            return
222
223        if "all" in action_name:
224            return
225
226        try:
227            table_filter = self.filter_map.get_filter(filter_name)
228            action = table_filter.get_action(action_name)
229            action.set_filter_params(action_params)
230            self.queryset = action.filter(self.queryset)
231        except KeyError:
232            # pass it to the user - programming error here
233            raise
234
235    def apply_orderby(self, orderby):
236        # Note that django will execute this when we try to retrieve the data
237        self.queryset = self.queryset.order_by(orderby)
238
239    def apply_search(self, search_term):
240        """Creates a query based on the model's search_allowed_fields"""
241
242        if not hasattr(self.queryset.model, 'search_allowed_fields'):
243            raise Exception("Search fields aren't defined in the model %s"
244                            % self.queryset.model)
245
246        search_queries = None
247        for st in search_term.split(" "):
248            queries = None
249            for field in self.queryset.model.search_allowed_fields:
250                query = Q(**{field + '__icontains': st})
251                if queries:
252                    queries |= query
253                else:
254                    queries = query
255
256            if search_queries:
257                search_queries &= queries
258            else:
259                search_queries = queries
260
261        self.queryset = self.queryset.filter(search_queries)
262
263    def get_data(self, request, **kwargs):
264        """
265        Returns the data for the page requested with the specified
266        parameters applied
267
268        filters: filter and action name, e.g. "outcome:build_succeeded"
269        filter_value: value to pass to the named filter+action, e.g. "on"
270        (for a toggle filter) or "2015-12-11,2015-12-12"
271        (for a date range filter)
272        """
273
274        page_num = request.GET.get("page", 1)
275        limit = request.GET.get("limit", 10)
276        search = request.GET.get("search", None)
277        filters = request.GET.get("filter", None)
278        filter_value = request.GET.get("filter_value", "on")
279        orderby = request.GET.get("orderby", None)
280        nocache = request.GET.get("nocache", None)
281
282        # Make a unique cache name
283        cache_name = self.__class__.__name__
284
285        for key, val in request.GET.items():
286            if key == 'nocache':
287                continue
288            cache_name = cache_name + str(key) + str(val)
289
290        for key, val in kwargs.items():
291            cache_name = cache_name + str(key) + str(val)
292
293        # No special chars allowed in the cache name apart from dash
294        cache_name = re.sub(r'[^A-Za-z0-9-]', "", cache_name)
295
296        if nocache:
297            cache.delete(cache_name)
298
299        data = cache.get(cache_name)
300
301        if data:
302            logger.debug("Got cache data for table '%s'" % self.title)
303            return data
304
305        self.setup_columns(**kwargs)
306
307        if search:
308            self.apply_search(search)
309        if filters:
310            self.apply_filter(filters, filter_value, **kwargs)
311        if orderby:
312            self.apply_orderby(orderby)
313
314        paginator = Paginator(self.queryset, limit)
315
316        try:
317            page = paginator.page(page_num)
318        except EmptyPage:
319            page = paginator.page(1)
320
321        data = {
322            'total': self.queryset.count(),
323            'default_orderby': self.default_orderby,
324            'columns': self.columns,
325            'rows': [],
326            'error': "ok",
327        }
328
329        try:
330            for model_obj in page.object_list:
331                # Use collection to maintain the order
332                required_data = collections.OrderedDict()
333
334                for col in self.columns:
335                    field = col['field_name']
336                    if not field:
337                        field = col['static_data_name']
338                    if not field:
339                        raise NoFieldOrDataName("Must supply a field_name or"
340                                                "static_data_name for column"
341                                                "%s.%s" %
342                                                (self.__class__.__name__, col)
343                                                )
344
345                    # Check if we need to process some static data
346                    if "static_data_name" in col and col['static_data_name']:
347                        # Overwrite the field_name with static_data_name
348                        # so that this can be used as the html class name
349                        col['field_name'] = col['static_data_name']
350
351                        try:
352                            # Render the template given
353                            required_data[col['static_data_name']] = \
354                                    self.render_static_data(
355                                        col['static_data_template'], model_obj)
356                        except (TemplateSyntaxError,
357                                VariableDoesNotExist) as e:
358                            logger.error("could not render template code"
359                                         "%s %s %s",
360                                         col['static_data_template'],
361                                         e, self.__class__.__name__)
362                            required_data[col['static_data_name']] =\
363                                '<!--error-->'
364
365                    else:
366                        # Traverse to any foriegn key in the field
367                        # e.g. recipe__layer_version__name
368                        model_data = None
369
370                        if "__" in field:
371                            for subfield in field.split("__"):
372                                if not model_data:
373                                    # The first iteration is always going to
374                                    # be on the actual model object instance.
375                                    # Subsequent ones are on the result of
376                                    # that. e.g. forieng key objects
377                                    model_data = getattr(model_obj,
378                                                         subfield)
379                                else:
380                                    model_data = getattr(model_data,
381                                                         subfield)
382
383                        else:
384                            model_data = getattr(model_obj,
385                                                 col['field_name'])
386
387                        # We might have a model function as the field so
388                        # call it to return the data needed
389                        if isinstance(model_data, types.MethodType):
390                            model_data = model_data()
391
392                        required_data[col['field_name']] = model_data
393
394                data['rows'].append(required_data)
395
396        except FieldError:
397            # pass  it to the user - programming-error here
398            raise
399
400        data = json.dumps(data, indent=2, cls=DjangoJSONEncoder)
401        cache.set(cache_name, data, 60*30)
402
403        return data
404
405
406class ToasterTypeAhead(View):
407    """ A typeahead mechanism to support the front end typeahead widgets """
408    MAX_RESULTS = 6
409
410    class MissingFieldsException(Exception):
411        pass
412
413    def __init__(self, *args, **kwargs):
414        super(ToasterTypeAhead, self).__init__()
415
416    def get(self, request, *args, **kwargs):
417        def response(data):
418            return HttpResponse(json.dumps(data,
419                                           indent=2,
420                                           cls=DjangoJSONEncoder),
421                                content_type="application/json")
422
423        error = "ok"
424
425        search_term = request.GET.get("search", None)
426        if search_term is None:
427            # We got no search value so return empty reponse
428            return response({'error': error, 'results': []})
429
430        try:
431            prj = Project.objects.get(pk=kwargs['pid'])
432        except KeyError:
433            prj = None
434
435        results = self.apply_search(search_term,
436                                    prj,
437                                    request)[:ToasterTypeAhead.MAX_RESULTS]
438
439        if len(results) > 0:
440            try:
441                self.validate_fields(results[0])
442            except self.MissingFieldsException as e:
443                error = e
444
445        data = {'results': results,
446                'error': error}
447
448        return response(data)
449
450    def validate_fields(self, result):
451        if 'name' in result is False or 'detail' in result is False:
452            raise self.MissingFieldsException(
453                "name and detail are required fields")
454
455    def apply_search(self, search_term, prj):
456        """ Override this function to implement search. Return an array of
457        dictionaries with a minium of a name and detail field"""
458        pass
459
460
461class MostRecentBuildsView(View):
462    def _was_yesterday_or_earlier(self, completed_on):
463        now = timezone.now()
464        delta = now - completed_on
465
466        if delta.days >= 1:
467            return True
468
469        return False
470
471    def get(self, request, *args, **kwargs):
472        """
473        Returns a list of builds in JSON format.
474        """
475        project = None
476
477        project_id = request.GET.get('project_id', None)
478        if project_id:
479            try:
480                project = Project.objects.get(pk=project_id)
481            except:
482                # if project lookup fails, assume no project
483                pass
484
485        recent_build_objs = Build.get_recent(project)
486        recent_builds = []
487
488        for build_obj in recent_build_objs:
489            dashboard_url = reverse('builddashboard', args=(build_obj.pk,))
490            buildtime_url = reverse('buildtime', args=(build_obj.pk,))
491            rebuild_url = \
492                reverse('xhr_buildrequest', args=(build_obj.project.pk,))
493            cancel_url = \
494                reverse('xhr_buildrequest', args=(build_obj.project.pk,))
495
496            build = {}
497            build['id'] = build_obj.pk
498            build['dashboard_url'] = dashboard_url
499
500            buildrequest_id = None
501            if hasattr(build_obj, 'buildrequest'):
502                buildrequest_id = build_obj.buildrequest.pk
503            build['buildrequest_id'] = buildrequest_id
504
505            if build_obj.recipes_to_parse > 0:
506                build['recipes_parsed_percentage'] = \
507                    int((build_obj.recipes_parsed /
508                         build_obj.recipes_to_parse) * 100)
509            else:
510                build['recipes_parsed_percentage'] = 0
511            if build_obj.repos_to_clone > 0:
512                build['repos_cloned_percentage'] = \
513                    int((build_obj.repos_cloned /
514                         build_obj.repos_to_clone) * 100)
515            else:
516                build['repos_cloned_percentage'] = 0
517
518            build['progress_item'] = build_obj.progress_item
519
520            tasks_complete_percentage = 0
521            if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED):
522                tasks_complete_percentage = 100
523            elif build_obj.outcome == Build.IN_PROGRESS:
524                tasks_complete_percentage = build_obj.completeper()
525            build['tasks_complete_percentage'] = tasks_complete_percentage
526
527            build['state'] = build_obj.get_state()
528
529            build['errors'] = build_obj.errors.count()
530            build['dashboard_errors_url'] = dashboard_url + '#errors'
531
532            build['warnings'] = build_obj.warnings.count()
533            build['dashboard_warnings_url'] = dashboard_url + '#warnings'
534
535            build['buildtime'] = sectohms(build_obj.timespent_seconds)
536            build['buildtime_url'] = buildtime_url
537
538            build['rebuild_url'] = rebuild_url
539            build['cancel_url'] = cancel_url
540
541            build['is_default_project_build'] = build_obj.project.is_default
542
543            build['build_targets_json'] = \
544                template_json(get_tasks(build_obj.target_set.all()))
545
546            # convert completed_on time to user's timezone
547            completed_on = timezone.localtime(build_obj.completed_on)
548
549            completed_on_template = '%H:%M'
550            if self._was_yesterday_or_earlier(completed_on):
551                completed_on_template = '%d/%m/%Y ' + completed_on_template
552            build['completed_on'] = completed_on.strftime(
553                completed_on_template)
554
555            targets = []
556            target_objs = build_obj.get_sorted_target_list()
557            for target_obj in target_objs:
558                if target_obj.task:
559                    targets.append(target_obj.target + ':' + target_obj.task)
560                else:
561                    targets.append(target_obj.target)
562            build['targets'] = ' '.join(targets)
563
564            # abbreviated form of the full target list
565            abbreviated_targets = ''
566            num_targets = len(targets)
567            if num_targets > 0:
568                abbreviated_targets = targets[0]
569            if num_targets > 1:
570                abbreviated_targets += (' +%s' % (num_targets - 1))
571            build['targets_abbreviated'] = abbreviated_targets
572
573            recent_builds.append(build)
574
575        return JsonResponse(recent_builds, safe=False)
576