1#
2# Helper functions for committing data to git and pushing upstream
3#
4# Copyright (c) 2017, Intel Corporation.
5# Copyright (c) 2019, Linux Foundation
6#
7# This program is free software; you can redistribute it and/or modify it
8# under the terms and conditions of the GNU General Public License,
9# version 2, as published by the Free Software Foundation.
10#
11# This program is distributed in the hope it will be useful, but WITHOUT
12# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
14# more details.
15#
16
17import os
18import re
19import sys
20from operator import attrgetter
21from collections import namedtuple
22from oeqa.utils.git import GitRepo, GitError
23
24class ArchiveError(Exception):
25    """Internal error handling of this script"""
26
27def format_str(string, fields):
28    """Format string using the given fields (dict)"""
29    try:
30        return string.format(**fields)
31    except KeyError as err:
32        raise ArchiveError("Unable to expand string '{}': unknown field {} "
33                           "(valid fields are: {})".format(
34                               string, err, ', '.join(sorted(fields.keys()))))
35
36
37def init_git_repo(path, no_create, bare, log):
38    """Initialize local Git repository"""
39    path = os.path.abspath(path)
40    if os.path.isfile(path):
41        raise ArchiveError("Invalid Git repo at {}: path exists but is not a "
42                           "directory".format(path))
43    if not os.path.isdir(path) or not os.listdir(path):
44        if no_create:
45            raise ArchiveError("No git repo at {}, refusing to create "
46                               "one".format(path))
47        if not os.path.isdir(path):
48            try:
49                os.mkdir(path)
50            except (FileNotFoundError, PermissionError) as err:
51                raise ArchiveError("Failed to mkdir {}: {}".format(path, err))
52        if not os.listdir(path):
53            log.info("Initializing a new Git repo at %s", path)
54            repo = GitRepo.init(path, bare)
55    try:
56        repo = GitRepo(path, is_topdir=True)
57    except GitError:
58        raise ArchiveError("Non-empty directory that is not a Git repository "
59                           "at {}\nPlease specify an existing Git repository, "
60                           "an empty directory or a non-existing directory "
61                           "path.".format(path))
62    return repo
63
64
65def git_commit_data(repo, data_dir, branch, message, exclude, notes, log):
66    """Commit data into a Git repository"""
67    log.info("Committing data into to branch %s", branch)
68    tmp_index = os.path.join(repo.git_dir, 'index.oe-git-archive')
69    try:
70        # Create new tree object from the data
71        env_update = {'GIT_INDEX_FILE': tmp_index,
72                      'GIT_WORK_TREE': os.path.abspath(data_dir)}
73        repo.run_cmd('add .', env_update)
74
75        # Remove files that are excluded
76        if exclude:
77            repo.run_cmd(['rm', '--cached'] + [f for f in exclude], env_update)
78
79        tree = repo.run_cmd('write-tree', env_update)
80
81        # Create new commit object from the tree
82        parent = repo.rev_parse(branch)
83        if not parent:
84            parent = repo.rev_parse("origin/" + branch)
85        git_cmd = ['commit-tree', tree, '-m', message]
86        if parent:
87            git_cmd += ['-p', parent]
88        commit = repo.run_cmd(git_cmd, env_update)
89
90        # Create git notes
91        for ref, filename in notes:
92            ref = ref.format(branch_name=branch)
93            repo.run_cmd(['notes', '--ref', ref, 'add',
94                          '-F', os.path.abspath(filename), commit])
95
96        # Update branch head
97        git_cmd = ['update-ref', 'refs/heads/' + branch, commit]
98        repo.run_cmd(git_cmd)
99
100        # Update current HEAD, if we're on branch 'branch'
101        if not repo.bare and repo.get_current_branch() == branch:
102            log.info("Updating %s HEAD to latest commit", repo.top_dir)
103            repo.run_cmd('reset --hard')
104
105        return commit
106    finally:
107        if os.path.exists(tmp_index):
108            os.unlink(tmp_index)
109
110
111def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern,
112                       keywords):
113    """Generate tag name and message, with support for running id number"""
114    keyws = keywords.copy()
115    # Tag number is handled specially: if not defined, we autoincrement it
116    if 'tag_number' not in keyws:
117        # Fill in all other fields than 'tag_number'
118        keyws['tag_number'] = '{tag_number}'
119        tag_re = format_str(name_pattern, keyws)
120        # Replace parentheses for proper regex matching
121        tag_re = tag_re.replace('(', '\(').replace(')', '\)') + '$'
122        # Inject regex group pattern for 'tag_number'
123        tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})')
124
125        keyws['tag_number'] = 0
126        for existing_tag in repo.run_cmd('tag').splitlines():
127            match = re.match(tag_re, existing_tag)
128
129            if match and int(match.group('tag_number')) >= keyws['tag_number']:
130                keyws['tag_number'] = int(match.group('tag_number')) + 1
131
132    tag_name = format_str(name_pattern, keyws)
133    msg_subj= format_str(msg_subj_pattern.strip(), keyws)
134    msg_body = format_str(msg_body_pattern, keyws)
135    return tag_name, msg_subj + '\n\n' + msg_body
136
137def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_msg_body, branch_name, no_tag, tagname, tag_msg_subject, tag_msg_body, exclude, notes, push, keywords, log):
138
139    if not os.path.isdir(data_dir):
140        raise ArchiveError("Not a directory: {}".format(data_dir))
141
142    data_repo = init_git_repo(git_dir, no_create, bare, log)
143
144    # Expand strings early in order to avoid getting into inconsistent
145    # state (e.g. no tag even if data was committed)
146    commit_msg = format_str(commit_msg_subject.strip(), keywords)
147    commit_msg += '\n\n' + format_str(commit_msg_body, keywords)
148    branch_name = format_str(branch_name, keywords)
149    tag_name = None
150    if not no_tag and tagname:
151        tag_name, tag_msg = expand_tag_strings(data_repo, tagname,
152                                               tag_msg_subject,
153                                               tag_msg_body, keywords)
154
155    # Commit data
156    commit = git_commit_data(data_repo, data_dir, branch_name,
157                             commit_msg, exclude, notes, log)
158
159    # Create tag
160    if tag_name:
161        log.info("Creating tag %s", tag_name)
162        data_repo.run_cmd(['tag', '-a', '-m', tag_msg, tag_name, commit])
163
164    # Push data to remote
165    if push:
166        cmd = ['push', '--tags']
167        # If no remote is given we push with the default settings from
168        # gitconfig
169        if push is not True:
170            notes_refs = ['refs/notes/' + ref.format(branch_name=branch_name)
171                           for ref, _ in notes]
172            cmd.extend([push, branch_name] + notes_refs)
173        log.info("Pushing data to remote")
174        data_repo.run_cmd(cmd)
175
176# Container class for tester revisions
177TestedRev = namedtuple('TestedRev', 'commit commit_number tags')
178
179def get_test_runs(log, repo, tag_name, **kwargs):
180    """Get a sorted list of test runs, matching given pattern"""
181    # First, get field names from the tag name pattern
182    field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)]
183    undef_fields = [f for f in field_names if f not in kwargs.keys()]
184
185    # Fields for formatting tag name pattern
186    str_fields = dict([(f, '*') for f in field_names])
187    str_fields.update(kwargs)
188
189    # Get a list of all matching tags
190    tag_pattern = tag_name.format(**str_fields)
191    tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines()
192    log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern)
193
194    # Parse undefined fields from tag names
195    str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names])
196    str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)'
197    str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})'
198    str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})'
199    str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})'
200    # escape parenthesis in fields in order to not messa up the regexp
201    fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()])
202    str_fields.update(fixed_fields)
203    tag_re = re.compile(tag_name.format(**str_fields))
204
205    # Parse fields from tags
206    revs = []
207    for tag in tags:
208        m = tag_re.match(tag)
209        groups = m.groupdict()
210        revs.append([groups[f] for f in undef_fields] + [tag])
211
212    # Return field names and a sorted list of revs
213    return undef_fields, sorted(revs)
214
215def get_test_revs(log, repo, tag_name, **kwargs):
216    """Get list of all tested revisions"""
217    fields, runs = get_test_runs(log, repo, tag_name, **kwargs)
218
219    revs = {}
220    commit_i = fields.index('commit')
221    commit_num_i = fields.index('commit_number')
222    for run in runs:
223        commit = run[commit_i]
224        commit_num = run[commit_num_i]
225        tag = run[-1]
226        if not commit in revs:
227            revs[commit] = TestedRev(commit, commit_num, [tag])
228        else:
229            assert commit_num == revs[commit].commit_number, "Commit numbers do not match"
230            revs[commit].tags.append(tag)
231
232    # Return in sorted table
233    revs = sorted(revs.values(), key=attrgetter('commit_number'))
234    log.debug("Found %d tested revisions:\n    %s", len(revs),
235              "\n    ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs]))
236    return revs
237
238def rev_find(revs, attr, val):
239    """Search from a list of TestedRev"""
240    for i, rev in enumerate(revs):
241        if getattr(rev, attr) == val:
242            return i
243    raise ValueError("Unable to find '{}' value '{}'".format(attr, val))
244
245