1# 2# Helper functions for committing data to git and pushing upstream 3# 4# Copyright (c) 2017, Intel Corporation. 5# Copyright (c) 2019, Linux Foundation 6# 7# This program is free software; you can redistribute it and/or modify it 8# under the terms and conditions of the GNU General Public License, 9# version 2, as published by the Free Software Foundation. 10# 11# This program is distributed in the hope it will be useful, but WITHOUT 12# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14# more details. 15# 16 17import os 18import re 19import sys 20from operator import attrgetter 21from collections import namedtuple 22from oeqa.utils.git import GitRepo, GitError 23 24class ArchiveError(Exception): 25 """Internal error handling of this script""" 26 27def format_str(string, fields): 28 """Format string using the given fields (dict)""" 29 try: 30 return string.format(**fields) 31 except KeyError as err: 32 raise ArchiveError("Unable to expand string '{}': unknown field {} " 33 "(valid fields are: {})".format( 34 string, err, ', '.join(sorted(fields.keys())))) 35 36 37def init_git_repo(path, no_create, bare, log): 38 """Initialize local Git repository""" 39 path = os.path.abspath(path) 40 if os.path.isfile(path): 41 raise ArchiveError("Invalid Git repo at {}: path exists but is not a " 42 "directory".format(path)) 43 if not os.path.isdir(path) or not os.listdir(path): 44 if no_create: 45 raise ArchiveError("No git repo at {}, refusing to create " 46 "one".format(path)) 47 if not os.path.isdir(path): 48 try: 49 os.mkdir(path) 50 except (FileNotFoundError, PermissionError) as err: 51 raise ArchiveError("Failed to mkdir {}: {}".format(path, err)) 52 if not os.listdir(path): 53 log.info("Initializing a new Git repo at %s", path) 54 repo = GitRepo.init(path, bare) 55 try: 56 repo = GitRepo(path, is_topdir=True) 57 except GitError: 58 raise ArchiveError("Non-empty directory that is not a Git repository " 59 "at {}\nPlease specify an existing Git repository, " 60 "an empty directory or a non-existing directory " 61 "path.".format(path)) 62 return repo 63 64 65def git_commit_data(repo, data_dir, branch, message, exclude, notes, log): 66 """Commit data into a Git repository""" 67 log.info("Committing data into to branch %s", branch) 68 tmp_index = os.path.join(repo.git_dir, 'index.oe-git-archive') 69 try: 70 # Create new tree object from the data 71 env_update = {'GIT_INDEX_FILE': tmp_index, 72 'GIT_WORK_TREE': os.path.abspath(data_dir)} 73 repo.run_cmd('add .', env_update) 74 75 # Remove files that are excluded 76 if exclude: 77 repo.run_cmd(['rm', '--cached'] + [f for f in exclude], env_update) 78 79 tree = repo.run_cmd('write-tree', env_update) 80 81 # Create new commit object from the tree 82 parent = repo.rev_parse(branch) 83 if not parent: 84 parent = repo.rev_parse("origin/" + branch) 85 git_cmd = ['commit-tree', tree, '-m', message] 86 if parent: 87 git_cmd += ['-p', parent] 88 commit = repo.run_cmd(git_cmd, env_update) 89 90 # Create git notes 91 for ref, filename in notes: 92 ref = ref.format(branch_name=branch) 93 repo.run_cmd(['notes', '--ref', ref, 'add', 94 '-F', os.path.abspath(filename), commit]) 95 96 # Update branch head 97 git_cmd = ['update-ref', 'refs/heads/' + branch, commit] 98 repo.run_cmd(git_cmd) 99 100 # Update current HEAD, if we're on branch 'branch' 101 if not repo.bare and repo.get_current_branch() == branch: 102 log.info("Updating %s HEAD to latest commit", repo.top_dir) 103 repo.run_cmd('reset --hard') 104 105 return commit 106 finally: 107 if os.path.exists(tmp_index): 108 os.unlink(tmp_index) 109 110 111def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern, 112 keywords): 113 """Generate tag name and message, with support for running id number""" 114 keyws = keywords.copy() 115 # Tag number is handled specially: if not defined, we autoincrement it 116 if 'tag_number' not in keyws: 117 # Fill in all other fields than 'tag_number' 118 keyws['tag_number'] = '{tag_number}' 119 tag_re = format_str(name_pattern, keyws) 120 # Replace parentheses for proper regex matching 121 tag_re = tag_re.replace('(', '\(').replace(')', '\)') + '$' 122 # Inject regex group pattern for 'tag_number' 123 tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})') 124 125 keyws['tag_number'] = 0 126 for existing_tag in repo.run_cmd('tag').splitlines(): 127 match = re.match(tag_re, existing_tag) 128 129 if match and int(match.group('tag_number')) >= keyws['tag_number']: 130 keyws['tag_number'] = int(match.group('tag_number')) + 1 131 132 tag_name = format_str(name_pattern, keyws) 133 msg_subj= format_str(msg_subj_pattern.strip(), keyws) 134 msg_body = format_str(msg_body_pattern, keyws) 135 return tag_name, msg_subj + '\n\n' + msg_body 136 137def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_msg_body, branch_name, no_tag, tagname, tag_msg_subject, tag_msg_body, exclude, notes, push, keywords, log): 138 139 if not os.path.isdir(data_dir): 140 raise ArchiveError("Not a directory: {}".format(data_dir)) 141 142 data_repo = init_git_repo(git_dir, no_create, bare, log) 143 144 # Expand strings early in order to avoid getting into inconsistent 145 # state (e.g. no tag even if data was committed) 146 commit_msg = format_str(commit_msg_subject.strip(), keywords) 147 commit_msg += '\n\n' + format_str(commit_msg_body, keywords) 148 branch_name = format_str(branch_name, keywords) 149 tag_name = None 150 if not no_tag and tagname: 151 tag_name, tag_msg = expand_tag_strings(data_repo, tagname, 152 tag_msg_subject, 153 tag_msg_body, keywords) 154 155 # Commit data 156 commit = git_commit_data(data_repo, data_dir, branch_name, 157 commit_msg, exclude, notes, log) 158 159 # Create tag 160 if tag_name: 161 log.info("Creating tag %s", tag_name) 162 data_repo.run_cmd(['tag', '-a', '-m', tag_msg, tag_name, commit]) 163 164 # Push data to remote 165 if push: 166 cmd = ['push', '--tags'] 167 # If no remote is given we push with the default settings from 168 # gitconfig 169 if push is not True: 170 notes_refs = ['refs/notes/' + ref.format(branch_name=branch_name) 171 for ref, _ in notes] 172 cmd.extend([push, branch_name] + notes_refs) 173 log.info("Pushing data to remote") 174 data_repo.run_cmd(cmd) 175 176# Container class for tester revisions 177TestedRev = namedtuple('TestedRev', 'commit commit_number tags') 178 179def get_test_runs(log, repo, tag_name, **kwargs): 180 """Get a sorted list of test runs, matching given pattern""" 181 # First, get field names from the tag name pattern 182 field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)] 183 undef_fields = [f for f in field_names if f not in kwargs.keys()] 184 185 # Fields for formatting tag name pattern 186 str_fields = dict([(f, '*') for f in field_names]) 187 str_fields.update(kwargs) 188 189 # Get a list of all matching tags 190 tag_pattern = tag_name.format(**str_fields) 191 tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines() 192 log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern) 193 194 # Parse undefined fields from tag names 195 str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names]) 196 str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)' 197 str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})' 198 str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})' 199 str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})' 200 # escape parenthesis in fields in order to not messa up the regexp 201 fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()]) 202 str_fields.update(fixed_fields) 203 tag_re = re.compile(tag_name.format(**str_fields)) 204 205 # Parse fields from tags 206 revs = [] 207 for tag in tags: 208 m = tag_re.match(tag) 209 groups = m.groupdict() 210 revs.append([groups[f] for f in undef_fields] + [tag]) 211 212 # Return field names and a sorted list of revs 213 return undef_fields, sorted(revs) 214 215def get_test_revs(log, repo, tag_name, **kwargs): 216 """Get list of all tested revisions""" 217 fields, runs = get_test_runs(log, repo, tag_name, **kwargs) 218 219 revs = {} 220 commit_i = fields.index('commit') 221 commit_num_i = fields.index('commit_number') 222 for run in runs: 223 commit = run[commit_i] 224 commit_num = run[commit_num_i] 225 tag = run[-1] 226 if not commit in revs: 227 revs[commit] = TestedRev(commit, commit_num, [tag]) 228 else: 229 assert commit_num == revs[commit].commit_number, "Commit numbers do not match" 230 revs[commit].tags.append(tag) 231 232 # Return in sorted table 233 revs = sorted(revs.values(), key=attrgetter('commit_number')) 234 log.debug("Found %d tested revisions:\n %s", len(revs), 235 "\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs])) 236 return revs 237 238def rev_find(revs, attr, val): 239 """Search from a list of TestedRev""" 240 for i, rev in enumerate(revs): 241 if getattr(rev, attr) == val: 242 return i 243 raise ValueError("Unable to find '{}' value '{}'".format(attr, val)) 244 245