1# 2# Helper functions for committing data to git and pushing upstream 3# 4# Copyright (c) 2017, Intel Corporation. 5# Copyright (c) 2019, Linux Foundation 6# 7# SPDX-License-Identifier: GPL-2.0-only 8# 9 10import os 11import re 12import sys 13from operator import attrgetter 14from collections import namedtuple 15from oeqa.utils.git import GitRepo, GitError 16 17class ArchiveError(Exception): 18 """Internal error handling of this script""" 19 20def format_str(string, fields): 21 """Format string using the given fields (dict)""" 22 try: 23 return string.format(**fields) 24 except KeyError as err: 25 raise ArchiveError("Unable to expand string '{}': unknown field {} " 26 "(valid fields are: {})".format( 27 string, err, ', '.join(sorted(fields.keys())))) 28 29 30def init_git_repo(path, no_create, bare, log): 31 """Initialize local Git repository""" 32 path = os.path.abspath(path) 33 if os.path.isfile(path): 34 raise ArchiveError("Invalid Git repo at {}: path exists but is not a " 35 "directory".format(path)) 36 if not os.path.isdir(path) or not os.listdir(path): 37 if no_create: 38 raise ArchiveError("No git repo at {}, refusing to create " 39 "one".format(path)) 40 if not os.path.isdir(path): 41 try: 42 os.mkdir(path) 43 except (FileNotFoundError, PermissionError) as err: 44 raise ArchiveError("Failed to mkdir {}: {}".format(path, err)) 45 if not os.listdir(path): 46 log.info("Initializing a new Git repo at %s", path) 47 repo = GitRepo.init(path, bare) 48 try: 49 repo = GitRepo(path, is_topdir=True) 50 except GitError: 51 raise ArchiveError("Non-empty directory that is not a Git repository " 52 "at {}\nPlease specify an existing Git repository, " 53 "an empty directory or a non-existing directory " 54 "path.".format(path)) 55 return repo 56 57 58def git_commit_data(repo, data_dir, branch, message, exclude, notes, log): 59 """Commit data into a Git repository""" 60 log.info("Committing data into to branch %s", branch) 61 tmp_index = os.path.join(repo.git_dir, 'index.oe-git-archive') 62 try: 63 # Create new tree object from the data 64 env_update = {'GIT_INDEX_FILE': tmp_index, 65 'GIT_WORK_TREE': os.path.abspath(data_dir)} 66 repo.run_cmd('add .', env_update) 67 68 # Remove files that are excluded 69 if exclude: 70 repo.run_cmd(['rm', '--cached', '--ignore-unmatch'] + [f for f in exclude], env_update) 71 72 tree = repo.run_cmd('write-tree', env_update) 73 74 # Create new commit object from the tree 75 parent = repo.rev_parse(branch) 76 if not parent: 77 parent = repo.rev_parse("origin/" + branch) 78 git_cmd = ['commit-tree', tree, '-m', message] 79 if parent: 80 git_cmd += ['-p', parent] 81 commit = repo.run_cmd(git_cmd, env_update) 82 83 # Create git notes 84 for ref, filename in notes: 85 ref = ref.format(branch_name=branch) 86 repo.run_cmd(['notes', '--ref', ref, 'add', 87 '-F', os.path.abspath(filename), commit]) 88 89 # Update branch head 90 git_cmd = ['update-ref', 'refs/heads/' + branch, commit] 91 repo.run_cmd(git_cmd) 92 93 # Update current HEAD, if we're on branch 'branch' 94 if not repo.bare and repo.get_current_branch() == branch: 95 log.info("Updating %s HEAD to latest commit", repo.top_dir) 96 repo.run_cmd('reset --hard') 97 98 return commit 99 finally: 100 if os.path.exists(tmp_index): 101 os.unlink(tmp_index) 102 103def get_tags(repo, log, pattern=None, url=None): 104 """ Fetch remote tags from current repository 105 106 A pattern can be provided to filter returned tags list 107 An URL can be provided if local repository has no valid remote configured 108 """ 109 110 base_cmd = ['ls-remote', '--refs', '--tags', '-q'] 111 cmd = base_cmd.copy() 112 113 # First try to fetch tags from repository configured remote 114 cmd.append('origin') 115 if pattern: 116 cmd.append("refs/tags/"+pattern) 117 try: 118 tags_refs = repo.run_cmd(cmd) 119 tags = ["".join(d.split()[1].split('/', 2)[2:]) for d in tags_refs.splitlines()] 120 except GitError as e: 121 # If it fails, retry with repository url if one is provided 122 if url: 123 log.info("No remote repository configured, use provided url") 124 cmd = base_cmd.copy() 125 cmd.append(url) 126 if pattern: 127 cmd.append(pattern) 128 tags_refs = repo.run_cmd(cmd) 129 tags = ["".join(d.split()[1].split('/', 2)[2:]) for d in tags_refs.splitlines()] 130 else: 131 log.info("Read local tags only, some remote tags may be missed") 132 cmd = ["tag"] 133 if pattern: 134 cmd += ["-l", pattern] 135 tags = repo.run_cmd(cmd).splitlines() 136 137 return tags 138 139def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern, 140 url, log, keywords): 141 """Generate tag name and message, with support for running id number""" 142 keyws = keywords.copy() 143 # Tag number is handled specially: if not defined, we autoincrement it 144 if 'tag_number' not in keyws: 145 # Fill in all other fields than 'tag_number' 146 keyws['tag_number'] = '{tag_number}' 147 tag_re = format_str(name_pattern, keyws) 148 # Replace parentheses for proper regex matching 149 tag_re = tag_re.replace('(', r'\(').replace(')', r'\)') + '$' 150 # Inject regex group pattern for 'tag_number' 151 tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})') 152 153 keyws['tag_number'] = 0 154 for existing_tag in get_tags(repo, log, url=url): 155 match = re.match(tag_re, existing_tag) 156 157 if match and int(match.group('tag_number')) >= keyws['tag_number']: 158 keyws['tag_number'] = int(match.group('tag_number')) + 1 159 160 tag_name = format_str(name_pattern, keyws) 161 msg_subj= format_str(msg_subj_pattern.strip(), keyws) 162 msg_body = format_str(msg_body_pattern, keyws) 163 return tag_name, msg_subj + '\n\n' + msg_body 164 165def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_msg_body, branch_name, no_tag, tagname, tag_msg_subject, tag_msg_body, exclude, notes, push, keywords, log): 166 167 if not os.path.isdir(data_dir): 168 raise ArchiveError("Not a directory: {}".format(data_dir)) 169 170 data_repo = init_git_repo(git_dir, no_create, bare, log) 171 172 # Expand strings early in order to avoid getting into inconsistent 173 # state (e.g. no tag even if data was committed) 174 commit_msg = format_str(commit_msg_subject.strip(), keywords) 175 commit_msg += '\n\n' + format_str(commit_msg_body, keywords) 176 branch_name = format_str(branch_name, keywords) 177 tag_name = None 178 if not no_tag and tagname: 179 tag_name, tag_msg = expand_tag_strings(data_repo, tagname, 180 tag_msg_subject, 181 tag_msg_body, 182 push, log, keywords) 183 184 # Commit data 185 commit = git_commit_data(data_repo, data_dir, branch_name, 186 commit_msg, exclude, notes, log) 187 188 # Create tag 189 if tag_name: 190 log.info("Creating tag %s", tag_name) 191 data_repo.run_cmd(['tag', '-a', '-m', tag_msg, tag_name, commit]) 192 193 # Push data to remote 194 if push: 195 cmd = ['push', '--tags'] 196 # If no remote is given we push with the default settings from 197 # gitconfig 198 if push is not True: 199 notes_refs = ['refs/notes/' + ref.format(branch_name=branch_name) 200 for ref, _ in notes] 201 cmd.extend([push, branch_name] + notes_refs) 202 log.info("Pushing data to remote") 203 data_repo.run_cmd(cmd) 204 205 return tag_name 206 207# Container class for tester revisions 208TestedRev = namedtuple('TestedRev', 'commit commit_number tags') 209 210def get_test_runs(log, repo, tag_name, **kwargs): 211 """Get a sorted list of test runs, matching given pattern""" 212 # First, get field names from the tag name pattern 213 field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)] 214 undef_fields = [f for f in field_names if f not in kwargs.keys()] 215 216 # Fields for formatting tag name pattern 217 str_fields = dict([(f, '*') for f in field_names]) 218 str_fields.update(kwargs) 219 220 # Get a list of all matching tags 221 tag_pattern = tag_name.format(**str_fields) 222 tags = get_tags(repo, log, pattern=tag_pattern) 223 log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern) 224 225 # Parse undefined fields from tag names 226 str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names]) 227 str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)' 228 str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})' 229 str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})' 230 str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})' 231 # escape parenthesis in fields in order to not messa up the regexp 232 fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()]) 233 str_fields.update(fixed_fields) 234 tag_re = re.compile(tag_name.format(**str_fields)) 235 236 # Parse fields from tags 237 revs = [] 238 for tag in tags: 239 m = tag_re.match(tag) 240 if not m: 241 continue 242 groups = m.groupdict() 243 revs.append([groups[f] for f in undef_fields] + [tag]) 244 245 # Return field names and a sorted list of revs 246 return undef_fields, sorted(revs) 247 248def get_test_revs(log, repo, tag_name, **kwargs): 249 """Get list of all tested revisions""" 250 fields, runs = get_test_runs(log, repo, tag_name, **kwargs) 251 252 revs = {} 253 commit_i = fields.index('commit') 254 commit_num_i = fields.index('commit_number') 255 for run in runs: 256 commit = run[commit_i] 257 commit_num = run[commit_num_i] 258 tag = run[-1] 259 if not commit in revs: 260 revs[commit] = TestedRev(commit, commit_num, [tag]) 261 else: 262 if commit_num != revs[commit].commit_number: 263 # Historically we have incorrect commit counts of '1' in the repo so fix these up 264 if int(revs[commit].commit_number) < 5: 265 tags = revs[commit].tags 266 revs[commit] = TestedRev(commit, commit_num, [tags]) 267 elif int(commit_num) < 5: 268 pass 269 else: 270 sys.exit("Commit numbers for commit %s don't match (%s vs %s)" % (commit, commit_num, revs[commit].commit_number)) 271 revs[commit].tags.append(tag) 272 273 # Return in sorted table 274 revs = sorted(revs.values(), key=attrgetter('commit_number')) 275 log.debug("Found %d tested revisions:\n %s", len(revs), 276 "\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs])) 277 return revs 278 279def rev_find(revs, attr, val): 280 """Search from a list of TestedRev""" 281 for i, rev in enumerate(revs): 282 if getattr(rev, attr) == val: 283 return i 284 raise ValueError("Unable to find '{}' value '{}'".format(attr, val)) 285 286