1"""
2BitBake 'Fetch' implementation for perforce
3
4Supported SRC_URI options are:
5
6- module
7   The top-level location to fetch while preserving the remote paths
8
9   The value of module can point to either a directory or a file. The result,
10   in both cases, is that the fetcher will preserve all file paths starting
11   from the module path. That is, the top-level directory in the module value
12   will also be the top-level directory in P4DIR.
13
14- remotepath
15   If the value "keep" is given, the full depot location of each file is
16   preserved in P4DIR. This option overrides the effect of the module option.
17
18"""
19
20# Copyright (C) 2003, 2004  Chris Larson
21# Copyright (C) 2016 Kodak Alaris, Inc.
22#
23# SPDX-License-Identifier: GPL-2.0-only
24#
25# Based on functions from the base bb module, Copyright 2003 Holger Schurig
26
27import os
28import bb
29from   bb.fetch2 import FetchMethod
30from   bb.fetch2 import FetchError
31from   bb.fetch2 import logger
32from   bb.fetch2 import runfetchcmd
33
34class PerforceProgressHandler (bb.progress.BasicProgressHandler):
35    """
36    Implements basic progress information for perforce, based on the number of
37    files to be downloaded.
38
39    The p4 print command will print one line per file, therefore it can be used
40    to "count" the number of files already completed and give an indication of
41    the progress.
42    """
43    def __init__(self, d, num_files):
44        self._num_files = num_files
45        self._count = 0
46        super(PerforceProgressHandler, self).__init__(d)
47
48        # Send an initial progress event so the bar gets shown
49        self._fire_progress(-1)
50
51    def write(self, string):
52        self._count = self._count + 1
53
54        percent = int(100.0 * float(self._count) / float(self._num_files))
55
56        # In case something goes wrong, we try to preserve our sanity
57        if percent > 100:
58            percent = 100
59
60        self.update(percent)
61
62        super(PerforceProgressHandler, self).write(string)
63
64class Perforce(FetchMethod):
65    """ Class to fetch from perforce repositories """
66    def supports(self, ud, d):
67        """ Check to see if a given url can be fetched with perforce. """
68        return ud.type in ['p4']
69
70    def urldata_init(self, ud, d):
71        """
72        Initialize perforce specific variables within url data.  If P4CONFIG is
73        provided by the env, use it.  If P4PORT is specified by the recipe, use
74        its values, which may override the settings in P4CONFIG.
75        """
76        ud.basecmd = d.getVar("FETCHCMD_p4") or "/usr/bin/env p4"
77
78        ud.dldir = d.getVar("P4DIR") or (d.getVar("DL_DIR") + "/p4")
79
80        path = ud.url.split('://')[1]
81        path = path.split(';')[0]
82        delim = path.find('@');
83        if delim != -1:
84            (ud.user, ud.pswd) = path.split('@')[0].split(':')
85            ud.path = path.split('@')[1]
86        else:
87            ud.path = path
88
89        ud.usingp4config = False
90        p4port = d.getVar('P4PORT')
91
92        if p4port:
93            logger.debug('Using recipe provided P4PORT: %s' % p4port)
94            ud.host = p4port
95        else:
96            logger.debug('Trying to use P4CONFIG to automatically set P4PORT...')
97            ud.usingp4config = True
98            p4cmd = '%s info | grep "Server address"' % ud.basecmd
99            bb.fetch2.check_network_access(d, p4cmd, ud.url)
100            ud.host = runfetchcmd(p4cmd, d, True)
101            ud.host = ud.host.split(': ')[1].strip()
102            logger.debug('Determined P4PORT to be: %s' % ud.host)
103            if not ud.host:
104                raise FetchError('Could not determine P4PORT from P4CONFIG')
105
106        # Fetcher options
107        ud.module = ud.parm.get('module')
108        ud.keepremotepath = (ud.parm.get('remotepath', '') == 'keep')
109
110        if ud.path.find('/...') >= 0:
111            ud.pathisdir = True
112        else:
113            ud.pathisdir = False
114
115        # Avoid using the "/..." syntax in SRC_URI when a module value is given
116        if ud.pathisdir and ud.module:
117            raise FetchError('SRC_URI depot path cannot not end in /... when a module value is given')
118
119        cleanedpath = ud.path.replace('/...', '').replace('/', '.')
120        cleanedhost = ud.host.replace(':', '.')
121
122        cleanedmodule = ""
123        # Merge the path and module into the final depot location
124        if ud.module:
125            if ud.module.find('/') == 0:
126                raise FetchError('module cannot begin with /')
127            ud.path = os.path.join(ud.path, ud.module)
128
129            # Append the module path to the local pkg name
130            cleanedmodule = ud.module.replace('/...', '').replace('/', '.')
131            cleanedpath += '--%s' % cleanedmodule
132
133        ud.pkgdir = os.path.join(ud.dldir, cleanedhost, cleanedpath)
134
135        ud.setup_revisions(d)
136
137        ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleanedmodule, ud.revision))
138
139    def _buildp4command(self, ud, d, command, depot_filename=None):
140        """
141        Build a p4 commandline.  Valid commands are "changes", "print", and
142        "files".  depot_filename is the full path to the file in the depot
143        including the trailing '#rev' value.
144        """
145        p4opt = ""
146
147        if ud.user:
148            p4opt += ' -u "%s"' % (ud.user)
149
150        if ud.pswd:
151            p4opt += ' -P "%s"' % (ud.pswd)
152
153        if ud.host and not ud.usingp4config:
154            p4opt += ' -p %s' % (ud.host)
155
156        if hasattr(ud, 'revision') and ud.revision:
157            pathnrev = '%s@%s' % (ud.path, ud.revision)
158        else:
159            pathnrev = '%s' % (ud.path)
160
161        if depot_filename:
162            if ud.keepremotepath:
163                # preserve everything, remove the leading //
164                filename = depot_filename.lstrip('/')
165            elif ud.module:
166                # remove everything up to the module path
167                modulepath = ud.module.rstrip('/...')
168                filename = depot_filename[depot_filename.rfind(modulepath):]
169            elif ud.pathisdir:
170                # Remove leading (visible) path to obtain the filepath
171                filename = depot_filename[len(ud.path)-1:]
172            else:
173                # Remove everything, except the filename
174                filename = depot_filename[depot_filename.rfind('/'):]
175
176            filename = filename[:filename.find('#')] # Remove trailing '#rev'
177
178        if command == 'changes':
179            p4cmd = '%s%s changes -m 1 //%s' % (ud.basecmd, p4opt, pathnrev)
180        elif command == 'print':
181            if depot_filename is not None:
182                p4cmd = '%s%s print -o "p4/%s" "%s"' % (ud.basecmd, p4opt, filename, depot_filename)
183            else:
184                raise FetchError('No depot file name provided to p4 %s' % command, ud.url)
185        elif command == 'files':
186            p4cmd = '%s%s files //%s' % (ud.basecmd, p4opt, pathnrev)
187        else:
188            raise FetchError('Invalid p4 command %s' % command, ud.url)
189
190        return p4cmd
191
192    def _p4listfiles(self, ud, d):
193        """
194        Return a list of the file names which are present in the depot using the
195        'p4 files' command, including trailing '#rev' file revision indicator
196        """
197        p4cmd = self._buildp4command(ud, d, 'files')
198        bb.fetch2.check_network_access(d, p4cmd, ud.url)
199        p4fileslist = runfetchcmd(p4cmd, d, True)
200        p4fileslist = [f.rstrip() for f in p4fileslist.splitlines()]
201
202        if not p4fileslist:
203            raise FetchError('Unable to fetch listing of p4 files from %s@%s' % (ud.host, ud.path))
204
205        count = 0
206        filelist = []
207
208        for filename in p4fileslist:
209            item = filename.split(' - ')
210            lastaction = item[1].split()
211            logger.debug('File: %s Last Action: %s' % (item[0], lastaction[0]))
212            if lastaction[0] == 'delete':
213                continue
214            filelist.append(item[0])
215
216        return filelist
217
218    def download(self, ud, d):
219        """ Get the list of files, fetch each one """
220        filelist = self._p4listfiles(ud, d)
221        if not filelist:
222            raise FetchError('No files found in depot %s@%s' % (ud.host, ud.path))
223
224        bb.utils.remove(ud.pkgdir, True)
225        bb.utils.mkdirhier(ud.pkgdir)
226
227        progresshandler = PerforceProgressHandler(d, len(filelist))
228
229        for afile in filelist:
230            p4fetchcmd = self._buildp4command(ud, d, 'print', afile)
231            bb.fetch2.check_network_access(d, p4fetchcmd, ud.url)
232            runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir, log=progresshandler)
233
234        runfetchcmd('tar -czf %s p4' % (ud.localpath), d, cleanup=[ud.localpath], workdir=ud.pkgdir)
235
236    def clean(self, ud, d):
237        """ Cleanup p4 specific files and dirs"""
238        bb.utils.remove(ud.localpath)
239        bb.utils.remove(ud.pkgdir, True)
240
241    def supports_srcrev(self):
242        return True
243
244    def _revision_key(self, ud, d, name):
245        """ Return a unique key for the url """
246        return 'p4:%s' % ud.pkgdir
247
248    def _latest_revision(self, ud, d, name):
249        """ Return the latest upstream scm revision number """
250        p4cmd = self._buildp4command(ud, d, "changes")
251        bb.fetch2.check_network_access(d, p4cmd, ud.url)
252        tip = runfetchcmd(p4cmd, d, True)
253
254        if not tip:
255            raise FetchError('Could not determine the latest perforce changelist')
256
257        tipcset = tip.split(' ')[1]
258        logger.debug('p4 tip found to be changelist %s' % tipcset)
259        return tipcset
260
261    def sortable_revision(self, ud, d, name):
262        """ Return a sortable revision number """
263        return False, self._build_revision(ud, d)
264
265    def _build_revision(self, ud, d):
266        return ud.revision
267
268