1# 2# SPDX-License-Identifier: GPL-2.0-only 3# 4 5import logging 6import os.path 7import errno 8import prserv 9import time 10 11try: 12 import sqlite3 13except ImportError: 14 from pysqlite2 import dbapi2 as sqlite3 15 16logger = logging.getLogger("BitBake.PRserv") 17 18sqlversion = sqlite3.sqlite_version_info 19if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): 20 raise Exception("sqlite3 version 3.3.0 or later is required.") 21 22# 23# "No History" mode - for a given query tuple (version, pkgarch, checksum), 24# the returned value will be the largest among all the values of the same 25# (version, pkgarch). This means the PR value returned can NOT be decremented. 26# 27# "History" mode - Return a new higher value for previously unseen query 28# tuple (version, pkgarch, checksum), otherwise return historical value. 29# Value can decrement if returning to a previous build. 30# 31 32class PRTable(object): 33 def __init__(self, conn, table, nohist, read_only): 34 self.conn = conn 35 self.nohist = nohist 36 self.read_only = read_only 37 self.dirty = False 38 if nohist: 39 self.table = "%s_nohist" % table 40 else: 41 self.table = "%s_hist" % table 42 43 if self.read_only: 44 table_exists = self._execute( 45 "SELECT count(*) FROM sqlite_master \ 46 WHERE type='table' AND name='%s'" % (self.table)) 47 if not table_exists: 48 raise prserv.NotFoundError 49 else: 50 self._execute("CREATE TABLE IF NOT EXISTS %s \ 51 (version TEXT NOT NULL, \ 52 pkgarch TEXT NOT NULL, \ 53 checksum TEXT NOT NULL, \ 54 value INTEGER, \ 55 PRIMARY KEY (version, pkgarch, checksum));" % self.table) 56 57 def _execute(self, *query): 58 """Execute a query, waiting to acquire a lock if necessary""" 59 start = time.time() 60 end = start + 20 61 while True: 62 try: 63 return self.conn.execute(*query) 64 except sqlite3.OperationalError as exc: 65 if 'is locked' in str(exc) and end > time.time(): 66 continue 67 raise exc 68 69 def sync(self): 70 if not self.read_only: 71 self.conn.commit() 72 self._execute("BEGIN EXCLUSIVE TRANSACTION") 73 74 def sync_if_dirty(self): 75 if self.dirty: 76 self.sync() 77 self.dirty = False 78 79 def _getValueHist(self, version, pkgarch, checksum): 80 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 81 (version, pkgarch, checksum)) 82 row=data.fetchone() 83 if row is not None: 84 return row[0] 85 else: 86 #no value found, try to insert 87 if self.read_only: 88 data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table), 89 (version, pkgarch)) 90 row = data.fetchone() 91 if row is not None: 92 return row[0] 93 else: 94 return 0 95 96 try: 97 self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));" 98 % (self.table,self.table), 99 (version,pkgarch, checksum,version, pkgarch)) 100 except sqlite3.IntegrityError as exc: 101 logger.error(str(exc)) 102 103 self.dirty = True 104 105 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 106 (version, pkgarch, checksum)) 107 row=data.fetchone() 108 if row is not None: 109 return row[0] 110 else: 111 raise prserv.NotFoundError 112 113 def _getValueNohist(self, version, pkgarch, checksum): 114 data=self._execute("SELECT value FROM %s \ 115 WHERE version=? AND pkgarch=? AND checksum=? AND \ 116 value >= (select max(value) from %s where version=? AND pkgarch=?);" 117 % (self.table, self.table), 118 (version, pkgarch, checksum, version, pkgarch)) 119 row=data.fetchone() 120 if row is not None: 121 return row[0] 122 else: 123 #no value found, try to insert 124 if self.read_only: 125 data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table), 126 (version, pkgarch)) 127 row = data.fetchone() 128 if row is not None: 129 return row[0] 130 else: 131 return 0 132 133 try: 134 self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));" 135 % (self.table,self.table), 136 (version, pkgarch, checksum, version, pkgarch)) 137 except sqlite3.IntegrityError as exc: 138 logger.error(str(exc)) 139 self.conn.rollback() 140 141 self.dirty = True 142 143 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 144 (version, pkgarch, checksum)) 145 row=data.fetchone() 146 if row is not None: 147 return row[0] 148 else: 149 raise prserv.NotFoundError 150 151 def getValue(self, version, pkgarch, checksum): 152 if self.nohist: 153 return self._getValueNohist(version, pkgarch, checksum) 154 else: 155 return self._getValueHist(version, pkgarch, checksum) 156 157 def _importHist(self, version, pkgarch, checksum, value): 158 if self.read_only: 159 return None 160 161 val = None 162 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 163 (version, pkgarch, checksum)) 164 row = data.fetchone() 165 if row is not None: 166 val=row[0] 167 else: 168 #no value found, try to insert 169 try: 170 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), 171 (version, pkgarch, checksum, value)) 172 except sqlite3.IntegrityError as exc: 173 logger.error(str(exc)) 174 175 self.dirty = True 176 177 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 178 (version, pkgarch, checksum)) 179 row = data.fetchone() 180 if row is not None: 181 val = row[0] 182 return val 183 184 def _importNohist(self, version, pkgarch, checksum, value): 185 if self.read_only: 186 return None 187 188 try: 189 #try to insert 190 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), 191 (version, pkgarch, checksum,value)) 192 except sqlite3.IntegrityError as exc: 193 #already have the record, try to update 194 try: 195 self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?" 196 % (self.table), 197 (value,version,pkgarch,checksum,value)) 198 except sqlite3.IntegrityError as exc: 199 logger.error(str(exc)) 200 201 self.dirty = True 202 203 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table, 204 (version,pkgarch,checksum,value)) 205 row=data.fetchone() 206 if row is not None: 207 return row[0] 208 else: 209 return None 210 211 def importone(self, version, pkgarch, checksum, value): 212 if self.nohist: 213 return self._importNohist(version, pkgarch, checksum, value) 214 else: 215 return self._importHist(version, pkgarch, checksum, value) 216 217 def export(self, version, pkgarch, checksum, colinfo): 218 metainfo = {} 219 #column info 220 if colinfo: 221 metainfo['tbl_name'] = self.table 222 metainfo['core_ver'] = prserv.__version__ 223 metainfo['col_info'] = [] 224 data = self._execute("PRAGMA table_info(%s);" % self.table) 225 for row in data: 226 col = {} 227 col['name'] = row['name'] 228 col['type'] = row['type'] 229 col['notnull'] = row['notnull'] 230 col['dflt_value'] = row['dflt_value'] 231 col['pk'] = row['pk'] 232 metainfo['col_info'].append(col) 233 234 #data info 235 datainfo = [] 236 237 if self.nohist: 238 sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \ 239 (SELECT version,pkgarch,max(value) as maxvalue FROM %s GROUP BY version,pkgarch) as T2 \ 240 WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table) 241 else: 242 sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table 243 sqlarg = [] 244 where = "" 245 if version: 246 where += "AND T1.version=? " 247 sqlarg.append(str(version)) 248 if pkgarch: 249 where += "AND T1.pkgarch=? " 250 sqlarg.append(str(pkgarch)) 251 if checksum: 252 where += "AND T1.checksum=? " 253 sqlarg.append(str(checksum)) 254 255 sqlstmt += where + ";" 256 257 if len(sqlarg): 258 data = self._execute(sqlstmt, tuple(sqlarg)) 259 else: 260 data = self._execute(sqlstmt) 261 for row in data: 262 if row['version']: 263 col = {} 264 col['version'] = row['version'] 265 col['pkgarch'] = row['pkgarch'] 266 col['checksum'] = row['checksum'] 267 col['value'] = row['value'] 268 datainfo.append(col) 269 return (metainfo, datainfo) 270 271 def dump_db(self, fd): 272 writeCount = 0 273 for line in self.conn.iterdump(): 274 writeCount = writeCount + len(line) + 1 275 fd.write(line) 276 fd.write('\n') 277 return writeCount 278 279class PRData(object): 280 """Object representing the PR database""" 281 def __init__(self, filename, nohist=True, read_only=False): 282 self.filename=os.path.abspath(filename) 283 self.nohist=nohist 284 self.read_only = read_only 285 #build directory hierarchy 286 try: 287 os.makedirs(os.path.dirname(self.filename)) 288 except OSError as e: 289 if e.errno != errno.EEXIST: 290 raise e 291 uri = "file:%s%s" % (self.filename, "?mode=ro" if self.read_only else "") 292 logger.debug("Opening PRServ database '%s'" % (uri)) 293 self.connection=sqlite3.connect(uri, uri=True, isolation_level="EXCLUSIVE", check_same_thread = False) 294 self.connection.row_factory=sqlite3.Row 295 if not self.read_only: 296 self.connection.execute("pragma synchronous = off;") 297 self.connection.execute("PRAGMA journal_mode = MEMORY;") 298 self._tables={} 299 300 def disconnect(self): 301 self.connection.close() 302 303 def __getitem__(self,tblname): 304 if not isinstance(tblname, str): 305 raise TypeError("tblname argument must be a string, not '%s'" % 306 type(tblname)) 307 if tblname in self._tables: 308 return self._tables[tblname] 309 else: 310 tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist, self.read_only) 311 return tableobj 312 313 def __delitem__(self, tblname): 314 if tblname in self._tables: 315 del self._tables[tblname] 316 logger.info("drop table %s" % (tblname)) 317 self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) 318