1# 2# Copyright BitBake Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6 7import logging 8import os.path 9import errno 10import prserv 11import time 12 13try: 14 import sqlite3 15except ImportError: 16 from pysqlite2 import dbapi2 as sqlite3 17 18logger = logging.getLogger("BitBake.PRserv") 19 20sqlversion = sqlite3.sqlite_version_info 21if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): 22 raise Exception("sqlite3 version 3.3.0 or later is required.") 23 24# 25# "No History" mode - for a given query tuple (version, pkgarch, checksum), 26# the returned value will be the largest among all the values of the same 27# (version, pkgarch). This means the PR value returned can NOT be decremented. 28# 29# "History" mode - Return a new higher value for previously unseen query 30# tuple (version, pkgarch, checksum), otherwise return historical value. 31# Value can decrement if returning to a previous build. 32# 33 34class PRTable(object): 35 def __init__(self, conn, table, nohist, read_only): 36 self.conn = conn 37 self.nohist = nohist 38 self.read_only = read_only 39 self.dirty = False 40 if nohist: 41 self.table = "%s_nohist" % table 42 else: 43 self.table = "%s_hist" % table 44 45 if self.read_only: 46 table_exists = self._execute( 47 "SELECT count(*) FROM sqlite_master \ 48 WHERE type='table' AND name='%s'" % (self.table)) 49 if not table_exists: 50 raise prserv.NotFoundError 51 else: 52 self._execute("CREATE TABLE IF NOT EXISTS %s \ 53 (version TEXT NOT NULL, \ 54 pkgarch TEXT NOT NULL, \ 55 checksum TEXT NOT NULL, \ 56 value INTEGER, \ 57 PRIMARY KEY (version, pkgarch, checksum));" % self.table) 58 59 def _execute(self, *query): 60 """Execute a query, waiting to acquire a lock if necessary""" 61 start = time.time() 62 end = start + 20 63 while True: 64 try: 65 return self.conn.execute(*query) 66 except sqlite3.OperationalError as exc: 67 if "is locked" in str(exc) and end > time.time(): 68 continue 69 raise exc 70 71 def sync(self): 72 if not self.read_only: 73 self.conn.commit() 74 self._execute("BEGIN EXCLUSIVE TRANSACTION") 75 76 def sync_if_dirty(self): 77 if self.dirty: 78 self.sync() 79 self.dirty = False 80 81 def test_package(self, version, pkgarch): 82 """Returns whether the specified package version is found in the database for the specified architecture""" 83 84 # Just returns the value if found or None otherwise 85 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=?;" % self.table, 86 (version, pkgarch)) 87 row=data.fetchone() 88 if row is not None: 89 return True 90 else: 91 return False 92 93 def test_value(self, version, pkgarch, value): 94 """Returns whether the specified value is found in the database for the specified package and architecture""" 95 96 # Just returns the value if found or None otherwise 97 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and value=?;" % self.table, 98 (version, pkgarch, value)) 99 row=data.fetchone() 100 if row is not None: 101 return True 102 else: 103 return False 104 105 def find_value(self, version, pkgarch, checksum): 106 """Returns the value for the specified checksum if found or None otherwise.""" 107 108 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 109 (version, pkgarch, checksum)) 110 row=data.fetchone() 111 if row is not None: 112 return row[0] 113 else: 114 return None 115 116 def find_max_value(self, version, pkgarch): 117 """Returns the greatest value for (version, pkgarch), or None if not found. Doesn't create a new value""" 118 119 data = self._execute("SELECT max(value) FROM %s where version=? AND pkgarch=?;" % (self.table), 120 (version, pkgarch)) 121 row = data.fetchone() 122 if row is not None: 123 return row[0] 124 else: 125 return None 126 127 def _get_value_hist(self, version, pkgarch, checksum): 128 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 129 (version, pkgarch, checksum)) 130 row=data.fetchone() 131 if row is not None: 132 return row[0] 133 else: 134 #no value found, try to insert 135 if self.read_only: 136 data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table), 137 (version, pkgarch)) 138 row = data.fetchone() 139 if row is not None: 140 return row[0] 141 else: 142 return 0 143 144 try: 145 self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));" 146 % (self.table, self.table), 147 (version, pkgarch, checksum, version, pkgarch)) 148 except sqlite3.IntegrityError as exc: 149 logger.error(str(exc)) 150 151 self.dirty = True 152 153 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 154 (version, pkgarch, checksum)) 155 row=data.fetchone() 156 if row is not None: 157 return row[0] 158 else: 159 raise prserv.NotFoundError 160 161 def _get_value_no_hist(self, version, pkgarch, checksum): 162 data=self._execute("SELECT value FROM %s \ 163 WHERE version=? AND pkgarch=? AND checksum=? AND \ 164 value >= (select max(value) from %s where version=? AND pkgarch=?);" 165 % (self.table, self.table), 166 (version, pkgarch, checksum, version, pkgarch)) 167 row=data.fetchone() 168 if row is not None: 169 return row[0] 170 else: 171 #no value found, try to insert 172 if self.read_only: 173 data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table), 174 (version, pkgarch)) 175 return data.fetchone()[0] 176 177 try: 178 self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));" 179 % (self.table, self.table), 180 (version, pkgarch, checksum, version, pkgarch)) 181 except sqlite3.IntegrityError as exc: 182 logger.error(str(exc)) 183 self.conn.rollback() 184 185 self.dirty = True 186 187 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 188 (version, pkgarch, checksum)) 189 row=data.fetchone() 190 if row is not None: 191 return row[0] 192 else: 193 raise prserv.NotFoundError 194 195 def get_value(self, version, pkgarch, checksum): 196 if self.nohist: 197 return self._get_value_no_hist(version, pkgarch, checksum) 198 else: 199 return self._get_value_hist(version, pkgarch, checksum) 200 201 def _import_hist(self, version, pkgarch, checksum, value): 202 if self.read_only: 203 return None 204 205 val = None 206 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 207 (version, pkgarch, checksum)) 208 row = data.fetchone() 209 if row is not None: 210 val=row[0] 211 else: 212 #no value found, try to insert 213 try: 214 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), 215 (version, pkgarch, checksum, value)) 216 except sqlite3.IntegrityError as exc: 217 logger.error(str(exc)) 218 219 self.dirty = True 220 221 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 222 (version, pkgarch, checksum)) 223 row = data.fetchone() 224 if row is not None: 225 val = row[0] 226 return val 227 228 def _import_no_hist(self, version, pkgarch, checksum, value): 229 if self.read_only: 230 return None 231 232 try: 233 #try to insert 234 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), 235 (version, pkgarch, checksum, value)) 236 except sqlite3.IntegrityError as exc: 237 #already have the record, try to update 238 try: 239 self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?" 240 % (self.table), 241 (value, version, pkgarch, checksum, value)) 242 except sqlite3.IntegrityError as exc: 243 logger.error(str(exc)) 244 245 self.dirty = True 246 247 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table, 248 (version, pkgarch, checksum, value)) 249 row=data.fetchone() 250 if row is not None: 251 return row[0] 252 else: 253 return None 254 255 def importone(self, version, pkgarch, checksum, value): 256 if self.nohist: 257 return self._import_no_hist(version, pkgarch, checksum, value) 258 else: 259 return self._import_hist(version, pkgarch, checksum, value) 260 261 def export(self, version, pkgarch, checksum, colinfo): 262 metainfo = {} 263 #column info 264 if colinfo: 265 metainfo["tbl_name"] = self.table 266 metainfo["core_ver"] = prserv.__version__ 267 metainfo["col_info"] = [] 268 data = self._execute("PRAGMA table_info(%s);" % self.table) 269 for row in data: 270 col = {} 271 col["name"] = row["name"] 272 col["type"] = row["type"] 273 col["notnull"] = row["notnull"] 274 col["dflt_value"] = row["dflt_value"] 275 col["pk"] = row["pk"] 276 metainfo["col_info"].append(col) 277 278 #data info 279 datainfo = [] 280 281 if self.nohist: 282 sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \ 283 (SELECT version, pkgarch, max(value) as maxvalue FROM %s GROUP BY version, pkgarch) as T2 \ 284 WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table) 285 else: 286 sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table 287 sqlarg = [] 288 where = "" 289 if version: 290 where += "AND T1.version=? " 291 sqlarg.append(str(version)) 292 if pkgarch: 293 where += "AND T1.pkgarch=? " 294 sqlarg.append(str(pkgarch)) 295 if checksum: 296 where += "AND T1.checksum=? " 297 sqlarg.append(str(checksum)) 298 299 sqlstmt += where + ";" 300 301 if len(sqlarg): 302 data = self._execute(sqlstmt, tuple(sqlarg)) 303 else: 304 data = self._execute(sqlstmt) 305 for row in data: 306 if row["version"]: 307 col = {} 308 col["version"] = row["version"] 309 col["pkgarch"] = row["pkgarch"] 310 col["checksum"] = row["checksum"] 311 col["value"] = row["value"] 312 datainfo.append(col) 313 return (metainfo, datainfo) 314 315 def dump_db(self, fd): 316 writeCount = 0 317 for line in self.conn.iterdump(): 318 writeCount = writeCount + len(line) + 1 319 fd.write(line) 320 fd.write("\n") 321 return writeCount 322 323class PRData(object): 324 """Object representing the PR database""" 325 def __init__(self, filename, nohist=True, read_only=False): 326 self.filename=os.path.abspath(filename) 327 self.nohist=nohist 328 self.read_only = read_only 329 #build directory hierarchy 330 try: 331 os.makedirs(os.path.dirname(self.filename)) 332 except OSError as e: 333 if e.errno != errno.EEXIST: 334 raise e 335 uri = "file:%s%s" % (self.filename, "?mode=ro" if self.read_only else "") 336 logger.debug("Opening PRServ database '%s'" % (uri)) 337 self.connection=sqlite3.connect(uri, uri=True, isolation_level="EXCLUSIVE", check_same_thread = False) 338 self.connection.row_factory=sqlite3.Row 339 if not self.read_only: 340 self.connection.execute("pragma synchronous = off;") 341 self.connection.execute("PRAGMA journal_mode = MEMORY;") 342 self._tables={} 343 344 def disconnect(self): 345 self.connection.close() 346 347 def __getitem__(self, tblname): 348 if not isinstance(tblname, str): 349 raise TypeError("tblname argument must be a string, not '%s'" % 350 type(tblname)) 351 if tblname in self._tables: 352 return self._tables[tblname] 353 else: 354 tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist, self.read_only) 355 return tableobj 356 357 def __delitem__(self, tblname): 358 if tblname in self._tables: 359 del self._tables[tblname] 360 logger.info("drop table %s" % (tblname)) 361 self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) 362