import logging
import os.path
import errno
import prserv
import time
try:
import sqlite3
except ImportError:
from pysqlite2 import dbapi2 as sqlite3
logger = logging.getLogger("BitBake.PRserv")
sqlversion = sqlite3.sqlite_version_info
if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
raise Exception("sqlite3 version 3.3.0 or later is required.")
#
# "No History" mode - for a given query tuple (version, pkgarch, checksum),
# the returned value will be the largest among all the values of the same
# (version, pkgarch). This means the PR value returned can NOT be decremented.
#
# "History" mode - Return a new higher value for previously unseen query
# tuple (version, pkgarch, checksum), otherwise return historical value.
# Value can decrement if returning to a previous build.
#
class PRTable(object):
def __init__(self, conn, table, nohist):
self.conn = conn
self.nohist = nohist
self.dirty = False
if nohist:
self.table = "%s_nohist" % table
else:
self.table = "%s_hist" % table
self._execute("CREATE TABLE IF NOT EXISTS %s \
(version TEXT NOT NULL, \
pkgarch TEXT NOT NULL, \
checksum TEXT NOT NULL, \
value INTEGER, \
PRIMARY KEY (version, pkgarch, checksum));" % self.table)
def _execute(self, *query):
"""Execute a query, waiting to acquire a lock if necessary"""
start = time.time()
end = start + 20
while True:
try:
return self.conn.execute(*query)
except sqlite3.OperationalError as exc:
if 'is locked' in str(exc) and end > time.time():
continue
raise exc
def sync(self):
self.conn.commit()
self._execute("BEGIN EXCLUSIVE TRANSACTION")
def sync_if_dirty(self):
if self.dirty:
self.sync()
self.dirty = False
def _getValueHist(self, version, pkgarch, checksum):
data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
(version, pkgarch, checksum))
row=data.fetchone()
if row != None:
return row[0]
else:
#no value found, try to insert
try:
self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
% (self.table,self.table),
(version,pkgarch, checksum,version, pkgarch))
except sqlite3.IntegrityError as exc:
logger.error(str(exc))
self.dirty = True
data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
(version, pkgarch, checksum))
row=data.fetchone()
if row != None:
return row[0]
else:
raise prserv.NotFoundError
def _getValueNohist(self, version, pkgarch, checksum):
data=self._execute("SELECT value FROM %s \
WHERE version=? AND pkgarch=? AND checksum=? AND \
value >= (select max(value) from %s where version=? AND pkgarch=?);"
% (self.table, self.table),
(version, pkgarch, checksum, version, pkgarch))
row=data.fetchone()
if row != None:
return row[0]
else:
#no value found, try to insert
try:
self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
% (self.table,self.table),
(version, pkgarch, checksum, version, pkgarch))
except sqlite3.IntegrityError as exc:
logger.error(str(exc))
self.conn.rollback()
self.dirty = True
data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
(version, pkgarch, checksum))
row=data.fetchone()
if row != None:
return row[0]
else:
raise prserv.NotFoundError
def getValue(self, version, pkgarch, checksum):
if self.nohist:
return self._getValueNohist(version, pkgarch, checksum)
else:
return self._getValueHist(version, pkgarch, checksum)
def _importHist(self, version, pkgarch, checksum, value):
val = None
data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
(version, pkgarch, checksum))
row = data.fetchone()
if row != None:
val=row[0]
else:
#no value found, try to insert
try:
self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
(version, pkgarch, checksum, value))
except sqlite3.IntegrityError as exc:
logger.error(str(exc))
self.dirty = True
data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
(version, pkgarch, checksum))
row = data.fetchone()
if row != None:
val = row[0]
return val
def _importNohist(self, version, pkgarch, checksum, value):
try:
#try to insert
self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
(version, pkgarch, checksum,value))
except sqlite3.IntegrityError as exc:
#already have the record, try to update
try:
self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?"
% (self.table),
(value,version,pkgarch,checksum,value))
except sqlite3.IntegrityError as exc:
logger.error(str(exc))
self.dirty = True
data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table,
(version,pkgarch,checksum,value))
row=data.fetchone()
if row != None:
return row[0]
else:
return None
def importone(self, version, pkgarch, checksum, value):
if self.nohist:
return self._importNohist(version, pkgarch, checksum, value)
else:
return self._importHist(version, pkgarch, checksum, value)
def export(self, version, pkgarch, checksum, colinfo):
metainfo = {}
#column info
if colinfo:
metainfo['tbl_name'] = self.table
metainfo['core_ver'] = prserv.__version__
metainfo['col_info'] = []
data = self._execute("PRAGMA table_info(%s);" % self.table)
for row in data:
col = {}
col['name'] = row['name']
col['type'] = row['type']
col['notnull'] = row['notnull']
col['dflt_value'] = row['dflt_value']
col['pk'] = row['pk']
metainfo['col_info'].append(col)
#data info
datainfo = []
if self.nohist:
sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
(SELECT version,pkgarch,max(value) as maxvalue FROM %s GROUP BY version,pkgarch) as T2 \
WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
else:
sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
sqlarg = []
where = ""
if version:
where += "AND T1.version=? "
sqlarg.append(str(version))
if pkgarch:
where += "AND T1.pkgarch=? "
sqlarg.append(str(pkgarch))
if checksum:
where += "AND T1.checksum=? "
sqlarg.append(str(checksum))
sqlstmt += where + ";"
if len(sqlarg):
data = self._execute(sqlstmt, tuple(sqlarg))
else:
data = self._execute(sqlstmt)
for row in data:
if row['version']:
col = {}
col['version'] = row['version']
col['pkgarch'] = row['pkgarch']
col['checksum'] = row['checksum']
col['value'] = row['value']
datainfo.append(col)
return (metainfo, datainfo)
def dump_db(self, fd):
writeCount = 0
for line in self.conn.iterdump():
writeCount = writeCount + len(line) + 1
fd.write(line)
fd.write('\n')
return writeCount
class PRData(object):
"""Object representing the PR database"""
def __init__(self, filename, nohist=True):
self.filename=os.path.abspath(filename)
self.nohist=nohist
#build directory hierarchy
try:
os.makedirs(os.path.dirname(self.filename))
except OSError as e:
if e.errno != errno.EEXIST:
raise e
self.connection=sqlite3.connect(self.filename, isolation_level="EXCLUSIVE", check_same_thread = False)
self.connection.row_factory=sqlite3.Row
self.connection.execute("pragma synchronous = off;")
self.connection.execute("PRAGMA journal_mode = WAL;")
self._tables={}
def disconnect(self):
self.connection.close()
def __getitem__(self,tblname):
if not isinstance(tblname, str):
raise TypeError("tblname argument must be a string, not '%s'" %
type(tblname))
if tblname in self._tables:
return self._tables[tblname]
else:
tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist)
return tableobj
def __delitem__(self, tblname):
if tblname in self._tables:
del self._tables[tblname]
logger.info("drop table %s" % (tblname))
self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname)