Author: fw
Date: 2005-09-14 13:27:19 +0000 (Wed, 14 Sep 2005)
New Revision: 1977
Modified:
bin/update-vulnerabilities
lib/python/bugs.py
lib/python/security_db.py
Log:
Ongoing work to implement version tracking. The current approach does
not scale with the number of architectures, though.
lib/python/security_db.py (DB):
Add nicknames member.
(DB.initSchema):
Add nicknames and package_status tables.
Add index package_notes_bug.
(DB._synthesizeReleases):
New method to build testing etc. distributions.
(DB.calculateVulnerabilities):
Update to use new tables. Return list of problems detected.
lib/python/bugs.py (PackageNote.releaseStatus):
New method to check for affected releases.
bin/update-vulnerabilities:
Print list of detected problems.
Modified: bin/update-vulnerabilities
==================================================================---
bin/update-vulnerabilities 2005-09-14 13:21:11 UTC (rev 1976)
+++ bin/update-vulnerabilities 2005-09-14 13:27:19 UTC (rev 1977)
@@ -28,5 +28,10 @@
assert os.path.exists(db_file)
db = security_db.DB(db_file, verbose=True)
c = db.writeTxn()
-db.calculateVulnerabilities(c)
+warnings = db.calculateVulnerabilities(c)
+if warnings:
+ db.rollback(c)
+ for x in warnings:
+ print x
+ sys.exit(1)
db.commit(c)
Modified: lib/python/bugs.py
==================================================================---
lib/python/bugs.py 2005-09-14 13:21:11 UTC (rev 1976)
+++ lib/python/bugs.py 2005-09-14 13:27:19 UTC (rev 1977)
@@ -103,6 +103,26 @@
else:
return "unfixed"
+ def releaseStatus(self, cursor):
+ """Returns a pair of lists (VULNERABLE, NON-VULNERABLE).
+
+ The lists consits of triples (release, archive, architecture).
+ """
+
+ vulnerable = []
+ non_vulnerable = []
+
+ for (release, archive, architecture, vuln) in cursor.execute(
+ """SELECT release, archive, architecture, vulnerable
+ FROM package_status WHERE note = ?""", (self.id,)):
+ t = (release, archive,architecture)
+ if vuln:
+ vulnerable.append(t)
+ else:
+ non_vulnerable.append(t)
+
+ return (vulnerable, non_vulnerable)
+
def writeDB(self, cursor, bug_name):
"""Writes the object to an SQLite database.
Modified: lib/python/security_db.py
==================================================================---
lib/python/security_db.py 2005-09-14 13:21:11 UTC (rev 1976)
+++ lib/python/security_db.py 2005-09-14 13:27:19 UTC (rev 1977)
@@ -63,6 +63,9 @@
def __init__(self, name, verbose=False):
self.db = apsw.Connection(name)
self.verbose = verbose
+ self.nicknames = {''etch'':
''testing'',
+ ''sarge'' :
''stable'',
+ ''woody'':
''oldstable''}
def cursor(self):
"""Creates a new database cursor.
@@ -97,6 +100,13 @@
(file TEXT NOT NULL PRIMARY KEY,
inodeprint TEXT NOT NULL)""")
+ cursor.execute(
+ """CREATE TABLE nicknames
+ (realname TEXT NOT NULL,
+ nickname TEXT NOT NULL)""")
+ cursor.executemany("INSERT INTO nicknames VALUES (?, ?)",
+ self.nicknames.items())
+
cursor.execute("""CREATE TABLE version_linear_order
(id INTEGER NOT NULL PRIMARY KEY,
version TEXT NOT NULL UNIQUE)""")
@@ -128,6 +138,8 @@
fixed_version_id INTEGER NOT NULL DEFAULT 0,
release TEXT NOT NULL,
urgency TEXT NOT NULL)""")
+ cursor.execute(
+ "CREATE INDEX package_notes_bug ON
package_notes(bug_name)")
cursor.execute("""CREATE TABLE debian_bugs
(bug INTEGER NOT NULL,
@@ -163,6 +175,23 @@
reason TEXT NOT NULL,
PRIMARY KEY (bug_name, release, note))""")
+ cursor.execute("""CREATE TABLE package_status
+ (note INTEGER NOT NULL,
+ release TEXT NOT NULL,
+ archive TEXT NOT NULL,
+ architecture TEXT NOT NULL,
+ vulnerable INTEGER NOT NULL,
+ bug_name TEXT NOT NULL,
+ package TEXT NOT NULL,
+ source TEXT NOT NULL,
+ PRIMARY KEY (note, release, archive, architecture))""")
+ cursor.execute(
+ "CREATE INDEX package_status_bug ON
package_status(bug_name)")
+ cursor.execute(
+ "CREATE INDEX package_status_package ON
package_status(package)")
+ cursor.execute(
+ "CREATE INDEX package_status_source ON
package_status(source)")
+
def updateSources(self, cursor, release, archive, packages):
"""Reads a Sources file and adds it to the database.
@@ -290,8 +319,7 @@
cursor.execute("""INSERT INTO inodeprints (file,
inodeprint)
VALUES (?, ?)""", (filename, current_print))
return result
-
-
+
def maybeUpdateSources(self, cursor, release, archive, filename):
"""Reads the Sources file filename if it has been
modified."""
self._maybeUpdate(cursor, (release, archive), filename,
@@ -508,17 +536,183 @@
if self.verbose:
print " finished"
+ def _synthesizeReleases(self, cursor):
+ """Creates the package lists for testing, stable and
oldstable.
+
+ These package lists include security updates.
+ """
+
+ if self.verbose:
+ print "synthesizeReleases:"
+ print " clear old data"
+ print " source packages"
+ cursor.execute(
+ """DELETE FROM source_packages
+ WHERE release IN (''stable'',
''oldstable'', ''testing'')""")
+ if self.verbose:
+ print " binary packages"
+ cursor.execute(
+ """DELETE FROM binary_packages
+ WHERE release IN (''stable'',
''oldstable'', ''testing'')""")
+
+ for (realname, nickname) in self.nicknames.items():
+ if self.verbose:
+ print " synthesize %s to %s" % (realname, nickname)
+ print " source packages"
+ cursor.execute(
+ """INSERT INTO source_packages
+ SELECT package, ?, archive, '''',
MAX(version_id) AS vid
+ FROM source_packages WHERE release IN (?, ?)
+ GROUP BY package, archive""",
+ (nickname, realname, realname + ''-security''))
+
+ if self.verbose:
+ print " binary packages"
+ cursor.execute(
+ """INSERT INTO binary_packages
+ SELECT DISTINCT package, ?, archive, architecture,
'''',
+ MAX (version_id) AS vid, source, source_version
+ FROM binary_packages WHERE release IN (?, ?)
+ GROUP BY package, archive, architecture""",
+ (nickname, realname, realname + ''-security''))
+
+ if self.verbose:
+ print " patch version strings"
+ print " source packages"
+ cursor.execute(
+ """UPDATE source_packages
+ SET version = (SELECT version FROM version_linear_order
+ WHERE id = version_id)
+ WHERE version = ''''""")
+ if self.verbose:
+ print " binary packages"
+ cursor.execute(
+ """UPDATE binary_packages
+ SET version = (SELECT version FROM version_linear_order
+ WHERE id = version_id)
+ WHERE version = ''''""")
+
+ if self.verbose:
+ print " finished"
+
def calculateVulnerabilities(self, cursor):
"""Calculate vulnerable packages.
To each package note, a release-specific vulnerability status
is attached. Currently, only etch/testing is processed.
+
+ Returns a list strings describing inconsistencies.
"""
+ result = []
+
self._updateVersions(cursor)
+ self._synthesizeReleases(cursor)
if self.verbose:
print "calculateVulnerabilities:"
+ print " check for version consistency in package notes"
+ for (bug_name, pkg_name, rel, unstable_ver, rel_ver) \
+ in list(cursor.execute(
+ """SELECT a.bug_name, a.package, a.release,
+ a.fixed_version, b.fixed_version
+ FROM package_notes a, package_notes b
+ WHERE a.bug_name = b.bug_name AND a.package = b.package
+ AND a.release = '''' AND b.release <>
''''
+ AND a.fixed_version_id < b.fixed_version_id""")):
+ b = bugs.BugFromDB(cursor, bug_name)
+ result.append("%s:%d: inconsistent versions for package
%s"
+ % (b.source_file, b.source_line, pkg_name))
+ result.append("%s:%d: unstable: %s"
+ % (b.source_file, b.source_line, rel_ver))
+ result.append("%s:%d: release %s: %s"
+ % (b.source_file, b.source_line, `rel`, rel_ver))
+
+ if self.verbose:
+ print " create temporary tables"
+ cursor.execute(
+ """CREATE TEMPORARY TABLE tmp_bug_releases
+ (bug_name TEXT NOT NULL,
+ release TEXT NOT NULL,
+ PRIMARY KEY (bug_name, release))""")
+ for (bug_name, release) in list(cursor.execute(
+ """SELECT DISTINCT bug_name, release FROM
package_notes
+ WHERE release <> ''''""")):
+ data = [(bug_name, release),
+ (bug_name, release + ''-security'')]
+ try:
+ data.append((bug_name, self.nicknames[release]))
+ except KeyError:
+ pass
+ cursor.executemany("INSERT INTO tmp_bug_releases VALUES (?,
?)",
+ data)
+
+ if self.verbose:
+ print " remove old status"
+ cursor.execute("DELETE FROM package_status")
+ if self.verbose:
+ print " calculate package status"
+ print " source packages (unqualified)"
+
+ # If there is a single package note qualified with a specific
+ # release, ignore all the unqualified annotations (even for
+ # other packages). This is implemented by looking at the
+ # tmp_bug_releases table.
+
+ cursor.execute(
+ """INSERT INTO package_status
+ SELECT n.id, p.release, p.archive, '''' AS
architecture,
+ p.version_id < n.fixed_version_id,
+ n.bug_name, p.package, p.package AS source
+ FROM package_notes AS n, source_packages AS p
+ WHERE n.release = '''' AND p.package = n.package
+ AND NOT EXISTS (SELECT * FROM tmp_bug_releases AS t
+ WHERE t.bug_name = n.bug_name
+ AND t.release = n.release)""")
+ if self.verbose:
+ print " source packages (qualified)"
+ cursor.execute(
+ """INSERT INTO package_status
+ SELECT n.id, p.release, p.archive, '''' AS
architecture,
+ p.version_id < n.fixed_version_id,
+ n.bug_name, p.package, p.package AS source
+ FROM package_notes AS n, source_packages AS p
+ WHERE p.package = n.package
+ AND (p.release = n.release
+ OR p.release = n.release || ''-security''
+ OR p.release = (SELECT nickname FROM nicknames
+ WHERE realname =
n.release))""")
+
+ # Same story for binary packages.
+
+ if self.verbose:
+ print " binary packages"
+ cursor.execute(
+ """INSERT INTO package_status
+ SELECT n.id, p.release, p.archive, p.architecture,
+ p.version_id < n.fixed_version_id AS fixed,
+ n.bug_name, p.package, p.source
+ FROM package_notes AS n, binary_packages AS p
+ WHERE n.release = '''' AND p.package = n.package
+ AND NOT EXISTS (SELECT * FROM tmp_bug_releases AS t
+ WHERE t.bug_name = n.bug_name
+ AND t.release = n.release)""")
+ if self.verbose:
+ print " binary packages (qualified)"
+ cursor.execute(
+ """INSERT INTO package_status
+ SELECT n.id, p.release, p.archive, p.architecture AS architecture,
+ p.version_id < n.fixed_version_id,
+ n.bug_name, p.package, p.package AS source
+ FROM package_notes AS n, binary_packages AS p
+ WHERE p.package = n.package
+ AND (p.release = n.release
+ OR p.release = n.release || ''-security''
+ OR p.release = (SELECT nickname FROM nicknames
+ WHERE realname =
n.release))""")
+
+
+ if self.verbose:
print " clearing old data"
cursor.execute("DELETE FROM bugs_status")
@@ -617,6 +811,8 @@
if self.verbose:
print " finished"
+ return result
+
def check(self, cursor=None):
"""Runs a simple consistency check and prints the
results."""