Author: fw
Date: 2005-09-14 08:59:50 +0000 (Wed, 14 Sep 2005)
New Revision: 1973
Modified:
bin/update-packages
bin/update-vulnerabilities
lib/python/security_db.py
Log:
Add table version_linear_order, which will enable us to make version
comparisons in pure SQL.
lib/python/security_db.py (DB):
Add verbose flag to constructor.
(DB.initSchema):
Add table version_linear_order. Add version ID fields to tables
package_notes, source_packages, binary_packages.
(DB._maybeUpdate):
Use self.verbose.
(DB.availableReleases):
New method to get a least of releases in the database.
(DB._updateVersions):
Calculate a linear order of versions. This will be used to speed up
the vulnerability rating process.
(DB.calculateVulnerabilities):
Invoke _updateVersions. Use self.verbose.
bin/update-vulnerabilities, bin/update-packages:
Set database verbose flag.
Modified: bin/update-packages
==================================================================---
bin/update-packages 2005-09-14 08:30:46 UTC (rev 1972)
+++ bin/update-packages 2005-09-14 08:59:50 UTC (rev 1973)
@@ -70,7 +70,7 @@
def cmd_import(args):
db_file = root_path + ''/data/security.db''
- db = security_db.DB(db_file)
+ db = security_db.DB(db_file, verbose=True)
try:
for release, archs in explodeReleases(args):
for archive in archives:
Modified: bin/update-vulnerabilities
==================================================================---
bin/update-vulnerabilities 2005-09-14 08:30:46 UTC (rev 1972)
+++ bin/update-vulnerabilities 2005-09-14 08:59:50 UTC (rev 1973)
@@ -26,7 +26,7 @@
db_file = root_path + ''/data/security.db''
assert os.path.exists(db_file)
-db = security_db.DB(db_file)
+db = security_db.DB(db_file, verbose=True)
c = db.writeTxn()
db.calculateVulnerabilities(c)
db.commit(c)
Modified: lib/python/security_db.py
==================================================================---
lib/python/security_db.py 2005-09-14 08:30:46 UTC (rev 1972)
+++ lib/python/security_db.py 2005-09-14 08:59:50 UTC (rev 1973)
@@ -60,8 +60,9 @@
misnomer because these objects are quite versatile.
"""
- def __init__(self, name):
+ def __init__(self, name, verbose=False):
self.db = apsw.Connection(name)
+ self.verbose = verbose
def cursor(self):
"""Creates a new database cursor.
@@ -96,10 +97,15 @@
(file TEXT NOT NULL PRIMARY KEY,
inodeprint TEXT NOT NULL)""")
+ cursor.execute("""CREATE TABLE version_linear_order
+ (id INTEGER NOT NULL PRIMARY KEY,
+ version TEXT NOT NULL UNIQUE)""")
+
cursor.execute("""CREATE TABLE source_packages
(package TEXT NOT NULL,
release TEXT NOT NULL, archive TEXT NOT NULL,
version TEXT NOT NULL,
+ version_id INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (package, release, archive));""")
cursor.execute("""CREATE TABLE binary_packages
@@ -107,6 +113,7 @@
release TEXT NOT NULL, archive TEXT NOT NULL,
architecture TEXT NOT NULL,
version TEXT NOT NULL,
+ version_id INTEGER NOT NULL DEFAULT 0,
source TEXT NOT NULL, source_version TEXT NOT NULL,
PRIMARY KEY (package, release, archive,
architecture));""")
cursor.execute("""CREATE INDEX binary_packages_source
@@ -118,6 +125,7 @@
package TEXT NOT NULL,
fixed_version TEXT
CHECK (fixed_version IS NULL OR fixed_version <>
''''),
+ fixed_version_id INTEGER NOT NULL DEFAULT 0,
release TEXT NOT NULL,
urgency TEXT NOT NULL)""")
@@ -268,6 +276,8 @@
("SELECT inodeprint FROM inodeprints WHERE file = ?",
(filename,)):
if old_print == current_print:
return
+ if self.verbose:
+ print "maybeUpdate: updating", `args`
result = apply(action, (cursor,) + args
+ (debian_support.PackageFile(filename),))
cursor.execute("""UPDATE inodeprints SET inodeprint
= ?
@@ -284,13 +294,11 @@
def maybeUpdateSources(self, cursor, release, archive, filename):
"""Reads the Sources file filename if it has been
modified."""
-
self._maybeUpdate(cursor, (release, archive), filename,
self.updateSources)
def maybeUpdatePackages(self, cursor, release, archive, arch, filename):
"""Reads the Packages file filename if it has been
modified."""
-
self._maybeUpdate(cursor, (release, archive, arch), filename,
self.updatePackages)
@@ -383,6 +391,32 @@
return warnings
+ def availableReleases(self, cursor=None):
+ """Returns a list of tuples (RELEASE, ARCHIVE,
ARCHITECTURE-LIST)."""
+ if cursor is None:
+ cursor = self.cursor()
+
+ releases = {}
+ for r in cursor.execute(
+ "SELECT DISTINCT release, archive FROM source_packages"):
+ releases[r] = [''(sources)'']
+
+ for (rel, archive, arch) in cursor.execute(
+ """SELECT DISTINCT release, archive, architecture
+ FROM binary_packages"""):
+ try:
+ releases[(rel, archive)].append(arch)
+ except KeyError:
+ releases[(rel, archive)] = [arch]
+
+ result = []
+ for ((rel, archive), archs) in releases.items():
+ archs.sort()
+ result.append((rel, archive, archs))
+ result.sort()
+
+ return result
+
def getVersion(self, cursor, release, package):
"""Returns the version number for package in release.
@@ -418,6 +452,62 @@
return True
return False
+ def _updateVersions(self, cursor):
+ """Updates the linear version table."""
+
+ cursor.execute("DELETE FROM version_linear_order");
+
+ if self.verbose:
+ print "updateVersions:"
+ print " reading"
+
+ versions = []
+ for (v,) in cursor.execute(
+ """SELECT DISTINCT *
+ FROM (SELECT fixed_version FROM package_notes
+ WHERE fixed_version IS NOT NULL
+ UNION ALL SELECT version FROM source_packages
+ UNION ALL SELECT version FROM binary_packages)"""):
+ if v is None:
+ continue
+ versions.append(debian_support.Version(v))
+
+ if self.verbose:
+ print " calculating linear oder"
+ versions.sort()
+
+ if self.verbose:
+ print " storing linear order"
+ for v in versions:
+ cursor.execute(
+ "INSERT INTO version_linear_order (version) VALUES
(?)",
+ (str(v),))
+
+ if self.verbose:
+ print " updating package notes"
+ cursor.execute(
+ """UPDATE package_notes
+ SET fixed_version_id = (SELECT id FROM version_linear_order
+ WHERE version = package_notes.fixed_version)
+ WHERE fixed_version IS NOT NULL""")
+
+ if self.verbose:
+ print " updating source packages"
+ cursor.execute(
+ """UPDATE source_packages
+ SET version_id = (SELECT id FROM version_linear_order
+ WHERE version = source_packages.version)""")
+
+ if self.verbose:
+ print " updating binary packages"
+ cursor.execute(
+ """UPDATE binary_packages
+ SET version_id = (SELECT id FROM version_linear_order
+ WHERE version = binary_packages.version)""")
+
+ if self.verbose:
+ print " finished"
+
def calculateVulnerabilities(self, cursor):
"""Calculate vulnerable packages.
@@ -425,6 +515,11 @@
is attached. Currently, only etch/testing is processed.
"""
+ self._updateVersions(cursor)
+
+ if self.verbose:
+ print "calculateVulnerabilities:"
+ print " clearing old data"
cursor.execute("DELETE FROM bugs_status")
def markVulnerable(bug, release, note, reason):
@@ -474,10 +569,14 @@
# First handle the DSAs. Cache results in DSA_status (used
# for CAN/CVE below).
+ if self.verbose:
+ print " reading DSAs"
bug_names = list(cursor.execute(
"""SELECT name FROM bugs
WHERE name LIKE ''DSA-%'' AND NOT
not_for_us"""))
DSA_status = {}
+ if self.verbose:
+ print " rating DSAs"
for (bug_name,) in bug_names:
b = bugs.BugFromDB(cursor, bug_name)
DSA_status[bug_name] = calcVuln(b)
@@ -486,10 +585,14 @@
# package annotations, but it references a non-vulnerable DSA,
# we assume that the current is not affect either.
+ if self.verbose:
+ print " reading other entries"
bug_names = list(cursor.execute(
"""SELECT name FROM bugs
WHERE (NOT not_for_us)
AND NOT (name LIKE ''DSA-%'' OR name LIKE
''DTSA-%'')"""))
+ if self.verbose:
+ print " rating other entries"
for (bug_name,) in bug_names:
b = bugs.BugFromDB(cursor, bug_name)
if b.notes:
@@ -511,6 +614,9 @@
if not dsa_found:
markVulnerable(b, ''etch'', 0, ''status
is unclear'')
+ if self.verbose:
+ print " finished"
+
def check(self, cursor=None):
"""Runs a simple consistency check and prints the
results."""