Author: fw
Date: 2005-09-21 17:46:59 +0000 (Wed, 21 Sep 2005)
New Revision: 2072
Modified:
lib/python/bugs.py
lib/python/security_db.py
Log:
Add "FIXES:" and "FIXED-BY:" directives.
lib/python/bugs.py (PackageNote):
New attribute "bug_origin".
(PackageNote.writeDB):
No longer skipr writing when self.id has been set (so that writeDB
can be used for cloning notes). Write the bug_origin attribute.
(PackageNoteFromDB):
Read the bug_origin attribute.
(BugBase):
Initialize the xref_fixes and xref_fixedby attributes.
(BugBase.writeDB):
Write them.
(Bug):
Pass through xref_fixes and xref_fixedby in constructor.
(BugFroMDB):
Load them.
(FileBase):
New regexps re_xref_fixes_required, re_xref_fixes,
re_xref_fixedby_required, re_xref_fixedby.
(FileBase.__iter__):
Record FIXES: and FIXED-BY:.
lib/python/security_db.py (DB):
Bump schema version.
(DB.initSchema):
Add bug_origin column to the packages_notes table.
Add copy_notes column to bugs_xref.
(DB.readBugs):
Remove incremental reading. Add new code that copies package notes,
as requested by the FIXES: and FIXED-BY: directives.
Modified: lib/python/bugs.py
==================================================================---
lib/python/bugs.py 2005-09-21 15:15:42 UTC (rev 2071)
+++ lib/python/bugs.py 2005-09-21 17:46:59 UTC (rev 2072)
@@ -67,6 +67,7 @@
self.urgency = urgency
self.bugs = []
self.package_kind = "unknown"
+ self.bug_origin = None
def affects(self, version, release=None):
"""Returns true if this package note affects the given
version.
@@ -131,16 +132,9 @@
return map(lambda ver: (ver, versions[str(ver)]), l)
return (sort(vulnerable_versions), sort(fixed_versions))
- def writeDB(self, cursor, bug_name):
- """Writes the object to an SQLite database.
+ def writeDB(self, cursor, bug_name, bug_origin=''''):
+ """Writes the object to an SQLite
database."""
- If the id attibute is already set, it is assumed that the
- object has already been written.
- """
-
- if self.id is not None:
- return
-
if self.fixed_version:
v = str(self.fixed_version)
else:
@@ -150,10 +144,10 @@
else:
r = ''''
cursor.execute("""INSERT INTO package_notes
- (bug_name, package, fixed_version, release, urgency)
- VALUES (?, ?, ?, ?, ?)""",
+ (bug_name, package, fixed_version, release, urgency, bug_origin)
+ VALUES (?, ?, ?, ?, ?, ?)""",
(bug_name, self.package, v, r,
- str(self.urgency)))
+ str(self.urgency), bug_origin))
for (rowid,) in cursor.execute(''SELECT
last_insert_rowid()''):
self.id = rowid
for b in self.bugs:
@@ -182,10 +176,10 @@
class PackageNoteFromDB(PackageNote):
def __init__(self, cursor, nid):
- for bug_name, package, fixed_version, release, urgency, package_kind\
- in cursor.execute\
+ for (bug_name, package, fixed_version, release, urgency,
+ package_kind, bug_origin) in cursor.execute\
("""SELECT bug_name, package, fixed_version,
release, urgency,
- package_kind
+ package_kind, bug_origin
FROM package_notes WHERE id = ?""", (nid,)):
PackageNote.__init__(package, fixed_version, release, urgency)
self.id = nid
@@ -241,6 +235,8 @@
self.comments = comments
self.notes = []
self.xref = []
+ self.xref_fixes = []
+ self.xref_fixedby = []
self.not_for_us = False
def isFromCVE(self):
@@ -313,11 +309,26 @@
raise ValueError, \
"cross reference to %s appears multiple times"
% x
+ # We use INSERT OR REPLACE below because the copy annotations
+ # must override plain cross-references.
+
+ for x in self.xref_fixes:
+ cursor.execute("""INSERT OR REPLACE INTO bugs_xref
+ (source, target, copy_notes) VALUES (?, ?, 1)""",
+ (self.name, x))
+
+ for x in self.xref_fixedby:
+ cursor.execute("""INSERT OR REPLACE INTO bugs_xref
+ (source, target, copy_notes) VALUES (?, ?, 1)""",
+ (x, self.name))
+ # Swap x and self.name because FIXED-BY: works in the other
+ # direction.
+
class Bug(BugBase):
"""Class for bugs for which we have some
data."""
def __init__(self, fname, lineno, date, name, description, comments, notes,
- xref, not_for_us=False):
+ xref, xref_fixes, xref_fixedby, not_for_us=False):
assert len(notes) == 0 or isinstance(notes[0], PackageNote)
assert len(xref) == 0 or type(xref[0]) == types.StringType
assert type(not_for_us) == types.BooleanType
@@ -325,6 +336,8 @@
description, comments)
self.notes = notes
self.xref = xref
+ self.xref_fixes = xref_fixes
+ self.xref_fixedby = xref_fixedby
self.not_for_us = not_for_us
def mergeNotes(self):
@@ -388,7 +401,7 @@
Bug.__init__(self, data[''source_file''],
data[''source_line''],
data[''release_date''], name,
data[''description''], comments=[],
- notes=[], xref=[],
+ notes=[], xref=[], xref_fixes=[], xref_fixedby=[],
not_for_us=not not data[''not_for_us''])
for (x,) in cursor.execute\
(''SELECT target FROM bugs_xref WHERE source = ?'',
(name,)):
@@ -401,18 +414,28 @@
self.comments.append((t, c))
# temporary list required because loadBugs needs the cursor
- for nid, package, fixed_version, release, urgency, package_kind \
- in list(cursor.execute
+ for (nid, package, fixed_version, release, urgency, package_kind,
+ bug_origin) in list(cursor.execute
("""SELECT id, package, fixed_version, release,
urgency,
- package_kind
+ package_kind, bug_origin
FROM package_notes WHERE bug_name = ?""", (name,))):
n = PackageNote(package, fixed_version, release, urgency)
n.id = nid
n.bug_name = name
n.package_kind = package_kind
+ n.bug_origin = bug_origin
n.loadBugs(cursor)
self.notes.append(n)
+ for (target,) in cursor.execute(
+ "SELECT target FROM bugs_xref WHERE source = ? AND
copy_notes",
+ (name,)):
+ self.xref_fixes.append(target)
+ for (target,) in cursor.execute(
+ "SELECT source FROM bugs_xref WHERE target = ? AND
copy_notes",
+ (name,)):
+ self.xref_fixedby.append(target)
+
def getDebianBugs(self, cursor):
"""Returns a list of Debian bugs to which the bug report
refers."""
return map(lambda (x,): x, cursor.execute(
@@ -465,6 +488,13 @@
re_xref_entry = re.compile(''^(?:(?:CAN|CVE)-\d{4}-\d{4}''
+ r''|VU#\d{6}''
+
r''|DSA-\d+(?:-\d+)?|DTSA-\d+-\d+)$'')
+ re_xref_entry_own = re.compile(
+
''^(?:(?:CAN|CVE)-\d{4}-\d{4}|DSA-\d+(?:-\d+)?|DTSA-\d+-\d+)$'')
+
+ re_xref_fixes_required = re.compile(r''^FIXES'')
+ re_xref_fixes = re.compile(r''^FIXES:\s+(.*?)\s*$'')
+ re_xref_fixedby_required = re.compile(r''^FIXED-BY'')
+ re_xref_fixedby = re.compile(r''^FIXED-BY:\s+(.*?)\s*$'')
# temporary hack, until we know what "!" actually means.
re_package_required = re.compile(r''^(?:\[.*\]\s*)?[-!]'')
@@ -562,6 +592,8 @@
not_for_us = None
xref = []
+ xref_fixes = []
+ xref_fixedby = []
pkg_notes = []
comments = []
cve_reserved = False
@@ -569,21 +601,38 @@
first_bug = 0
for (lineno, r) in record:
- if self.re_xref_required.match(r):
- match = self.re_xref.match(r)
- if match:
- (xref_string,) = match.groups()
- for x in self.re_whitespace.split(xref_string):
- if self.re_xref_entry.match(x):
- xref.append(x)
- else:
- self.raiseSyntaxError\
- ("invalid cross reference " +
`x`, lineno)
- continue
+ def handle_xref(re_required, re_real, re_entry, target):
+ if re_required.match(r):
+ match = re_real.match(r)
+ if match:
+ (xref_string,) = match.groups()
+ for x in self.re_whitespace.split(xref_string):
+ if re_entry.match(x):
+ target.append(x)
+ else:
+ self.raiseSyntaxError\
+ ("invalid cross reference " +
`x`,
+ lineno)
+ return True
+ else:
+ self.raiseSyntaxError(
+ "expected cross reference, got: " +
`r`,
+ lineno)
else:
- self.raiseSyntaxError("expected cross reference,
got: "
- + `r`, lineno)
-
+ return False
+
+ if handle_xref(self.re_xref_required, self.re_xref,
+ self.re_xref_entry, xref):
+ continue
+ if handle_xref(self.re_xref_fixes_required,
+ self.re_xref_fixes,
+ self.re_xref_entry_own, xref_fixes):
+ continue
+ if handle_xref(self.re_xref_fixedby_required,
+ self.re_xref_fixedby,
+ self.re_xref_entry_own, xref_fixedby):
+ continue
+
if self.re_package_required.match(r):
match = self.re_package.match(r)
if match:
@@ -658,7 +707,9 @@
yield self.finishBug(Bug(self.file.name, first_lineno,
date, record_name, description,
comments,
- notes=pkg_notes, xref=xref))
+ notes=pkg_notes, xref=xref,
+ xref_fixes=xref_fixes,
+ xref_fixedby=xref_fixedby))
else:
yield BugReservedCVE(self.file.name, first_lineno,
record_name, comments)
@@ -684,13 +735,18 @@
first_lineno)
yield self.finishBug(Bug(self.file.name, first_lineno, date,
record_name, description, comments,
- notes=[], xref=xref, not_for_us=True))
+ notes=[], xref=xref,
+ xref_fixes=xref_fixes,
+ xref_fixedby=xref_fixedby,
+ not_for_us=True))
else:
if not self.isUniqueName(record_name):
record_name = ''FAKE-%07d-%06d'' %
(first_bug, first_lineno)
yield self.finishBug(Bug(self.file.name, first_lineno, date,
record_name, description,
- comments, notes=pkg_notes, xref=xref))
+ comments, notes=pkg_notes, xref=xref,
+ xref_fixes=xref_fixes,
+ xref_fixedby=xref_fixedby))
def finishBug(self, bug):
"""Applies a transformation to the bug after it has been
Modified: lib/python/security_db.py
==================================================================---
lib/python/security_db.py 2005-09-21 15:15:42 UTC (rev 2071)
+++ lib/python/security_db.py 2005-09-21 17:46:59 UTC (rev 2072)
@@ -93,7 +93,7 @@
self.db = apsw.Connection(name)
self.verbose = verbose
- self.schema_version = 9
+ self.schema_version = 10
c = self.cursor()
for (v,) in c.execute("PRAGMA user_version"):
@@ -195,7 +195,8 @@
fixed_version_id INTEGER NOT NULL DEFAULT 0,
release TEXT NOT NULL,
package_kind TEXT NOT NULL DEFAULT ''unknown'',
- urgency TEXT NOT NULL)""")
+ urgency TEXT NOT NULL,
+ bug_origin TEXT NOT NULL DEFAULT
'''')""")
cursor.execute(
"""CREATE UNIQUE INDEX package_notes_bug
ON package_notes(bug_name, package, release)""")
@@ -226,7 +227,11 @@
(source TEXT NOT NULL,
target TEXT NOT NULL,
normalized_target TEXT NOT NULL DEFAULT '''',
+ copy_notes INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (source, target))""")
+ cursor.execute(
+ """CREATE INDEX bugs_xref_normalized_target
+ ON bugs_xref(normalized_target)""")
cursor.execute("""CREATE TABLE bug_status
(bug_name TEXT NOT NULL,
@@ -539,50 +544,32 @@
if self.verbose:
print "readBugs:"
- def clear_db(filename):
- cursor.execute(
- """CREATE TEMPORARY TABLE bugs_to_delete
- (tbd TEXT NOT NULL PRIMARY KEY)""")
- cursor.execute(
- """INSERT INTO bugs_to_delete
- SELECT name FROM bugs WHERE source_file = ?""",
- (filename,))
+ def clear_db(cleared=[False]):
+ # Avoid clearing the database multiple times.
+ if cleared[0]:
+ return
+ else:
+ cleared[0] = True
+
+ cursor.execute("DELETE FROM debian_bugs")
+ cursor.execute("DELETE FROM bugs")
+ cursor.execute("DELETE FROM package_notes")
+ cursor.execute("DELETE FROM bugs_notes")
+ cursor.execute("DELETE FROM bugs_xref")
- cursor.execute(
- """DELETE FROM debian_bugs
- WHERE EXISTS (SELECT 1
- FROM package_notes AS p, bugs_to_delete AS b
- WHERE p.id = debian_bugs.note
- AND p.bug_name = b.tbd)""")
-
- cursor.execute("""DELETE FROM bugs
- WHERE EXISTS (SELECT * FROM bugs_to_delete
- WHERE tbd = name)""")
- cursor.execute("""DELETE FROM package_notes
- WHERE EXISTS (SELECT * FROM bugs_to_delete
- WHERE tbd = bug_name)""")
- cursor.execute("""DELETE FROM bugs_notes
- WHERE EXISTS (SELECT * FROM bugs_to_delete
- WHERE tbd = bug_name)""")
- cursor.execute("""DELETE FROM bugs_xref
- WHERE EXISTS (SELECT * FROM bugs_to_delete
- WHERE tbd = source)""")
-
# The *_status tables are regenerated anyway, no need to
# delete them here.
- cursor.execute("""DROP TABLE
bugs_to_delete""")
-
self._clearVersions(cursor)
-
- def do_parse(source):
+
+ def do_parse(source, cleared=[False]):
errors = []
+
+ clear_db()
if self.verbose:
print " reading " + `source.name`
- clear_db(source.name)
-
for bug in source:
try:
bug.writeDB(cursor)
@@ -592,43 +579,46 @@
if errors:
raise InsertError(errors)
- def read_one(source):
- filename = source.name
+ def has_changed(filename):
current_print = self.filePrint(filename)
-
for (old_print,) in cursor.execute(
"SELECT inodeprint FROM inodeprints WHERE file = ?",
(filename,)):
if old_print == current_print:
return False
- do_parse(source)
- cursor.execute(
- "UPDATE inodeprints SET inodeprint = ? WHERE file =
?",
- (current_print, filename))
- return True
-
- # No inodeprints entry, load file and add one.
- do_parse(source)
- cursor.execute(
- "INSERT INTO inodeprints (file, inodeprint) VALUES (?,
?)",
- (filename, current_print))
+ else:
+ return True
return True
+ sources = ((bugs.CANFile, ''/CAN/list''),
+ (bugs.CVEFile, ''/CVE/list''),
+ (bugs.DSAFile, ''/DSA/list''),
+ (bugs.DTSAFile, ''/DTSA/list''))
+
unchanged = True
- if read_one(bugs.CANFile(path + ''/CAN/list'')):
- unchanged = False
- if read_one(bugs.CVEFile(path + ''/CVE/list'')):
- unchanged = False
- if read_one(bugs.DSAFile(path + ''/DSA/list'')):
- unchanged = False
- if read_one(bugs.DTSAFile(path + ''/DTSA/list'')):
- unchanged = False
-
+ for (_, name) in sources:
+ if has_changed(path + name):
+ unchanged = False
+ break
if unchanged:
if self.verbose:
print " finished (no changes)"
return
+ clear_db()
+
+ def read_one(source):
+ filename = source.name
+ current_print = self.filePrint(filename)
+
+ do_parse(source)
+ cursor.execute(
+ """INSERT OR REPLACE INTO inodeprints
(inodeprint, file)
+ VALUES (?, ?)""", (current_print, filename))
+
+ for (cls, name) in sources:
+ read_one(cls(path + name))
+
errors = []
if self.verbose:
@@ -656,10 +646,11 @@
if self.verbose:
print " normalize CAN/CVE references"
-
+
+ cursor.execute("UPDATE bugs_xref SET normalized_target =
target")
for source, target in list(cursor.execute\
("""SELECT source, target FROM bugs_xref
- WHERE normalized_target = ''''""")):
+ WHERE target LIKE ''CAN-%'' OR target LIKE
''CVE-%''""")):
if bugs.BugBase.re_cve_name.match(target):
can_target = ''CAN-'' + target[4:]
cve_target = ''CVE-'' + target[4:]
@@ -695,6 +686,64 @@
("%s: %d: reference to unknwown advisory %s"
% (b.source_file, b.source_line, target))
+ if self.verbose:
+ print " apply FIXES"
+
+ target_sources = {}
+ for source, target in list(cursor.execute(
+ """SELECT source, normalized_target
+ FROM bugs_xref WHERE copy_notes""")):
+ if target_sources.has_key(target):
+ target_sources[target][source] = True
+ else:
+ target_sources[target] = {source : True}
+
+ # Recursively collect all sources for each target. Add new
+ # sources until the set of sources stabilizes.
+ for sources in target_sources.values():
+ while 1:
+ old_size = len(sources.keys())
+ for src in sources.keys():
+ for s in target_sources.get(src, {}).keys():
+ sources[s] = True
+ if len(sources.keys()) == old_size:
+ break
+
+ # Copy all the notes from all sources.
+ for (target, sources) in target_sources.items():
+ sources = sources.keys()
+ for source in sources:
+ source_bug = bugs.BugFromDB(cursor, source)
+ for n in source_bug.notes:
+ # Only copy "real" notes, not notes which have
+ # already bee ncopied.
+ if n.bug_origin:
+ continue
+ if n.release:
+ rel = str(n.release)
+ else:
+ rel = ''''
+ present = False
+ for (version, origin) in list(cursor.execute(
+ """SELECT fixed_version, bug_origin
+ FROM package_notes
+ WHERE bug_name = ? AND package = ? AND release =
?""",
+ (target, n.package, rel))):
+ if version <> str(n.fixed_version):
+ bug = bugs.BugFromDB(cursor, origin or target)
+ errors.append(
+ ("%s: %d: version %s for package %s "
+ + "conflicts with %s")
+ % (bug.source_file, bug.source_line,
+ version, n.package, bug_source.name))
+ errors.append("%s: %d: location of %s"
+ % (source_bug.source_file,
+ source_bug.source_line,
+ source_bug.name))
+ present = True
+ if not present:
+ n.writeDB(cursor, target, bug_origin=source)
+
if errors:
raise InsertError(errors)