Author: fw
Date: 2005-09-15 10:11:44 +0000 (Thu, 15 Sep 2005)
New Revision: 1994
Added:
bin/update-db
Removed:
bin/update-bug-list-db
bin/update-packages
bin/update-vulnerabilities
Modified:
Makefile
bin/check-syntax
lib/python/bugs.py
lib/python/security_db.py
Log:
Implement bin/update-db, to update the database with a single command.
Most processing is skipped if no input files have been modified.
lib/python/security_db.py (SchemaMismatch):
New exception.
(DB):
Handle schema versioning.
(DB.initSchema):
Add subrelease column to source_packages and binary_packages.
Set user_version.
Remove stray commit.
(DB._parseFile):
Return information to the caller if the file is unchanged.
(DB.readPackages):
Move deletion code to callees.
(DB._readSourcePackages, DB._readBinaryPackages):
Implement incremental updates. Add subrelease.
Need to invoke _clearVersions if any changes are made.
(DB.deleteBugs, DB.finishBugs):
Moved into readBugs.
(DB.insertBugs):
Rename ...
(DB.readBugs):
... to this one. Implement incremental updates.
Invoke _clearVersions if necessary.
(DB._clearVersions):
Add.
(DB._updateVersions):
Skip processing if _clearVersions has not been invoked.
(DB.getVersion, DB.releaseContainsPackage, DB._synthesizeReleases):
Obsolete, remove.
(test):
Update.
lib/python/bugs.py (CANFile, CVEFile):
Split into two classes, which handle the differences between the two
files.
bin/check-syntax:
Update accordingly.
bin/update-db:
New database update script. Implements incremental updates.
Makefile:
Remove references to bin/update-packages. Simplify drastically.
Modified: Makefile
==================================================================--- Makefile
2005-09-15 09:14:17 UTC (rev 1993)
+++ Makefile 2005-09-15 10:11:44 UTC (rev 1994)
@@ -14,29 +14,9 @@
PACKAGE_FILES = $(wildcard data/packages/*_Sources) \
$(wildcard data/packages/*_Packages)
-all: stamps/bug-lists-imported stamps/packages-imported \
- stamps/calc-vulns
+all:
+ $(PYTHON) bin/update-db
-stamps/bug-lists-imported: bin/update-bug-list-db \
- $(BUG_LISTS) $(PYTHON_MODULES)
- $(PYTHON) bin/update-bug-list-db
- touch $@
-
-# No dependencies on the Python files. This part of the code should
-# be quite stable. We only run the packages import if "make
-# update-packages" has been invoked before.
-stamps/packages-imported: $(PACKAGE_FILES)
- if test -e stamps/packages-downloaded ; then \
- $(PYTHON) bin/update-packages import ; \
- fi
- touch $@
-
-stamps/calc-vulns: stamps/bug-lists-imported stamps/packages-imported
- if test -e stamps/packages-downloaded ; then \
- $(PYTHON) bin/update-vulnerabilities ; \
- fi
- touch $@
-
clean:
-rm data/security.db
-rm stamps/*-*
@@ -64,9 +44,43 @@
$(PYTHON) bin/check-syntax DTSA data/DTSA/list
touch $@
-.PHONY: update-packages
+.PHONY: update-packages update-etch-security
update-packages:
- $(PYTHON) bin/update-packages download $(MIRROR) $(RELEASES)
- $(PYTHON) bin/update-packages download \
- http://security.debian.org/ $(SECURITY_RELEASES)
+ set -e ; for rel in woody sarge etch sid ; do \
+ for archive in main contrib non-free ; do \
+ $(PYTHON) bin/apt-update-file \
+ $(MIRROR)/dists/$$rel/$$archive/source/Sources \
+ data/packages/$${rel}__$${archive}_Sources ; \
+ done ; \
+ for arch in i386 ia64 ; do \
+ for archive in main contrib non-free ; do \
+ $(PYTHON) bin/apt-update-file \
+ $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \
+ data/packages/$${rel}__$${archive}_$${arch}_Packages ; \
+ done ; \
+ done ; \
+ done
touch stamps/packages-downloaded
+
+ST_MIRROR =
http://secure-testing.debian.net/debian-secure-testing/dists/etch/security-updates
+ST_FILE = data/packages/etch_security_
+update-testing-security:
+ $(PYTHON) bin/apt-update-file \
+ $(ST_MIRROR)/main/source/Sources $(ST_FILE)main_Sources
+ $(PYTHON) bin/apt-update-file \
+ $(ST_MIRROR)/main/binary-i386/Packages $(ST_FILE)main_i386_Packages
+ $(PYTHON) bin/apt-update-file \
+ $(ST_MIRROR)/main/binary-ia64/Packages $(ST_FILE)main_ia64_Packages
+
+SEC_MIRROR = http://security.debian.org/dists
+update-security:
+ for archive in woody sarge ; do \
+ $(PYTHON) bin/apt-update-file \
+ $(SEC_MIRROR)/$$archive/updates/main/source/Sources \
+ data/packages/$${archive}_security_main_Sources ; \
+ for arch in i386 ia64 ; do \
+ $(PYTHON) bin/apt-update-file \
+ $(SEC_MIRROR)/$$archive/updates/main/binary-$$arch/Packages \
+ data/packages/$${archive}_security_main_$${arch}_Packages ; \
+ done ; \
+ done
Modified: bin/check-syntax
==================================================================---
bin/check-syntax 2005-09-15 09:14:17 UTC (rev 1993)
+++ bin/check-syntax 2005-09-15 10:11:44 UTC (rev 1994)
@@ -46,7 +46,7 @@
def parse_CAN(name):
- do_parse(bugs.CVEFile(name))
+ do_parse(bugs.CANFile(name))
def parse_CVE(name):
f = bugs.CVEFile(name)
Deleted: bin/update-bug-list-db
==================================================================---
bin/update-bug-list-db 2005-09-15 09:14:17 UTC (rev 1993)
+++ bin/update-bug-list-db 2005-09-15 10:11:44 UTC (rev 1994)
@@ -1,55 +0,0 @@
-#!/usr/bin/python
-
-import os
-import os.path
-import string
-import sys
-
-def setup_paths():
- check_file = ''lib/python/debian_support.py''
- path = os.getcwd()
- while 1:
- if os.path.exists("%s/%s" % (path, check_file)):
- sys.path = [path + ''/lib/python''] + sys.path
- return path
- idx = string.rfind(path, ''/'')
- if idx == -1:
- raise ImportError, "could not setup paths"
- path = path[0:idx]
-root_path = setup_paths()
-
-import bugs
-import debian_support
-import security_db
-
-db_file = root_path + ''/data/security.db''
-new_file = not os.path.exists(db_file)
-db = security_db.DB(db_file)
-if new_file:
- db.initSchema()
-cursor = db.writeTxn()
-db.deleteBugs(cursor)
-try:
- db.insertBugs(cursor, bugs.CVEFile(root_path +
''/data/CAN/list''))
- db.insertBugs(cursor, bugs.CVEFile(root_path +
''/data/CVE/list'',
- no_version_needs_note=False))
- db.insertBugs(cursor, bugs.DSAFile(root_path +
''/data/DSA/list''))
- db.insertBugs(cursor, bugs.DTSAFile(root_path +
''/data/DTSA/list''))
-except debian_support.ParseError, e:
- db.rollback(cursor)
- e.printOut(sys.stderr)
- sys.exit(1)
-except security_db.InsertError, e:
- db.rollback(cursor)
- for err in e.errors:
- print err
- sys.exit(1)
-
-warnings = db.finishBugs(cursor)
-if warnings:
- db.rollback(cursor)
- for x in warnings:
- print x
- sys.exit(1)
-else:
- db.commit(cursor)
Added: bin/update-db
==================================================================---
bin/update-db 2005-09-15 09:14:17 UTC (rev 1993)
+++ bin/update-db 2005-09-15 10:11:44 UTC (rev 1994)
@@ -0,0 +1,64 @@
+#!/usr/bin/python
+
+import os
+import os.path
+import string
+import sys
+
+def setup_paths():
+ check_file = ''lib/python/debian_support.py''
+ path = os.getcwd()
+ while 1:
+ if os.path.exists("%s/%s" % (path, check_file)):
+ sys.path = [path + ''/lib/python''] + sys.path
+ return path
+ idx = string.rfind(path, ''/'')
+ if idx == -1:
+ raise ImportError, "could not setup paths"
+ path = path[0:idx]
+os.chdir(setup_paths())
+
+import bugs
+import debian_support
+import security_db
+
+db_file = ''data/security.db''
+try:
+ db = security_db.DB(db_file, verbose=True)
+except security_db.SchemaMismatch:
+ os.unlink(db_file)
+ db = security_db.DB(db_file, verbose=True)
+
+cursor = db.writeTxn()
+
+# Bug lists (CAN/CVE/DSA/DTSA)
+
+try:
+ warnings = db.readBugs(cursor, ''data'')
+except debian_support.ParseError, e:
+ e.printOut(sys.stderr)
+ sys.exit(1)
+except security_db.InsertError, e:
+ for err in e.errors:
+ print err
+ sys.exit(1)
+if warnings:
+ for x in warnings:
+ print x
+ sys.exit(1)
+
+# Packages
+
+db.readPackages(cursor, ''data/packages'')
+
+# Calculate vulnerability information.
+
+warnings = db.calculateVulnerabilities(cursor)
+if warnings:
+ for x in warnings:
+ print x
+ sys.exit(1)
+
+# Everything worked well.
+
+db.commit(cursor)
Property changes on: bin/update-db
___________________________________________________________________
Name: svn:executable
+ *
Deleted: bin/update-packages
==================================================================---
bin/update-packages 2005-09-15 09:14:17 UTC (rev 1993)
+++ bin/update-packages 2005-09-15 10:11:44 UTC (rev 1994)
@@ -1,95 +0,0 @@
-#!/usr/bin/python
-
-# This script downloads and imports Debian package files.
-
-import errno
-import os
-import os.path
-import string
-import sys
-
-def setup_paths():
- check_file = ''lib/python/debian_support.py''
- path = os.getcwd()
- while 1:
- if os.path.exists("%s/%s" % (path, check_file)):
- sys.path = [path + ''/lib/python''] + sys.path
- return path
- idx = string.rfind(path, ''/'')
- if idx == -1:
- raise ImportError, "could not setup paths"
- path = path[0:idx]
-root_path = setup_paths()
-
-import debian_support
-import security_db
-
-def explodeReleases(args):
- for arg in args:
- (release, archs) = arg.split(''='')
- # FIXME: What shall we do with these?
- # if debian_support.internRelease(release) is None:
- # sys.stderr.write("error: unknown release: %s\n" %
release)
- # sys.exit(1)
- yield release, archs.split('','')
-
-archives = (''main'', ''contrib'',
''non-free'')
-
-def nameSources(release, archive):
- return ''%s/data/packages/%s_%s_Sources'' % (root_path,
release, archive)
-
-def namePackages(release, archive, arch):
- return ''%s/data/packages/%s_%s_%s_Packages'' % (root_path,
release,
- archive, arch)
-
-def cmd_download(args):
- url_base = args[0]
- if url_base[-1] != ''/'':
- url_base += ''/''
-
- for release, archs in explodeReleases(args[1:]):
- # Security updates are stored in a different directory.
- if release[-9:] == ''-security'':
- rrel = release[:-9] + ''/updates''
- else:
- rrel = release
-
- for archive in archives:
- print "Updating source package %s/%s" % (release,
archive)
- debian_support.updateFile("%sdists/%s/%s/source/Sources"
- % (url_base, rrel, archive),
- nameSources(release, archive),
- verbose=True)
- for arch in archs:
- print "Updating binary package %s/%s/%s" \
- % (release, archive, arch)
-
debian_support.updateFile("%sdists/%s/%s/binary-%s/Packages"
- % (url_base, rrel, archive, arch),
- namePackages(release, archive, arch),
- verbose=True)
-
-def cmd_import(args):
- db_file = root_path + ''/data/security.db''
- new_file = not os.path.exists(db_file)
- db = security_db.DB(db_file, verbose=True)
- if new_file:
- db.initSchema()
- c = db.writeTxn()
- db.readPackages(c, root_path + ''/data/packages'')
- db.commit(c)
-
-cmds = {"download" : cmd_download,
- "import" : cmd_import}
-
-if len(sys.argv) < 2 or not cmds.has_key(sys.argv[1]):
- sys.stderr.write(\
-"""usage: update-packages download URL-BASE RELEASE=ARCH...
- update-packages import
-""")
- sys.exit(1)
-try:
- cmds[sys.argv[1]](sys.argv[2:])
-except debian_support.ParseError, e:
- e.printOut(sys.stderr)
- sys.exit(1)
-
Deleted: bin/update-vulnerabilities
==================================================================---
bin/update-vulnerabilities 2005-09-15 09:14:17 UTC (rev 1993)
+++ bin/update-vulnerabilities 2005-09-15 10:11:44 UTC (rev 1994)
@@ -1,37 +0,0 @@
-#!/usr/bin/python
-
-# This script recalculates the vulnerability information in the
-# security database.
-
-import errno
-import os
-import os.path
-import string
-import sys
-
-def setup_paths():
- check_file = ''lib/python/debian_support.py''
- path = os.getcwd()
- while 1:
- if os.path.exists("%s/%s" % (path, check_file)):
- sys.path = [path + ''/lib/python''] + sys.path
- return path
- idx = string.rfind(path, ''/'')
- if idx == -1:
- raise ImportError, "could not setup paths"
- path = path[0:idx]
-root_path = setup_paths()
-
-import security_db
-
-db_file = root_path + ''/data/security.db''
-assert os.path.exists(db_file)
-db = security_db.DB(db_file, verbose=True)
-c = db.writeTxn()
-warnings = db.calculateVulnerabilities(c)
-if warnings:
- db.rollback(c)
- for x in warnings:
- print x
- sys.exit(1)
-db.commit(c)
Modified: lib/python/bugs.py
==================================================================---
lib/python/bugs.py 2005-09-15 09:14:17 UTC (rev 1993)
+++ lib/python/bugs.py 2005-09-15 10:11:44 UTC (rev 1994)
@@ -616,15 +616,15 @@
yield Bug(self.file.name, first_lineno, date,
record_name, description,
comments, notes=pkg_notes, xref=xref)
-
-class CVEFile(FileBase):
- """A CVE file, as used by the Debian testing security
team."""
+
+class CANFile(FileBase):
+ """A CAN file, as used by the Debian testing security
team."""
- re_cve =
re.compile(r''^((?:CAN|CVE)-\d{4}-(?:\d{4}|XXXX))\s+(.*?)\s*$'')
+ re_cve =
re.compile(r''^(CAN-\d{4}-(?:\d{4}|XXXX))\s+(.*?)\s*$'')
- def __init__(self, name, fileObj=None, no_version_needs_note=True):
+ def __init__(self, name, fileObj=None):
FileBase.__init__(self, name, fileObj)
- self.no_version_needs_note = no_version_needs_note
+ self.no_version_needs_note = True
def isUniqueName(self, name):
return BugBase.re_cve_name.match(name) is not None
@@ -632,6 +632,34 @@
def matchHeader(self, line):
match = self.re_cve.match(line)
if not match:
+ self.raiseSyntaxError("expected CAN record, got: %s" %
`line`)
+ (record_name, description) = match.groups()
+ (cve, desc) = match.groups()
+ if desc:
+ if desc[0] == ''('':
+ if desc[-1] <> '')'':
+ self.raiseSyntaxError("missing closing
parenthesis")
+ else:
+ desc = desc[1:-1]
+ elif desc[0] == ''['':
+ if desc[-1] <> '']'':
+ self.raiseSyntaxError("missing closing bracket")
+ else:
+ desc = desc[1:-1]
+ return (None, cve, desc)
+
+class CVEFile(FileBase):
+ """A CVE file, as used by the Debian testing security
team."""
+
+ re_cve = re.compile(r''^(CVE-\d{4}-\d{4})\s+(.*?)\s*$'')
+
+ def __init__(self, name, fileObj=None):
+ FileBase.__init__(self, name, fileObj)
+ self.no_version_needs_note = False
+
+ def matchHeader(self, line):
+ match = self.re_cve.match(line)
+ if not match:
self.raiseSyntaxError("expected CVE record, got: %s" %
`line`)
(record_name, description) = match.groups()
(cve, desc) = match.groups()
Modified: lib/python/security_db.py
==================================================================---
lib/python/security_db.py 2005-09-15 09:14:17 UTC (rev 1993)
+++ lib/python/security_db.py 2005-09-15 10:11:44 UTC (rev 1994)
@@ -73,6 +73,11 @@
result.sort()
return result
+class SchemaMismatch(Exception):
+ """Raised to indicate a schema mismatch.
+
+ The caller is expected to remove and regenerate the
database."""
+
class DB:
"""Access to the security database.
@@ -91,6 +96,15 @@
''sarge'' :
''stable'',
''woody'':
''oldstable''}
+ c = self.cursor()
+ for (v,) in c.execute("PRAGMA user_version"):
+ if v == 0:
+ self.initSchema()
+ if v <> 1:
+ raise SchemaMismatch, `v`
+ return
+ assert False
+
def cursor(self):
"""Creates a new database cursor.
@@ -140,22 +154,24 @@
"""CREATE TABLE source_packages
(name TEXT NOT NULL,
release TEXT NOT NULL,
+ subrelease TEXT NOT NULL,
archive TEXT NOT NULL,
version TEXT NOT NULL,
version_id INTEGER NOT NULL DEFAULT 0,
- PRIMARY KEY (name, release, archive))""")
+ PRIMARY KEY (name, release, subrelease,
archive))""")
cursor.execute(
"""CREATE TABLE binary_packages
(name TEXT NOT NULL,
release TEXT NOT NULL,
+ subrelease TEXT NOT NULL,
archive TEXT NOT NULL,
version TEXT NOT NULL,
source TEXT NOT NULL,
source_version TEXT NOT NULL,
archs TEXT NOT NULL,
version_id INTEGER NOT NULL DEFAULT 0,
- PRIMARY KEY (name, release, archive, version, source,
+ PRIMARY KEY (name, release, subrelease, archive, version, source,
source_version))""")
cursor.execute(
"""CREATE INDEX binary_packages_source
@@ -225,8 +241,13 @@
"""CREATE INDEX binary_package_status_package
ON binary_package_status(package)""")
- self.commit(cursor)
+ # Put this at the end. Any exception will leave the schema
+ # version at 0, so we automatically recreate the schema once
+ # the application is started after the underlying error has
+ # been fixed.
+ cursor.execute("PRAGMA user_version = 1")
+
def filePrint(self, filename):
"""Returns a fingerprint string for
filename."""
@@ -284,26 +305,23 @@
"SELECT inodeprint, parsed FROM inodeprints WHERE file =
?",
(filename,)):
if old_print == current_print:
- return cPickle.load(cStringIO.StringIO(contents))
+ return (True, cPickle.load(cStringIO.StringIO(contents)))
result = do_parse(debian_support.PackageFile(filename))
cursor.execute("""UPDATE inodeprints SET inodeprint
= ?, parsed = ?
WHERE file = ?""", (current_print, toString(result),
filename))
- return result
+ return (False, result)
# No inodeprints entry, load file and add one.
result = do_parse(debian_support.PackageFile(filename))
cursor.execute("""INSERT INTO inodeprints (file,
inodeprint, parsed)
VALUES (?, ?, ?)""", (filename, current_print,
toString(result)))
- return result
+ return (False, result)
def readPackages(self, cursor, directory):
"""Reads a directory of package files."""
if self.verbose:
print "readPackages:"
- print " deleting old data"
- cursor.execute("DELETE FROM source_packages")
- cursor.execute("DELETE FROM binary_packages")
self._readSourcePackages(cursor, directory)
self._readBinaryPackages(cursor, directory)
@@ -314,7 +332,7 @@
def _readSourcePackages(self, cursor, directory):
"""Reads from directory with source package
files."""
- re_sources =
re.compile(r''.*/([a-z-]+)_([a-z-]+)_Sources$'')
+ re_sources =
re.compile(r''.*/([a-z-]+)_([a-z-]*)_([a-z-]+)_Sources$'')
if self.verbose:
@@ -325,48 +343,71 @@
if match is None:
raise ValueError, "invalid file name: " + `filename`
- (release, archive) = match.groups()
- parsed = self._parseFile(cursor, filename)
+ (release, subrelease, archive) = match.groups()
+ (unchanged, parsed) = self._parseFile(cursor, filename)
+ if unchanged:
+ continue
+ cursor.execute(
+ """DELETE FROM source_packages
+ WHERE release = ? AND subrelease = ? AND archive =
?""",
+ (release, subrelease, archive))
+ self._clearVersions(cursor)
+
def gen():
for (name, version, source, source_version) in parsed:
assert source is None
assert source_version is None
- yield name, release, archive, version
+ yield name, release, subrelease, archive, version
cursor.executemany(
"""INSERT INTO source_packages
- (name, release, archive, version) VALUES (?, ?, ?,
?)""",
+ (name, release, subrelease, archive, version)
+ VALUES (?, ?, ?, ?, ?)""",
gen())
def _readBinaryPackages(self, cursor, directory):
"""Reads from a directory with binary package
files."""
re_packages \
- =
re.compile(r''.*/([a-z-]+)_([a-z-]+)_([a-z0-9]+)_Packages$'')
+ = re.compile(
+
r''.*/([a-z-]+)_([a-z-]*)_([a-z-]+)_([a-z0-9]+)_Packages$'')
if self.verbose:
print " reading binary packages"
packages = {}
+ unchanged = True
for filename in glob.glob(directory + ''/*_Packages''):
match = re_packages.match(filename)
if match is None:
raise ValueError, "invalid file name: " + `filename`
- (release, archive, architecture) = match.groups()
- parsed = self._parseFile(cursor, filename)
+ (release, subrelease, archive, architecture) = match.groups()
+ (unch, parsed) = self._parseFile(cursor, filename)
+ unchanged = unchanged and unch
for (name, version, source, source_version) in parsed:
if source is None:
source = name
if source_version is None:
source_version = version
- key = (name, release, archive, version, source, source_version)
+ key = (name, release, subrelease, archive, version,
+ source, source_version)
if packages.has_key(key):
packages[key][architecture] = 1
else:
packages[key] = {architecture : 1}
+ if unchanged:
+ if self.verbose:
+ print " finished (no changes)"
+ return
+
+ if self.verbose:
+ print " deleting old data"
+ cursor.execute("DELETE FROM binary_packages")
+ self._clearVersions(cursor)
+
l = packages.keys()
if len(l) == 0:
@@ -385,41 +426,110 @@
cursor.executemany(
"""INSERT INTO binary_packages
- (name, release, archive, version,
+ (name, release, subrelease, archive, version,
source, source_version, archs)
- VALUES (?, ?, ?, ?, ?, ?, ?)""",
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
gen())
- def deleteBugs(self, cursor):
- """Deletes all record bug reports from the
database."""
- cursor.execute("DELETE FROM package_notes")
- cursor.execute("DELETE FROM debian_bugs")
- cursor.execute("DELETE FROM bugs")
- cursor.execute("DELETE FROM bugs_notes")
- cursor.execute("DELETE FROM bugs_xref")
+ def readBugs(self, cursor, path):
+ if self.verbose:
+ print "readBugs:"
- def insertBugs(self, cursor, source):
- """Reads the CAN/CVE/DSA/DTSA file and writes them to
the database."""
+ def clear_db(filename):
+ cursor.execute(
+ """CREATE TEMPORARY TABLE bugs_to_delete
+ (tbd TEXT NOT NULL PRIMARY KEY)""")
+ cursor.execute(
+ """INSERT INTO bugs_to_delete
+ SELECT name FROM bugs WHERE source_file = ?""",
+ (filename,))
- errors = []
- for bug in source:
- try:
- bug.writeDB(cursor)
- except ValueError, e:
- errors.append("%s: %d: error: %s"
- % (bug.source_file, bug.source_line, e))
- if errors:
- raise InsertError(errors)
+ cursor.execute(
+ """DELETE FROM debian_bugs
+ WHERE EXISTS (SELECT 1
+ FROM package_notes AS p, bugs_to_delete AS b
+ WHERE p.id = debian_bugs.note
+ AND p.bug_name = b.tbd)""")
- def finishBugs(self, cursor):
- """After inserting new bugs, update cross-references.
+ cursor.execute("""DELETE FROM bugs
+ WHERE EXISTS (SELECT * FROM bugs_to_delete
+ WHERE tbd = name)""")
+ cursor.execute("""DELETE FROM package_notes
+ WHERE EXISTS (SELECT * FROM bugs_to_delete
+ WHERE tbd = bug_name)""")
+ cursor.execute("""DELETE FROM bugs_notes
+ WHERE EXISTS (SELECT * FROM bugs_to_delete
+ WHERE tbd = bug_name)""")
+ cursor.execute("""DELETE FROM bugs_xref
+ WHERE EXISTS (SELECT * FROM bugs_to_delete
+ WHERE tbd = source)""")
- Returns a list of warning messages."""
+ # The *_status tables are regenerated anyway, no need to
+ # delete them here.
+
+ cursor.execute("""DROP TABLE
bugs_to_delete""")
+
+ self._clearVersions(cursor)
+
+ def do_parse(source):
+ errors = []
+
+ if self.verbose:
+ print " reading " + `source.name`
- warnings = []
+ clear_db(source.name)
+
+ for bug in source:
+ try:
+ bug.writeDB(cursor)
+ except ValueError, e:
+ errors.append("%s: %d: error: %s"
+ % (bug.source_file, bug.source_line, e))
+ if errors:
+ raise InsertError(errors)
- # Check that there are no CAN/CVE collisions.
+ def read_one(source):
+ filename = source.name
+ current_print = self.filePrint(filename)
+ for (old_print,) in cursor.execute(
+ "SELECT inodeprint FROM inodeprints WHERE file = ?",
+ (filename,)):
+ if old_print == current_print:
+ return False
+ do_parse(source)
+ cursor.execute(
+ "UPDATE inodeprints SET inodeprint = ? WHERE file =
?",
+ (current_print, filename))
+ return True
+
+ # No inodeprints entry, load file and add one.
+ do_parse(source)
+ cursor.execute(
+ "INSERT INTO inodeprints (file, inodeprint) VALUES (?,
?)",
+ (filename, current_print))
+ return True
+
+ unchanged = True
+ if read_one(bugs.CANFile(path + ''/CAN/list'')):
+ unchanged = False
+ if read_one(bugs.CVEFile(path + ''/CVE/list'')):
+ unchanged = False
+ if read_one(bugs.DSAFile(path + ''/DSA/list'')):
+ unchanged = False
+ if read_one(bugs.DTSAFile(path + ''/DTSA/list'')):
+ unchanged = False
+
+ if unchanged:
+ if self.verbose:
+ print " finished (no changes)"
+ return
+
+ errors = []
+
+ if self.verbose:
+ print " checking CAN/CVE collisions"
+
for b1, b2 in list(cursor.execute\
("""SELECT b1.name, b2.name FROM bugs AS b1, bugs AS
b2
WHERE b1.name LIKE ''CVE-%''
@@ -427,19 +537,22 @@
b1 = bugs.BugFromDB(cursor, b1)
b2 = bugs.BugFromDB(cursor, b2)
- warnings.append("%s:%d: duplicate CVE entries %s and %s"
- % (b1.source_file, b1.source_line,
- b1.name, b2.name))
- warnings.append("%s:%d: location of %s"
- % (b1.source_file, b1.source_line, b1.name))
- warnings.append("%s:%d: location of %s"
- % (b2.source_file, b2.source_line, b2.name))
+ errors.append("%s:%d: duplicate CVE entries %s and %s"
+ % (b1.source_file, b1.source_line,
+ b1.name, b2.name))
+ errors.append("%s:%d: location of %s"
+ % (b1.source_file, b1.source_line, b1.name))
+ errors.append("%s:%d: location of %s"
+ % (b2.source_file, b2.source_line, b2.name))
# Normalize the CAN/CVE references to the entry which is
# actually in the database. After the CAN -> CVE transition,
# this can go away (but we should check that the
# cross-references are valid).
+ if self.verbose:
+ print " normalize CAN/CVE references"
+
for source, target in list(cursor.execute\
("""SELECT source, target FROM bugs_xref
WHERE normalized_target = ''''""")):
@@ -458,11 +571,12 @@
break
if not found:
b = bugs.BugFromDB(cursor, source)
- warnings.append\
+ errors.append\
("%s: %d: reference to unknwown CVE entry %s"
% (b.source_file, b.source_line, target))
- # Check that the DSA/DTSA references are valid.
+ if self.verbose:
+ print " check DSA/DTSA references"
for source, target in list(cursor.execute
("""SELECT source, target FROM bugs_xref
@@ -473,12 +587,16 @@
found = True
if not found:
b = bugs.BugFromDB(cursor, source)
- warnings.append\
+ errors.append\
("%s: %d: reference to unknwown advisory %s"
% (b.source_file, b.source_line, target))
- return warnings
+ if errors:
+ raise InsertErrors(errors)
+ if self.verbose:
+ print " finished"
+
def availableReleases(self, cursor=None):
"""Returns a list of tuples (RELEASE, ARCHIVE,
SOURCES-PRESENT, ARCHITECTURE-LIST)."""
@@ -487,20 +605,21 @@
releases = {}
for r in cursor.execute(
- "SELECT DISTINCT release, archive FROM source_packages"):
+ """SELECT DISTINCT release, subrelease, archive
+ FROM source_packages"""):
releases[r] = (True, [])
- for (rel, archive, archs) in cursor.execute(
- """SELECT DISTINCT release, archive, archs
+ for (rel, subrel, archive, archs) in cursor.execute(
+ """SELECT DISTINCT release, subrelease, archive,
archs
FROM binary_packages"""):
- key = (rel, archive)
+ key = (rel, subrel, archive)
if not releases.has_key(key):
releases[key] = (False, [])
releases[key][1][:] = mergeLists(releases[key][1], archs)
result = []
- for ((rel, archive), (sources, archs)) in releases.items():
- result.append((rel, archive, sources, archs))
+ for ((rel, subrel, archive), (sources, archs)) in releases.items():
+ result.append((rel, subrel, archive, sources, archs))
result.sort()
return result
@@ -517,48 +636,21 @@
WHERE name = source AND version <> source_version
ORDER BY name, release, archive"""))
- def getVersion(self, cursor, release, package):
- """Returns the version number for package in release.
+ def _clearVersions(self, cursor):
+ cursor.execute("DELETE FROM version_linear_order")
- Package can be a source or binary package. Binary package
- versions take precedence.
-
- Security updates etc. are not considered."""
-
- versions = list(cursor.execute(
- """SELECT version FROM binary_packages
- WHERE package = ? AND release = ?""", (package,
release)))
- if versions:
- return min(map(lambda (v,): debian_support.Version(v), versions))
-
- versions = list(cursor.execute(
- """SELECT version FROM source_packages
- WHERE package = ? AND release = ?""", (package,
release)))
- if versions:
- assert len(versions) == 1
- return debian_support.Version(versions[0][0])
-
- return None
-
- def releaseContainsPackage(self, cursor, release, package):
- """Returns True if the source or binary package exists
in release."""
- for (c,) in cursor.execute(
- """SELECT version FROM binary_packages
- WHERE package = ? AND release = ?""", (package,
release)):
- return True
- for (c,) in cursor.execute(
- """SELECT version FROM source_packages
- WHERE package = ? AND release = ?""", (package,
release)):
- return True
- return False
-
def _updateVersions(self, cursor):
"""Updates the linear version table."""
- cursor.execute("DELETE FROM version_linear_order");
+ if self.verbose:
+ print "updateVersions:"
+ for x in cursor.execute("SELECT * FROM version_linear_order LIMIT
1"):
+ if self.verbose:
+ print " finished (no changes)"
+ return
+
if self.verbose:
- print "updateVersions:"
print " reading"
versions = []
@@ -606,66 +698,6 @@
if self.verbose:
print " finished"
- def _synthesizeReleases(self, cursor):
- """Creates the package lists for testing, stable and
oldstable.
-
- These package lists include security updates.
- """
-
- if self.verbose:
- print "synthesizeReleases:"
- print " clear old data"
- print " source packages"
- cursor.execute(
- """DELETE FROM source_packages
- WHERE release IN (''stable'',
''oldstable'', ''testing'')""")
- if self.verbose:
- print " binary packages"
- cursor.execute(
- """DELETE FROM binary_packages
- WHERE release IN (''stable'',
''oldstable'', ''testing'')""")
-
- for (realname, nickname) in self.nicknames.items():
- if self.verbose:
- print " synthesize %s to %s" % (realname, nickname)
- print " source packages"
- cursor.execute(
- """INSERT INTO source_packages
- SELECT name, ?, archive, '''', MAX(version_id)
AS vid
- FROM source_packages WHERE release IN (?, ?)
- GROUP BY name, archive""",
- (nickname, realname, realname + ''-security''))
-
- if self.verbose:
- print " binary packages"
- cursor.execute(
- """INSERT INTO binary_packages
- SELECT DISTINCT name, ?, archive,
- MAX (version_id) AS vid, source, source_version,
- ''''
- FROM binary_packages WHERE release IN (?, ?)
- GROUP BY name, archive, archs""",
- (nickname, realname, realname + ''-security''))
-
- if self.verbose:
- print " patch version strings"
- print " source packages"
- cursor.execute(
- """UPDATE source_packages
- SET version = (SELECT version FROM version_linear_order
- WHERE id = version_id)
- WHERE version = ''''""")
- if self.verbose:
- print " binary packages"
- cursor.execute(
- """UPDATE binary_packages
- SET version = (SELECT version FROM version_linear_order
- WHERE id = version_id)
- WHERE version = ''''""")
-
- if self.verbose:
- print " finished"
-
def calculateVulnerabilities(self, cursor):
"""Calculate vulnerable packages.
@@ -949,19 +981,15 @@
assert mergeLists(''a,c'', [''b'',
''de'']) == [''a'', ''b'',
''c'', ''de'']
import os
- if os.path.exists(''test_security.db''):
- os.unlink(''test_security.db'')
- db = DB(''test_security.db'')
- db.initSchema()
+ db_file = ''test_security.db''
+ try:
+ db = DB(db_file)
+ except SchemaMismatch:
+ os.unlink(db_file)
+ db = DB(db_file)
cursor = db.writeTxn()
- db.deleteBugs(cursor)
- db.insertBugs(cursor,
bugs.CVEFile(''../../data/CAN/list''))
- db.insertBugs(cursor, bugs.CVEFile(''../../data/CVE/list'',
- no_version_needs_note=False))
- db.insertBugs(cursor,
bugs.DSAFile(''../../data/DSA/list''))
- db.insertBugs(cursor,
bugs.DTSAFile(''../../data/DTSA/list''))
- db.finishBugs(cursor)
+ db.readBugs(cursor, ''../../data'')
db.commit(cursor)
b = bugs.BugFromDB(cursor, ''CAN-2005-2491'')