Florian Weimer
2005-Sep-13 09:12 UTC
[Secure-testing-commits] r1947 - / bin data data/packages lib/python
Author: fw
Date: 2005-09-13 09:12:19 +0000 (Tue, 13 Sep 2005)
New Revision: 1947
Added:
bin/update-packages
data/packages/
Modified:
Makefile
lib/python/security_db.py
Log:
Add support for downloading Packages and Sources files.
After this change, you must run "make clean update-packages all" to
download the package data.
lib/python/security_db.py(DB.initSchema):
Add table inodeprints.
(DB.filePrint, DB._maybeUpdate, DB.maybeUpdateSources,
DB.maybeUpdatePackages):
Add.
(test):
Update to new file locations.
bin/update-packages:
New file. Used by the makefile.
data/packages:
New directory. Used to store the downloaded files.
Makefile:
Add "update-packages" and "clean" targets, and the
necessary targets
to support them.
Modified: Makefile
==================================================================--- Makefile
2005-09-13 09:03:54 UTC (rev 1946)
+++ Makefile 2005-09-13 09:12:19 UTC (rev 1947)
@@ -2,13 +2,38 @@
PYTHON_MODULES = $(wildcard lib/python/*.py)
BUG_LISTS = $(wildcard data/*/list)
-all: stamps/bug-lists-imported
+# Adjust these if necessary. The architecture selection is arther
+# arbitrary at the moment. More architectures can be added later.
+MIRROR = http://merkel.debian.org/~aba/debian/
+RELEASES_with_proposed_updates = sarge=i386,ia64 etch=i386,ia64
+RELEASES = sid=i386,ia64 $(RELEASES_with_proposed_updates) \
+ $(subst =,-proposed-updates=, $(RELEASES_with_proposed_updates))
+SECURITY_RELEASES = sarge-security=i386,ia64
+PACKAGE_FILES = $(wildcard data/packages/*_Sources) \
+ $(wildcard data/packages/*_Packages)
+
+all: stamps/bug-lists-imported stamps/packages-imported
+
stamps/bug-lists-imported: bin/update-bug-list-db \
$(BUG_LISTS) $(PYTHON_MODULES)
$(PYTHON) bin/update-bug-list-db
touch $@
+# No dependencies on the Python files. This part of the code should
+# be quite stable. We only run the packages import if "make
+# update-packages" has been invoked before.
+stamps/packages-imported: $(PACKAGE_FILES)
+ if test -e stamps/packages-downloaded ; then \
+ $(PYTHON) bin/update-packages import \
+ $(RELEASES) $(SECURITY_RELEASES) ; \
+ fi
+ touch $@
+
+clean:
+ -rm data/security.db
+ -rm stamps/*-*
+
.PHONY: check check-syntax
test check: check-syntax
@@ -31,3 +56,10 @@
stamps/DTSA-syntax: data/DTSA/list bin/check-syntax $(PYTHON_MODULES)
$(PYTHON) bin/check-syntax DTSA data/DTSA/list
touch $@
+
+.PHONY: update-packages
+update-packages:
+ $(PYTHON) bin/update-packages download $(MIRROR) $(RELEASES)
+ $(PYTHON) bin/update-packages download \
+ http://security.debian.org/ $(SECURITY_RELEASES)
+ touch stamps/packages-downloaded
Added: bin/update-packages
==================================================================---
bin/update-packages 2005-09-13 09:03:54 UTC (rev 1946)
+++ bin/update-packages 2005-09-13 09:12:19 UTC (rev 1947)
@@ -0,0 +1,109 @@
+#!/usr/bin/python
+
+# This script download and imports Debian package files.
+
+import errno
+import os
+import os.path
+import string
+import sys
+
+def setup_paths():
+ check_file = ''lib/python/debian_support.py''
+ path = os.getcwd()
+ while 1:
+ if os.path.exists("%s/%s" % (path, check_file)):
+ sys.path = [path + ''/lib/python''] + sys.path
+ return path
+ idx = string.rfind(path, ''/'')
+ if idx == -1:
+ raise ImportError, "could not setup paths"
+ path = path[0:idx]
+root_path = setup_paths()
+
+import debian_support
+import security_db
+
+def explodeReleases(args):
+ for arg in args:
+ (release, archs) = arg.split(''='')
+ # FIXME: What shall we do with these?
+ # if debian_support.internRelease(release) is None:
+ # sys.stderr.write("error: unknown release: %s\n" %
release)
+ # sys.exit(1)
+ yield release, archs.split('','')
+
+archives = (''main'', ''contrib'',
''non-free'')
+
+def nameSources(release, archive):
+ return ''%s/data/packages/%s_%s_Sources'' % (root_path,
release, archive)
+
+def namePackages(release, archive, arch):
+ return ''%s/data/packages/%s_%s_%s_Packages'' % (root_path,
release,
+ archive, arch)
+
+def cmd_download(args):
+ url_base = args[0]
+ if url_base[-1] != ''/'':
+ url_base += ''/''
+
+ for release, archs in explodeReleases(args[1:]):
+ # Security updates are stored in a different directory.
+ if release[-9:] == ''-security'':
+ rrel = release[:-9] + ''/updates''
+ else:
+ rrel = release
+
+ for archive in archives:
+ print "Updating source package %s/%s" % (release,
archive)
+ debian_support.updateFile("%sdists/%s/%s/source/Sources"
+ % (url_base, rrel, archive),
+ nameSources(release, archive),
+ verbose=True)
+ for arch in archs:
+ print "Updating binary package %s/%s/%s" \
+ % (release, archive, arch)
+
debian_support.updateFile("%sdists/%s/%s/binary-%s/Packages"
+ % (url_base, rrel, archive, arch),
+ namePackages(release, archive, arch),
+ verbose=True)
+
+def cmd_import(args):
+ db_file = root_path + ''/data/security.db''
+ db = security_db.DB(db_file)
+ try:
+ for release, archs in explodeReleases(args):
+ for archive in archives:
+ c = db.writeTxn()
+ db.maybeUpdateSources(c, release, archive,
+ nameSources(release, archive))
+ db.commit(c)
+ for arch in archs:
+ c = db.writeTxn()
+ db.maybeUpdatePackages(c, release, archive, arch,
+ namePackages(release,
+ archive, arch))
+ db.commit(c)
+ except IOError, e:
+ if e.errno == errno.ENOENT:
+ sys.stderr.write(
+"""error: could not open a file
+error: please run ''update-packages download'' first
+""")
+ sys.exit(1)
+
+cmds = {"download" : cmd_download,
+ "import" : cmd_import}
+
+if len(sys.argv) < 3 or not cmds.has_key(sys.argv[1]):
+ sys.stderr.write(\
+"""usage: update-packages download URL-BASE RELEASE=ARCH...
+ update-packages import RELEASE=ARCH...
+""")
+ sys.exit(1)
+try:
+ cmds[sys.argv[1]](sys.argv[2:])
+except debian_support.ParseError, e:
+ e.printOut(sys.stderr)
+ sys.exit(1)
+
Property changes on: bin/update-packages
___________________________________________________________________
Name: svn:executable
+ *
Property changes on: data/packages
___________________________________________________________________
Name: svn:ignore
+ *_Sources
*_Packages
Modified: lib/python/security_db.py
==================================================================---
lib/python/security_db.py 2005-09-13 09:03:54 UTC (rev 1946)
+++ lib/python/security_db.py 2005-09-13 09:12:19 UTC (rev 1947)
@@ -29,6 +29,7 @@
import apsw
import bugs
import debian_support
+import os
import re
import sys
import types
@@ -91,6 +92,10 @@
"""Creates the database schema."""
cursor = self.cursor()
+ cursor.execute("""CREATE TABLE inodeprints
+ (file TEXT NOT NULL PRIMARY KEY,
+ inodeprint TEXT NOT NULL)""")
+
cursor.execute("""CREATE TABLE source_packages
(package TEXT NOT NULL,
release TEXT NOT NULL, archive TEXT NOT NULL,
@@ -178,7 +183,6 @@
+ ''VALUES (?, ?, ?, ?)'',
(pkg_name, release, archive, str(pkg_version)))
-
def updatePackages(self, cursor,
release, archive, architecture,
packages):
@@ -240,6 +244,49 @@
str(pkg_version),
pkg_source, str(pkg_source_version)))
+ def filePrint(self, filename):
+ """Returns a fingerprint string for
filename."""
+
+ st = os.stat(filename)
+ # The "1" is a version number which can be used to trigger a
+ # re-read if the code has changed in an incompatible way.
+ return `(st.st_size, st.st_ino, st.st_mtime, 1)`
+
+ def _maybeUpdate(self, cursor, args, filename, action):
+ """Internal routine used for conditional
update."""
+
+ current_print = self.filePrint(filename)
+
+ for (old_print,) in cursor.execute\
+ ("SELECT inodeprint FROM inodeprints WHERE file = ?",
(filename,)):
+ if old_print == current_print:
+ return
+ result = apply(action, (cursor,) + args
+ + (debian_support.PackageFile(filename),))
+ cursor.execute("""UPDATE inodeprints SET inodeprint
= ?
+ WHERE file = ?""", (current_print, filename))
+ return result
+
+ # No inodeprints entry, load file and add one.
+ result = apply(action, (cursor,) + args
+ + (debian_support.PackageFile(filename),))
+ cursor.execute("""INSERT INTO inodeprints (file,
inodeprint)
+ VALUES (?, ?)""", (filename, current_print))
+ return result
+
+
+ def maybeUpdateSources(self, cursor, release, archive, filename):
+ """Reads the Sources file filename if it has been
modified."""
+
+ self._maybeUpdate(cursor, (release, archive), filename,
+ self.updateSources)
+
+ def maybeUpdatePackages(self, cursor, release, archive, arch, filename):
+ """Reads the Packages file filename if it has been
modified."""
+
+ self._maybeUpdate(cursor, (release, archive, arch), filename,
+ self.updatePackages)
+
def deleteBugs(self, cursor):
"""Deletes all record bug reports from the
database."""
cursor.execute("DELETE FROM package_notes")
@@ -383,23 +430,19 @@
def test():
import os
- os.unlink(''test_security.db'')
+ if os.path.exists(''test_security.db''):
+ os.unlink(''test_security.db'')
db = DB(''test_security.db'')
db.initSchema()
- data_prefix = ''../../data/packages/sarge/''
- if False:
- cursor = db.writeTxn()
- db.updateSources(cursor, ''sarge'',
'''',
- debian_support.PackageFile(data_prefix +
''Sources''))
- db.updateSources(cursor, ''sarge'',
''security'',
- debian_support.PackageFile(data_prefix +
''Sources.security''))
- db.updatePackages(cursor, ''sarge'',
'''', ''i386'',
- debian_support.PackageFile(data_prefix +
''Packages.i386''))
- db.updatePackages(cursor, ''sarge'',
''security'', ''i386'',
- debian_support.PackageFile(data_prefix
- +
''Packages.security-i386''))
- db.commit(cursor)
+ data_prefix = ''../../data/packages/''
+ cursor = db.writeTxn()
+ db.updateSources(cursor, ''sarge'',
''main'',
+ debian_support.PackageFile(data_prefix +
''sarge_main_Sources''))
+ db.updatePackages(cursor, ''sarge'',
''main'', ''i386'',
+ debian_support.PackageFile(data_prefix
+ +
''sarge_main_i386_Packages''))
+ db.commit(cursor)
# db.check(cursor)