Author: fw
Date: 2010-05-07 19:26:36 +0000 (Fri, 07 May 2010)
New Revision: 14625
Modified:
lib/python/parsers.py
Log:
lib/python/parsers.py: implement the inner annotation parsers
Modified: lib/python/parsers.py
==================================================================---
lib/python/parsers.py 2010-05-07 18:51:15 UTC (rev 14624)
+++ lib/python/parsers.py 2010-05-07 19:26:36 UTC (rev 14625)
@@ -18,8 +18,10 @@
import operator
import re
+import debian_support
+import regexpcase
+import xcollections
import xpickle
-import debian_support
FORMAT = "1"
@@ -60,6 +62,135 @@
data[pkg_name] = pkg_version
return data
+def _sortedtuple(seq):
+ l = list(seq)
+ l.sort()
+ return tuple(l)
+
+Message = xcollections.namedtuple("Message", "file line level
message")
+def addmessage(messages, file, line, level, msg):
+ if level not in ("error", "warning"):
+ raise ValueError("invalid message level: " + repr(level))
+ messages.append(Message(file, line, level, msg))
+
+FlagAnnotation = xcollections.namedtuple("FlagAnnotation", "line
type")
+StringAnnotation = xcollections.namedtuple("StringAnnotation",
+ "line type description")
+XrefAnnotation = xcollections.namedtuple("XrefAnnotation", "line
type bugs")
+PackageAnnotation = xcollections.namedtuple(
+ "PackageAnnotation",
+ "line type release package kind version description "
+ + "urgency debian_bugs bug_filed")
+
+def _annotationdispatcher():
+ # Parser for inner annotations, like (bug #1345; low)
+ urgencies=set("unimportant low medium high".split())
+ @regexpcase.rule(''(bug filed|%s)'' %
''|''.join(urgencies))
+ def innerflag(groups, file, line, messages, flags, bugs):
+ f = groups[0]
+ if f in flags:
+ addmessage(messages, file, line, "error",
+ "duplicate flag: " + repr(f))
+ else:
+ flags.add(f)
+ @regexpcase.rule(r''bug #(\d+)'')
+ def innerbug(groups, file, line, messages, flags, bugs):
+ no = int(groups[0])
+ if no in bugs:
+ messages.add(file, line, "error",
+ "duplicate bug number: " + groups[0])
+ else:
+ bugs.add(no)
+ def innerdefault(text, file, line, messages, flags, bugs):
+ addmessage(messages, file, line, "error",
+ "invalid inner annotation: " + repr(text))
+ innerdispatch = regexpcase.RegexpCase((innerflag, innerbug),
+ default=innerdefault)
+
+ def parseinner(file, line, messages, inner):
+ if not inner:
+ return (None, (), False)
+ flags = set()
+ bugs = set()
+ for innerann in inner.split(";"):
+ innerdispatch(innerann.strip(), file, line, messages, flags, bugs)
+
+ urgency = urgencies.intersection(flags)
+ if urgency:
+ if len(urgency) > 1:
+ addmessage(messages, file, line, "error",
+ "multiple urgencies: " + ",
".join(urgency))
+ else:
+ urgency = urgency.pop()
+ else:
+ urgency = None
+
+ bug_filed = "bug filed" in flags
+ if bugs and bug_filed:
+ addmessage(messages, file, line, "error",
+ "''bug filed'' and bug numbers
listed")
+ bug_filed = False
+
+ return (urgency, _sortedtuple(bugs), bug_filed)
+
+ # Parsers for indented annotations (NOT-FOR-US:, " - foo
<unfixed>" etc.)
+
+
@regexpcase.rule(r''(?:\[([a-z]+)\]\s)?-\s([A-Za-z0-9:.+-]+)\s*''
+ +
r''(?:\s([A-Za-z0-9:.+~-]+)\s*)?(?:\s\((.*)\))?'')
+ def package_version(groups, file, line, messages, anns):
+ release, package, version, inner = groups
+ inner = parseinner(file, line, messages, inner)
+ if version is None:
+ kind = "unfixed"
+ else:
+ kind = "fixed"
+ anns.append(PackageAnnotation(
+ *((line, "package", release, package, kind, version,
None)
+ + inner)))
+
+ pseudo_freetext = "no-dsa not-affected".split()
+ pseudo_struct = set("unfixed removed end-of-life itp
undetermined".split())
+
@regexpcase.rule(r''(?:\[([a-z]+)\]\s)?-\s([A-Za-z0-9:.+-]+)''
+ +
r''\s+<([a-z-]+)>\s*(?:\s\((.*)\))?'')
+ def package_pseudo(groups, file, line, messages, anns):
+ release, package, version, inner = groups
+ if version in pseudo_freetext:
+ anns.append(PackageAnnotation(
+ line, "package", release, package, version, None,
inner,
+ None, (), False))
+ elif version in pseudo_struct:
+ inner = parseinner(file, line, messages, inner)
+ if version == "itp" and not inner[1]:
+ addmessage(messages, file, line, "error",
+ "<itp> needs Debian bug reference")
+ anns.append(PackageAnnotation(
+ *((line, "package", release, package, version,
None, None)
+ + inner)))
+ else:
+ addmessage(messages, file, line, "error",
+ "invalid pseudo-version: " + repr(version))
+
+ @regexpcase.rule(r''\{(.*)\}'')
+ def xref(groups, file, line, messages, anns):
+ x = _sortedtuple(groups[0].strip().split())
+ if x:
+ anns.append(XrefAnnotation(line, "xref", x))
+ else:
+ addmessage(messages, file, line, "error", "empty
cross-reference")
+
+ return regexpcase.RegexpCase(
+ ((r''(RESERVED|REJECTED)'',
+ lambda groups, file, line, messages, anns:
+ anns.append(FlagAnnotation(line, groups[0]))),
+ (r''(NOT-FOR-US|NOTE|TODO):\s+(\S.*)'',
+ lambda groups, file, line, messages, anns:
+ anns.append(StringAnnotation(line, *groups))),
+ package_version, package_pseudo, xref),
+ prefix=r"\s+", suffix=r"\s*",
+ default=lambda text, file, line, messages, anns:
+ addmessage(messages, file, line, "error", "invalid
annotation"))
+_annotationdispatcher = _annotationdispatcher()
+
def _test():
o = binarypackages("../../data/packages/sid__main_i386_Packages")
assert type(o) == type(())
@@ -69,5 +200,84 @@
assert type(o) == type({})
assert "bash" in o
+ for (line, res, xmsgs) in [
+ ('' - foo <unfixed>'',
+ PackageAnnotation(17, "package", None, "foo",
"unfixed", None,
+ None, None, (), False), ()),
+ ('' - foo'',
+ PackageAnnotation(17, "package", None, "foo",
"unfixed", None,
+ None, None, (), False), ()),
+ ('' [lenny] - foo <unfixed>'',
+ PackageAnnotation(17, "package", "lenny",
"foo", "unfixed", None,
+ None, None, (), False), ()),
+ ('' [lenny] - foo <undetermined> (bug
#1234)'',
+ PackageAnnotation(17, "package", "lenny",
"foo", "undetermined",
+ None, None, None, (1234,), False), ()),
+ ('' [lenny] - foo <itp> (bug #1234)'',
+ PackageAnnotation(17, "package", "lenny",
"foo", "itp", None,
+ None, None, (1234,), False), ()),
+ ('' [lenny] - foo <itp>'',
+ PackageAnnotation(17, "package", "lenny",
"foo", "itp", None,
+ None, None, (), False),
+ (Message("CVE", 17, "error",
+ "<itp> needs Debian bug reference"),)),
+ ('' [lenny] - foo 1.0'',
+ PackageAnnotation(17, "package", "lenny",
"foo", "fixed", "1.0" ,
+ None, None, (), False), ()),
+ ('' [lenny] - foo <unfixed> (bug filed)'',
+ PackageAnnotation(17, "package", "lenny",
"foo", "unfixed", None,
+ None, None, (), True), ()),
+ ('' [lenny] - foo <unfixed> (bug filed; bug
#1234)'',
+ PackageAnnotation(17, "package", "lenny",
"foo", "unfixed", None,
+ None, None, (1234,), False),
+ (Message("CVE", 17, "error",
+ "''bug filed'' and bug numbers
listed"),)),
+ ('' [lenny] - foo <unfixed> (low)'',
+ PackageAnnotation(17, "package", "lenny",
"foo", "unfixed", None,
+ None, "low", (), False), ()),
+ ('' [lenny] - foo <unfixed> (low; low)'',
+ PackageAnnotation(17, "package", "lenny",
"foo", "unfixed", None,
+ None, "low", (), False),
+ (Message("CVE", 17, "error", "duplicate
flag: ''low''"),)),
+ ('' [lenny] - foo <unfixed> (bug #1234;
garbled)'',
+ PackageAnnotation(17, "package", "lenny",
"foo", "unfixed", None,
+ None, None, (1234,), False),
+ (Message("CVE", 17, "error",
+ "invalid inner annotation:
''garbled''"),)),
+ ('' [lenny] - foo <no-dsa> (explanation goes
here)'',
+ PackageAnnotation(17, "package", "lenny",
"foo", "no-dsa", None,
+ "explanation goes here", None, (),
False), ()),
+ ('' [lenny] - foo <not-affected> (explanation goes
here)'',
+ PackageAnnotation(17, "package", "lenny",
"foo", "not-affected",
+ None,
+ "explanation goes here", None, (),
False), ()),
+ (''\t{CVE-2009-1234 CVE-2009-1235}'',
+ XrefAnnotation(17, "xref",
+ tuple("CVE-2009-1234
CVE-2009-1235".split())),
+ ()),
+ (''\t{}'', None,
+ (Message("CVE", 17, "error", "empty
cross-reference"),)),
+ ('' NOT-FOR-US: Plan 9'',
+ StringAnnotation(17, "NOT-FOR-US", "Plan 9"),
()),
+ ('' TODO: to-do'', StringAnnotation(17,
"TODO", "to-do"), ()),
+ ('' NOTE: note'', StringAnnotation(17,
"NOTE", "note"), ()),
+ ('' RESERVED'', FlagAnnotation(17,
''RESERVED''), ()),
+ ('' REJECTED'', FlagAnnotation(17,
''REJECTED''), ()),
+ ('' garbled'', None,
+ (Message("CVE", 17, "error", "invalid
annotation"),)),
+ ('' [lenny] - foo <garbled> (bug #1234)'',
None,
+ (Message("CVE", 17, "error",
+ "invalid pseudo-version:
''garbled''"),)),
+ ]:
+ anns = []
+ msgs = []
+ _annotationdispatcher(line, "CVE", 17, msgs, anns)
+ assert tuple(msgs) == xmsgs, repr(msgs)
+ if anns:
+ r = anns[0]
+ else:
+ r = None
+ assert r == res, repr(anns)
+
if __name__ == "__main__":
_test()