[pacman-dev] [PATCH] pactest: add test.expectfailure option for tests

[Dan McGee]

This will enable us to mark tests we know currently fail to differentiate
them from those that we know should pass. Regressions should be easier to
spot this way.

Signed-off-by: Dan McGee <dan at archlinux.org>
---
 pactest/pmenv.py                 |   31 ++++++++++++++++++++++++++-----
 pactest/pmtest.py                |    1 +
 pactest/tests/fileconflict001.py |    2 ++
 pactest/tests/fileconflict002.py |    2 ++
 pactest/tests/fileconflict004.py |    2 ++
 pactest/tests/sync403.py         |    2 ++
 pactest/tests/trans001.py        |    2 ++
 pactest/tests/upgrade046.py      |    2 ++
 pactest/tests/upgrade051.py      |    2 ++
 9 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/pactest/pmenv.py b/pactest/pmenv.py
index f2327f9..b86abe9 100755
--- a/pactest/pmenv.py
+++ b/pactest/pmenv.py
@@ -86,13 +86,25 @@ class pmenv:
         """
         passed = 0
         tpassed = []
+        failed = 0
         tfailed = []
+        expectedfail = 0
+        texpectedfail = []
+        unexpectedpass = 0
+        tunexpectedpass = []
         for test in self.testcases:
             fail = test.result["fail"]
-            if fail == 0:
+            if fail == 0 and not test.expectfailure:
                 passed += 1
                 tpassed.append(test)
+            elif fail != 0 and test.expectfailure:
+                expectedfail += 1
+                texpectedfail.append(test)
+            elif fail == 0: # and not test.expectfail
+                unexpectedpass += 1
+                tunexpectedpass.append(test)
             else:
+                failed += 1
                 tfailed.append(test)
 
         def _printtest(t):
@@ -114,17 +126,26 @@ class pmenv:
         print "=========="*8
         print "Results"
         print "----------"*8
+        print " Passed:"
         for test in tpassed: _printtest(test)
         print "----------"*8
+        print " Expected Failures:"
+        for test in texpectedfail: _printtest(test)
+        print "----------"*8
+        print " Unexpected Passes:"
+        for test in tunexpectedpass: _printtest(test)
+        print "----------"*8
+        print " Failed:"
         for test in tfailed: _printtest(test)
         print "----------"*8
 
         total = len(self.testcases)
-        failed = total - passed
-        print "TOTAL = %3u" % total
+        print "Total            = %3u" % total
         if total:
-            print "PASS  = %3u (%6.2f%%)" % (passed, float(passed) * 100 / total)
-            print "FAIL  = %3u (%6.2f%%)" % (failed, float(failed) * 100 / total)
+            print "Pass             = %3u (%6.2f%%)" % (passed, float(passed) * 100 / total)
+            print "Expected Fail    = %3u (%6.2f%%)" % (expectedfail, float(expectedfail) * 100 / total)
+            print "Unexpected Pass  = %3u (%6.2f%%)" % (unexpectedpass, float(unexpectedpass) * 100 / total)
+            print "Fail             = %3u (%6.2f%%)" % (failed, float(failed) * 100 / total)
         print ""
 
 if __name__ == "__main__":
diff --git a/pactest/pmtest.py b/pactest/pmtest.py
index e8f6fa8..f31563b 100755
--- a/pactest/pmtest.py
+++ b/pactest/pmtest.py
@@ -90,6 +90,7 @@ class pmtest:
         # Test rules
         self.rules = []
         self.files = []
+        self.expectfailure = False
         
         if os.path.isfile(self.name):
             execfile(self.name)
diff --git a/pactest/tests/fileconflict001.py b/pactest/tests/fileconflict001.py
index 4c2069e..8c13911 100644
--- a/pactest/tests/fileconflict001.py
+++ b/pactest/tests/fileconflict001.py
@@ -18,3 +18,5 @@ self.args = "-U %s" % " ".join([p.filename() for p in p1, p2])
 self.addrule("PACMAN_RETCODE=1")
 self.addrule("!PKG_EXIST=pkg1")
 self.addrule("!PKG_EXIST=pkg2")
+
+self.expectfailure = True
diff --git a/pactest/tests/fileconflict002.py b/pactest/tests/fileconflict002.py
index c54f6da..f70873b 100644
--- a/pactest/tests/fileconflict002.py
+++ b/pactest/tests/fileconflict002.py
@@ -14,3 +14,5 @@ self.args = "-U %s" % " ".join([p.filename() for p in p1, p2])
 self.addrule("PACMAN_RETCODE=1")
 self.addrule("!PKG_EXIST=pkg1")
 self.addrule("!PKG_EXIST=pkg2")
+
+self.expectfailure = True
diff --git a/pactest/tests/fileconflict004.py b/pactest/tests/fileconflict004.py
index a5347cc..2396ced 100644
--- a/pactest/tests/fileconflict004.py
+++ b/pactest/tests/fileconflict004.py
@@ -17,3 +17,5 @@ self.addrule("PACMAN_RETCODE=0")
 self.addrule("PKG_EXIST=pkg1")
 self.addrule("PKG_VERSION=pkg1|2.0-1")
 self.addrule("FILE_TYPE=test|link")
+
+self.expectfailure = True
diff --git a/pactest/tests/sync403.py b/pactest/tests/sync403.py
index b8d0101..d8ab763 100644
--- a/pactest/tests/sync403.py
+++ b/pactest/tests/sync403.py
@@ -19,3 +19,5 @@ self.addrule("PACMAN_RETCODE=0")
 self.addrule("PKG_EXIST=pkg1")
 self.addrule("!PKG_EXIST=pkg2")
 self.addrule("PKG_EXIST=pkg3")
+
+self.expectfailure = True
diff --git a/pactest/tests/trans001.py b/pactest/tests/trans001.py
index b9889b1..b3d7883 100644
--- a/pactest/tests/trans001.py
+++ b/pactest/tests/trans001.py
@@ -18,3 +18,5 @@ self.addrule("PACMAN_RETCODE=1")
 self.addrule("!PKG_EXIST=pkg1")
 self.addrule("PKG_EXIST=pkg2")
 self.addrule("PKG_EXIST=pkg3")
+
+self.expectfailure = True
diff --git a/pactest/tests/upgrade046.py b/pactest/tests/upgrade046.py
index 60164b7..1239064 100644
--- a/pactest/tests/upgrade046.py
+++ b/pactest/tests/upgrade046.py
@@ -29,3 +29,5 @@ self.addrule("FILE_MODIFIED=bin/dummy")
 self.addrule("FILE_MODIFIED=bin/foobar")
 self.addrule("FILE_EXIST=usr/share/file")
 self.addrule("FILE_MODIFIED=usr/share/file")
+
+self.expectfailure = True
diff --git a/pactest/tests/upgrade051.py b/pactest/tests/upgrade051.py
index e8c69eb..4346c24 100644
--- a/pactest/tests/upgrade051.py
+++ b/pactest/tests/upgrade051.py
@@ -13,3 +13,5 @@ self.args = "-U %s" % p.filename()
 self.addrule("PACMAN_RETCODE=0")
 self.addrule("!PKG_EXIST=pkg1")
 self.addrule("PKG_EXIST=pkg2")
+
+self.expectfailure = True
-- 
1.5.6.4