Use pytest as a test runner instead of unit test

'./test <controller> -s spec1 -s spec2' becomes:
'pytest --controller <controller> -k "spec1 or spec2"'

This uses pytest's test selection, which allows finer selection of which tests
to run (for example, it will allow running all tests but those requiring one
feature or combination of features).
It also allows running only a particular test (or set of test) by
filtering on their name or file name.

pytest also shows a much nicer output while testing (grouped by file,
percentage of tests run, manages the verbosity); and it captures all the output
and only shows it if the test fails, which makes --show-io irrelevant.

conftest.py

@@ -0,0 +1,94 @@
+import importlib
+import sys
+import unittest
+
+import pytest
+import _pytest.unittest
+
+from irctest.cases import _IrcTestCase, BaseClientTestCase, BaseServerTestCase
+from irctest.basecontrollers import BaseClientController, BaseServerController
+
+def pytest_addoption(parser):
+    """Called by pytest, registers CLI options passed to the pytest command."""
+    parser.addoption("--controller", help="Which module to use to run the tested software.")
+    parser.addoption('--openssl-bin', type=str, default='openssl',
+            help='The openssl binary to use')
+
+
+def pytest_configure(config):
+    """Called by pytest, after it parsed the command-line."""
+    module_name = config.getoption("controller")
+
+    if module_name is None:
+        pytest.exit("--controller is required.", 1)
+
+    try:
+        module = importlib.import_module(module_name)
+    except ImportError:
+        pytest.exit('Cannot import module {}'.format(module_name), 1)
+
+    controller_class = module.get_irctest_controller_class()
+    if issubclass(controller_class, BaseClientController):
+        from irctest import client_tests as module
+    elif issubclass(controller_class, BaseServerController):
+        from irctest import server_tests as module
+    else:
+        pytest.exit(
+            r'{}.Controller should be a subclass of '
+            r'irctest.basecontroller.Base{{Client,Server}}Controller'
+            .format(module_name),
+            1
+        )
+    _IrcTestCase.controllerClass = controller_class
+    _IrcTestCase.controllerClass.openssl_bin = config.getoption("openssl_bin")
+    _IrcTestCase.show_io = True  # TODO
+
+
+def pytest_collection_modifyitems(session, config, items):
+    """Called by pytest after finishing the test collection,
+    and before actually running the tests.
+
+    This function filters out client tests if running with a server controller,
+    and vice versa.
+    """
+
+    # First, check if we should run server tests or client tests
+    if issubclass(_IrcTestCase.controllerClass, BaseServerController):
+        server_tests = True
+    elif issubclass(_IrcTestCase.controllerClass, BaseClientController):
+        server_tests = False
+    else:
+        assert False, (
+            f"{_IrcTestCase.controllerClass} inherits neither "
+            f"BaseClientController or BaseServerController"
+        )
+
+    filtered_items = []
+
+    # Iterate over each of the test functions (they are pytest "Nodes")
+    for item in items:
+        # we only use unittest-style test function here
+        assert isinstance(item, _pytest.unittest.TestCaseFunction)
+
+        # unittest-style test functions have the node of UnitTest class as parent
+        assert isinstance(item.parent, _pytest.unittest.UnitTestCase)
+
+        # and that node references the UnitTest class
+        assert issubclass(item.parent.cls, unittest.TestCase)
+
+        # and in this project, TestCase classes all inherit either from BaseClientController
+        # or BaseServerController.
+        if issubclass(item.parent.cls, BaseServerTestCase):
+            if server_tests:
+                filtered_items.append(item)
+        elif issubclass(item.parent.cls, BaseClientTestCase):
+            if not server_tests:
+                filtered_items.append(item)
+        else:
+            assert False, (
+                f"{item}'s class inherits neither BaseServerTestCase "
+                "or BaseClientTestCase"
+            )
+
+    # Finally, rewrite in-place the list of tests pytest will run
+    items[:] = filtered_items

irctest/cases.py

@@ -5,6 +5,8 @@ import tempfile
 import unittest
 import functools
 
+import pytest
+
 from . import runner
 from . import client_mock
 from .irc_utils import capabilities
@@ -463,12 +465,9 @@ class SpecificationSelector:
             raise ValueError('Invalid set of specifications: {}'
                     .format(specifications))
         def decorator(f):
-            @functools.wraps(f)
-            def newf(self):
-                if specifications.isdisjoint(self.testedSpecifications):
-                    raise runner.NotRequiredBySpecifications()
-                if strict and not self.strictTests:
-                    raise runner.SkipStrictTest()
-                return f(self)
-            return newf
+            for specification in specifications:
+                f = getattr(pytest.mark, specification.value)(f)
+            if strict:
+                f = pytest.mark.strict(f)
+            return f
         return decorator

pytest.ini

@@ -0,0 +1,12 @@
+[pytest]
+markers =
+    RFC1459
+    RFC2812
+    RFC-deprecated
+    IRCv3.1
+    IRCv3.2
+    IRCv3.2-deprecated
+    message-tags
+    multiline
+    Oragono
+    strict

test.py

@@ -1,93 +0,0 @@
-#!/usr/bin/env python3
-import sys
-import unittest
-import argparse
-import unittest
-import functools
-import importlib
-from irctest.cases import _IrcTestCase
-from irctest.runner import TextTestRunner
-from irctest.specifications import Specifications
-from irctest.basecontrollers import BaseClientController, BaseServerController
-
-def main(args):
-    try:
-        module = importlib.import_module(args.module)
-    except ImportError:
-        print('Cannot import module {}'.format(args.module), file=sys.stderr)
-        exit(1)
-
-    controller_class = module.get_irctest_controller_class()
-    if issubclass(controller_class, BaseClientController):
-        from irctest import client_tests as module
-    elif issubclass(controller_class, BaseServerController):
-        from irctest import server_tests as module
-    else:
-        print(r'{}.Controller should be a subclass of '
-                r'irctest.basecontroller.Base{{Client,Server}}Controller'
-                .format(args.module),
-                file=sys.stderr)
-        exit(1)
-    _IrcTestCase.controllerClass = controller_class
-    _IrcTestCase.controllerClass.openssl_bin = args.openssl_bin
-    _IrcTestCase.show_io = args.show_io
-    _IrcTestCase.strictTests = not args.loose
-    if args.specification:
-        try:
-            _IrcTestCase.testedSpecifications = frozenset(
-                Specifications.of_name(x) for x in args.specification
-                )
-        except ValueError:
-            print('Invalid set of specifications: {}'
-                    .format(', '.join(args.specification)))
-            exit(1)
-    else:
-        specs = list(Specifications)
-        # remove deprecated specs
-        specs.remove(Specifications.RFCDeprecated)
-        specs.remove(Specifications.IRC302Deprecated)
-
-        _IrcTestCase.testedSpecifications = frozenset(specs)
-    print('Testing {} on specification(s): {}'.format(
-        controller_class.software_name,
-        ', '.join(sorted(map(lambda x:x.value,
-            _IrcTestCase.testedSpecifications)))))
-    ts = module.discover()
-    testRunner = TextTestRunner(
-            verbosity=args.verbose,
-            descriptions=True,
-            )
-    testLoader = unittest.loader.defaultTestLoader
-    result = testRunner.run(ts)
-    if result.failures or result.errors:
-        exit(1)
-    else:
-        exit(0)
-
-
-parser = argparse.ArgumentParser(
-        description='A script to test interoperability of IRC software.')
-parser.add_argument('module', type=str,
-        help='The module used to run the tested program.')
-parser.add_argument('--openssl-bin', type=str, default='openssl',
-        help='The openssl binary to use')
-parser.add_argument('--show-io', action='store_true',
-        help='Show input/outputs with the tested program.')
-parser.add_argument('-v', '--verbose', action='count', default=1,
-        help='Verbosity. Give this option multiple times to make '
-        'it even more verbose.')
-parser.add_argument('-s', '--specification', type=str, action='append',
-        help=('The set of specifications to test the program with. '
-        'Valid values: {}. '
-        'Use this option multiple times to test with multiple '
-        'specifications. If it is not given, defaults to all that '
-        'are not deprecated.')
-        .format(', '.join(x.value for x in Specifications)))
-parser.add_argument('-l', '--loose', action='store_true',
-        help='Disables strict checks of conformity to the specification. '
-        'Strict means the specification is unclear, and the most restrictive '
-        'interpretation is choosen.')
-
-
-args = parser.parse_args()
-main(args)