From 2bc68a22085104dc4a96f3a456ee91041d742308 Mon Sep 17 00:00:00 2001 From: Val Lorentz Date: Tue, 12 Apr 2022 22:36:28 +0200 Subject: [PATCH] Use xfail instead of deselection for known failures (#155) --- Makefile | 112 +----------------- irctest/cases.py | 27 +++++ irctest/client_tests/sasl.py | 1 + irctest/controllers/anope_services.py | 2 + irctest/controllers/atheme_services.py | 2 + irctest/controllers/irc2.py | 8 +- irctest/controllers/ircu2.py | 2 +- irctest/controllers/plexus4.py | 2 +- irctest/dashboard/format.py | 5 + irctest/dashboard/style.css | 3 + irctest/server_tests/account_tag.py | 3 + irctest/server_tests/bot_mode.py | 8 ++ irctest/server_tests/buffering.py | 10 ++ irctest/server_tests/cap.py | 8 ++ irctest/server_tests/chathistory.py | 31 ++++- irctest/server_tests/chmodes/key.py | 15 +++ .../server_tests/connection_registration.py | 8 ++ irctest/server_tests/help.py | 28 +++++ irctest/server_tests/info.py | 3 + irctest/server_tests/invite.py | 8 ++ irctest/server_tests/kick.py | 4 + irctest/server_tests/list.py | 2 + irctest/server_tests/lusers.py | 20 ++++ irctest/server_tests/messages.py | 12 ++ irctest/server_tests/quit.py | 1 + irctest/server_tests/regressions.py | 9 +- irctest/server_tests/sasl.py | 14 +++ irctest/server_tests/statusmsg.py | 10 ++ irctest/server_tests/wallops.py | 3 + irctest/server_tests/who.py | 21 ++++ irctest/server_tests/whois.py | 3 + irctest/server_tests/whowas.py | 37 ++++++ 32 files changed, 308 insertions(+), 114 deletions(-) diff --git a/Makefile b/Makefile index a2d7dd2..e933806 100644 --- a/Makefile +++ b/Makefile @@ -7,102 +7,47 @@ PYTEST_ARGS ?= # Will be appended at the end of the -k argument to pytest EXTRA_SELECTORS ?= -# testPlainLarge fails because it doesn't handle split AUTHENTICATE (reported on IRC) -ANOPE_SELECTORS := \ - and not testPlainLarge - -# buffering tests cannot pass because of issues with UTF-8 handling: https://github.com/DALnet/bahamut/issues/196 -# mask tests in test_who.py fail because they are not implemented. -# some HelpTestCase::*[HELP] tests fail because Bahamut forwards /HELP to HelpServ (but not /HELPOP) -# testWhowasMultiTarget fails because Bahamut returns the results in query order instead of chronological order BAHAMUT_SELECTORS := \ not Ergo \ and not deprecated \ and not strict \ and not IRCv3 \ - and not buffering \ - and not (testWho and not whois and mask) \ - and not testWhoStar \ - and (not HelpTestCase or HELPOP) \ - and not testWhowasMultiTarget \ $(EXTRA_SELECTORS) -# testQuitErrors is very flaky -# AccountTagTestCase.testInvite fails because https://github.com/solanum-ircd/solanum/issues/166 -# testKickDefaultComment fails because it uses the nick of the kickee rather than the kicker. -# testWhoisNumerics[oper] fails because charybdis uses RPL_WHOISSPECIAL instead of RPL_WHOISOPERATOR -# testWhowasNoSuchNick fails because of a typo (solved in https://github.com/solanum-ircd/solanum/commit/08b7b6bd7e60a760ad47b58cbe8075b45d66166f) CHARYBDIS_SELECTORS := \ not Ergo \ and not deprecated \ and not strict \ - and not testQuitErrors \ - and not testKickDefaultComment \ - and not (AccountTagTestCase and testInvite) \ - and not (testWhoisNumerics and oper) \ - and not testWhowasNoSuchNick \ $(EXTRA_SELECTORS) -# testInfoNosuchserver does not apply to Ergo: Ergo ignores the optional argument ERGO_SELECTORS := \ not deprecated \ - and not testInfoNosuchserver \ $(EXTRA_SELECTORS) -# testInviteUnopped is the only strict test that Hybrid fails HYBRID_SELECTORS := \ not Ergo \ - and not testInviteUnopped \ and not deprecated \ $(EXTRA_SELECTORS) -# testBotPrivateMessage and testBotChannelMessage fail because https://github.com/inspircd/inspircd/pull/1910 is not released yet -# WHOWAS tests fail because https://github.com/inspircd/inspircd/pull/1967 and https://github.com/inspircd/inspircd/pull/1968 are not released yet INSPIRCD_SELECTORS := \ not Ergo \ and not deprecated \ and not strict \ - and not testNoticeNonexistentChannel \ - and not testBotPrivateMessage and not testBotChannelMessage \ - and not whowas \ $(EXTRA_SELECTORS) -# buffering tests fail because ircu2 discards the whole buffer on long lines (TODO: refine how we exclude these tests) -# testQuit and testQuitErrors fail because ircu2 does not send ERROR or QUIT -# lusers "full" tests fail because they depend on Modern behavior, not just RFC2812 -# statusmsg tests fail because STATUSMSG is present in ISUPPORT, but it not actually supported as PRIVMSG target -# testKeyValidation[empty] fails because ircu2 returns ERR_NEEDMOREPARAMS on empty keys: https://github.com/UndernetIRC/ircu2/issues/13 -# testKickDefaultComment fails because it uses the nick of the kickee rather than the kicker. -# testEmptyRealname fails because it uses a default value instead of ERR_NEEDMOREPARAMS. # HelpTestCase fails because it returns NOTICEs instead of numerics -# testWhowasCountZero fails: https://github.com/UndernetIRC/ircu2/pull/19 IRCU2_SELECTORS := \ not Ergo \ and not deprecated \ and not strict \ - and not buffering \ - and not testQuit \ - and not (lusers and full) \ - and not statusmsg \ - and not (testKeyValidation and empty) \ - and not testKickDefaultComment \ - and not testEmptyRealname \ - and not HelpTestCase \ - and not testWhowasCountZero \ $(EXTRA_SELECTORS) # same justification as ircu2 -# lusers "unregistered" tests fail because Nefarious doesn't seem to distinguish unregistered users from normal ones +# lusers "unregistered" tests fail because NEFARIOUS_SELECTORS := \ not Ergo \ and not deprecated \ and not strict \ - and not buffering \ - and not testQuit \ - and not (lusers and unregistered) \ - and not statusmsg \ - and not (testKeyValidation and empty) \ - and not testEmptyRealname \ $(EXTRA_SELECTORS) # same justification as ircu2 @@ -110,24 +55,12 @@ SNIRCD_SELECTORS := \ not Ergo \ and not deprecated \ and not strict \ - and not buffering \ - and not testQuit \ - and not (lusers and full) \ - and not statusmsg \ $(EXTRA_SELECTORS) -# testListEmpty and testListOne fails because irc2 deprecated LIST -# testKickDefaultComment fails because it uses the nick of the kickee rather than the kicker. -# testWallopsPrivileges fails because it ignores the command instead of replying ERR_UNKNOWNCOMMAND -# HelpTestCase fails because it returns NOTICEs instead of numerics IRC2_SELECTORS := \ not Ergo \ and not deprecated \ and not strict \ - and not testListEmpty and not testListOne \ - and not testKickDefaultComment \ - and not testWallopsPrivileges \ - and not HelpTestCase \ $(EXTRA_SELECTORS) MAMMON_SELECTORS := \ @@ -136,28 +69,14 @@ MAMMON_SELECTORS := \ and not strict \ $(EXTRA_SELECTORS) -# testKeyValidation[spaces] and testKeyValidation[empty] fail because ngIRCd does not validate them https://github.com/ngircd/ngircd/issues/290 -# testStarNick: wat -# testEmptyRealname fails because it uses a default value instead of ERR_NEEDMOREPARAMS. -# chathistory tests fail because they need nicks longer than 9 chars -# HelpTestCase::*[HELP] fails because it returns NOTICEs instead of numerics NGIRCD_SELECTORS := \ not Ergo \ and not deprecated \ and not strict \ - and not (testKeyValidation and (spaces or empty)) \ - and not testStarNick \ - and not testEmptyRealname \ - and not chathistory \ - and (not HelpTestCase or HELPOP) \ $(EXTRA_SELECTORS) -# testInviteUnopped is the only strict test that Plexus4 fails -# testInviteInviteOnly fails because Plexus4 allows non-op to invite if (and only if) the channel is not invite-only PLEXUS4_SELECTORS := \ not Ergo \ - and not testInviteUnopped \ - and not testInviteInviteOnly \ and not deprecated \ $(EXTRA_SELECTORS) @@ -168,46 +87,27 @@ LIMNORIA_SELECTORS := \ (foo or not foo) \ $(EXTRA_SELECTORS) -# testQuitErrors is too flaky for CI -# testKickDefaultComment fails because solanum uses the nick of the kickee rather than the kicker. SOLANUM_SELECTORS := \ not Ergo \ and not deprecated \ and not strict \ - and not testQuitErrors \ - and not testKickDefaultComment \ $(EXTRA_SELECTORS) +# Same as Limnoria SOPEL_SELECTORS := \ - not testPlainNotAvailable \ + (foo or not foo) \ $(EXTRA_SELECTORS) -# testNoticeNonexistentChannel fails: https://bugs.unrealircd.org/view.php?id=5949 -# regressions::testTagCap fails: https://bugs.unrealircd.org/view.php?id=5948 -# messages::testLineTooLong fails: https://bugs.unrealircd.org/view.php?id=5947 -# testCapRemovalByClient and testNakWhole fail pending https://github.com/unrealircd/unrealircd/pull/148 # Tests marked with arbitrary_client_tags can't pass because Unreal whitelists which tags it relays # Tests marked with react_tag can't pass because Unreal blocks +draft/react https://github.com/unrealircd/unrealircd/pull/149 # Tests marked with private_chathistory can't pass because Unreal does not implement CHATHISTORY for DMs -# testChathistory[BETWEEN] fails: https://bugs.unrealircd.org/view.php?id=5952 -# testChathistory[AROUND] fails: https://bugs.unrealircd.org/view.php?id=5953 -# testWhoAllOpers fails because Unreal skips results when the mask is too broad -# HELP and HELPOP tests fail because Unreal uses custom numerics https://github.com/unrealircd/unrealircd/pull/184 UNREALIRCD_SELECTORS := \ not Ergo \ and not deprecated \ and not strict \ - and not testNoticeNonexistentChannel \ - and not (regressions.py and testTagCap) \ - and not (messages.py and testLineTooLong) \ - and not (cap.py and (testCapRemovalByClient or testNakWhole)) \ - and not (account_tag.py and testInvite) \ and not arbitrary_client_tags \ and not react_tag \ and not private_chathistory \ - and not (testChathistory and (between or around)) \ - and not testWhoAllOpers \ - and not HelpTestCase \ $(EXTRA_SELECTORS) .PHONY: all flakes bahamut charybdis ergo inspircd ircu2 snircd irc2 mammon nefarious limnoria sopel solanum unrealircd @@ -238,7 +138,7 @@ bahamut-anope: --services-controller=irctest.controllers.anope_services \ -m 'services' \ -n 10 \ - -k '$(BAHAMUT_SELECTORS) $(ANOPE_SELECTORS)' + -k '$(BAHAMUT_SELECTORS)' charybdis: $(PYTEST) $(PYTEST_ARGS) \ @@ -275,7 +175,7 @@ inspircd-anope: --controller=irctest.controllers.inspircd \ --services-controller=irctest.controllers.anope_services \ -m 'services' \ - -k '$(INSPIRCD_SELECTORS) $(ANOPE_SELECTORS)' + -k '$(INSPIRCD_SELECTORS)' ircu2: $(PYTEST) $(PYTEST_ARGS) \ @@ -373,4 +273,4 @@ unrealircd-anope: --controller=irctest.controllers.unrealircd \ --services-controller=irctest.controllers.anope_services \ -m 'services' \ - -k '$(UNREALIRCD_SELECTORS) $(ANOPE_SELECTORS)' + -k '$(UNREALIRCD_SELECTORS)' diff --git a/irctest/cases.py b/irctest/cases.py index e782e11..cc15916 100644 --- a/irctest/cases.py +++ b/irctest/cases.py @@ -765,6 +765,33 @@ def skipUnlessHasSasl(f: Callable[..., _TReturn]) -> Callable[..., _TReturn]: return newf +def xfailIf( + condition: Callable[..., bool], reason: str +) -> Callable[[Callable[..., _TReturn]], Callable[..., _TReturn]]: + # Works about the same as skipUnlessHasMechanism + def decorator(f: Callable[..., _TReturn]) -> Callable[..., _TReturn]: + @functools.wraps(f) + def newf(self: _TSelf, *args: Any, **kwargs: Any) -> _TReturn: + if condition(self): + try: + return f(self, *args, **kwargs) + except Exception: + pytest.xfail(reason) + assert False # make mypy happy + else: + return f(self, *args, **kwargs) + + return newf + + return decorator + + +def xfailIfSoftware( + names: List[str], reason: str +) -> Callable[[Callable[..., _TReturn]], Callable[..., _TReturn]]: + return xfailIf(lambda testcase: testcase.controller.software_name in names, reason) + + def mark_services(cls: TClass) -> TClass: cls.run_services = True return pytest.mark.services(cls) # type: ignore diff --git a/irctest/client_tests/sasl.py b/irctest/client_tests/sasl.py index ca771b7..44f5e76 100644 --- a/irctest/client_tests/sasl.py +++ b/irctest/client_tests/sasl.py @@ -61,6 +61,7 @@ class SaslTestCase(cases.BaseClientTestCase): self.assertEqual(m, Message({}, None, "CAP", ["END"])) @cases.skipUnlessHasMechanism("PLAIN") + @cases.xfailIfSoftware(["Sopel"], "Sopel requests SASL PLAIN even if not available") def testPlainNotAvailable(self): """`sasl=EXTERNAL` is advertized, whereas the client is configured to use PLAIN. diff --git a/irctest/controllers/anope_services.py b/irctest/controllers/anope_services.py index f620dd9..e25e88d 100644 --- a/irctest/controllers/anope_services.py +++ b/irctest/controllers/anope_services.py @@ -73,6 +73,8 @@ module {{ name = "ns_cert" }} class AnopeController(BaseServicesController, DirectoryBasedController): """Collaborator for server controllers that rely on Anope""" + software_name = "Anope" + def run(self, protocol: str, server_hostname: str, server_port: int) -> None: self.create_config() diff --git a/irctest/controllers/atheme_services.py b/irctest/controllers/atheme_services.py index 8994677..485d6d7 100644 --- a/irctest/controllers/atheme_services.py +++ b/irctest/controllers/atheme_services.py @@ -56,6 +56,8 @@ saslserv {{ class AthemeController(BaseServicesController, DirectoryBasedController): """Mixin for server controllers that rely on Atheme""" + software_name = "Atheme" + def run(self, protocol: str, server_hostname: str, server_port: int) -> None: self.create_config() diff --git a/irctest/controllers/irc2.py b/irctest/controllers/irc2.py index 56defc8..0281fb6 100644 --- a/irctest/controllers/irc2.py +++ b/irctest/controllers/irc2.py @@ -29,8 +29,8 @@ O:*:operpassword:operuser:::: """ -class Ircu2Controller(BaseServerController, DirectoryBasedController): - binary_name: str +class Irc2Controller(BaseServerController, DirectoryBasedController): + software_name = "irc2" services_protocol: str supports_sts = False @@ -89,5 +89,5 @@ class Ircu2Controller(BaseServerController, DirectoryBasedController): ) -def get_irctest_controller_class() -> Type[Ircu2Controller]: - return Ircu2Controller +def get_irctest_controller_class() -> Type[Irc2Controller]: + return Irc2Controller diff --git a/irctest/controllers/ircu2.py b/irctest/controllers/ircu2.py index 9b0d4bd..6bf9916 100644 --- a/irctest/controllers/ircu2.py +++ b/irctest/controllers/ircu2.py @@ -51,7 +51,7 @@ features {{ class Ircu2Controller(BaseServerController, DirectoryBasedController): - software_name = "Ircu2" + software_name = "ircu2" supports_sts = False extban_mute_char = None diff --git a/irctest/controllers/plexus4.py b/irctest/controllers/plexus4.py index 395b4e4..a968e56 100644 --- a/irctest/controllers/plexus4.py +++ b/irctest/controllers/plexus4.py @@ -74,7 +74,7 @@ operator {{ class Plexus4Controller(BaseHybridController): - software_name = "Hybrid" + software_name = "Plexus4" binary_name = "ircd" services_protocol = "plexus" diff --git a/irctest/dashboard/format.py b/irctest/dashboard/format.py index be211e1..721dc04 100644 --- a/irctest/dashboard/format.py +++ b/irctest/dashboard/format.py @@ -208,6 +208,9 @@ def build_module_html( cell.set("class", "skipped") if result.type == "pytest.skip": text = "s" + elif result.type == "pytest.xfail": + text = "X" + cell.set("class", "expected-failure") else: text = result.type elif result.success: @@ -231,6 +234,8 @@ def build_module_html( a.text = text or "?" else: cell.text = text or "?" + if result.message: + cell.set("title", result.message) return root diff --git a/irctest/dashboard/style.css b/irctest/dashboard/style.css index ed0b737..628fd8e 100644 --- a/irctest/dashboard/style.css +++ b/irctest/dashboard/style.css @@ -46,6 +46,9 @@ table.test-matrix .skipped { table.test-matrix .failure { background-color: red; } +table.test-matrix .expected-failure { + background-color: orange; +} /* Rotate headers, thanks to https://css-tricks.com/rotated-table-column-headers/ */ th.job-name { diff --git a/irctest/server_tests/account_tag.py b/irctest/server_tests/account_tag.py index 6a19755..45cd942 100644 --- a/irctest/server_tests/account_tag.py +++ b/irctest/server_tests/account_tag.py @@ -55,6 +55,9 @@ class AccountTagTestCase(cases.BaseServerTestCase): @cases.mark_capabilities("account-tag") @cases.skipUnlessHasMechanism("PLAIN") + @cases.xfailIfSoftware( + ["Charybdis"], "https://github.com/solanum-ircd/solanum/issues/166" + ) def testInvite(self): self.connectClient("foo", capabilities=["account-tag"], skip_if_cap_nak=True) self.getMessages(1) diff --git a/irctest/server_tests/bot_mode.py b/irctest/server_tests/bot_mode.py index 224c77e..1b9cf7f 100644 --- a/irctest/server_tests/bot_mode.py +++ b/irctest/server_tests/bot_mode.py @@ -67,6 +67,10 @@ class BotModeTestCase(cases.BaseServerTestCase): message, command=RPL_WHOISBOT, params=["usernick", "botnick", ANYSTR] ) + @cases.xfailIfSoftware( + ["InspIRCd"], + "Uses only vendor tags for now: https://github.com/inspircd/inspircd/pull/1910", + ) def testBotPrivateMessage(self): self._initBot() @@ -84,6 +88,10 @@ class BotModeTestCase(cases.BaseServerTestCase): tags={"draft/bot": None, **ANYDICT}, ) + @cases.xfailIfSoftware( + ["InspIRCd"], + "Uses only vendor tags for now: https://github.com/inspircd/inspircd/pull/1910", + ) def testBotChannelMessage(self): self._initBot() diff --git a/irctest/server_tests/buffering.py b/irctest/server_tests/buffering.py index 4adf01d..53e9f9a 100644 --- a/irctest/server_tests/buffering.py +++ b/irctest/server_tests/buffering.py @@ -32,6 +32,16 @@ def _sendBytePerByte(self, line): class BufferingTestCase(cases.BaseServerTestCase): + @cases.xfailIfSoftware( + ["Bahamut"], + "cannot pass because of issues with UTF-8 handling: " + "https://github.com/DALnet/bahamut/issues/196", + ) + @cases.xfailIfSoftware( + ["ircu2", "Nefarious", "snircd"], + "ircu2 discards the whole buffer on long lines " + "(TODO: refine how we exclude these tests)", + ) @pytest.mark.parametrize( "sender_function,colon", [ diff --git a/irctest/server_tests/cap.py b/irctest/server_tests/cap.py index 9b74447..c079078 100644 --- a/irctest/server_tests/cap.py +++ b/irctest/server_tests/cap.py @@ -78,6 +78,10 @@ class CapTestCase(cases.BaseServerTestCase): ) @cases.mark_specifications("IRCv3") + @cases.xfailIfSoftware( + ["UnrealIRCd"], + "UnrealIRCd sends a trailing space on CAP NAK: https://github.com/unrealircd/unrealircd/pull/148", + ) def testNakWhole(self): """“The capability identifier set must be accepted as a whole, or rejected entirely.” @@ -125,6 +129,10 @@ class CapTestCase(cases.BaseServerTestCase): ) @cases.mark_specifications("IRCv3") + @cases.xfailIfSoftware( + ["UnrealIRCd"], + "UnrealIRCd sends a trailing space on CAP NAK: https://github.com/unrealircd/unrealircd/pull/148", + ) def testCapRemovalByClient(self): """Test CAP LIST and removal of caps via CAP REQ :-tagname.""" cap1 = "echo-message" diff --git a/irctest/server_tests/chathistory.py b/irctest/server_tests/chathistory.py index a264ae5..53b4d16 100644 --- a/irctest/server_tests/chathistory.py +++ b/irctest/server_tests/chathistory.py @@ -2,12 +2,13 @@ `IRCv3 draft chathistory `_ """ +import functools import secrets import time import pytest -from irctest import cases +from irctest import cases, runner from irctest.irc_utils.junkdrawer import random_name from irctest.patma import ANYSTR @@ -42,6 +43,16 @@ def validate_chathistory_batch(msgs): return result +def skip_ngircd(f): + @functools.wraps(f) + def newf(self, *args, **kwargs): + if self.controller.software_name == "ngIRCd": + raise runner.NotImplementedByController("nicks longer 9 characters") + return f(self, *args, **kwargs) + + return newf + + @cases.mark_specifications("IRCv3") @cases.mark_services class ChathistoryTestCase(cases.BaseServerTestCase): @@ -49,6 +60,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase): def config() -> cases.TestCaseControllerConfig: return cases.TestCaseControllerConfig(chathistory=True) + @skip_ngircd def testInvalidTargets(self): bar, pw = random_name("bar"), random_name("pw") self.controller.registerUser(self, bar, pw) @@ -94,6 +106,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase): ) @pytest.mark.private_chathistory + @skip_ngircd def testMessagesToSelf(self): bar, pw = random_name("bar"), random_name("pw") self.controller.registerUser(self, bar, pw) @@ -166,7 +179,19 @@ class ChathistoryTestCase(cases.BaseServerTestCase): self.assertEqual(len(set(msg.time for msg in echo_messages)), num_messages) @pytest.mark.parametrize("subcommand", SUBCOMMANDS) + @skip_ngircd def testChathistory(self, subcommand): + if subcommand == "BETWEEN" and self.controller.software_name == "UnrealIRCd": + pytest.xfail( + "CHATHISTORY BETWEEN does not apply bounds correct " + "https://bugs.unrealircd.org/view.php?id=5952" + ) + if subcommand == "AROUND" and self.controller.software_name == "UnrealIRCd": + pytest.xfail( + "CHATHISTORY AROUND excludes 'central' messages " + "https://bugs.unrealircd.org/view.php?id=5953" + ) + self.connectClient( "bar", capabilities=[ @@ -198,6 +223,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase): self.validate_chathistory(subcommand, echo_messages, 1, chname) @pytest.mark.parametrize("subcommand", SUBCOMMANDS) + @skip_ngircd def testChathistoryEventPlayback(self, subcommand): self.connectClient( "bar", @@ -231,6 +257,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase): @pytest.mark.parametrize("subcommand", SUBCOMMANDS) @pytest.mark.private_chathistory + @skip_ngircd def testChathistoryDMs(self, subcommand): c1 = "foo" + secrets.token_hex(12) c2 = "bar" + secrets.token_hex(12) @@ -553,6 +580,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase): self.assertIn(echo_messages[7], result) @pytest.mark.arbitrary_client_tags + @skip_ngircd def testChathistoryTagmsg(self): c1 = "foo" + secrets.token_hex(12) c2 = "bar" + secrets.token_hex(12) @@ -651,6 +679,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase): @pytest.mark.arbitrary_client_tags @pytest.mark.private_chathistory + @skip_ngircd def testChathistoryDMClientOnlyTags(self): # regression test for Ergo #1411 c1 = "foo" + secrets.token_hex(12) diff --git a/irctest/server_tests/chmodes/key.py b/irctest/server_tests/chmodes/key.py index 6878c90..1f8773a 100644 --- a/irctest/server_tests/chmodes/key.py +++ b/irctest/server_tests/chmodes/key.py @@ -64,6 +64,21 @@ class KeyTestCase(cases.BaseServerTestCase): -- https://modern.ircdocs.horse/#key-channel-mode -- https://github.com/ircdocs/modern-irc/pull/111 """ + if key == "" and self.controller.software_name in ( + "ircu2", + "Nefarious", + "snircd", + ): + pytest.xfail( + "ircu2 returns ERR_NEEDMOREPARAMS on empty keys: " + "https://github.com/UndernetIRC/ircu2/issues/13" + ) + if (key == "" or " " in key) and self.controller.software_name == "ngIRCd": + pytest.xfail( + "ngIRCd does not validate channel keys: " + "https://github.com/ngircd/ngircd/issues/290" + ) + self.connectClient("bar") self.joinChannel(1, "#chan") self.sendLine(1, f"MODE #chan +k :{key}") diff --git a/irctest/server_tests/connection_registration.py b/irctest/server_tests/connection_registration.py index e4a1c0b..d726cfc 100644 --- a/irctest/server_tests/connection_registration.py +++ b/irctest/server_tests/connection_registration.py @@ -84,6 +84,10 @@ class ConnectionRegistrationTestCase(cases.BaseServerTestCase): self.getMessages(1) @cases.mark_specifications("RFC2812") + @cases.xfailIfSoftware(["Charybdis", "Solanum"], "very flaky") + @cases.xfailIfSoftware( + ["ircu2", "Nefarious", "snircd"], "ircu2 does not send ERROR" + ) def testQuitErrors(self): """“A client session is terminated with a quit message. The server acknowledges this by sending an ERROR message to the client.” @@ -164,6 +168,10 @@ class ConnectionRegistrationTestCase(cases.BaseServerTestCase): "neither got 001.", ) + @cases.xfailIfSoftware( + ["ircu2", "Nefarious", "ngIRCd"], + "uses a default value instead of ERR_NEEDMOREPARAMS", + ) def testEmptyRealname(self): """ Syntax: diff --git a/irctest/server_tests/help.py b/irctest/server_tests/help.py index 9058ea3..ff5069e 100644 --- a/irctest/server_tests/help.py +++ b/irctest/server_tests/help.py @@ -2,6 +2,7 @@ The HELP and HELPOP command (`Modern `__) """ +import functools import re import pytest @@ -17,6 +18,30 @@ from irctest.numerics import ( from irctest.patma import ANYSTR, StrRe +def with_xfails(f): + @functools.wraps(f) + def newf(self, command, *args, **kwargs): + if command == "HELP" and self.controller.software_name == "Bahamut": + raise runner.NotImplementedByController( + "fail because Bahamut forwards /HELP to HelpServ (but not /HELPOP)" + ) + + if self.controller.software_name in ("irc2", "ircu2", "ngIRCd"): + raise runner.NotImplementedByController( + "numerics in reply to /HELP and /HELPOP (uses NOTICE instead)" + ) + + if self.controller.software_name == "UnrealIRCd": + raise runner.NotImplementedByController( + "fails because Unreal uses custom numerics " + "https://github.com/unrealircd/unrealircd/pull/184" + ) + + return f(self, command, *args, **kwargs) + + return newf + + class HelpTestCase(cases.BaseServerTestCase): def _assertValidHelp(self, messages, subject): if subject != ANYSTR: @@ -46,6 +71,7 @@ class HelpTestCase(cases.BaseServerTestCase): @pytest.mark.parametrize("command", ["HELP", "HELPOP"]) @cases.mark_specifications("Modern") + @with_xfails def testHelpNoArg(self, command): self.connectClient("nick") self.sendLine(1, f"{command}") @@ -59,6 +85,7 @@ class HelpTestCase(cases.BaseServerTestCase): @pytest.mark.parametrize("command", ["HELP", "HELPOP"]) @cases.mark_specifications("Modern") + @with_xfails def testHelpPrivmsg(self, command): self.connectClient("nick") self.sendLine(1, f"{command} PRIVMSG") @@ -71,6 +98,7 @@ class HelpTestCase(cases.BaseServerTestCase): @pytest.mark.parametrize("command", ["HELP", "HELPOP"]) @cases.mark_specifications("Modern") + @with_xfails def testHelpUnknownSubject(self, command): self.connectClient("nick") self.sendLine(1, f"{command} THISISNOTACOMMAND") diff --git a/irctest/server_tests/info.py b/irctest/server_tests/info.py index 8e3ed61..d1a2613 100644 --- a/irctest/server_tests/info.py +++ b/irctest/server_tests/info.py @@ -87,6 +87,9 @@ class InfoTestCase(cases.BaseServerTestCase): @pytest.mark.parametrize("target", ["invalid.server.example", "invalidserver"]) @cases.mark_specifications("RFC1459", "RFC2812", deprecated=True) + @cases.xfailIfSoftware( + ["Ergo"], "does not apply to Ergo, which ignores the optional argument" + ) def testInfoNosuchserver(self, target): """ diff --git a/irctest/server_tests/invite.py b/irctest/server_tests/invite.py index f441849..2f9c305 100644 --- a/irctest/server_tests/invite.py +++ b/irctest/server_tests/invite.py @@ -200,6 +200,9 @@ class InviteTestCase(cases.BaseServerTestCase): self._testInvite(opped=True, invite_only=invite_only) @cases.mark_specifications("RFC1459", "RFC2812", "Modern", strict=True) + @cases.xfailIfSoftware( + ["Hybrid", "Plexus4"], "the only strict test that Hybrid fails" + ) def testInviteUnopped(self): """Tests invites from unopped users on not-invite-only chans.""" self._testInvite(opped=False, invite_only=False) @@ -237,6 +240,11 @@ class InviteTestCase(cases.BaseServerTestCase): ) @cases.mark_specifications("RFC1459", "RFC2812", "Modern") + @cases.xfailIfSoftware( + ["Plexus4"], + "Plexus4 allows non-op to invite if (and only if) the channel is not " + "invite-only", + ) def testInviteInviteOnly(self): """ "To invite a user to a channel which is invite only (MODE diff --git a/irctest/server_tests/kick.py b/irctest/server_tests/kick.py index d2acc72..afec031 100644 --- a/irctest/server_tests/kick.py +++ b/irctest/server_tests/kick.py @@ -96,6 +96,10 @@ class KickTestCase(cases.BaseServerTestCase): self.assertMessageMatch(m3, command="KICK", params=["#chan", "bar", ANYSTR]) @cases.mark_specifications("RFC2812") + @cases.xfailIfSoftware( + ["Charybdis", "ircu2", "irc2", "Solanum"], + "uses the nick of the kickee rather than the kicker.", + ) def testKickDefaultComment(self): """ "If a "comment" is diff --git a/irctest/server_tests/list.py b/irctest/server_tests/list.py index 0c57289..7a902e3 100644 --- a/irctest/server_tests/list.py +++ b/irctest/server_tests/list.py @@ -12,6 +12,7 @@ from irctest import cases class ListTestCase(cases.BaseServerTestCase): @cases.mark_specifications("RFC1459", "RFC2812") + @cases.xfailIfSoftware(["irc2"], "irc2 deprecated LIST") def testListEmpty(self): """ @@ -40,6 +41,7 @@ class ListTestCase(cases.BaseServerTestCase): ) @cases.mark_specifications("RFC1459", "RFC2812") + @cases.xfailIfSoftware(["irc2"], "irc2 deprecated LIST") def testListOne(self): """When a channel exists, LIST should get it in a reply. diff --git a/irctest/server_tests/lusers.py b/irctest/server_tests/lusers.py index 7eb59d6..786c582 100644 --- a/irctest/server_tests/lusers.py +++ b/irctest/server_tests/lusers.py @@ -153,6 +153,10 @@ class BasicLusersTestCase(LusersTestCase): self.getLusers("bar", True) @cases.mark_specifications("Modern") + @cases.xfailIfSoftware( + ["ircu2", "Nefarious", "snircd"], + "test depends on Modern behavior, not just RFC2812", + ) def testLusersFull(self): self.connectClient("bar", name="bar") lusers = self.getLusers("bar", False) @@ -170,10 +174,22 @@ class BasicLusersTestCase(LusersTestCase): class LusersUnregisteredTestCase(LusersTestCase): @cases.mark_specifications("RFC2812") + @cases.xfailIfSoftware( + ["Nefarious"], + "Nefarious doesn't seem to distinguish unregistered users from normal ones", + ) def testLusersRfc2812(self): self.doLusersTest(True) @cases.mark_specifications("Modern") + @cases.xfailIfSoftware( + ["Nefarious"], + "Nefarious doesn't seem to distinguish unregistered users from normal ones", + ) + @cases.xfailIfSoftware( + ["ircu2", "Nefarious", "snircd"], + "test depends on Modern behavior, not just RFC2812", + ) def testLusersFull(self): self.doLusersTest(False) @@ -237,6 +253,10 @@ class LusersUnregisteredDefaultInvisibleTestCase(LusersUnregisteredTestCase): ) @cases.mark_specifications("Ergo") + @cases.xfailIfSoftware( + ["Nefarious"], + "Nefarious doesn't seem to distinguish unregistered users from normal ones", + ) def testLusers(self): self.doLusersTest(False) lusers = self.getLusers("bar", False) diff --git a/irctest/server_tests/messages.py b/irctest/server_tests/messages.py index a4e8544..b33be17 100644 --- a/irctest/server_tests/messages.py +++ b/irctest/server_tests/messages.py @@ -51,6 +51,15 @@ class NoticeTestCase(cases.BaseServerTestCase): ) @cases.mark_specifications("RFC1459", "RFC2812") + @cases.xfailIfSoftware( + ["InspIRCd"], + "replies with ERR_NOSUCHCHANNEL to NOTICE to non-existent channels", + ) + @cases.xfailIfSoftware( + ["UnrealIRCd"], + "replies with ERR_NOSUCHCHANNEL to NOTICE to non-existent channels: " + "https://bugs.unrealircd.org/view.php?id=5949", + ) def testNoticeNonexistentChannel(self): """ "automatic replies must never be @@ -71,6 +80,9 @@ class NoticeTestCase(cases.BaseServerTestCase): class TagsTestCase(cases.BaseServerTestCase): @cases.mark_capabilities("message-tags") + @cases.xfailIfSoftware( + ["UnrealIRCd"], "https://bugs.unrealircd.org/view.php?id=5947" + ) def testLineTooLong(self): self.connectClient("bar", capabilities=["message-tags"], skip_if_cap_nak=True) self.connectClient( diff --git a/irctest/server_tests/quit.py b/irctest/server_tests/quit.py index 022447f..a62c123 100644 --- a/irctest/server_tests/quit.py +++ b/irctest/server_tests/quit.py @@ -16,6 +16,7 @@ from irctest.patma import StrRe class ChannelQuitTestCase(cases.BaseServerTestCase): @cases.mark_specifications("RFC2812") + @cases.xfailIfSoftware(["ircu2", "Nefarious", "snircd"], "ircu2 does not echo QUIT") def testQuit(self): """“Once a user has joined a channel, he receives information about all commands his server receives affecting the channel. This diff --git a/irctest/server_tests/regressions.py b/irctest/server_tests/regressions.py index 15ff21e..c09fdf5 100644 --- a/irctest/server_tests/regressions.py +++ b/irctest/server_tests/regressions.py @@ -4,7 +4,7 @@ Regression tests for bugs in `Ergo `_. import time -from irctest import cases +from irctest import cases, runner from irctest.numerics import ERR_ERRONEUSNICKNAME, ERR_NICKNAMEINUSE, RPL_WELCOME from irctest.patma import ANYDICT @@ -57,6 +57,12 @@ class RegressionsTestCase(cases.BaseServerTestCase): @cases.mark_capabilities("message-tags", "batch", "echo-message", "server-time") def testTagCap(self): + if self.controller.software_name == "UnrealIRCd": + raise runner.NotImplementedByController( + "Arbitrary +draft/reply values (TODO: adapt this test to use real " + "values so their pass Unreal's validation) " + "https://bugs.unrealircd.org/view.php?id=5948" + ) # regression test for oragono #754 self.connectClient( "alice", @@ -99,6 +105,7 @@ class RegressionsTestCase(cases.BaseServerTestCase): ) @cases.mark_specifications("RFC1459") + @cases.xfailIfSoftware(["ngIRCd"], "wat") def testStarNick(self): self.addClient(1) self.sendLine(1, "NICK *") diff --git a/irctest/server_tests/sasl.py b/irctest/server_tests/sasl.py index d680491..47f53cd 100644 --- a/irctest/server_tests/sasl.py +++ b/irctest/server_tests/sasl.py @@ -171,6 +171,13 @@ class SaslTestCase(cases.BaseServerTestCase): @cases.mark_specifications("IRCv3") @cases.skipUnlessHasMechanism("PLAIN") + @cases.xfailIf( + lambda self: ( + self.controller.services_controller is not None + and self.controller.services_controller.software_name == "Anope" + ), + "Anope does not handle split AUTHENTICATE (reported on IRC)", + ) def testPlainLarge(self): """Test the client splits large AUTHENTICATE messages whose payload is not a multiple of 400. @@ -233,6 +240,13 @@ class SaslTestCase(cases.BaseServerTestCase): @cases.mark_specifications("IRCv3") @cases.skipUnlessHasMechanism("PLAIN") + @cases.xfailIf( + lambda self: ( + self.controller.services_controller is not None + and self.controller.services_controller.software_name == "Anope" + ), + "Anope does not handle split AUTHENTICATE (reported on IRC)", + ) def testPlainLargeEquals400(self): """Test the client splits large AUTHENTICATE messages whose payload is not a multiple of 400. diff --git a/irctest/server_tests/statusmsg.py b/irctest/server_tests/statusmsg.py index fb063f8..1a1910d 100644 --- a/irctest/server_tests/statusmsg.py +++ b/irctest/server_tests/statusmsg.py @@ -17,6 +17,11 @@ class StatusmsgTestCase(cases.BaseServerTestCase): self.assertEqual(self.server_support["STATUSMSG"], "~&@%+") @cases.mark_isupport("STATUSMSG") + @cases.xfailIfSoftware( + ["ircu2", "Nefarious", "snircd"], + "STATUSMSG is present in ISUPPORT, but it not actually supported as PRIVMSG " + "target (only for WALLCOPS/WALLCHOPS/...)", + ) def testStatusmsgFromOp(self): """Test that STATUSMSG are sent to the intended recipients, with the intended prefixes.""" @@ -68,6 +73,11 @@ class StatusmsgTestCase(cases.BaseServerTestCase): self.assertEqual(len(unprivilegedMessages), 0) @cases.mark_isupport("STATUSMSG") + @cases.xfailIfSoftware( + ["ircu2", "Nefarious", "snircd"], + "STATUSMSG is present in ISUPPORT, but it not actually supported as PRIVMSG " + "target (only for WALLCOPS/WALLCHOPS/...)", + ) def testStatusmsgFromRegular(self): """Test that STATUSMSG are sent to the intended recipients, with the intended prefixes.""" diff --git a/irctest/server_tests/wallops.py b/irctest/server_tests/wallops.py index c1d493b..e0d00e0 100644 --- a/irctest/server_tests/wallops.py +++ b/irctest/server_tests/wallops.py @@ -66,6 +66,9 @@ class WallopsTestCase(cases.BaseServerTestCase): ) @cases.mark_specifications("Modern") + @cases.xfailIfSoftware( + ["irc2"], "irc2 ignores the command instead of replying ERR_UNKNOWNCOMMAND" + ) def testWallopsPrivileges(self): """ https://github.com/ircdocs/modern-irc/pull/118 diff --git a/irctest/server_tests/who.py b/irctest/server_tests/who.py index 8510e69..f5e8903 100644 --- a/irctest/server_tests/who.py +++ b/irctest/server_tests/who.py @@ -87,6 +87,9 @@ class BaseWhoTestCase: class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase): @cases.mark_specifications("Modern") def testWhoStar(self): + if self.controller.software_name == "Bahamut": + raise runner.NotImplementedByController("WHO mask") + self._init() self.sendLine(2, "WHO *") @@ -115,6 +118,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase): ) @cases.mark_specifications("Modern") def testWhoNick(self, mask): + if "*" in mask and self.controller.software_name == "Bahamut": + raise runner.NotImplementedByController("WHO mask") + self._init() self.sendLine(2, f"WHO {mask}") @@ -142,6 +148,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase): ids=["username", "realname-mask", "hostname"], ) def testWhoUsernameRealName(self, mask): + if "*" in mask and self.controller.software_name == "Bahamut": + raise runner.NotImplementedByController("WHO mask") + self._init() self.sendLine(2, f"WHO :{mask}") @@ -192,6 +201,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase): ) @cases.mark_specifications("Modern") def testWhoNickAway(self, mask): + if "*" in mask and self.controller.software_name == "Bahamut": + raise runner.NotImplementedByController("WHO mask") + self._init() self.sendLine(1, "AWAY :be right back") @@ -218,6 +230,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase): ) @cases.mark_specifications("Modern") def testWhoNickOper(self, mask): + if "*" in mask and self.controller.software_name == "Bahamut": + raise runner.NotImplementedByController("WHO mask") + self._init() self.sendLine(1, "OPER operuser operpassword") @@ -249,6 +264,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase): ) @cases.mark_specifications("Modern") def testWhoNickAwayAndOper(self, mask): + if "*" in mask and self.controller.software_name == "Bahamut": + raise runner.NotImplementedByController("WHO mask") + self._init() self.sendLine(1, "OPER operuser operpassword") @@ -280,6 +298,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase): @pytest.mark.parametrize("mask", ["#chan", "#CHAN"], ids=["exact", "casefolded"]) @cases.mark_specifications("Modern") def testWhoChan(self, mask): + if "*" in mask and self.controller.software_name == "Bahamut": + raise runner.NotImplementedByController("WHO mask") + self._init() self.sendLine(1, "OPER operuser operpassword") diff --git a/irctest/server_tests/whois.py b/irctest/server_tests/whois.py index ec432a5..a22b34f 100644 --- a/irctest/server_tests/whois.py +++ b/irctest/server_tests/whois.py @@ -29,6 +29,9 @@ from irctest.patma import ANYSTR, StrRe class _WhoisTestMixin(cases.BaseServerTestCase): def _testWhoisNumerics(self, authenticate, away, oper): + if oper and self.controller.software_name == "Charybdis": + pytest.xfail("charybdis uses RPL_WHOISSPECIAL instead of RPL_WHOISOPERATOR") + if authenticate: self.connectClient("nick1") self.controller.registerUser(self, "val", "sesame") diff --git a/irctest/server_tests/whowas.py b/irctest/server_tests/whowas.py index dd65b54..a3d9dc0 100644 --- a/irctest/server_tests/whowas.py +++ b/irctest/server_tests/whowas.py @@ -163,6 +163,10 @@ class WhowasTestCase(cases.BaseServerTestCase): ) @cases.mark_specifications("RFC1459", "RFC2812") + @cases.xfailIfSoftware( + ["InspIRCd"], + "Feature not released yet: https://github.com/inspircd/inspircd/pull/1967", + ) def testWhowasMultiple(self): """ "The history is searched backward, returning the most recent entry first." @@ -172,6 +176,10 @@ class WhowasTestCase(cases.BaseServerTestCase): self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS nick2") @cases.mark_specifications("RFC1459", "RFC2812") + @cases.xfailIfSoftware( + ["InspIRCd"], + "Feature not released yet: https://github.com/inspircd/inspircd/pull/1968", + ) def testWhowasCount1(self): """ "If there are multiple entries, up to replies will be returned" @@ -181,6 +189,10 @@ class WhowasTestCase(cases.BaseServerTestCase): self._testWhowasMultiple(second_result=False, whowas_command="WHOWAS nick2 1") @cases.mark_specifications("RFC1459", "RFC2812") + @cases.xfailIfSoftware( + ["InspIRCd"], + "Feature not released yet: https://github.com/inspircd/inspircd/pull/1968", + ) def testWhowasCount2(self): """ "If there are multiple entries, up to replies will be returned" @@ -190,6 +202,10 @@ class WhowasTestCase(cases.BaseServerTestCase): self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS nick2 2") @cases.mark_specifications("RFC1459", "RFC2812") + @cases.xfailIfSoftware( + ["InspIRCd"], + "Feature not released yet: https://github.com/inspircd/inspircd/pull/1968", + ) def testWhowasCountNegative(self): """ "If a non-positive number is passed as being , then a full search @@ -200,6 +216,13 @@ class WhowasTestCase(cases.BaseServerTestCase): self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS nick2 -1") @cases.mark_specifications("RFC1459", "RFC2812") + @cases.xfailIfSoftware( + ["ircu2"], "Fix not released yet: https://github.com/UndernetIRC/ircu2/pull/19" + ) + @cases.xfailIfSoftware( + ["InspIRCd"], + "Feature not released yet: https://github.com/inspircd/inspircd/pull/1967", + ) def testWhowasCountZero(self): """ "If a non-positive number is passed as being , then a full search @@ -215,6 +238,9 @@ class WhowasTestCase(cases.BaseServerTestCase): "Wildcards are allowed in the parameter." -- https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3 """ + if self.controller.software_name == "Bahamut": + raise runner.NotImplementedByController("WHOWAS mask") + self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS *ck2") @cases.mark_specifications("RFC1459", "RFC2812", deprecated=True) @@ -250,6 +276,12 @@ class WhowasTestCase(cases.BaseServerTestCase): ) @cases.mark_specifications("RFC1459", "RFC2812") + @cases.xfailIfSoftware( + ["Charybdis"], + "fails because of a typo (solved in " + "https://github.com/solanum-ircd/solanum/commit/" + "08b7b6bd7e60a760ad47b58cbe8075b45d66166f)", + ) def testWhowasNoSuchNick(self): """ https://datatracker.ietf.org/doc/html/rfc1459#section-4.5.3 @@ -285,6 +317,11 @@ class WhowasTestCase(cases.BaseServerTestCase): """ https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3 """ + if self.controller.software_name == "Bahamut": + pytest.xfail( + "Bahamut returns entries in query order instead of chronological order" + ) + self.connectClient("nick1") targmax = dict(