val/irctest
1
0
Fork 0
mirror of https://github.com/progval/irctest.git synced 2025-04-05 06:49:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use xfail instead of deselection for known failures (#155)

Browse Source
This commit is contained in:
Val Lorentz
2022-04-12 22:36:28 +02:00
committed by GitHub
parent 10b6f8d6da
commit 2bc68a2208
32 changed files with 308 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
Makefile
View File

@ -7,102 +7,47 @@ PYTEST_ARGS ?=
# Will be appended at the end of the -k argument to pytest
EXTRA_SELECTORS ?=
# testPlainLarge fails because it doesn't handle split AUTHENTICATE (reported on IRC)
ANOPE_SELECTORS := \
and not testPlainLarge
# buffering tests cannot pass because of issues with UTF-8 handling: https://github.com/DALnet/bahamut/issues/196
# mask tests in test_who.py fail because they are not implemented.
# some HelpTestCase::*[HELP] tests fail because Bahamut forwards /HELP to HelpServ (but not /HELPOP)
# testWhowasMultiTarget fails because Bahamut returns the results in query order instead of chronological order
BAHAMUT_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not IRCv3 \
and not buffering \
and not (testWho and not whois and mask) \
and not testWhoStar \
and (not HelpTestCase or HELPOP) \
and not testWhowasMultiTarget \
$(EXTRA_SELECTORS)
# testQuitErrors is very flaky
# AccountTagTestCase.testInvite fails because https://github.com/solanum-ircd/solanum/issues/166
# testKickDefaultComment fails because it uses the nick of the kickee rather than the kicker.
# testWhoisNumerics[oper] fails because charybdis uses RPL_WHOISSPECIAL instead of RPL_WHOISOPERATOR
# testWhowasNoSuchNick fails because of a typo (solved in https://github.com/solanum-ircd/solanum/commit/08b7b6bd7e60a760ad47b58cbe8075b45d66166f)
CHARYBDIS_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not testQuitErrors \
and not testKickDefaultComment \
and not (AccountTagTestCase and testInvite) \
and not (testWhoisNumerics and oper) \
and not testWhowasNoSuchNick \
$(EXTRA_SELECTORS)
# testInfoNosuchserver does not apply to Ergo: Ergo ignores the optional <target> argument
ERGO_SELECTORS := \
not deprecated \
and not testInfoNosuchserver \
$(EXTRA_SELECTORS)
# testInviteUnopped is the only strict test that Hybrid fails
HYBRID_SELECTORS := \
not Ergo \
and not testInviteUnopped \
and not deprecated \
$(EXTRA_SELECTORS)
# testBotPrivateMessage and testBotChannelMessage fail because https://github.com/inspircd/inspircd/pull/1910 is not released yet
# WHOWAS tests fail because https://github.com/inspircd/inspircd/pull/1967 and https://github.com/inspircd/inspircd/pull/1968 are not released yet
INSPIRCD_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not testNoticeNonexistentChannel \
and not testBotPrivateMessage and not testBotChannelMessage \
and not whowas \
$(EXTRA_SELECTORS)
# buffering tests fail because ircu2 discards the whole buffer on long lines (TODO: refine how we exclude these tests)
# testQuit and testQuitErrors fail because ircu2 does not send ERROR or QUIT
# lusers "full" tests fail because they depend on Modern behavior, not just RFC2812
# statusmsg tests fail because STATUSMSG is present in ISUPPORT, but it not actually supported as PRIVMSG target
# testKeyValidation[empty] fails because ircu2 returns ERR_NEEDMOREPARAMS on empty keys: https://github.com/UndernetIRC/ircu2/issues/13
# testKickDefaultComment fails because it uses the nick of the kickee rather than the kicker.
# testEmptyRealname fails because it uses a default value instead of ERR_NEEDMOREPARAMS.
# HelpTestCase fails because it returns NOTICEs instead of numerics
# testWhowasCountZero fails: https://github.com/UndernetIRC/ircu2/pull/19
IRCU2_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not buffering \
and not testQuit \
and not (lusers and full) \
and not statusmsg \
and not (testKeyValidation and empty) \
and not testKickDefaultComment \
and not testEmptyRealname \
and not HelpTestCase \
and not testWhowasCountZero \
$(EXTRA_SELECTORS)
# same justification as ircu2
# lusers "unregistered" tests fail because Nefarious doesn't seem to distinguish unregistered users from normal ones
# lusers "unregistered" tests fail because
NEFARIOUS_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not buffering \
and not testQuit \
and not (lusers and unregistered) \
and not statusmsg \
and not (testKeyValidation and empty) \
and not testEmptyRealname \
$(EXTRA_SELECTORS)
# same justification as ircu2
@ -110,24 +55,12 @@ SNIRCD_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not buffering \
and not testQuit \
and not (lusers and full) \
and not statusmsg \
$(EXTRA_SELECTORS)
# testListEmpty and testListOne fails because irc2 deprecated LIST
# testKickDefaultComment fails because it uses the nick of the kickee rather than the kicker.
# testWallopsPrivileges fails because it ignores the command instead of replying ERR_UNKNOWNCOMMAND
# HelpTestCase fails because it returns NOTICEs instead of numerics
IRC2_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not testListEmpty and not testListOne \
and not testKickDefaultComment \
and not testWallopsPrivileges \
and not HelpTestCase \
$(EXTRA_SELECTORS)
MAMMON_SELECTORS := \
@ -136,28 +69,14 @@ MAMMON_SELECTORS := \
and not strict \
$(EXTRA_SELECTORS)
# testKeyValidation[spaces] and testKeyValidation[empty] fail because ngIRCd does not validate them https://github.com/ngircd/ngircd/issues/290
# testStarNick: wat
# testEmptyRealname fails because it uses a default value instead of ERR_NEEDMOREPARAMS.
# chathistory tests fail because they need nicks longer than 9 chars
# HelpTestCase::*[HELP] fails because it returns NOTICEs instead of numerics
NGIRCD_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not (testKeyValidation and (spaces or empty)) \
and not testStarNick \
and not testEmptyRealname \
and not chathistory \
and (not HelpTestCase or HELPOP) \
$(EXTRA_SELECTORS)
# testInviteUnopped is the only strict test that Plexus4 fails
# testInviteInviteOnly fails because Plexus4 allows non-op to invite if (and only if) the channel is not invite-only
PLEXUS4_SELECTORS := \
not Ergo \
and not testInviteUnopped \
and not testInviteInviteOnly \
and not deprecated \
$(EXTRA_SELECTORS)
@ -168,46 +87,27 @@ LIMNORIA_SELECTORS := \
(foo or not foo) \
$(EXTRA_SELECTORS)
# testQuitErrors is too flaky for CI
# testKickDefaultComment fails because solanum uses the nick of the kickee rather than the kicker.
SOLANUM_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not testQuitErrors \
and not testKickDefaultComment \
$(EXTRA_SELECTORS)
# Same as Limnoria
SOPEL_SELECTORS := \
not testPlainNotAvailable \
(foo or not foo) \
$(EXTRA_SELECTORS)
# testNoticeNonexistentChannel fails: https://bugs.unrealircd.org/view.php?id=5949
# regressions::testTagCap fails: https://bugs.unrealircd.org/view.php?id=5948
# messages::testLineTooLong fails: https://bugs.unrealircd.org/view.php?id=5947
# testCapRemovalByClient and testNakWhole fail pending https://github.com/unrealircd/unrealircd/pull/148
# Tests marked with arbitrary_client_tags can't pass because Unreal whitelists which tags it relays
# Tests marked with react_tag can't pass because Unreal blocks +draft/react https://github.com/unrealircd/unrealircd/pull/149
# Tests marked with private_chathistory can't pass because Unreal does not implement CHATHISTORY for DMs
# testChathistory[BETWEEN] fails: https://bugs.unrealircd.org/view.php?id=5952
# testChathistory[AROUND] fails: https://bugs.unrealircd.org/view.php?id=5953
# testWhoAllOpers fails because Unreal skips results when the mask is too broad
# HELP and HELPOP tests fail because Unreal uses custom numerics https://github.com/unrealircd/unrealircd/pull/184
UNREALIRCD_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not testNoticeNonexistentChannel \
and not (regressions.py and testTagCap) \
and not (messages.py and testLineTooLong) \
and not (cap.py and (testCapRemovalByClient or testNakWhole)) \
and not (account_tag.py and testInvite) \
and not arbitrary_client_tags \
and not react_tag \
and not private_chathistory \
and not (testChathistory and (between or around)) \
and not testWhoAllOpers \
and not HelpTestCase \
$(EXTRA_SELECTORS)
.PHONY: all flakes bahamut charybdis ergo inspircd ircu2 snircd irc2 mammon nefarious limnoria sopel solanum unrealircd
@ -238,7 +138,7 @@ bahamut-anope:
--services-controller=irctest.controllers.anope_services \
-m 'services' \
-n 10 \
-k '$(BAHAMUT_SELECTORS) $(ANOPE_SELECTORS)'
-k '$(BAHAMUT_SELECTORS)'
charybdis:
$(PYTEST) $(PYTEST_ARGS) \
@ -275,7 +175,7 @@ inspircd-anope:
--controller=irctest.controllers.inspircd \
--services-controller=irctest.controllers.anope_services \
-m 'services' \
-k '$(INSPIRCD_SELECTORS) $(ANOPE_SELECTORS)'
-k '$(INSPIRCD_SELECTORS)'
ircu2:
$(PYTEST) $(PYTEST_ARGS) \
@ -373,4 +273,4 @@ unrealircd-anope:
--controller=irctest.controllers.unrealircd \
--services-controller=irctest.controllers.anope_services \
-m 'services' \
-k '$(UNREALIRCD_SELECTORS) $(ANOPE_SELECTORS)'
-k '$(UNREALIRCD_SELECTORS)'

27
irctest/cases.py
View File

@ -765,6 +765,33 @@ def skipUnlessHasSasl(f: Callable[..., _TReturn]) -> Callable[..., _TReturn]:
return newf
def xfailIf(
condition: Callable[..., bool], reason: str
) -> Callable[[Callable[..., _TReturn]], Callable[..., _TReturn]]:
# Works about the same as skipUnlessHasMechanism
def decorator(f: Callable[..., _TReturn]) -> Callable[..., _TReturn]:
@functools.wraps(f)
def newf(self: _TSelf, *args: Any, **kwargs: Any) -> _TReturn:
if condition(self):
try:
return f(self, *args, **kwargs)
except Exception:
pytest.xfail(reason)
assert False # make mypy happy
else:
return f(self, *args, **kwargs)
return newf
return decorator
def xfailIfSoftware(
names: List[str], reason: str
) -> Callable[[Callable[..., _TReturn]], Callable[..., _TReturn]]:
return xfailIf(lambda testcase: testcase.controller.software_name in names, reason)
def mark_services(cls: TClass) -> TClass:
cls.run_services = True
return pytest.mark.services(cls) # type: ignore

1
irctest/client_tests/sasl.py
View File

@ -61,6 +61,7 @@ class SaslTestCase(cases.BaseClientTestCase):
self.assertEqual(m, Message({}, None, "CAP", ["END"]))
@cases.skipUnlessHasMechanism("PLAIN")
@cases.xfailIfSoftware(["Sopel"], "Sopel requests SASL PLAIN even if not available")
def testPlainNotAvailable(self):
"""`sasl=EXTERNAL` is advertized, whereas the client is configured
to use PLAIN.

2
irctest/controllers/anope_services.py
View File

@ -73,6 +73,8 @@ module {{ name = "ns_cert" }}
class AnopeController(BaseServicesController, DirectoryBasedController):
"""Collaborator for server controllers that rely on Anope"""
software_name = "Anope"
def run(self, protocol: str, server_hostname: str, server_port: int) -> None:
self.create_config()

2
irctest/controllers/atheme_services.py
View File

@ -56,6 +56,8 @@ saslserv {{
class AthemeController(BaseServicesController, DirectoryBasedController):
"""Mixin for server controllers that rely on Atheme"""
software_name = "Atheme"
def run(self, protocol: str, server_hostname: str, server_port: int) -> None:
self.create_config()

8
irctest/controllers/irc2.py
View File

@ -29,8 +29,8 @@ O:*:operpassword:operuser::::
"""
class Ircu2Controller(BaseServerController, DirectoryBasedController):
binary_name: str
class Irc2Controller(BaseServerController, DirectoryBasedController):
software_name = "irc2"
services_protocol: str
supports_sts = False
@ -89,5 +89,5 @@ class Ircu2Controller(BaseServerController, DirectoryBasedController):
)
def get_irctest_controller_class() -> Type[Ircu2Controller]:
return Ircu2Controller
def get_irctest_controller_class() -> Type[Irc2Controller]:
return Irc2Controller

2
irctest/controllers/ircu2.py
View File

@ -51,7 +51,7 @@ features {{
class Ircu2Controller(BaseServerController, DirectoryBasedController):
software_name = "Ircu2"
software_name = "ircu2"
supports_sts = False
extban_mute_char = None

2
irctest/controllers/plexus4.py
View File

@ -74,7 +74,7 @@ operator {{
class Plexus4Controller(BaseHybridController):
software_name = "Hybrid"
software_name = "Plexus4"
binary_name = "ircd"
services_protocol = "plexus"

5
irctest/dashboard/format.py
View File

@ -208,6 +208,9 @@ def build_module_html(
cell.set("class", "skipped")
if result.type == "pytest.skip":
text = "s"
elif result.type == "pytest.xfail":
text = "X"
cell.set("class", "expected-failure")
else:
text = result.type
elif result.success:
@ -231,6 +234,8 @@ def build_module_html(
a.text = text or "?"
else:
cell.text = text or "?"
if result.message:
cell.set("title", result.message)
return root

3
irctest/dashboard/style.css
View File

@ -46,6 +46,9 @@ table.test-matrix .skipped {
table.test-matrix .failure {
background-color: red;
}
table.test-matrix .expected-failure {
background-color: orange;
}
/* Rotate headers, thanks to https://css-tricks.com/rotated-table-column-headers/ */
th.job-name {

3
irctest/server_tests/account_tag.py
View File

@ -55,6 +55,9 @@ class AccountTagTestCase(cases.BaseServerTestCase):
@cases.mark_capabilities("account-tag")
@cases.skipUnlessHasMechanism("PLAIN")
@cases.xfailIfSoftware(
["Charybdis"], "https://github.com/solanum-ircd/solanum/issues/166"
)
def testInvite(self):
self.connectClient("foo", capabilities=["account-tag"], skip_if_cap_nak=True)
self.getMessages(1)

8
irctest/server_tests/bot_mode.py
View File

@ -67,6 +67,10 @@ class BotModeTestCase(cases.BaseServerTestCase):
message, command=RPL_WHOISBOT, params=["usernick", "botnick", ANYSTR]
)
@cases.xfailIfSoftware(
["InspIRCd"],
"Uses only vendor tags for now: https://github.com/inspircd/inspircd/pull/1910",
)
def testBotPrivateMessage(self):
self._initBot()
@ -84,6 +88,10 @@ class BotModeTestCase(cases.BaseServerTestCase):
tags={"draft/bot": None, **ANYDICT},
)
@cases.xfailIfSoftware(
["InspIRCd"],
"Uses only vendor tags for now: https://github.com/inspircd/inspircd/pull/1910",
)
def testBotChannelMessage(self):
self._initBot()

10
irctest/server_tests/buffering.py
View File

@ -32,6 +32,16 @@ def _sendBytePerByte(self, line):
class BufferingTestCase(cases.BaseServerTestCase):
@cases.xfailIfSoftware(
["Bahamut"],
"cannot pass because of issues with UTF-8 handling: "
"https://github.com/DALnet/bahamut/issues/196",
)
@cases.xfailIfSoftware(
["ircu2", "Nefarious", "snircd"],
"ircu2 discards the whole buffer on long lines "
"(TODO: refine how we exclude these tests)",
)
@pytest.mark.parametrize(
"sender_function,colon",
[

8
irctest/server_tests/cap.py
View File

@ -78,6 +78,10 @@ class CapTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("IRCv3")
@cases.xfailIfSoftware(
["UnrealIRCd"],
"UnrealIRCd sends a trailing space on CAP NAK: https://github.com/unrealircd/unrealircd/pull/148",
)
def testNakWhole(self):
"""“The capability identifier set must be accepted as a whole, or
rejected entirely.”
@ -125,6 +129,10 @@ class CapTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("IRCv3")
@cases.xfailIfSoftware(
["UnrealIRCd"],
"UnrealIRCd sends a trailing space on CAP NAK: https://github.com/unrealircd/unrealircd/pull/148",
)
def testCapRemovalByClient(self):
"""Test CAP LIST and removal of caps via CAP REQ :-tagname."""
cap1 = "echo-message"

31
irctest/server_tests/chathistory.py
View File

@ -2,12 +2,13 @@
`IRCv3 draft chathistory <https://ircv3.net/specs/extensions/chathistory>`_
"""
import functools
import secrets
import time
import pytest
from irctest import cases
from irctest import cases, runner
from irctest.irc_utils.junkdrawer import random_name
from irctest.patma import ANYSTR
@ -42,6 +43,16 @@ def validate_chathistory_batch(msgs):
return result
def skip_ngircd(f):
@functools.wraps(f)
def newf(self, *args, **kwargs):
if self.controller.software_name == "ngIRCd":
raise runner.NotImplementedByController("nicks longer 9 characters")
return f(self, *args, **kwargs)
return newf
@cases.mark_specifications("IRCv3")
@cases.mark_services
class ChathistoryTestCase(cases.BaseServerTestCase):
@ -49,6 +60,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
def config() -> cases.TestCaseControllerConfig:
return cases.TestCaseControllerConfig(chathistory=True)
@skip_ngircd
def testInvalidTargets(self):
bar, pw = random_name("bar"), random_name("pw")
self.controller.registerUser(self, bar, pw)
@ -94,6 +106,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
)
@pytest.mark.private_chathistory
@skip_ngircd
def testMessagesToSelf(self):
bar, pw = random_name("bar"), random_name("pw")
self.controller.registerUser(self, bar, pw)
@ -166,7 +179,19 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
self.assertEqual(len(set(msg.time for msg in echo_messages)), num_messages)
@pytest.mark.parametrize("subcommand", SUBCOMMANDS)
@skip_ngircd
def testChathistory(self, subcommand):
if subcommand == "BETWEEN" and self.controller.software_name == "UnrealIRCd":
pytest.xfail(
"CHATHISTORY BETWEEN does not apply bounds correct "
"https://bugs.unrealircd.org/view.php?id=5952"
)
if subcommand == "AROUND" and self.controller.software_name == "UnrealIRCd":
pytest.xfail(
"CHATHISTORY AROUND excludes 'central' messages "
"https://bugs.unrealircd.org/view.php?id=5953"
)
self.connectClient(
"bar",
capabilities=[
@ -198,6 +223,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
self.validate_chathistory(subcommand, echo_messages, 1, chname)
@pytest.mark.parametrize("subcommand", SUBCOMMANDS)
@skip_ngircd
def testChathistoryEventPlayback(self, subcommand):
self.connectClient(
"bar",
@ -231,6 +257,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
@pytest.mark.parametrize("subcommand", SUBCOMMANDS)
@pytest.mark.private_chathistory
@skip_ngircd
def testChathistoryDMs(self, subcommand):
c1 = "foo" + secrets.token_hex(12)
c2 = "bar" + secrets.token_hex(12)
@ -553,6 +580,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
self.assertIn(echo_messages[7], result)
@pytest.mark.arbitrary_client_tags
@skip_ngircd
def testChathistoryTagmsg(self):
c1 = "foo" + secrets.token_hex(12)
c2 = "bar" + secrets.token_hex(12)
@ -651,6 +679,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
@pytest.mark.arbitrary_client_tags
@pytest.mark.private_chathistory
@skip_ngircd
def testChathistoryDMClientOnlyTags(self):
# regression test for Ergo #1411
c1 = "foo" + secrets.token_hex(12)

15
irctest/server_tests/chmodes/key.py
View File

@ -64,6 +64,21 @@ class KeyTestCase(cases.BaseServerTestCase):
-- https://modern.ircdocs.horse/#key-channel-mode
-- https://github.com/ircdocs/modern-irc/pull/111
"""
if key == "" and self.controller.software_name in (
"ircu2",
"Nefarious",
"snircd",
):
pytest.xfail(
"ircu2 returns ERR_NEEDMOREPARAMS on empty keys: "
"https://github.com/UndernetIRC/ircu2/issues/13"
)
if (key == "" or " " in key) and self.controller.software_name == "ngIRCd":
pytest.xfail(
"ngIRCd does not validate channel keys: "
"https://github.com/ngircd/ngircd/issues/290"
)
self.connectClient("bar")
self.joinChannel(1, "#chan")
self.sendLine(1, f"MODE #chan +k :{key}")

8
irctest/server_tests/connection_registration.py
View File

@ -84,6 +84,10 @@ class ConnectionRegistrationTestCase(cases.BaseServerTestCase):
self.getMessages(1)
@cases.mark_specifications("RFC2812")
@cases.xfailIfSoftware(["Charybdis", "Solanum"], "very flaky")
@cases.xfailIfSoftware(
["ircu2", "Nefarious", "snircd"], "ircu2 does not send ERROR"
)
def testQuitErrors(self):
"""“A client session is terminated with a quit message. The server
acknowledges this by sending an ERROR message to the client.”
@ -164,6 +168,10 @@ class ConnectionRegistrationTestCase(cases.BaseServerTestCase):
"neither got 001.",
)
@cases.xfailIfSoftware(
["ircu2", "Nefarious", "ngIRCd"],
"uses a default value instead of ERR_NEEDMOREPARAMS",
)
def testEmptyRealname(self):
"""
Syntax:

28
irctest/server_tests/help.py
View File

@ -2,6 +2,7 @@
The HELP and HELPOP command (`Modern <https://modern.ircdocs.horse/#help-message>`__)
"""
import functools
import re
import pytest
@ -17,6 +18,30 @@ from irctest.numerics import (
from irctest.patma import ANYSTR, StrRe
def with_xfails(f):
@functools.wraps(f)
def newf(self, command, *args, **kwargs):
if command == "HELP" and self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController(
"fail because Bahamut forwards /HELP to HelpServ (but not /HELPOP)"
)
if self.controller.software_name in ("irc2", "ircu2", "ngIRCd"):
raise runner.NotImplementedByController(
"numerics in reply to /HELP and /HELPOP (uses NOTICE instead)"
)
if self.controller.software_name == "UnrealIRCd":
raise runner.NotImplementedByController(
"fails because Unreal uses custom numerics "
"https://github.com/unrealircd/unrealircd/pull/184"
)
return f(self, command, *args, **kwargs)
return newf
class HelpTestCase(cases.BaseServerTestCase):
def _assertValidHelp(self, messages, subject):
if subject != ANYSTR:
@ -46,6 +71,7 @@ class HelpTestCase(cases.BaseServerTestCase):
@pytest.mark.parametrize("command", ["HELP", "HELPOP"])
@cases.mark_specifications("Modern")
@with_xfails
def testHelpNoArg(self, command):
self.connectClient("nick")
self.sendLine(1, f"{command}")
@ -59,6 +85,7 @@ class HelpTestCase(cases.BaseServerTestCase):
@pytest.mark.parametrize("command", ["HELP", "HELPOP"])
@cases.mark_specifications("Modern")
@with_xfails
def testHelpPrivmsg(self, command):
self.connectClient("nick")
self.sendLine(1, f"{command} PRIVMSG")
@ -71,6 +98,7 @@ class HelpTestCase(cases.BaseServerTestCase):
@pytest.mark.parametrize("command", ["HELP", "HELPOP"])
@cases.mark_specifications("Modern")
@with_xfails
def testHelpUnknownSubject(self, command):
self.connectClient("nick")
self.sendLine(1, f"{command} THISISNOTACOMMAND")

3
irctest/server_tests/info.py
View File

@ -87,6 +87,9 @@ class InfoTestCase(cases.BaseServerTestCase):
@pytest.mark.parametrize("target", ["invalid.server.example", "invalidserver"])
@cases.mark_specifications("RFC1459", "RFC2812", deprecated=True)
@cases.xfailIfSoftware(
["Ergo"], "does not apply to Ergo, which ignores the optional <target> argument"
)
def testInfoNosuchserver(self, target):
"""
<https://datatracker.ietf.org/doc/html/rfc1459#section-4.3.8>

8
irctest/server_tests/invite.py
View File

@ -200,6 +200,9 @@ class InviteTestCase(cases.BaseServerTestCase):
self._testInvite(opped=True, invite_only=invite_only)
@cases.mark_specifications("RFC1459", "RFC2812", "Modern", strict=True)
@cases.xfailIfSoftware(
["Hybrid", "Plexus4"], "the only strict test that Hybrid fails"
)
def testInviteUnopped(self):
"""Tests invites from unopped users on not-invite-only chans."""
self._testInvite(opped=False, invite_only=False)
@ -237,6 +240,11 @@ class InviteTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("RFC1459", "RFC2812", "Modern")
@cases.xfailIfSoftware(
["Plexus4"],
"Plexus4 allows non-op to invite if (and only if) the channel is not "
"invite-only",
)
def testInviteInviteOnly(self):
"""
"To invite a user to a channel which is invite only (MODE

4
irctest/server_tests/kick.py
View File

@ -96,6 +96,10 @@ class KickTestCase(cases.BaseServerTestCase):
self.assertMessageMatch(m3, command="KICK", params=["#chan", "bar", ANYSTR])
@cases.mark_specifications("RFC2812")
@cases.xfailIfSoftware(
["Charybdis", "ircu2", "irc2", "Solanum"],
"uses the nick of the kickee rather than the kicker.",
)
def testKickDefaultComment(self):
"""
"If a "comment" is

2
irctest/server_tests/list.py
View File

@ -12,6 +12,7 @@ from irctest import cases
class ListTestCase(cases.BaseServerTestCase):
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(["irc2"], "irc2 deprecated LIST")
def testListEmpty(self):
"""<https://tools.ietf.org/html/rfc1459#section-4.2.6>
<https://tools.ietf.org/html/rfc2812#section-3.2.6>
@ -40,6 +41,7 @@ class ListTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(["irc2"], "irc2 deprecated LIST")
def testListOne(self):
"""When a channel exists, LIST should get it in a reply.
<https://tools.ietf.org/html/rfc1459#section-4.2.6>

20
irctest/server_tests/lusers.py
View File

@ -153,6 +153,10 @@ class BasicLusersTestCase(LusersTestCase):
self.getLusers("bar", True)
@cases.mark_specifications("Modern")
@cases.xfailIfSoftware(
["ircu2", "Nefarious", "snircd"],
"test depends on Modern behavior, not just RFC2812",
)
def testLusersFull(self):
self.connectClient("bar", name="bar")
lusers = self.getLusers("bar", False)
@ -170,10 +174,22 @@ class BasicLusersTestCase(LusersTestCase):
class LusersUnregisteredTestCase(LusersTestCase):
@cases.mark_specifications("RFC2812")
@cases.xfailIfSoftware(
["Nefarious"],
"Nefarious doesn't seem to distinguish unregistered users from normal ones",
)
def testLusersRfc2812(self):
self.doLusersTest(True)
@cases.mark_specifications("Modern")
@cases.xfailIfSoftware(
["Nefarious"],
"Nefarious doesn't seem to distinguish unregistered users from normal ones",
)
@cases.xfailIfSoftware(
["ircu2", "Nefarious", "snircd"],
"test depends on Modern behavior, not just RFC2812",
)
def testLusersFull(self):
self.doLusersTest(False)
@ -237,6 +253,10 @@ class LusersUnregisteredDefaultInvisibleTestCase(LusersUnregisteredTestCase):
)
@cases.mark_specifications("Ergo")
@cases.xfailIfSoftware(
["Nefarious"],
"Nefarious doesn't seem to distinguish unregistered users from normal ones",
)
def testLusers(self):
self.doLusersTest(False)
lusers = self.getLusers("bar", False)

12
irctest/server_tests/messages.py
View File

@ -51,6 +51,15 @@ class NoticeTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(
["InspIRCd"],
"replies with ERR_NOSUCHCHANNEL to NOTICE to non-existent channels",
)
@cases.xfailIfSoftware(
["UnrealIRCd"],
"replies with ERR_NOSUCHCHANNEL to NOTICE to non-existent channels: "
"https://bugs.unrealircd.org/view.php?id=5949",
)
def testNoticeNonexistentChannel(self):
"""
"automatic replies must never be
@ -71,6 +80,9 @@ class NoticeTestCase(cases.BaseServerTestCase):
class TagsTestCase(cases.BaseServerTestCase):
@cases.mark_capabilities("message-tags")
@cases.xfailIfSoftware(
["UnrealIRCd"], "https://bugs.unrealircd.org/view.php?id=5947"
)
def testLineTooLong(self):
self.connectClient("bar", capabilities=["message-tags"], skip_if_cap_nak=True)
self.connectClient(

1
irctest/server_tests/quit.py
View File

@ -16,6 +16,7 @@ from irctest.patma import StrRe
class ChannelQuitTestCase(cases.BaseServerTestCase):
@cases.mark_specifications("RFC2812")
@cases.xfailIfSoftware(["ircu2", "Nefarious", "snircd"], "ircu2 does not echo QUIT")
def testQuit(self):
"""“Once a user has joined a channel, he receives information about
all commands his server receives affecting the channel. This

9
irctest/server_tests/regressions.py
View File

@ -4,7 +4,7 @@ Regression tests for bugs in `Ergo <https://ergo.chat/>`_.
import time
from irctest import cases
from irctest import cases, runner
from irctest.numerics import ERR_ERRONEUSNICKNAME, ERR_NICKNAMEINUSE, RPL_WELCOME
from irctest.patma import ANYDICT
@ -57,6 +57,12 @@ class RegressionsTestCase(cases.BaseServerTestCase):
@cases.mark_capabilities("message-tags", "batch", "echo-message", "server-time")
def testTagCap(self):
if self.controller.software_name == "UnrealIRCd":
raise runner.NotImplementedByController(
"Arbitrary +draft/reply values (TODO: adapt this test to use real "
"values so their pass Unreal's validation) "
"https://bugs.unrealircd.org/view.php?id=5948"
)
# regression test for oragono #754
self.connectClient(
"alice",
@ -99,6 +105,7 @@ class RegressionsTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("RFC1459")
@cases.xfailIfSoftware(["ngIRCd"], "wat")
def testStarNick(self):
self.addClient(1)
self.sendLine(1, "NICK *")

14
irctest/server_tests/sasl.py
View File

@ -171,6 +171,13 @@ class SaslTestCase(cases.BaseServerTestCase):
@cases.mark_specifications("IRCv3")
@cases.skipUnlessHasMechanism("PLAIN")
@cases.xfailIf(
lambda self: (
self.controller.services_controller is not None
and self.controller.services_controller.software_name == "Anope"
),
"Anope does not handle split AUTHENTICATE (reported on IRC)",
)
def testPlainLarge(self):
"""Test the client splits large AUTHENTICATE messages whose payload
is not a multiple of 400.
@ -233,6 +240,13 @@ class SaslTestCase(cases.BaseServerTestCase):
@cases.mark_specifications("IRCv3")
@cases.skipUnlessHasMechanism("PLAIN")
@cases.xfailIf(
lambda self: (
self.controller.services_controller is not None
and self.controller.services_controller.software_name == "Anope"
),
"Anope does not handle split AUTHENTICATE (reported on IRC)",
)
def testPlainLargeEquals400(self):
"""Test the client splits large AUTHENTICATE messages whose payload
is not a multiple of 400.

10
irctest/server_tests/statusmsg.py
View File

@ -17,6 +17,11 @@ class StatusmsgTestCase(cases.BaseServerTestCase):
self.assertEqual(self.server_support["STATUSMSG"], "~&@%+")
@cases.mark_isupport("STATUSMSG")
@cases.xfailIfSoftware(
["ircu2", "Nefarious", "snircd"],
"STATUSMSG is present in ISUPPORT, but it not actually supported as PRIVMSG "
"target (only for WALLCOPS/WALLCHOPS/...)",
)
def testStatusmsgFromOp(self):
"""Test that STATUSMSG are sent to the intended recipients,
with the intended prefixes."""
@ -68,6 +73,11 @@ class StatusmsgTestCase(cases.BaseServerTestCase):
self.assertEqual(len(unprivilegedMessages), 0)
@cases.mark_isupport("STATUSMSG")
@cases.xfailIfSoftware(
["ircu2", "Nefarious", "snircd"],
"STATUSMSG is present in ISUPPORT, but it not actually supported as PRIVMSG "
"target (only for WALLCOPS/WALLCHOPS/...)",
)
def testStatusmsgFromRegular(self):
"""Test that STATUSMSG are sent to the intended recipients,
with the intended prefixes."""

3
irctest/server_tests/wallops.py
View File

@ -66,6 +66,9 @@ class WallopsTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("Modern")
@cases.xfailIfSoftware(
["irc2"], "irc2 ignores the command instead of replying ERR_UNKNOWNCOMMAND"
)
def testWallopsPrivileges(self):
"""
https://github.com/ircdocs/modern-irc/pull/118

21
irctest/server_tests/who.py
View File

@ -87,6 +87,9 @@ class BaseWhoTestCase:
class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
@cases.mark_specifications("Modern")
def testWhoStar(self):
if self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHO mask")
self._init()
self.sendLine(2, "WHO *")
@ -115,6 +118,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
)
@cases.mark_specifications("Modern")
def testWhoNick(self, mask):
if "*" in mask and self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHO mask")
self._init()
self.sendLine(2, f"WHO {mask}")
@ -142,6 +148,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
ids=["username", "realname-mask", "hostname"],
)
def testWhoUsernameRealName(self, mask):
if "*" in mask and self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHO mask")
self._init()
self.sendLine(2, f"WHO :{mask}")
@ -192,6 +201,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
)
@cases.mark_specifications("Modern")
def testWhoNickAway(self, mask):
if "*" in mask and self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHO mask")
self._init()
self.sendLine(1, "AWAY :be right back")
@ -218,6 +230,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
)
@cases.mark_specifications("Modern")
def testWhoNickOper(self, mask):
if "*" in mask and self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHO mask")
self._init()
self.sendLine(1, "OPER operuser operpassword")
@ -249,6 +264,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
)
@cases.mark_specifications("Modern")
def testWhoNickAwayAndOper(self, mask):
if "*" in mask and self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHO mask")
self._init()
self.sendLine(1, "OPER operuser operpassword")
@ -280,6 +298,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
@pytest.mark.parametrize("mask", ["#chan", "#CHAN"], ids=["exact", "casefolded"])
@cases.mark_specifications("Modern")
def testWhoChan(self, mask):
if "*" in mask and self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHO mask")
self._init()
self.sendLine(1, "OPER operuser operpassword")

3
irctest/server_tests/whois.py
View File

@ -29,6 +29,9 @@ from irctest.patma import ANYSTR, StrRe
class _WhoisTestMixin(cases.BaseServerTestCase):
def _testWhoisNumerics(self, authenticate, away, oper):
if oper and self.controller.software_name == "Charybdis":
pytest.xfail("charybdis uses RPL_WHOISSPECIAL instead of RPL_WHOISOPERATOR")
if authenticate:
self.connectClient("nick1")
self.controller.registerUser(self, "val", "sesame")

37
irctest/server_tests/whowas.py
View File

@ -163,6 +163,10 @@ class WhowasTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(
["InspIRCd"],
"Feature not released yet: https://github.com/inspircd/inspircd/pull/1967",
)
def testWhowasMultiple(self):
"""
"The history is searched backward, returning the most recent entry first."
@ -172,6 +176,10 @@ class WhowasTestCase(cases.BaseServerTestCase):
self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS nick2")
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(
["InspIRCd"],
"Feature not released yet: https://github.com/inspircd/inspircd/pull/1968",
)
def testWhowasCount1(self):
"""
"If there are multiple entries, up to <count> replies will be returned"
@ -181,6 +189,10 @@ class WhowasTestCase(cases.BaseServerTestCase):
self._testWhowasMultiple(second_result=False, whowas_command="WHOWAS nick2 1")
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(
["InspIRCd"],
"Feature not released yet: https://github.com/inspircd/inspircd/pull/1968",
)
def testWhowasCount2(self):
"""
"If there are multiple entries, up to <count> replies will be returned"
@ -190,6 +202,10 @@ class WhowasTestCase(cases.BaseServerTestCase):
self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS nick2 2")
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(
["InspIRCd"],
"Feature not released yet: https://github.com/inspircd/inspircd/pull/1968",
)
def testWhowasCountNegative(self):
"""
"If a non-positive number is passed as being <count>, then a full search
@ -200,6 +216,13 @@ class WhowasTestCase(cases.BaseServerTestCase):
self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS nick2 -1")
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(
["ircu2"], "Fix not released yet: https://github.com/UndernetIRC/ircu2/pull/19"
)
@cases.xfailIfSoftware(
["InspIRCd"],
"Feature not released yet: https://github.com/inspircd/inspircd/pull/1967",
)
def testWhowasCountZero(self):
"""
"If a non-positive number is passed as being <count>, then a full search
@ -215,6 +238,9 @@ class WhowasTestCase(cases.BaseServerTestCase):
"Wildcards are allowed in the <target> parameter."
-- https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3
"""
if self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHOWAS mask")
self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS *ck2")
@cases.mark_specifications("RFC1459", "RFC2812", deprecated=True)
@ -250,6 +276,12 @@ class WhowasTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(
["Charybdis"],
"fails because of a typo (solved in "
"https://github.com/solanum-ircd/solanum/commit/"
"08b7b6bd7e60a760ad47b58cbe8075b45d66166f)",
)
def testWhowasNoSuchNick(self):
"""
https://datatracker.ietf.org/doc/html/rfc1459#section-4.5.3
@ -285,6 +317,11 @@ class WhowasTestCase(cases.BaseServerTestCase):
"""
https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3
"""
if self.controller.software_name == "Bahamut":
pytest.xfail(
"Bahamut returns entries in query order instead of chronological order"
)
self.connectClient("nick1")
targmax = dict(