val
/
irctest
mirror of https://github.com/progval/irctest.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use xfail instead of deselection for known failures (#155)

This commit is contained in:
Val Lorentz 2022-04-12 22:36:28 +02:00 committed by GitHub
parent 10b6f8d6da
commit 2bc68a2208
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 308 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
Makefile
View File

@ -7,102 +7,47 @@ PYTEST_ARGS ?=
# Will be appended at the end of the -k argument to pytest
EXTRA_SELECTORS ?=
# testPlainLarge fails because it doesn't handle split AUTHENTICATE (reported on IRC)
ANOPE_SELECTORS := \
and not testPlainLarge
# buffering tests cannot pass because of issues with UTF-8 handling: https://github.com/DALnet/bahamut/issues/196
# mask tests in test_who.py fail because they are not implemented.
# some HelpTestCase::*[HELP] tests fail because Bahamut forwards /HELP to HelpServ (but not /HELPOP)
# testWhowasMultiTarget fails because Bahamut returns the results in query order instead of chronological order
BAHAMUT_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not IRCv3 \
and not buffering \
and not (testWho and not whois and mask) \
and not testWhoStar \
and (not HelpTestCase or HELPOP) \
and not testWhowasMultiTarget \
$(EXTRA_SELECTORS)
# testQuitErrors is very flaky
# AccountTagTestCase.testInvite fails because https://github.com/solanum-ircd/solanum/issues/166
# testKickDefaultComment fails because it uses the nick of the kickee rather than the kicker.
# testWhoisNumerics[oper] fails because charybdis uses RPL_WHOISSPECIAL instead of RPL_WHOISOPERATOR
# testWhowasNoSuchNick fails because of a typo (solved in https://github.com/solanum-ircd/solanum/commit/08b7b6bd7e60a760ad47b58cbe8075b45d66166f)
CHARYBDIS_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not testQuitErrors \
and not testKickDefaultComment \
and not (AccountTagTestCase and testInvite) \
and not (testWhoisNumerics and oper) \
and not testWhowasNoSuchNick \
$(EXTRA_SELECTORS)
# testInfoNosuchserver does not apply to Ergo: Ergo ignores the optional <target> argument
ERGO_SELECTORS := \
not deprecated \
and not testInfoNosuchserver \
$(EXTRA_SELECTORS)
# testInviteUnopped is the only strict test that Hybrid fails
HYBRID_SELECTORS := \
not Ergo \
and not testInviteUnopped \
and not deprecated \
$(EXTRA_SELECTORS)
# testBotPrivateMessage and testBotChannelMessage fail because https://github.com/inspircd/inspircd/pull/1910 is not released yet
# WHOWAS tests fail because https://github.com/inspircd/inspircd/pull/1967 and https://github.com/inspircd/inspircd/pull/1968 are not released yet
INSPIRCD_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not testNoticeNonexistentChannel \
and not testBotPrivateMessage and not testBotChannelMessage \
and not whowas \
$(EXTRA_SELECTORS)
# buffering tests fail because ircu2 discards the whole buffer on long lines (TODO: refine how we exclude these tests)
# testQuit and testQuitErrors fail because ircu2 does not send ERROR or QUIT
# lusers "full" tests fail because they depend on Modern behavior, not just RFC2812
# statusmsg tests fail because STATUSMSG is present in ISUPPORT, but it not actually supported as PRIVMSG target
# testKeyValidation[empty] fails because ircu2 returns ERR_NEEDMOREPARAMS on empty keys: https://github.com/UndernetIRC/ircu2/issues/13
# testKickDefaultComment fails because it uses the nick of the kickee rather than the kicker.
# testEmptyRealname fails because it uses a default value instead of ERR_NEEDMOREPARAMS.
# HelpTestCase fails because it returns NOTICEs instead of numerics
# testWhowasCountZero fails: https://github.com/UndernetIRC/ircu2/pull/19
IRCU2_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not buffering \
and not testQuit \
and not (lusers and full) \
and not statusmsg \
and not (testKeyValidation and empty) \
and not testKickDefaultComment \
and not testEmptyRealname \
and not HelpTestCase \
and not testWhowasCountZero \
$(EXTRA_SELECTORS)
# same justification as ircu2
# lusers "unregistered" tests fail because Nefarious doesn't seem to distinguish unregistered users from normal ones
# lusers "unregistered" tests fail because
NEFARIOUS_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not buffering \
and not testQuit \
and not (lusers and unregistered) \
and not statusmsg \
and not (testKeyValidation and empty) \
and not testEmptyRealname \
$(EXTRA_SELECTORS)
# same justification as ircu2
@ -110,24 +55,12 @@ SNIRCD_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not buffering \
and not testQuit \
and not (lusers and full) \
and not statusmsg \
$(EXTRA_SELECTORS)
# testListEmpty and testListOne fails because irc2 deprecated LIST
# testKickDefaultComment fails because it uses the nick of the kickee rather than the kicker.
# testWallopsPrivileges fails because it ignores the command instead of replying ERR_UNKNOWNCOMMAND
# HelpTestCase fails because it returns NOTICEs instead of numerics
IRC2_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not testListEmpty and not testListOne \
and not testKickDefaultComment \
and not testWallopsPrivileges \
and not HelpTestCase \
$(EXTRA_SELECTORS)
MAMMON_SELECTORS := \
@ -136,28 +69,14 @@ MAMMON_SELECTORS := \
and not strict \
$(EXTRA_SELECTORS)
# testKeyValidation[spaces] and testKeyValidation[empty] fail because ngIRCd does not validate them https://github.com/ngircd/ngircd/issues/290
# testStarNick: wat
# testEmptyRealname fails because it uses a default value instead of ERR_NEEDMOREPARAMS.
# chathistory tests fail because they need nicks longer than 9 chars
# HelpTestCase::*[HELP] fails because it returns NOTICEs instead of numerics
NGIRCD_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not (testKeyValidation and (spaces or empty)) \
and not testStarNick \
and not testEmptyRealname \
and not chathistory \
and (not HelpTestCase or HELPOP) \
$(EXTRA_SELECTORS)
# testInviteUnopped is the only strict test that Plexus4 fails
# testInviteInviteOnly fails because Plexus4 allows non-op to invite if (and only if) the channel is not invite-only
PLEXUS4_SELECTORS := \
not Ergo \
and not testInviteUnopped \
and not testInviteInviteOnly \
and not deprecated \
$(EXTRA_SELECTORS)
@ -168,46 +87,27 @@ LIMNORIA_SELECTORS := \
(foo or not foo) \
$(EXTRA_SELECTORS)
# testQuitErrors is too flaky for CI
# testKickDefaultComment fails because solanum uses the nick of the kickee rather than the kicker.
SOLANUM_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not testQuitErrors \
and not testKickDefaultComment \
$(EXTRA_SELECTORS)
# Same as Limnoria
SOPEL_SELECTORS := \
not testPlainNotAvailable \
(foo or not foo) \
$(EXTRA_SELECTORS)
# testNoticeNonexistentChannel fails: https://bugs.unrealircd.org/view.php?id=5949
# regressions::testTagCap fails: https://bugs.unrealircd.org/view.php?id=5948
# messages::testLineTooLong fails: https://bugs.unrealircd.org/view.php?id=5947
# testCapRemovalByClient and testNakWhole fail pending https://github.com/unrealircd/unrealircd/pull/148
# Tests marked with arbitrary_client_tags can't pass because Unreal whitelists which tags it relays
# Tests marked with react_tag can't pass because Unreal blocks +draft/react https://github.com/unrealircd/unrealircd/pull/149
# Tests marked with private_chathistory can't pass because Unreal does not implement CHATHISTORY for DMs
# testChathistory[BETWEEN] fails: https://bugs.unrealircd.org/view.php?id=5952
# testChathistory[AROUND] fails: https://bugs.unrealircd.org/view.php?id=5953
# testWhoAllOpers fails because Unreal skips results when the mask is too broad
# HELP and HELPOP tests fail because Unreal uses custom numerics https://github.com/unrealircd/unrealircd/pull/184
UNREALIRCD_SELECTORS := \
not Ergo \
and not deprecated \
and not strict \
and not testNoticeNonexistentChannel \
and not (regressions.py and testTagCap) \
and not (messages.py and testLineTooLong) \
and not (cap.py and (testCapRemovalByClient or testNakWhole)) \
and not (account_tag.py and testInvite) \
and not arbitrary_client_tags \
and not react_tag \
and not private_chathistory \
and not (testChathistory and (between or around)) \
and not testWhoAllOpers \
and not HelpTestCase \
$(EXTRA_SELECTORS)
.PHONY: all flakes bahamut charybdis ergo inspircd ircu2 snircd irc2 mammon nefarious limnoria sopel solanum unrealircd
@ -238,7 +138,7 @@ bahamut-anope:
--services-controller=irctest.controllers.anope_services \
-m 'services' \
-n 10 \
-k '$(BAHAMUT_SELECTORS) $(ANOPE_SELECTORS)'
-k '$(BAHAMUT_SELECTORS)'
charybdis:
$(PYTEST) $(PYTEST_ARGS) \
@ -275,7 +175,7 @@ inspircd-anope:
--controller=irctest.controllers.inspircd \
--services-controller=irctest.controllers.anope_services \
-m 'services' \
-k '$(INSPIRCD_SELECTORS) $(ANOPE_SELECTORS)'
-k '$(INSPIRCD_SELECTORS)'
ircu2:
$(PYTEST) $(PYTEST_ARGS) \
@ -373,4 +273,4 @@ unrealircd-anope:
--controller=irctest.controllers.unrealircd \
--services-controller=irctest.controllers.anope_services \
-m 'services' \
-k '$(UNREALIRCD_SELECTORS) $(ANOPE_SELECTORS)'
-k '$(UNREALIRCD_SELECTORS)'

27
irctest/cases.py
View File

@ -765,6 +765,33 @@ def skipUnlessHasSasl(f: Callable[..., _TReturn]) -> Callable[..., _TReturn]:
return newf
def xfailIf(
condition: Callable[..., bool], reason: str
) -> Callable[[Callable[..., _TReturn]], Callable[..., _TReturn]]:
# Works about the same as skipUnlessHasMechanism
def decorator(f: Callable[..., _TReturn]) -> Callable[..., _TReturn]:
@functools.wraps(f)
def newf(self: _TSelf, *args: Any, **kwargs: Any) -> _TReturn:
if condition(self):
try:
return f(self, *args, **kwargs)
except Exception:
pytest.xfail(reason)
assert False # make mypy happy
else:
return f(self, *args, **kwargs)
return newf
return decorator
def xfailIfSoftware(
names: List[str], reason: str
) -> Callable[[Callable[..., _TReturn]], Callable[..., _TReturn]]:
return xfailIf(lambda testcase: testcase.controller.software_name in names, reason)
def mark_services(cls: TClass) -> TClass:
cls.run_services = True
return pytest.mark.services(cls) # type: ignore

1
irctest/client_tests/sasl.py
View File

@ -61,6 +61,7 @@ class SaslTestCase(cases.BaseClientTestCase):
self.assertEqual(m, Message({}, None, "CAP", ["END"]))
@cases.skipUnlessHasMechanism("PLAIN")
@cases.xfailIfSoftware(["Sopel"], "Sopel requests SASL PLAIN even if not available")
def testPlainNotAvailable(self):
"""`sasl=EXTERNAL` is advertized, whereas the client is configured
to use PLAIN.

2
irctest/controllers/anope_services.py
View File

@ -73,6 +73,8 @@ module {{ name = "ns_cert" }}
class AnopeController(BaseServicesController, DirectoryBasedController):
"""Collaborator for server controllers that rely on Anope"""
software_name = "Anope"
def run(self, protocol: str, server_hostname: str, server_port: int) -> None:
self.create_config()

2
irctest/controllers/atheme_services.py
View File

@ -56,6 +56,8 @@ saslserv {{
class AthemeController(BaseServicesController, DirectoryBasedController):
"""Mixin for server controllers that rely on Atheme"""
software_name = "Atheme"
def run(self, protocol: str, server_hostname: str, server_port: int) -> None:
self.create_config()

8
irctest/controllers/irc2.py
View File

@ -29,8 +29,8 @@ O:*:operpassword:operuser::::
"""
class Ircu2Controller(BaseServerController, DirectoryBasedController):
binary_name: str
class Irc2Controller(BaseServerController, DirectoryBasedController):
software_name = "irc2"
services_protocol: str
supports_sts = False
@ -89,5 +89,5 @@ class Ircu2Controller(BaseServerController, DirectoryBasedController):
)
def get_irctest_controller_class() -> Type[Ircu2Controller]:
return Ircu2Controller
def get_irctest_controller_class() -> Type[Irc2Controller]:
return Irc2Controller

2
irctest/controllers/ircu2.py
View File

@ -51,7 +51,7 @@ features {{
class Ircu2Controller(BaseServerController, DirectoryBasedController):
software_name = "Ircu2"
software_name = "ircu2"
supports_sts = False
extban_mute_char = None

2
irctest/controllers/plexus4.py
View File

@ -74,7 +74,7 @@ operator {{
class Plexus4Controller(BaseHybridController):
software_name = "Hybrid"
software_name = "Plexus4"
binary_name = "ircd"
services_protocol = "plexus"

5
irctest/dashboard/format.py
View File

@ -208,6 +208,9 @@ def build_module_html(
cell.set("class", "skipped")
if result.type == "pytest.skip":
text = "s"
elif result.type == "pytest.xfail":
text = "X"
cell.set("class", "expected-failure")
else:
text = result.type
elif result.success:
@ -231,6 +234,8 @@ def build_module_html(
a.text = text or "?"
else:
cell.text = text or "?"
if result.message:
cell.set("title", result.message)
return root

3
irctest/dashboard/style.css
View File

@ -46,6 +46,9 @@ table.test-matrix .skipped {
table.test-matrix .failure {
background-color: red;
}
table.test-matrix .expected-failure {
background-color: orange;
}
/* Rotate headers, thanks to https://css-tricks.com/rotated-table-column-headers/ */
th.job-name {

3
irctest/server_tests/account_tag.py
View File

@ -55,6 +55,9 @@ class AccountTagTestCase(cases.BaseServerTestCase):
@cases.mark_capabilities("account-tag")
@cases.skipUnlessHasMechanism("PLAIN")
@cases.xfailIfSoftware(
["Charybdis"], "https://github.com/solanum-ircd/solanum/issues/166"
)
def testInvite(self):
self.connectClient("foo", capabilities=["account-tag"], skip_if_cap_nak=True)
self.getMessages(1)

8
irctest/server_tests/bot_mode.py
View File

@ -67,6 +67,10 @@ class BotModeTestCase(cases.BaseServerTestCase):
message, command=RPL_WHOISBOT, params=["usernick", "botnick", ANYSTR]
)
@cases.xfailIfSoftware(
["InspIRCd"],
"Uses only vendor tags for now: https://github.com/inspircd/inspircd/pull/1910",
)
def testBotPrivateMessage(self):
self._initBot()
@ -84,6 +88,10 @@ class BotModeTestCase(cases.BaseServerTestCase):
tags={"draft/bot": None, **ANYDICT},
)
@cases.xfailIfSoftware(
["InspIRCd"],
"Uses only vendor tags for now: https://github.com/inspircd/inspircd/pull/1910",
)
def testBotChannelMessage(self):
self._initBot()

10
irctest/server_tests/buffering.py
View File

@ -32,6 +32,16 @@ def _sendBytePerByte(self, line):
class BufferingTestCase(cases.BaseServerTestCase):
@cases.xfailIfSoftware(
["Bahamut"],
"cannot pass because of issues with UTF-8 handling: "
"https://github.com/DALnet/bahamut/issues/196",
)
@cases.xfailIfSoftware(
["ircu2", "Nefarious", "snircd"],
"ircu2 discards the whole buffer on long lines "
"(TODO: refine how we exclude these tests)",
)
@pytest.mark.parametrize(
"sender_function,colon",
[

8
irctest/server_tests/cap.py
View File

@ -78,6 +78,10 @@ class CapTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("IRCv3")
@cases.xfailIfSoftware(
["UnrealIRCd"],
"UnrealIRCd sends a trailing space on CAP NAK: https://github.com/unrealircd/unrealircd/pull/148",
)
def testNakWhole(self):
"""“The capability identifier set must be accepted as a whole, or
rejected entirely.”
@ -125,6 +129,10 @@ class CapTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("IRCv3")
@cases.xfailIfSoftware(
["UnrealIRCd"],
"UnrealIRCd sends a trailing space on CAP NAK: https://github.com/unrealircd/unrealircd/pull/148",
)
def testCapRemovalByClient(self):
"""Test CAP LIST and removal of caps via CAP REQ :-tagname."""
cap1 = "echo-message"

31
irctest/server_tests/chathistory.py
View File

@ -2,12 +2,13 @@
`IRCv3 draft chathistory <https://ircv3.net/specs/extensions/chathistory>`_
"""
import functools
import secrets
import time
import pytest
from irctest import cases
from irctest import cases, runner
from irctest.irc_utils.junkdrawer import random_name
from irctest.patma import ANYSTR
@ -42,6 +43,16 @@ def validate_chathistory_batch(msgs):
return result
def skip_ngircd(f):
@functools.wraps(f)
def newf(self, *args, **kwargs):
if self.controller.software_name == "ngIRCd":
raise runner.NotImplementedByController("nicks longer 9 characters")
return f(self, *args, **kwargs)
return newf
@cases.mark_specifications("IRCv3")
@cases.mark_services
class ChathistoryTestCase(cases.BaseServerTestCase):
@ -49,6 +60,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
def config() -> cases.TestCaseControllerConfig:
return cases.TestCaseControllerConfig(chathistory=True)
@skip_ngircd
def testInvalidTargets(self):
bar, pw = random_name("bar"), random_name("pw")
self.controller.registerUser(self, bar, pw)
@ -94,6 +106,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
)
@pytest.mark.private_chathistory
@skip_ngircd
def testMessagesToSelf(self):
bar, pw = random_name("bar"), random_name("pw")
self.controller.registerUser(self, bar, pw)
@ -166,7 +179,19 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
self.assertEqual(len(set(msg.time for msg in echo_messages)), num_messages)
@pytest.mark.parametrize("subcommand", SUBCOMMANDS)
@skip_ngircd
def testChathistory(self, subcommand):
if subcommand == "BETWEEN" and self.controller.software_name == "UnrealIRCd":
pytest.xfail(
"CHATHISTORY BETWEEN does not apply bounds correct "
"https://bugs.unrealircd.org/view.php?id=5952"
)
if subcommand == "AROUND" and self.controller.software_name == "UnrealIRCd":
pytest.xfail(
"CHATHISTORY AROUND excludes 'central' messages "
"https://bugs.unrealircd.org/view.php?id=5953"
)
self.connectClient(
"bar",
capabilities=[
@ -198,6 +223,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
self.validate_chathistory(subcommand, echo_messages, 1, chname)
@pytest.mark.parametrize("subcommand", SUBCOMMANDS)
@skip_ngircd
def testChathistoryEventPlayback(self, subcommand):
self.connectClient(
"bar",
@ -231,6 +257,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
@pytest.mark.parametrize("subcommand", SUBCOMMANDS)
@pytest.mark.private_chathistory
@skip_ngircd
def testChathistoryDMs(self, subcommand):
c1 = "foo" + secrets.token_hex(12)
c2 = "bar" + secrets.token_hex(12)
@ -553,6 +580,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
self.assertIn(echo_messages[7], result)
@pytest.mark.arbitrary_client_tags
@skip_ngircd
def testChathistoryTagmsg(self):
c1 = "foo" + secrets.token_hex(12)
c2 = "bar" + secrets.token_hex(12)
@ -651,6 +679,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
@pytest.mark.arbitrary_client_tags
@pytest.mark.private_chathistory
@skip_ngircd
def testChathistoryDMClientOnlyTags(self):
# regression test for Ergo #1411
c1 = "foo" + secrets.token_hex(12)

15
irctest/server_tests/chmodes/key.py
View File

@ -64,6 +64,21 @@ class KeyTestCase(cases.BaseServerTestCase):
-- https://modern.ircdocs.horse/#key-channel-mode
-- https://github.com/ircdocs/modern-irc/pull/111
"""
if key == "" and self.controller.software_name in (
"ircu2",
"Nefarious",
"snircd",
):
pytest.xfail(
"ircu2 returns ERR_NEEDMOREPARAMS on empty keys: "
"https://github.com/UndernetIRC/ircu2/issues/13"
)
if (key == "" or " " in key) and self.controller.software_name == "ngIRCd":
pytest.xfail(
"ngIRCd does not validate channel keys: "
"https://github.com/ngircd/ngircd/issues/290"
)
self.connectClient("bar")
self.joinChannel(1, "#chan")
self.sendLine(1, f"MODE #chan +k :{key}")

8
irctest/server_tests/connection_registration.py
View File

@ -84,6 +84,10 @@ class ConnectionRegistrationTestCase(cases.BaseServerTestCase):
self.getMessages(1)
@cases.mark_specifications("RFC2812")
@cases.xfailIfSoftware(["Charybdis", "Solanum"], "very flaky")
@cases.xfailIfSoftware(
["ircu2", "Nefarious", "snircd"], "ircu2 does not send ERROR"
)
def testQuitErrors(self):
"""“A client session is terminated with a quit message. The server
acknowledges this by sending an ERROR message to the client.”
@ -164,6 +168,10 @@ class ConnectionRegistrationTestCase(cases.BaseServerTestCase):
"neither got 001.",
)
@cases.xfailIfSoftware(
["ircu2", "Nefarious", "ngIRCd"],
"uses a default value instead of ERR_NEEDMOREPARAMS",
)
def testEmptyRealname(self):
"""
Syntax:

28
irctest/server_tests/help.py
View File

@ -2,6 +2,7 @@
The HELP and HELPOP command (`Modern <https://modern.ircdocs.horse/#help-message>`__)
"""
import functools
import re
import pytest
@ -17,6 +18,30 @@ from irctest.numerics import (
from irctest.patma import ANYSTR, StrRe
def with_xfails(f):
@functools.wraps(f)
def newf(self, command, *args, **kwargs):
if command == "HELP" and self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController(
"fail because Bahamut forwards /HELP to HelpServ (but not /HELPOP)"
)
if self.controller.software_name in ("irc2", "ircu2", "ngIRCd"):
raise runner.NotImplementedByController(
"numerics in reply to /HELP and /HELPOP (uses NOTICE instead)"
)
if self.controller.software_name == "UnrealIRCd":
raise runner.NotImplementedByController(
"fails because Unreal uses custom numerics "
"https://github.com/unrealircd/unrealircd/pull/184"
)
return f(self, command, *args, **kwargs)
return newf
class HelpTestCase(cases.BaseServerTestCase):
def _assertValidHelp(self, messages, subject):
if subject != ANYSTR:
@ -46,6 +71,7 @@ class HelpTestCase(cases.BaseServerTestCase):
@pytest.mark.parametrize("command", ["HELP", "HELPOP"])
@cases.mark_specifications("Modern")
@with_xfails
def testHelpNoArg(self, command):
self.connectClient("nick")
self.sendLine(1, f"{command}")
@ -59,6 +85,7 @@ class HelpTestCase(cases.BaseServerTestCase):
@pytest.mark.parametrize("command", ["HELP", "HELPOP"])
@cases.mark_specifications("Modern")
@with_xfails
def testHelpPrivmsg(self, command):
self.connectClient("nick")
self.sendLine(1, f"{command} PRIVMSG")
@ -71,6 +98,7 @@ class HelpTestCase(cases.BaseServerTestCase):
@pytest.mark.parametrize("command", ["HELP", "HELPOP"])
@cases.mark_specifications("Modern")
@with_xfails
def testHelpUnknownSubject(self, command):
self.connectClient("nick")
self.sendLine(1, f"{command} THISISNOTACOMMAND")

3
irctest/server_tests/info.py
View File

@ -87,6 +87,9 @@ class InfoTestCase(cases.BaseServerTestCase):
@pytest.mark.parametrize("target", ["invalid.server.example", "invalidserver"])
@cases.mark_specifications("RFC1459", "RFC2812", deprecated=True)
@cases.xfailIfSoftware(
["Ergo"], "does not apply to Ergo, which ignores the optional <target> argument"
)
def testInfoNosuchserver(self, target):
"""
<https://datatracker.ietf.org/doc/html/rfc1459#section-4.3.8>

8
irctest/server_tests/invite.py
View File

@ -200,6 +200,9 @@ class InviteTestCase(cases.BaseServerTestCase):
self._testInvite(opped=True, invite_only=invite_only)
@cases.mark_specifications("RFC1459", "RFC2812", "Modern", strict=True)
@cases.xfailIfSoftware(
["Hybrid", "Plexus4"], "the only strict test that Hybrid fails"
)
def testInviteUnopped(self):
"""Tests invites from unopped users on not-invite-only chans."""
self._testInvite(opped=False, invite_only=False)
@ -237,6 +240,11 @@ class InviteTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("RFC1459", "RFC2812", "Modern")
@cases.xfailIfSoftware(
["Plexus4"],
"Plexus4 allows non-op to invite if (and only if) the channel is not "
"invite-only",
)
def testInviteInviteOnly(self):
"""
"To invite a user to a channel which is invite only (MODE

4
irctest/server_tests/kick.py
View File

@ -96,6 +96,10 @@ class KickTestCase(cases.BaseServerTestCase):
self.assertMessageMatch(m3, command="KICK", params=["#chan", "bar", ANYSTR])
@cases.mark_specifications("RFC2812")
@cases.xfailIfSoftware(
["Charybdis", "ircu2", "irc2", "Solanum"],
"uses the nick of the kickee rather than the kicker.",
)
def testKickDefaultComment(self):
"""
"If a "comment" is

2
irctest/server_tests/list.py
View File

@ -12,6 +12,7 @@ from irctest import cases
class ListTestCase(cases.BaseServerTestCase):
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(["irc2"], "irc2 deprecated LIST")
def testListEmpty(self):
"""<https://tools.ietf.org/html/rfc1459#section-4.2.6>
<https://tools.ietf.org/html/rfc2812#section-3.2.6>
@ -40,6 +41,7 @@ class ListTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(["irc2"], "irc2 deprecated LIST")
def testListOne(self):
"""When a channel exists, LIST should get it in a reply.
<https://tools.ietf.org/html/rfc1459#section-4.2.6>

20
irctest/server_tests/lusers.py
View File

@ -153,6 +153,10 @@ class BasicLusersTestCase(LusersTestCase):
self.getLusers("bar", True)
@cases.mark_specifications("Modern")
@cases.xfailIfSoftware(
["ircu2", "Nefarious", "snircd"],
"test depends on Modern behavior, not just RFC2812",
)
def testLusersFull(self):
self.connectClient("bar", name="bar")
lusers = self.getLusers("bar", False)
@ -170,10 +174,22 @@ class BasicLusersTestCase(LusersTestCase):
class LusersUnregisteredTestCase(LusersTestCase):
@cases.mark_specifications("RFC2812")
@cases.xfailIfSoftware(
["Nefarious"],
"Nefarious doesn't seem to distinguish unregistered users from normal ones",
)
def testLusersRfc2812(self):
self.doLusersTest(True)
@cases.mark_specifications("Modern")
@cases.xfailIfSoftware(
["Nefarious"],
"Nefarious doesn't seem to distinguish unregistered users from normal ones",
)
@cases.xfailIfSoftware(
["ircu2", "Nefarious", "snircd"],
"test depends on Modern behavior, not just RFC2812",
)
def testLusersFull(self):
self.doLusersTest(False)
@ -237,6 +253,10 @@ class LusersUnregisteredDefaultInvisibleTestCase(LusersUnregisteredTestCase):
)
@cases.mark_specifications("Ergo")
@cases.xfailIfSoftware(
["Nefarious"],
"Nefarious doesn't seem to distinguish unregistered users from normal ones",
)
def testLusers(self):
self.doLusersTest(False)
lusers = self.getLusers("bar", False)

12
irctest/server_tests/messages.py
View File

@ -51,6 +51,15 @@ class NoticeTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(
["InspIRCd"],
"replies with ERR_NOSUCHCHANNEL to NOTICE to non-existent channels",
)
@cases.xfailIfSoftware(
["UnrealIRCd"],
"replies with ERR_NOSUCHCHANNEL to NOTICE to non-existent channels: "
"https://bugs.unrealircd.org/view.php?id=5949",
)
def testNoticeNonexistentChannel(self):
"""
"automatic replies must never be
@ -71,6 +80,9 @@ class NoticeTestCase(cases.BaseServerTestCase):
class TagsTestCase(cases.BaseServerTestCase):
@cases.mark_capabilities("message-tags")
@cases.xfailIfSoftware(
["UnrealIRCd"], "https://bugs.unrealircd.org/view.php?id=5947"
)
def testLineTooLong(self):
self.connectClient("bar", capabilities=["message-tags"], skip_if_cap_nak=True)
self.connectClient(

1
irctest/server_tests/quit.py
View File

@ -16,6 +16,7 @@ from irctest.patma import StrRe
class ChannelQuitTestCase(cases.BaseServerTestCase):
@cases.mark_specifications("RFC2812")
@cases.xfailIfSoftware(["ircu2", "Nefarious", "snircd"], "ircu2 does not echo QUIT")
def testQuit(self):
"""“Once a user has joined a channel, he receives information about
all commands his server receives affecting the channel. This

9
irctest/server_tests/regressions.py
View File

@ -4,7 +4,7 @@ Regression tests for bugs in `Ergo <https://ergo.chat/>`_.
import time
from irctest import cases
from irctest import cases, runner
from irctest.numerics import ERR_ERRONEUSNICKNAME, ERR_NICKNAMEINUSE, RPL_WELCOME
from irctest.patma import ANYDICT
@ -57,6 +57,12 @@ class RegressionsTestCase(cases.BaseServerTestCase):
@cases.mark_capabilities("message-tags", "batch", "echo-message", "server-time")
def testTagCap(self):
if self.controller.software_name == "UnrealIRCd":
raise runner.NotImplementedByController(
"Arbitrary +draft/reply values (TODO: adapt this test to use real "
"values so their pass Unreal's validation) "
"https://bugs.unrealircd.org/view.php?id=5948"
)
# regression test for oragono #754
self.connectClient(
"alice",
@ -99,6 +105,7 @@ class RegressionsTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("RFC1459")
@cases.xfailIfSoftware(["ngIRCd"], "wat")
def testStarNick(self):
self.addClient(1)
self.sendLine(1, "NICK *")

14
irctest/server_tests/sasl.py
View File

@ -171,6 +171,13 @@ class SaslTestCase(cases.BaseServerTestCase):
@cases.mark_specifications("IRCv3")
@cases.skipUnlessHasMechanism("PLAIN")
@cases.xfailIf(
lambda self: (
self.controller.services_controller is not None
and self.controller.services_controller.software_name == "Anope"
),
"Anope does not handle split AUTHENTICATE (reported on IRC)",
)
def testPlainLarge(self):
"""Test the client splits large AUTHENTICATE messages whose payload
is not a multiple of 400.
@ -233,6 +240,13 @@ class SaslTestCase(cases.BaseServerTestCase):
@cases.mark_specifications("IRCv3")
@cases.skipUnlessHasMechanism("PLAIN")
@cases.xfailIf(
lambda self: (
self.controller.services_controller is not None
and self.controller.services_controller.software_name == "Anope"
),
"Anope does not handle split AUTHENTICATE (reported on IRC)",
)
def testPlainLargeEquals400(self):
"""Test the client splits large AUTHENTICATE messages whose payload
is not a multiple of 400.

10
irctest/server_tests/statusmsg.py
View File

@ -17,6 +17,11 @@ class StatusmsgTestCase(cases.BaseServerTestCase):
self.assertEqual(self.server_support["STATUSMSG"], "~&@%+")
@cases.mark_isupport("STATUSMSG")
@cases.xfailIfSoftware(
["ircu2", "Nefarious", "snircd"],
"STATUSMSG is present in ISUPPORT, but it not actually supported as PRIVMSG "
"target (only for WALLCOPS/WALLCHOPS/...)",
)
def testStatusmsgFromOp(self):
"""Test that STATUSMSG are sent to the intended recipients,
with the intended prefixes."""
@ -68,6 +73,11 @@ class StatusmsgTestCase(cases.BaseServerTestCase):
self.assertEqual(len(unprivilegedMessages), 0)
@cases.mark_isupport("STATUSMSG")
@cases.xfailIfSoftware(
["ircu2", "Nefarious", "snircd"],
"STATUSMSG is present in ISUPPORT, but it not actually supported as PRIVMSG "
"target (only for WALLCOPS/WALLCHOPS/...)",
)
def testStatusmsgFromRegular(self):
"""Test that STATUSMSG are sent to the intended recipients,
with the intended prefixes."""

3
irctest/server_tests/wallops.py
View File

@ -66,6 +66,9 @@ class WallopsTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("Modern")
@cases.xfailIfSoftware(
["irc2"], "irc2 ignores the command instead of replying ERR_UNKNOWNCOMMAND"
)
def testWallopsPrivileges(self):
"""
https://github.com/ircdocs/modern-irc/pull/118

21
irctest/server_tests/who.py
View File

@ -87,6 +87,9 @@ class BaseWhoTestCase:
class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
@cases.mark_specifications("Modern")
def testWhoStar(self):
if self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHO mask")
self._init()
self.sendLine(2, "WHO *")
@ -115,6 +118,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
)
@cases.mark_specifications("Modern")
def testWhoNick(self, mask):
if "*" in mask and self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHO mask")
self._init()
self.sendLine(2, f"WHO {mask}")
@ -142,6 +148,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
ids=["username", "realname-mask", "hostname"],
)
def testWhoUsernameRealName(self, mask):
if "*" in mask and self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHO mask")
self._init()
self.sendLine(2, f"WHO :{mask}")
@ -192,6 +201,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
)
@cases.mark_specifications("Modern")
def testWhoNickAway(self, mask):
if "*" in mask and self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHO mask")
self._init()
self.sendLine(1, "AWAY :be right back")
@ -218,6 +230,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
)
@cases.mark_specifications("Modern")
def testWhoNickOper(self, mask):
if "*" in mask and self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHO mask")
self._init()
self.sendLine(1, "OPER operuser operpassword")
@ -249,6 +264,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
)
@cases.mark_specifications("Modern")
def testWhoNickAwayAndOper(self, mask):
if "*" in mask and self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHO mask")
self._init()
self.sendLine(1, "OPER operuser operpassword")
@ -280,6 +298,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
@pytest.mark.parametrize("mask", ["#chan", "#CHAN"], ids=["exact", "casefolded"])
@cases.mark_specifications("Modern")
def testWhoChan(self, mask):
if "*" in mask and self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHO mask")
self._init()
self.sendLine(1, "OPER operuser operpassword")

3
irctest/server_tests/whois.py
View File

@ -29,6 +29,9 @@ from irctest.patma import ANYSTR, StrRe
class _WhoisTestMixin(cases.BaseServerTestCase):
def _testWhoisNumerics(self, authenticate, away, oper):
if oper and self.controller.software_name == "Charybdis":
pytest.xfail("charybdis uses RPL_WHOISSPECIAL instead of RPL_WHOISOPERATOR")
if authenticate:
self.connectClient("nick1")
self.controller.registerUser(self, "val", "sesame")

37
irctest/server_tests/whowas.py
View File

@ -163,6 +163,10 @@ class WhowasTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(
["InspIRCd"],
"Feature not released yet: https://github.com/inspircd/inspircd/pull/1967",
)
def testWhowasMultiple(self):
"""
"The history is searched backward, returning the most recent entry first."
@ -172,6 +176,10 @@ class WhowasTestCase(cases.BaseServerTestCase):
self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS nick2")
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(
["InspIRCd"],
"Feature not released yet: https://github.com/inspircd/inspircd/pull/1968",
)
def testWhowasCount1(self):
"""
"If there are multiple entries, up to <count> replies will be returned"
@ -181,6 +189,10 @@ class WhowasTestCase(cases.BaseServerTestCase):
self._testWhowasMultiple(second_result=False, whowas_command="WHOWAS nick2 1")
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(
["InspIRCd"],
"Feature not released yet: https://github.com/inspircd/inspircd/pull/1968",
)
def testWhowasCount2(self):
"""
"If there are multiple entries, up to <count> replies will be returned"
@ -190,6 +202,10 @@ class WhowasTestCase(cases.BaseServerTestCase):
self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS nick2 2")
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(
["InspIRCd"],
"Feature not released yet: https://github.com/inspircd/inspircd/pull/1968",
)
def testWhowasCountNegative(self):
"""
"If a non-positive number is passed as being <count>, then a full search
@ -200,6 +216,13 @@ class WhowasTestCase(cases.BaseServerTestCase):
self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS nick2 -1")
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(
["ircu2"], "Fix not released yet: https://github.com/UndernetIRC/ircu2/pull/19"
)
@cases.xfailIfSoftware(
["InspIRCd"],
"Feature not released yet: https://github.com/inspircd/inspircd/pull/1967",
)
def testWhowasCountZero(self):
"""
"If a non-positive number is passed as being <count>, then a full search
@ -215,6 +238,9 @@ class WhowasTestCase(cases.BaseServerTestCase):
"Wildcards are allowed in the <target> parameter."
-- https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3
"""
if self.controller.software_name == "Bahamut":
raise runner.NotImplementedByController("WHOWAS mask")
self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS *ck2")
@cases.mark_specifications("RFC1459", "RFC2812", deprecated=True)
@ -250,6 +276,12 @@ class WhowasTestCase(cases.BaseServerTestCase):
)
@cases.mark_specifications("RFC1459", "RFC2812")
@cases.xfailIfSoftware(
["Charybdis"],
"fails because of a typo (solved in "
"https://github.com/solanum-ircd/solanum/commit/"
"08b7b6bd7e60a760ad47b58cbe8075b45d66166f)",
)
def testWhowasNoSuchNick(self):
"""
https://datatracker.ietf.org/doc/html/rfc1459#section-4.5.3
@ -285,6 +317,11 @@ class WhowasTestCase(cases.BaseServerTestCase):
"""
https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3
"""
if self.controller.software_name == "Bahamut":
pytest.xfail(
"Bahamut returns entries in query order instead of chronological order"
)
self.connectClient("nick1")
targmax = dict(