From 77272f83fb71c84e252afc404717e82aaa9f504e Mon Sep 17 00:00:00 2001 From: Val Lorentz Date: Sat, 10 Jul 2021 16:33:32 +0200 Subject: [PATCH] Fix Hybrid support + enable it on CI (#82) * Fix Hybrid support + enable it on CI * Can't make Hybrid linking work on Github CI because the reverse DNS is 'cpu-pool.com' for some reason, and I don't want to hardcode it, so I give up. --- .github/workflows/test-devel.yml | 70 +++++++++++ .github/workflows/test-stable.yml | 70 +++++++++++ Makefile | 11 ++ irctest/basecontrollers.py | 2 +- irctest/controllers/anope_services.py | 2 +- irctest/controllers/base_hybrid.py | 97 +++++++++++++++ irctest/controllers/charybdis.py | 91 ++------------ irctest/controllers/hybrid.py | 116 +++++++----------- irctest/server_tests/test_bot_mode.py | 12 +- .../server_tests/test_channel_operations.py | 13 ++ irctest/server_tests/test_sasl.py | 5 +- workflows.yml | 21 +++- 12 files changed, 345 insertions(+), 165 deletions(-) create mode 100644 irctest/controllers/base_hybrid.py diff --git a/.github/workflows/test-devel.yml b/.github/workflows/test-devel.yml index 6379206..5ff4662 100644 --- a/.github/workflows/test-devel.yml +++ b/.github/workflows/test-devel.yml @@ -29,6 +29,43 @@ jobs: name: installed-anope path: ~/artefacts-*.tar.gz retention-days: 1 + build-hybrid: + runs-on: ubuntu-latest + steps: + - name: Create directories + run: cd ~/; mkdir -p .local/ go/ + - name: Cache dependencies + uses: actions/cache@v2 + with: + key: ${{ runner.os }}-hybrid-devel + path: |- + ~/.cache + $GITHUB_WORKSPACE/ircd-hybrid + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Checkout Hybrid + uses: actions/checkout@v2 + with: + path: ircd-hybrid + ref: 8.2.x + repository: ircd-hybrid/ircd-hybrid + - name: Build Hybrid + run: | + cd $GITHUB_WORKSPACE/ircd-hybrid/ + ./configure --prefix=$HOME/.local/ + make -j 4 + make install + - name: Make artefact tarball + run: cd ~; tar -czf artefacts-hybrid.tar.gz .local/ go/ + - name: Upload build artefacts + uses: actions/upload-artifact@v2 + with: + name: installed-hybrid + path: ~/artefacts-*.tar.gz + retention-days: 1 build-inspircd: runs-on: ubuntu-latest steps: @@ -145,6 +182,7 @@ jobs: name: Publish Unit Tests Results needs: - test-ergo + - test-hybrid - test-inspircd - test-inspircd-anope - test-limnoria @@ -202,6 +240,38 @@ jobs: with: name: pytest results ergo (devel) path: pytest.xml + test-hybrid: + needs: + - build-hybrid + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Download build artefacts + uses: actions/download-artifact@v2 + with: + name: installed-hybrid + path: '~' + - name: Unpack artefacts + run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \; + - name: Install Atheme + run: sudo apt-get install atheme-services + - name: Install irctest dependencies + run: |- + python -m pip install --upgrade pip + pip install pytest -r requirements.txt + - name: Test with pytest + run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH make + hybrid + - if: always() + name: Publish results + uses: actions/upload-artifact@v2 + with: + name: pytest results hybrid (devel) + path: pytest.xml test-inspircd: needs: - build-inspircd diff --git a/.github/workflows/test-stable.yml b/.github/workflows/test-stable.yml index 1531ff2..d7a5e57 100644 --- a/.github/workflows/test-stable.yml +++ b/.github/workflows/test-stable.yml @@ -67,6 +67,43 @@ jobs: name: installed-charybdis path: ~/artefacts-*.tar.gz retention-days: 1 + build-hybrid: + runs-on: ubuntu-latest + steps: + - name: Create directories + run: cd ~/; mkdir -p .local/ go/ + - name: Cache dependencies + uses: actions/cache@v2 + with: + key: ${{ runner.os }}-hybrid-stable + path: |- + ~/.cache + $GITHUB_WORKSPACE/ircd-hybrid + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Checkout Hybrid + uses: actions/checkout@v2 + with: + path: ircd-hybrid + ref: 8.2.38 + repository: ircd-hybrid/ircd-hybrid + - name: Build Hybrid + run: | + cd $GITHUB_WORKSPACE/ircd-hybrid/ + ./configure --prefix=$HOME/.local/ + make -j 4 + make install + - name: Make artefact tarball + run: cd ~; tar -czf artefacts-hybrid.tar.gz .local/ go/ + - name: Upload build artefacts + uses: actions/upload-artifact@v2 + with: + name: installed-hybrid + path: ~/artefacts-*.tar.gz + retention-days: 1 build-inspircd: runs-on: ubuntu-latest steps: @@ -184,6 +221,7 @@ jobs: needs: - test-charybdis - test-ergo + - test-hybrid - test-inspircd - test-inspircd-anope - test-inspircd-atheme @@ -274,6 +312,38 @@ jobs: with: name: pytest results ergo (stable) path: pytest.xml + test-hybrid: + needs: + - build-hybrid + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Download build artefacts + uses: actions/download-artifact@v2 + with: + name: installed-hybrid + path: '~' + - name: Unpack artefacts + run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \; + - name: Install Atheme + run: sudo apt-get install atheme-services + - name: Install irctest dependencies + run: |- + python -m pip install --upgrade pip + pip install pytest -r requirements.txt + - name: Test with pytest + run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH make + hybrid + - if: always() + name: Publish results + uses: actions/upload-artifact@v2 + with: + name: pytest results hybrid (stable) + path: pytest.xml test-inspircd: needs: - build-inspircd diff --git a/Makefile b/Makefile index 0543a89..b1d9682 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,11 @@ ERGO_SELECTORS := \ not deprecated \ $(EXTRA_SELECTORS) +HYBRID_SELECTORS := \ + not Ergo \ + and not deprecated \ + $(EXTRA_SELECTORS) + # testNoticeNonexistentChannel fails because of https://github.com/inspircd/inspircd/issues/1849 # testDirectMessageEcho fails because of https://github.com/inspircd/inspircd/issues/1851 # testKeyValidation fails because of https://github.com/inspircd/inspircd/issues/1850 @@ -108,6 +113,12 @@ ergo: --controller irctest.controllers.ergo \ -k "$(ERGO_SELECTORS)" +hybrid: + $(PYTEST) $(PYTEST_ARGS) \ + --controller irctest.controllers.hybrid \ + -m 'not services' \ + -k "$(HYBRID_SELECTORS)" + inspircd: $(PYTEST) $(PYTEST_ARGS) \ --controller=irctest.controllers.inspircd \ diff --git a/irctest/basecontrollers.py b/irctest/basecontrollers.py index 2795690..42e9ff3 100644 --- a/irctest/basecontrollers.py +++ b/irctest/basecontrollers.py @@ -224,7 +224,7 @@ class BaseServerController(_BaseController): # test_lusers.py (eg. this happens with Charybdis 3.5.0) c.send(b"QUIT :chkport\r\n") data = b"" - while b"chkport" not in data: + while b"chkport" not in data and b"ERROR" not in data: data += c.recv(1024) c.close() diff --git a/irctest/controllers/anope_services.py b/irctest/controllers/anope_services.py index 025d98d..9aaaf71 100644 --- a/irctest/controllers/anope_services.py +++ b/irctest/controllers/anope_services.py @@ -76,7 +76,7 @@ class AnopeController(BaseServicesController, DirectoryBasedController): def run(self, protocol: str, server_hostname: str, server_port: int) -> None: self.create_config() - assert protocol in ("inspircd3", "charybdis", "unreal4") + assert protocol in ("inspircd3", "charybdis", "hybrid", "unreal4") with self.open_file("conf/services.conf") as fd: fd.write( diff --git a/irctest/controllers/base_hybrid.py b/irctest/controllers/base_hybrid.py new file mode 100644 index 0000000..11a5bc9 --- /dev/null +++ b/irctest/controllers/base_hybrid.py @@ -0,0 +1,97 @@ +import os +import subprocess +from typing import Optional, Set + +from irctest.basecontrollers import ( + BaseServerController, + DirectoryBasedController, + NotImplementedByController, +) +from irctest.irc_utils.junkdrawer import find_hostname_and_port + +TEMPLATE_SSL_CONFIG = """ + ssl_private_key = "{key_path}"; + ssl_cert = "{pem_path}"; + ssl_dh_params = "{dh_path}"; +""" + + +class BaseHybridController(BaseServerController, DirectoryBasedController): + """A base class for all controllers derived from ircd-hybrid (Hybrid itself, + Charybdis, Solanum, ...)""" + + binary_name: str + services_protocol: str + + supports_sts = False + extban_mute_char = None + + template_config: str + + def create_config(self) -> None: + super().create_config() + with self.open_file("server.conf"): + pass + + def run( + self, + hostname: str, + port: int, + *, + password: Optional[str], + ssl: bool, + run_services: bool, + valid_metadata_keys: Optional[Set[str]] = None, + invalid_metadata_keys: Optional[Set[str]] = None, + ) -> None: + if valid_metadata_keys or invalid_metadata_keys: + raise NotImplementedByController( + "Defining valid and invalid METADATA keys." + ) + assert self.proc is None + self.port = port + self.hostname = hostname + self.create_config() + (services_hostname, services_port) = find_hostname_and_port() + password_field = 'password = "{}";'.format(password) if password else "" + if ssl: + self.gen_ssl() + ssl_config = TEMPLATE_SSL_CONFIG.format( + key_path=self.key_path, pem_path=self.pem_path, dh_path=self.dh_path + ) + else: + ssl_config = "" + with self.open_file("server.conf") as fd: + fd.write( + (self.template_config).format( + hostname=hostname, + port=port, + services_hostname=services_hostname, + services_port=services_port, + password_field=password_field, + ssl_config=ssl_config, + ) + ) + assert self.directory + self.proc = subprocess.Popen( + [ + self.binary_name, + "-foreground", + "-configfile", + os.path.join(self.directory, "server.conf"), + "-pidfile", + os.path.join(self.directory, "server.pid"), + ], + # stderr=subprocess.DEVNULL, + ) + + if run_services: + self.wait_for_port() + self.services_controller = self.services_controller_class( + self.test_config, self + ) + self.services_controller.run( + protocol=self.services_protocol, + server_hostname=hostname, + server_port=port, + ) diff --git a/irctest/controllers/charybdis.py b/irctest/controllers/charybdis.py index e489fd6..2729441 100644 --- a/irctest/controllers/charybdis.py +++ b/irctest/controllers/charybdis.py @@ -1,13 +1,6 @@ -import os -import subprocess -from typing import Optional, Set, Type +from typing import Type -from irctest.basecontrollers import ( - BaseServerController, - DirectoryBasedController, - NotImplementedByController, -) -from irctest.irc_utils.junkdrawer import find_hostname_and_port +from .base_hybrid import BaseHybridController TEMPLATE_CONFIG = """ serverinfo {{ @@ -49,7 +42,7 @@ channel {{ connect "services.example.org" {{ host = "localhost"; # Used to validate incoming connection - port = 0; # We don't want the servers to connect to services + port = 0; # We don't need servers to connect to services send_password = "password"; accept_password = "password"; class = "server"; @@ -60,85 +53,15 @@ service {{ }}; """ -TEMPLATE_SSL_CONFIG = """ - ssl_private_key = "{key_path}"; - ssl_cert = "{pem_path}"; - ssl_dh_params = "{dh_path}"; -""" - -class CharybdisController(BaseServerController, DirectoryBasedController): +class CharybdisController(BaseHybridController): software_name = "Charybdis" binary_name = "charybdis" + services_protocol = "charybdis" + supported_sasl_mechanisms = {"PLAIN"} - supports_sts = False - extban_mute_char = None - def create_config(self) -> None: - super().create_config() - with self.open_file("server.conf"): - pass - - def run( - self, - hostname: str, - port: int, - *, - password: Optional[str], - ssl: bool, - run_services: bool, - valid_metadata_keys: Optional[Set[str]] = None, - invalid_metadata_keys: Optional[Set[str]] = None, - ) -> None: - if valid_metadata_keys or invalid_metadata_keys: - raise NotImplementedByController( - "Defining valid and invalid METADATA keys." - ) - assert self.proc is None - self.port = port - self.hostname = hostname - self.create_config() - (services_hostname, services_port) = find_hostname_and_port() - password_field = 'password = "{}";'.format(password) if password else "" - if ssl: - self.gen_ssl() - ssl_config = TEMPLATE_SSL_CONFIG.format( - key_path=self.key_path, pem_path=self.pem_path, dh_path=self.dh_path - ) - else: - ssl_config = "" - with self.open_file("server.conf") as fd: - fd.write( - TEMPLATE_CONFIG.format( - hostname=hostname, - port=port, - services_hostname=services_hostname, - services_port=services_port, - password_field=password_field, - ssl_config=ssl_config, - ) - ) - assert self.directory - self.proc = subprocess.Popen( - [ - self.binary_name, - "-foreground", - "-configfile", - os.path.join(self.directory, "server.conf"), - "-pidfile", - os.path.join(self.directory, "server.pid"), - ], - # stderr=subprocess.DEVNULL, - ) - - if run_services: - self.wait_for_port() - self.services_controller = self.services_controller_class( - self.test_config, self - ) - self.services_controller.run( - protocol="charybdis", server_hostname=hostname, server_port=port - ) + template_config = TEMPLATE_CONFIG def get_irctest_controller_class() -> Type[CharybdisController]: diff --git a/irctest/controllers/hybrid.py b/irctest/controllers/hybrid.py index 99b3963..bfc9dcc 100644 --- a/irctest/controllers/hybrid.py +++ b/irctest/controllers/hybrid.py @@ -1,30 +1,54 @@ -import os -import subprocess -from typing import Optional, Set, Type +from typing import Set, Type -from irctest.basecontrollers import ( - BaseServerController, - DirectoryBasedController, - NotImplementedByController, -) +from .base_hybrid import BaseHybridController TEMPLATE_CONFIG = """ serverinfo {{ name = "My.Little.Server"; sid = "42X"; description = "test server"; + + # Hybrid defaults to 9 + max_nick_length = 20; {ssl_config} }}; + +general {{ + throttle_count = 100; # We need to connect lots of clients quickly + sasl_service = "SaslServ"; + + # Allow PART/QUIT reasons quickly + anti_spam_exit_message_time = 0; + + # Allow all commands quickly + pace_wait_simple = 0; + pace_wait = 0; +}}; + listen {{ + defer_accept = yes; + host = "{hostname}"; port = {port}; }}; -general {{ - disable_auth = yes; - anti_nick_flood = no; - max_nick_changes = 256; - throttle_count = 512; + +class {{ + name = "server"; + ping_time = 5 minutes; + connectfreq = 5 minutes; }}; +connect {{ + name = "services.example.org"; + host = "localhost"; # Used to validate incoming connection + port = 0; # We don't need servers to connect to services + send_password = "password"; + accept_password = "password"; + class = "server"; +}}; +service {{ + name = "services.example.org"; +}}; + auth {{ user = "*"; flags = exceed_limit; @@ -32,71 +56,15 @@ auth {{ }}; """ -TEMPLATE_SSL_CONFIG = """ - rsa_private_key_file = "{key_path}"; - ssl_certificate_file = "{pem_path}"; - ssl_dh_param_file = "{dh_path}"; -""" - -class HybridController(BaseServerController, DirectoryBasedController): +class HybridController(BaseHybridController): software_name = "Hybrid" - supports_sts = False + binary_name = "ircd" + services_protocol = "hybrid" + supported_sasl_mechanisms: Set[str] = set() - def create_config(self) -> None: - super().create_config() - with self.open_file("server.conf"): - pass - - def run( - self, - hostname: str, - port: int, - *, - password: Optional[str], - ssl: bool, - run_services: bool, - valid_metadata_keys: Optional[Set[str]] = None, - invalid_metadata_keys: Optional[Set[str]] = None, - ) -> None: - if valid_metadata_keys or invalid_metadata_keys: - raise NotImplementedByController( - "Defining valid and invalid METADATA keys." - ) - assert self.proc is None - self.create_config() - self.port = port - password_field = 'password = "{}";'.format(password) if password else "" - if ssl: - self.gen_ssl() - ssl_config = TEMPLATE_SSL_CONFIG.format( - key_path=self.key_path, pem_path=self.pem_path, dh_path=self.dh_path - ) - else: - ssl_config = "" - with self.open_file("server.conf") as fd: - fd.write( - TEMPLATE_CONFIG.format( - hostname=hostname, - port=port, - password_field=password_field, - ssl_config=ssl_config, - ) - ) - assert self.directory - self.proc = subprocess.Popen( - [ - "ircd", - "-foreground", - "-configfile", - os.path.join(self.directory, "server.conf"), - "-pidfile", - os.path.join(self.directory, "server.pid"), - ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) + template_config = TEMPLATE_CONFIG def get_irctest_controller_class() -> Type[HybridController]: diff --git a/irctest/server_tests/test_bot_mode.py b/irctest/server_tests/test_bot_mode.py index 258f4e9..0879391 100644 --- a/irctest/server_tests/test_bot_mode.py +++ b/irctest/server_tests/test_bot_mode.py @@ -71,7 +71,9 @@ class BotModeTestCase(cases.BaseServerTestCase): def testBotPrivateMessage(self): self._initBot() - self.connectClient("usernick", "user", capabilities=["message-tags"]) + self.connectClient( + "usernick", "user", capabilities=["message-tags"], skip_if_cap_nak=True + ) self.sendLine("bot", "PRIVMSG usernick :beep boop") self.getMessages("bot") # Synchronizes @@ -86,7 +88,9 @@ class BotModeTestCase(cases.BaseServerTestCase): def testBotChannelMessage(self): self._initBot() - self.connectClient("usernick", "user", capabilities=["message-tags"]) + self.connectClient( + "usernick", "user", capabilities=["message-tags"], skip_if_cap_nak=True + ) self.sendLine("bot", "JOIN #chan") self.sendLine("user", "JOIN #chan") @@ -106,7 +110,9 @@ class BotModeTestCase(cases.BaseServerTestCase): def testBotWhox(self): self._initBot() - self.connectClient("usernick", "user", capabilities=["message-tags"]) + self.connectClient( + "usernick", "user", capabilities=["message-tags"], skip_if_cap_nak=True + ) self.sendLine("bot", "JOIN #chan") self.sendLine("user", "JOIN #chan") diff --git a/irctest/server_tests/test_channel_operations.py b/irctest/server_tests/test_channel_operations.py index c33a077..578d0b0 100644 --- a/irctest/server_tests/test_channel_operations.py +++ b/irctest/server_tests/test_channel_operations.py @@ -248,6 +248,10 @@ class JoinTestCase(cases.BaseServerTestCase): self.getMessages(1) self.getMessages(2) + # Despite `anti_spam_exit_message_time = 0`, hybrid does not immediately + # allow custom PART reasons. + time.sleep(1) + self.sendLine(1, "PART #chan :bye everyone") # both the PART'ing client and the other channel member should receive # a PART line: @@ -276,6 +280,10 @@ class JoinTestCase(cases.BaseServerTestCase): self.getMessages(1) self.getMessages(2) + # Despite `anti_spam_exit_message_time = 0`, hybrid does not immediately + # allow custom PART reasons. + time.sleep(1) + self.sendLine(1, "PART #chan :bye everyone") # both the PART'ing client and the other channel member should receive # a PART line: @@ -816,6 +824,11 @@ class ChannelQuitTestCase(cases.BaseServerTestCase): self.getMessages(2) self.getMessages(1) + + # Despite `anti_spam_exit_message_time = 0`, hybrid does not immediately + # allow custom PART reasons. + time.sleep(1) + self.sendLine(2, "QUIT :qux out") self.getMessages(2) m = self.getMessage(1) diff --git a/irctest/server_tests/test_sasl.py b/irctest/server_tests/test_sasl.py index 9dee3e2..11a10a7 100644 --- a/irctest/server_tests/test_sasl.py +++ b/irctest/server_tests/test_sasl.py @@ -1,6 +1,6 @@ import base64 -from irctest import cases +from irctest import cases, runner from irctest.patma import ANYSTR @@ -145,6 +145,9 @@ class SaslTestCase(cases.BaseServerTestCase, cases.OptionalityHelper): """“If authentication fails, a 904 or 905 numeric will be sent” -- """ + if not self.controller.supported_sasl_mechanisms: + raise runner.CapabilityNotSupported("sasl") + self.controller.registerUser(self, "jilles", "sesame") self.addClient() self.sendLine(1, "CAP LS 302") diff --git a/workflows.yml b/workflows.yml index 332e3d0..9a51d7d 100644 --- a/workflows.yml +++ b/workflows.yml @@ -5,7 +5,7 @@ software: ############################# - # Charybdis family: + # Hybrid family: charybdis: name: Charybdis repository: charybdis-ircd/charybdis @@ -23,6 +23,22 @@ software: make -j 4 make install + hybrid: + name: Hybrid + repository: ircd-hybrid/ircd-hybrid + refs: + stable: "8.2.38" + release: null + devel: "8.2.x" + devel_release: null + path: ircd-hybrid + separate_build_job: true + build_script: | + cd $GITHUB_WORKSPACE/ircd-hybrid/ + ./configure --prefix=$HOME/.local/ + make -j 4 + make install + solanum: name: Solanum repository: solanum-ircd/solanum @@ -143,6 +159,9 @@ tests: charybdis: software: [charybdis] + hybrid: + software: [hybrid] + solanum: software: [solanum]