Fix Hybrid support + enable it on CI (#82)

* Fix Hybrid support + enable it on CI

* Can't make Hybrid linking work on Github CI

because the reverse DNS is 'cpu-pool.com' for some reason, and I don't
want to hardcode it, so I give up.

.github/workflows/test-devel.yml

@@ -29,6 +29,43 @@ jobs:
         name: installed-anope
         path: ~/artefacts-*.tar.gz
         retention-days: 1
+  build-hybrid:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Create directories
+      run: cd ~/; mkdir -p .local/ go/
+    - name: Cache dependencies
+      uses: actions/cache@v2
+      with:
+        key: ${{ runner.os }}-hybrid-devel
+        path: |-
+          ~/.cache
+          $GITHUB_WORKSPACE/ircd-hybrid
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - name: Checkout Hybrid
+      uses: actions/checkout@v2
+      with:
+        path: ircd-hybrid
+        ref: 8.2.x
+        repository: ircd-hybrid/ircd-hybrid
+    - name: Build Hybrid
+      run: |
+        cd $GITHUB_WORKSPACE/ircd-hybrid/
+        ./configure --prefix=$HOME/.local/
+        make -j 4
+        make install
+    - name: Make artefact tarball
+      run: cd ~; tar -czf artefacts-hybrid.tar.gz .local/ go/
+    - name: Upload build artefacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: installed-hybrid
+        path: ~/artefacts-*.tar.gz
+        retention-days: 1
   build-inspircd:
     runs-on: ubuntu-latest
     steps:
@@ -145,6 +182,7 @@ jobs:
     name: Publish Unit Tests Results
     needs:
     - test-ergo
+    - test-hybrid
     - test-inspircd
     - test-inspircd-anope
     - test-limnoria
@@ -202,6 +240,38 @@ jobs:
       with:
         name: pytest results ergo (devel)
         path: pytest.xml
+  test-hybrid:
+    needs:
+    - build-hybrid
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - name: Download build artefacts
+      uses: actions/download-artifact@v2
+      with:
+        name: installed-hybrid
+        path: '~'
+    - name: Unpack artefacts
+      run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
+    - name: Install Atheme
+      run: sudo apt-get install atheme-services
+    - name: Install irctest dependencies
+      run: |-
+        python -m pip install --upgrade pip
+        pip install pytest -r requirements.txt
+    - name: Test with pytest
+      run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
+        hybrid
+    - if: always()
+      name: Publish results
+      uses: actions/upload-artifact@v2
+      with:
+        name: pytest results hybrid (devel)
+        path: pytest.xml
   test-inspircd:
     needs:
     - build-inspircd

.github/workflows/test-stable.yml

@@ -67,6 +67,43 @@ jobs:
         name: installed-charybdis
         path: ~/artefacts-*.tar.gz
         retention-days: 1
+  build-hybrid:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Create directories
+      run: cd ~/; mkdir -p .local/ go/
+    - name: Cache dependencies
+      uses: actions/cache@v2
+      with:
+        key: ${{ runner.os }}-hybrid-stable
+        path: |-
+          ~/.cache
+          $GITHUB_WORKSPACE/ircd-hybrid
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - name: Checkout Hybrid
+      uses: actions/checkout@v2
+      with:
+        path: ircd-hybrid
+        ref: 8.2.38
+        repository: ircd-hybrid/ircd-hybrid
+    - name: Build Hybrid
+      run: |
+        cd $GITHUB_WORKSPACE/ircd-hybrid/
+        ./configure --prefix=$HOME/.local/
+        make -j 4
+        make install
+    - name: Make artefact tarball
+      run: cd ~; tar -czf artefacts-hybrid.tar.gz .local/ go/
+    - name: Upload build artefacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: installed-hybrid
+        path: ~/artefacts-*.tar.gz
+        retention-days: 1
   build-inspircd:
     runs-on: ubuntu-latest
     steps:
@@ -184,6 +221,7 @@ jobs:
     needs:
     - test-charybdis
     - test-ergo
+    - test-hybrid
     - test-inspircd
     - test-inspircd-anope
     - test-inspircd-atheme
@@ -274,6 +312,38 @@ jobs:
       with:
         name: pytest results ergo (stable)
         path: pytest.xml
+  test-hybrid:
+    needs:
+    - build-hybrid
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - name: Download build artefacts
+      uses: actions/download-artifact@v2
+      with:
+        name: installed-hybrid
+        path: '~'
+    - name: Unpack artefacts
+      run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
+    - name: Install Atheme
+      run: sudo apt-get install atheme-services
+    - name: Install irctest dependencies
+      run: |-
+        python -m pip install --upgrade pip
+        pip install pytest -r requirements.txt
+    - name: Test with pytest
+      run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
+        hybrid
+    - if: always()
+      name: Publish results
+      uses: actions/upload-artifact@v2
+      with:
+        name: pytest results hybrid (stable)
+        path: pytest.xml
   test-inspircd:
     needs:
     - build-inspircd

Makefile

@@ -26,6 +26,11 @@ ERGO_SELECTORS := \
 	not deprecated \
 	$(EXTRA_SELECTORS)
 
+HYBRID_SELECTORS := \
+	not Ergo \
+	and not deprecated \
+	$(EXTRA_SELECTORS)
+
 # testNoticeNonexistentChannel fails because of https://github.com/inspircd/inspircd/issues/1849
 # testDirectMessageEcho fails because of https://github.com/inspircd/inspircd/issues/1851
 # testKeyValidation fails because of https://github.com/inspircd/inspircd/issues/1850
@@ -108,6 +113,12 @@ ergo:
 		--controller irctest.controllers.ergo \
 		-k "$(ERGO_SELECTORS)"
 
+hybrid:
+	$(PYTEST) $(PYTEST_ARGS) \
+		--controller irctest.controllers.hybrid \
+		-m 'not services' \
+		-k "$(HYBRID_SELECTORS)"
+
 inspircd:
 	$(PYTEST) $(PYTEST_ARGS) \
 		--controller=irctest.controllers.inspircd \

irctest/basecontrollers.py

@@ -224,7 +224,7 @@ class BaseServerController(_BaseController):
                 # test_lusers.py (eg. this happens with Charybdis 3.5.0)
                 c.send(b"QUIT :chkport\r\n")
                 data = b""
-                while b"chkport" not in data:
+                while b"chkport" not in data and b"ERROR" not in data:
                     data += c.recv(1024)
 
                 c.close()

irctest/controllers/anope_services.py

@@ -76,7 +76,7 @@ class AnopeController(BaseServicesController, DirectoryBasedController):
     def run(self, protocol: str, server_hostname: str, server_port: int) -> None:
         self.create_config()
 
-        assert protocol in ("inspircd3", "charybdis", "unreal4")
+        assert protocol in ("inspircd3", "charybdis", "hybrid", "unreal4")
 
         with self.open_file("conf/services.conf") as fd:
             fd.write(

irctest/controllers/base_hybrid.py

@@ -0,0 +1,97 @@
+import os
+import subprocess
+from typing import Optional, Set
+
+from irctest.basecontrollers import (
+    BaseServerController,
+    DirectoryBasedController,
+    NotImplementedByController,
+)
+from irctest.irc_utils.junkdrawer import find_hostname_and_port
+
+TEMPLATE_SSL_CONFIG = """
+    ssl_private_key = "{key_path}";
+    ssl_cert = "{pem_path}";
+    ssl_dh_params = "{dh_path}";
+"""
+
+
+class BaseHybridController(BaseServerController, DirectoryBasedController):
+    """A base class for all controllers derived from ircd-hybrid (Hybrid itself,
+    Charybdis, Solanum, ...)"""
+
+    binary_name: str
+    services_protocol: str
+
+    supports_sts = False
+    extban_mute_char = None
+
+    template_config: str
+
+    def create_config(self) -> None:
+        super().create_config()
+        with self.open_file("server.conf"):
+            pass
+
+    def run(
+        self,
+        hostname: str,
+        port: int,
+        *,
+        password: Optional[str],
+        ssl: bool,
+        run_services: bool,
+        valid_metadata_keys: Optional[Set[str]] = None,
+        invalid_metadata_keys: Optional[Set[str]] = None,
+    ) -> None:
+        if valid_metadata_keys or invalid_metadata_keys:
+            raise NotImplementedByController(
+                "Defining valid and invalid METADATA keys."
+            )
+        assert self.proc is None
+        self.port = port
+        self.hostname = hostname
+        self.create_config()
+        (services_hostname, services_port) = find_hostname_and_port()
+        password_field = 'password = "{}";'.format(password) if password else ""
+        if ssl:
+            self.gen_ssl()
+            ssl_config = TEMPLATE_SSL_CONFIG.format(
+                key_path=self.key_path, pem_path=self.pem_path, dh_path=self.dh_path
+            )
+        else:
+            ssl_config = ""
+        with self.open_file("server.conf") as fd:
+            fd.write(
+                (self.template_config).format(
+                    hostname=hostname,
+                    port=port,
+                    services_hostname=services_hostname,
+                    services_port=services_port,
+                    password_field=password_field,
+                    ssl_config=ssl_config,
+                )
+            )
+        assert self.directory
+        self.proc = subprocess.Popen(
+            [
+                self.binary_name,
+                "-foreground",
+                "-configfile",
+                os.path.join(self.directory, "server.conf"),
+                "-pidfile",
+                os.path.join(self.directory, "server.pid"),
+            ],
+            # stderr=subprocess.DEVNULL,
+        )
+
+        if run_services:
+            self.wait_for_port()
+            self.services_controller = self.services_controller_class(
+                self.test_config, self
+            )
+            self.services_controller.run(
+                protocol=self.services_protocol,
+                server_hostname=hostname,
+                server_port=port,
+            )

irctest/controllers/charybdis.py

@@ -1,13 +1,6 @@
-import os
+from typing import Type
-import subprocess
-from typing import Optional, Set, Type
 
-from irctest.basecontrollers import (
+from .base_hybrid import BaseHybridController
-    BaseServerController,
-    DirectoryBasedController,
-    NotImplementedByController,
-)
-from irctest.irc_utils.junkdrawer import find_hostname_and_port
 
 TEMPLATE_CONFIG = """
 serverinfo {{
@@ -49,7 +42,7 @@ channel {{
 
 connect "services.example.org" {{
     host = "localhost";  # Used to validate incoming connection
-    port = 0;  # We don't want the servers to connect to services
+    port = 0;  # We don't need servers to connect to services
     send_password = "password";
     accept_password = "password";
     class = "server";
@@ -60,85 +53,15 @@ service {{
 }};
 """
 
-TEMPLATE_SSL_CONFIG = """
-    ssl_private_key = "{key_path}";
-    ssl_cert = "{pem_path}";
-    ssl_dh_params = "{dh_path}";
-"""
 
-
+class CharybdisController(BaseHybridController):
-class CharybdisController(BaseServerController, DirectoryBasedController):
     software_name = "Charybdis"
     binary_name = "charybdis"
+    services_protocol = "charybdis"
+
     supported_sasl_mechanisms = {"PLAIN"}
-    supports_sts = False
-    extban_mute_char = None
 
-    def create_config(self) -> None:
+    template_config = TEMPLATE_CONFIG
-        super().create_config()
-        with self.open_file("server.conf"):
-            pass
-
-    def run(
-        self,
-        hostname: str,
-        port: int,
-        *,
-        password: Optional[str],
-        ssl: bool,
-        run_services: bool,
-        valid_metadata_keys: Optional[Set[str]] = None,
-        invalid_metadata_keys: Optional[Set[str]] = None,
-    ) -> None:
-        if valid_metadata_keys or invalid_metadata_keys:
-            raise NotImplementedByController(
-                "Defining valid and invalid METADATA keys."
-            )
-        assert self.proc is None
-        self.port = port
-        self.hostname = hostname
-        self.create_config()
-        (services_hostname, services_port) = find_hostname_and_port()
-        password_field = 'password = "{}";'.format(password) if password else ""
-        if ssl:
-            self.gen_ssl()
-            ssl_config = TEMPLATE_SSL_CONFIG.format(
-                key_path=self.key_path, pem_path=self.pem_path, dh_path=self.dh_path
-            )
-        else:
-            ssl_config = ""
-        with self.open_file("server.conf") as fd:
-            fd.write(
-                TEMPLATE_CONFIG.format(
-                    hostname=hostname,
-                    port=port,
-                    services_hostname=services_hostname,
-                    services_port=services_port,
-                    password_field=password_field,
-                    ssl_config=ssl_config,
-                )
-            )
-        assert self.directory
-        self.proc = subprocess.Popen(
-            [
-                self.binary_name,
-                "-foreground",
-                "-configfile",
-                os.path.join(self.directory, "server.conf"),
-                "-pidfile",
-                os.path.join(self.directory, "server.pid"),
-            ],
-            # stderr=subprocess.DEVNULL,
-        )
-
-        if run_services:
-            self.wait_for_port()
-            self.services_controller = self.services_controller_class(
-                self.test_config, self
-            )
-            self.services_controller.run(
-                protocol="charybdis", server_hostname=hostname, server_port=port
-            )
 
 
 def get_irctest_controller_class() -> Type[CharybdisController]:

irctest/controllers/hybrid.py

@@ -1,30 +1,54 @@
-import os
+from typing import Set, Type
-import subprocess
-from typing import Optional, Set, Type
 
-from irctest.basecontrollers import (
+from .base_hybrid import BaseHybridController
-    BaseServerController,
-    DirectoryBasedController,
-    NotImplementedByController,
-)
 
 TEMPLATE_CONFIG = """
 serverinfo {{
     name = "My.Little.Server";
     sid = "42X";
     description = "test server";
+
+    # Hybrid defaults to 9
+    max_nick_length = 20;
 {ssl_config}
 }};
+
+general {{
+    throttle_count = 100;  # We need to connect lots of clients quickly
+    sasl_service = "SaslServ";
+
+    # Allow PART/QUIT reasons quickly
+    anti_spam_exit_message_time = 0;
+
+    # Allow all commands quickly
+    pace_wait_simple = 0;
+    pace_wait = 0;
+}};
+
 listen {{
+    defer_accept = yes;
+
     host = "{hostname}";
     port = {port};
 }};
-general {{
+
-    disable_auth = yes;
+class {{
-    anti_nick_flood = no;
+    name = "server";
-    max_nick_changes = 256;
+    ping_time = 5 minutes;
-    throttle_count = 512;
+    connectfreq = 5 minutes;
 }};
+connect {{
+    name = "services.example.org";
+    host = "localhost";  # Used to validate incoming connection
+    port = 0;  # We don't need servers to connect to services
+    send_password = "password";
+    accept_password = "password";
+    class = "server";
+}};
+service {{
+    name = "services.example.org";
+}};
+
 auth {{
     user = "*";
     flags = exceed_limit;
@@ -32,71 +56,15 @@ auth {{
 }};
 """
 
-TEMPLATE_SSL_CONFIG = """
-    rsa_private_key_file = "{key_path}";
-    ssl_certificate_file = "{pem_path}";
-    ssl_dh_param_file = "{dh_path}";
-"""
 
-
+class HybridController(BaseHybridController):
-class HybridController(BaseServerController, DirectoryBasedController):
     software_name = "Hybrid"
-    supports_sts = False
+    binary_name = "ircd"
+    services_protocol = "hybrid"
+
     supported_sasl_mechanisms: Set[str] = set()
 
-    def create_config(self) -> None:
+    template_config = TEMPLATE_CONFIG
-        super().create_config()
-        with self.open_file("server.conf"):
-            pass
-
-    def run(
-        self,
-        hostname: str,
-        port: int,
-        *,
-        password: Optional[str],
-        ssl: bool,
-        run_services: bool,
-        valid_metadata_keys: Optional[Set[str]] = None,
-        invalid_metadata_keys: Optional[Set[str]] = None,
-    ) -> None:
-        if valid_metadata_keys or invalid_metadata_keys:
-            raise NotImplementedByController(
-                "Defining valid and invalid METADATA keys."
-            )
-        assert self.proc is None
-        self.create_config()
-        self.port = port
-        password_field = 'password = "{}";'.format(password) if password else ""
-        if ssl:
-            self.gen_ssl()
-            ssl_config = TEMPLATE_SSL_CONFIG.format(
-                key_path=self.key_path, pem_path=self.pem_path, dh_path=self.dh_path
-            )
-        else:
-            ssl_config = ""
-        with self.open_file("server.conf") as fd:
-            fd.write(
-                TEMPLATE_CONFIG.format(
-                    hostname=hostname,
-                    port=port,
-                    password_field=password_field,
-                    ssl_config=ssl_config,
-                )
-            )
-        assert self.directory
-        self.proc = subprocess.Popen(
-            [
-                "ircd",
-                "-foreground",
-                "-configfile",
-                os.path.join(self.directory, "server.conf"),
-                "-pidfile",
-                os.path.join(self.directory, "server.pid"),
-            ],
-            stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL,
-        )
 
 
 def get_irctest_controller_class() -> Type[HybridController]:

irctest/server_tests/test_bot_mode.py

@@ -71,7 +71,9 @@ class BotModeTestCase(cases.BaseServerTestCase):
     def testBotPrivateMessage(self):
         self._initBot()
 
-        self.connectClient("usernick", "user", capabilities=["message-tags"])
+        self.connectClient(
+            "usernick", "user", capabilities=["message-tags"], skip_if_cap_nak=True
+        )
 
         self.sendLine("bot", "PRIVMSG usernick :beep boop")
         self.getMessages("bot")  # Synchronizes
@@ -86,7 +88,9 @@ class BotModeTestCase(cases.BaseServerTestCase):
     def testBotChannelMessage(self):
         self._initBot()
 
-        self.connectClient("usernick", "user", capabilities=["message-tags"])
+        self.connectClient(
+            "usernick", "user", capabilities=["message-tags"], skip_if_cap_nak=True
+        )
 
         self.sendLine("bot", "JOIN #chan")
         self.sendLine("user", "JOIN #chan")
@@ -106,7 +110,9 @@ class BotModeTestCase(cases.BaseServerTestCase):
     def testBotWhox(self):
         self._initBot()
 
-        self.connectClient("usernick", "user", capabilities=["message-tags"])
+        self.connectClient(
+            "usernick", "user", capabilities=["message-tags"], skip_if_cap_nak=True
+        )
 
         self.sendLine("bot", "JOIN #chan")
         self.sendLine("user", "JOIN #chan")

irctest/server_tests/test_channel_operations.py

@@ -248,6 +248,10 @@ class JoinTestCase(cases.BaseServerTestCase):
         self.getMessages(1)
         self.getMessages(2)
 
+        # Despite `anti_spam_exit_message_time = 0`, hybrid does not immediately
+        # allow custom PART reasons.
+        time.sleep(1)
+
         self.sendLine(1, "PART #chan :bye everyone")
         # both the PART'ing client and the other channel member should receive
         # a PART line:
@@ -276,6 +280,10 @@ class JoinTestCase(cases.BaseServerTestCase):
         self.getMessages(1)
         self.getMessages(2)
 
+        # Despite `anti_spam_exit_message_time = 0`, hybrid does not immediately
+        # allow custom PART reasons.
+        time.sleep(1)
+
         self.sendLine(1, "PART #chan :bye everyone")
         # both the PART'ing client and the other channel member should receive
         # a PART line:
@@ -816,6 +824,11 @@ class ChannelQuitTestCase(cases.BaseServerTestCase):
         self.getMessages(2)
 
         self.getMessages(1)
+
+        # Despite `anti_spam_exit_message_time = 0`, hybrid does not immediately
+        # allow custom PART reasons.
+        time.sleep(1)
+
         self.sendLine(2, "QUIT :qux out")
         self.getMessages(2)
         m = self.getMessage(1)

irctest/server_tests/test_sasl.py

@@ -1,6 +1,6 @@
 import base64
 
-from irctest import cases
+from irctest import cases, runner
 from irctest.patma import ANYSTR
 
 
@@ -145,6 +145,9 @@ class SaslTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
         """“If authentication fails, a 904 or 905 numeric will be sent”
         -- <http://ircv3.net/specs/extensions/sasl-3.1.html#the-authenticate-command>
         """
+        if not self.controller.supported_sasl_mechanisms:
+            raise runner.CapabilityNotSupported("sasl")
+
         self.controller.registerUser(self, "jilles", "sesame")
         self.addClient()
         self.sendLine(1, "CAP LS 302")

workflows.yml

@@ -5,7 +5,7 @@
 
 software:
     #############################
-    # Charybdis family:
+    # Hybrid family:
     charybdis:
         name: Charybdis
         repository: charybdis-ircd/charybdis
@@ -23,6 +23,22 @@ software:
             make -j 4
             make install
 
+    hybrid:
+        name: Hybrid
+        repository: ircd-hybrid/ircd-hybrid
+        refs:
+            stable: "8.2.38"
+            release: null
+            devel: "8.2.x"
+            devel_release: null
+        path: ircd-hybrid
+        separate_build_job: true
+        build_script: |
+            cd $GITHUB_WORKSPACE/ircd-hybrid/
+            ./configure --prefix=$HOME/.local/
+            make -j 4
+            make install
+
     solanum:
         name: Solanum
         repository: solanum-ircd/solanum
@@ -143,6 +159,9 @@ tests:
     charybdis:
         software: [charybdis]
 
+    hybrid:
+        software: [hybrid]
+
     solanum:
         software: [solanum]