Add support for Dlk-Services (#176)

.github/workflows/test-devel.yml

@@ -400,6 +400,7 @@ jobs:
     - test-unrealircd-5
     - test-unrealircd-anope
     - test-unrealircd-atheme
+    - test-unrealircd-dlk
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -1127,6 +1128,52 @@ jobs:
       with:
         name: pytest-results_unrealircd-atheme_devel
         path: pytest.xml
+  test-unrealircd-dlk:
+    needs:
+    - build-unrealircd
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - name: Download build artefacts
+      uses: actions/download-artifact@v2
+      with:
+        name: installed-unrealircd
+        path: '~'
+    - name: Unpack artefacts
+      run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
+    - name: Checkout Dlk
+      uses: actions/checkout@v2
+      with:
+        path: Dlk-Services
+        ref: main
+        repository: DalekIRC/Dalek-Services
+    - name: Build Dlk
+      run: |
+        pip install pifpaf
+        wget -q https://raw.githubusercontent.com/wp-cli/builds/gh-pages/phar/wp-cli.phar
+        wget -q https://wordpress.org/latest.zip -O wordpress-latest.zip
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
+    - name: Install irctest dependencies
+      run: |-
+        python -m pip install --upgrade pip
+        pip install pytest pytest-xdist -r requirements.txt
+    - name: Test with pytest
+      run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/unrealircd/sbin:~/.local/unrealircd/bin:$PATH
+        IRCTEST_DLK_PATH="${{ github.workspace }}/Dlk-Services" IRCTEST_WP_CLI_PATH="${{
+        github.workspace }}/wp-cli.phar" IRCTEST_WP_ZIP_PATH="${{ github.workspace
+        }}/wordpress-latest.zip" make unrealircd-dlk
+      timeout-minutes: 30
+    - if: always()
+      name: Publish results
+      uses: actions/upload-artifact@v2
+      with:
+        name: pytest-results_unrealircd-dlk_devel
+        path: pytest.xml
 name: irctest with devel versions
 'on':
   schedule:

.github/workflows/test-stable.yml

@@ -443,6 +443,7 @@ jobs:
     - test-unrealircd-5
     - test-unrealircd-anope
     - test-unrealircd-atheme
+    - test-unrealircd-dlk
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -1285,6 +1286,52 @@ jobs:
       with:
         name: pytest-results_unrealircd-atheme_stable
         path: pytest.xml
+  test-unrealircd-dlk:
+    needs:
+    - build-unrealircd
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - name: Download build artefacts
+      uses: actions/download-artifact@v2
+      with:
+        name: installed-unrealircd
+        path: '~'
+    - name: Unpack artefacts
+      run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
+    - name: Checkout Dlk
+      uses: actions/checkout@v2
+      with:
+        path: Dlk-Services
+        ref: effd18652fc1c847d1959089d9cca9ff9837a8c0
+        repository: DalekIRC/Dalek-Services
+    - name: Build Dlk
+      run: |
+        pip install pifpaf
+        wget -q https://raw.githubusercontent.com/wp-cli/builds/gh-pages/phar/wp-cli.phar
+        wget -q https://wordpress.org/latest.zip -O wordpress-latest.zip
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
+    - name: Install irctest dependencies
+      run: |-
+        python -m pip install --upgrade pip
+        pip install pytest pytest-xdist -r requirements.txt
+    - name: Test with pytest
+      run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/unrealircd/sbin:~/.local/unrealircd/bin:$PATH
+        IRCTEST_DLK_PATH="${{ github.workspace }}/Dlk-Services" IRCTEST_WP_CLI_PATH="${{
+        github.workspace }}/wp-cli.phar" IRCTEST_WP_ZIP_PATH="${{ github.workspace
+        }}/wordpress-latest.zip" make unrealircd-dlk
+      timeout-minutes: 30
+    - if: always()
+      name: Publish results
+      uses: actions/upload-artifact@v2
+      with:
+        name: pytest-results_unrealircd-dlk_stable
+        path: pytest.xml
 name: irctest with stable versions
 'on':
   pull_request: null

Makefile

@@ -274,3 +274,10 @@ unrealircd-anope:
 		--services-controller=irctest.controllers.anope_services \
 		-m 'services' \
 		-k '$(UNREALIRCD_SELECTORS)'
+
+unrealircd-dlk:
+	pifpaf run mysql -- $(PYTEST) $(PYTEST_ARGS) \
+		--controller=irctest.controllers.unrealircd \
+		--services-controller=irctest.controllers.dlk_services \
+		-m 'services' \
+		-k '$(UNREALIRCD_SELECTORS)'

irctest/basecontrollers.py

@@ -301,10 +301,11 @@ class BaseServicesController(_BaseController):
                 c.sendLine("PONG :" + msg.params[0])
         c.getMessages()
 
-        timeout = time.time() + 5
+        timeout = time.time() + 3
         while True:
-            c.sendLine(f"PRIVMSG {self.server_controller.nickserv} :HELP")
+            c.sendLine(f"PRIVMSG {self.server_controller.nickserv} :help")
-            msgs = self.getNickServResponse(c)
+
+            msgs = self.getNickServResponse(c, timeout=1)
             for msg in msgs:
                 if msg.command == "401":
                     # NickServ not available yet
@@ -330,11 +331,12 @@ class BaseServicesController(_BaseController):
         c.disconnect()
         self.services_up = True
 
-    def getNickServResponse(self, client: Any) -> List[Message]:
+    def getNickServResponse(self, client: Any, timeout: int = 0) -> List[Message]:
         """Wrapper aroung getMessages() that waits longer, because NickServ
         is queried asynchronously."""
         msgs: List[Message] = []
-        while not msgs:
+        start_time = time.time()
+        while not msgs and (not timeout or start_time + timeout > time.time()):
             time.sleep(0.05)
             msgs = client.getMessages()
         return msgs

irctest/controllers/dlk_services.py

@@ -0,0 +1,245 @@
+import os
+from pathlib import Path
+import secrets
+import subprocess
+from typing import Optional, Type
+
+import irctest
+from irctest.basecontrollers import BaseServicesController, DirectoryBasedController
+import irctest.cases
+import irctest.runner
+
+TEMPLATE_DLK_CONFIG = """\
+info {{
+    SID "00A";
+    network-name "testnetwork";
+    services-name "services.example.org";
+    admin-email "admin@example.org";
+}}
+
+link {{
+    hostname "{server_hostname}";
+    port "{server_port}";
+    password "password";
+}}
+
+log {{
+    debug "yes";
+}}
+
+sql {{
+    port "3306";
+    username "pifpaf";
+    password "pifpaf";
+    database "pifpaf";
+    sockfile "{mysql_socket}";
+    prefix "{dlk_prefix}";
+}}
+
+wordpress {{
+    prefix "{wp_prefix}";
+}}
+
+"""
+
+TEMPLATE_DLK_WP_CONFIG = """
+<?php
+
+global $wpconfig;
+$wpconfig = [
+
+    "dbprefix" => "{wp_prefix}",
+
+
+    "default_avatar" => "https://valware.uk/wp-content/plugins/ultimate-member/assets/img/default_avatar.jpg",
+    "forumschan" => "#DLK-Support",
+
+];
+"""
+
+TEMPLATE_WP_CONFIG = """
+define( 'DB_NAME', 'pifpaf' );
+define( 'DB_USER', 'pifpaf' );
+define( 'DB_PASSWORD', 'pifpaf' );
+define( 'DB_HOST', 'localhost:{mysql_socket}' );
+define( 'DB_CHARSET', 'utf8' );
+define( 'DB_COLLATE', '' );
+
+define( 'AUTH_KEY',         'put your unique phrase here' );
+define( 'SECURE_AUTH_KEY',  'put your unique phrase here' );
+define( 'LOGGED_IN_KEY',    'put your unique phrase here' );
+define( 'NONCE_KEY',        'put your unique phrase here' );
+define( 'AUTH_SALT',        'put your unique phrase here' );
+define( 'SECURE_AUTH_SALT', 'put your unique phrase here' );
+define( 'LOGGED_IN_SALT',   'put your unique phrase here' );
+define( 'NONCE_SALT',       'put your unique phrase here' );
+
+$table_prefix = '{wp_prefix}';
+
+define( 'WP_DEBUG', false );
+
+if (!defined('ABSPATH')) {{
+    define( 'ABSPATH', '{wp_path}' );
+}}
+
+/* That's all, stop editing! Happy publishing. */
+
+/** Absolute path to the WordPress directory. */
+
+
+/** Sets up WordPress vars and included files. */
+require_once ABSPATH . 'wp-settings.php';
+"""
+
+
+class DlkController(BaseServicesController, DirectoryBasedController):
+    """Mixin for server controllers that rely on DLK"""
+
+    software_name = "Dlk-Services"
+
+    def run_sql(self, sql: str) -> None:
+        mysql_socket = os.environ["PIFPAF_MYSQL_SOCKET"]
+        subprocess.run(
+            ["mysql", "-S", mysql_socket, "pifpaf"],
+            input=sql.encode(),
+            check=True,
+        )
+
+    def run(self, protocol: str, server_hostname: str, server_port: int) -> None:
+        self.create_config()
+
+        if protocol == "unreal4":
+            protocol = "unreal5"
+        assert protocol in ("unreal5",), protocol
+
+        mysql_socket = os.environ["PIFPAF_MYSQL_SOCKET"]
+
+        assert self.directory
+
+        try:
+            self.wp_cli_path = Path(os.environ["IRCTEST_WP_CLI_PATH"])
+            if not self.wp_cli_path.is_file():
+                raise KeyError()
+        except KeyError:
+            raise RuntimeError(
+                "$IRCTEST_WP_CLI_PATH must be set to a WP-CLI executable (eg. "
+                "downloaded from <https://raw.githubusercontent.com/wp-cli/builds/"
+                "gh-pages/phar/wp-cli.phar>)"
+            ) from None
+
+        try:
+            self.dlk_path = Path(os.environ["IRCTEST_DLK_PATH"])
+            if not self.dlk_path.is_dir():
+                raise KeyError()
+        except KeyError:
+            raise RuntimeError("$IRCTEST_DLK_PATH is not set") from None
+        self.dlk_path = self.dlk_path.resolve()
+
+        # Unpack a fresh Wordpress install in the temporary directory.
+        # In theory we could have a common Wordpress install and only wp-config.php
+        # in the temporary directory; but wp-cli assumes wp-config.php must be
+        # in a Wordpress directory, and fails in various places if it isn't.
+        # Rather than symlinking everything to make it work, let's just copy
+        # the whole code, it's not that big.
+        try:
+            wp_zip_path = Path(os.environ["IRCTEST_WP_ZIP_PATH"])
+            if not wp_zip_path.is_file():
+                raise KeyError()
+        except KeyError:
+            raise RuntimeError(
+                "$IRCTEST_WP_ZIP_PATH must be set to a Wordpress source zipball "
+                "(eg. downloaded from <https://wordpress.org/latest.zip>)"
+            ) from None
+        subprocess.run(
+            ["unzip", wp_zip_path, "-d", self.directory], stdout=subprocess.DEVNULL
+        )
+        self.wp_path = self.directory / "wordpress"
+
+        rand_hex = secrets.token_hex(6)
+        self.wp_prefix = f"wp{rand_hex}_"
+        self.dlk_prefix = f"dlk{rand_hex}_"
+        template_vars = dict(
+            protocol=protocol,
+            server_hostname=server_hostname,
+            server_port=server_port,
+            mysql_socket=mysql_socket,
+            wp_path=self.wp_path,
+            wp_prefix=self.wp_prefix,
+            dlk_prefix=self.dlk_prefix,
+        )
+
+        # Configure Wordpress
+        wp_config_path = self.directory / "wp-config.php"
+        with open(wp_config_path, "w") as fd:
+            fd.write(TEMPLATE_WP_CONFIG.format(**template_vars))
+
+        subprocess.run(
+            [
+                "php",
+                self.wp_cli_path,
+                "core",
+                "install",
+                "--url=http://localhost/",
+                "--title=irctest site",
+                "--admin_user=adminuser",
+                "--admin_email=adminuser@example.org",
+                f"--path={self.wp_path}",
+            ],
+            check=True,
+        )
+
+        # Configure Dlk
+        dlk_log_dir = self.directory / "logs"
+        dlk_conf_dir = self.directory / "conf"
+        dlk_conf_path = dlk_conf_dir / "dalek.conf"
+        os.mkdir(dlk_conf_dir)
+        with open(dlk_conf_path, "w") as fd:
+            fd.write(TEMPLATE_DLK_CONFIG.format(**template_vars))
+        dlk_wp_config_path = dlk_conf_dir / "wordpress.conf"
+        with open(dlk_wp_config_path, "w") as fd:
+            fd.write(TEMPLATE_DLK_WP_CONFIG.format(**template_vars))
+        (dlk_conf_dir / "modules.conf").symlink_to(self.dlk_path / "conf/modules.conf")
+
+        self.proc = subprocess.Popen(
+            [
+                "php",
+                "src/dalek",
+            ],
+            cwd=self.dlk_path,
+            env={
+                **os.environ,
+                "DALEK_CONF_DIR": str(dlk_conf_dir),
+                "DALEK_LOG_DIR": str(dlk_log_dir),
+            },
+        )
+
+    def terminate(self) -> None:
+        super().terminate()
+
+    def kill(self) -> None:
+        super().kill()
+
+    def registerUser(
+        self,
+        case: irctest.cases.BaseServerTestCase,
+        username: str,
+        password: Optional[str] = None,
+    ) -> None:
+        assert password
+        subprocess.run(
+            [
+                "php",
+                self.wp_cli_path,
+                "user",
+                "create",
+                username,
+                f"{username}@example.org",
+                f"--user_pass={password}",
+                f"--path={self.wp_path}",
+            ],
+            check=True,
+        )
+
+
+def get_irctest_controller_class() -> Type[DlkController]:
+    return DlkController

irctest/dashboard/format.py

@@ -303,7 +303,7 @@ def write_html_pages(
             for result in results
         )
         assert is_client != is_server, (job, is_client, is_server)
-        if job.endswith(("-atheme", "-anope")):
+        if job.endswith(("-atheme", "-anope", "-dlk")):
             assert is_server
             job_categories[job] = "server-with-services"
         elif is_server:

irctest/server_tests/sasl.py

@@ -178,6 +178,14 @@ class SaslTestCase(cases.BaseServerTestCase):
         ),
         "Anope does not handle split AUTHENTICATE (reported on IRC)",
     )
+    @cases.xfailIf(
+        lambda self: (
+            self.controller.services_controller is not None
+            and self.controller.services_controller.software_name == "Dlk-Services"
+        ),
+        "Dlk does not handle split AUTHENTICATE "
+        "https://github.com/DalekIRC/Dalek-Services/issues/28",
+    )
     def testPlainLarge(self):
         """Test the client splits large AUTHENTICATE messages whose payload
         is not a multiple of 400.

irctest/server_tests/who.py

@@ -37,8 +37,8 @@ class BaseWhoTestCase:
         self.sendLine(1, f"USER {self.username} 0 * :{self.realname}")
         if auth:
             self.sendLine(1, "CAP END")
-            self.getRegistrationMessage(1)
         self.skipToWelcome(1)
+        self.getMessages(1)
         self.sendLine(1, "JOIN #chan")
 
         self.getMessages(1)

irctest/server_tests/whois.py

@@ -71,7 +71,10 @@ class _WhoisTestMixin(cases.BaseServerTestCase):
             last_message,
             command=RPL_ENDOFWHOIS,
             params=["nick1", "nick2", ANYSTR],
-            fail_msg=f"Last message was not RPL_ENDOFWHOIS ({RPL_ENDOFWHOIS})",
+            fail_msg=(
+                f"Expected RPL_ENDOFWHOIS ({RPL_ENDOFWHOIS}) as last message, "
+                f"got {{msg}}"
+            ),
         )
 
         unexpected_messages = []

make_workflows.py

@@ -146,7 +146,7 @@ def get_test_job(*, config, test_config, test_id, version_flavor, jobs):
     for software_id in test_config.get("software", []):
         software_config = config["software"][software_id]
 
-        env += test_config.get("env", {}).get(version_flavor.value, "") + " "
+        env += software_config.get("env", "") + " "
         if "prefix" in software_config:
             env += (
                 f"PATH={software_config['prefix']}/sbin"

workflows.yml

@@ -304,6 +304,7 @@ software:
 
     #############################
     # Services:
+
     anope:
         name: Anope
         repository: anope/anope
@@ -321,6 +322,24 @@ software:
             make -C build -j 4
             make -C build install
 
+    dlk:
+        name: Dlk
+        repository: DalekIRC/Dalek-Services
+        separate_build_job: false
+        path: Dlk-Services
+        refs:
+            stable: &dlk_stable "effd18652fc1c847d1959089d9cca9ff9837a8c0"
+            release: *dlk_stable
+            devel: "main"
+            devel_release: *dlk_stable
+        build_script: |
+            pip install pifpaf
+            wget -q https://raw.githubusercontent.com/wp-cli/builds/gh-pages/phar/wp-cli.phar
+            wget -q https://wordpress.org/latest.zip -O wordpress-latest.zip
+        env: >-
+            IRCTEST_DLK_PATH="${{ github.workspace }}/Dlk-Services"
+            IRCTEST_WP_CLI_PATH="${{ github.workspace }}/wp-cli.phar"
+            IRCTEST_WP_ZIP_PATH="${{ github.workspace }}/wordpress-latest.zip"
 
 
     #############################
@@ -425,6 +444,9 @@ tests:
     unrealircd-anope:
         software: [unrealircd, anope]
 
+    unrealircd-dlk:
+        software: [unrealircd, dlk]
+
 
     limnoria:
         software: [limnoria]