Initialize MySQL

This starts each test with a clean database, so we can remove chan/nick
randomization from stateful tests (chathistory and roleplay).

It will also allow testing Ergo with a MySQL backend for the KV store
instead of buntdb.

Additionally, this makes it much easier to run these tests, than having
to manually configure such a database.

.github/deploy_to_netlify.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import pprint
+import re
+import subprocess
+import sys
+import urllib.request
+
+event_name = os.environ["GITHUB_EVENT_NAME"]
+
+is_pull_request = is_push = False
+if event_name.startswith("pull_request"):
+    is_pull_request = True
+elif event_name.startswith("push"):
+    is_push = True
+elif event_name.startswith("schedule"):
+    # Don't publish; scheduled workflows run against the latest commit of every
+    # implementation, so they are likely to have failed tests for the wrong reasons
+    sys.exit(0)
+else:
+    print("Unexpected event name:", event_name)
+
+with open(os.environ["GITHUB_EVENT_PATH"]) as fd:
+    github_event = json.load(fd)
+
+pprint.pprint(github_event)
+
+context_suffix = ""
+
+command = ["netlify", "deploy", "--dir=dashboard/"]
+if is_pull_request:
+    pr_number = github_event["number"]
+    sha = github_event.get("after") or github_event["pull_request"]["head"]["sha"]
+    # Aliases can't exceed 37 chars
+    command.extend(["--alias", f"pr-{pr_number}-{sha[0:10]}"])
+    context_suffix = " (pull_request)"
+elif is_push:
+    ref = github_event["ref"]
+    m = re.match("refs/heads/(.*)", ref)
+    if m:
+        branch = m.group(1)
+        sha = github_event["head_commit"]["id"]
+
+        if branch in ("main", "master"):
+            command.extend(["--prod"])
+        else:
+            command.extend(["--alias", f"br-{branch[0:23]}-{sha[0:10]}"])
+            context_suffix = " (push)"
+    else:
+        # TODO
+        pass
+
+
+print("Running", command)
+proc = subprocess.run(command, capture_output=True)
+
+output = proc.stdout.decode()
+assert proc.returncode == 0, (output, proc.stderr.decode())
+print(output)
+
+m = re.search("https://[^ ]*--[^ ]*netlify.app", output)
+assert m
+netlify_site_url = m.group(0)
+target_url = f"{netlify_site_url}/index.xhtml"
+
+print("Published to", netlify_site_url)
+
+
+def send_status() -> None:
+    statuses_url = github_event["repository"]["statuses_url"].format(sha=sha)
+
+    payload = {
+        "state": "success",
+        "context": f"Dashboard{context_suffix}",
+        "description": "Table of all test results",
+        "target_url": target_url,
+    }
+    request = urllib.request.Request(
+        statuses_url,
+        data=json.dumps(payload).encode(),
+        headers={
+            "Authorization": f'token {os.environ["GITHUB_TOKEN"]}',
+            "Content-Type": "text/json",
+            "Accept": "application/vnd.github+json",
+        },
+    )
+
+    response = urllib.request.urlopen(request)
+
+    assert response.status == 201, response.read()
+
+
+send_status()
+
+
+def send_pr_comment() -> None:
+    comments_url = github_event["pull_request"]["_links"]["comments"]["href"]
+
+    payload = {
+        "body": f"[Test results]({target_url})",
+    }
+    request = urllib.request.Request(
+        comments_url,
+        data=json.dumps(payload).encode(),
+        headers={
+            "Authorization": f'token {os.environ["GITHUB_TOKEN"]}',
+            "Content-Type": "text/json",
+            "Accept": "application/vnd.github+json",
+        },
+    )
+
+    response = urllib.request.urlopen(request)
+
+    assert response.status == 201, response.read()
+
+
+if is_pull_request:
+    send_pr_comment()

.github/workflows/test-devel.yml

@@ -315,6 +315,8 @@ jobs:
         CFLAGS="-O0 -march=x86-64" CXXFLAGS="$CFLAGS" ./Config -quick
         make -j 4
         make install
+        # Prevent download of geoIP database on first startup
+        sed -i 's/loadmodule "geoip_classic";//' ~/.local/unrealircd/conf/modules.default.conf
     - name: Make artefact tarball
       run: cd ~; tar -czf artefacts-unrealircd.tar.gz .local/ go/
     - name: Upload build artefacts
@@ -359,6 +361,8 @@ jobs:
         CFLAGS="-O0 -march=x86-64" CXXFLAGS="$CFLAGS" ./Config -quick
         make -j 4
         make install
+        # Prevent download of geoIP database on first startup
+        sed -i 's/loadmodule "geoip_classic";//' ~/.local/unrealircd/conf/modules.default.conf
     - name: Make artefact tarball
       run: cd ~; tar -czf artefacts-unrealircd-5.tar.gz .local/ go/
     - name: Upload build artefacts
@@ -369,7 +373,7 @@ jobs:
         retention-days: 1
   publish-test-results:
     if: success() || failure()
-    name: Publish Unit Tests Results
+    name: Publish Dashboard
     needs:
     - test-bahamut
     - test-bahamut-anope
@@ -380,6 +384,7 @@ jobs:
     - test-inspircd-anope
     - test-ircu2
     - test-limnoria
+    - test-nefarious
     - test-ngircd
     - test-ngircd-anope
     - test-ngircd-atheme
@@ -397,27 +402,23 @@ jobs:
       uses: actions/download-artifact@v2
       with:
         path: artifacts
-    - if: github.event_name == 'pull_request'
-      name: Publish Unit Test Results
-      uses: actions/github-script@v4
-      with:
-        result-encoding: string
-        script: |
-          let body = '';
-          const options = {};
-          options.listeners = {
-              stdout: (data) => {
-                  body += data.toString();
-              }
-          };
-          await exec.exec('bash', ['-c', 'shopt -s globstar; python3 report.py artifacts/**/*.xml'], options);
-          github.issues.createComment({
-            issue_number: context.issue.number,
-            owner: context.repo.owner,
-            repo: context.repo.repo,
-            body: body,
-          });
-          return body;
+    - name: Install dashboard dependencies
+      run: |-
+        python -m pip install --upgrade pip
+        pip install defusedxml docutils -r requirements.txt
+    - name: Generate dashboard
+      run: |-
+        shopt -s globstar
+        python3 -m irctest.dashboard.format dashboard/ artifacts/**/*.xml
+        echo '/ /index.xhtml' > dashboard/_redirects
+    - name: Install netlify-cli
+      run: npm i -g netlify-cli
+    - env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+        NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+      name: Deploy to Netlify
+      run: ./.github/deploy_to_netlify.py
   test-bahamut:
     needs:
     - build-bahamut
@@ -435,8 +436,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -444,11 +445,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
         bahamut
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results bahamut (devel)
+        name: pytest-results_bahamut_devel
         path: pytest.xml
   test-bahamut-anope:
     needs:
@@ -473,8 +475,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -482,11 +484,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH   make
         bahamut-anope
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results bahamut-anope (devel)
+        name: pytest-results_bahamut-anope_devel
         path: pytest.xml
   test-bahamut-atheme:
     needs:
@@ -505,8 +508,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -514,11 +517,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
         bahamut-atheme
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results bahamut-atheme (devel)
+        name: pytest-results_bahamut-atheme_devel
         path: pytest.xml
   test-ergo:
     needs: []
@@ -544,8 +548,8 @@ jobs:
         cd $GITHUB_WORKSPACE/ergo/
         make build
         make install
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -553,11 +557,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/go/sbin:~/go/bin:$PATH
         make ergo
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results ergo (devel)
+        name: pytest-results_ergo_devel
         path: pytest.xml
   test-hybrid:
     needs:
@@ -582,8 +587,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -591,11 +596,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH   make
         hybrid
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results hybrid (devel)
+        name: pytest-results_hybrid_devel
         path: pytest.xml
   test-inspircd:
     needs:
@@ -614,8 +620,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -623,11 +629,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/inspircd/sbin:~/.local/inspircd/bin:$PATH
         make inspircd
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results inspircd (devel)
+        name: pytest-results_inspircd_devel
         path: pytest.xml
   test-inspircd-anope:
     needs:
@@ -652,8 +659,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -661,11 +668,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/inspircd/sbin:~/.local/inspircd/bin:$PATH  make
         inspircd-anope
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results inspircd-anope (devel)
+        name: pytest-results_inspircd-anope_devel
         path: pytest.xml
   test-ircu2:
     needs: []
@@ -690,8 +698,8 @@ jobs:
         ./configure --prefix=$HOME/.local/ --with-maxcon=1024 --enable-debug
         make -j 4
         make install
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -699,11 +707,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
         ircu2
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results ircu2 (devel)
+        name: pytest-results_ircu2_devel
         path: pytest.xml
   test-limnoria:
     needs: []
@@ -717,8 +726,8 @@ jobs:
     - name: Install dependencies
       run: pip install git+https://github.com/ProgVal/Limnoria.git@testing cryptography
         pyxmpp2-scram
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -726,11 +735,50 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
         limnoria
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results limnoria (devel)
+        name: pytest-results_limnoria_devel
+        path: pytest.xml
+  test-nefarious:
+    needs: []
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - name: Checkout nefarious
+      uses: actions/checkout@v2
+      with:
+        path: nefarious
+        ref: master
+        repository: evilnet/nefarious2
+    - name: Build nefarious
+      run: |
+        cd $GITHUB_WORKSPACE/nefarious
+        ./configure --prefix=$HOME/.local/ --enable-debug
+        make -j 4
+        make install
+        cp $GITHUB_WORKSPACE/data/nefarious/* $HOME/.local/lib
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
+    - name: Install irctest dependencies
+      run: |-
+        python -m pip install --upgrade pip
+        pip install pytest pytest-xdist -r requirements.txt
+    - name: Test with pytest
+      run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
+        nefarious
+      timeout-minutes: 30
+    - if: always()
+      name: Publish results
+      uses: actions/upload-artifact@v2
+      with:
+        name: pytest-results_nefarious_devel
         path: pytest.xml
   test-ngircd:
     needs:
@@ -749,8 +797,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -758,11 +806,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local//sbin:~/.local//bin:$PATH
         make ngircd
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results ngircd (devel)
+        name: pytest-results_ngircd_devel
         path: pytest.xml
   test-ngircd-anope:
     needs:
@@ -787,8 +836,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -796,11 +845,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local//sbin:~/.local//bin:$PATH  make
         ngircd-anope
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results ngircd-anope (devel)
+        name: pytest-results_ngircd-anope_devel
         path: pytest.xml
   test-ngircd-atheme:
     needs:
@@ -819,8 +869,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -828,11 +878,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local//sbin:~/.local//bin:$PATH
         make ngircd-atheme
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results ngircd-atheme (devel)
+        name: pytest-results_ngircd-atheme_devel
         path: pytest.xml
   test-plexus4:
     needs:
@@ -857,8 +908,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -866,11 +917,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH   make
         plexus4
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results plexus4 (devel)
+        name: pytest-results_plexus4_devel
         path: pytest.xml
   test-solanum:
     needs:
@@ -889,8 +941,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -898,11 +950,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
         solanum
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results solanum (devel)
+        name: pytest-results_solanum_devel
         path: pytest.xml
   test-sopel:
     needs: []
@@ -915,8 +968,8 @@ jobs:
         python-version: 3.7
     - name: Install dependencies
       run: pip install git+https://github.com/sopel-irc/sopel.git
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -924,11 +977,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
         sopel
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results sopel (devel)
+        name: pytest-results_sopel_devel
         path: pytest.xml
   test-unrealircd:
     needs:
@@ -947,8 +1001,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -956,11 +1010,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/unrealircd/sbin:~/.local/unrealircd/bin:$PATH
         make unrealircd
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results unrealircd (devel)
+        name: pytest-results_unrealircd_devel
         path: pytest.xml
   test-unrealircd-5:
     needs:
@@ -979,8 +1034,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -988,11 +1043,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/unrealircd/sbin:~/.local/unrealircd/bin:$PATH
         make unrealircd-5
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results unrealircd-5 (devel)
+        name: pytest-results_unrealircd-5_devel
         path: pytest.xml
   test-unrealircd-anope:
     needs:
@@ -1017,8 +1073,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -1026,11 +1082,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/unrealircd/sbin:~/.local/unrealircd/bin:$PATH  make
         unrealircd-anope
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results unrealircd-anope (devel)
+        name: pytest-results_unrealircd-anope_devel
         path: pytest.xml
   test-unrealircd-atheme:
     needs:
@@ -1049,8 +1106,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -1058,11 +1115,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/unrealircd/sbin:~/.local/unrealircd/bin:$PATH
         make unrealircd-atheme
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results unrealircd-atheme (devel)
+        name: pytest-results_unrealircd-atheme_devel
         path: pytest.xml
 name: irctest with devel versions
 'on':

.github/workflows/test-devel_release.yml

@@ -71,7 +71,7 @@ jobs:
         retention-days: 1
   publish-test-results:
     if: success() || failure()
-    name: Publish Unit Tests Results
+    name: Publish Dashboard
     needs:
     - test-inspircd
     - test-inspircd-anope
@@ -83,27 +83,23 @@ jobs:
       uses: actions/download-artifact@v2
       with:
         path: artifacts
-    - if: github.event_name == 'pull_request'
-      name: Publish Unit Test Results
-      uses: actions/github-script@v4
-      with:
-        result-encoding: string
-        script: |
-          let body = '';
-          const options = {};
-          options.listeners = {
-              stdout: (data) => {
-                  body += data.toString();
-              }
-          };
-          await exec.exec('bash', ['-c', 'shopt -s globstar; python3 report.py artifacts/**/*.xml'], options);
-          github.issues.createComment({
-            issue_number: context.issue.number,
-            owner: context.repo.owner,
-            repo: context.repo.repo,
-            body: body,
-          });
-          return body;
+    - name: Install dashboard dependencies
+      run: |-
+        python -m pip install --upgrade pip
+        pip install defusedxml docutils -r requirements.txt
+    - name: Generate dashboard
+      run: |-
+        shopt -s globstar
+        python3 -m irctest.dashboard.format dashboard/ artifacts/**/*.xml
+        echo '/ /index.xhtml' > dashboard/_redirects
+    - name: Install netlify-cli
+      run: npm i -g netlify-cli
+    - env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+        NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+      name: Deploy to Netlify
+      run: ./.github/deploy_to_netlify.py
   test-inspircd:
     needs:
     - build-inspircd
@@ -121,8 +117,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -130,11 +126,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/inspircd/sbin:~/.local/inspircd/bin:$PATH
         make inspircd
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results inspircd (devel_release)
+        name: pytest-results_inspircd_devel_release
         path: pytest.xml
   test-inspircd-anope:
     needs:
@@ -159,8 +156,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -168,11 +165,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/inspircd/sbin:~/.local/inspircd/bin:$PATH  make
         inspircd-anope
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results inspircd-anope (devel_release)
+        name: pytest-results_inspircd-anope_devel_release
         path: pytest.xml
   test-inspircd-atheme:
     needs:
@@ -191,8 +189,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -200,11 +198,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/inspircd/sbin:~/.local/inspircd/bin:$PATH
         make inspircd-atheme
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results inspircd-atheme (devel_release)
+        name: pytest-results_inspircd-atheme_devel_release
         path: pytest.xml
 name: irctest with devel_release versions
 'on':

.github/workflows/test-stable.yml

@@ -342,7 +342,7 @@ jobs:
       uses: actions/checkout@v2
       with:
         path: unrealircd
-        ref: daa0c11f285c7123ba9fa2966dee2d1a17729f1e
+        ref: cedd23ae9cdd5985ce16e9869cbdb808479c3fc4
         repository: unrealircd/unrealircd
     - name: Build UnrealIRCd 6
       run: |
@@ -355,6 +355,8 @@ jobs:
         CFLAGS="-O0 -march=x86-64" CXXFLAGS="$CFLAGS" ./Config -quick
         make -j 4
         make install
+        # Prevent download of geoIP database on first startup
+        sed -i 's/loadmodule "geoip_classic";//' ~/.local/unrealircd/conf/modules.default.conf
     - name: Make artefact tarball
       run: cd ~; tar -czf artefacts-unrealircd.tar.gz .local/ go/
     - name: Upload build artefacts
@@ -399,6 +401,8 @@ jobs:
         CFLAGS="-O0 -march=x86-64" CXXFLAGS="$CFLAGS" ./Config -quick
         make -j 4
         make install
+        # Prevent download of geoIP database on first startup
+        sed -i 's/loadmodule "geoip_classic";//' ~/.local/unrealircd/conf/modules.default.conf
     - name: Make artefact tarball
       run: cd ~; tar -czf artefacts-unrealircd-5.tar.gz .local/ go/
     - name: Upload build artefacts
@@ -409,7 +413,7 @@ jobs:
         retention-days: 1
   publish-test-results:
     if: success() || failure()
-    name: Publish Unit Tests Results
+    name: Publish Dashboard
     needs:
     - test-bahamut
     - test-bahamut-anope
@@ -423,6 +427,7 @@ jobs:
     - test-irc2
     - test-ircu2
     - test-limnoria
+    - test-nefarious
     - test-ngircd
     - test-ngircd-anope
     - test-ngircd-atheme
@@ -440,27 +445,23 @@ jobs:
       uses: actions/download-artifact@v2
       with:
         path: artifacts
-    - if: github.event_name == 'pull_request'
-      name: Publish Unit Test Results
-      uses: actions/github-script@v4
-      with:
-        result-encoding: string
-        script: |
-          let body = '';
-          const options = {};
-          options.listeners = {
-              stdout: (data) => {
-                  body += data.toString();
-              }
-          };
-          await exec.exec('bash', ['-c', 'shopt -s globstar; python3 report.py artifacts/**/*.xml'], options);
-          github.issues.createComment({
-            issue_number: context.issue.number,
-            owner: context.repo.owner,
-            repo: context.repo.repo,
-            body: body,
-          });
-          return body;
+    - name: Install dashboard dependencies
+      run: |-
+        python -m pip install --upgrade pip
+        pip install defusedxml docutils -r requirements.txt
+    - name: Generate dashboard
+      run: |-
+        shopt -s globstar
+        python3 -m irctest.dashboard.format dashboard/ artifacts/**/*.xml
+        echo '/ /index.xhtml' > dashboard/_redirects
+    - name: Install netlify-cli
+      run: npm i -g netlify-cli
+    - env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+        NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+      name: Deploy to Netlify
+      run: ./.github/deploy_to_netlify.py
   test-bahamut:
     needs:
     - build-bahamut
@@ -478,8 +479,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -487,11 +488,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
         bahamut
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results bahamut (stable)
+        name: pytest-results_bahamut_stable
         path: pytest.xml
   test-bahamut-anope:
     needs:
@@ -516,8 +518,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -525,11 +527,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH   make
         bahamut-anope
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results bahamut-anope (stable)
+        name: pytest-results_bahamut-anope_stable
         path: pytest.xml
   test-bahamut-atheme:
     needs:
@@ -548,8 +551,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -557,11 +560,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
         bahamut-atheme
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results bahamut-atheme (stable)
+        name: pytest-results_bahamut-atheme_stable
         path: pytest.xml
   test-charybdis:
     needs:
@@ -580,8 +584,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -589,11 +593,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
         charybdis
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results charybdis (stable)
+        name: pytest-results_charybdis_stable
         path: pytest.xml
   test-ergo:
     needs: []
@@ -619,8 +624,8 @@ jobs:
         cd $GITHUB_WORKSPACE/ergo/
         make build
         make install
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -628,11 +633,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/go/sbin:~/go/bin:$PATH
         make ergo
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results ergo (stable)
+        name: pytest-results_ergo_stable
         path: pytest.xml
   test-hybrid:
     needs:
@@ -657,8 +663,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -666,11 +672,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH   make
         hybrid
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results hybrid (stable)
+        name: pytest-results_hybrid_stable
         path: pytest.xml
   test-inspircd:
     needs:
@@ -689,8 +696,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -698,11 +705,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/inspircd/sbin:~/.local/inspircd/bin:$PATH
         make inspircd
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results inspircd (stable)
+        name: pytest-results_inspircd_stable
         path: pytest.xml
   test-inspircd-anope:
     needs:
@@ -727,8 +735,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -736,11 +744,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/inspircd/sbin:~/.local/inspircd/bin:$PATH  make
         inspircd-anope
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results inspircd-anope (stable)
+        name: pytest-results_inspircd-anope_stable
         path: pytest.xml
   test-inspircd-atheme:
     needs:
@@ -759,8 +768,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -768,11 +777,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/inspircd/sbin:~/.local/inspircd/bin:$PATH
         make inspircd-atheme
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results inspircd-atheme (stable)
+        name: pytest-results_inspircd-atheme_stable
         path: pytest.xml
   test-irc2:
     needs: []
@@ -783,38 +793,33 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.7
-    - name: Get source code
-      run: curl http://ftp.irc.org/ftp/irc/server/irc2.11.2p3.tgz | tar -zx
-    - name: Configure
-      run: 'cd $GITHUB_WORKSPACE/irc2.11.2p3
-
+    - name: Checkout irc2
+      uses: actions/checkout@v2
+      with:
+        path: irc2.11.2p3
+        ref: 59649f24c3a5c27bad5648b48774f27475bccfd3
+        repository: irc-archive/irc2-mirror
+    - name: Build irc2
+      run: |
+        # Configure
+        cd $GITHUB_WORKSPACE/irc2.11.2p3
         ./configure --prefix=$HOME/.local/
-
         cd x86*
-
         echo "#define CMDLINE_CONFIG/" >> config.h
-
         echo "#define DEFAULT_SPLIT_USERS 0" >> config.h
-
         echo "#define DEFAULT_SPLIT_SERVERS 0" >> config.h
-
         #echo "#undef LIST_ALIS_NOTE" >> config.h
-
         # TODO: find a better way to make it not fork...
+        echo "#define fork() (0)" >> config.h
 
-        echo "#define fork() (0)" >> config.h'
-    - name: Compile and install
-      run: 'cd $GITHUB_WORKSPACE/irc2.11.2p3/x86*
-
+        # Compile and install
+        cd $GITHUB_WORKSPACE/irc2.11.2p3/x86*
         make -j 4 all
-
         make install
-
         mkdir -p $HOME/.local/bin
-
-        cp $HOME/.local/sbin/ircd $HOME/.local/bin/ircd'
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+        cp $HOME/.local/sbin/ircd $HOME/.local/bin/ircd
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -822,11 +827,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
         irc2
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results irc2 (stable)
+        name: pytest-results_irc2_stable
         path: pytest.xml
   test-ircu2:
     needs: []
@@ -851,8 +857,8 @@ jobs:
         ./configure --prefix=$HOME/.local/ --with-maxcon=1024 --enable-debug
         make -j 4
         make install
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -860,11 +866,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
         ircu2
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results ircu2 (stable)
+        name: pytest-results_ircu2_stable
         path: pytest.xml
   test-limnoria:
     needs: []
@@ -877,8 +884,8 @@ jobs:
         python-version: 3.7
     - name: Install dependencies
       run: pip install limnoria==2022.03.17 cryptography pyxmpp2-scram
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -886,11 +893,50 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
         limnoria
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results limnoria (stable)
+        name: pytest-results_limnoria_stable
+        path: pytest.xml
+  test-nefarious:
+    needs: []
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - name: Checkout nefarious
+      uses: actions/checkout@v2
+      with:
+        path: nefarious
+        ref: 985704168ecada12d9e53b46df6087ef9d9fb40b
+        repository: evilnet/nefarious2
+    - name: Build nefarious
+      run: |
+        cd $GITHUB_WORKSPACE/nefarious
+        ./configure --prefix=$HOME/.local/ --enable-debug
+        make -j 4
+        make install
+        cp $GITHUB_WORKSPACE/data/nefarious/* $HOME/.local/lib
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
+    - name: Install irctest dependencies
+      run: |-
+        python -m pip install --upgrade pip
+        pip install pytest pytest-xdist -r requirements.txt
+    - name: Test with pytest
+      run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
+        nefarious
+      timeout-minutes: 30
+    - if: always()
+      name: Publish results
+      uses: actions/upload-artifact@v2
+      with:
+        name: pytest-results_nefarious_stable
         path: pytest.xml
   test-ngircd:
     needs:
@@ -909,8 +955,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -918,11 +964,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local//sbin:~/.local//bin:$PATH
         make ngircd
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results ngircd (stable)
+        name: pytest-results_ngircd_stable
         path: pytest.xml
   test-ngircd-anope:
     needs:
@@ -947,8 +994,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -956,11 +1003,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local//sbin:~/.local//bin:$PATH  make
         ngircd-anope
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results ngircd-anope (stable)
+        name: pytest-results_ngircd-anope_stable
         path: pytest.xml
   test-ngircd-atheme:
     needs:
@@ -979,8 +1027,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -988,11 +1036,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local//sbin:~/.local//bin:$PATH
         make ngircd-atheme
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results ngircd-atheme (stable)
+        name: pytest-results_ngircd-atheme_stable
         path: pytest.xml
   test-plexus4:
     needs:
@@ -1017,8 +1066,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -1026,11 +1075,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH   make
         plexus4
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results plexus4 (stable)
+        name: pytest-results_plexus4_stable
         path: pytest.xml
   test-solanum:
     needs:
@@ -1049,8 +1099,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -1058,11 +1108,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
         solanum
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results solanum (stable)
+        name: pytest-results_solanum_stable
         path: pytest.xml
   test-sopel:
     needs: []
@@ -1075,8 +1126,8 @@ jobs:
         python-version: 3.7
     - name: Install dependencies
       run: pip install sopel==7.1.8
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -1084,11 +1135,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  make
         sopel
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results sopel (stable)
+        name: pytest-results_sopel_stable
         path: pytest.xml
   test-unrealircd:
     needs:
@@ -1107,8 +1159,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -1116,11 +1168,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/unrealircd/sbin:~/.local/unrealircd/bin:$PATH
         make unrealircd
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results unrealircd (stable)
+        name: pytest-results_unrealircd_stable
         path: pytest.xml
   test-unrealircd-5:
     needs:
@@ -1139,8 +1192,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -1148,11 +1201,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/unrealircd/sbin:~/.local/unrealircd/bin:$PATH
         make unrealircd-5
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results unrealircd-5 (stable)
+        name: pytest-results_unrealircd-5_stable
         path: pytest.xml
   test-unrealircd-anope:
     needs:
@@ -1177,8 +1231,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -1186,11 +1240,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/unrealircd/sbin:~/.local/unrealircd/bin:$PATH  make
         unrealircd-anope
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results unrealircd-anope (stable)
+        name: pytest-results_unrealircd-anope_stable
         path: pytest.xml
   test-unrealircd-atheme:
     needs:
@@ -1209,8 +1264,8 @@ jobs:
         path: '~'
     - name: Unpack artefacts
       run: cd ~; find -name 'artefacts-*.tar.gz' -exec tar -xzf '{}' \;
-    - name: Install Atheme
-      run: sudo apt-get install atheme-services
+    - name: Install system dependencies
+      run: sudo apt-get install atheme-services faketime
     - name: Install irctest dependencies
       run: |-
         python -m pip install --upgrade pip
@@ -1218,11 +1273,12 @@ jobs:
     - name: Test with pytest
       run: PYTEST_ARGS='--junit-xml pytest.xml' PATH=$HOME/.local/bin:$PATH  PATH=~/.local/unrealircd/sbin:~/.local/unrealircd/bin:$PATH
         make unrealircd-atheme
+      timeout-minutes: 30
     - if: always()
       name: Publish results
       uses: actions/upload-artifact@v2
       with:
-        name: pytest results unrealircd-atheme (stable)
+        name: pytest-results_unrealircd-atheme_stable
         path: pytest.xml
 name: irctest with stable versions
 'on':

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ exclude: ^irctest/scram
 
 repos:
   - repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 22.3.0
     hooks:
       - id: black
         language_version: python3

Makefile

@@ -7,88 +7,47 @@ PYTEST_ARGS ?=
 # Will be appended at the end of the -k argument to pytest
 EXTRA_SELECTORS ?=
 
-# testPlainLarge fails because it doesn't handle split AUTHENTICATE (reported on IRC)
-ANOPE_SELECTORS := \
-	and not testPlainLarge
-
-# buffering tests cannot pass because of issues with UTF-8 handling: https://github.com/DALnet/bahamut/issues/196
-# mask tests in test_who.py fail because they are not implemented.
-# some HelpTestCase::*[HELP] tests fail because Bahamut forwards /HELP to HelpServ (but not /HELPOP)
-# testWhowasMultiTarget fails because Bahamut returns the results in query order instead of chronological order
 BAHAMUT_SELECTORS := \
 	not Ergo \
 	and not deprecated \
 	and not strict \
 	and not IRCv3 \
-	and not buffering \
-	and not (testWho and not whois and mask) \
-	and not testWhoStar \
-	and (not HelpTestCase or HELPOP) \
-	and not testWhowasMultiTarget \
 	$(EXTRA_SELECTORS)
 
-# testQuitErrors is very flaky
-# AccountTagTestCase.testInvite fails because https://github.com/solanum-ircd/solanum/issues/166
-# testKickDefaultComment fails because it uses the nick of the kickee rather than the kicker.
-# testWhoisNumerics[oper] fails because charybdis uses RPL_WHOISSPECIAL instead of RPL_WHOISOPERATOR
-# testWhowasNoSuchNick fails because of a typo (solved in https://github.com/solanum-ircd/solanum/commit/08b7b6bd7e60a760ad47b58cbe8075b45d66166f)
 CHARYBDIS_SELECTORS := \
 	not Ergo \
 	and not deprecated \
 	and not strict \
-	and not testQuitErrors \
-	and not testKickDefaultComment \
-	and not (AccountTagTestCase and testInvite) \
-	and not (testWhoisNumerics and oper) \
-	and not testWhowasNoSuchNick \
 	$(EXTRA_SELECTORS)
 
-# testInfoNosuchserver does not apply to Ergo: Ergo ignores the optional <target> argument
 ERGO_SELECTORS := \
 	not deprecated \
-	and not testInfoNosuchserver \
 	$(EXTRA_SELECTORS)
 
-# testInviteUnopped is the only strict test that Hybrid fails
 HYBRID_SELECTORS := \
 	not Ergo \
-	and not testInviteUnopped \
 	and not deprecated \
 	$(EXTRA_SELECTORS)
 
-# testBotPrivateMessage and testBotChannelMessage fail because https://github.com/inspircd/inspircd/pull/1910 is not released yet
-# WHOWAS tests fail because https://github.com/inspircd/inspircd/pull/1967 and https://github.com/inspircd/inspircd/pull/1968 are not released yet
 INSPIRCD_SELECTORS := \
 	not Ergo \
 	and not deprecated \
 	and not strict \
-	and not testNoticeNonexistentChannel \
-	and not testBotPrivateMessage and not testBotChannelMessage \
-	and not whowas \
 	$(EXTRA_SELECTORS)
 
-# buffering tests fail because ircu2 discards the whole buffer on long lines (TODO: refine how we exclude these tests)
-# testQuit and testQuitErrors fail because ircu2 does not send ERROR or QUIT
-# lusers "full" tests fail because they depend on Modern behavior, not just RFC2812
-# statusmsg tests fail because STATUSMSG is present in ISUPPORT, but it not actually supported as PRIVMSG target
-# testKeyValidation[empty] fails because ircu2 returns ERR_NEEDMOREPARAMS on empty keys: https://github.com/UndernetIRC/ircu2/issues/13
-# testKickDefaultComment fails because it uses the nick of the kickee rather than the kicker.
-# testEmptyRealname fails because it uses a default value instead of ERR_NEEDMOREPARAMS.
 # HelpTestCase fails because it returns NOTICEs instead of numerics
-# testWhowasCountZero fails: https://github.com/UndernetIRC/ircu2/pull/19
 IRCU2_SELECTORS := \
 	not Ergo \
 	and not deprecated \
 	and not strict \
-	and not buffering \
-	and not testQuit \
-	and not (lusers and full) \
-	and not statusmsg \
-	and not (testKeyValidation and empty) \
-	and not testKickDefaultComment \
-	and not testEmptyRealname \
-	and not HelpTestCase \
-	and not testWhowasCountZero \
+	$(EXTRA_SELECTORS)
+
+# same justification as ircu2
+# lusers "unregistered" tests fail because 
+NEFARIOUS_SELECTORS := \
+	not Ergo \
+	and not deprecated \
+	and not strict \
 	$(EXTRA_SELECTORS)
 
 # same justification as ircu2
@@ -96,24 +55,12 @@ SNIRCD_SELECTORS := \
 	not Ergo \
 	and not deprecated \
 	and not strict \
-	and not buffering \
-	and not testQuit \
-	and not (lusers and full) \
-	and not statusmsg \
 	$(EXTRA_SELECTORS)
 
-# testListEmpty and testListOne fails because irc2 deprecated LIST
-# testKickDefaultComment fails because it uses the nick of the kickee rather than the kicker.
-# testWallopsPrivileges fails because it ignores the command instead of replying ERR_UNKNOWNCOMMAND
-# HelpTestCase fails because it returns NOTICEs instead of numerics
 IRC2_SELECTORS := \
 	not Ergo \
 	and not deprecated \
 	and not strict \
-	and not testListEmpty and not testListOne \
-	and not testKickDefaultComment \
-	and not testWallopsPrivileges \
-	and not HelpTestCase \
 	$(EXTRA_SELECTORS)
 
 MAMMON_SELECTORS := \
@@ -122,28 +69,14 @@ MAMMON_SELECTORS := \
 	and not strict \
 	$(EXTRA_SELECTORS)
 
-# testKeyValidation[spaces] and testKeyValidation[empty] fail because ngIRCd does not validate them https://github.com/ngircd/ngircd/issues/290
-# testStarNick: wat
-# testEmptyRealname fails because it uses a default value instead of ERR_NEEDMOREPARAMS.
-# chathistory tests fail because they need nicks longer than 9 chars
-# HelpTestCase::*[HELP] fails because it returns NOTICEs instead of numerics
 NGIRCD_SELECTORS := \
 	not Ergo \
 	and not deprecated \
 	and not strict \
-	and not (testKeyValidation and (spaces or empty)) \
-	and not testStarNick \
-	and not testEmptyRealname \
-	and not chathistory \
-	and (not HelpTestCase or HELPOP) \
 	$(EXTRA_SELECTORS)
 
-# testInviteUnopped is the only strict test that Plexus4 fails
-# testInviteInviteOnly fails because Plexus4 allows non-op to invite if (and only if) the channel is not invite-only
 PLEXUS4_SELECTORS := \
 	not Ergo \
-	and not testInviteUnopped \
-	and not testInviteInviteOnly \
 	and not deprecated \
 	$(EXTRA_SELECTORS)
 
@@ -154,51 +87,33 @@ LIMNORIA_SELECTORS := \
 	(foo or not foo) \
 	$(EXTRA_SELECTORS)
 
-# testQuitErrors is too flaky for CI
-# testKickDefaultComment fails because solanum uses the nick of the kickee rather than the kicker.
 SOLANUM_SELECTORS := \
 	not Ergo \
 	and not deprecated \
 	and not strict \
-	and not testQuitErrors \
-	and not testKickDefaultComment \
 	$(EXTRA_SELECTORS)
 
+# Same as Limnoria
 SOPEL_SELECTORS := \
-	not testPlainNotAvailable \
+	(foo or not foo) \
 	$(EXTRA_SELECTORS)
 
-# testNoticeNonexistentChannel fails: https://bugs.unrealircd.org/view.php?id=5949
-# regressions::testTagCap fails: https://bugs.unrealircd.org/view.php?id=5948
-# messages::testLineTooLong fails: https://bugs.unrealircd.org/view.php?id=5947
-# testCapRemovalByClient and testNakWhole fail pending https://github.com/unrealircd/unrealircd/pull/148
 # Tests marked with arbitrary_client_tags can't pass because Unreal whitelists which tags it relays
 # Tests marked with react_tag can't pass because Unreal blocks +draft/react https://github.com/unrealircd/unrealircd/pull/149
 # Tests marked with private_chathistory can't pass because Unreal does not implement CHATHISTORY for DMs
-# testChathistory[BETWEEN] fails: https://bugs.unrealircd.org/view.php?id=5952
-# testChathistory[AROUND] fails: https://bugs.unrealircd.org/view.php?id=5953
-# testWhoAllOpers fails because Unreal skips results when the mask is too broad
-# HELP and HELPOP tests fail because Unreal uses custom numerics https://github.com/unrealircd/unrealircd/pull/184
+
 UNREALIRCD_SELECTORS := \
 	not Ergo \
 	and not deprecated \
 	and not strict \
-	and not testNoticeNonexistentChannel \
-	and not (regressions.py and testTagCap) \
-	and not (messages.py and testLineTooLong) \
-	and not (cap.py and (testCapRemovalByClient or testNakWhole)) \
-	and not (account_tag.py and testInvite) \
 	and not arbitrary_client_tags \
 	and not react_tag \
 	and not private_chathistory \
-	and not (testChathistory and (between or around)) \
-	and not testWhoAllOpers \
-	and not HelpTestCase \
 	$(EXTRA_SELECTORS)
 
-.PHONY: all flakes bahamut charybdis ergo inspircd ircu2 snircd irc2 mammon limnoria sopel solanum unrealircd
+.PHONY: all flakes bahamut charybdis ergo inspircd ircu2 snircd irc2 mammon nefarious limnoria sopel solanum unrealircd
 
-all: flakes bahamut charybdis ergo inspircd ircu2 snircd irc2 mammon limnoria sopel solanum unrealircd
+all: flakes bahamut charybdis ergo inspircd ircu2 snircd irc2 mammon nefarious limnoria sopel solanum unrealircd
 
 flakes:
 	find irctest/ -name "*.py" -not -path "irctest/scram/*" -print0 | xargs -0 pyflakes3
@@ -224,7 +139,7 @@ bahamut-anope:
 		--services-controller=irctest.controllers.anope_services \
 		-m 'services' \
 		-n 10 \
-		-k '$(BAHAMUT_SELECTORS) $(ANOPE_SELECTORS)'
+		-k '$(BAHAMUT_SELECTORS)'
 
 charybdis:
 	$(PYTEST) $(PYTEST_ARGS) \
@@ -261,7 +176,7 @@ inspircd-anope:
 		--controller=irctest.controllers.inspircd \
 		--services-controller=irctest.controllers.anope_services \
 		-m 'services' \
-		-k '$(INSPIRCD_SELECTORS) $(ANOPE_SELECTORS)'
+		-k '$(INSPIRCD_SELECTORS)'
 
 ircu2:
 	$(PYTEST) $(PYTEST_ARGS) \
@@ -270,6 +185,13 @@ ircu2:
 		-n 10 \
 		-k '$(IRCU2_SELECTORS)'
 
+nefarious:
+	$(PYTEST) $(PYTEST_ARGS) \
+		--controller=irctest.controllers.nefarious \
+		-m 'not services' \
+		-n 10 \
+		-k '$(NEFARIOUS_SELECTORS)'
+
 snircd:
 	$(PYTEST) $(PYTEST_ARGS) \
 		--controller=irctest.controllers.snircd \
@@ -352,4 +274,4 @@ unrealircd-anope:
 		--controller=irctest.controllers.unrealircd \
 		--services-controller=irctest.controllers.anope_services \
 		-m 'services' \
-		-k '$(UNREALIRCD_SELECTORS) $(ANOPE_SELECTORS)'
+		-k '$(UNREALIRCD_SELECTORS)'

README.md

@@ -18,6 +18,7 @@ have no side effect.
 Install irctest and dependencies:
 
 ```
+sudo apt install faketime  # Optional, but greatly speeds up irctest/server_tests/list.py
 cd ~
 git clone https://github.com/ProgVal/irctest.git
 cd irctest

data/nefarious/ircd.pem

@@ -0,0 +1,83 @@
+-----BEGIN PRIVATE KEY-----
+MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQDT0URxi7/l7ZGe
+tkPv9Yh8h2s9BpbAR4Wq8sakgqETWg/nE/JQM5dPxroVbtZWWQXuJEFsgBKbASLa
+/eg5cyJv4Uu5WIZpG1LxdPEEIOSMWjzoAGwoLxbTRGrS7qNXsknB9RwDuq8lPQiK
+kiAahg1Cn1vRrQ4cRrG+AkQWpRHJEDoLjCSo8IcAsKAZlw/eGtAcmeNvkr5AujEw
+XjIwx2FoDyKaNGRH5Z7gLWvCKBNxQuJuMTzh8guLqdGbE4hH3rqyICbW5DGPaOZL
+LErWuJ7kEhLZG2HDW5JaXOr0QfFYAA8pl9/qCuFMdoxRUKRcYBoxoMmz6dlsmipN
+7vIj+TT6TemwcAT25pwMJIVS4WC4+BZilNH2eWKD9hZA8Kq7FDPu+1rxOJaLbE/b
+vpK8jZeRdqFzE1eBCgPkw8D8V0r7J18d+DsmgOe2kRycaia/t9M4rhqe0FXjX1X1
+lzQ52grxgc28Ejd1fGQXIJmdTh4BqKqTzxup0izS7dgFP1Ezm6Z4O+wklpL5uQF2
+Ex4X6QEj76iCxH+J/01/cvbxMe3iuGXECbO/y1FIrg7FKzZSrQo4aP63lS7Y7aq0
+t2t6kOS83ebhnpgHClgFs8/C3ayzYBBtbK63PYthwO8Rt6WamCIZFF5tA3XoI4Ak
+fZcWD18loZai+QznVzbLNINf++rTwwIDAQABAoICAQCs1tT3piZHU2yAyo9zLbJa
+kxGxcT/v1CzBSmtG8ATJyrKxRzhxszdj9HABbzjcqrXJFbKA+5yy+OFdOtSUlFtk
+Wb21lwPOnmo29sp4KPL1h+itEzMuMwZ4DBry1aFZvPSsnPpoHJwwUbY3hHdHzVzi
+oTCGTqT188Wzmxu+MqHppCEJLSj45ZPzvyxU1UwwW0a4H+ZTM7WlEYlzw1lHLlpQ
+VBFTLS8q77aNjOKiQptiz0X+zpS0dhJvu3l7BhwtMRS8prmqnffG4r0QWCsVPP8C
+cbEJkWtbwswQikN6Xpi1yw6UTQZ8brZa810aOSh07EJTfrU35rjxAndEspbJPd+4
+Zz6fKNnRA7A4fFap2hF4TSP/UF12b4ulQ8FfcMMTFszWko5M6fWFuCeWfNbyXML5
+fmn+NmSOboj7LkDqbpxtyaIVXXb2Y3F6A2fNllm/mxaGrRoEGNaH3o0qBgWRzJJB
+TDSvIQtJddzL+iMaqxz4ufXAREJernZmPa3vlkVGLINNQUC9JLrB5eFjLzPiZN2U
+8RgQ9YX5tjoJ+DtPWuMFDiCS1ZE20/UBOEYTeqIVuKdK3AjJDMFSjg8fRvsWRqZe
+zsHv6tCtIFZFxYRxtrRGTUPQF+1QD6zBjYxZZk1B4n3uYBGVQFM/LnNHUxRnJBx1
+PunD4ICOY97xd2hcPmGiCQKCAQEA8NCXYaHzhv6fg95H/iMuJVcOCKrJ5rVr4poG
+SD0KZtS7SLzUYat8WcuoSubh5Eb2hHtrsnLrSOTnwQUO61f4gCRm2sEqHYsOAd7+
+mNe1jfil0UBVqqL9GBcGYJkc5+DHgUlJQaxMV+4YLt8fD0KfZEnHaDAYX3kUdz+p
+be//YAKv+JmxWcUdBF60AUWPjbCJT/1pfJeY8nEBFiYtlYKKN24+4OiRdJ2yRGzt
+ZtNHaWy5EFF70yVgPX5MGQ7Z2JpejzK+lt+9nG4h1uJ4M2X4YrGVrRCn1W8jwqm/
+bXest3E6wkkLoWDm9EaeYj00DUgMOviPyP4ckyxilG+Fny4JbwKCAQEA4SyUV03X
+KEoL5sOD69sLu3TpnIQz73u9an9W/f2f7rsGwmCcR9RdYGV29ltwfBvOm0FnPQio
+GliN+3PTWAL6bb8VYo2IR53VKpVHKSQUlzDOD9PGObXw1CT/+0zoMP7FBA4dTJDf
+xQ63AQNpSCGdwbxZygPWzLV5O1WxMeXhnQRL1EBvMyJ52od0+HbajDXg5mNiBKNQ
+AtVhB9pEu47Blu/KBqWjfh/GeBLPZB7MHmGNBYbBGGskKRLG4sIbwShs9cx8UM0/
+9dxXkX2d8hjnSD/0ZBh54HHUaEvKAKfpz1L8AC0ll7biCAy0CZK23AoZu/KT8zJ+
+qvz3AuJcW3lo7QKCAQEAzfGltNJawPUamBzNttKBUV+s2c6tkkdO92C/xKGnNp/x
+dtg+TTTpyKV5zGy9fIsPoecnCFptS06vwAvCYZQ/Kd93stcFXHSiSwlY9H9tfffK
+XzTEzoRLLIHsa0omRUufcrqpEqf2NjChr9wS5OsWAx9xkHGpNmUHEqB4FlPsM0C5
+G0LdQCdplGYlTP0fMo5qL+VJhErle1kXE8kcrMMRzyvSTGe4lWGTph79vDUt2kQn
+1IPLAJzzPEO5cqiXtzz1Z0N/aOn5b0FkYTAWmeY30LeMiJA46Df+/ihLVKPHKq6E
+EMmFT8LeYMPQCbXLwRv/kaMm3D4tU9PejpD9Vk95swKCAQAtULBlxXeIVyaAAVba
+L1H0Hroo0n41MtzSwt+568G05JSep5yr4/QKw0CmoY5Im7v/iLEDGmviKXIhaZTd
+wHOvhGYEWGFVsFDG6hXRFL7EEoFVtBPPZ2sY9n1BkJ+lxI/XmhORZhJycNypaotU
+hddets4HFrCyr86+/ybS2OWHmOa9x13Zl5WYQexrWFfxIaKqGtQOBOPEPjbxwp5U
+dI1HF+i7X7hAWJqzbW2pQ31mm9EqjIztoho73diCp/e37q/G46kdBcFadEZ3NCWG
+JDbfVmeTgU19usq5Vo9HhINMQvIOAwfuuVJRtmTBDHKaY7n8FfxqU/4j4RbA0Ncv
+XYadAoIBAQC7yh4/UZCGhklUhhk/667OfchWvWGriCSaYGmQPdMzxjnIjAvvIUe9
+riOTBSZXYYLmZHsmY/jK7KMGB3AsLTypSa9+ddAWqWn2dvOYyxNiAaSJK/RZfA9A
+ocVfvvkhOfNAYIF+A+fyJ2pznsDkBf9tPkhN7kovl+mr/e25qZb1d09377770Pi7
+thzEi+JLrRgYVLrCrPi2j4l7/Va/UaAPz+Dtu2GCT9vXgnhZtpb8R1kTViZFryTv
+k+qbNYJzVm61Vit9mVAGe+WuzhlclJnN6LIZGG3zYHIulRAJrH1XDauHZfHzCKgi
+FnMesy4thDMH/MhUfRtbylZTq45gtvCA
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIUYHD08+9S32VTD9IEsr2Oe1dH3VEwDQYJKoZIhvcNAQEL
+BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjA0MDQxODE2NTZaFw0yMzA0
+MDQxODE2NTZaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggIiMA0GCSqGSIb3DQEB
+AQUAA4ICDwAwggIKAoICAQDT0URxi7/l7ZGetkPv9Yh8h2s9BpbAR4Wq8sakgqET
+Wg/nE/JQM5dPxroVbtZWWQXuJEFsgBKbASLa/eg5cyJv4Uu5WIZpG1LxdPEEIOSM
+WjzoAGwoLxbTRGrS7qNXsknB9RwDuq8lPQiKkiAahg1Cn1vRrQ4cRrG+AkQWpRHJ
+EDoLjCSo8IcAsKAZlw/eGtAcmeNvkr5AujEwXjIwx2FoDyKaNGRH5Z7gLWvCKBNx
+QuJuMTzh8guLqdGbE4hH3rqyICbW5DGPaOZLLErWuJ7kEhLZG2HDW5JaXOr0QfFY
+AA8pl9/qCuFMdoxRUKRcYBoxoMmz6dlsmipN7vIj+TT6TemwcAT25pwMJIVS4WC4
++BZilNH2eWKD9hZA8Kq7FDPu+1rxOJaLbE/bvpK8jZeRdqFzE1eBCgPkw8D8V0r7
+J18d+DsmgOe2kRycaia/t9M4rhqe0FXjX1X1lzQ52grxgc28Ejd1fGQXIJmdTh4B
+qKqTzxup0izS7dgFP1Ezm6Z4O+wklpL5uQF2Ex4X6QEj76iCxH+J/01/cvbxMe3i
+uGXECbO/y1FIrg7FKzZSrQo4aP63lS7Y7aq0t2t6kOS83ebhnpgHClgFs8/C3ayz
+YBBtbK63PYthwO8Rt6WamCIZFF5tA3XoI4AkfZcWD18loZai+QznVzbLNINf++rT
+wwIDAQABo1MwUTAdBgNVHQ4EFgQU+9eHi2eqy0f3fDS0GjqkijGDDocwHwYDVR0j
+BBgwFoAU+9eHi2eqy0f3fDS0GjqkijGDDocwDwYDVR0TAQH/BAUwAwEB/zANBgkq
+hkiG9w0BAQsFAAOCAgEAAJXO3qUc/PW75pI2dt1cKv20VqozkfEf7P0eeVisCDxn
+1p3QhVgI2lEe9kzdHp7t42g5xLkUhQEVmBcKm9xbl+k2D1X0+T8px1x6ZiWfbhXL
+ptc/qCIXjPCgVN3s+Kasii3hHkZxKGZz/ySmBmfDJZjQZtbZzQWpvvX6SD4s7sjo
+gWbZW3qvQ0bFTGdD1IjKYGaxK6aSrNkAIutiAX4RczJ1QSwb9Z2EIen+ABAvOZS9
+xv3LiiidWcuOT7WzXEa4QvOslCEkAF+jj6mGYB7NWtly0kj4AEPvI4IoYTi9dohS
+CA0zd1DTfjRwpAnT5P4sj4mpjLyRBumeeVGpCZhUxfKpFjIB2AnlgxrU+LPq5c9R
+ZZ9Q5oeLxjRPjpqBeWwgnbjXstQCL9g0U7SsEemsv+zmvG5COhAmG5Wce/65ILlg
+450H4bcn1ul0xvxz9hat6tqEZry3HcNE/CGDT+tXuhHpqOXkY1/c78C0QbWjWodR
+tCvlXW00a+7TlEhNr4XBNdqtIQfYS9K9yiVVNfZLPEsN/SA3BGXmrr+du1/E4Ria
+CkVpmBdJsVu5eMaUj1arsCqI4fwHzljtojJe/pCzZBVkOaSWQEQ+LL4iVnMas68m
+qyshtNf4KNiM55OQmyTiFHMTIxCtdEcHaR3mUxR7GrIhc/bxyxUUBtMAuUX0Kjs=
+-----END CERTIFICATE-----

irctest/basecontrollers.py

@@ -189,6 +189,10 @@ class BaseServerController(_BaseController):
     """Character used for the 'mute' extban"""
     nickserv = "NickServ"
 
+    def __init__(self, *args: Any, **kwargs: Any):
+        super().__init__(*args, **kwargs)
+        self.faketime_enabled = False
+
     def get_hostname_and_port(self) -> Tuple[str, int]:
         return find_hostname_and_port()
 
@@ -202,6 +206,7 @@ class BaseServerController(_BaseController):
         run_services: bool,
         valid_metadata_keys: Optional[Set[str]],
         invalid_metadata_keys: Optional[Set[str]],
+        faketime: Optional[str],
     ) -> None:
         raise NotImplementedError()
 

irctest/cases.py

@@ -69,6 +69,30 @@ TController = TypeVar("TController", bound=basecontrollers._BaseController)
 T = TypeVar("T")
 
 
+def retry(f: TCallable) -> TCallable:
+    """Retry the function if it raises ConnectionClosed; as a workaround for flaky
+    connection, such as::
+
+        1: connects to server.
+        1 -> S: NICK foo
+        1 -> S: USER username * * :Realname
+        S -> 1: :My.Little.Server NOTICE * :*** Found your hostname (cached)
+        S -> 1: :My.Little.Server NOTICE * :*** Checking Ident
+        S -> 1: :My.Little.Server NOTICE * :*** No Ident response
+        S -> 1: ERROR :Closing Link: cpu-pool.com (Use a different port)
+    """
+
+    @functools.wraps(f)
+    def newf(*args, **kwargs):  # type: ignore
+        try:
+            return f(*args, **kwargs)
+        except ConnectionClosed:
+            time.sleep(1)
+            return f(*args, **kwargs)
+
+    return newf  # type: ignore
+
+
 class ChannelJoinException(Exception):
     def __init__(self, code: str, params: List[str]):
         super().__init__(f"Failed to join channel ({code}): {params}")
@@ -508,6 +532,12 @@ class BaseServerTestCase(
     server_support: Optional[Dict[str, Optional[str]]]
     run_services = False
 
+    faketime: Optional[str] = None
+    """If not None and the controller supports it and libfaketime is available,
+    runs the server using faketime and this value set as the $FAKETIME env variable.
+    Tests must check ``self.controller.faketime_enabled`` is True before
+    relying on this."""
+
     __new__ = object.__new__  # pytest won't collect Generic[] subclasses otherwise
 
     def setUp(self) -> None:
@@ -522,6 +552,7 @@ class BaseServerTestCase(
             invalid_metadata_keys=self.invalid_metadata_keys,
             ssl=self.ssl,
             run_services=self.run_services,
+            faketime=self.faketime,
         )
         self.clients: Dict[TClientName, client_mock.ClientMock] = {}
 
@@ -654,6 +685,7 @@ class BaseServerTestCase(
         m = self.getRegistrationMessage(client)
         self.assertIn(m.command, ["900", "903"], str(m))
 
+    @retry
     def connectClient(
         self,
         nick: str,
@@ -672,7 +704,7 @@ class BaseServerTestCase(
         client = self.addClient(name, show_io=show_io)
         if capabilities:
             self.sendLine(client, "CAP LS 302")
-            m = self.getRegistrationMessage(client)
+            self.getCapLs(client)
             self.requestCapabilities(client, capabilities, skip_if_cap_nak)
         if password is not None:
             if "sasl" not in (capabilities or ()):
@@ -702,6 +734,12 @@ class BaseServerTestCase(
                         self.server_support[param] = None
             welcome.append(m)
 
+        self.targmax: Dict[str, Optional[str]] = dict(
+            item.split(":", 1)  # type: ignore
+            for item in (self.server_support.get("TARGMAX") or "").split(",")
+            if item
+        )
+
         return welcome
 
     def joinClient(self, client: TClientName, channel: str) -> None:
@@ -732,50 +770,55 @@ class BaseServerTestCase(
                     raise ChannelJoinException(msg.command, msg.params)
 
 
-_TSelf = TypeVar("_TSelf", bound="OptionalityHelper")
+_TSelf = TypeVar("_TSelf", bound="_IrcTestCase")
 _TReturn = TypeVar("_TReturn")
 
 
-class OptionalityHelper(Generic[TController]):
-    controller: TController
-
-    def checkSaslSupport(self) -> None:
-        if self.controller.supported_sasl_mechanisms:
-            return
-        raise runner.NotImplementedByController("SASL")
-
-    def checkMechanismSupport(self, mechanism: str) -> None:
-        if mechanism in self.controller.supported_sasl_mechanisms:
-            return
-        raise runner.OptionalSaslMechanismNotSupported(mechanism)
-
-    @staticmethod
-    def skipUnlessHasMechanism(
-        mech: str,
-    ) -> Callable[[Callable[..., _TReturn]], Callable[..., _TReturn]]:
-        # Just a function returning a function that takes functions and
-        # returns functions, nothing to see here.
-        # If Python didn't have such an awful syntax for callables, it would be:
-        # str -> ((TSelf -> TReturn) -> (TSelf -> TReturn))
-        def decorator(f: Callable[..., _TReturn]) -> Callable[..., _TReturn]:
-            @functools.wraps(f)
-            def newf(self: _TSelf, *args: Any, **kwargs: Any) -> _TReturn:
-                self.checkMechanismSupport(mech)
-                return f(self, *args, **kwargs)
-
-            return newf
-
-        return decorator
-
-    @staticmethod
-    def skipUnlessHasSasl(f: Callable[..., _TReturn]) -> Callable[..., _TReturn]:
+def skipUnlessHasMechanism(
+    mech: str,
+) -> Callable[[Callable[..., _TReturn]], Callable[..., _TReturn]]:
+    # Just a function returning a function that takes functions and
+    # returns functions, nothing to see here.
+    # If Python didn't have such an awful syntax for callables, it would be:
+    # str -> ((TSelf -> TReturn) -> (TSelf -> TReturn))
+    def decorator(f: Callable[..., _TReturn]) -> Callable[..., _TReturn]:
         @functools.wraps(f)
         def newf(self: _TSelf, *args: Any, **kwargs: Any) -> _TReturn:
-            self.checkSaslSupport()
+            if mech not in self.controller.supported_sasl_mechanisms:
+                raise runner.OptionalSaslMechanismNotSupported(mech)
             return f(self, *args, **kwargs)
 
         return newf
 
+    return decorator
+
+
+def xfailIf(
+    condition: Callable[..., bool], reason: str
+) -> Callable[[Callable[..., _TReturn]], Callable[..., _TReturn]]:
+    # Works about the same as skipUnlessHasMechanism
+    def decorator(f: Callable[..., _TReturn]) -> Callable[..., _TReturn]:
+        @functools.wraps(f)
+        def newf(self: _TSelf, *args: Any, **kwargs: Any) -> _TReturn:
+            if condition(self):
+                try:
+                    return f(self, *args, **kwargs)
+                except Exception:
+                    pytest.xfail(reason)
+                    assert False  # make mypy happy
+            else:
+                return f(self, *args, **kwargs)
+
+        return newf
+
+    return decorator
+
+
+def xfailIfSoftware(
+    names: List[str], reason: str
+) -> Callable[[Callable[..., _TReturn]], Callable[..., _TReturn]]:
+    return xfailIf(lambda testcase: testcase.controller.software_name in names, reason)
+
 
 def mark_services(cls: TClass) -> TClass:
     cls.run_services = True

irctest/client_tests/cap.py

@@ -1,3 +1,5 @@
+"""Format of ``CAP LS`` sent by IRCv3 clients."""
+
 from irctest import cases
 from irctest.irc_utils.message_parser import Message
 

irctest/client_tests/sasl.py

@@ -1,3 +1,8 @@
+"""SASL authentication from clients, for all known mechanisms.
+
+For now, only `SASLv3.1 <https://ircv3.net/specs/extensions/sasl-3.1>`_
+is tested, not `SASLv3.2 <https://ircv3.net/specs/extensions/sasl-3.2>`_."""
+
 import base64
 
 import pytest
@@ -34,8 +39,8 @@ class IdentityHash:
         return self._data
 
 
-class SaslTestCase(cases.BaseClientTestCase, cases.OptionalityHelper):
-    @cases.OptionalityHelper.skipUnlessHasMechanism("PLAIN")
+class SaslTestCase(cases.BaseClientTestCase):
+    @cases.skipUnlessHasMechanism("PLAIN")
     def testPlain(self):
         """Test PLAIN authentication with correct username/password."""
         auth = authentication.Authentication(
@@ -55,7 +60,8 @@ class SaslTestCase(cases.BaseClientTestCase, cases.OptionalityHelper):
         m = self.negotiateCapabilities(["sasl"], False)
         self.assertEqual(m, Message({}, None, "CAP", ["END"]))
 
-    @cases.OptionalityHelper.skipUnlessHasMechanism("PLAIN")
+    @cases.skipUnlessHasMechanism("PLAIN")
+    @cases.xfailIfSoftware(["Sopel"], "Sopel requests SASL PLAIN even if not available")
     def testPlainNotAvailable(self):
         """`sasl=EXTERNAL` is advertized, whereas the client is configured
         to use PLAIN.
@@ -85,7 +91,7 @@ class SaslTestCase(cases.BaseClientTestCase, cases.OptionalityHelper):
         self.assertMessageMatch(m, command="CAP")
 
     @pytest.mark.parametrize("pattern", ["barbaz", "éèà"])
-    @cases.OptionalityHelper.skipUnlessHasMechanism("PLAIN")
+    @cases.skipUnlessHasMechanism("PLAIN")
     def testPlainLarge(self, pattern):
         """Test the client splits large AUTHENTICATE messages whose payload
         is not a multiple of 400.
@@ -114,7 +120,7 @@ class SaslTestCase(cases.BaseClientTestCase, cases.OptionalityHelper):
         m = self.negotiateCapabilities(["sasl"], False)
         self.assertEqual(m, Message({}, None, "CAP", ["END"]))
 
-    @cases.OptionalityHelper.skipUnlessHasMechanism("PLAIN")
+    @cases.skipUnlessHasMechanism("PLAIN")
     @pytest.mark.parametrize("pattern", ["quux", "éè"])
     def testPlainLargeMultiple(self, pattern):
         """Test the client splits large AUTHENTICATE messages whose payload
@@ -145,7 +151,7 @@ class SaslTestCase(cases.BaseClientTestCase, cases.OptionalityHelper):
         self.assertEqual(m, Message({}, None, "CAP", ["END"]))
 
     @pytest.mark.skipif(ecdsa is None, reason="python3-ecdsa is not available")
-    @cases.OptionalityHelper.skipUnlessHasMechanism("ECDSA-NIST256P-CHALLENGE")
+    @cases.skipUnlessHasMechanism("ECDSA-NIST256P-CHALLENGE")
     def testEcdsa(self):
         """Test ECDSA authentication."""
         auth = authentication.Authentication(
@@ -179,7 +185,7 @@ class SaslTestCase(cases.BaseClientTestCase, cases.OptionalityHelper):
         m = self.negotiateCapabilities(["sasl"], False)
         self.assertEqual(m, Message({}, None, "CAP", ["END"]))
 
-    @cases.OptionalityHelper.skipUnlessHasMechanism("SCRAM-SHA-256")
+    @cases.skipUnlessHasMechanism("SCRAM-SHA-256")
     def testScram(self):
         """Test SCRAM-SHA-256 authentication."""
         auth = authentication.Authentication(
@@ -221,7 +227,7 @@ class SaslTestCase(cases.BaseClientTestCase, cases.OptionalityHelper):
         self.assertEqual(m.command, "AUTHENTICATE", m)
         self.assertEqual(m.params, ["+"], m)
 
-    @cases.OptionalityHelper.skipUnlessHasMechanism("SCRAM-SHA-256")
+    @cases.skipUnlessHasMechanism("SCRAM-SHA-256")
     def testScramBadPassword(self):
         """Test SCRAM-SHA-256 authentication with a bad password."""
         auth = authentication.Authentication(
@@ -256,8 +262,8 @@ class SaslTestCase(cases.BaseClientTestCase, cases.OptionalityHelper):
             authenticator.response(msg)
 
 
-class Irc302SaslTestCase(cases.BaseClientTestCase, cases.OptionalityHelper):
-    @cases.OptionalityHelper.skipUnlessHasMechanism("PLAIN")
+class Irc302SaslTestCase(cases.BaseClientTestCase):
+    @cases.skipUnlessHasMechanism("PLAIN")
     def testPlainNotAvailable(self):
         """Test the client does not try to authenticate using a mechanism the
         server does not advertise.

irctest/client_tests/tls.py

@@ -1,3 +1,5 @@
+"""Clients should validate certificates; either with a CA or fingerprints."""
+
 import socket
 import ssl
 
@@ -138,7 +140,7 @@ class TlsTestCase(cases.BaseClientTestCase):
             self.getMessage()
 
 
-class StsTestCase(cases.BaseClientTestCase, cases.OptionalityHelper):
+class StsTestCase(cases.BaseClientTestCase):
     def setUp(self):
         super().setUp()
         self.insecure_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

irctest/controllers/anope_services.py

@@ -73,6 +73,8 @@ module {{ name = "ns_cert" }}
 class AnopeController(BaseServicesController, DirectoryBasedController):
     """Collaborator for server controllers that rely on Anope"""
 
+    software_name = "Anope"
+
     def run(self, protocol: str, server_hostname: str, server_port: int) -> None:
         self.create_config()
 

irctest/controllers/atheme_services.py

@@ -56,6 +56,8 @@ saslserv {{
 class AthemeController(BaseServicesController, DirectoryBasedController):
     """Mixin for server controllers that rely on Atheme"""
 
+    software_name = "Atheme"
+
     def run(self, protocol: str, server_hostname: str, server_port: int) -> None:
         self.create_config()
 

irctest/controllers/bahamut.py

@@ -102,6 +102,7 @@ class BahamutController(BaseServerController, DirectoryBasedController):
         valid_metadata_keys: Optional[Set[str]] = None,
         invalid_metadata_keys: Optional[Set[str]] = None,
         restricted_metadata_keys: Optional[Set[str]] = None,
+        faketime: Optional[str],
     ) -> None:
         if valid_metadata_keys or invalid_metadata_keys:
             raise NotImplementedByController(
@@ -136,15 +137,21 @@ class BahamutController(BaseServerController, DirectoryBasedController):
                     # pem_path=self.pem_path,
                 )
             )
+
+        if faketime and shutil.which("faketime"):
+            faketime_cmd = ["faketime", "-f", faketime]
+            self.faketime_enabled = True
+        else:
+            faketime_cmd = []
+
         self.proc = subprocess.Popen(
             [
-                # "strace", "-f", "-e", "file",
+                *faketime_cmd,
                 "ircd",
                 "-t",  # don't fork
                 "-f",
                 os.path.join(self.directory, "server.conf"),
             ],
-            # stdout=subprocess.DEVNULL,
         )
 
         if run_services:

irctest/controllers/base_hybrid.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import subprocess
 from typing import Optional, Set
 
@@ -43,6 +44,7 @@ class BaseHybridController(BaseServerController, DirectoryBasedController):
         run_services: bool,
         valid_metadata_keys: Optional[Set[str]] = None,
         invalid_metadata_keys: Optional[Set[str]] = None,
+        faketime: Optional[str],
     ) -> None:
         if valid_metadata_keys or invalid_metadata_keys:
             raise NotImplementedByController(
@@ -73,8 +75,16 @@ class BaseHybridController(BaseServerController, DirectoryBasedController):
                 )
             )
         assert self.directory
+
+        if faketime and shutil.which("faketime"):
+            faketime_cmd = ["faketime", "-f", faketime]
+            self.faketime_enabled = True
+        else:
+            faketime_cmd = []
+
         self.proc = subprocess.Popen(
             [
+                *faketime_cmd,
                 self.binary_name,
                 "-foreground",
                 "-configfile",

irctest/controllers/ergo.py

@@ -1,8 +1,9 @@
 import copy
 import json
 import os
+import shutil
 import subprocess
-from typing import Any, Dict, Optional, Set, Type, Union
+from typing import Any, Dict, List, Optional, Set, Type, Union
 
 from irctest.basecontrollers import (
     BaseServerController,
@@ -138,6 +139,7 @@ class ErgoController(BaseServerController, DirectoryBasedController):
     supported_sasl_mechanisms = {"PLAIN", "SCRAM-SHA-256"}
     supports_sts = True
     extban_mute_char = "m"
+    mysql_proc: Optional[subprocess.Popen] = None
 
     def create_config(self) -> None:
         super().create_config()
@@ -155,6 +157,7 @@ class ErgoController(BaseServerController, DirectoryBasedController):
         valid_metadata_keys: Optional[Set[str]] = None,
         invalid_metadata_keys: Optional[Set[str]] = None,
         restricted_metadata_keys: Optional[Set[str]] = None,
+        faketime: Optional[str],
         config: Optional[Any] = None,
     ) -> None:
         if valid_metadata_keys or invalid_metadata_keys:
@@ -171,7 +174,7 @@ class ErgoController(BaseServerController, DirectoryBasedController):
         enable_chathistory = self.test_config.chathistory
         enable_roleplay = self.test_config.ergo_roleplay
         if enable_chathistory or enable_roleplay:
-            config = self.addMysqlToConfig(config)
+            self.addDatabaseToConfig(config)
 
         if enable_roleplay:
             config["roleplay"] = {"enabled": True}
@@ -202,10 +205,27 @@ class ErgoController(BaseServerController, DirectoryBasedController):
         self._write_config()
         subprocess.call(["ergo", "initdb", "--conf", self._config_path, "--quiet"])
         subprocess.call(["ergo", "mkcerts", "--conf", self._config_path, "--quiet"])
+
+        if faketime and shutil.which("faketime"):
+            faketime_cmd = ["faketime", "-f", faketime]
+            self.faketime_enabled = True
+        else:
+            faketime_cmd = []
+
         self.proc = subprocess.Popen(
-            ["ergo", "run", "--conf", self._config_path, "--quiet"]
+            [*faketime_cmd, "ergo", "run", "--conf", self._config_path, "--quiet"]
         )
 
+    def terminate(self) -> None:
+        if self.mysql_proc is not None:
+            self.mysql_proc.terminate()
+        super().terminate()
+
+    def kill(self) -> None:
+        if self.mysql_proc is not None:
+            self.mysql_proc.kill()
+        super().kill()
+
     def wait_for_services(self) -> None:
         # Nothing to wait for, they start at the same time as Ergo.
         pass
@@ -257,32 +277,107 @@ class ErgoController(BaseServerController, DirectoryBasedController):
         config.update(LOGGING_CONFIG)
         return config
 
-    def addMysqlToConfig(self, config: Optional[Dict] = None) -> Dict:
-        mysql_password = os.getenv("MYSQL_PASSWORD")
-        if config is None:
-            config = self.baseConfig()
-        if not mysql_password:
-            return config
-        config["datastore"]["mysql"] = {
-            "enabled": True,
-            "host": "localhost",
-            "user": "ergo",
-            "password": mysql_password,
-            "history-database": "ergo_history",
-            "timeout": "3s",
-        }
-        config["accounts"]["multiclient"] = {
-            "enabled": True,
-            "allowed-by-default": True,
-            "always-on": "disabled",
-        }
-        config["history"]["persistent"] = {
-            "enabled": True,
-            "unregistered-channels": True,
-            "registered-channels": "opt-out",
-            "direct-messages": "opt-out",
-        }
-        return config
+    def addDatabaseToConfig(self, config: Dict) -> None:
+        history_backend = os.environ.get("ERGO_HISTORY_BACKEND", "memory")
+        if history_backend == "memory":
+            # nothing to do, this is the default
+            pass
+        elif history_backend == "mysql":
+            socket_path = self.startMysql()
+            self.createMysqlDatabase(socket_path, "ergo_history")
+            config["datastore"]["mysql"] = {
+                "enabled": True,
+                "socket-path": socket_path,
+                "history-database": "ergo_history",
+                "timeout": "3s",
+            }
+            config["history"]["persistent"] = {
+                "enabled": True,
+                "unregistered-channels": True,
+                "registered-channels": "opt-out",
+                "direct-messages": "opt-out",
+            }
+        else:
+            raise ValueError(
+                f"Invalid $ERGO_HISTORY_BACKEND value: {history_backend}. "
+                f"It should be 'memory' (the default) or 'mysql'"
+            )
+
+    def startMysql(self) -> str:
+        """Starts a new MySQL server listening on a UNIX socket, returns the socket
+        path"""
+        # Function based on pifpaf's MySQL driver:
+        # https://github.com/jd/pifpaf/blob/3.1.5/pifpaf/drivers/mysql.py
+        assert self.directory
+        mysql_dir = os.path.join(self.directory, "mysql")
+        socket_path = os.path.join(mysql_dir, "mysql.socket")
+        os.mkdir(mysql_dir)
+
+        print("Starting MySQL...")
+        try:
+            subprocess.check_call(
+                [
+                    "mysqld",
+                    "--no-defaults",
+                    "--tmpdir=" + mysql_dir,
+                    "--initialize-insecure",
+                    "--datadir=" + mysql_dir,
+                ],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+        except subprocess.CalledProcessError:
+            # Initialize the old way
+            subprocess.check_call(
+                [
+                    "mysql_install_db",
+                    "--no-defaults",
+                    "--tmpdir=" + mysql_dir,
+                    "--datadir=" + mysql_dir,
+                ],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+        self.mysql_proc = subprocess.Popen(
+            [
+                "mysqld",
+                "--no-defaults",
+                "--tmpdir=" + mysql_dir,
+                "--datadir=" + mysql_dir,
+                "--socket=" + socket_path,
+                "--skip-networking",
+                "--skip-grant-tables",
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+        )
+
+        mysql_stdout = self.mysql_proc.stdout
+        assert mysql_stdout is not None  # for mypy...
+        lines: List[bytes] = []
+        while self.mysql_proc.returncode is None:
+            line = mysql_stdout.readline()
+            lines.append(lines)
+            if b"mysqld: ready for connections." in line:
+                break
+        assert self.mysql_proc.returncode is None, (
+            "MySQL unexpected stopped: " + b"\n".join(lines).decode()
+        )
+        print("MySQL started")
+
+        return socket_path
+
+    def createMysqlDatabase(self, socket_path: str, database_name: str) -> None:
+        subprocess.check_call(
+            [
+                "mysql",
+                "--no-defaults",
+                "-S",
+                socket_path,
+                "-e",
+                f"CREATE DATABASE {database_name};",
+            ]
+        )
 
     def rehash(self, case: BaseServerTestCase, config: Dict) -> None:
         self._config = config

irctest/controllers/external_server.py

@@ -42,6 +42,7 @@ class ExternalServerController(BaseServerController):
         valid_metadata_keys: Optional[Set[str]] = None,
         invalid_metadata_keys: Optional[Set[str]] = None,
         restricted_metadata_keys: Optional[Set[str]] = None,
+        faketime: Optional[str],
     ) -> None:
         pass
 

irctest/controllers/inspircd.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import subprocess
 from typing import Optional, Set, Type
 
@@ -59,8 +60,10 @@ TEMPLATE_CONFIG = """
       target="services.example.org">
 
 # Protocol:
+<module name="banexception">
 <module name="botmode">
 <module name="cap">
+<module name="inviteexception">
 <module name="ircv3">
 <module name="ircv3_accounttag">
 <module name="ircv3_batch">
@@ -74,7 +77,6 @@ TEMPLATE_CONFIG = """
 <module name="monitor">
 <module name="m_muteban">  # for testing mute extbans
 <module name="namesx"> # For multi-prefix
-<module name="noctcp">
 <module name="sasl">
 
 # HELP/HELPOP
@@ -84,7 +86,7 @@ TEMPLATE_CONFIG = """
 
 # Misc:
 <log method="file" type="*" level="debug" target="/tmp/ircd-{port}.log">
-<server name="My.Little.Server" description="testnet" id="000" network="testnet">
+<server name="My.Little.Server" description="test server" id="000" network="testnet">
 """
 
 TEMPLATE_SSL_CONFIG = """
@@ -115,6 +117,7 @@ class InspircdController(BaseServerController, DirectoryBasedController):
         valid_metadata_keys: Optional[Set[str]] = None,
         invalid_metadata_keys: Optional[Set[str]] = None,
         restricted_metadata_keys: Optional[Set[str]] = None,
+        faketime: Optional[str] = None,
     ) -> None:
         if valid_metadata_keys or invalid_metadata_keys:
             raise NotImplementedByController(
@@ -148,8 +151,16 @@ class InspircdController(BaseServerController, DirectoryBasedController):
                 )
             )
         assert self.directory
+
+        if faketime and shutil.which("faketime"):
+            faketime_cmd = ["faketime", "-f", faketime]
+            self.faketime_enabled = True
+        else:
+            faketime_cmd = []
+
         self.proc = subprocess.Popen(
             [
+                *faketime_cmd,
                 "inspircd",
                 "--nofork",
                 "--config",

irctest/controllers/irc2.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import subprocess
 from typing import Optional, Set, Type
 
@@ -10,7 +11,7 @@ from irctest.basecontrollers import (
 
 TEMPLATE_CONFIG = """
 # M:<Server NAME>:<YOUR Internet IP#>:<Geographic Location>:<Port>:<SID>:
-M:My.Little.Server:{hostname}:Somewhere:{port}:0042:
+M:My.Little.Server:{hostname}:test server:{port}:0042:
 
 # A:<Your Name/Location>:<Your E-Mail Addr>:<other info>::<network name>:
 A:Organization, IRC dept.:Daemon <ircd@example.irc.org>:Client Server::IRCnet:
@@ -29,8 +30,8 @@ O:*:operpassword:operuser::::
 """
 
 
-class Ircu2Controller(BaseServerController, DirectoryBasedController):
-    binary_name: str
+class Irc2Controller(BaseServerController, DirectoryBasedController):
+    software_name = "irc2"
     services_protocol: str
 
     supports_sts = False
@@ -51,6 +52,7 @@ class Ircu2Controller(BaseServerController, DirectoryBasedController):
         run_services: bool,
         valid_metadata_keys: Optional[Set[str]] = None,
         invalid_metadata_keys: Optional[Set[str]] = None,
+        faketime: Optional[str],
     ) -> None:
         if valid_metadata_keys or invalid_metadata_keys:
             raise NotImplementedByController(
@@ -76,8 +78,16 @@ class Ircu2Controller(BaseServerController, DirectoryBasedController):
                     pidfile=pidfile,
                 )
             )
+
+        if faketime and shutil.which("faketime"):
+            faketime_cmd = ["faketime", "-f", faketime]
+            self.faketime_enabled = True
+        else:
+            faketime_cmd = []
+
         self.proc = subprocess.Popen(
             [
+                *faketime_cmd,
                 "ircd",
                 "-s",  # no iauth
                 "-p",
@@ -89,5 +99,5 @@ class Ircu2Controller(BaseServerController, DirectoryBasedController):
         )
 
 
-def get_irctest_controller_class() -> Type[Ircu2Controller]:
-    return Ircu2Controller
+def get_irctest_controller_class() -> Type[Irc2Controller]:
+    return Irc2Controller

irctest/controllers/ircu2.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import subprocess
 from typing import Optional, Set, Type
 
@@ -51,6 +52,7 @@ features {{
 
 
 class Ircu2Controller(BaseServerController, DirectoryBasedController):
+    software_name = "ircu2"
     supports_sts = False
     extban_mute_char = None
 
@@ -69,6 +71,7 @@ class Ircu2Controller(BaseServerController, DirectoryBasedController):
         run_services: bool,
         valid_metadata_keys: Optional[Set[str]] = None,
         invalid_metadata_keys: Optional[Set[str]] = None,
+        faketime: Optional[str],
     ) -> None:
         if valid_metadata_keys or invalid_metadata_keys:
             raise NotImplementedByController(
@@ -94,8 +97,16 @@ class Ircu2Controller(BaseServerController, DirectoryBasedController):
                     pidfile=pidfile,
                 )
             )
+
+        if faketime and shutil.which("faketime"):
+            faketime_cmd = ["faketime", "-f", faketime]
+            self.faketime_enabled = True
+        else:
+            faketime_cmd = []
+
         self.proc = subprocess.Popen(
             [
+                *faketime_cmd,
                 "ircd",
                 "-n",  # don't detach
                 "-f",

irctest/controllers/mammon.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import subprocess
 from typing import Optional, Set, Type
 
@@ -92,6 +93,7 @@ class MammonController(BaseServerController, DirectoryBasedController):
         valid_metadata_keys: Optional[Set[str]] = None,
         invalid_metadata_keys: Optional[Set[str]] = None,
         restricted_metadata_keys: Optional[Set[str]] = None,
+        faketime: Optional[str],
     ) -> None:
         if password is not None:
             raise NotImplementedByController("PASS command")
@@ -113,8 +115,16 @@ class MammonController(BaseServerController, DirectoryBasedController):
         # with self.open_file('server.yml', 'r') as fd:
         #    print(fd.read())
         assert self.directory
+
+        if faketime and shutil.which("faketime"):
+            faketime_cmd = ["faketime", "-f", faketime]
+            self.faketime_enabled = True
+        else:
+            faketime_cmd = []
+
         self.proc = subprocess.Popen(
             [
+                *faketime_cmd,
                 "mammond",
                 "--nofork",  # '--debug',
                 "--config",

irctest/controllers/nefarious.py

@@ -0,0 +1,11 @@
+from typing import Type
+
+from .ircu2 import Ircu2Controller
+
+
+class NefariousController(Ircu2Controller):
+    software_name = "Nefarious"
+
+
+def get_irctest_controller_class() -> Type[NefariousController]:
+    return NefariousController

irctest/controllers/ngircd.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import subprocess
 from typing import Optional, Set, Type
 
@@ -12,7 +13,7 @@ from irctest.irc_utils.junkdrawer import find_hostname_and_port
 TEMPLATE_CONFIG = """
 [Global]
     Name = My.Little.Server
-    Info = ExampleNET Server
+    Info = test server
     Bind = {hostname}
     Ports = {port}
     AdminInfo1 = Bob Smith
@@ -56,6 +57,7 @@ class NgircdController(BaseServerController, DirectoryBasedController):
         valid_metadata_keys: Optional[Set[str]] = None,
         invalid_metadata_keys: Optional[Set[str]] = None,
         restricted_metadata_keys: Optional[Set[str]] = None,
+        faketime: Optional[str],
     ) -> None:
         if valid_metadata_keys or invalid_metadata_keys:
             raise NotImplementedByController(
@@ -81,6 +83,7 @@ class NgircdController(BaseServerController, DirectoryBasedController):
             fd.write("\n")
 
         assert self.directory
+
         with self.open_file("server.conf") as fd:
             fd.write(
                 TEMPLATE_CONFIG.format(
@@ -94,8 +97,16 @@ class NgircdController(BaseServerController, DirectoryBasedController):
                     empty_file=os.path.join(self.directory, "empty.txt"),
                 )
             )
+
+        if faketime and shutil.which("faketime"):
+            faketime_cmd = ["faketime", "-f", faketime]
+            self.faketime_enabled = True
+        else:
+            faketime_cmd = []
+
         self.proc = subprocess.Popen(
             [
+                *faketime_cmd,
                 "ngircd",
                 "--nodaemon",
                 "--config",

irctest/controllers/plexus4.py

@@ -74,7 +74,7 @@ operator {{
 
 
 class Plexus4Controller(BaseHybridController):
-    software_name = "Hybrid"
+    software_name = "Plexus4"
     binary_name = "ircd"
     services_protocol = "plexus"
 

irctest/controllers/snircd.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import subprocess
 from typing import Optional, Set, Type
 
@@ -69,6 +70,7 @@ class SnircdController(BaseServerController, DirectoryBasedController):
         run_services: bool,
         valid_metadata_keys: Optional[Set[str]] = None,
         invalid_metadata_keys: Optional[Set[str]] = None,
+        faketime: Optional[str],
     ) -> None:
         if valid_metadata_keys or invalid_metadata_keys:
             raise NotImplementedByController(
@@ -94,8 +96,16 @@ class SnircdController(BaseServerController, DirectoryBasedController):
                     pidfile=pidfile,
                 )
             )
+
+        if faketime and shutil.which("faketime"):
+            faketime_cmd = ["faketime", "-f", faketime]
+            self.faketime_enabled = True
+        else:
+            faketime_cmd = []
+
         self.proc = subprocess.Popen(
             [
+                *faketime_cmd,
                 "ircd",
                 "-n",  # don't detach
                 "-f",

irctest/controllers/unrealircd.py

@@ -22,7 +22,7 @@ include "help/help.conf";
 
 me {{
     name "My.Little.Server";
-    info "ExampleNET Server";
+    info "test server";
     sid "001";
 }}
 admin {{
@@ -100,6 +100,9 @@ set {{
         }}
     }}
     modes-on-join "+H 100:1d";  // Enables CHATHISTORY
+
+    {set_extras}
+
 }}
 
 tld {{
@@ -139,6 +142,7 @@ class UnrealircdController(BaseServerController, DirectoryBasedController):
     supports_sts = False
 
     extban_mute_char = "quiet" if installed_version() >= 6 else "q"
+    software_version = installed_version()
 
     def create_config(self) -> None:
         super().create_config()
@@ -156,6 +160,7 @@ class UnrealircdController(BaseServerController, DirectoryBasedController):
         valid_metadata_keys: Optional[Set[str]] = None,
         invalid_metadata_keys: Optional[Set[str]] = None,
         restricted_metadata_keys: Optional[Set[str]] = None,
+        faketime: Optional[str],
     ) -> None:
         if valid_metadata_keys or invalid_metadata_keys:
             raise NotImplementedByController(
@@ -185,13 +190,26 @@ class UnrealircdController(BaseServerController, DirectoryBasedController):
                 loadmodule "cloak_md5";
                 """
             )
+            set_extras = textwrap.indent(
+                textwrap.dedent(
+                    """
+                    // Remove RPL_WHOISSPECIAL used to advertise security groups
+                    whois-details {
+                        security-groups { everyone none; self none; oper none; }
+                    }
+                    """
+                ),
+                "    ",
+            )
         else:
             extras = ""
+            set_extras = ""
 
         with self.open_file("empty.txt") as fd:
             fd.write("\n")
 
         assert self.directory
+
         with self.open_file("unrealircd.conf") as fd:
             fd.write(
                 TEMPLATE_CONFIG.format(
@@ -206,6 +224,7 @@ class UnrealircdController(BaseServerController, DirectoryBasedController):
                     pem_path=self.pem_path,
                     empty_file=os.path.join(self.directory, "empty.txt"),
                     extras=extras,
+                    set_extras=set_extras,
                 )
             )
 
@@ -225,9 +244,16 @@ class UnrealircdController(BaseServerController, DirectoryBasedController):
                 proot_cmd = ["proot", "-b", f"{tmpdir}:{unrealircd_prefix}/tmp"]
                 self.using_proot = True
 
+        if faketime and shutil.which("faketime"):
+            faketime_cmd = ["faketime", "-f", faketime]
+            self.faketime_enabled = True
+        else:
+            faketime_cmd = []
+
         self.proc = subprocess.Popen(
             [
                 *proot_cmd,
+                *faketime_cmd,
                 "unrealircd",
                 "-t",
                 "-F",  # BOOT_NOFORK

irctest/dashboard/format.py

@@ -0,0 +1,438 @@
+import base64
+import dataclasses
+import gzip
+import hashlib
+import importlib
+from pathlib import Path
+import re
+import sys
+from typing import (
+    IO,
+    Callable,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
+    TypeVar,
+)
+import xml.etree.ElementTree as ET
+
+from defusedxml.ElementTree import parse as parse_xml
+import docutils.core
+
+NETLIFY_CHAR_BLACKLIST = frozenset('":<>|*?\r\n#')
+"""Characters not allowed in output filenames"""
+
+
+@dataclasses.dataclass
+class CaseResult:
+    module_name: str
+    class_name: str
+    test_name: str
+    job: str
+    success: bool
+    skipped: bool
+    system_out: Optional[str]
+    details: Optional[str] = None
+    type: Optional[str] = None
+    message: Optional[str] = None
+
+    def output_filename(self):
+        test_name = self.test_name
+        if len(test_name) > 50 or set(test_name) & NETLIFY_CHAR_BLACKLIST:
+            # File name too long or otherwise invalid. This should be good enough:
+            m = re.match(r"(?P<function_name>\w+?)\[(?P<params>.+)\]", test_name)
+            assert m, "File name is too long but has no parameter."
+            test_name = f'{m.group("function_name")}[{md5sum(m.group("params"))}]'
+        return f"{self.job}_{self.module_name}.{self.class_name}.{test_name}.txt"
+
+
+TK = TypeVar("TK")
+TV = TypeVar("TV")
+
+
+def md5sum(text: str) -> str:
+    return base64.urlsafe_b64encode(hashlib.md5(text.encode()).digest()).decode()
+
+
+def group_by(list_: Iterable[TV], key: Callable[[TV], TK]) -> Dict[TK, List[TV]]:
+    groups: Dict[TK, List[TV]] = {}
+    for value in list_:
+        groups.setdefault(key(value), []).append(value)
+
+    return groups
+
+
+def iter_job_results(job_file_name: Path, job: ET.ElementTree) -> Iterator[CaseResult]:
+    (suite,) = job.getroot()
+    for case in suite:
+        if "name" not in case.attrib:
+            continue
+
+        success = True
+        skipped = False
+        details = None
+        system_out = None
+        extra = {}
+        for child in case:
+            if child.tag == "skipped":
+                success = True
+                skipped = True
+                details = None
+                extra = child.attrib
+            elif child.tag in ("failure", "error"):
+                success = False
+                skipped = False
+                details = child.text
+                extra = child.attrib
+            elif child.tag == "system-out":
+                assert (
+                    system_out is None
+                    # for some reason, skipped tests have two system-out;
+                    # and the second one contains test teardown
+                    or child.text.startswith(system_out.rstrip())
+                ), ("Duplicate system-out tag", repr(system_out), repr(child.text))
+                system_out = child.text
+            else:
+                assert False, child
+
+        (module_name, class_name) = case.attrib["classname"].rsplit(".", 1)
+        m = re.match(
+            r"(.*/)?pytest[ -]results[ _](?P<name>.*)"
+            r"[ _][(]?(stable|release|devel|devel_release)[)]?/pytest.xml(.gz)?",
+            str(job_file_name),
+        )
+        assert m, job_file_name
+        yield CaseResult(
+            module_name=module_name,
+            class_name=class_name,
+            test_name=case.attrib["name"],
+            job=m.group("name"),
+            success=success,
+            skipped=skipped,
+            details=details,
+            system_out=system_out,
+            **extra,
+        )
+
+
+def rst_to_element(s: str) -> ET.Element:
+    html = docutils.core.publish_parts(s, writer_name="xhtml")["html_body"]
+    htmltree = ET.fromstring(html)
+    return htmltree
+
+
+def append_docstring(element: ET.Element, obj: object) -> None:
+    if obj.__doc__ is None:
+        return
+
+    element.append(rst_to_element(obj.__doc__))
+
+
+def build_job_html(job: str, results: List[CaseResult]) -> ET.Element:
+    jobs = sorted({result.job for result in results})
+    root = ET.Element("html")
+    head = ET.SubElement(root, "head")
+    ET.SubElement(head, "title").text = job
+    ET.SubElement(head, "link", rel="stylesheet", type="text/css", href="./style.css")
+
+    body = ET.SubElement(root, "body")
+
+    ET.SubElement(body, "h1").text = job
+
+    table = build_test_table(jobs, results)
+    table.set("class", "job-results test-matrix")
+    body.append(table)
+
+    return root
+
+
+def build_module_html(
+    jobs: List[str], results: List[CaseResult], module_name: str
+) -> ET.Element:
+    module = importlib.import_module(module_name)
+
+    root = ET.Element("html")
+    head = ET.SubElement(root, "head")
+    ET.SubElement(head, "title").text = module_name
+    ET.SubElement(head, "link", rel="stylesheet", type="text/css", href="./style.css")
+
+    body = ET.SubElement(root, "body")
+
+    ET.SubElement(body, "h1").text = module_name
+
+    append_docstring(body, module)
+
+    table = build_test_table(jobs, results)
+    table.set("class", "module-results test-matrix")
+    body.append(table)
+
+    return root
+
+
+def build_test_table(jobs: List[str], results: List[CaseResult]) -> ET.Element:
+    results_by_module_and_class = group_by(
+        results, lambda r: (r.module_name, r.class_name)
+    )
+
+    table = ET.Element("table")
+
+    job_row = ET.Element("tr")
+    ET.SubElement(job_row, "th")  # column of case name
+    for job in jobs:
+        cell = ET.SubElement(job_row, "th")
+        ET.SubElement(ET.SubElement(cell, "div"), "span").text = job
+        cell.set("class", "job-name")
+
+    for ((module_name, class_name), class_results) in sorted(
+        results_by_module_and_class.items()
+    ):
+        module = importlib.import_module(module_name)
+
+        # Header row: class name
+        header_row = ET.SubElement(table, "tr")
+        th = ET.SubElement(header_row, "th", colspan=str(len(jobs) + 1))
+        row_anchor = f"{class_name}"
+        section_header = ET.SubElement(
+            ET.SubElement(th, "h2"),
+            "a",
+            href=f"#{row_anchor}",
+            id=row_anchor,
+        )
+        section_header.text = class_name
+        append_docstring(th, getattr(module, class_name))
+
+        # Header row: one column for each implementation
+        table.append(job_row)
+
+        # One row for each test:
+        results_by_test = group_by(class_results, key=lambda r: r.test_name)
+        for (test_name, test_results) in sorted(results_by_test.items()):
+            row_anchor = f"{class_name}.{test_name}"
+            if len(row_anchor) >= 50:
+                # Too long; give up on generating readable URL
+                # TODO: only hash test parameter
+                row_anchor = md5sum(row_anchor)
+
+            row = ET.SubElement(table, "tr", id=row_anchor)
+
+            cell = ET.SubElement(row, "th")
+            cell.set("class", "test-name")
+            cell_link = ET.SubElement(cell, "a", href=f"#{row_anchor}")
+            cell_link.text = test_name
+
+            results_by_job = group_by(test_results, key=lambda r: r.job)
+            for job_name in jobs:
+                cell = ET.SubElement(row, "td")
+                try:
+                    (result,) = results_by_job[job_name]
+                except KeyError:
+                    cell.set("class", "deselected")
+                    cell.text = "d"
+                    continue
+
+                text: Optional[str]
+
+                if result.skipped:
+                    cell.set("class", "skipped")
+                    if result.type == "pytest.skip":
+                        text = "s"
+                    elif result.type == "pytest.xfail":
+                        text = "X"
+                        cell.set("class", "expected-failure")
+                    else:
+                        text = result.type
+                elif result.success:
+                    cell.set("class", "success")
+                    if result.type:
+                        # dead code?
+                        text = result.type
+                    else:
+                        text = "."
+                else:
+                    cell.set("class", "failure")
+                    if result.type:
+                        # dead code?
+                        text = result.type
+                    else:
+                        text = "f"
+
+                if result.system_out:
+                    # There is a log file; link to it.
+                    a = ET.SubElement(cell, "a", href=f"./{result.output_filename()}")
+                    a.text = text or "?"
+                else:
+                    cell.text = text or "?"
+                if result.message:
+                    cell.set("title", result.message)
+
+    return table
+
+
+def write_html_pages(
+    output_dir: Path, results: List[CaseResult]
+) -> List[Tuple[str, str, str]]:
+    """Returns the list of (module_name, file_name)."""
+    output_dir.mkdir(parents=True, exist_ok=True)
+    results_by_module = group_by(results, lambda r: r.module_name)
+
+    # used as columns
+    jobs = list(sorted({r.job for r in results}))
+
+    job_categories = {}
+    for job in jobs:
+        is_client = any(
+            "client_tests" in result.module_name and result.job == job
+            for result in results
+        )
+        is_server = any(
+            "server_tests" in result.module_name and result.job == job
+            for result in results
+        )
+        assert is_client != is_server, (job, is_client, is_server)
+        if job.endswith(("-atheme", "-anope")):
+            assert is_server
+            job_categories[job] = "server-with-services"
+        elif is_server:
+            job_categories[job] = "server"  # with or without services
+        else:
+            assert is_client
+            job_categories[job] = "client"
+
+    pages = []
+
+    for (module_name, module_results) in sorted(results_by_module.items()):
+        # Filter out client jobs if this is a server test module, and vice versa
+        module_categories = {
+            job_categories[result.job]
+            for result in results
+            if result.module_name == module_name and not result.skipped
+        }
+
+        module_jobs = [job for job in jobs if job_categories[job] in module_categories]
+
+        root = build_module_html(module_jobs, module_results, module_name)
+        file_name = f"{module_name}.xhtml"
+        write_xml_file(output_dir / file_name, root)
+        pages.append(("module", module_name, file_name))
+
+    for category in ("server", "client"):
+        for job in [job for job in job_categories if job_categories[job] == category]:
+            job_results = [
+                result
+                for result in results
+                if result.job == job or result.job.startswith(job + "-")
+            ]
+            root = build_job_html(job, job_results)
+            file_name = f"{job}.xhtml"
+            write_xml_file(output_dir / file_name, root)
+            pages.append(("job", job, file_name))
+
+    return pages
+
+
+def write_test_outputs(output_dir: Path, results: List[CaseResult]) -> None:
+    """Writes stdout files of each test."""
+    for result in results:
+        if result.system_out is None:
+            continue
+        output_file = output_dir / result.output_filename()
+        with output_file.open("wt") as fd:
+            fd.write(result.system_out)
+
+
+def write_html_index(output_dir: Path, pages: List[Tuple[str, str, str]]) -> None:
+    root = ET.Element("html")
+    head = ET.SubElement(root, "head")
+    ET.SubElement(head, "title").text = "irctest dashboard"
+    ET.SubElement(head, "link", rel="stylesheet", type="text/css", href="./style.css")
+
+    body = ET.SubElement(root, "body")
+
+    ET.SubElement(body, "h1").text = "irctest dashboard"
+
+    module_pages = []
+    job_pages = []
+    for (page_type, title, file_name) in sorted(pages):
+        if page_type == "module":
+            module_pages.append((title, file_name))
+        elif page_type == "job":
+            job_pages.append((title, file_name))
+        else:
+            assert False, page_type
+
+    ET.SubElement(body, "h2").text = "Tests by command/specification"
+
+    dl = ET.SubElement(body, "dl")
+    dl.set("class", "module-index")
+
+    for (module_name, file_name) in sorted(module_pages):
+        module = importlib.import_module(module_name)
+
+        link = ET.SubElement(ET.SubElement(dl, "dt"), "a", href=f"./{file_name}")
+        link.text = module_name
+        append_docstring(ET.SubElement(dl, "dd"), module)
+
+    ET.SubElement(body, "h2").text = "Tests by implementation"
+
+    ul = ET.SubElement(body, "ul")
+    ul.set("class", "job-index")
+
+    for (job, file_name) in sorted(job_pages):
+        link = ET.SubElement(ET.SubElement(ul, "li"), "a", href=f"./{file_name}")
+        link.text = job
+
+    write_xml_file(output_dir / "index.xhtml", root)
+
+
+def write_assets(output_dir: Path) -> None:
+    css_path = output_dir / "style.css"
+    source_css_path = Path(__file__).parent / "style.css"
+    with css_path.open("wt") as fd:
+        with source_css_path.open() as source_fd:
+            fd.write(source_fd.read())
+
+
+def write_xml_file(filename: Path, root: ET.Element) -> None:
+    # Hacky: ET expects the namespace to be present in every tag we create instead;
+    # but it would be excessively verbose.
+    root.set("xmlns", "http://www.w3.org/1999/xhtml")
+
+    # Serialize
+    s = ET.tostring(root)
+
+    with filename.open("wb") as fd:
+        fd.write(s)
+
+
+def parse_xml_file(filename: Path) -> ET.ElementTree:
+    fd: IO
+    if filename.suffix == ".gz":
+        with gzip.open(filename, "rb") as fd:  # type: ignore
+            return parse_xml(fd)  # type: ignore
+    else:
+        with open(filename) as fd:
+            return parse_xml(fd)  # type: ignore
+
+
+def main(output_path: Path, filenames: List[Path]) -> int:
+    results = [
+        result
+        for filename in filenames
+        for result in iter_job_results(filename, parse_xml_file(filename))
+    ]
+
+    pages = write_html_pages(output_path, results)
+
+    write_html_index(output_path, pages)
+    write_test_outputs(output_path, results)
+    write_assets(output_path)
+
+    return 0
+
+
+if __name__ == "__main__":
+    (_, output_path, *filenames) = sys.argv
+    exit(main(Path(output_path), list(map(Path, filenames))))

irctest/dashboard/github_download.py

@@ -0,0 +1,87 @@
+import dataclasses
+import gzip
+import io
+import json
+from pathlib import Path
+import sys
+from typing import Iterator
+import urllib.parse
+import urllib.request
+import zipfile
+
+
+@dataclasses.dataclass
+class Artifact:
+    repo: str
+    run_id: int
+    name: str
+    download_url: str
+
+    @property
+    def public_download_url(self):
+        # GitHub API is not available publicly for artifacts, we need to use
+        # a third-party proxy to access it...
+        name = urllib.parse.quote(self.name)
+        return f"https://nightly.link/{repo}/actions/runs/{self.run_id}/{name}.zip"
+
+
+def iter_run_artifacts(repo: str, run_id: int) -> Iterator[Artifact]:
+    request = urllib.request.Request(
+        f"https://api.github.com/repos/{repo}/actions/runs/{run_id}/artifacts"
+        "?per_page=100",
+        headers={"Accept": "application/vnd.github.v3+json"},
+    )
+
+    response = urllib.request.urlopen(request)
+
+    for artifact in json.load(response)["artifacts"]:
+        if not artifact["name"].startswith(("pytest-results_", "pytest results ")):
+            continue
+        if artifact["expired"]:
+            continue
+        yield Artifact(
+            repo=repo,
+            run_id=run_id,
+            name=artifact["name"],
+            download_url=artifact["archive_download_url"],
+        )
+
+
+def download_artifact(output_name: Path, url: str) -> None:
+    if output_name.exists():
+        return
+    response = urllib.request.urlopen(url)
+    archive_bytes = response.read()  # Can't stream it, it's a ZIP
+    with zipfile.ZipFile(io.BytesIO(archive_bytes)) as archive:
+        with archive.open("pytest.xml") as input_fd:
+            pytest_xml = input_fd.read()
+
+    tmp_output_path = output_name.with_suffix(".tmp")
+    with gzip.open(tmp_output_path, "wb") as output_fd:
+        output_fd.write(pytest_xml)
+
+    # Atomically write to the output path, so that we don't write partial files in case
+    # the download process is interrupted
+    tmp_output_path.rename(output_name)
+
+
+def main(output_dir: Path, repo: str, run_id: int) -> int:
+    output_dir.mkdir(parents=True, exist_ok=True)
+    run_path = output_dir / str(run_id)
+    run_path.mkdir(exist_ok=True)
+
+    for artifact in iter_run_artifacts(repo, run_id):
+        artifact_path = run_path / artifact.name / "pytest.xml.gz"
+        artifact_path.parent.mkdir(exist_ok=True)
+        try:
+            download_artifact(artifact_path, artifact.download_url)
+        except Exception:
+            download_artifact(artifact_path, artifact.public_download_url)
+        print("downloaded", artifact.name)
+
+    return 0
+
+
+if __name__ == "__main__":
+    (_, output_path, repo, run_id) = sys.argv
+    exit(main(Path(output_path), repo, int(run_id)))

irctest/dashboard/style.css

@@ -0,0 +1,67 @@
+@media (prefers-color-scheme: dark) {
+    body {
+        background-color: #121212;
+        color: rgba(255, 255, 255, 0.87);
+    }
+    a {
+        filter: invert(0.85) hue-rotate(180deg);
+    }
+}
+
+dl.module-index {
+    column-width: 40em; /* Magic constant for 2 columns on average laptop/desktop */
+}
+
+/* Only 1px solid border between cells */
+table.test-matrix {
+    border-spacing: 0;
+    border-collapse: collapse;
+}
+table.test-matrix td {
+    text-align: center;
+    border: 1px solid grey;
+}
+
+/* Make link take the whole cell */
+table.test-matrix td a {
+    display: block;
+    margin: 0;
+    padding: 0;
+    width: 100%;
+    height: 100%;
+    color: black;
+    text-decoration: none;
+}
+
+/* Test matrix colors */
+table.test-matrix .deselected {
+    background-color: grey;
+}
+table.test-matrix .success {
+    background-color: green;
+}
+table.test-matrix .skipped {
+    background-color: yellow;
+}
+table.test-matrix .failure {
+    background-color: red;
+}
+table.test-matrix .expected-failure {
+    background-color: orange;
+}
+
+/* Rotate headers, thanks to https://css-tricks.com/rotated-table-column-headers/ */
+table.module-results th.job-name {
+  height: 140px;
+  white-space: nowrap;
+}
+table.module-results th.job-name > div {
+  transform:
+    translate(28px, 50px)
+    rotate(315deg);
+  width: 40px;
+}
+table.module-results th.job-name > div > span {
+  border-bottom: 1px solid grey;
+  padding-left: 0px;
+}

irctest/numerics.py

@@ -66,6 +66,7 @@ RPL_WHOISIDLE = "317"
 RPL_ENDOFWHOIS = "318"
 RPL_WHOISCHANNELS = "319"
 RPL_WHOISSPECIAL = "320"
+RPL_LISTSTART = "321"
 RPL_LIST = "322"
 RPL_LISTEND = "323"
 RPL_CHANNELMODEIS = "324"
@@ -141,6 +142,7 @@ ERR_USERONCHANNEL = "443"
 ERR_NOLOGIN = "444"
 ERR_SUMMONDISABLED = "445"
 ERR_USERSDISABLED = "446"
+ERR_FORBIDDENCHANNEL = "448"
 ERR_NOTREGISTERED = "451"
 ERR_NEEDMOREPARAMS = "461"
 ERR_ALREADYREGISTRED = "462"

irctest/patma.py

@@ -155,13 +155,15 @@ def match_dict(
     for (expected_key, expected_value) in expected.items():
         if isinstance(expected_key, RemainingKeys):
             remaining_keys_wildcard = (expected_key.key, expected_value)
-        elif isinstance(expected_key, Operator):
-            raise NotImplementedError(f"Unsupported operator: {expected_key}")
         else:
-            if expected_key not in got:
-                return False
-            got_value = got.pop(expected_key)
-            if not match_string(got_value, expected_value):
+            for key in got:
+                if match_string(key, expected_key) and match_string(
+                    got[key], expected_value
+                ):
+                    got.pop(key)
+                    break
+            else:
+                # Found no (key, value) pair matching the request
                 return False
 
     if remaining_keys_wildcard:

irctest/self_tests/cases.py

@@ -1,3 +1,5 @@
+"""Internal checks of assertion implementations."""
+
 from typing import Dict, List, Tuple
 
 import pytest
@@ -177,6 +179,39 @@ MESSAGE_SPECS: List[Tuple[Dict, List[str], List[str], List[str]]] = [
             "expected tags to match {'tag1': 'bar', RemainingKeys(ANYSTR): ANYOPTSTR}, got {}",
         ]
     ),
+    (
+        # the specification:
+        dict(
+            tags={StrRe("tag[12]"): "bar", **ANYDICT},
+            command="PRIVMSG",
+            params=["#chan", "hello"],
+        ),
+        # matches:
+        [
+            "@tag1=bar PRIVMSG #chan :hello",
+            "@tag1=bar;tag2= PRIVMSG #chan :hello",
+            "@tag1=bar :foo!baz@qux PRIVMSG #chan :hello",
+            "@tag2=bar PRIVMSG #chan :hello",
+            "@tag1=bar;tag2= PRIVMSG #chan :hello",
+            "@tag1=;tag2=bar PRIVMSG #chan :hello",
+        ],
+        # and does not match:
+        [
+            "PRIVMG #chan :hello",
+            "@tag1=value1 PRIVMSG #chan :hello",
+            "PRIVMSG #chan hello2",
+            "PRIVMSG #chan2 hello",
+            ":foo!baz@qux PRIVMSG #chan hello",
+        ],
+        # and they each error with:
+        [
+            "expected command to be PRIVMSG, got PRIVMG",
+            "expected tags to match {StrRe(r'tag[12]'): 'bar', RemainingKeys(ANYSTR): ANYOPTSTR}, got {'tag1': 'value1'}",
+            "expected params to match ['#chan', 'hello'], got ['#chan', 'hello2']",
+            "expected params to match ['#chan', 'hello'], got ['#chan2', 'hello']",
+            "expected tags to match {StrRe(r'tag[12]'): 'bar', RemainingKeys(ANYSTR): ANYOPTSTR}, got {}",
+        ]
+    ),
     (
         # the specification:
         dict(

irctest/server_tests/account_registration.py

@@ -1,3 +1,8 @@
+"""
+`Draft IRCv3 account-registration
+<https://ircv3.net/specs/extensions/account-registration>`_
+"""
+
 from irctest import cases
 from irctest.patma import ANYSTR
 

irctest/server_tests/account_tag.py

@@ -1,12 +1,12 @@
 """
-<http://ircv3.net/specs/extensions/account-tag-3.2.html>
+`IRCv3 account-tag <https://ircv3.net/specs/extensions/account-tag>`_
 """
 
 from irctest import cases
 
 
 @cases.mark_services
-class AccountTagTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
+class AccountTagTestCase(cases.BaseServerTestCase):
     def connectRegisteredClient(self, nick):
         self.addClient()
         self.sendLine(2, "CAP LS 302")
@@ -40,7 +40,7 @@ class AccountTagTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
         self.skipToWelcome(2)
 
     @cases.mark_capabilities("account-tag")
-    @cases.OptionalityHelper.skipUnlessHasMechanism("PLAIN")
+    @cases.skipUnlessHasMechanism("PLAIN")
     def testPrivmsg(self):
         self.connectClient("foo", capabilities=["account-tag"], skip_if_cap_nak=True)
         self.getMessages(1)
@@ -54,7 +54,10 @@ class AccountTagTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
         )
 
     @cases.mark_capabilities("account-tag")
-    @cases.OptionalityHelper.skipUnlessHasMechanism("PLAIN")
+    @cases.skipUnlessHasMechanism("PLAIN")
+    @cases.xfailIfSoftware(
+        ["Charybdis"], "https://github.com/solanum-ircd/solanum/issues/166"
+    )
     def testInvite(self):
         self.connectClient("foo", capabilities=["account-tag"], skip_if_cap_nak=True)
         self.getMessages(1)

irctest/server_tests/away.py

@@ -1,3 +1,8 @@
+"""
+AWAY command (`RFC 2812 <https://datatracker.ietf.org/doc/html/rfc2812#section-4.1>`__,
+`Modern <https://modern.ircdocs.horse/#away-message>`__)
+"""
+
 from irctest import cases
 from irctest.numerics import RPL_AWAY, RPL_NOWAWAY, RPL_UNAWAY, RPL_USERHOST
 from irctest.patma import StrRe
@@ -32,7 +37,7 @@ class AwayTestCase(cases.BaseServerTestCase):
         """
         "The server acknowledges the change in away status by returning the
         `RPL_NOWAWAY` and `RPL_UNAWAY` numerics."
-        -- https://github.com/ircdocs/modern-irc/pull/100
+        -- https://modern.ircdocs.horse/#away-message
         """
         self.connectClient("bar")
         self.sendLine(1, "AWAY :I'm not here right now")
@@ -48,7 +53,7 @@ class AwayTestCase(cases.BaseServerTestCase):
         """
         "Servers SHOULD notify clients when a user they're interacting with
         is away when relevant"
-        -- https://github.com/ircdocs/modern-irc/pull/100
+        -- https://modern.ircdocs.horse/#away-message
 
         "<client> <nick> :<message>"
         -- https://modern.ircdocs.horse/#rplaway-301
@@ -75,7 +80,7 @@ class AwayTestCase(cases.BaseServerTestCase):
         """
         "Servers SHOULD notify clients when a user they're interacting with
         is away when relevant"
-        -- https://github.com/ircdocs/modern-irc/pull/100
+        -- https://modern.ircdocs.horse/#away-message
 
         "<client> <nick> :<message>"
         -- https://modern.ircdocs.horse/#rplaway-301
@@ -113,7 +118,7 @@ class AwayTestCase(cases.BaseServerTestCase):
         """
         "Servers SHOULD notify clients when a user they're interacting with
         is away when relevant"
-        -- https://github.com/ircdocs/modern-irc/pull/100
+        -- https://modern.ircdocs.horse/#away-message
 
         "<client> <nick> :<message>"
         -- https://modern.ircdocs.horse/#rplaway-301

irctest/server_tests/away_notify.py

@@ -1,11 +1,11 @@
 """
-<https://ircv3.net/specs/extensions/away-notify-3.1>
+`IRCv3 away-notify <https://ircv3.net/specs/extensions/away-notify>`_
 """
 
 from irctest import cases
 
 
-class AwayNotifyTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
+class AwayNotifyTestCase(cases.BaseServerTestCase):
     @cases.mark_capabilities("away-notify")
     def testAwayNotify(self):
         """Basic away-notify test."""

irctest/server_tests/bot_mode.py

@@ -1,6 +1,5 @@
 """
-Draft bot mode specification, as defined in
-<https://ircv3.net/specs/extensions/bot-mode>
+`IRCv3 bot mode <https://ircv3.net/specs/extensions/bot-mode>`_
 """
 
 from irctest import cases, runner
@@ -68,6 +67,10 @@ class BotModeTestCase(cases.BaseServerTestCase):
             message, command=RPL_WHOISBOT, params=["usernick", "botnick", ANYSTR]
         )
 
+    @cases.xfailIfSoftware(
+        ["InspIRCd"],
+        "Uses only vendor tags for now: https://github.com/inspircd/inspircd/pull/1910",
+    )
     def testBotPrivateMessage(self):
         self._initBot()
 
@@ -82,9 +85,13 @@ class BotModeTestCase(cases.BaseServerTestCase):
             self.getMessage("user"),
             command="PRIVMSG",
             params=["usernick", "beep boop"],
-            tags={"draft/bot": None, **ANYDICT},
+            tags={StrRe("(draft/)?bot"): None, **ANYDICT},
         )
 
+    @cases.xfailIfSoftware(
+        ["InspIRCd"],
+        "Uses only vendor tags for now: https://github.com/inspircd/inspircd/pull/1910",
+    )
     def testBotChannelMessage(self):
         self._initBot()
 
@@ -104,7 +111,7 @@ class BotModeTestCase(cases.BaseServerTestCase):
             self.getMessage("user"),
             command="PRIVMSG",
             params=["#chan", "beep boop"],
-            tags={"draft/bot": None, **ANYDICT},
+            tags={StrRe("(draft/)?bot"): None, **ANYDICT},
         )
 
     def testBotWhox(self):

irctest/server_tests/bouncer.py

@@ -1,3 +1,9 @@
+"""
+`Ergo <https://ergo.chat/>`_-specific tests of
+`multiclient features
+<https://github.com/ergochat/ergo/blob/master/docs/MANUAL.md#multiclient-bouncer>`_
+"""
+
 from irctest import cases
 from irctest.irc_utils.sasl import sasl_plain_blob
 from irctest.numerics import ERR_NICKNAMEINUSE, RPL_WELCOME

irctest/server_tests/buffering.py

@@ -1,5 +1,7 @@
-"""Sends packets with various length to check the server reassembles them
-correctly. Also checks truncation"""
+"""
+Sends packets with various length to check the server reassembles them
+correctly. Also checks truncation.
+"""
 
 import socket
 import time
@@ -30,6 +32,16 @@ def _sendBytePerByte(self, line):
 
 
 class BufferingTestCase(cases.BaseServerTestCase):
+    @cases.xfailIfSoftware(
+        ["Bahamut"],
+        "cannot pass because of issues with UTF-8 handling: "
+        "https://github.com/DALnet/bahamut/issues/196",
+    )
+    @cases.xfailIfSoftware(
+        ["ircu2", "Nefarious", "snircd"],
+        "ircu2 discards the whole buffer on long lines "
+        "(TODO: refine how we exclude these tests)",
+    )
     @pytest.mark.parametrize(
         "sender_function,colon",
         [

irctest/server_tests/cap.py

@@ -1,9 +1,14 @@
+"""
+`IRCv3 Capability negotiation
+<https://ircv3.net/specs/extensions/capability-negotiation>`_
+"""
+
 from irctest import cases
 from irctest.patma import ANYSTR
 from irctest.runner import CapabilityNotSupported, ImplementationChoice
 
 
-class CapTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
+class CapTestCase(cases.BaseServerTestCase):
     @cases.mark_specifications("IRCv3")
     def testNoReq(self):
         """Test the server handles gracefully clients which do not send
@@ -73,6 +78,10 @@ class CapTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
         )
 
     @cases.mark_specifications("IRCv3")
+    @cases.xfailIfSoftware(
+        ["UnrealIRCd"],
+        "UnrealIRCd sends a trailing space on CAP NAK: https://github.com/unrealircd/unrealircd/pull/148",
+    )
     def testNakWhole(self):
         """“The capability identifier set must be accepted as a whole, or
         rejected entirely.”
@@ -120,6 +129,10 @@ class CapTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
         )
 
     @cases.mark_specifications("IRCv3")
+    @cases.xfailIfSoftware(
+        ["UnrealIRCd"],
+        "UnrealIRCd sends a trailing space on CAP NAK: https://github.com/unrealircd/unrealircd/pull/148",
+    )
     def testCapRemovalByClient(self):
         """Test CAP LIST and removal of caps via CAP REQ :-tagname."""
         cap1 = "echo-message"
@@ -172,3 +185,60 @@ class CapTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
         enabled_caps.discard("cap-notify")  # implicitly added by some impls
         self.assertEqual(enabled_caps, {cap1})
         self.assertNotIn("time", cap_list.tags)
+
+    @cases.mark_specifications("IRCv3")
+    def testIrc301CapLs(self):
+        """
+        Current version:
+
+        "The LS subcommand is used to list the capabilities supported by the server.
+        The client should send an LS subcommand with no other arguments to solicit
+        a list of all capabilities."
+
+        "If a client has not indicated support for CAP LS 302 features,
+        the server MUST NOT send these new features to the client."
+        -- <https://ircv3.net/specs/core/capability-negotiation.html>
+
+        Before the v3.1 / v3.2 merge:
+
+        IRCv3.1: “The LS subcommand is used to list the capabilities
+        supported by the server. The client should send an LS subcommand with
+        no other arguments to solicit a list of all capabilities.”
+        -- <http://ircv3.net/specs/core/capability-negotiation-3.1.html#the-cap-ls-subcommand>
+
+        IRCv3.2: “Servers MUST NOT send messages described by this document if
+        the client only supports version 3.1.”
+        -- <http://ircv3.net/specs/core/capability-negotiation-3.2.html#version-in-cap-ls>
+        """  # noqa
+        self.addClient()
+        self.sendLine(1, "CAP LS")
+        m = self.getRegistrationMessage(1)
+        self.assertNotEqual(
+            m.params[2],
+            "*",
+            m,
+            fail_msg="Server replied with multi-line CAP LS to a "
+            "“CAP LS” (ie. IRCv3.1) request: {msg}",
+        )
+        self.assertFalse(
+            any("=" in cap for cap in m.params[2].split()),
+            "Server replied with a name-value capability in "
+            "CAP LS reply as a response to “CAP LS” (ie. IRCv3.1) "
+            "request: {}".format(m),
+        )
+
+    @cases.mark_specifications("IRCv3")
+    def testEmptyCapList(self):
+        """“If no capabilities are active, an empty parameter must be sent.”
+        -- <http://ircv3.net/specs/core/capability-negotiation-3.1.html#the-cap-list-subcommand>
+        """  # noqa
+        self.addClient()
+        self.sendLine(1, "CAP LIST")
+        m = self.getRegistrationMessage(1)
+        self.assertMessageMatch(
+            m,
+            command="CAP",
+            params=["*", "LIST", ""],
+            fail_msg="Sending “CAP LIST” as first message got a reply "
+            "that is not “CAP * LIST :”: {msg}",
+        )

irctest/server_tests/channel.py

@@ -1,3 +1,7 @@
+"""
+Channel casemapping
+"""
+
 import pytest
 
 from irctest import cases, client_mock, runner
@@ -18,7 +22,7 @@ class ChannelCaseSensitivityTestCase(cases.BaseServerTestCase):
         self.connectClient("foo")
         self.connectClient("bar")
         if self.server_support["CASEMAPPING"] != casemapping:
-            raise runner.NotImplementedByController(
+            raise runner.ImplementationChoice(
                 "Casemapping {} not implemented".format(casemapping)
             )
         self.joinClient(1, name1)
@@ -43,7 +47,7 @@ class ChannelCaseSensitivityTestCase(cases.BaseServerTestCase):
         self.connectClient("foo")
         self.connectClient("bar")
         if self.server_support["CASEMAPPING"] != casemapping:
-            raise runner.NotImplementedByController(
+            raise runner.ImplementationChoice(
                 "Casemapping {} not implemented".format(casemapping)
             )
         self.joinClient(1, name1)

irctest/server_tests/channel_forward.py

@@ -1,3 +1,9 @@
+"""
+`Ergo <https://ergo.chat/>`_-specific tests of channel forwarding
+
+TODO: Should be extended to other servers, once a specification is written.
+"""
+
 from irctest import cases
 from irctest.numerics import ERR_CHANOPRIVSNEEDED, ERR_INVALIDMODEPARAM, ERR_LINKCHANNEL
 

irctest/server_tests/channel_rename.py

@@ -1,24 +1,22 @@
+"""
+`Draft IRCv3 channel-rename <https://ircv3.net/specs/extensions/channel-rename>`_
+"""
+
 from irctest import cases
 from irctest.numerics import ERR_CHANOPRIVSNEEDED
 
-MODERN_CAPS = [
-    "server-time",
-    "message-tags",
-    "batch",
-    "labeled-response",
-    "echo-message",
-    "account-tag",
-]
 RENAME_CAP = "draft/channel-rename"
 
 
+@cases.mark_specifications("IRCv3")
 class ChannelRenameTestCase(cases.BaseServerTestCase):
     """Basic tests for channel-rename."""
 
-    @cases.mark_specifications("Ergo")
     def testChannelRename(self):
-        self.connectClient("bar", name="bar", capabilities=MODERN_CAPS + [RENAME_CAP])
-        self.connectClient("baz", name="baz", capabilities=MODERN_CAPS)
+        self.connectClient(
+            "bar", name="bar", capabilities=[RENAME_CAP], skip_if_cap_nak=True
+        )
+        self.connectClient("baz", name="baz")
         self.joinChannel("bar", "#bar")
         self.joinChannel("baz", "#bar")
         self.getMessages("bar")

irctest/server_tests/chathistory.py

@@ -1,9 +1,14 @@
+"""
+`IRCv3 draft chathistory <https://ircv3.net/specs/extensions/chathistory>`_
+"""
+
+import functools
 import secrets
 import time
 
 import pytest
 
-from irctest import cases
+from irctest import cases, runner
 from irctest.irc_utils.junkdrawer import random_name
 from irctest.patma import ANYSTR
 
@@ -13,8 +18,6 @@ EVENT_PLAYBACK_CAP = "draft/event-playback"
 # Keep this in sync with validate_chathistory()
 SUBCOMMANDS = ["LATEST", "BEFORE", "AFTER", "BETWEEN", "AROUND"]
 
-MYSQL_PASSWORD = ""
-
 
 def validate_chathistory_batch(msgs):
     batch_tag = None
@@ -38,6 +41,16 @@ def validate_chathistory_batch(msgs):
     return result
 
 
+def skip_ngircd(f):
+    @functools.wraps(f)
+    def newf(self, *args, **kwargs):
+        if self.controller.software_name == "ngIRCd":
+            raise runner.OptionalExtensionNotSupported("nicks longer 9 characters")
+        return f(self, *args, **kwargs)
+
+    return newf
+
+
 @cases.mark_specifications("IRCv3")
 @cases.mark_services
 class ChathistoryTestCase(cases.BaseServerTestCase):
@@ -45,6 +58,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
     def config() -> cases.TestCaseControllerConfig:
         return cases.TestCaseControllerConfig(chathistory=True)
 
+    @skip_ngircd
     def testInvalidTargets(self):
         bar, pw = random_name("bar"), random_name("pw")
         self.controller.registerUser(self, bar, pw)
@@ -90,6 +104,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
         )
 
     @pytest.mark.private_chathistory
+    @skip_ngircd
     def testMessagesToSelf(self):
         bar, pw = random_name("bar"), random_name("pw")
         self.controller.registerUser(self, bar, pw)
@@ -162,7 +177,19 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
         self.assertEqual(len(set(msg.time for msg in echo_messages)), num_messages)
 
     @pytest.mark.parametrize("subcommand", SUBCOMMANDS)
+    @skip_ngircd
     def testChathistory(self, subcommand):
+        if subcommand == "BETWEEN" and self.controller.software_name == "UnrealIRCd":
+            pytest.xfail(
+                "CHATHISTORY BETWEEN does not apply bounds correct "
+                "https://bugs.unrealircd.org/view.php?id=5952"
+            )
+        if subcommand == "AROUND" and self.controller.software_name == "UnrealIRCd":
+            pytest.xfail(
+                "CHATHISTORY AROUND excludes 'central' messages "
+                "https://bugs.unrealircd.org/view.php?id=5953"
+            )
+
         self.connectClient(
             "bar",
             capabilities=[
@@ -194,6 +221,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
         self.validate_chathistory(subcommand, echo_messages, 1, chname)
 
     @pytest.mark.parametrize("subcommand", SUBCOMMANDS)
+    @skip_ngircd
     def testChathistoryEventPlayback(self, subcommand):
         self.connectClient(
             "bar",
@@ -227,6 +255,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
 
     @pytest.mark.parametrize("subcommand", SUBCOMMANDS)
     @pytest.mark.private_chathistory
+    @skip_ngircd
     def testChathistoryDMs(self, subcommand):
         c1 = "foo" + secrets.token_hex(12)
         c2 = "bar" + secrets.token_hex(12)
@@ -549,6 +578,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
         self.assertIn(echo_messages[7], result)
 
     @pytest.mark.arbitrary_client_tags
+    @skip_ngircd
     def testChathistoryTagmsg(self):
         c1 = "foo" + secrets.token_hex(12)
         c2 = "bar" + secrets.token_hex(12)
@@ -647,6 +677,7 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
 
     @pytest.mark.arbitrary_client_tags
     @pytest.mark.private_chathistory
+    @skip_ngircd
     def testChathistoryDMClientOnlyTags(self):
         # regression test for Ergo #1411
         c1 = "foo" + secrets.token_hex(12)

irctest/server_tests/chmodes/auditorium.py

@@ -1,3 +1,9 @@
+"""
+`Ergo <https://ergo.chat/>`_-specific tests of auditorium mode
+
+TODO: Should be extended to other servers, once a specification is written.
+"""
+
 import math
 import time
 

irctest/server_tests/chmodes/ban.py

@@ -1,4 +1,12 @@
-from irctest import cases
+"""
+Channel ban (`RFC 1459
+<https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.3.1>`__,
+`RFC 2812 <https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.3>`__,
+`Modern <https://modern.ircdocs.horse/#ban-channel-mode>`__)
+and ban exception (`Modern <https://modern.ircdocs.horse/#exception-channel-mode>`__)
+"""
+
+from irctest import cases, runner
 from irctest.numerics import ERR_BANNEDFROMCHAN, RPL_BANLIST, RPL_ENDOFBANLIST
 from irctest.patma import ANYSTR, StrRe
 
@@ -26,7 +34,7 @@ class BanModeTestCase(cases.BaseServerTestCase):
 
     @cases.mark_specifications("Modern")
     def testBanList(self):
-        """https://github.com/ircdocs/modern-irc/pull/125"""
+        """`RPL_BANLIST <https://modern.ircdocs.horse/#rplbanlist-367>`_"""
         self.connectClient("chanop")
         self.joinChannel(1, "#chan")
         self.getMessages(1)
@@ -70,6 +78,65 @@ class BanModeTestCase(cases.BaseServerTestCase):
             ],
         )
 
+    @cases.mark_specifications("Modern")
+    def testBanException(self):
+        """`Exception mode <https://modern.ircdocs.horse/#exception-channel-mode`_,
+        detected using `ISUPPORT EXCEPTS
+        <https://modern.ircdocs.horse/#excepts-parameter>`_ and checked against
+        `ISUPPORT CHANMODES <https://modern.ircdocs.horse/#chanmodes-parameter>`_"""
+        self.connectClient("chanop", name="chanop")
+
+        if "EXCEPTS" in self.server_support:
+            mode = self.server_support["EXCEPTS"] or "e"
+            if "CHANMODES" in self.server_support:
+                self.assertIn(
+                    mode,
+                    self.server_support["CHANMODES"],
+                    fail_msg="ISUPPORT EXCEPTS is present, but '{item}' is missing "
+                    "from 'CHANMODES={list}'",
+                )
+                self.assertIn(
+                    mode,
+                    self.server_support["CHANMODES"].split(",")[0],
+                    fail_msg="ISUPPORT EXCEPTS is present, but '{item}' is not "
+                    "in group A",
+                )
+        else:
+            mode = "e"
+            if "CHANMODES" in self.server_support:
+                if "e" not in self.server_support["CHANMODES"]:
+                    raise runner.OptionalExtensionNotSupported(
+                        "Ban exception (or mode letter is not +e)"
+                    )
+                self.assertIn(
+                    mode,
+                    self.server_support["CHANMODES"].split(",")[0],
+                    fail_msg="Mode +e (assumed to be ban exception) is present, "
+                    "but 'e' is not in group A",
+                )
+            else:
+                raise runner.OptionalExtensionNotSupported("ISUPPORT CHANMODES")
+
+        self.sendLine("chanop", "JOIN #chan")
+        self.getMessages("chanop")
+        self.sendLine("chanop", "MODE #chan +b ba*!*@*")
+        self.getMessages("chanop")
+
+        # banned client cannot join
+        self.connectClient("Bar", name="bar")
+        self.sendLine("bar", "JOIN #chan")
+        self.assertMessageMatch(self.getMessage("bar"), command=ERR_BANNEDFROMCHAN)
+
+        # chanop sets exception
+        self.sendLine("chanop", "MODE #chan +e *ar!*@*")
+        self.assertMessageMatch(self.getMessage("chanop"), command="MODE")
+
+        # client can now join
+        self.sendLine("bar", "JOIN #chan")
+        self.assertMessageMatch(self.getMessage("bar"), command="JOIN")
+
+    # TODO: Add testBanExceptionList, once the numerics are specified in Modern
+
     @cases.mark_specifications("Ergo")
     def testCaseInsensitive(self):
         """Some clients allow unsetting modes if their argument matches

irctest/server_tests/chmodes/ergo.py

@@ -1,3 +1,7 @@
+"""
+Various Ergo-specific channel modes
+"""
+
 from irctest import cases
 from irctest.numerics import ERR_CANNOTSENDTOCHAN, ERR_CHANOPRIVSNEEDED
 

irctest/server_tests/chmodes/key.py

@@ -1,3 +1,10 @@
+"""
+Channel key (`RFC 1459
+<https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.3.1>`__,
+`RFC 2812 <https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.3>`__,
+`Modern <https://modern.ircdocs.horse/#key-channel-mode>`__)
+"""
+
 import pytest
 
 from irctest import cases
@@ -20,10 +27,16 @@ class KeyTestCase(cases.BaseServerTestCase):
 
         self.connectClient("qux")
         self.getMessages(2)
+        # JOIN with a missing key MUST receive ERR_BADCHANNELKEY:
         self.sendLine(2, "JOIN #chan")
-        reply = self.getMessages(2)
-        self.assertNotIn("JOIN", {msg.command for msg in reply})
-        self.assertIn(ERR_BADCHANNELKEY, {msg.command for msg in reply})
+        reply_cmds = {msg.command for msg in self.getMessages(2)}
+        self.assertNotIn("JOIN", reply_cmds)
+        self.assertIn(ERR_BADCHANNELKEY, reply_cmds)
+        # similarly for JOIN with an incorrect key:
+        self.sendLine(2, "JOIN #chan bees")
+        reply_cmds = {msg.command for msg in self.getMessages(2)}
+        self.assertNotIn("JOIN", reply_cmds)
+        self.assertIn(ERR_BADCHANNELKEY, reply_cmds)
 
         self.sendLine(2, "JOIN #chan beer")
         reply = self.getMessages(2)
@@ -57,6 +70,21 @@ class KeyTestCase(cases.BaseServerTestCase):
         -- https://modern.ircdocs.horse/#key-channel-mode
         -- https://github.com/ircdocs/modern-irc/pull/111
         """
+        if key == "" and self.controller.software_name in (
+            "ircu2",
+            "Nefarious",
+            "snircd",
+        ):
+            pytest.xfail(
+                "ircu2 returns ERR_NEEDMOREPARAMS on empty keys: "
+                "https://github.com/UndernetIRC/ircu2/issues/13"
+            )
+        if (key == "" or " " in key) and self.controller.software_name == "ngIRCd":
+            pytest.xfail(
+                "ngIRCd does not validate channel keys: "
+                "https://github.com/ngircd/ngircd/issues/290"
+            )
+
         self.connectClient("bar")
         self.joinChannel(1, "#chan")
         self.sendLine(1, f"MODE #chan +k :{key}")

irctest/server_tests/chmodes/moderated.py

@@ -1,3 +1,9 @@
+"""
+Channel moderation mode (`RFC 2812
+<https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.3>`__,
+`Modern <https://modern.ircdocs.horse/#ban-channel-mode>`__)
+"""
+
 from irctest import cases
 from irctest.numerics import ERR_CANNOTSENDTOCHAN
 

irctest/server_tests/chmodes/mute_extban.py

@@ -1,3 +1,7 @@
+"""
+Mute extban, currently no specifications or ways to discover it.
+"""
+
 from irctest import cases, runner
 from irctest.numerics import ERR_CANNOTSENDTOCHAN, ERR_CHANOPRIVSNEEDED
 from irctest.patma import ANYLIST, StrRe
@@ -194,7 +198,7 @@ class MuteExtbanTestCase(cases.BaseServerTestCase):
             self.getMessages(client)
 
         # +e grants an exemption to +b
-        self.sendLine("chanop", f"MODE #chan +e {prefix}{self.char()}:*!~evan@*")
+        self.sendLine("chanop", f"MODE #chan +e {prefix}{self.char()}:*!*evan@*")
         replies = {msg.command for msg in self.getMessages("chanop")}
         self.assertIn("MODE", replies)
         self.assertNotIn(ERR_CHANOPRIVSNEEDED, replies)

irctest/server_tests/chmodes/noctcp.py

@@ -1,69 +0,0 @@
-from irctest import cases, runner
-from irctest.numerics import ERR_CANNOTSENDTOCHAN
-from irctest.patma import ANYSTR
-
-
-class NoctcpModeTestCase(cases.BaseServerTestCase):
-    @cases.mark_specifications("Modern")
-    def testNoctcpMode(self):
-        """
-        "This mode is used in almost all IRC software today. The standard mode letter
-        used for it is `"+C"`.
-        When this mode is set, should not send [CTCP](/ctcp.html) messages, except
-        CTCP Action (also known as `/me`) to the channel.
-        When blocking a message because of this mode, servers SHOULD use
-        ERR_CANNOTSENDTOCHAN"
-        -- TODO add link
-        """
-        self.connectClient("chanop")
-
-        if "C" not in self.server_support.get("CHANMODES", ""):
-            raise runner.NotImplementedByController("+C (noctcp) channel mode")
-
-        # Both users join:
-
-        self.sendLine(1, "JOIN #chan")
-        self.getMessages(1)  # synchronize
-
-        self.connectClient("user")
-        self.sendLine(2, "JOIN #chan")
-        self.getMessages(2)
-        self.getMessages(1)
-
-        # Send ACTION and PING, both should go through:
-
-        self.sendLine(2, "PRIVMSG #chan :\x01ACTION is testing\x01")
-        self.sendLine(2, "PRIVMSG #chan :\x01PING 12345\x01")
-        self.assertEqual(self.getMessages(2), [])
-
-        self.assertEqual(
-            [(m.command, m.params[1]) for m in self.getMessages(1)],
-            [
-                ("PRIVMSG", "\x01ACTION is testing\x01"),
-                ("PRIVMSG", "\x01PING 12345\x01"),
-            ],
-        )
-
-        # Set mode +C:
-
-        self.sendLine(1, "MODE #chan +C")
-        self.getMessages(1)
-        self.getMessages(2)
-
-        # Send ACTION and PING, only ACTION should go through:
-
-        self.sendLine(2, "PRIVMSG #chan :\x01ACTION is testing\x01")
-        self.assertEqual(self.getMessages(2), [])
-        self.sendLine(2, "PRIVMSG #chan :\x01PING 12345\x01")
-        self.assertMessageMatch(
-            self.getMessage(2),
-            command=ERR_CANNOTSENDTOCHAN,
-            params=["user", "#chan", ANYSTR],
-        )
-
-        self.assertEqual(
-            [(m.command, m.params[1]) for m in self.getMessages(1)],
-            [
-                ("PRIVMSG", "\x01ACTION is testing\x01"),
-            ],
-        )

irctest/server_tests/chmodes/secret.py

@@ -1,3 +1,10 @@
+"""
+Channel secrecy mode (`RFC 1459
+<https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.3.1>`__,
+`RFC 2812 <https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.3>`__,
+`Modern <https://modern.ircdocs.horse/#secret-channel-mode>`__)
+"""
+
 from irctest import cases
 from irctest.numerics import RPL_LIST
 

irctest/server_tests/confusables.py

@@ -1,3 +1,8 @@
+"""
+`Ergo <https://ergo.chat/>`_-specific tests for nick collisions based on Unicode
+confusable characters
+"""
+
 from irctest import cases
 from irctest.numerics import ERR_NICKNAMEINUSE, RPL_WELCOME
 

irctest/server_tests/connection_registration.py

@@ -1,11 +1,13 @@
 """
 Tests section 4.1 of RFC 1459.
 <https://tools.ietf.org/html/rfc1459#section-4.1>
+
+TODO: cross-reference Modern and RFC 2812 too
 """
 
 from irctest import cases
 from irctest.client_mock import ConnectionClosed
-from irctest.numerics import ERR_NEEDMOREPARAMS
+from irctest.numerics import ERR_NEEDMOREPARAMS, ERR_PASSWDMISMATCH
 from irctest.patma import ANYSTR, StrRe
 
 
@@ -36,8 +38,14 @@ class PasswordedConnectionRegistrationTestCase(cases.BaseServerTestCase):
             m.command, "001", msg="Got 001 after NICK+USER but missing PASS"
         )
 
-    @cases.mark_specifications("RFC1459", "RFC2812")
+    @cases.mark_specifications("Modern")
     def testWrongPassword(self):
+        """
+        "If the password supplied does not match the password expected by the server,
+        then the server SHOULD send ERR_PASSWDMISMATCH and MUST close the connection
+        with ERROR."
+        -- https://github.com/ircdocs/modern-irc/pull/172
+        """
         self.addClient()
         self.sendLine(1, "PASS {}".format(self.password + "garbage"))
         self.sendLine(1, "NICK foo")
@@ -46,6 +54,13 @@ class PasswordedConnectionRegistrationTestCase(cases.BaseServerTestCase):
         self.assertNotEqual(
             m.command, "001", msg="Got 001 after NICK+USER but incorrect PASS"
         )
+        self.assertIn(m.command, {ERR_PASSWDMISMATCH, "ERROR"})
+
+        if m.command == "ERR_PASSWDMISMATCH":
+            m = self.getRegistrationMessage(1)
+            self.assertEqual(
+                m.command, "ERROR", msg="ERR_PASSWDMISMATCH not followed by ERROR."
+            )
 
     @cases.mark_specifications("RFC1459", "RFC2812", strict=True)
     def testPassAfterNickuser(self):
@@ -82,6 +97,10 @@ class ConnectionRegistrationTestCase(cases.BaseServerTestCase):
             self.getMessages(1)
 
     @cases.mark_specifications("RFC2812")
+    @cases.xfailIfSoftware(["Charybdis", "Solanum"], "very flaky")
+    @cases.xfailIfSoftware(
+        ["ircu2", "Nefarious", "snircd"], "ircu2 does not send ERROR"
+    )
     def testQuitErrors(self):
         """“A client session is terminated with a quit message.  The server
         acknowledges this by sending an ERROR message to the client.”
@@ -162,6 +181,10 @@ class ConnectionRegistrationTestCase(cases.BaseServerTestCase):
             "neither got 001.",
         )
 
+    @cases.xfailIfSoftware(
+        ["ircu2", "Nefarious", "ngIRCd"],
+        "uses a default value instead of ERR_NEEDMOREPARAMS",
+    )
     def testEmptyRealname(self):
         """
         Syntax:
@@ -183,60 +206,3 @@ class ConnectionRegistrationTestCase(cases.BaseServerTestCase):
             command=ERR_NEEDMOREPARAMS,
             params=[StrRe(r"(\*|foo)"), "USER", ANYSTR],
         )
-
-    @cases.mark_specifications("IRCv3")
-    def testIrc301CapLs(self):
-        """
-        Current version:
-
-        "The LS subcommand is used to list the capabilities supported by the server.
-        The client should send an LS subcommand with no other arguments to solicit
-        a list of all capabilities."
-
-        "If a client has not indicated support for CAP LS 302 features,
-        the server MUST NOT send these new features to the client."
-        -- <https://ircv3.net/specs/core/capability-negotiation.html>
-
-        Before the v3.1 / v3.2 merge:
-
-        IRCv3.1: “The LS subcommand is used to list the capabilities
-        supported by the server. The client should send an LS subcommand with
-        no other arguments to solicit a list of all capabilities.”
-        -- <http://ircv3.net/specs/core/capability-negotiation-3.1.html#the-cap-ls-subcommand>
-
-        IRCv3.2: “Servers MUST NOT send messages described by this document if
-        the client only supports version 3.1.”
-        -- <http://ircv3.net/specs/core/capability-negotiation-3.2.html#version-in-cap-ls>
-        """  # noqa
-        self.addClient()
-        self.sendLine(1, "CAP LS")
-        m = self.getRegistrationMessage(1)
-        self.assertNotEqual(
-            m.params[2],
-            "*",
-            m,
-            fail_msg="Server replied with multi-line CAP LS to a "
-            "“CAP LS” (ie. IRCv3.1) request: {msg}",
-        )
-        self.assertFalse(
-            any("=" in cap for cap in m.params[2].split()),
-            "Server replied with a name-value capability in "
-            "CAP LS reply as a response to “CAP LS” (ie. IRCv3.1) "
-            "request: {}".format(m),
-        )
-
-    @cases.mark_specifications("IRCv3")
-    def testEmptyCapList(self):
-        """“If no capabilities are active, an empty parameter must be sent.”
-        -- <http://ircv3.net/specs/core/capability-negotiation-3.1.html#the-cap-list-subcommand>
-        """  # noqa
-        self.addClient()
-        self.sendLine(1, "CAP LIST")
-        m = self.getRegistrationMessage(1)
-        self.assertMessageMatch(
-            m,
-            command="CAP",
-            params=["*", "LIST", ""],
-            fail_msg="Sending “CAP LIST” as first message got a reply "
-            "that is not “CAP * LIST :”: {msg}",
-        )

irctest/server_tests/echo_message.py

@@ -1,11 +1,10 @@
 """
-<http://ircv3.net/specs/extensions/echo-message-3.2.html>
+`IRCv3 echo-message <https://ircv3.net/specs/extensions/echo-message>`_
 """
 
 import pytest
 
 from irctest import cases
-from irctest.basecontrollers import NotImplementedByController
 from irctest.irc_utils.junkdrawer import random_name
 from irctest.patma import ANYDICT
 
@@ -23,31 +22,18 @@ class EchoMessageTestCase(cases.BaseServerTestCase):
     @cases.mark_capabilities("echo-message")
     def testEchoMessage(self, command, solo, server_time):
         """<http://ircv3.net/specs/extensions/echo-message-3.2.html>"""
-        self.addClient()
-        self.sendLine(1, "CAP LS 302")
-        capabilities = self.getCapLs(1)
-        if "echo-message" not in capabilities:
-            raise NotImplementedByController("echo-message")
-        if server_time and "server-time" not in capabilities:
-            raise NotImplementedByController("server-time")
-
-        # TODO: check also without this
-        self.sendLine(
-            1,
-            "CAP REQ :echo-message{}".format(" server-time" if server_time else ""),
-        )
-        self.getRegistrationMessage(1)
-        # TODO: Remove this one the trailing space issue is fixed in Charybdis
-        # and Mammon:
-        # self.assertMessageMatch(m, command='CAP',
-        #        params=['*', 'ACK', 'echo-message'] +
-        #        (['server-time'] if server_time else []),
-        #        fail_msg='Did not ACK advertised capabilities: {msg}')
-        self.sendLine(1, "USER f * * :foo")
-        self.sendLine(1, "NICK baz")
-        self.sendLine(1, "CAP END")
-        self.skipToWelcome(1)
-        self.getMessages(1)
+        if server_time:
+            self.connectClient(
+                "baz",
+                capabilities=["echo-message", "server-time"],
+                skip_if_cap_nak=True,
+            )
+        else:
+            self.connectClient(
+                "baz",
+                capabilities=["echo-message", "server-time"],
+                skip_if_cap_nak=True,
+            )
 
         self.sendLine(1, "JOIN #chan")
 

irctest/server_tests/ergo/services.py

@@ -1,3 +1,7 @@
+"""
+`Ergo <https://ergo.chat/>`-specific tests of NickServ.
+"""
+
 from irctest import cases
 from irctest.numerics import RPL_YOUREOPER
 

irctest/server_tests/extended_join.py

@@ -1,12 +1,12 @@
 """
-<http://ircv3.net/specs/extensions/extended-join-3.1.html>
+`IRCv3 extended-join <https://ircv3.net/specs/extensions/extended-join>`_
 """
 
 from irctest import cases
 
 
 @cases.mark_services
-class MetadataTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
+class MetadataTestCase(cases.BaseServerTestCase):
     def connectRegisteredClient(self, nick):
         self.addClient()
         self.sendLine(2, "CAP LS 302")
@@ -50,7 +50,7 @@ class MetadataTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
         )
 
     @cases.mark_capabilities("extended-join")
-    @cases.OptionalityHelper.skipUnlessHasMechanism("PLAIN")
+    @cases.skipUnlessHasMechanism("PLAIN")
     def testLoggedIn(self):
         self.connectClient("foo", capabilities=["extended-join"], skip_if_cap_nak=True)
         self.joinChannel(1, "#chan")

irctest/server_tests/help.py

@@ -1,7 +1,8 @@
 """
-The HELP and HELPOP command.
+The HELP and HELPOP command (`Modern <https://modern.ircdocs.horse/#help-message>`__)
 """
 
+import functools
 import re
 
 import pytest
@@ -17,6 +18,30 @@ from irctest.numerics import (
 from irctest.patma import ANYSTR, StrRe
 
 
+def with_xfails(f):
+    @functools.wraps(f)
+    def newf(self, command, *args, **kwargs):
+        if command == "HELP" and self.controller.software_name == "Bahamut":
+            raise runner.ImplementationChoice(
+                "fail because Bahamut forwards /HELP to HelpServ (but not /HELPOP)"
+            )
+
+        if self.controller.software_name in ("irc2", "ircu2", "ngIRCd"):
+            raise runner.ImplementationChoice(
+                "numerics in reply to /HELP and /HELPOP (uses NOTICE instead)"
+            )
+
+        if self.controller.software_name == "UnrealIRCd":
+            raise runner.ImplementationChoice(
+                "fails because Unreal uses custom numerics "
+                "https://github.com/unrealircd/unrealircd/pull/184"
+            )
+
+        return f(self, command, *args, **kwargs)
+
+    return newf
+
+
 class HelpTestCase(cases.BaseServerTestCase):
     def _assertValidHelp(self, messages, subject):
         if subject != ANYSTR:
@@ -46,6 +71,7 @@ class HelpTestCase(cases.BaseServerTestCase):
 
     @pytest.mark.parametrize("command", ["HELP", "HELPOP"])
     @cases.mark_specifications("Modern")
+    @with_xfails
     def testHelpNoArg(self, command):
         self.connectClient("nick")
         self.sendLine(1, f"{command}")
@@ -59,6 +85,7 @@ class HelpTestCase(cases.BaseServerTestCase):
 
     @pytest.mark.parametrize("command", ["HELP", "HELPOP"])
     @cases.mark_specifications("Modern")
+    @with_xfails
     def testHelpPrivmsg(self, command):
         self.connectClient("nick")
         self.sendLine(1, f"{command} PRIVMSG")
@@ -71,6 +98,7 @@ class HelpTestCase(cases.BaseServerTestCase):
 
     @pytest.mark.parametrize("command", ["HELP", "HELPOP"])
     @cases.mark_specifications("Modern")
+    @with_xfails
     def testHelpUnknownSubject(self, command):
         self.connectClient("nick")
         self.sendLine(1, f"{command} THISISNOTACOMMAND")

irctest/server_tests/info.py

@@ -1,5 +1,8 @@
 """
-The INFO command.
+The INFO command  (`RFC 1459
+<https://datatracker.ietf.org/doc/html/rfc1459#section-4.3.8>`__,
+`RFC 2812 <https://datatracker.ietf.org/doc/html/rfc2812#section-3.4.10>`__,
+`Modern <https://modern.ircdocs.horse/#info-message>`__)
 """
 
 import pytest
@@ -84,6 +87,9 @@ class InfoTestCase(cases.BaseServerTestCase):
 
     @pytest.mark.parametrize("target", ["invalid.server.example", "invalidserver"])
     @cases.mark_specifications("RFC1459", "RFC2812", deprecated=True)
+    @cases.xfailIfSoftware(
+        ["Ergo"], "does not apply to Ergo, which ignores the optional <target> argument"
+    )
     def testInfoNosuchserver(self, target):
         """
         <https://datatracker.ietf.org/doc/html/rfc1459#section-4.3.8>

irctest/server_tests/invite.py

@@ -1,10 +1,18 @@
+"""
+The INVITE command  (`RFC 1459
+<https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.7>`__,
+`RFC 2812 <https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.7>`__,
+`Modern <https://modern.ircdocs.horse/#invite-message>`__)
+"""
+
 import pytest
 
-from irctest import cases
+from irctest import cases, runner
 from irctest.numerics import (
     ERR_BANNEDFROMCHAN,
     ERR_CHANOPRIVSNEEDED,
     ERR_INVITEONLYCHAN,
+    ERR_NEEDMOREPARAMS,
     ERR_NOSUCHNICK,
     ERR_NOTONCHANNEL,
     ERR_USERONCHANNEL,
@@ -193,6 +201,9 @@ class InviteTestCase(cases.BaseServerTestCase):
         self._testInvite(opped=True, invite_only=invite_only)
 
     @cases.mark_specifications("RFC1459", "RFC2812", "Modern", strict=True)
+    @cases.xfailIfSoftware(
+        ["Hybrid", "Plexus4"], "the only strict test that Hybrid fails"
+    )
     def testInviteUnopped(self):
         """Tests invites from unopped users on not-invite-only chans."""
         self._testInvite(opped=False, invite_only=False)
@@ -230,6 +241,11 @@ class InviteTestCase(cases.BaseServerTestCase):
         )
 
     @cases.mark_specifications("RFC1459", "RFC2812", "Modern")
+    @cases.xfailIfSoftware(
+        ["Plexus4"],
+        "Plexus4 allows non-op to invite if (and only if) the channel is not "
+        "invite-only",
+    )
     def testInviteInviteOnly(self):
         """
         "To invite a user to a channel which is invite only (MODE
@@ -357,6 +373,87 @@ class InviteTestCase(cases.BaseServerTestCase):
             params=["foo", "bar", "#chan", ANYSTR],
         )
 
+    @cases.mark_specifications("RFC2812", "Modern")
+    @cases.xfailIfSoftware(
+        ["ircu2"],
+        "Uses 346/347 instead of 336/337 to reply to INVITE "
+        "https://github.com/UndernetIRC/ircu2/pull/20",
+    )
+    def testInviteList(self):
+        self.connectClient("foo")
+        self.connectClient("bar")
+        self.getMessages(1)
+        self.getMessages(2)
+
+        self.sendLine(1, "JOIN #chan")
+        self.getMessages(1)
+
+        self.sendLine(1, "INVITE bar #chan")
+        self.getMessages(1)
+        self.getMessages(2)
+
+        self.sendLine(2, "INVITE")
+        m = self.getMessage(2)
+        if m.command == ERR_NEEDMOREPARAMS:
+            raise runner.OptionalExtensionNotSupported("INVITE with no parameter")
+        if m.command != "337":
+            # Hybrid always sends an empty list; so skip this.
+            self.assertMessageMatch(
+                m,
+                command="336",
+                params=["bar", "#chan"],
+            )
+            m = self.getMessage(2)
+        self.assertMessageMatch(
+            m,
+            command="337",
+            params=["bar", ANYSTR],
+        )
+
+    @cases.mark_isupport("INVEX")
+    @cases.mark_specifications("Modern")
+    def testInvexList(self):
+        self.connectClient("foo")
+        self.getMessages(1)
+
+        if "INVEX" in self.server_support:
+            invex = self.server_support.get("INVEX") or "I"
+        else:
+            raise runner.IsupportTokenNotSupported("INVEX")
+
+        self.sendLine(1, "JOIN #chan")
+        self.getMessages(1)
+
+        self.sendLine(1, f"MODE #chan +{invex} bar!*@*")
+        self.getMessages(1)
+
+        self.sendLine(1, f"MODE #chan +{invex}")
+        m = self.getMessage(1)
+        if len(m.params) == 3:
+            # Old format
+            self.assertMessageMatch(
+                m,
+                command="346",
+                params=["foo", "#chan", "bar!*@*"],
+            )
+        else:
+            self.assertMessageMatch(
+                m,
+                command="346",
+                params=[
+                    "foo",
+                    "#chan",
+                    "bar!*@*",
+                    StrRe("foo(!.*@.*)?"),
+                    StrRe("[0-9]+"),
+                ],
+            )
+        self.assertMessageMatch(
+            self.getMessage(1),
+            command="347",
+            params=["foo", "#chan", ANYSTR],
+        )
+
     @cases.mark_specifications("Ergo")
     def testInviteExemptsFromBan(self):
         # regression test for ergochat/ergo#1876;

irctest/server_tests/isupport.py

@@ -1,3 +1,8 @@
+"""
+RPL_ISUPPORT: `format <https://modern.ircdocs.horse/#rplisupport-005>`__
+and various `tokens <https://modern.ircdocs.horse/#rplisupport-parameters>`__
+"""
+
 import re
 
 from irctest import cases, runner
@@ -11,7 +16,7 @@ class IsupportTestCase(cases.BaseServerTestCase):
         self.connectClient("foo")
 
         if "PREFIX" not in self.server_support:
-            raise runner.NotImplementedByController("PREFIX")
+            raise runner.IsupportTokenNotSupported("PREFIX")
 
         if self.server_support["PREFIX"] == "":
             # "The value is OPTIONAL and when it is not specified indicates that no
@@ -75,7 +80,7 @@ class IsupportTestCase(cases.BaseServerTestCase):
         self.connectClient("foo")
 
         if "TARGMAX" not in self.server_support:
-            raise runner.NotImplementedByController("TARGMAX")
+            raise runner.IsupportTokenNotSupported("TARGMAX")
 
         parts = self.server_support["TARGMAX"].split(",")
         for part in parts:

irctest/server_tests/join.py

@@ -1,5 +1,30 @@
-from irctest import cases
+"""
+The JOIN command  (`RFC 1459
+<https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.1>`__,
+`RFC 2812 <https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.1>`__,
+`Modern <https://modern.ircdocs.horse/#join-message>`__)
+"""
+
+from irctest import cases, runner
 from irctest.irc_utils import ambiguities
+from irctest.numerics import (
+    ERR_BADCHANMASK,
+    ERR_FORBIDDENCHANNEL,
+    ERR_NOSUCHCHANNEL,
+    RPL_ENDOFNAMES,
+    RPL_NAMREPLY,
+)
+from irctest.patma import ANYSTR, StrRe
+
+ERR_BADCHANNAME = "479"  # Hybrid only, and conflicts with others
+
+
+JOIN_ERROR_NUMERICS = {
+    ERR_BADCHANMASK,
+    ERR_NOSUCHCHANNEL,
+    ERR_FORBIDDENCHANNEL,
+    ERR_BADCHANNAME,
+}
 
 
 class JoinTestCase(cases.BaseServerTestCase):
@@ -19,13 +44,22 @@ class JoinTestCase(cases.BaseServerTestCase):
         self.connectClient("foo")
         self.sendLine(1, "JOIN #chan")
         received_commands = {m.command for m in self.getMessages(1)}
-        expected_commands = {"353", "366"}  # RPL_NAMREPLY  # RPL_ENDOFNAMES
-        self.assertTrue(
-            expected_commands.issubset(received_commands),
+        expected_commands = {RPL_NAMREPLY, RPL_ENDOFNAMES, "JOIN"}
+        acceptable_commands = expected_commands | {"MODE"}
+        self.assertLessEqual(  # set inclusion
+            expected_commands,
+            received_commands,
             "Server sent {} commands, but at least {} were expected.".format(
                 received_commands, expected_commands
             ),
         )
+        self.assertLessEqual(  # ditto
+            received_commands,
+            acceptable_commands,
+            "Server sent {} commands, but only {} were expected.".format(
+                received_commands, acceptable_commands
+            ),
+        )
 
     @cases.mark_specifications("RFC2812")
     def testJoinNamreply(self):
@@ -110,3 +144,95 @@ class JoinTestCase(cases.BaseServerTestCase):
                     '"foo" with an optional "+" or "@" prefix, but got: '
                     "{msg}",
                 )
+
+    def testJoinPartiallyInvalid(self):
+        """TODO: specify this in Modern"""
+        self.connectClient("foo")
+        if int(self.targmax.get("JOIN") or "4") < 2:
+            raise runner.OptionalExtensionNotSupported("multi-channel JOIN")
+
+        self.sendLine(1, "JOIN #valid,inv@lid")
+        messages = self.getMessages(1)
+        received_commands = {m.command for m in messages}
+        expected_commands = {RPL_NAMREPLY, RPL_ENDOFNAMES, "JOIN"}
+        acceptable_commands = expected_commands | JOIN_ERROR_NUMERICS | {"MODE"}
+        self.assertLessEqual(
+            expected_commands,
+            received_commands,
+            "Server sent {} commands, but at least {} were expected.".format(
+                received_commands, expected_commands
+            ),
+        )
+        self.assertLessEqual(
+            received_commands,
+            acceptable_commands,
+            "Server sent {} commands, but only {} were expected.".format(
+                received_commands, acceptable_commands
+            ),
+        )
+
+        nb_errors = 0
+        for m in messages:
+            if m.command in JOIN_ERROR_NUMERICS:
+                nb_errors += 1
+                self.assertMessageMatch(m, params=["foo", "inv@lid", ANYSTR])
+
+        self.assertEqual(
+            nb_errors,
+            1,
+            fail_msg="Expected 1 error when joining channels '#valid' and 'inv@lid', "
+            "got {got}",
+        )
+
+    @cases.mark_capabilities("batch", "labeled-response")
+    def testJoinPartiallyInvalidLabeledResponse(self):
+        """TODO: specify this in Modern"""
+        self.connectClient(
+            "foo", capabilities=["batch", "labeled-response"], skip_if_cap_nak=True
+        )
+        if int(self.targmax.get("JOIN") or "4") < 2:
+            raise runner.OptionalExtensionNotSupported("multi-channel JOIN")
+
+        self.sendLine(1, "@label=label1 JOIN #valid,inv@lid")
+        messages = self.getMessages(1)
+
+        first_msg = messages.pop(0)
+        last_msg = messages.pop(-1)
+        self.assertMessageMatch(
+            first_msg, command="BATCH", params=[StrRe(r"\+.*"), "labeled-response"]
+        )
+        batch_id = first_msg.params[0][1:]
+        self.assertMessageMatch(last_msg, command="BATCH", params=["-" + batch_id])
+
+        received_commands = {m.command for m in messages}
+        expected_commands = {RPL_NAMREPLY, RPL_ENDOFNAMES, "JOIN"}
+        acceptable_commands = expected_commands | JOIN_ERROR_NUMERICS | {"MODE"}
+        self.assertLessEqual(
+            expected_commands,
+            received_commands,
+            "Server sent {} commands, but at least {} were expected.".format(
+                received_commands, expected_commands
+            ),
+        )
+        self.assertLessEqual(
+            received_commands,
+            acceptable_commands,
+            "Server sent {} commands, but only {} were expected.".format(
+                received_commands, acceptable_commands
+            ),
+        )
+
+        nb_errors = 0
+        for m in messages:
+            self.assertIn("batch", m.tags)
+            self.assertEqual(m.tags["batch"], batch_id)
+            if m.command in JOIN_ERROR_NUMERICS:
+                nb_errors += 1
+                self.assertMessageMatch(m, params=["foo", "inv@lid", ANYSTR])
+
+        self.assertEqual(
+            nb_errors,
+            1,
+            fail_msg="Expected 1 error when joining channels '#valid' and 'inv@lid', "
+            "got {got}",
+        )

irctest/server_tests/kick.py

@@ -1,3 +1,10 @@
+"""
+The KICK command  (`RFC 1459
+<https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.1>`__,
+`RFC 2812 <https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.>`__,
+`Modern <https://modern.ircdocs.horse/#kick-message>`__)
+"""
+
 import pytest
 
 from irctest import cases, client_mock, runner
@@ -89,6 +96,10 @@ class KickTestCase(cases.BaseServerTestCase):
             self.assertMessageMatch(m3, command="KICK", params=["#chan", "bar", ANYSTR])
 
     @cases.mark_specifications("RFC2812")
+    @cases.xfailIfSoftware(
+        ["Charybdis", "ircu2", "irc2", "Solanum"],
+        "uses the nick of the kickee rather than the kicker.",
+    )
     def testKickDefaultComment(self):
         """
         "If a "comment" is
@@ -219,13 +230,8 @@ class KickTestCase(cases.BaseServerTestCase):
         self.connectClient("qux")
         self.joinChannel(4, "#chan")
 
-        targmax = dict(
-            item.split(":", 1)
-            for item in self.server_support.get("TARGMAX", "").split(",")
-            if item
-        )
-        if targmax.get("KICK", "1") == "1":
-            raise runner.NotImplementedByController("Multi-target KICK")
+        if self.targmax.get("KICK", "1") == "1":
+            raise runner.OptionalExtensionNotSupported("Multi-target KICK")
 
         # TODO: check foo is an operator
 

irctest/server_tests/labeled_responses.py

@@ -1,8 +1,8 @@
 """
+`IRCv3 labeled-response <https://ircv3.net/specs/extensions/labeled-response>`_
+
 This specification is a little hard to test because all labels are optional;
 so there may be many false positives.
-
-<https://ircv3.net/specs/extensions/labeled-response.html>
 """
 
 import re
@@ -14,7 +14,7 @@ from irctest.numerics import ERR_UNKNOWNCOMMAND
 from irctest.patma import ANYDICT, ANYOPTSTR, NotStrRe, RemainingKeys, StrRe
 
 
-class LabeledResponsesTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
+class LabeledResponsesTestCase(cases.BaseServerTestCase):
     @cases.mark_capabilities("echo-message", "batch", "labeled-response")
     def testLabeledPrivmsgResponsesToMultipleClients(self):
         self.connectClient(

irctest/server_tests/links.py

@@ -0,0 +1,136 @@
+from irctest import cases, runner
+from irctest.numerics import ERR_UNKNOWNCOMMAND, RPL_ENDOFLINKS, RPL_LINKS
+from irctest.patma import ANYSTR, StrRe
+
+
+class LinksTestCase(cases.BaseServerTestCase):
+    @cases.mark_specifications("RFC1459", "RFC2812", "Modern")
+    def testLinksSingleServer(self):
+        """
+        Only testing the parameter-less case.
+
+        https://datatracker.ietf.org/doc/html/rfc1459#section-4.3.3
+        https://datatracker.ietf.org/doc/html/rfc2812#section-3.4.5
+        https://github.com/ircdocs/modern-irc/pull/175
+
+        "
+        364     RPL_LINKS
+                        "<mask> <server> :<hopcount> <server info>"
+        365     RPL_ENDOFLINKS
+                        "<mask> :End of /LINKS list"
+
+                - In replying to the LINKS message, a server must send
+                  replies back using the RPL_LINKS numeric and mark the
+                  end of the list using an RPL_ENDOFLINKS reply.
+        "
+        -- https://datatracker.ietf.org/doc/html/rfc1459#page-51
+        -- https://datatracker.ietf.org/doc/html/rfc2812#page-48
+
+        RPL_LINKS: "<client> * <server> :<hopcount> <server info>"
+        RPL_ENDOFLINKS: "<client> * :End of /LINKS list"
+        -- https://github.com/ircdocs/modern-irc/pull/175/files
+        """
+        self.connectClient("nick")
+        self.sendLine(1, "LINKS")
+        messages = self.getMessages(1)
+        if messages[0].command == ERR_UNKNOWNCOMMAND:
+            raise runner.OptionalCommandNotSupported("LINKS")
+
+        # Ignore '/LINKS has been disabled' from ircu2
+        messages = [m for m in messages if m.command != "NOTICE"]
+
+        self.assertMessageMatch(
+            messages.pop(-1),
+            command=RPL_ENDOFLINKS,
+            params=["nick", "*", ANYSTR],
+        )
+
+        if not messages:
+            # This server probably redacts links
+            return
+
+        self.assertMessageMatch(
+            messages[0],
+            command=RPL_LINKS,
+            params=[
+                "nick",
+                "My.Little.Server",
+                "My.Little.Server",
+                StrRe("0 (0042 )?test server"),
+            ],
+        )
+
+
+@cases.mark_services
+class ServicesLinksTestCase(cases.BaseServerTestCase):
+    # On every IRCd but Ergo, services are linked.
+    # Ergo does not implement LINKS at all, so this test is skipped.
+    @cases.mark_specifications("RFC1459", "RFC2812", "Modern")
+    def testLinksWithServices(self):
+        """
+        Only testing the parameter-less case.
+
+        https://datatracker.ietf.org/doc/html/rfc1459#section-4.3.3
+        https://datatracker.ietf.org/doc/html/rfc2812#section-3.4.5
+
+        "
+        364     RPL_LINKS
+                        "<mask> <server> :<hopcount> <server info>"
+        365     RPL_ENDOFLINKS
+                        "<mask> :End of /LINKS list"
+
+                - In replying to the LINKS message, a server must send
+                  replies back using the RPL_LINKS numeric and mark the
+                  end of the list using an RPL_ENDOFLINKS reply.
+        "
+        -- https://datatracker.ietf.org/doc/html/rfc1459#page-51
+        -- https://datatracker.ietf.org/doc/html/rfc2812#page-48
+
+        RPL_LINKS: "<client> * <server> :<hopcount> <server info>"
+        RPL_ENDOFLINKS: "<client> * :End of /LINKS list"
+        -- https://github.com/ircdocs/modern-irc/pull/175/files
+        """
+        self.connectClient("nick")
+        self.sendLine(1, "LINKS")
+        messages = self.getMessages(1)
+
+        if messages[0].command == ERR_UNKNOWNCOMMAND:
+            raise runner.OptionalCommandNotSupported("LINKS")
+
+        # Ignore '/LINKS has been disabled' from ircu2
+        messages = [m for m in messages if m.command != "NOTICE"]
+
+        self.assertMessageMatch(
+            messages.pop(-1),
+            command=RPL_ENDOFLINKS,
+            params=["nick", "*", ANYSTR],
+        )
+
+        if not messages:
+            # This server redacts links
+            return
+
+        messages.sort(key=lambda m: m.params[-1])
+
+        self.assertMessageMatch(
+            messages.pop(0),
+            command=RPL_LINKS,
+            params=[
+                "nick",
+                "My.Little.Server",
+                "My.Little.Server",
+                StrRe("0 (0042 )?test server"),
+            ],
+        )
+        self.assertMessageMatch(
+            messages.pop(0),
+            command=RPL_LINKS,
+            params=[
+                "nick",
+                "services.example.org",
+                "My.Little.Server",
+                StrRe("1 .+"),  # SID instead of description for Anope...
+            ],
+        )
+
+        self.assertEqual(messages, [])

irctest/server_tests/list.py

@@ -1,40 +1,71 @@
-from irctest import cases
+"""
+The LIST command  (`RFC 1459
+<https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.6>`__,
+`RFC 2812 <https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.6>`__,
+`Modern <https://modern.ircdocs.horse/#list-message>`__)
+"""
+
+import time
+
+from irctest import cases, runner
+from irctest.numerics import RPL_LIST, RPL_LISTEND, RPL_LISTSTART
 
 
-class ListTestCase(cases.BaseServerTestCase):
+class _BasedListTestCase(cases.BaseServerTestCase):
+    def _parseChanList(self, client):
+        channels = set()
+        while True:
+            m = self.getMessage(client)
+            if m.command == RPL_LISTEND:
+                break
+            if m.command == RPL_LIST:
+                if m.params[1].startswith("&"):
+                    # skip local pseudo-channels listed by ngircd and ircu
+                    continue
+                channels.add(m.params[1])
+
+        return channels
+
+
+class ListTestCase(_BasedListTestCase):
     @cases.mark_specifications("RFC1459", "RFC2812")
+    @cases.xfailIfSoftware(["irc2"], "irc2 deprecated LIST")
     def testListEmpty(self):
         """<https://tools.ietf.org/html/rfc1459#section-4.2.6>
         <https://tools.ietf.org/html/rfc2812#section-3.2.6>
+        <https://modern.ircdocs.horse/#list-message>
         """
         self.connectClient("foo")
         self.connectClient("bar")
         self.getMessages(1)
         self.sendLine(2, "LIST")
         m = self.getMessage(2)
-        if m.command == "321":
-            # skip RPL_LISTSTART
+        if m.command == RPL_LISTSTART:
+            # skip
             m = self.getMessage(2)
         # skip local pseudo-channels listed by ngircd and ircu
-        while m.command == "322" and m.params[1].startswith("&"):
+        while m.command == RPL_LIST and m.params[1].startswith("&"):
             m = self.getMessage(2)
         self.assertNotEqual(
             m.command,
-            "322",  # RPL_LIST
+            RPL_LIST,
             "LIST response gives (at least) one channel, whereas there " "is none.",
         )
         self.assertMessageMatch(
             m,
-            command="323",  # RPL_LISTEND
+            command=RPL_LISTEND,
             fail_msg="Second reply to LIST is not 322 (RPL_LIST) "
             "or 323 (RPL_LISTEND), or but: {msg}",
         )
 
     @cases.mark_specifications("RFC1459", "RFC2812")
+    @cases.xfailIfSoftware(["irc2"], "irc2 deprecated LIST")
     def testListOne(self):
         """When a channel exists, LIST should get it in a reply.
         <https://tools.ietf.org/html/rfc1459#section-4.2.6>
         <https://tools.ietf.org/html/rfc2812#section-3.2.6>
+
+        <https://modern.ircdocs.horse/#list-message>
         """
         self.connectClient("foo")
         self.connectClient("bar")
@@ -42,34 +73,331 @@ class ListTestCase(cases.BaseServerTestCase):
         self.getMessages(1)
         self.sendLine(2, "LIST")
         m = self.getMessage(2)
-        if m.command == "321":
-            # skip RPL_LISTSTART
+        if m.command == RPL_LISTSTART:
+            # skip
             m = self.getMessage(2)
         self.assertNotEqual(
             m.command,
-            "323",  # RPL_LISTEND
+            RPL_LISTEND,
             fail_msg="LIST response ended (ie. 323, aka RPL_LISTEND) "
             "without listing any channel, whereas there is one.",
         )
         self.assertMessageMatch(
             m,
-            command="322",  # RPL_LIST
+            command=RPL_LIST,
             fail_msg="Second reply to LIST is not 322 (RPL_LIST), "
             "nor 323 (RPL_LISTEND) but: {msg}",
         )
         m = self.getMessage(2)
         # skip local pseudo-channels listed by ngircd and ircu
-        while m.command == "322" and m.params[1].startswith("&"):
+        while m.command == RPL_LIST and m.params[1].startswith("&"):
             m = self.getMessage(2)
         self.assertNotEqual(
             m.command,
-            "322",  # RPL_LIST
+            RPL_LIST,
             fail_msg="LIST response gives (at least) two channels, "
             "whereas there is only one.",
         )
         self.assertMessageMatch(
             m,
-            command="323",  # RPL_LISTEND
+            command=RPL_LISTEND,
             fail_msg="Third reply to LIST is not 322 (RPL_LIST) "
             "or 323 (RPL_LISTEND), or but: {msg}",
         )
+
+    @cases.mark_specifications("RFC1459", "RFC2812")
+    @cases.xfailIfSoftware(
+        ["Charybdis", "Solanum"],
+        "Charybdis and Solanum insert ERR_NOSUCHNICK reply in LIST",
+    )
+    def testListNonexistent(self):
+        """LIST on a nonexistent channel does not send an error
+        response.
+        <https://tools.ietf.org/html/rfc1459#section-4.2.6>
+        <https://tools.ietf.org/html/rfc2812#section-3.2.6>
+        """
+        self.connectClient("bar")
+        self.sendLine(1, "LIST #nonexistent")
+        responses = {msg.command for msg in self.getMessages(1)}
+        # successful response MUST include RPL_LISTEND:
+        self.assertIn(RPL_LISTEND, responses)
+        # and MUST NOT include RPL_LIST (since there is no matching channel)
+        # or any error numerics:
+        self.assertLessEqual(responses, {RPL_LISTSTART, RPL_LISTEND})
+
+    @cases.mark_isupport("ELIST")
+    @cases.mark_specifications("Modern")
+    def testListMask(self):
+        """
+        "M: Searching based on mask."
+        -- <https://modern.ircdocs.horse/#elist-parameter>
+        -- https://datatracker.ietf.org/doc/html/draft-hardy-irc-isupport-00#section-4.8
+        """
+        self.connectClient("foo")
+
+        if "M" not in self.server_support.get("ELIST", ""):
+            raise runner.OptionalExtensionNotSupported("ELIST=M")
+
+        self.connectClient("bar")
+        self.sendLine(1, "JOIN #chan1")
+        self.getMessages(1)
+        self.sendLine(1, "JOIN #chan2")
+        self.getMessages(1)
+
+        self.sendLine(2, "LIST *an1")
+        self.assertEqual(self._parseChanList(2), {"#chan1"})
+
+        self.sendLine(2, "LIST *an2")
+        self.assertEqual(self._parseChanList(2), {"#chan2"})
+
+        self.sendLine(2, "LIST #c*n2")
+        self.assertEqual(self._parseChanList(2), {"#chan2"})
+
+        self.sendLine(2, "LIST *an3")
+        self.assertEqual(self._parseChanList(2), set())
+
+        self.sendLine(2, "LIST #ch*")
+        self.assertEqual(self._parseChanList(2), {"#chan1", "#chan2"})
+
+    @cases.mark_isupport("ELIST")
+    @cases.mark_specifications("Modern")
+    def testListNotMask(self):
+        """
+        " N: Searching based on a non-matching mask. i.e., the opposite of M."
+        -- <https://modern.ircdocs.horse/#elist-parameter>
+        -- https://datatracker.ietf.org/doc/html/draft-hardy-irc-isupport-00#section-4.8
+        """
+        self.connectClient("foo")
+
+        if "N" not in self.server_support.get("ELIST", ""):
+            raise runner.OptionalExtensionNotSupported("ELIST=N")
+
+        self.sendLine(1, "JOIN #chan1")
+        self.getMessages(1)
+        self.sendLine(1, "JOIN #chan2")
+        self.getMessages(1)
+
+        self.connectClient("bar")
+
+        self.sendLine(2, "LIST !*an1")
+        self.assertEqual(self._parseChanList(2), {"#chan2"})
+
+        self.sendLine(2, "LIST !*an2")
+        self.assertEqual(self._parseChanList(2), {"#chan1"})
+
+        self.sendLine(2, "LIST !#c*n2")
+        self.assertEqual(self._parseChanList(2), {"#chan1"})
+
+        self.sendLine(2, "LIST !*an3")
+        self.assertEqual(self._parseChanList(2), {"#chan1", "#chan2"})
+
+        self.sendLine(2, "LIST !#ch*")
+        self.assertEqual(self._parseChanList(2), set())
+
+    @cases.mark_isupport("ELIST")
+    @cases.mark_specifications("Modern")
+    def testListUsers(self):
+        """
+        "U: Searching based on user count within the channel, via the "<val" and
+        ">val" modifiers to search for a channel that has less or more than val users,
+        respectively."
+        -- <https://modern.ircdocs.horse/#elist-parameter>
+        -- https://datatracker.ietf.org/doc/html/draft-hardy-irc-isupport-00#section-4.8
+        """
+        self.connectClient("foo")
+
+        if "U" not in self.server_support.get("ELIST", ""):
+            raise runner.OptionalExtensionNotSupported("ELIST=U")
+
+        self.sendLine(1, "JOIN #chan1")
+        self.getMessages(1)
+        self.sendLine(1, "JOIN #chan2")
+        self.getMessages(1)
+
+        self.connectClient("bar")
+        self.sendLine(2, "JOIN #chan2")
+        self.getMessages(2)
+
+        self.connectClient("baz")
+
+        self.sendLine(3, "LIST >0")
+        self.assertEqual(self._parseChanList(3), {"#chan1", "#chan2"})
+
+        self.sendLine(3, "LIST <1")
+        self.assertEqual(self._parseChanList(3), set())
+
+        self.sendLine(3, "LIST <100")
+        self.assertEqual(self._parseChanList(3), {"#chan1", "#chan2"})
+
+        self.sendLine(3, "LIST >1")
+        self.assertEqual(self._parseChanList(3), {"#chan2"})
+
+        self.sendLine(3, "LIST <2")
+        self.assertEqual(self._parseChanList(3), {"#chan1"})
+
+        self.sendLine(3, "LIST <100")
+        self.assertEqual(self._parseChanList(3), {"#chan1", "#chan2"})
+
+
+class FaketimeListTestCase(_BasedListTestCase):
+    faketime = "+1y x30"  # for every wall clock second, 1 minute passed for the server
+
+    def _sleep_minutes(self, n):
+        for _ in range(n):
+            if self.controller.faketime_enabled:
+                # From the server's point of view, 1 min will pass
+                time.sleep(2)
+            else:
+                time.sleep(60)
+
+            # reply to pings
+            self.getMessages(1)
+            self.getMessages(2)
+
+    @cases.mark_isupport("ELIST")
+    @cases.mark_specifications("Modern")
+    @cases.xfailIfSoftware(
+        ["Plexus4", "Hybrid"],
+        "Hybrid and Plexus4 filter on ELIST=C with the opposite meaning",
+    )
+    @cases.xfailIf(
+        lambda self: bool(
+            self.controller.software_name == "UnrealIRCd"
+            and self.controller.software_version == 5
+        ),
+        "UnrealIRCd <6.0.3 filters on ELIST=C with the opposite meaning",
+    )
+    def testListCreationTime(self):
+        """
+        " C: Searching based on channel creation time, via the "C<val" and "C>val"
+        modifiers to search for a channel creation time that is higher or lower
+        than val."
+        -- <https://modern.ircdocs.horse/#elist-parameter>
+        -- https://datatracker.ietf.org/doc/html/draft-hardy-irc-isupport-00#section-4.8
+
+        Unfortunately, this is ambiguous, because "val" is a time delta (in minutes),
+        not a timestamp.
+
+        On InspIRCd and Charybdis/Solanum, "C<val" is interpreted as "the channel was
+        created less than <val> minutes ago
+
+        On UnrealIRCd, Plexus, and Hybrid, it is interpreted as "the channel's creation
+        time is a timestamp lower than <val> minutes ago" (ie. the exact opposite)
+
+        "C: Searching based on channel creation time, via the "C<val" and "C>val"
+        modifiers to search for a channel that was created either less than `val`
+        minutes ago, or more than `val` minutes ago, respectively"
+        -- https://github.com/ircdocs/modern-irc/pull/171
+        """
+        self.connectClient("foo")
+
+        if "C" not in self.server_support.get("ELIST", ""):
+            raise runner.OptionalExtensionNotSupported("ELIST=C")
+
+        self.connectClient("bar")
+        self.sendLine(1, "JOIN #chan1")
+        self.getMessages(1)
+
+        # Helps debugging
+        self.sendLine(1, "TIME")
+        self.getMessages(1)
+
+        self._sleep_minutes(2)
+
+        # Helps debugging
+        self.sendLine(1, "TIME")
+        self.getMessages(1)
+
+        self.sendLine(1, "JOIN #chan2")
+        self.getMessages(1)
+
+        self._sleep_minutes(1)
+
+        self.sendLine(2, "LIST C>2")
+        self.assertEqual(self._parseChanList(2), {"#chan1"})
+
+        self.sendLine(2, "LIST C<2")
+        self.assertEqual(self._parseChanList(2), {"#chan2"})
+
+        self.sendLine(2, "LIST C<0")
+        if self.controller.software_name == "InspIRCd":
+            self.assertEqual(self._parseChanList(2), {"#chan1", "#chan2"})
+        else:
+            self.assertEqual(self._parseChanList(2), set())
+
+        self.sendLine(2, "LIST C>0")
+        self.assertEqual(self._parseChanList(2), {"#chan1", "#chan2"})
+
+        self.sendLine(2, "LIST C<10")
+        self.assertEqual(self._parseChanList(2), {"#chan1", "#chan2"})
+
+    @cases.mark_isupport("ELIST")
+    @cases.mark_specifications("Modern")
+    @cases.xfailIf(
+        lambda self: bool(
+            self.controller.software_name == "UnrealIRCd"
+            and self.controller.software_version == 5
+        ),
+        "UnrealIRCd <6.0.3 advertises ELIST=T but does not implement it",
+    )
+    def testListTopicTime(self):
+        """
+        "T: Searching based on topic time, via the "T<val" and "T>val"
+        modifiers to search for a topic time that is lower or higher than
+        val respectively."
+        -- <https://modern.ircdocs.horse/#elist-parameter>
+        -- https://datatracker.ietf.org/doc/html/draft-hardy-irc-isupport-00#section-4.8
+
+        See testListCreationTime's docstring for comments on this.
+
+        "T: Searching based on topic set time, via the "T<val" and "T>val" modifiers
+        to search for a topic time that was set less than `val` minutes ago, or more
+        than `val` minutes ago, respectively."
+        -- https://github.com/ircdocs/modern-irc/pull/171
+        """
+        self.connectClient("foo")
+
+        if "T" not in self.server_support.get("ELIST", ""):
+            raise runner.OptionalExtensionNotSupported("ELIST=T")
+
+        self.connectClient("bar")
+        self.sendLine(1, "JOIN #chan1")
+        self.sendLine(1, "JOIN #chan2")
+        self.getMessages(1)
+
+        self.sendLine(1, "TOPIC #chan1 :First channel")
+        self.getMessages(1)
+
+        # Helps debugging
+        self.sendLine(1, "TIME")
+        self.getMessages(1)
+
+        self._sleep_minutes(2)
+
+        # Helps debugging
+        self.sendLine(1, "TIME")
+        self.getMessages(1)
+
+        self.sendLine(1, "TOPIC #chan2 :Second channel")
+        self.getMessages(1)
+
+        self._sleep_minutes(1)
+
+        self.sendLine(1, "LIST T>2")
+        self.assertEqual(self._parseChanList(1), {"#chan1"})
+
+        self.sendLine(1, "LIST T<2")
+        self.assertEqual(self._parseChanList(1), {"#chan2"})
+
+        self.sendLine(1, "LIST T<0")
+        if self.controller.software_name == "InspIRCd":
+            # Insp internally represents "LIST T>0" like "LIST"
+            self.assertEqual(self._parseChanList(1), {"#chan1", "#chan2"})
+        else:
+            self.assertEqual(self._parseChanList(1), set())
+
+        self.sendLine(1, "LIST T>0")
+        self.assertEqual(self._parseChanList(1), {"#chan1", "#chan2"})
+
+        self.sendLine(1, "LIST T<10")
+        self.assertEqual(self._parseChanList(1), {"#chan1", "#chan2"})

irctest/server_tests/lusers.py

@@ -1,3 +1,11 @@
+"""
+The LUSERS command  (`RFC 2812
+<https://datatracker.ietf.org/doc/html/rfc2812#section-3.4.2>`__,
+`Modern <https://modern.ircdocs.horse/#lusers-message>`__),
+which provides statistics on user counts.
+"""
+
+
 from dataclasses import dataclass
 import re
 from typing import Optional
@@ -145,6 +153,10 @@ class BasicLusersTestCase(LusersTestCase):
         self.getLusers("bar", True)
 
     @cases.mark_specifications("Modern")
+    @cases.xfailIfSoftware(
+        ["ircu2", "Nefarious", "snircd"],
+        "test depends on Modern behavior, not just RFC2812",
+    )
     def testLusersFull(self):
         self.connectClient("bar", name="bar")
         lusers = self.getLusers("bar", False)
@@ -162,10 +174,22 @@ class BasicLusersTestCase(LusersTestCase):
 
 class LusersUnregisteredTestCase(LusersTestCase):
     @cases.mark_specifications("RFC2812")
+    @cases.xfailIfSoftware(
+        ["Nefarious"],
+        "Nefarious doesn't seem to distinguish unregistered users from normal ones",
+    )
     def testLusersRfc2812(self):
         self.doLusersTest(True)
 
     @cases.mark_specifications("Modern")
+    @cases.xfailIfSoftware(
+        ["Nefarious"],
+        "Nefarious doesn't seem to distinguish unregistered users from normal ones",
+    )
+    @cases.xfailIfSoftware(
+        ["ircu2", "Nefarious", "snircd"],
+        "test depends on Modern behavior, not just RFC2812",
+    )
     def testLusersFull(self):
         self.doLusersTest(False)
 
@@ -229,6 +253,10 @@ class LusersUnregisteredDefaultInvisibleTestCase(LusersUnregisteredTestCase):
         )
 
     @cases.mark_specifications("Ergo")
+    @cases.xfailIfSoftware(
+        ["Nefarious"],
+        "Nefarious doesn't seem to distinguish unregistered users from normal ones",
+    )
     def testLusers(self):
         self.doLusersTest(False)
         lusers = self.getLusers("bar", False)

irctest/server_tests/message_tags.py

@@ -1,5 +1,5 @@
 """
-https://ircv3.net/specs/extensions/message-tags.html
+`IRCv3 message-tags <https://ircv3.net/specs/extensions/message-tags>`_
 """
 
 import pytest
@@ -10,7 +10,7 @@ from irctest.numerics import ERR_INPUTTOOLONG
 from irctest.patma import ANYDICT, ANYSTR, StrRe
 
 
-class MessageTagsTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
+class MessageTagsTestCase(cases.BaseServerTestCase):
     @pytest.mark.arbitrary_client_tags
     @cases.mark_capabilities("message-tags")
     def testBasic(self):

irctest/server_tests/messages.py

@@ -1,6 +1,5 @@
 """
-Section 3.2 of RFC 2812
-<https://tools.ietf.org/html/rfc2812#section-3.3>
+The PRIVMSG and NOTICE commands.
 """
 
 from irctest import cases
@@ -52,6 +51,15 @@ class NoticeTestCase(cases.BaseServerTestCase):
         )
 
     @cases.mark_specifications("RFC1459", "RFC2812")
+    @cases.xfailIfSoftware(
+        ["InspIRCd"],
+        "replies with ERR_NOSUCHCHANNEL to NOTICE to non-existent channels",
+    )
+    @cases.xfailIfSoftware(
+        ["UnrealIRCd"],
+        "replies with ERR_NOSUCHCHANNEL to NOTICE to non-existent channels: "
+        "https://bugs.unrealircd.org/view.php?id=5949",
+    )
     def testNoticeNonexistentChannel(self):
         """
         "automatic replies must never be
@@ -72,6 +80,9 @@ class NoticeTestCase(cases.BaseServerTestCase):
 
 class TagsTestCase(cases.BaseServerTestCase):
     @cases.mark_capabilities("message-tags")
+    @cases.xfailIfSoftware(
+        ["UnrealIRCd"], "https://bugs.unrealircd.org/view.php?id=5947"
+    )
     def testLineTooLong(self):
         self.connectClient("bar", capabilities=["message-tags"], skip_if_cap_nak=True)
         self.connectClient(

irctest/server_tests/metadata.py

@@ -1,6 +1,5 @@
 """
-Tests METADATA features.
-<http://ircv3.net/specs/core/metadata-3.2.html>
+`Deprecated IRCv3 Metadata <https://ircv3.net/specs/core/metadata-3.2>`_
 """
 
 from irctest import cases

irctest/server_tests/monitor.py

@@ -1,9 +1,8 @@
 """
-<http://ircv3.net/specs/core/monitor-3.2.html>
+`IRCv3 MONITOR <https://ircv3.net/specs/extensions/monitor>`_
 """
 
-from irctest import cases
-from irctest.basecontrollers import NotImplementedByController
+from irctest import cases, runner
 from irctest.client_mock import NoMessageException
 from irctest.numerics import (
     RPL_ENDOFMONLIST,
@@ -17,7 +16,7 @@ from irctest.patma import ANYSTR, StrRe
 class MonitorTestCase(cases.BaseServerTestCase):
     def check_server_support(self):
         if "MONITOR" not in self.server_support:
-            raise NotImplementedByController("MONITOR")
+            raise runner.IsupportTokenNotSupported("MONITOR")
 
     def assertMononline(self, client, nick, m=None):
         if not m:

irctest/server_tests/multi_prefix.py

@@ -1,6 +1,5 @@
 """
-Tests multi-prefix.
-<http://ircv3.net/specs/extensions/multi-prefix-3.1.html>
+`IRCv3 multi-prefix <https://ircv3.net/specs/extensions/multi-prefix>`_
 """
 
 from irctest import cases

irctest/server_tests/multiline.py

@@ -1,5 +1,5 @@
 """
-draft/multiline
+`Draft IRCv3 multiline <https://ircv3.net/specs/extensions/multiline>`_
 """
 
 from irctest import cases
@@ -12,7 +12,7 @@ CONCAT_TAG = "draft/multiline-concat"
 base_caps = ["message-tags", "batch", "echo-message", "server-time", "labeled-response"]
 
 
-class MultilineTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
+class MultilineTestCase(cases.BaseServerTestCase):
     @cases.mark_capabilities("draft/multiline")
     def testBasic(self):
         self.connectClient(

irctest/server_tests/names.py

@@ -1,9 +1,114 @@
-from irctest import cases
-from irctest.numerics import RPL_ENDOFNAMES
-from irctest.patma import ANYSTR
+"""
+The NAMES command  (`RFC 1459
+<https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.5>`__,
+`RFC 2812 <https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.5>`__,
+`Modern <https://modern.ircdocs.horse/#names-message>`__)
+"""
+
+from irctest import cases, runner
+from irctest.numerics import RPL_ENDOFNAMES, RPL_NAMREPLY
+from irctest.patma import ANYSTR, StrRe
 
 
 class NamesTestCase(cases.BaseServerTestCase):
+    def _testNames(self, symbol):
+        self.connectClient("nick1")
+        self.sendLine(1, "JOIN #chan")
+        self.getMessages(1)
+        self.connectClient("nick2")
+        self.sendLine(2, "JOIN #chan")
+        self.getMessages(2)
+        self.getMessages(1)
+
+        self.sendLine(1, "NAMES #chan")
+
+        # TODO: It is technically allowed to have one line for each;
+        # but noone does that.
+        self.assertMessageMatch(
+            self.getMessage(1),
+            command=RPL_NAMREPLY,
+            params=[
+                "nick1",
+                *(["="] if symbol else []),
+                "#chan",
+                StrRe("(nick2 @nick1|@nick1 nick2)"),
+            ],
+        )
+
+        self.assertMessageMatch(
+            self.getMessage(1),
+            command=RPL_ENDOFNAMES,
+            params=["nick1", "#chan", ANYSTR],
+        )
+
+    @cases.mark_specifications("RFC1459", deprecated=True)
+    def testNames1459(self):
+        """
+        https://modern.ircdocs.horse/#names-message
+        https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.5
+        https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.5
+        """
+        self._testNames(symbol=False)
+
+    @cases.mark_specifications("RFC1459", "RFC2812", "Modern")
+    def testNames2812(self):
+        """
+        https://modern.ircdocs.horse/#names-message
+        https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.5
+        https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.5
+        """
+        self._testNames(symbol=True)
+
+    def _testNamesMultipleChannels(self, symbol):
+        self.connectClient("nick1")
+
+        if self.targmax.get("NAMES", "1") == "1":
+            raise runner.OptionalExtensionNotSupported("Multi-target NAMES")
+
+        self.sendLine(1, "JOIN #chan1")
+        self.sendLine(1, "JOIN #chan2")
+        self.getMessages(1)
+
+        self.sendLine(1, "NAMES #chan1,#chan2")
+
+        # TODO: order is unspecified
+        self.assertMessageMatch(
+            self.getMessage(1),
+            command=RPL_NAMREPLY,
+            params=["nick1", *(["="] if symbol else []), "#chan1", "@nick1"],
+        )
+        self.assertMessageMatch(
+            self.getMessage(1),
+            command=RPL_NAMREPLY,
+            params=["nick1", *(["="] if symbol else []), "#chan2", "@nick1"],
+        )
+
+        self.assertMessageMatch(
+            self.getMessage(1),
+            command=RPL_ENDOFNAMES,
+            params=["nick1", "#chan1,#chan2", ANYSTR],
+        )
+
+    @cases.mark_isupport("TARGMAX")
+    @cases.mark_specifications("RFC1459", deprecated=True)
+    def testNamesMultipleChannels1459(self):
+        """
+        https://modern.ircdocs.horse/#names-message
+        https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.5
+        https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.5
+        """
+        self._testNamesMultipleChannels(symbol=False)
+
+    @cases.mark_isupport("TARGMAX")
+    @cases.mark_specifications("RFC1459", "RFC2812", "Modern")
+    def testNamesMultipleChannels2812(self):
+        """
+        https://modern.ircdocs.horse/#names-message
+        https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.5
+        https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.5
+        """
+        self._testNamesMultipleChannels(symbol=True)
+
     @cases.mark_specifications("RFC1459", "RFC2812", "Modern")
     def testNamesInvalidChannel(self):
         """
@@ -47,3 +152,101 @@ class NamesTestCase(cases.BaseServerTestCase):
             command=RPL_ENDOFNAMES,
             params=["foo", "#nonexisting", ANYSTR],
         )
+
+    def _testNamesNoArgumentPublic(self, symbol):
+        self.connectClient("nick1")
+        self.getMessages(1)
+        self.sendLine(1, "JOIN #chan1")
+        self.connectClient("nick2")
+        self.sendLine(2, "JOIN #chan2")
+        self.sendLine(2, "MODE #chan2 -sp")
+        self.getMessages(1)
+        self.getMessages(2)
+
+        self.sendLine(1, "NAMES")
+
+        # TODO: order is unspecified
+        self.assertMessageMatch(
+            self.getMessage(1),
+            command=RPL_NAMREPLY,
+            params=["nick1", *(["="] if symbol else []), "#chan1", "@nick1"],
+        )
+        self.assertMessageMatch(
+            self.getMessage(1),
+            command=RPL_NAMREPLY,
+            params=["nick1", *(["="] if symbol else []), "#chan2", "@nick2"],
+        )
+
+        self.assertMessageMatch(
+            self.getMessage(1),
+            command=RPL_ENDOFNAMES,
+            params=["nick1", ANYSTR, ANYSTR],
+        )
+
+    @cases.mark_specifications("RFC1459", deprecated=True)
+    def testNamesNoArgumentPublic1459(self):
+        """
+        "If no <channel> parameter is given, a list of all channels and their
+        occupants is returned."
+        -- https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.5
+        -- https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.5
+        """
+        self._testNamesNoArgumentPublic(symbol=False)
+
+    @cases.mark_specifications("RFC2812", deprecated=True)
+    def testNamesNoArgumentPublic2812(self):
+        """
+        "If no <channel> parameter is given, a list of all channels and their
+        occupants is returned."
+        -- https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.5
+        -- https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.5
+        """
+        self._testNamesNoArgumentPublic(symbol=True)
+
+    def _testNamesNoArgumentPrivate(self, symbol):
+        self.connectClient("nick1")
+        self.getMessages(1)
+        self.sendLine(1, "JOIN #chan1")
+        self.connectClient("nick2")
+        self.sendLine(2, "JOIN #chan2")
+        self.sendLine(2, "MODE #chan2 +sp")
+        self.getMessages(1)
+        self.getMessages(2)
+
+        self.sendLine(1, "NAMES")
+
+        self.assertMessageMatch(
+            self.getMessage(1),
+            command=RPL_NAMREPLY,
+            params=["nick1", *(["="] if symbol else []), "#chan1", "@nick1"],
+        )
+
+        self.assertMessageMatch(
+            self.getMessage(1),
+            command=RPL_ENDOFNAMES,
+            params=["nick1", ANYSTR, ANYSTR],
+        )
+
+    @cases.mark_specifications("RFC1459", deprecated=True)
+    def testNamesNoArgumentPrivate1459(self):
+        """
+        "If no <channel> parameter is given, a list of all channels and their
+        occupants is returned.  At the end of this list, a list of users who
+        are visible but either not on any channel or not on a visible channel
+        are listed as being on `channel' "*"."
+        -- https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.5
+        -- https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.5
+        """
+        self._testNamesNoArgumentPrivate(symbol=False)
+
+    @cases.mark_specifications("RFC2812", deprecated=True)
+    def testNamesNoArgumentPrivate2812(self):
+        """
+        "If no <channel> parameter is given, a list of all channels and their
+        occupants is returned.  At the end of this list, a list of users who
+        are visible but either not on any channel or not on a visible channel
+        are listed as being on `channel' "*"."
+        -- https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.5
+        -- https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.5
+        """
+        self._testNamesNoArgumentPrivate(symbol=True)

irctest/server_tests/part.py

@@ -1,3 +1,12 @@
+"""
+The PART command  (`RFC 1459
+<https://datatracker.ietf.org/doc/html/rfc1459#section-6.1>`__,
+`RFC 2812 <https://datatracker.ietf.org/doc/html/rfc2812#section-5.2>`__,
+`Modern <https://modern.ircdocs.horse/#part-message>`__)
+
+TODO: cross-reference Modern
+"""
+
 import time
 
 from irctest import cases

irctest/server_tests/pingpong.py

@@ -1,3 +1,7 @@
+"""
+The PING and PONG commands
+"""
+
 from irctest import cases
 from irctest.numerics import ERR_NEEDMOREPARAMS, ERR_NOORIGIN
 from irctest.patma import ANYSTR

irctest/server_tests/quit.py

@@ -1,3 +1,12 @@
+"""
+The QUITcommand  (`RFC 1459
+<https://datatracker.ietf.org/doc/html/rfc1459#section-4.1.6>`__,
+`RFC 2812 <https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.1>`__,
+`Modern <https://modern.ircdocs.horse/#quit-message>`__)
+
+TODO: cross-reference RFC 1459 and Modern
+"""
+
 import time
 
 from irctest import cases
@@ -7,6 +16,7 @@ from irctest.patma import StrRe
 
 class ChannelQuitTestCase(cases.BaseServerTestCase):
     @cases.mark_specifications("RFC2812")
+    @cases.xfailIfSoftware(["ircu2", "Nefarious", "snircd"], "ircu2 does not echo QUIT")
     def testQuit(self):
         """“Once a user has joined a channel, he receives information about
         all commands his server receives affecting the channel.  This

irctest/server_tests/readq.py

@@ -1,9 +1,12 @@
+"""
+`Ergo <https://ergo.chat/>`_-specific tests of responses to DoS attacks
+using long lines.
+"""
+
 from irctest import cases
 
 
 class ReadqTestCase(cases.BaseServerTestCase):
-    """Test responses to DoS attacks using long lines."""
-
     @cases.mark_specifications("Ergo")
     @cases.mark_capabilities("message-tags")
     def testReadqTags(self):

irctest/server_tests/regressions.py

@@ -1,10 +1,10 @@
 """
-Regression tests for bugs in oragono.
+Regression tests for bugs in `Ergo <https://ergo.chat/>`_.
 """
 
 import time
 
-from irctest import cases
+from irctest import cases, runner
 from irctest.numerics import ERR_ERRONEUSNICKNAME, ERR_NICKNAMEINUSE, RPL_WELCOME
 from irctest.patma import ANYDICT
 
@@ -57,6 +57,12 @@ class RegressionsTestCase(cases.BaseServerTestCase):
 
     @cases.mark_capabilities("message-tags", "batch", "echo-message", "server-time")
     def testTagCap(self):
+        if self.controller.software_name == "UnrealIRCd":
+            raise runner.ImplementationChoice(
+                "Arbitrary +draft/reply values (TODO: adapt this test to use real "
+                "values so their pass Unreal's validation) "
+                "https://bugs.unrealircd.org/view.php?id=5948"
+            )
         # regression test for oragono #754
         self.connectClient(
             "alice",
@@ -99,6 +105,7 @@ class RegressionsTestCase(cases.BaseServerTestCase):
         )
 
     @cases.mark_specifications("RFC1459")
+    @cases.xfailIfSoftware(["ngIRCd"], "wat")
     def testStarNick(self):
         self.addClient(1)
         self.sendLine(1, "NICK *")

irctest/server_tests/relaymsg.py

@@ -1,3 +1,7 @@
+"""
+RELAYMSG command of `Ergo <https://ergo.chat/>`_
+"""
+
 from irctest import cases
 from irctest.irc_utils.junkdrawer import random_name
 from irctest.patma import ANYSTR

irctest/server_tests/roleplay.py

@@ -1,3 +1,7 @@
+"""
+Roleplay features of `Ergo <https://ergo.chat/>`_
+"""
+
 from irctest import cases
 from irctest.irc_utils.junkdrawer import random_name
 from irctest.numerics import ERR_CANNOTSENDRP

irctest/server_tests/sasl.py

@@ -12,9 +12,9 @@ class RegistrationTestCase(cases.BaseServerTestCase):
 
 
 @cases.mark_services
-class SaslTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
+class SaslTestCase(cases.BaseServerTestCase):
     @cases.mark_specifications("IRCv3")
-    @cases.OptionalityHelper.skipUnlessHasMechanism("PLAIN")
+    @cases.skipUnlessHasMechanism("PLAIN")
     def testPlain(self):
         """PLAIN authentication with correct username/password."""
         self.controller.registerUser(self, "foo", "sesame")
@@ -54,7 +54,7 @@ class SaslTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
         )
 
     @cases.mark_specifications("IRCv3")
-    @cases.OptionalityHelper.skipUnlessHasMechanism("PLAIN")
+    @cases.skipUnlessHasMechanism("PLAIN")
     def testPlainNonAscii(self):
         password = "é" * 100
         authstring = base64.b64encode(
@@ -82,7 +82,7 @@ class SaslTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
         )
 
     @cases.mark_specifications("IRCv3")
-    @cases.OptionalityHelper.skipUnlessHasMechanism("PLAIN")
+    @cases.skipUnlessHasMechanism("PLAIN")
     def testPlainNoAuthzid(self):
         """“message   = [authzid] UTF8NUL authcid UTF8NUL passwd
 
@@ -170,7 +170,14 @@ class SaslTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
         )
 
     @cases.mark_specifications("IRCv3")
-    @cases.OptionalityHelper.skipUnlessHasMechanism("PLAIN")
+    @cases.skipUnlessHasMechanism("PLAIN")
+    @cases.xfailIf(
+        lambda self: (
+            self.controller.services_controller is not None
+            and self.controller.services_controller.software_name == "Anope"
+        ),
+        "Anope does not handle split AUTHENTICATE (reported on IRC)",
+    )
     def testPlainLarge(self):
         """Test the client splits large AUTHENTICATE messages whose payload
         is not a multiple of 400.
@@ -232,7 +239,14 @@ class SaslTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
     # message's length too big for it to be valid.
 
     @cases.mark_specifications("IRCv3")
-    @cases.OptionalityHelper.skipUnlessHasMechanism("PLAIN")
+    @cases.skipUnlessHasMechanism("PLAIN")
+    @cases.xfailIf(
+        lambda self: (
+            self.controller.services_controller is not None
+            and self.controller.services_controller.software_name == "Anope"
+        ),
+        "Anope does not handle split AUTHENTICATE (reported on IRC)",
+    )
     def testPlainLargeEquals400(self):
         """Test the client splits large AUTHENTICATE messages whose payload
         is not a multiple of 400.
@@ -277,7 +291,7 @@ class SaslTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
     # message's length too big for it to be valid.
 
     @cases.mark_specifications("IRCv3")
-    @cases.OptionalityHelper.skipUnlessHasMechanism("SCRAM-SHA-256")
+    @cases.skipUnlessHasMechanism("SCRAM-SHA-256")
     def testScramSha256Success(self):
         self.controller.registerUser(self, "Scramtest", "sesame")
 
@@ -333,7 +347,7 @@ class SaslTestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
         self.confirmSuccessfulAuth()
 
     @cases.mark_specifications("IRCv3")
-    @cases.OptionalityHelper.skipUnlessHasMechanism("SCRAM-SHA-256")
+    @cases.skipUnlessHasMechanism("SCRAM-SHA-256")
     def testScramSha256Failure(self):
         self.controller.registerUser(self, "Scramtest", "sesame")
 

irctest/server_tests/statusmsg.py

@@ -1,3 +1,10 @@
+"""
+STATUSMSG ISUPPORT token and related PRIVMSG (`Modern
+<https://modern.ircdocs.horse/#statusmsg-parameter>`__)
+
+TODO: cross-reference Modern
+"""
+
 from irctest import cases, runner
 from irctest.numerics import RPL_NAMREPLY
 
@@ -10,6 +17,11 @@ class StatusmsgTestCase(cases.BaseServerTestCase):
         self.assertEqual(self.server_support["STATUSMSG"], "~&@%+")
 
     @cases.mark_isupport("STATUSMSG")
+    @cases.xfailIfSoftware(
+        ["ircu2", "Nefarious", "snircd"],
+        "STATUSMSG is present in ISUPPORT, but it not actually supported as PRIVMSG "
+        "target (only for WALLCOPS/WALLCHOPS/...)",
+    )
     def testStatusmsgFromOp(self):
         """Test that STATUSMSG are sent to the intended recipients,
         with the intended prefixes."""
@@ -61,6 +73,11 @@ class StatusmsgTestCase(cases.BaseServerTestCase):
         self.assertEqual(len(unprivilegedMessages), 0)
 
     @cases.mark_isupport("STATUSMSG")
+    @cases.xfailIfSoftware(
+        ["ircu2", "Nefarious", "snircd"],
+        "STATUSMSG is present in ISUPPORT, but it not actually supported as PRIVMSG "
+        "target (only for WALLCOPS/WALLCHOPS/...)",
+    )
     def testStatusmsgFromRegular(self):
         """Test that STATUSMSG are sent to the intended recipients,
         with the intended prefixes."""

irctest/server_tests/topic.py

@@ -1,3 +1,10 @@
+"""
+The TOPIC command  (`RFC 1459
+<https://datatracker.ietf.org/doc/html/rfc1459#section-4.2.1>`__,
+`RFC 2812 <https://datatracker.ietf.org/doc/html/rfc2812#section-3.2.1>`__,
+`Modern <https://modern.ircdocs.horse/#topic-message>`__)
+"""
+
 from irctest import cases, client_mock, runner
 from irctest.numerics import ERR_CHANOPRIVSNEEDED, RPL_NOTOPIC, RPL_TOPIC, RPL_TOPICTIME
 

irctest/server_tests/umodes/registeredonly.py

@@ -0,0 +1,124 @@
+"""
+Test the registered-only DM user mode (commonly +R).
+"""
+
+from irctest import cases
+from irctest.numerics import ERR_NEEDREGGEDNICK
+
+
+@cases.mark_services
+class RegisteredOnlyUmodeTestCase(cases.BaseServerTestCase):
+    @cases.mark_specifications("Ergo")
+    def testRegisteredOnlyUserMode(self):
+        """Test the +R registered-only mode."""
+        self.controller.registerUser(self, "evan", "sesame")
+        self.controller.registerUser(self, "carmen", "pink")
+
+        self.connectClient(
+            "evan",
+            name="evan",
+            account="evan",
+            password="sesame",
+            capabilities=["sasl"],
+        )
+        self.connectClient("shivaram", name="shivaram")
+        self.sendLine("evan", "MODE evan +R")
+        self.assertMessageMatch(
+            self.getMessage("evan"), command="MODE", params=["evan", "+R"]
+        )
+
+        # this DM should be blocked by +R registered-only
+        self.getMessages("shivaram")
+        self.sendLine("shivaram", "PRIVMSG evan :hey there")
+        self.assertMessageMatch(
+            self.getMessage("shivaram"),
+            command=ERR_NEEDREGGEDNICK,
+        )
+        self.assertEqual(self.getMessages("evan"), [])
+
+        self.connectClient(
+            "carmen",
+            name="carmen",
+            account="carmen",
+            password="pink",
+            capabilities=["sasl"],
+        )
+        self.getMessages("evan")
+        self.sendLine("carmen", "PRIVMSG evan :hey there")
+        self.assertEqual(self.getMessages("carmen"), [])
+        # this message should go through fine:
+        self.assertMessageMatch(
+            self.getMessage("evan"),
+            command="PRIVMSG",
+            params=["evan", "hey there"],
+        )
+
+    @cases.mark_specifications("Ergo")
+    def testRegisteredOnlyUserModeAcceptCommand(self):
+        """Test that the ACCEPT command can authorize another user
+        to send the accept-er direct messages, overriding the
+        +R registered-only mode."""
+        self.controller.registerUser(self, "evan", "sesame")
+        self.connectClient(
+            "evan",
+            name="evan",
+            account="evan",
+            password="sesame",
+            capabilities=["sasl"],
+        )
+        self.connectClient("shivaram", name="shivaram")
+        self.sendLine("evan", "MODE evan +R")
+        self.assertMessageMatch(
+            self.getMessage("evan"), command="MODE", params=["evan", "+R"]
+        )
+        self.sendLine("evan", "ACCEPT shivaram")
+        self.getMessages("evan")
+
+        self.sendLine("shivaram", "PRIVMSG evan :hey there")
+        self.assertEqual(self.getMessages("shivaram"), [])
+        self.assertMessageMatch(
+            self.getMessage("evan"),
+            command="PRIVMSG",
+            params=["evan", "hey there"],
+        )
+
+        self.sendLine("evan", "ACCEPT -shivaram")
+        self.getMessages("evan")
+        self.sendLine("shivaram", "PRIVMSG evan :how's it going")
+        self.assertMessageMatch(
+            self.getMessage("shivaram"),
+            command=ERR_NEEDREGGEDNICK,
+        )
+        self.assertEqual(self.getMessages("evan"), [])
+
+    @cases.mark_specifications("Ergo")
+    def testRegisteredOnlyUserModeAutoAcceptOnDM(self):
+        """Test that sending someone a DM automatically authorizes them to
+        reply, overriding the +R registered-only mode."""
+        self.controller.registerUser(self, "evan", "sesame")
+        self.connectClient(
+            "evan",
+            name="evan",
+            account="evan",
+            password="sesame",
+            capabilities=["sasl"],
+        )
+        self.connectClient("shivaram", name="shivaram")
+        self.sendLine("evan", "MODE evan +R")
+        self.assertMessageMatch(
+            self.getMessage("evan"), command="MODE", params=["evan", "+R"]
+        )
+        self.sendLine("evan", "PRIVMSG shivaram :hey there")
+        self.getMessages("evan")
+        self.assertMessageMatch(
+            self.getMessage("shivaram"),
+            command="PRIVMSG",
+            params=["shivaram", "hey there"],
+        )
+        self.sendLine("shivaram", "PRIVMSG evan :how's it going")
+        self.assertEqual(self.getMessages("shivaram"), [])
+        self.assertMessageMatch(
+            self.getMessage("evan"),
+            command="PRIVMSG",
+            params=["evan", "how's it going"],
+        )

irctest/server_tests/utf8.py

@@ -1,8 +1,15 @@
+"""
+`Ergo <https://ergo.chat/>`_-specific tests of non-Unicode filtering
+
+TODO: turn this into a test of `IRCv3 UTF8ONLY
+<https://ircv3.net/specs/extensions/utf8-only>`_
+"""
+
 from irctest import cases
 from irctest.patma import ANYSTR
 
 
-class Utf8TestCase(cases.BaseServerTestCase, cases.OptionalityHelper):
+class Utf8TestCase(cases.BaseServerTestCase):
     @cases.mark_specifications("Ergo")
     def testUtf8Validation(self):
         self.connectClient(

irctest/server_tests/wallops.py

@@ -1,3 +1,9 @@
+"""
+The WALLOPS command  (`RFC 2812
+<https://datatracker.ietf.org/doc/html/rfc2812#section-3.7>`__,
+`Modern <https://modern.ircdocs.horse/#wallops-message>`__)
+"""
+
 from irctest import cases, runner
 from irctest.numerics import ERR_NOPRIVILEGES, ERR_UNKNOWNCOMMAND, RPL_YOUREOPER
 from irctest.patma import ANYSTR, StrRe
@@ -38,7 +44,7 @@ class WallopsTestCase(cases.BaseServerTestCase):
 
         messages = self.getMessages(1)
         if ERR_UNKNOWNCOMMAND in (message.command for message in messages):
-            raise runner.NotImplementedByController("WALLOPS")
+            raise runner.OptionalCommandNotSupported("WALLOPS")
         for message in messages:
             self.assertMessageMatch(
                 message,
@@ -60,6 +66,9 @@ class WallopsTestCase(cases.BaseServerTestCase):
         )
 
     @cases.mark_specifications("Modern")
+    @cases.xfailIfSoftware(
+        ["irc2"], "irc2 ignores the command instead of replying ERR_UNKNOWNCOMMAND"
+    )
     def testWallopsPrivileges(self):
         """
         https://github.com/ircdocs/modern-irc/pull/118
@@ -68,7 +77,7 @@ class WallopsTestCase(cases.BaseServerTestCase):
         self.sendLine(1, "WALLOPS :hi everyone")
         message = self.getMessage(1)
         if message.command == ERR_UNKNOWNCOMMAND:
-            raise runner.NotImplementedByController("WALLOPS")
+            raise runner.OptionalCommandNotSupported("WALLOPS")
         self.assertMessageMatch(
             message, command=ERR_NOPRIVILEGES, params=["nick1", ANYSTR]
         )

irctest/server_tests/who.py

@@ -1,3 +1,10 @@
+"""
+The WHO command  (`Modern <https://modern.ircdocs.horse/#who-message>`__)
+and `IRCv3 WHOX <https://ircv3.net/specs/extensions/whox>`_
+
+TODO: cross-reference RFC 1459 and RFC 2812
+"""
+
 import re
 
 import pytest
@@ -77,9 +84,12 @@ class BaseWhoTestCase:
             )
 
 
-class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase, cases.OptionalityHelper):
+class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
     @cases.mark_specifications("Modern")
     def testWhoStar(self):
+        if self.controller.software_name == "Bahamut":
+            raise runner.OptionalExtensionNotSupported("WHO mask")
+
         self._init()
 
         self.sendLine(2, "WHO *")
@@ -108,6 +118,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase, cases.OptionalityHe
     )
     @cases.mark_specifications("Modern")
     def testWhoNick(self, mask):
+        if "*" in mask and self.controller.software_name == "Bahamut":
+            raise runner.OptionalExtensionNotSupported("WHO mask")
+
         self._init()
 
         self.sendLine(2, f"WHO {mask}")
@@ -135,6 +148,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase, cases.OptionalityHe
         ids=["username", "realname-mask", "hostname"],
     )
     def testWhoUsernameRealName(self, mask):
+        if "*" in mask and self.controller.software_name == "Bahamut":
+            raise runner.OptionalExtensionNotSupported("WHO mask")
+
         self._init()
 
         self.sendLine(2, f"WHO :{mask}")
@@ -185,6 +201,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase, cases.OptionalityHe
     )
     @cases.mark_specifications("Modern")
     def testWhoNickAway(self, mask):
+        if "*" in mask and self.controller.software_name == "Bahamut":
+            raise runner.OptionalExtensionNotSupported("WHO mask")
+
         self._init()
 
         self.sendLine(1, "AWAY :be right back")
@@ -211,6 +230,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase, cases.OptionalityHe
     )
     @cases.mark_specifications("Modern")
     def testWhoNickOper(self, mask):
+        if "*" in mask and self.controller.software_name == "Bahamut":
+            raise runner.OptionalExtensionNotSupported("WHO mask")
+
         self._init()
 
         self.sendLine(1, "OPER operuser operpassword")
@@ -242,6 +264,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase, cases.OptionalityHe
     )
     @cases.mark_specifications("Modern")
     def testWhoNickAwayAndOper(self, mask):
+        if "*" in mask and self.controller.software_name == "Bahamut":
+            raise runner.OptionalExtensionNotSupported("WHO mask")
+
         self._init()
 
         self.sendLine(1, "OPER operuser operpassword")
@@ -273,6 +298,9 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase, cases.OptionalityHe
     @pytest.mark.parametrize("mask", ["#chan", "#CHAN"], ids=["exact", "casefolded"])
     @cases.mark_specifications("Modern")
     def testWhoChan(self, mask):
+        if "*" in mask and self.controller.software_name == "Bahamut":
+            raise runner.OptionalExtensionNotSupported("WHO mask")
+
         self._init()
 
         self.sendLine(1, "OPER operuser operpassword")
@@ -415,9 +443,7 @@ class WhoTestCase(BaseWhoTestCase, cases.BaseServerTestCase, cases.OptionalityHe
 
 
 @cases.mark_services
-class WhoServicesTestCase(
-    BaseWhoTestCase, cases.BaseServerTestCase, cases.OptionalityHelper
-):
+class WhoServicesTestCase(BaseWhoTestCase, cases.BaseServerTestCase):
     @cases.mark_specifications("IRCv3")
     @cases.mark_isupport("WHOX")
     def testWhoxAccount(self):

irctest/server_tests/whois.py

@@ -1,3 +1,9 @@
+"""
+The WHOIS command  (`Modern <https://modern.ircdocs.horse/#whois-message>`__)
+
+TODO: cross-reference RFC 1459 and RFC 2812
+"""
+
 import pytest
 
 from irctest import cases
@@ -23,6 +29,9 @@ from irctest.patma import ANYSTR, StrRe
 
 class _WhoisTestMixin(cases.BaseServerTestCase):
     def _testWhoisNumerics(self, authenticate, away, oper):
+        if oper and self.controller.software_name == "Charybdis":
+            pytest.xfail("charybdis uses RPL_WHOISSPECIAL instead of RPL_WHOISOPERATOR")
+
         if authenticate:
             self.connectClient("nick1")
             self.controller.registerUser(self, "val", "sesame")
@@ -158,7 +167,7 @@ class _WhoisTestMixin(cases.BaseServerTestCase):
         )
 
 
-class WhoisTestCase(_WhoisTestMixin, cases.BaseServerTestCase, cases.OptionalityHelper):
+class WhoisTestCase(_WhoisTestMixin, cases.BaseServerTestCase):
     @pytest.mark.parametrize(
         "server",
         ["", "My.Little.Server", "coolNick"],
@@ -204,11 +213,9 @@ class WhoisTestCase(_WhoisTestMixin, cases.BaseServerTestCase, cases.Optionality
 
 
 @cases.mark_services
-class ServicesWhoisTestCase(
-    _WhoisTestMixin, cases.BaseServerTestCase, cases.OptionalityHelper
-):
+class ServicesWhoisTestCase(_WhoisTestMixin, cases.BaseServerTestCase):
     @pytest.mark.parametrize("oper", [False, True], ids=["normal", "oper"])
-    @cases.OptionalityHelper.skipUnlessHasMechanism("PLAIN")
+    @cases.skipUnlessHasMechanism("PLAIN")
     @cases.mark_specifications("Modern")
     def testWhoisNumerics(self, oper):
         """Tests all numerics are in the exhaustive list defined in the Modern spec,
@@ -291,7 +298,7 @@ class ServicesWhoisTestCase(
             "RPL_WHOISCHANNELS should be sent for a non-invisible nick",
         )
 
-    @cases.OptionalityHelper.skipUnlessHasMechanism("PLAIN")
+    @cases.skipUnlessHasMechanism("PLAIN")
     @cases.mark_specifications("ircdocs")
     def testWhoisAccount(self):
         """Test numeric 330, RPL_WHOISACCOUNT.

irctest/server_tests/whowas.py

@@ -1,8 +1,19 @@
+"""
+The WHOSWAS command  (`RFC 1459
+<https://datatracker.ietf.org/doc/html/rfc1459#section-4.5.3>`__,
+`RFC 2812 <https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3>`__,
+`Modern <https://modern.ircdocs.horse/#whowas-message>`__)
+
+TODO: cross-reference Modern
+"""
+
+
 import pytest
 
 from irctest import cases, runner
 from irctest.exceptions import ConnectionClosed
 from irctest.numerics import (
+    ERR_NEEDMOREPARAMS,
     ERR_NONICKNAMEGIVEN,
     ERR_WASNOSUCHNICK,
     RPL_ENDOFWHOWAS,
@@ -78,6 +89,43 @@ class WhowasTestCase(cases.BaseServerTestCase):
             unexpected_messages, [], fail_msg="Unexpected numeric messages: {got}"
         )
 
+    @cases.mark_specifications("RFC1459", "RFC2812", "Modern")
+    def testWhowasEnd(self):
+        """
+        "At the end of all reply batches, there must be RPL_ENDOFWHOWAS"
+        -- https://datatracker.ietf.org/doc/html/rfc1459#page-50
+        -- https://datatracker.ietf.org/doc/html/rfc2812#page-45
+
+        "Servers MUST reply with either ERR_WASNOSUCHNICK or [...],
+        both followed with RPL_ENDOFWHOWAS"
+        -- https://github.com/ircdocs/modern-irc/pull/170
+        """
+        self.connectClient("nick1")
+
+        self.connectClient("nick2")
+        self.sendLine(2, "QUIT :bye")
+        try:
+            self.getMessages(2)
+        except ConnectionClosed:
+            pass
+
+        self.sendLine(1, "WHOWAS nick2")
+
+        messages = []
+        for _ in range(10):
+            messages.extend(self.getMessages(1))
+            if RPL_ENDOFWHOWAS in (m.command for m in messages):
+                break
+
+        last_message = messages.pop()
+
+        self.assertMessageMatch(
+            last_message,
+            command=RPL_ENDOFWHOWAS,
+            params=["nick1", "nick2", ANYSTR],
+            fail_msg=f"Last message was not RPL_ENDOFWHOWAS ({RPL_ENDOFWHOWAS})",
+        )
+
     def _testWhowasMultiple(self, second_result, whowas_command):
         """
         "The history is searched backward, returning the most recent entry first."
@@ -152,50 +200,78 @@ class WhowasTestCase(cases.BaseServerTestCase):
             fail_msg=f"Last message was not RPL_ENDOFWHOWAS ({RPL_ENDOFWHOWAS})",
         )
 
-    @cases.mark_specifications("RFC1459", "RFC2812")
+    @cases.mark_specifications("RFC1459", "RFC2812", "Modern")
+    @cases.xfailIfSoftware(
+        ["InspIRCd"],
+        "Feature not released yet: https://github.com/inspircd/inspircd/pull/1967",
+    )
     def testWhowasMultiple(self):
         """
         "The history is searched backward, returning the most recent entry first."
         -- https://datatracker.ietf.org/doc/html/rfc1459#section-4.5.3
         -- https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3
+        -- https://github.com/ircdocs/modern-irc/pull/170
         """
         self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS nick2")
 
-    @cases.mark_specifications("RFC1459", "RFC2812")
+    @cases.mark_specifications("RFC1459", "RFC2812", "Modern")
+    @cases.xfailIfSoftware(
+        ["InspIRCd"],
+        "Feature not released yet: https://github.com/inspircd/inspircd/pull/1968",
+    )
     def testWhowasCount1(self):
         """
         "If there are multiple entries, up to <count> replies will be returned"
         -- https://datatracker.ietf.org/doc/html/rfc1459#section-4.5.3
         -- https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3
+        -- https://github.com/ircdocs/modern-irc/pull/170
         """
         self._testWhowasMultiple(second_result=False, whowas_command="WHOWAS nick2 1")
 
-    @cases.mark_specifications("RFC1459", "RFC2812")
+    @cases.mark_specifications("RFC1459", "RFC2812", "Modern")
+    @cases.xfailIfSoftware(
+        ["InspIRCd"],
+        "Feature not released yet: https://github.com/inspircd/inspircd/pull/1968",
+    )
     def testWhowasCount2(self):
         """
         "If there are multiple entries, up to <count> replies will be returned"
         -- https://datatracker.ietf.org/doc/html/rfc1459#section-4.5.3
         -- https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3
+        -- https://github.com/ircdocs/modern-irc/pull/170
         """
         self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS nick2 2")
 
-    @cases.mark_specifications("RFC1459", "RFC2812")
+    @cases.mark_specifications("RFC1459", "RFC2812", "Modern")
+    @cases.xfailIfSoftware(
+        ["InspIRCd"],
+        "Feature not released yet: https://github.com/inspircd/inspircd/pull/1968",
+    )
     def testWhowasCountNegative(self):
         """
         "If a non-positive number is passed as being <count>, then a full search
         is done."
         -- https://datatracker.ietf.org/doc/html/rfc1459#section-4.5.3
         -- https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3
+        -- https://github.com/ircdocs/modern-irc/pull/170
         """
         self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS nick2 -1")
 
-    @cases.mark_specifications("RFC1459", "RFC2812")
+    @cases.mark_specifications("RFC1459", "RFC2812", "Modern")
+    @cases.xfailIfSoftware(
+        ["ircu2"], "Fix not released yet: https://github.com/UndernetIRC/ircu2/pull/19"
+    )
+    @cases.xfailIfSoftware(
+        ["InspIRCd"],
+        "Feature not released yet: https://github.com/inspircd/inspircd/pull/1967",
+    )
     def testWhowasCountZero(self):
         """
         "If a non-positive number is passed as being <count>, then a full search
         is done."
         -- https://datatracker.ietf.org/doc/html/rfc1459#section-4.5.3
         -- https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3
+        -- https://github.com/ircdocs/modern-irc/pull/170
         """
         self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS nick2 0")
 
@@ -204,11 +280,15 @@ class WhowasTestCase(cases.BaseServerTestCase):
         """
         "Wildcards are allowed in the <target> parameter."
         -- https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3
+        -- https://github.com/ircdocs/modern-irc/pull/170
         """
+        if self.controller.software_name == "Bahamut":
+            raise runner.OptionalExtensionNotSupported("WHOWAS mask")
+
         self._testWhowasMultiple(second_result=True, whowas_command="WHOWAS *ck2")
 
     @cases.mark_specifications("RFC1459", "RFC2812", deprecated=True)
-    def testWhowasNoParam(self):
+    def testWhowasNoParamRfc(self):
         """
         https://datatracker.ietf.org/doc/html/rfc1459#section-4.5.3
         https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3
@@ -239,11 +319,46 @@ class WhowasTestCase(cases.BaseServerTestCase):
             params=["nick1", "nick2", ANYSTR],
         )
 
-    @cases.mark_specifications("RFC1459", "RFC2812")
+    @cases.mark_specifications("Modern")
+    def testWhowasNoParamModern(self):
+        """
+        "If the `<nick>` argument is missing, they SHOULD send a single reply, using
+        either ERR_NONICKNAMEGIVEN or ERR_NEEDMOREPARAMS"
+        -- https://github.com/ircdocs/modern-irc/pull/170
+        """
+        # But no one seems to follow this. Most implementations use ERR_NEEDMOREPARAMS
+        # instead of ERR_NONICKNAMEGIVEN; and I couldn't find any that returns
+        # RPL_ENDOFWHOWAS either way.
+        self.connectClient("nick1")
+
+        self.sendLine(1, "WHOWAS")
+
+        m = self.getMessage(1)
+        if m.command == ERR_NONICKNAMEGIVEN:
+            self.assertMessageMatch(
+                m,
+                command=ERR_NONICKNAMEGIVEN,
+                params=["nick1", ANYSTR],
+            )
+        else:
+            self.assertMessageMatch(
+                m,
+                command=ERR_NEEDMOREPARAMS,
+                params=["nick1", "WHOWAS", ANYSTR],
+            )
+
+    @cases.mark_specifications("RFC1459", "RFC2812", "Modern")
+    @cases.xfailIfSoftware(
+        ["Charybdis"],
+        "fails because of a typo (solved in "
+        "https://github.com/solanum-ircd/solanum/commit/"
+        "08b7b6bd7e60a760ad47b58cbe8075b45d66166f)",
+    )
     def testWhowasNoSuchNick(self):
         """
         https://datatracker.ietf.org/doc/html/rfc1459#section-4.5.3
         https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3
+        -- https://github.com/ircdocs/modern-irc/pull/170
 
         and:
 
@@ -251,6 +366,12 @@ class WhowasTestCase(cases.BaseServerTestCase):
         (even if there was only one reply and it was an error)."
         -- https://datatracker.ietf.org/doc/html/rfc1459#page-50
         -- https://datatracker.ietf.org/doc/html/rfc2812#page-45
+
+        and:
+
+        "Servers MUST reply with either ERR_WASNOSUCHNICK or [...],
+        both followed with RPL_ENDOFWHOWAS"
+        -- https://github.com/ircdocs/modern-irc/pull/170
         """
         self.connectClient("nick1")
 
@@ -275,15 +396,15 @@ class WhowasTestCase(cases.BaseServerTestCase):
         """
         https://datatracker.ietf.org/doc/html/rfc2812#section-3.6.3
         """
+        if self.controller.software_name == "Bahamut":
+            pytest.xfail(
+                "Bahamut returns entries in query order instead of chronological order"
+            )
+
         self.connectClient("nick1")
 
-        targmax = dict(
-            item.split(":", 1)
-            for item in self.server_support.get("TARGMAX", "").split(",")
-            if item
-        )
-        if targmax.get("WHOWAS", "1") == "1":
-            raise runner.NotImplementedByController("Multi-target WHOWAS")
+        if self.targmax.get("WHOWAS", "1") == "1":
+            raise runner.OptionalExtensionNotSupported("Multi-target WHOWAS")
 
         self.connectClient("nick2", ident="ident2")
         self.sendLine(2, "QUIT :bye")

irctest/server_tests/znc_playback.py

@@ -1,3 +1,7 @@
+"""
+`Ergo <https://ergo.chat/>`_-specific tests of ZNC-like message playback
+"""
+
 import time
 
 from irctest import cases

irctest/specifications.py

@@ -50,6 +50,8 @@ class Capabilities(enum.Enum):
 @enum.unique
 class IsupportTokens(enum.Enum):
     BOT = "BOT"
+    ELIST = "ELIST"
+    INVEX = "INVEX"
     PREFIX = "PREFIX"
     MONITOR = "MONITOR"
     STATUSMSG = "STATUSMSG"

make_workflows.py

@@ -10,7 +10,6 @@ and keep them in sync.
 
 import enum
 import pathlib
-import textwrap
 
 import yaml
 
@@ -209,8 +208,8 @@ def get_test_job(*, config, test_config, test_id, version_flavor, jobs):
             *unpack,
             *install_steps,
             {
-                "name": "Install Atheme",
-                "run": "sudo apt-get install atheme-services",
+                "name": "Install system dependencies",
+                "run": "sudo apt-get install atheme-services faketime",
             },
             {
                 "name": "Install irctest dependencies",
@@ -226,6 +225,7 @@ def get_test_job(*, config, test_config, test_id, version_flavor, jobs):
             },
             {
                 "name": "Test with pytest",
+                "timeout-minutes": 30,
                 "run": (
                     f"PYTEST_ARGS='--junit-xml pytest.xml' "
                     f"PATH=$HOME/.local/bin:$PATH "
@@ -237,7 +237,7 @@ def get_test_job(*, config, test_config, test_id, version_flavor, jobs):
                 "if": "always()",
                 "uses": "actions/upload-artifact@v2",
                 "with": {
-                    "name": f"pytest results {test_id} ({version_flavor.value})",
+                    "name": f"pytest-results_{test_id}_{version_flavor.value}",
                     "path": "pytest.xml",
                 },
             },
@@ -351,7 +351,7 @@ def generate_workflow(config: dict, version_flavor: VersionFlavor):
             jobs[f"test-{test_id}"] = test_job
 
     jobs["publish-test-results"] = {
-        "name": "Publish Unit Tests Results",
+        "name": "Publish Dashboard",
         "needs": sorted({f"test-{test_id}" for test_id in config["tests"]} & set(jobs)),
         "runs-on": "ubuntu-latest",
         # the build-and-test job might be skipped, we don't need to run
@@ -365,32 +365,31 @@ def generate_workflow(config: dict, version_flavor: VersionFlavor):
                 "with": {"path": "artifacts"},
             },
             {
-                "name": "Publish Unit Test Results",
-                "uses": "actions/github-script@v4",
-                "if": "github.event_name == 'pull_request'",
-                "with": {
-                    "result-encoding": "string",
-                    "script": script(
-                        textwrap.dedent(
-                            """\
-                            let body = '';
-                            const options = {};
-                            options.listeners = {
-                                stdout: (data) => {
-                                    body += data.toString();
-                                }
-                            };
-                            await exec.exec('bash', ['-c', 'shopt -s globstar; python3 report.py artifacts/**/*.xml'], options);
-                            github.issues.createComment({
-                              issue_number: context.issue.number,
-                              owner: context.repo.owner,
-                              repo: context.repo.repo,
-                              body: body,
-                            });
-                            return body;
-                        """
-                        )
-                    ),
+                "name": "Install dashboard dependencies",
+                "run": script(
+                    "python -m pip install --upgrade pip",
+                    "pip install defusedxml docutils -r requirements.txt",
+                ),
+            },
+            {
+                "name": "Generate dashboard",
+                "run": script(
+                    "shopt -s globstar",
+                    "python3 -m irctest.dashboard.format dashboard/ artifacts/**/*.xml",
+                    "echo '/ /index.xhtml' > dashboard/_redirects",
+                ),
+            },
+            {
+                "name": "Install netlify-cli",
+                "run": "npm i -g netlify-cli",
+            },
+            {
+                "name": "Deploy to Netlify",
+                "run": "./.github/deploy_to_netlify.py",
+                "env": {
+                    "NETLIFY_SITE_ID": "${{ secrets.NETLIFY_SITE_ID }}",
+                    "NETLIFY_AUTH_TOKEN": "${{ secrets.NETLIFY_AUTH_TOKEN }}",
+                    "GITHUB_TOKEN": "${{ secrets.GITHUB_TOKEN }}",
                 },
             },
         ],

mypy.ini

@@ -12,6 +12,9 @@ disallow_untyped_defs = False
 [mypy-irctest.client_tests.*]
 disallow_untyped_defs = False
 
+[mypy-defusedxml.*]
+ignore_missing_imports = True
+
 [mypy-ecdsa]
 ignore_missing_imports = True
 

pytest.ini

@@ -32,6 +32,8 @@ markers =
 
     # isupport tokens
     BOT
+    ELIST
+    INVEX
     MONITOR
     PREFIX
     STATUSMSG
@@ -39,3 +41,6 @@ markers =
     WHOX
 
 python_classes = *TestCase Test*
+
+# Include stdout in pytest.xml files used by the dashboard.
+junit_logging = system-out

workflows.yml

@@ -159,32 +159,32 @@ software:
     irc2:
         name: irc2
         separate_build_job: false
-        install_steps:
-            stable:
-                - name: Get source code
-                  run: |-
-                      curl http://ftp.irc.org/ftp/irc/server/irc2.11.2p3.tgz | tar -zx
-                - name: Configure
-                  run: |-
-                      cd $GITHUB_WORKSPACE/irc2.11.2p3
-                      ./configure --prefix=$HOME/.local/
-                      cd x86*
-                      echo "#define CMDLINE_CONFIG/" >> config.h
-                      echo "#define DEFAULT_SPLIT_USERS 0" >> config.h
-                      echo "#define DEFAULT_SPLIT_SERVERS 0" >> config.h
-                      #echo "#undef LIST_ALIS_NOTE" >> config.h
-                      # TODO: find a better way to make it not fork...
-                      echo "#define fork() (0)" >> config.h
-                - name: Compile and install
-                  run: |-
-                      cd $GITHUB_WORKSPACE/irc2.11.2p3/x86*
-                      make -j 4 all
-                      make install
-                      mkdir -p $HOME/.local/bin
-                      cp $HOME/.local/sbin/ircd $HOME/.local/bin/ircd
+        repository : irc-archive/irc2-mirror
+        path: irc2.11.2p3
+        cache: true
+        refs:
+            stable: 59649f24c3a5c27bad5648b48774f27475bccfd3  #  irc2.11.2p3
             release: null
             devel: null
             devel_release: null
+        build_script: |
+            # Configure
+            cd $GITHUB_WORKSPACE/irc2.11.2p3
+            ./configure --prefix=$HOME/.local/
+            cd x86*
+            echo "#define CMDLINE_CONFIG/" >> config.h
+            echo "#define DEFAULT_SPLIT_USERS 0" >> config.h
+            echo "#define DEFAULT_SPLIT_SERVERS 0" >> config.h
+            #echo "#undef LIST_ALIS_NOTE" >> config.h
+            # TODO: find a better way to make it not fork...
+            echo "#define fork() (0)" >> config.h
+
+            # Compile and install
+            cd $GITHUB_WORKSPACE/irc2.11.2p3/x86*
+            make -j 4 all
+            make install
+            mkdir -p $HOME/.local/bin
+            cp $HOME/.local/sbin/ircd $HOME/.local/bin/ircd
 
     ircu2:
         name: ircu2
@@ -204,6 +204,23 @@ software:
             make -j 4
             make install
 
+    nefarious:
+        name: nefarious
+        repository: evilnet/nefarious2
+        refs:
+            stable: "985704168ecada12d9e53b46df6087ef9d9fb40b"
+            release: null
+            devel: "master"
+            devel_release: null
+        path: nefarious
+        separate_build_job: false
+        build_script: |
+            cd $GITHUB_WORKSPACE/nefarious
+            ./configure --prefix=$HOME/.local/ --enable-debug
+            make -j 4
+            make install
+            cp $GITHUB_WORKSPACE/data/nefarious/* $HOME/.local/lib
+
     ngircd:
         name: ngircd
         repository: ngircd/ngircd
@@ -250,8 +267,8 @@ software:
         name: UnrealIRCd 6
         repository: unrealircd/unrealircd
         refs:
-            stable: daa0c11f285c7123ba9fa2966dee2d1a17729f1e  # 6.0.2 + a few commits
-            release: 29fd2e772a6b4b9107daa4e3c237df454b055810  # 6.0.2
+            stable: cedd23ae9cdd5985ce16e9869cbdb808479c3fc4  # 6.0.3
+            release: cedd23ae9cdd5985ce16e9869cbdb808479c3fc4  # 6.0.3
             devel: unreal60_dev
             devel_release: null
         path: unrealircd
@@ -267,6 +284,8 @@ software:
             CFLAGS="-O0 -march=x86-64" CXXFLAGS="$CFLAGS" ./Config -quick
             make -j 4
             make install
+            # Prevent download of geoIP database on first startup
+            sed -i 's/loadmodule "geoip_classic";//' ~/.local/unrealircd/conf/modules.default.conf
 
     unrealircd-5:
         name: UnrealIRCd 5
@@ -358,16 +377,19 @@ tests:
     plexus4:
         software: [plexus4, anope]
 
-    # doesn't build because it can't find liblex for some reason
-    #snircd:
-    #    software: [snircd]
-
     irc2:
         software: [irc2]
 
     ircu2:
         software: [ircu2]
 
+    nefarious:
+        software: [nefarious]
+
+    # doesn't build because it can't find liblex for some reason
+    #snircd:
+    #    software: [snircd]
+
     unrealircd-5:
         software: [unrealircd-5]