enhanced chathistory test

irctest/cases.py

@@ -11,6 +11,7 @@ from . import runner
 from . import client_mock
 from .irc_utils import capabilities
 from .irc_utils import message_parser
+from .irc_utils.sasl import sasl_plain_blob
 from .exceptions import ConnectionClosed
 from .specifications import Specifications
 
@@ -235,12 +236,15 @@ class BaseServerTestCase(_IrcTestCase):
     invalid_metadata_keys = frozenset()
     def setUp(self):
         super().setUp()
+        config = None
+        if hasattr(self, 'customizedConfig'):
+            config = self.customizedConfig()
         self.server_support = {}
         self.find_hostname_and_port()
         self.controller.run(self.hostname, self.port, password=self.password,
                 valid_metadata_keys=self.valid_metadata_keys,
                 invalid_metadata_keys=self.invalid_metadata_keys,
-                ssl=self.ssl)
+                ssl=self.ssl, config=config)
         self.clients = {}
     def tearDown(self):
         self.controller.kill()
@@ -324,7 +328,7 @@ class BaseServerTestCase(_IrcTestCase):
                 return result
 
     def connectClient(self, nick, name=None, capabilities=None,
-            skip_if_cap_nak=False, show_io=None):
+            skip_if_cap_nak=False, show_io=None, password=None):
         client = self.addClient(name, show_io=show_io)
         if capabilities is not None and 0 < len(capabilities):
             self.sendLine(client, 'CAP REQ :{}'.format(' '.join(capabilities)))
@@ -340,6 +344,9 @@ class BaseServerTestCase(_IrcTestCase):
                             ', '.join(capabilities))
                 else:
                     raise
+            if password is not None:
+                self.sendLine(client, 'AUTHENTICATE PLAIN')
+                self.sendLine(client, sasl_plain_blob(nick, password))
             self.sendLine(client, 'CAP END')
         self.sendLine(client, 'NICK {}'.format(nick))
         self.sendLine(client, 'USER username * * :Realname')

irctest/controllers/oragono.py

@@ -6,6 +6,8 @@ import subprocess
 from irctest.basecontrollers import NotImplementedByController
 from irctest.basecontrollers import BaseServerController, DirectoryBasedController
 
+OPER_PWD = 'frenchfries'
+
 BASE_CONFIG = {
     "network": {
         "name": "OragonoTest",
@@ -73,6 +75,47 @@ BASE_CONFIG = {
        "client-length": 128,
        "chathistory-maxmessages": 100,
    },
+
+    'oper-classes': {
+        'server-admin': {
+            'title': 'Server Admin',
+            'capabilities': [
+                 "oper:local_kill",
+                 "oper:local_ban",
+                 "oper:local_unban",
+                 "nofakelag",
+                 "oper:remote_kill",
+                 "oper:remote_ban",
+                 "oper:remote_unban",
+                 "oper:rehash",
+                 "oper:die",
+                 "accreg",
+                 "sajoin",
+                 "samode",
+                 "vhosts",
+                 "chanreg",
+            ],
+        },
+    },
+
+    'opers': {
+        'root': {
+            'class': 'server-admin',
+            'whois-line': 'is a server admin',
+            # OPER_PWD
+            'password': '$2a$04$3GzUZB5JapaAbwn7sogpOu9NSiLOgnozVllm2e96LiNPrm61ZsZSq',
+        },
+    },
+}
+
+LOGGING_CONFIG = {
+    "logging": [
+       {
+           "method": "stderr",
+           "level": "debug",
+           "type": "*",
+       },
+    ]
 }
 
 def hash_password(password):
@@ -95,16 +138,17 @@ class OragonoController(BaseServerController, DirectoryBasedController):
 
     def run(self, hostname, port, password=None, ssl=False,
             restricted_metadata_keys=None,
-            valid_metadata_keys=None, invalid_metadata_keys=None):
+            valid_metadata_keys=None, invalid_metadata_keys=None, config=None):
         if valid_metadata_keys or invalid_metadata_keys:
             raise NotImplementedByController(
                     'Defining valid and invalid METADATA keys.')
 
         self.create_config()
-        config = copy.deepcopy(BASE_CONFIG)
+        if config is None:
+            config = copy.deepcopy(BASE_CONFIG)
 
         self.port = port
-        bind_address = ":%s" % (port,)
+        bind_address = "127.0.0.1:%s" % (port,)
         listener_conf = None # plaintext
         if ssl:
             self.key_path = os.path.join(self.directory, 'ssl.key')
@@ -119,14 +163,15 @@ class OragonoController(BaseServerController, DirectoryBasedController):
 
         assert self.proc is None
 
-        with self.open_file('server.yml', 'w') as fd:
+        self._config_path = os.path.join(self.directory, 'server.yml')
-            json.dump(config, fd)
+        self._config = config
+        self._write_config()
         subprocess.call(['oragono', 'initdb',
-            '--conf', os.path.join(self.directory, 'server.yml'), '--quiet'])
+            '--conf', self._config_path, '--quiet'])
         subprocess.call(['oragono', 'mkcerts',
-            '--conf', os.path.join(self.directory, 'server.yml'), '--quiet'])
+            '--conf', self._config_path, '--quiet'])
         self.proc = subprocess.Popen(['oragono', 'run',
-            '--conf', os.path.join(self.directory, 'server.yml'), '--quiet'])
+            '--conf', self._config_path, '--quiet'])
 
     def registerUser(self, case, username, password=None):
         # XXX: Move this somewhere else when
@@ -146,5 +191,35 @@ class OragonoController(BaseServerController, DirectoryBasedController):
         case.sendLine(client, 'QUIT')
         case.assertDisconnected(client)
 
+    def _write_config(self):
+        with open(self._config_path, 'w') as fd:
+            json.dump(self._config, fd)
+
+    def baseConfig(self):
+        return copy.deepcopy(BASE_CONFIG)
+
+    def getConfig(self):
+        return copy.deepcopy(self._config)
+
+    def addLoggingToConfig(self, config):
+        config.update(LOGGING_CONFIG)
+        return config
+
+    def rehash(self, case, config):
+        self._config = config
+        self._write_config()
+        client = 'operator_for_rehash'
+        case.connectClient(nick=client, name=client)
+        case.sendLine(client, 'OPER root %s' % (OPER_PWD,))
+        case.sendLine(client, 'REHASH')
+        case.getMessages(client)
+        case.sendLine(client, 'QUIT')
+        case.assertDisconnected(client)
+
+    def enable_debug_logging(self, case):
+        config = self.getConfig()
+        config.update(LOGGING_CONFIG)
+        self.rehash(case, config)
+
 def get_irctest_controller_class():
     return OragonoController

irctest/server_tests/test_chathistory.py

@@ -9,10 +9,12 @@ from irctest import cases
 CHATHISTORY_CAP = 'draft/chathistory'
 EVENT_PLAYBACK_CAP = 'draft/event-playback'
 
-HistoryMessage = namedtuple('HistoryMessage', ['time', 'msgid', 'text'])
+HistoryMessage = namedtuple('HistoryMessage', ['time', 'msgid', 'target', 'text'])
+
+MYSQL_PASSWORD = ""
 
 def to_history_message(msg):
-    return HistoryMessage(time=msg.tags.get('time'), msgid=msg.tags.get('msgid'), text=msg.params[1])
+    return HistoryMessage(time=msg.tags.get('time'), msgid=msg.tags.get('msgid'), target=msg.params[0], text=msg.params[1])
 
 def validate_chathistory_batch(msgs):
     batch_tag = None
@@ -32,6 +34,13 @@ def validate_chathistory_batch(msgs):
 
 class ChathistoryTestCase(cases.BaseServerTestCase):
 
+    def validate_echo_messages(self, num_messages, echo_messages):
+        # sanity checks: should have received the correct number of echo messages,
+        # all with distinct time tags (because we slept) and msgids
+        self.assertEqual(len(echo_messages), num_messages)
+        self.assertEqual(len(set(msg.msgid for msg in echo_messages)), num_messages)
+        self.assertEqual(len(set(msg.time for msg in echo_messages)), num_messages)
+
     @cases.SpecificationSelector.requiredBySpecification('Oragono')
     def testChathistory(self):
         self.connectClient('bar', capabilities=['message-tags', 'server-time', 'echo-message', 'batch', 'labeled-response', CHATHISTORY_CAP, EVENT_PLAYBACK_CAP])
@@ -40,103 +49,166 @@ class ChathistoryTestCase(cases.BaseServerTestCase):
         self.getMessages(1)
 
         NUM_MESSAGES = 10
-        INCLUSIVE_LIMIT = NUM_MESSAGES * 2
         echo_messages = []
         for i in range(NUM_MESSAGES):
             self.sendLine(1, 'PRIVMSG %s :this is message %d' % (chname, i))
             echo_messages.extend(to_history_message(msg) for msg in self.getMessages(1))
             time.sleep(0.002)
-        # sanity checks: should have received the correct number of echo messages,
-        # all with distinct time tags (because we slept) and msgids
-        self.assertEqual(len(echo_messages), NUM_MESSAGES)
-        self.assertEqual(len(set(msg.msgid for msg in echo_messages)), NUM_MESSAGES)
-        self.assertEqual(len(set(msg.time for msg in echo_messages)), NUM_MESSAGES)
 
-        self.sendLine(1, "CHATHISTORY LATEST %s * %d" % (chname, INCLUSIVE_LIMIT))
+        self.validate_echo_messages(NUM_MESSAGES, echo_messages)
-        result = validate_chathistory_batch(self.getMessages(1))
+        self.validate_chathistory(echo_messages, 1, chname)
+
+    def customizedConfig(self):
+        if MYSQL_PASSWORD == "":
+            return None
+
+        # enable mysql-backed history for all channels and logged-in clients
+        config = self.controller.baseConfig()
+        config['datastore']['mysql'] = {
+           "enabled": True,
+           "host": "localhost",
+           "user": "oragono",
+           "password": MYSQL_PASSWORD,
+           "history-database": "oragono_history",
+        }
+        config['history']['persistent'] = {
+            "enabled": True,
+            "unregistered-channels": True,
+            "registered-channels": "opt-out",
+            "clients": "opt-out",
+        }
+        return config
+
+    @cases.SpecificationSelector.requiredBySpecification('Oragono')
+    def testChathistoryDMs(self):
+        c1 = secrets.token_hex(12)
+        c2 = secrets.token_hex(12)
+        self.controller.registerUser(self, c1, c1)
+        self.controller.registerUser(self, c2, c2)
+        self.connectClient(c1, capabilities=['message-tags', 'server-time', 'echo-message', 'batch', 'labeled-response', CHATHISTORY_CAP, EVENT_PLAYBACK_CAP], password=c1)
+        self.connectClient(c2, capabilities=['message-tags', 'server-time', 'echo-message', 'batch', 'labeled-response', CHATHISTORY_CAP, EVENT_PLAYBACK_CAP], password=c2)
+        self.getMessages(1)
+        self.getMessages(2)
+
+        NUM_MESSAGES = 10
+        echo_messages = []
+        for i in range(NUM_MESSAGES):
+            user = (i % 2) + 1
+            if user == 1:
+                target = c2
+            else:
+                target = c1
+            self.getMessages(user)
+            self.sendLine(user, 'PRIVMSG %s :this is message %d' % (target, i))
+            echo_messages.extend(to_history_message(msg) for msg in self.getMessages(user))
+            time.sleep(0.002)
+
+        self.validate_echo_messages(NUM_MESSAGES, echo_messages)
+        self.validate_chathistory(echo_messages, 1, c2)
+        self.validate_chathistory(echo_messages, 1, '*')
+        self.validate_chathistory(echo_messages, 2, c1)
+        self.validate_chathistory(echo_messages, 2, '*')
+
+        c3 = secrets.token_hex(12)
+        self.controller.registerUser(self, c3, c3)
+        self.connectClient(c3, capabilities=['message-tags', 'server-time', 'echo-message', 'batch', 'labeled-response', CHATHISTORY_CAP, EVENT_PLAYBACK_CAP], password=c3)
+        self.sendLine(1, 'PRIVMSG %s :this is a message in a separate conversation' % (c3,))
+        self.getMessages(1)
+        self.sendLine(3, 'PRIVMSG %s :i agree that this is a separate conversation' % (c1,))
+        self.getMessages(3)
+
+        # additional messages with c3 should not show up in the c1-c2 history:
+        self.validate_chathistory(echo_messages, 1, c2)
+        self.validate_chathistory(echo_messages, 2, c1)
+
+    def validate_chathistory(self, echo_messages, user, chname):
+        INCLUSIVE_LIMIT = len(echo_messages) * 2
+
+        self.sendLine(user, "CHATHISTORY LATEST %s * %d" % (chname, INCLUSIVE_LIMIT))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages, result)
 
-        self.sendLine(1, "CHATHISTORY LATEST %s * %d" % (chname, 5))
+        self.sendLine(user, "CHATHISTORY LATEST %s * %d" % (chname, 5))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[-5:], result)
 
-        self.sendLine(1, "CHATHISTORY LATEST %s * %d" % (chname, 1))
+        self.sendLine(user, "CHATHISTORY LATEST %s * %d" % (chname, 1))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[-1:], result)
 
-        self.sendLine(1, "CHATHISTORY LATEST %s msgid=%s %d" % (chname, echo_messages[4].msgid, INCLUSIVE_LIMIT))
+        self.sendLine(user, "CHATHISTORY LATEST %s msgid=%s %d" % (chname, echo_messages[4].msgid, INCLUSIVE_LIMIT))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[5:], result)
 
-        self.sendLine(1, "CHATHISTORY LATEST %s timestamp=%s %d" % (chname, echo_messages[4].time, INCLUSIVE_LIMIT))
+        self.sendLine(user, "CHATHISTORY LATEST %s timestamp=%s %d" % (chname, echo_messages[4].time, INCLUSIVE_LIMIT))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[5:], result)
 
-        self.sendLine(1, "CHATHISTORY BEFORE %s msgid=%s %d" % (chname, echo_messages[6].msgid, INCLUSIVE_LIMIT))
+        self.sendLine(user, "CHATHISTORY BEFORE %s msgid=%s %d" % (chname, echo_messages[6].msgid, INCLUSIVE_LIMIT))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[:6], result)
 
-        self.sendLine(1, "CHATHISTORY BEFORE %s timestamp=%s %d" % (chname, echo_messages[6].time, INCLUSIVE_LIMIT))
+        self.sendLine(user, "CHATHISTORY BEFORE %s timestamp=%s %d" % (chname, echo_messages[6].time, INCLUSIVE_LIMIT))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[:6], result)
 
-        self.sendLine(1, "CHATHISTORY BEFORE %s timestamp=%s %d" % (chname, echo_messages[6].time, 2))
+        self.sendLine(user, "CHATHISTORY BEFORE %s timestamp=%s %d" % (chname, echo_messages[6].time, 2))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[4:6], result)
 
-        self.sendLine(1, "CHATHISTORY AFTER %s msgid=%s %d" % (chname, echo_messages[3].msgid, INCLUSIVE_LIMIT))
+        self.sendLine(user, "CHATHISTORY AFTER %s msgid=%s %d" % (chname, echo_messages[3].msgid, INCLUSIVE_LIMIT))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[4:], result)
 
-        self.sendLine(1, "CHATHISTORY AFTER %s timestamp=%s %d" % (chname, echo_messages[3].time, INCLUSIVE_LIMIT))
+        self.sendLine(user, "CHATHISTORY AFTER %s timestamp=%s %d" % (chname, echo_messages[3].time, INCLUSIVE_LIMIT))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[4:], result)
 
-        self.sendLine(1, "CHATHISTORY AFTER %s timestamp=%s %d" % (chname, echo_messages[3].time, 3))
+        self.sendLine(user, "CHATHISTORY AFTER %s timestamp=%s %d" % (chname, echo_messages[3].time, 3))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[4:7], result)
 
         # BETWEEN forwards and backwards
-        self.sendLine(1, "CHATHISTORY BETWEEN %s msgid=%s msgid=%s %d" % (chname, echo_messages[0].msgid, echo_messages[-1].msgid, INCLUSIVE_LIMIT))
+        self.sendLine(user, "CHATHISTORY BETWEEN %s msgid=%s msgid=%s %d" % (chname, echo_messages[0].msgid, echo_messages[-1].msgid, INCLUSIVE_LIMIT))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[1:-1], result)
 
-        self.sendLine(1, "CHATHISTORY BETWEEN %s msgid=%s msgid=%s %d" % (chname, echo_messages[-1].msgid, echo_messages[0].msgid, INCLUSIVE_LIMIT))
+        self.sendLine(user, "CHATHISTORY BETWEEN %s msgid=%s msgid=%s %d" % (chname, echo_messages[-1].msgid, echo_messages[0].msgid, INCLUSIVE_LIMIT))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[1:-1], result)
 
         # BETWEEN forwards and backwards with a limit, should get different results this time
-        self.sendLine(1, "CHATHISTORY BETWEEN %s msgid=%s msgid=%s %d" % (chname, echo_messages[0].msgid, echo_messages[-1].msgid, 3))
+        self.sendLine(user, "CHATHISTORY BETWEEN %s msgid=%s msgid=%s %d" % (chname, echo_messages[0].msgid, echo_messages[-1].msgid, 3))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[1:4], result)
 
-        self.sendLine(1, "CHATHISTORY BETWEEN %s msgid=%s msgid=%s %d" % (chname, echo_messages[-1].msgid, echo_messages[0].msgid, 3))
+        self.sendLine(user, "CHATHISTORY BETWEEN %s msgid=%s msgid=%s %d" % (chname, echo_messages[-1].msgid, echo_messages[0].msgid, 3))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[-4:-1], result)
 
         # same stuff again but with timestamps
-        self.sendLine(1, "CHATHISTORY BETWEEN %s timestamp=%s timestamp=%s %d" % (chname, echo_messages[0].time, echo_messages[-1].time, INCLUSIVE_LIMIT))
+        self.sendLine(user, "CHATHISTORY BETWEEN %s timestamp=%s timestamp=%s %d" % (chname, echo_messages[0].time, echo_messages[-1].time, INCLUSIVE_LIMIT))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[1:-1], result)
-        self.sendLine(1, "CHATHISTORY BETWEEN %s timestamp=%s timestamp=%s %d" % (chname, echo_messages[-1].time, echo_messages[0].time, INCLUSIVE_LIMIT))
+        self.sendLine(user, "CHATHISTORY BETWEEN %s timestamp=%s timestamp=%s %d" % (chname, echo_messages[-1].time, echo_messages[0].time, INCLUSIVE_LIMIT))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[1:-1], result)
-        self.sendLine(1, "CHATHISTORY BETWEEN %s timestamp=%s timestamp=%s %d" % (chname, echo_messages[0].time, echo_messages[-1].time, 3))
+        self.sendLine(user, "CHATHISTORY BETWEEN %s timestamp=%s timestamp=%s %d" % (chname, echo_messages[0].time, echo_messages[-1].time, 3))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[1:4], result)
-        self.sendLine(1, "CHATHISTORY BETWEEN %s timestamp=%s timestamp=%s %d" % (chname, echo_messages[-1].time, echo_messages[0].time, 3))
+        self.sendLine(user, "CHATHISTORY BETWEEN %s timestamp=%s timestamp=%s %d" % (chname, echo_messages[-1].time, echo_messages[0].time, 3))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[-4:-1], result)
 
         # AROUND
-        self.sendLine(1, "CHATHISTORY AROUND %s msgid=%s %d" % (chname, echo_messages[7].msgid, 1))
+        self.sendLine(user, "CHATHISTORY AROUND %s msgid=%s %d" % (chname, echo_messages[7].msgid, 1))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual([echo_messages[7]], result)
 
-        self.sendLine(1, "CHATHISTORY AROUND %s msgid=%s %d" % (chname, echo_messages[7].msgid, 3))
+        self.sendLine(user, "CHATHISTORY AROUND %s msgid=%s %d" % (chname, echo_messages[7].msgid, 3))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertEqual(echo_messages[6:9], result)
 
-        self.sendLine(1, "CHATHISTORY AROUND %s timestamp=%s %d" % (chname, echo_messages[7].time, 3))
+        self.sendLine(user, "CHATHISTORY AROUND %s timestamp=%s %d" % (chname, echo_messages[7].time, 3))
-        result = validate_chathistory_batch(self.getMessages(1))
+        result = validate_chathistory_batch(self.getMessages(user))
         self.assertIn(echo_messages[7], result)