From b18cc37a15c3e88fb8516f7dc8bc5ea7789040a2 Mon Sep 17 00:00:00 2001 From: Krytarik Raido Date: Fri, 9 Mar 2018 19:56:04 +0100 Subject: [PATCH] Encyclopedia: Port to SQLite 3 (with Matt Wheeler) --- Bantracker/cgi/bans.cgi | 1 + Encyclopedia/README.txt | 28 +++++++------- Encyclopedia/__init__.py | 10 +++-- Encyclopedia/config.py | 36 +++++++++--------- Encyclopedia/factoids.cgi | 36 ++++++++---------- Encyclopedia/plugin.py | 80 ++++++++++++++++++++------------------- Encyclopedia/test.py | 36 +++++++++--------- commoncgi.py | 2 +- 8 files changed, 116 insertions(+), 113 deletions(-) diff --git a/Bantracker/cgi/bans.cgi b/Bantracker/cgi/bans.cgi index 1398cde..02e6efc 100755 --- a/Bantracker/cgi/bans.cgi +++ b/Bantracker/cgi/bans.cgi @@ -17,6 +17,7 @@ import sys import time import urllib import ConfigParser +import sqlite CONFIG_FILENAME = "bantracker.conf" config = ConfigParser.RawConfigParser() diff --git a/Encyclopedia/README.txt b/Encyclopedia/README.txt index 27fcfa6..808218f 100644 --- a/Encyclopedia/README.txt +++ b/Encyclopedia/README.txt @@ -7,29 +7,29 @@ best, this example wil use myfactoids as name. Then create a directory to store your databases in (somewere in $botdir/data would be best). If you choose to enable this plugin during supybot-wizard the database will be created for you. If noy, you can create the database manually. -In the new directory create an SQLite2 database with the following command: +In the new directory create an SQLite 3 database with the following command: + +sqlite3 myfactoids.db -sqlite myfactoids.db Then copy/paste in the below 2 tables: CREATE TABLE facts ( - id INTEGER PRIMARY KEY, - author VARCHAR(100) NOT NULL, - name VARCHAR(20) NOT NULL, - added DATETIME, - value VARCHAR(200) NOT NULL, - popularity INTEGER NOT NULL DEFAULT 0 + id INTEGER PRIMARY KEY, + author TEXT NOT NULL, + name TEXT NOT NULL, + added TEXT NOT NULL, + value TEXT NOT NULL, + popularity INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE log ( - id INTEGER PRIMARY KEY, - author VARCHAR(100) NOT NULL, - name VARCHAR(20) NOT NULL, - added DATETIME, - oldvalue VARCHAR(200) NOT NULL + id INTEGER PRIMARY KEY, + author TEXT NOT NULL, + name TEXT NOT NULL, + added TEXT NOT NULL, + oldvalue TEXT NOT NULL ); - If you want to create more databases, repeat these last two steps. When the databases exist, you need to configure the bots to actually use them. diff --git a/Encyclopedia/__init__.py b/Encyclopedia/__init__.py index 106f45f..e806e7c 100644 --- a/Encyclopedia/__init__.py +++ b/Encyclopedia/__init__.py @@ -2,6 +2,7 @@ ### # Copyright (c) 2006-2007 Dennis Kaarsemaker # Copyright (c) 2008-2010 Terence Simpson +# Copyright (c) 2018- Krytarik Raido # # This program is free software; you can redistribute it and/or modify # it under the terms of version 2 of the GNU General Public License as @@ -22,12 +23,13 @@ funtionality has been moved to PackageInfo import supybot import supybot.world as world -__version__ = "2.3" -__author__ = supybot.Author("Terence Simpson", "tsimpson", "tsimpson@ubuntu.com") +__version__ = "2.4" +__author__ = supybot.Author("Krytarik Raido", "krytarik", "krytarik@tuxgarage.com") __contributors__ = { - supybot.Author("Dennis Kaarsemaker","Seveas","dennis@kaarsemaker.net"): ['Original Author'] + supybot.Author("Dennis Kaarsemaker", "Seveas", "dennis@kaarsemaker.net"): ['Original Author'], + supybot.Author("Terence Simpson", "tsimpson", "tsimpson@ubuntu.com"): ['Original Author'] } -__url__ = 'https://launchpad.net/ubuntu-bots/' +__url__ = 'https://launchpad.net/ubuntu-bots' import config reload(config) diff --git a/Encyclopedia/config.py b/Encyclopedia/config.py index fd30e2b..f8473b6 100644 --- a/Encyclopedia/config.py +++ b/Encyclopedia/config.py @@ -2,6 +2,7 @@ ### # Copyright (c) 2006-2007 Dennis Kaarsemaker # Copyright (c) 2008-2010 Terence Simpson +# Copyright (c) 2018- Krytarik Raido # # This program is free software; you can redistribute it and/or modify # it under the terms of version 2 of the GNU General Public License as @@ -21,7 +22,7 @@ def configure(advanced): from supybot.questions import yn, something, output from supybot.utils.str import format import os - import sqlite + import sqlite3 import re def anything(prompt, default=None): @@ -114,29 +115,29 @@ def configure(advanced): output("supybot.plugins.Encyclopedia.database will be set to %r" % db_file) Encyclopedia.database.setValue(db_dir) - if os.path.exists(os.path.join(db_dir, db_file + '.db')): + if os.path.exists(os.path.join(db_dir, '%s.db' % db_file)): return - con = sqlite.connect(os.path.join(db_dir, db_file + '.db')) + con = sqlite3.connect(os.path.join(db_dir, '%s.db' % db_file)) cur = con.cursor() try: cur.execute("""CREATE TABLE facts ( - id INTEGER PRIMARY KEY, - author VARCHAR(100) NOT NULL, - name VARCHAR(20) NOT NULL, - added DATETIME, - value VARCHAR(200) NOT NULL, - popularity INTEGER NOT NULL DEFAULT 0 -)""") -#""" + id INTEGER PRIMARY KEY, + author TEXT NOT NULL, + name TEXT NOT NULL, + added TEXT NOT NULL, + value TEXT NOT NULL, + popularity INTEGER NOT NULL DEFAULT 0 + )""") + cur.execute("""CREATE TABLE log ( - id INTEGER PRIMARY KEY, - author VARCHAR(100) NOT NULL, - name VARCHAR(20) NOT NULL, - added DATETIME, - oldvalue VARCHAR(200) NOT NULL -)""") + id INTEGER PRIMARY KEY, + author TEXT NOT NULL, + name TEXT NOT NULL, + added TEXT NOT NULL, + oldvalue TEXT NOT NULL + )""") except: con.rollback() @@ -144,7 +145,6 @@ def configure(advanced): else: con.commit() finally: - cur.close() con.close() Encyclopedia = conf.registerPlugin('Encyclopedia') diff --git a/Encyclopedia/factoids.cgi b/Encyclopedia/factoids.cgi index 428807d..9a3fe8c 100755 --- a/Encyclopedia/factoids.cgi +++ b/Encyclopedia/factoids.cgi @@ -1,7 +1,8 @@ #!/usr/bin/env python ### -# Copyright (c) 2006,2007 Dennis Kaarsemaker -# Copyright (c) 2008, 2009 Terence Simpson +# Copyright (c) 2006-2007 Dennis Kaarsemaker +# Copyright (c) 2008-2009 Terence Simpson +# Copyright (c) 2018- Krytarik Raido # # This program is free software; you can redistribute it and/or modify # it under the terms of version 2 of the GNU General Public License as @@ -18,6 +19,7 @@ import sys # This needs to be set to the location of the commoncgi.py file sys.path.append('/var/www/bot') from commoncgi import * +import sqlite3 ### Variables NUM_PER_PAGE=50.0 @@ -77,18 +79,12 @@ class Log: return self._added[:self._added.find('.')] return self._added -## SQLite function to check if a string contains a substring -## Because we can't pass % along to WHERE X LIKE %s -def contains(needle, haystack): - return needle.lower() in haystack.lower() and 1 or 0 - # Read POST if 'db' in form: database = form['db'].value if database not in databases: database = default_database -con = sqlite.connect(os.path.join(datadir, database + '.db')) -con.create_function('contains', 2, contains) +con = sqlite3.connect(os.path.join(datadir, '%s.db' % database)) cur = con.cursor() try: page = int(form['page'].value) @@ -107,26 +103,26 @@ if search: if not keys: keys = [''] query1 = "SELECT name, value, author, added, popularity FROM facts WHERE name NOT LIKE '%%-also' AND (" - query2 = "SELECT COUNT(name) FROM facts WHERE " + query2 = "SELECT COUNT(*) FROM facts WHERE " bogus = False for k in keys: - values.extend((k, k)) + values.extend(('%%%s%%' % k, '%%%s%%' % k)) if bogus: query1 += ' OR ' query2 += ' OR ' - query1 += "contains(%s, name) OR contains(%s, value)" - query2 += "contains(%s, name) OR contains(%s, value)" + query1 += 'name LIKE ? OR value LIKE ?' + query2 += 'name LIKE ? OR value LIKE ?' bogus=True - query1 += ') ORDER BY %s LIMIT %d, %d' % (order_by, NUM_PER_PAGE*page, NUM_PER_PAGE) - cur.execute(query1, values) + cur.execute(query1 + ') ORDER BY %s LIMIT %d, %d' % (order_by, NUM_PER_PAGE * page, NUM_PER_PAGE), values) factoids = [Factoid(*x) for x in cur.fetchall()] cur.execute(query2, values) total = cur.fetchall()[0][0] else: - cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE value NOT LIKE '%%' AND name NOT LIKE '%%-also' ORDER BY %s LIMIT %d, %d" % (order_by, page*NUM_PER_PAGE, NUM_PER_PAGE)) + cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE value NOT LIKE '%%' AND name NOT LIKE '%%-also' ORDER BY %s LIMIT %d, %d" % + (order_by, NUM_PER_PAGE * page, NUM_PER_PAGE)) factoids = [Factoid(*x) for x in cur.fetchall()] - cur.execute("""SELECT COUNT(*) FROM facts WHERE value NOT LIKE '%%'""") + cur.execute("SELECT COUNT(*) FROM facts WHERE value NOT LIKE '%%'") total = cur.fetchall()[0][0] # Pagination links @@ -168,14 +164,14 @@ def link(match): i = 0 for fact in factoids: name = fact.name - cur.execute("SELECT value FROM facts WHERE name = %s", fact.name + '-also') + cur.execute("SELECT value FROM facts WHERE name = ?", ('%s-also' % fact.name,)) more = cur.fetchall() if len(more): name += ' $hr$' + ' $hr$'.join([x[0] for x in more]) - cur.execute("SELECT name FROM facts WHERE value LIKE %s", ' ' + fact.name) + cur.execute("SELECT name FROM facts WHERE value = ?", (' %s' % fact.name,)) name += ' \n' + ' \n'.join([x[0] for x in cur.fetchall()]) data = [ q(x) for x in fact ] - cur.execute("SELECT author, added FROM log WHERE name = %s ORDER BY id DESC LIMIT 1", fact.name) + cur.execute("SELECT author, added FROM log WHERE name = ? ORDER BY id DESC LIMIT 1", (fact.name,)) edit = [Log(*x) for x in cur.fetchall()] # edit = [Log(author, added) for (author, added) in cur.fetchall()] if edit: diff --git a/Encyclopedia/plugin.py b/Encyclopedia/plugin.py index 484987e..90aafa7 100644 --- a/Encyclopedia/plugin.py +++ b/Encyclopedia/plugin.py @@ -2,6 +2,7 @@ ### # Copyright (c) 2006-2007 Dennis Kaarsemaker # Copyright (c) 2008-2010 Terence Simpson +# Copyright (c) 2018- Krytarik Raido # # This program is free software; you can redistribute it and/or modify # it under the terms of version 2 of the GNU General Public License as @@ -17,7 +18,7 @@ from supybot.commands import * import supybot.ircmsgs as ircmsgs import supybot.callbacks as callbacks -import sqlite, datetime, time, pytz +import sqlite3, datetime, time, pytz import supybot.registry as registry import supybot.ircdb as ircdb import supybot.conf as conf @@ -161,6 +162,13 @@ def checkUrl(s): return bool(urlRe.search(s)) +class DbWrapper(object): + def __init__(self, db, name): + self.db = db + self.name = name + self.time = time.time() + + class Encyclopedia(callbacks.Plugin): """!factoid: show factoid""" @@ -270,13 +278,12 @@ class Encyclopedia(callbacks.Plugin): db = self.registryValue('database',channel) if channel in self.databases: if self.databases[channel].time < time.time() - 3600 or self.databases[channel].name != db: - self.databases[channel].close() + self.databases[channel].db.close() self.databases.pop(channel) if channel not in self.databases: - self.databases[channel] = sqlite.connect(os.path.join(self.registryValue('datadir'), '%s.db' % db)) - self.databases[channel].name = db - self.databases[channel].time = time.time() - return self.databases[channel] + self.databases[channel] = DbWrapper( + sqlite3.connect(os.path.join(self.registryValue('datadir'), '%s.db' % db)), db) + return self.databases[channel].db def get_log_db(self, channel=None): db = "%s-log" % self.registryValue('database',channel) @@ -287,13 +294,11 @@ class Encyclopedia(callbacks.Plugin): channel = "%s-log" % channel if channel in self.databases: if self.databases[channel].time < time.time() - 3600 or self.databases[channel].name != db: - self.databases[channel].close() + self.databases[channel].db.close() self.databases.pop(channel) if channel not in self.databases: - self.databases[channel] = sqlite.connect(db_path) - self.databases[channel].name = db - self.databases[channel].time = time.time() - return self.databases[channel] + self.databases[channel] = DbWrapper(sqlite3.connect(db_path), db) + return self.databases[channel].db def addressed(self, recipients, text, irc, msg): nlen = len(irc.nick) @@ -346,9 +351,9 @@ class Encyclopedia(callbacks.Plugin): db = self.get_db(channel) cur = db.cursor() if deleted: - cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE name = %s", name) + cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE name = ?", (name,)) else: - cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE name = %s AND value NOT like '%%'", name) + cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE name = ? AND value NOT LIKE '%%'", (name,)) factoids = cur.fetchall() if len(factoids): f = factoids[0] @@ -371,20 +376,18 @@ class Encyclopedia(callbacks.Plugin): def factoid_info(self, channel, factoid): if not factoid: return + db = self.get_db(channel) + cur = db.cursor() if not factoid.value.startswith(''): # Try and find aliases - db = self.get_db(channel) - cur = db.cursor() - cur.execute("SELECT name FROM facts WHERE value = %s", ' ' + factoid.name) + cur.execute("SELECT name FROM facts WHERE value = ?", (' %s' % factoid.name,)) data = cur.fetchall() if data: factoid.value = " %s aliases: %s" % (factoid.name, ', '.join([x[0] for x in data])) else: - factoid.value = " %s has no aliases" % (factoid.name) + factoid.value = " %s has no aliases" % factoid.name # Author info - db = self.get_db(channel) - cur = db.cursor() - cur.execute("SELECT author, added FROM log WHERE name = %s", factoid.name) + cur.execute("SELECT author, added FROM log WHERE name = ?", (factoid.name,)) data = cur.fetchall() factoid.value += " - added by %s on %s" % (factoid.author[:factoid.author.find('!')], factoid.added[:factoid.added.find('.')]) if data: @@ -456,7 +459,7 @@ class Encyclopedia(callbacks.Plugin): if checkIgnored(msg.prefix,msg.args[0]): return # Are we being queried? - recipient, text = msg.args + recipient, text = msg.args[0], msg.args[1].decode('utf-8') if not self.registryValue('enabled', ircutils.isChannel(recipient) and recipient or None): # Encyclopedia is disabled here, do nothing return @@ -551,7 +554,7 @@ class Encyclopedia(callbacks.Plugin): retmsg = '' ret = self.registryValue('notfoundmsg') if ret.count('%') == ret.count('%s') == 1: - ret = ret % repr(text) + ret = ret % repr(text).lstrip('u') if channel.lower() == irc.nick.lower() or self.registryValue('privateNotFound', channel): myqueue(irc, msg.nick, ret) else: @@ -624,7 +627,7 @@ class Encyclopedia(callbacks.Plugin): factoid = retmsg = None def log_change(factoid): - cs.execute('''insert into log (author, name, added, oldvalue) values (%s, %s, %s, %s)''', + cs.execute("INSERT INTO log (author, name, added, oldvalue) VALUES (?, ?, ?, ?)", (editor, factoid.name, str(datetime.datetime.now(pytz.timezone("UTC"))), factoid.value)) db.commit() @@ -701,7 +704,7 @@ class Encyclopedia(callbacks.Plugin): ret = self.check_aliases(channel, factoid) if ret: return ret - cs.execute("UPDATE facts SET value=%s where name=%s", (factoid.value,factoid.name)) + cs.execute("UPDATE facts SET value = ? WHERE name = ?", (factoid.value, factoid.name)) db.commit() return retmsg @@ -725,7 +728,7 @@ class Encyclopedia(callbacks.Plugin): ret = self.check_aliases(channel, factoid) if ret: return ret - cs.execute("INSERT INTO facts (name, value, author, added) VALUES (%s, %s, %s, %s)", + cs.execute("INSERT INTO facts (name, value, author, added) VALUES (?, ?, ?, ?)", (name, value, editor, str(datetime.datetime.now(pytz.timezone("UTC"))))) db.commit() return "I'll remember that, %s" % editor[:editor.find('!')] @@ -767,7 +770,7 @@ class Encyclopedia(callbacks.Plugin): factoid = getattr(factoids,key) if (not display_info and not display_raw): cur = db.cursor() - cur.execute("UPDATE FACTS SET popularity = %d WHERE name = %s", factoid.popularity+1, factoid.name) + cur.execute("UPDATE facts SET popularity = ? WHERE name = ?", (factoid.popularity+1, factoid.name)) db.commit() if display_raw: ret.append(factoid.value) @@ -819,13 +822,13 @@ class Encyclopedia(callbacks.Plugin): return cur = db.cursor() now = str(datetime.datetime.now(pytz.timezone("UTC"))) - cur.execute("SELECT value FROM requests WHERE name = %s", name) + cur.execute("SELECT value FROM requests WHERE name = ?", (name,)) items = cur.fetchall() if len(items): for item in items: if item[0] == msg: return - cur.execute("INSERT INTO requests (type, name, value, oldval, who, date, rank) VALUES (%i, %s, %s, %s, %s, %s, 0)", + cur.execute("INSERT INTO requests (type, name, value, oldval, who, date, rank) VALUES (?, ?, ?, ?, ?, ?, 0)", (int(bool(tp)), name, msg, oldval, nick, now)) db.commit() @@ -835,8 +838,7 @@ class Encyclopedia(callbacks.Plugin): cur = db.cursor() ret = {} for k in keys: - k = k.replace("'","\'") - cur.execute("SELECT name,value FROM facts WHERE name LIKE '%%%s%%' OR VAlUE LIKE '%%%s%%'" % (k, k)) + cur.execute("SELECT name, value FROM facts WHERE name LIKE ? OR value LIKE ?", ('%%%s%%' % k, '%%%s%%' % k)) res = cur.fetchall() for r in res: val = r[1] @@ -883,11 +885,11 @@ class Encyclopedia(callbacks.Plugin): fd2.close() # Do some checking to make sure we have an SQLite database fd2 = open(tmp_db, 'rb') - data = fd2.read(47) - if data == '** This file contains an SQLite 2.1 database **': # OK, rename to dpath + check = fd2.read(15) + if check == 'SQLite format 3': # OK, rename to dpath os.rename(tmp_db, dpath) try: - self.databases[channel].close() + self.databases[channel].db.close() except: pass try: @@ -896,7 +898,7 @@ class Encyclopedia(callbacks.Plugin): pass else: # Remove the tmpparary file and raise an error os.remove(tmp_db) - raise RuntimeError, "Downloaded file was not a SQLite 2.1 database" + raise RuntimeError, "Downloaded file was not an SQLite 3 database" db = self.registryValue('database', channel) if not db: @@ -954,16 +956,16 @@ class Encyclopedia(callbacks.Plugin): if not author: author = msg.prefix def isLastEdit(name, id): - cur.execute("SELECT MAX(id) FROM log WHERE name=%s", (name,)) + cur.execute("SELECT max(id) FROM log WHERE name = ?", (name,)) return int(cur.fetchall()[0][0]) == id author = author.split('!', 1)[0] db = self.get_db(channel) cur = db.cursor() ret = {} log_ret = {} - cur.execute("SELECT name,value FROM facts WHERE author LIKE '%s%%'" % (author,)) + cur.execute("SELECT name, value FROM facts WHERE author LIKE ?", ('%s%%' % author,)) res = cur.fetchall() - cur.execute("SELECT id, name, oldvalue FROM log WHERE author LIKE '%s%%'" % (author,)) + cur.execute("SELECT id, name, oldvalue FROM log WHERE author LIKE ?", ('%s%%' % author,)) log_res = cur.fetchall() for r in res: val = r[1] @@ -1035,8 +1037,8 @@ class Encyclopedia(callbacks.Plugin): cur = db.cursor() sessid = hashlib.md5('%s%s%d' % (msg.prefix, time.time(), random.randint(1,100000))).hexdigest() - cur.execute("INSERT INTO sessions (session_id, user, time) VALUES (%s, %s, %d)", - (sessid, msg.nick, int(time.mktime(time.gmtime())) )) + cur.execute("INSERT INTO sessions (session_id, user, time) VALUES (?, ?, ?)", + (sessid, msg.nick, int(time.mktime(time.gmtime())))) db.commit() irc.reply("Login at http://jussi01.com/stdin/test/facts.cgi?sessid=%s" % sessid, private=True) diff --git a/Encyclopedia/test.py b/Encyclopedia/test.py index 5f8b1d4..47d9ddd 100644 --- a/Encyclopedia/test.py +++ b/Encyclopedia/test.py @@ -2,6 +2,7 @@ ### # Copyright (c) 2006 Dennis Kaarsemaker # Copyright (c) 2010 Elián Hanisch +# Copyright (c) 2018 Krytarik Raido # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -33,29 +34,30 @@ class EncyclopediaTestCase(ChannelPluginTestCase): self.createDB() def createDB(self): - import sqlite, os + import sqlite3, os dbfile = os.path.join(Econf.datadir(), '%s.db' %Econf.database()) try: os.remove(dbfile) except: pass - db = sqlite.connect(dbfile) - cursor = db.cursor() - cursor.execute('CREATE TABLE facts ('\ - 'id INTEGER PRIMARY KEY,'\ - 'author VARCHAR(100) NOT NULL,'\ - 'name VARCHAR(20) NOT NULL,'\ - 'added DATETIME,'\ - 'value VARCHAR(200) NOT NULL,'\ - 'popularity INTEGER NOT NULL DEFAULT 0);') - cursor.execute('CREATE TABLE log ('\ - 'id INTEGER PRIMARY KEY,'\ - 'author VARCHAR(100) NOT NULL,'\ - 'name VARCHAR(20) NOT NULL,'\ - 'added DATETIME,'\ - 'oldvalue VARCHAR(200) NOT NULL);') + db = sqlite3.connect(dbfile) + cur = db.cursor() + cur.execute("""CREATE TABLE facts ( + id INTEGER PRIMARY KEY, + author TEXT NOT NULL, + name TEXT NOT NULL, + added TEXT NOT NULL, + value TEXT NOT NULL, + popularity INTEGER NOT NULL DEFAULT 0 + )""") + cur.execute("""CREATE TABLE log ( + id INTEGER PRIMARY KEY, + author TEXT NOT NULL, + name TEXT NOT NULL, + added TEXT NOT NULL, + oldvalue TEXT NOT NULL + )""") db.commit() - cursor.close() db.close() self.getCallback().databases = {} diff --git a/commoncgi.py b/commoncgi.py index ea2497b..9d9f261 100644 --- a/commoncgi.py +++ b/commoncgi.py @@ -14,7 +14,7 @@ # ### -import cgi, cgitb, re, sys, math, os, hashlib, sqlite, random, time, datetime, pytz, Cookie, StringIO, urllib2 +import cgi, cgitb, re, sys, math, os, hashlib, random, time, datetime, pytz, Cookie, StringIO, urllib2 import cPickle as pickle cgitb.enable()