val
/
ubuntu-bots
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Activity

Encyclopedia: Port to SQLite 3 (with Matt Wheeler)

This commit is contained in:
Krytarik Raido 2018-03-09 19:56:04 +01:00
parent af7578e888
commit b18cc37a15
8 changed files with 116 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Bantracker/cgi/bans.cgi
View File

@ -17,6 +17,7 @@ import sys
import time
import urllib
import ConfigParser
import sqlite
CONFIG_FILENAME = "bantracker.conf"
config = ConfigParser.RawConfigParser()

22
Encyclopedia/README.txt
View File

@ -7,29 +7,29 @@ best, this example wil use myfactoids as name. Then create a directory to store
your databases in (somewere in $botdir/data would be best).
If you choose to enable this plugin during supybot-wizard the database will be
created for you. If noy, you can create the database manually.
In the new directory create an SQLite2 database with the following command:
In the new directory create an SQLite 3 database with the following command:
sqlite3 myfactoids.db
sqlite myfactoids.db
Then copy/paste in the below 2 tables:
CREATE TABLE facts (
id INTEGER PRIMARY KEY,
author VARCHAR(100) NOT NULL,
name VARCHAR(20) NOT NULL,
added DATETIME,
value VARCHAR(200) NOT NULL,
author TEXT NOT NULL,
name TEXT NOT NULL,
added TEXT NOT NULL,
value TEXT NOT NULL,
popularity INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE log (
id INTEGER PRIMARY KEY,
author VARCHAR(100) NOT NULL,
name VARCHAR(20) NOT NULL,
added DATETIME,
oldvalue VARCHAR(200) NOT NULL
author TEXT NOT NULL,
name TEXT NOT NULL,
added TEXT NOT NULL,
oldvalue TEXT NOT NULL
);
If you want to create more databases, repeat these last two steps.
When the databases exist, you need to configure the bots to actually use them.

10
Encyclopedia/__init__.py
View File

@ -2,6 +2,7 @@
###
# Copyright (c) 2006-2007 Dennis Kaarsemaker
# Copyright (c) 2008-2010 Terence Simpson
# Copyright (c) 2018- Krytarik Raido
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
@ -22,12 +23,13 @@ funtionality has been moved to PackageInfo
import supybot
import supybot.world as world
__version__ = "2.3"
__author__ = supybot.Author("Terence Simpson", "tsimpson", "tsimpson@ubuntu.com")
__version__ = "2.4"
__author__ = supybot.Author("Krytarik Raido", "krytarik", "krytarik@tuxgarage.com")
__contributors__ = {
supybot.Author("Dennis Kaarsemaker","Seveas","dennis@kaarsemaker.net"): ['Original Author']
supybot.Author("Dennis Kaarsemaker", "Seveas", "dennis@kaarsemaker.net"): ['Original Author'],
supybot.Author("Terence Simpson", "tsimpson", "tsimpson@ubuntu.com"): ['Original Author']
}
__url__ = 'https://launchpad.net/ubuntu-bots/'
__url__ = 'https://launchpad.net/ubuntu-bots'
import config
reload(config)

26
Encyclopedia/config.py
View File

@ -2,6 +2,7 @@
###
# Copyright (c) 2006-2007 Dennis Kaarsemaker
# Copyright (c) 2008-2010 Terence Simpson
# Copyright (c) 2018- Krytarik Raido
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
@ -21,7 +22,7 @@ def configure(advanced):
from supybot.questions import yn, something, output
from supybot.utils.str import format
import os
import sqlite
import sqlite3
import re
def anything(prompt, default=None):
@ -114,28 +115,28 @@ def configure(advanced):
output("supybot.plugins.Encyclopedia.database will be set to %r" % db_file)
Encyclopedia.database.setValue(db_dir)
if os.path.exists(os.path.join(db_dir, db_file + '.db')):
if os.path.exists(os.path.join(db_dir, '%s.db' % db_file)):
return
con = sqlite.connect(os.path.join(db_dir, db_file + '.db'))
con = sqlite3.connect(os.path.join(db_dir, '%s.db' % db_file))
cur = con.cursor()
try:
cur.execute("""CREATE TABLE facts (
id INTEGER PRIMARY KEY,
author VARCHAR(100) NOT NULL,
name VARCHAR(20) NOT NULL,
added DATETIME,
value VARCHAR(200) NOT NULL,
author TEXT NOT NULL,
name TEXT NOT NULL,
added TEXT NOT NULL,
value TEXT NOT NULL,
popularity INTEGER NOT NULL DEFAULT 0
)""")
#"""
cur.execute("""CREATE TABLE log (
id INTEGER PRIMARY KEY,
author VARCHAR(100) NOT NULL,
name VARCHAR(20) NOT NULL,
added DATETIME,
oldvalue VARCHAR(200) NOT NULL
author TEXT NOT NULL,
name TEXT NOT NULL,
added TEXT NOT NULL,
oldvalue TEXT NOT NULL
)""")
except:
@ -144,7 +145,6 @@ def configure(advanced):
else:
con.commit()
finally:
cur.close()
con.close()
Encyclopedia = conf.registerPlugin('Encyclopedia')

36
Encyclopedia/factoids.cgi
View File

@ -1,7 +1,8 @@
#!/usr/bin/env python
###
# Copyright (c) 2006,2007 Dennis Kaarsemaker
# Copyright (c) 2008, 2009 Terence Simpson
# Copyright (c) 2006-2007 Dennis Kaarsemaker
# Copyright (c) 2008-2009 Terence Simpson
# Copyright (c) 2018- Krytarik Raido
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
@ -18,6 +19,7 @@ import sys
# This needs to be set to the location of the commoncgi.py file
sys.path.append('/var/www/bot')
from commoncgi import *
import sqlite3
### Variables
NUM_PER_PAGE=50.0
@ -77,18 +79,12 @@ class Log:
return self._added[:self._added.find('.')]
return self._added
## SQLite function to check if a string contains a substring
## Because we can't pass % along to WHERE X LIKE %s
def contains(needle, haystack):
return needle.lower() in haystack.lower() and 1 or 0
# Read POST
if 'db' in form:
database = form['db'].value
if database not in databases:
database = default_database
con = sqlite.connect(os.path.join(datadir, database + '.db'))
con.create_function('contains', 2, contains)
con = sqlite3.connect(os.path.join(datadir, '%s.db' % database))
cur = con.cursor()
try: page = int(form['page'].value)
@ -107,26 +103,26 @@ if search:
if not keys:
keys = ['']
query1 = "SELECT name, value, author, added, popularity FROM facts WHERE name NOT LIKE '%%-also' AND ("
query2 = "SELECT COUNT(name) FROM facts WHERE "
query2 = "SELECT COUNT(*) FROM facts WHERE "
bogus = False
for k in keys:
values.extend((k, k))
values.extend(('%%%s%%' % k, '%%%s%%' % k))
if bogus:
query1 += ' OR '
query2 += ' OR '
query1 += "contains(%s, name) OR contains(%s, value)"
query2 += "contains(%s, name) OR contains(%s, value)"
query1 += 'name LIKE ? OR value LIKE ?'
query2 += 'name LIKE ? OR value LIKE ?'
bogus=True
query1 += ') ORDER BY %s LIMIT %d, %d' % (order_by, NUM_PER_PAGE*page, NUM_PER_PAGE)
cur.execute(query1, values)
cur.execute(query1 + ') ORDER BY %s LIMIT %d, %d' % (order_by, NUM_PER_PAGE * page, NUM_PER_PAGE), values)
factoids = [Factoid(*x) for x in cur.fetchall()]
cur.execute(query2, values)
total = cur.fetchall()[0][0]
else:
cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE value NOT LIKE '<alias>%%' AND name NOT LIKE '%%-also' ORDER BY %s LIMIT %d, %d" % (order_by, page*NUM_PER_PAGE, NUM_PER_PAGE))
cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE value NOT LIKE '<alias>%%' AND name NOT LIKE '%%-also' ORDER BY %s LIMIT %d, %d" %
(order_by, NUM_PER_PAGE * page, NUM_PER_PAGE))
factoids = [Factoid(*x) for x in cur.fetchall()]
cur.execute("""SELECT COUNT(*) FROM facts WHERE value NOT LIKE '<alias>%%'""")
cur.execute("SELECT COUNT(*) FROM facts WHERE value NOT LIKE '<alias>%%'")
total = cur.fetchall()[0][0]
# Pagination links
@ -168,14 +164,14 @@ def link(match):
i = 0
for fact in factoids:
name = fact.name
cur.execute("SELECT value FROM facts WHERE name = %s", fact.name + '-also')
cur.execute("SELECT value FROM facts WHERE name = ?", ('%s-also' % fact.name,))
more = cur.fetchall()
if len(more):
name += ' $hr$' + ' $hr$'.join([x[0] for x in more])
cur.execute("SELECT name FROM facts WHERE value LIKE %s", '<alias> ' + fact.name)
cur.execute("SELECT name FROM facts WHERE value = ?", ('<alias> %s' % fact.name,))
name += ' \n' + ' \n'.join([x[0] for x in cur.fetchall()])
data = [ q(x) for x in fact ]
cur.execute("SELECT author, added FROM log WHERE name = %s ORDER BY id DESC LIMIT 1", fact.name)
cur.execute("SELECT author, added FROM log WHERE name = ? ORDER BY id DESC LIMIT 1", (fact.name,))
edit = [Log(*x) for x in cur.fetchall()]
# edit = [Log(author, added) for (author, added) in cur.fetchall()]
if edit:

78
Encyclopedia/plugin.py
View File

@ -2,6 +2,7 @@
###
# Copyright (c) 2006-2007 Dennis Kaarsemaker
# Copyright (c) 2008-2010 Terence Simpson
# Copyright (c) 2018- Krytarik Raido
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
@ -17,7 +18,7 @@
from supybot.commands import *
import supybot.ircmsgs as ircmsgs
import supybot.callbacks as callbacks
import sqlite, datetime, time, pytz
import sqlite3, datetime, time, pytz
import supybot.registry as registry
import supybot.ircdb as ircdb
import supybot.conf as conf
@ -161,6 +162,13 @@ def checkUrl(s):
return bool(urlRe.search(s))
class DbWrapper(object):
def __init__(self, db, name):
self.db = db
self.name = name
self.time = time.time()
class Encyclopedia(callbacks.Plugin):
"""!factoid: show factoid"""
@ -270,13 +278,12 @@ class Encyclopedia(callbacks.Plugin):
db = self.registryValue('database',channel)
if channel in self.databases:
if self.databases[channel].time < time.time() - 3600 or self.databases[channel].name != db:
self.databases[channel].close()
self.databases[channel].db.close()
self.databases.pop(channel)
if channel not in self.databases:
self.databases[channel] = sqlite.connect(os.path.join(self.registryValue('datadir'), '%s.db' % db))
self.databases[channel].name = db
self.databases[channel].time = time.time()
return self.databases[channel]
self.databases[channel] = DbWrapper(
sqlite3.connect(os.path.join(self.registryValue('datadir'), '%s.db' % db)), db)
return self.databases[channel].db
def get_log_db(self, channel=None):
db = "%s-log" % self.registryValue('database',channel)
@ -287,13 +294,11 @@ class Encyclopedia(callbacks.Plugin):
channel = "%s-log" % channel
if channel in self.databases:
if self.databases[channel].time < time.time() - 3600 or self.databases[channel].name != db:
self.databases[channel].close()
self.databases[channel].db.close()
self.databases.pop(channel)
if channel not in self.databases:
self.databases[channel] = sqlite.connect(db_path)
self.databases[channel].name = db
self.databases[channel].time = time.time()
return self.databases[channel]
self.databases[channel] = DbWrapper(sqlite3.connect(db_path), db)
return self.databases[channel].db
def addressed(self, recipients, text, irc, msg):
nlen = len(irc.nick)
@ -346,9 +351,9 @@ class Encyclopedia(callbacks.Plugin):
db = self.get_db(channel)
cur = db.cursor()
if deleted:
cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE name = %s", name)
cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE name = ?", (name,))
else:
cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE name = %s AND value NOT like '<deleted>%%'", name)
cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE name = ? AND value NOT LIKE '<deleted>%%'", (name,))
factoids = cur.fetchall()
if len(factoids):
f = factoids[0]
@ -371,20 +376,18 @@ class Encyclopedia(callbacks.Plugin):
def factoid_info(self, channel, factoid):
if not factoid:
return
if not factoid.value.startswith('<alias>'):
# Try and find aliases
db = self.get_db(channel)
cur = db.cursor()
cur.execute("SELECT name FROM facts WHERE value = %s", '<alias> ' + factoid.name)
if not factoid.value.startswith('<alias>'):
# Try and find aliases
cur.execute("SELECT name FROM facts WHERE value = ?", ('<alias> %s' % factoid.name,))
data = cur.fetchall()
if data:
factoid.value = "<reply> %s aliases: %s" % (factoid.name, ', '.join([x[0] for x in data]))
else:
factoid.value = "<reply> %s has no aliases" % (factoid.name)
factoid.value = "<reply> %s has no aliases" % factoid.name
# Author info
db = self.get_db(channel)
cur = db.cursor()
cur.execute("SELECT author, added FROM log WHERE name = %s", factoid.name)
cur.execute("SELECT author, added FROM log WHERE name = ?", (factoid.name,))
data = cur.fetchall()
factoid.value += " - added by %s on %s" % (factoid.author[:factoid.author.find('!')], factoid.added[:factoid.added.find('.')])
if data:
@ -456,7 +459,7 @@ class Encyclopedia(callbacks.Plugin):
if checkIgnored(msg.prefix,msg.args[0]):
return
# Are we being queried?
recipient, text = msg.args
recipient, text = msg.args[0], msg.args[1].decode('utf-8')
if not self.registryValue('enabled', ircutils.isChannel(recipient) and recipient or None):
# Encyclopedia is disabled here, do nothing
return
@ -551,7 +554,7 @@ class Encyclopedia(callbacks.Plugin):
retmsg = ''
ret = self.registryValue('notfoundmsg')
if ret.count('%') == ret.count('%s') == 1:
ret = ret % repr(text)
ret = ret % repr(text).lstrip('u')
if channel.lower() == irc.nick.lower() or self.registryValue('privateNotFound', channel):
myqueue(irc, msg.nick, ret)
else:
@ -624,7 +627,7 @@ class Encyclopedia(callbacks.Plugin):
factoid = retmsg = None
def log_change(factoid):
cs.execute('''insert into log (author, name, added, oldvalue) values (%s, %s, %s, %s)''',
cs.execute("INSERT INTO log (author, name, added, oldvalue) VALUES (?, ?, ?, ?)",
(editor, factoid.name, str(datetime.datetime.now(pytz.timezone("UTC"))), factoid.value))
db.commit()
@ -701,7 +704,7 @@ class Encyclopedia(callbacks.Plugin):
ret = self.check_aliases(channel, factoid)
if ret:
return ret
cs.execute("UPDATE facts SET value=%s where name=%s", (factoid.value,factoid.name))
cs.execute("UPDATE facts SET value = ? WHERE name = ?", (factoid.value, factoid.name))
db.commit()
return retmsg
@ -725,7 +728,7 @@ class Encyclopedia(callbacks.Plugin):
ret = self.check_aliases(channel, factoid)
if ret:
return ret
cs.execute("INSERT INTO facts (name, value, author, added) VALUES (%s, %s, %s, %s)",
cs.execute("INSERT INTO facts (name, value, author, added) VALUES (?, ?, ?, ?)",
(name, value, editor, str(datetime.datetime.now(pytz.timezone("UTC")))))
db.commit()
return "I'll remember that, %s" % editor[:editor.find('!')]
@ -767,7 +770,7 @@ class Encyclopedia(callbacks.Plugin):
factoid = getattr(factoids,key)
if (not display_info and not display_raw):
cur = db.cursor()
cur.execute("UPDATE FACTS SET popularity = %d WHERE name = %s", factoid.popularity+1, factoid.name)
cur.execute("UPDATE facts SET popularity = ? WHERE name = ?", (factoid.popularity+1, factoid.name))
db.commit()
if display_raw:
ret.append(factoid.value)
@ -819,13 +822,13 @@ class Encyclopedia(callbacks.Plugin):
return
cur = db.cursor()
now = str(datetime.datetime.now(pytz.timezone("UTC")))
cur.execute("SELECT value FROM requests WHERE name = %s", name)
cur.execute("SELECT value FROM requests WHERE name = ?", (name,))
items = cur.fetchall()
if len(items):
for item in items:
if item[0] == msg:
return
cur.execute("INSERT INTO requests (type, name, value, oldval, who, date, rank) VALUES (%i, %s, %s, %s, %s, %s, 0)",
cur.execute("INSERT INTO requests (type, name, value, oldval, who, date, rank) VALUES (?, ?, ?, ?, ?, ?, 0)",
(int(bool(tp)), name, msg, oldval, nick, now))
db.commit()
@ -835,8 +838,7 @@ class Encyclopedia(callbacks.Plugin):
cur = db.cursor()
ret = {}
for k in keys:
k = k.replace("'","\'")
cur.execute("SELECT name,value FROM facts WHERE name LIKE '%%%s%%' OR VAlUE LIKE '%%%s%%'" % (k, k))
cur.execute("SELECT name, value FROM facts WHERE name LIKE ? OR value LIKE ?", ('%%%s%%' % k, '%%%s%%' % k))
res = cur.fetchall()
for r in res:
val = r[1]
@ -883,11 +885,11 @@ class Encyclopedia(callbacks.Plugin):
fd2.close()
# Do some checking to make sure we have an SQLite database
fd2 = open(tmp_db, 'rb')
data = fd2.read(47)
if data == '** This file contains an SQLite 2.1 database **': # OK, rename to dpath
check = fd2.read(15)
if check == 'SQLite format 3': # OK, rename to dpath
os.rename(tmp_db, dpath)
try:
self.databases[channel].close()
self.databases[channel].db.close()
except:
pass
try:
@ -896,7 +898,7 @@ class Encyclopedia(callbacks.Plugin):
pass
else: # Remove the tmpparary file and raise an error
os.remove(tmp_db)
raise RuntimeError, "Downloaded file was not a SQLite 2.1 database"
raise RuntimeError, "Downloaded file was not an SQLite 3 database"
db = self.registryValue('database', channel)
if not db:
@ -954,16 +956,16 @@ class Encyclopedia(callbacks.Plugin):
if not author:
author = msg.prefix
def isLastEdit(name, id):
cur.execute("SELECT MAX(id) FROM log WHERE name=%s", (name,))
cur.execute("SELECT max(id) FROM log WHERE name = ?", (name,))
return int(cur.fetchall()[0][0]) == id
author = author.split('!', 1)[0]
db = self.get_db(channel)
cur = db.cursor()
ret = {}
log_ret = {}
cur.execute("SELECT name,value FROM facts WHERE author LIKE '%s%%'" % (author,))
cur.execute("SELECT name, value FROM facts WHERE author LIKE ?", ('%s%%' % author,))
res = cur.fetchall()
cur.execute("SELECT id, name, oldvalue FROM log WHERE author LIKE '%s%%'" % (author,))
cur.execute("SELECT id, name, oldvalue FROM log WHERE author LIKE ?", ('%s%%' % author,))
log_res = cur.fetchall()
for r in res:
val = r[1]
@ -1035,7 +1037,7 @@ class Encyclopedia(callbacks.Plugin):
cur = db.cursor()
sessid = hashlib.md5('%s%s%d' % (msg.prefix, time.time(), random.randint(1,100000))).hexdigest()
cur.execute("INSERT INTO sessions (session_id, user, time) VALUES (%s, %s, %d)",
cur.execute("INSERT INTO sessions (session_id, user, time) VALUES (?, ?, ?)",
(sessid, msg.nick, int(time.mktime(time.gmtime()))))
db.commit()
irc.reply("Login at http://jussi01.com/stdin/test/facts.cgi?sessid=%s" % sessid, private=True)

36
Encyclopedia/test.py
View File

@ -2,6 +2,7 @@
###
# Copyright (c) 2006 Dennis Kaarsemaker
# Copyright (c) 2010 Elián Hanisch
# Copyright (c) 2018 Krytarik Raido
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -33,29 +34,30 @@ class EncyclopediaTestCase(ChannelPluginTestCase):
self.createDB()
def createDB(self):
import sqlite, os
import sqlite3, os
dbfile = os.path.join(Econf.datadir(), '%s.db' %Econf.database())
try:
os.remove(dbfile)
except:
pass
db = sqlite.connect(dbfile)
cursor = db.cursor()
cursor.execute('CREATE TABLE facts ('\
'id INTEGER PRIMARY KEY,'\
'author VARCHAR(100) NOT NULL,'\
'name VARCHAR(20) NOT NULL,'\
'added DATETIME,'\
'value VARCHAR(200) NOT NULL,'\
'popularity INTEGER NOT NULL DEFAULT 0);')
cursor.execute('CREATE TABLE log ('\
'id INTEGER PRIMARY KEY,'\
'author VARCHAR(100) NOT NULL,'\
'name VARCHAR(20) NOT NULL,'\
'added DATETIME,'\
'oldvalue VARCHAR(200) NOT NULL);')
db = sqlite3.connect(dbfile)
cur = db.cursor()
cur.execute("""CREATE TABLE facts (
id INTEGER PRIMARY KEY,
author TEXT NOT NULL,
name TEXT NOT NULL,
added TEXT NOT NULL,
value TEXT NOT NULL,
popularity INTEGER NOT NULL DEFAULT 0
)""")
cur.execute("""CREATE TABLE log (
id INTEGER PRIMARY KEY,
author TEXT NOT NULL,
name TEXT NOT NULL,
added TEXT NOT NULL,
oldvalue TEXT NOT NULL
)""")
db.commit()
cursor.close()
db.close()
self.getCallback().databases = {}

2
commoncgi.py
View File

@ -14,7 +14,7 @@
#
###
import cgi, cgitb, re, sys, math, os, hashlib, sqlite, random, time, datetime, pytz, Cookie, StringIO, urllib2
import cgi, cgitb, re, sys, math, os, hashlib, random, time, datetime, pytz, Cookie, StringIO, urllib2
import cPickle as pickle
cgitb.enable()