val/ubuntu-bots
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Activity

Encyclopedia: Restructure databases.

Browse Source 
* Add database conversion script.
This commit is contained in:
Krytarik Raido
2018-03-22 21:34:04 +01:00
parent 3368b2d6fe
commit 0cfbaa834e
7 changed files with 192 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
Encyclopedia/README.txt
View File

@ -11,7 +11,7 @@ In the new directory create an SQLite 3 database with the following command:
sqlite3 myfactoids.db
Then copy/paste in the below 2 tables:
Then copy/paste in the below 3 tables:
CREATE TABLE facts (
id INTEGER PRIMARY KEY,
@ -19,17 +19,29 @@ CREATE TABLE facts (
value TEXT NOT NULL,
author TEXT NOT NULL,
added TEXT NOT NULL,
editor TEXT,
edited TEXT,
popularity INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE log (
id INTEGER PRIMARY KEY,
type TEXT NOT NULL,
name TEXT NOT NULL,
oldvalue TEXT NOT NULL,
value TEXT NOT NULL,
author TEXT NOT NULL,
added TEXT NOT NULL
);
CREATE TABLE requests (
id INTEGER PRIMARY KEY,
type TEXT NOT NULL,
name TEXT NOT NULL,
value TEXT NOT NULL,
requester TEXT NOT NULL,
requested TEXT NOT NULL
);
If you want to create more databases, repeat these last two steps.
When the databases exist, you need to configure the bots to actually use them.

2
Encyclopedia/__init__.py
View File

@ -24,7 +24,7 @@ import supybot
import supybot.world as world
from imp import reload
__version__ = "2.6"
__version__ = "2.7"
__author__ = supybot.Author("Krytarik Raido", "krytarik", "krytarik@tuxgarage.com")
__contributors__ = {
supybot.Author("Dennis Kaarsemaker", "Seveas", "dennis@kaarsemaker.net"): ['Original Author'],

14
Encyclopedia/config.py
View File

@ -125,17 +125,29 @@ def configure(advanced):
value TEXT NOT NULL,
author TEXT NOT NULL,
added TEXT NOT NULL,
editor TEXT,
edited TEXT,
popularity INTEGER NOT NULL DEFAULT 0
)""")
cur.execute("""CREATE TABLE log (
id INTEGER PRIMARY KEY,
type TEXT NOT NULL,
name TEXT NOT NULL,
oldvalue TEXT NOT NULL,
value TEXT NOT NULL,
author TEXT NOT NULL,
added TEXT NOT NULL
)""")
cur.execute("""CREATE TABLE requests (
id INTEGER PRIMARY KEY,
type TEXT NOT NULL,
name TEXT NOT NULL,
value TEXT NOT NULL,
requester TEXT NOT NULL,
requested TEXT NOT NULL
)""")
except:
con.rollback()
raise

16
Encyclopedia/factoids.cgi
View File

@ -40,8 +40,9 @@ factoids = []
total = 0
class Factoid:
def __init__(self, name, value, author, added, popularity):
self.name, self.value, self.author, self.added, self.popularity = name, value, author, added, popularity
def __init__(self, name, value, author, added, editor, edited, popularity):
self.name, self.value, self.author, self.added, self.editor, self.edited, self.popularity = \
name, value, author, added, editor, edited, popularity
class Log:
def __init__(self, author, added):
@ -73,13 +74,13 @@ if search:
qterms += ' AND '
qterms += '(name LIKE ? OR value LIKE ? OR value LIKE ?)'
values.extend(['%%%s%%' % k.lower(), '%%%s%%' % k, '%%%s%%' % k.lower()])
cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE name NOT LIKE '%%-also' AND %s ORDER BY %s LIMIT %d, %d" %
cur.execute("SELECT name, value, author, added, editor, edited, popularity FROM facts WHERE name NOT LIKE '%%-also' AND %s ORDER BY %s LIMIT %d, %d" %
(qterms, order_by, NUM_PER_PAGE * (page - 1), NUM_PER_PAGE), values)
factoids = [Factoid(*x) for x in cur.fetchall()]
cur.execute("SELECT COUNT(*) FROM facts WHERE name NOT LIKE '%%-also' AND %s" % qterms, values)
total = cur.fetchall()[0][0]
else:
cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE value NOT LIKE '<alias>%%' AND name NOT LIKE '%%-also' ORDER BY %s LIMIT %d, %d" %
cur.execute("SELECT name, value, author, added, editor, edited, popularity FROM facts WHERE value NOT LIKE '<alias>%%' AND name NOT LIKE '%%-also' ORDER BY %s LIMIT %d, %d" %
(order_by, NUM_PER_PAGE * (page - 1), NUM_PER_PAGE))
factoids = [Factoid(*x) for x in cur.fetchall()]
cur.execute("SELECT COUNT(*) FROM facts WHERE value NOT LIKE '<alias>%%' AND name NOT LIKE '%%-also'")
@ -123,11 +124,8 @@ for fact in factoids:
cur.execute("SELECT value FROM facts WHERE name = ?", ('%s-also' % fact.name,))
value = ' '.join([fact.value] + [x[0] for x in cur.fetchall()])
data = ["Added by %s" % fact.author[:fact.author.find('!')], "Date: %s" % fact.added[:fact.added.rfind('.')]]
cur.execute("SELECT author, added FROM log WHERE name = ? ORDER BY id DESC LIMIT 1", (fact.name,))
edit = [Log(*x) for x in cur.fetchall()]
if edit:
log = edit[0]
data.extend(["Last edited by %s" % log.author[:log.author.find('!')], "Date: %s" % log.added[:log.added.rfind('.')]])
if fact.editor:
data.extend(["Last edited by %s" % fact.editor[:fact.editor.find('!')], "Date: %s" % fact.edited[:fact.edited.rfind('.')]])
data.append("Requested %s times" % fact.popularity)
print('''\

111
Encyclopedia/plugin.py
View File

@ -72,9 +72,11 @@ def checkAddressed(text, channel):
# Simple wrapper class for factoids
class Factoid:
def __init__(self, name, value, author=None, added=None, popularity=None):
def __init__(self, name, value, author=None, added=None,
editor=None, edited=None, popularity=None):
self.name = name; self.value = value
self.author = author; self.added = added
self.editor = editor; self.edited = edited
self.popularity = popularity
class FactoidSet:
@ -274,22 +276,6 @@ class Encyclopedia(callbacks.Plugin):
sqlite3.connect(os.path.join(self.registryValue('datadir'), '%s.db' % db)), db)
return self.databases[channel].db
def get_log_db(self, channel=None):
db = "%s-log" % self.registryValue('database', channel)
db_path = os.path.join(self.registryValue('datadir'), "%s.db" % db)
if not os.access(db_path, os.R_OK | os.W_OK):
self.log.warning("Encyclopedia: Could not access log database at '%s.db'" % db_path)
return
channel = "%s-log" % channel
if channel in self.databases \
and (self.databases[channel].time < time.time() - 3600 \
or self.databases[channel].name != db):
self.databases[channel].db.close()
self.databases.pop(channel)
if channel not in self.databases:
self.databases[channel] = DbWrapper(sqlite3.connect(db_path), db)
return self.databases[channel].db
def get_factoids(self, name, channel, display=None):
factoids = FactoidSet()
factoids.global_primary = self.get_single_factoid(channel, name, deleted=bool(display))
@ -329,9 +315,9 @@ class Encyclopedia(callbacks.Plugin):
db = self.get_db(channel)
cur = db.cursor()
if deleted:
cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE name = ?", (name.lower(),))
cur.execute("SELECT name, value, author, added, editor, edited, popularity FROM facts WHERE name = ?", (name.lower(),))
else:
cur.execute("SELECT name, value, author, added, popularity FROM facts WHERE name = ? AND value NOT LIKE '<deleted>%%'", (name.lower(),))
cur.execute("SELECT name, value, author, added, editor, edited, popularity FROM facts WHERE name = ? AND value NOT LIKE '<deleted>%%'", (name.lower(),))
factoids = cur.fetchall()
if factoids:
return Factoid(*factoids[0])
@ -368,11 +354,8 @@ class Encyclopedia(callbacks.Plugin):
ret.append("no aliases")
# Author info
ret.append("added by %s on %s" % (factoid.author[:factoid.author.find('!')], factoid.added[:factoid.added.rfind('.')]))
cur.execute("SELECT author, added FROM log WHERE name = ? ORDER BY id DESC LIMIT 1", (factoid.name,))
data = cur.fetchall()
if data:
editor, edited = data[0]
ret.append("last edited by %s on %s" % (editor[:editor.find('!')], edited[:edited.rfind('.')]))
if factoid.editor:
ret.append("last edited by %s on %s" % (factoid.editor[:factoid.editor.find('!')], factoid.edited[:factoid.edited.rfind('.')]))
# Popularity info
ret.append("requested %d times" % factoid.popularity)
return ' - '.join(ret)
@ -524,12 +507,12 @@ class Encyclopedia(callbacks.Plugin):
irc.reply("Your edit request has been forwarded to %s. " % relaychan
+ "Thank you for your attention to detail", private=True)
irc.reply("In %s, %s said: %s" % (msg.args[0], msg.nick, msg.args[1]), to=relaychan)
self.log_request(check[0], check[1], check[2], channel, msg.prefix)
self.log_request(check[0], check[1], channel, msg.prefix)
return
else:
queue_msg = False
getattr(self, 'factoid_%s' % edit)(check[1], check[2], channel, msg.prefix)
ret = check[3]
getattr(self, 'factoid_%s' % edit)(check[1], channel, msg.prefix)
ret = check[2]
if not ret:
if len(text.split()) > 4:
@ -591,8 +574,9 @@ class Encyclopedia(callbacks.Plugin):
for s in L:
irc.reply(s, to=nick)
def factoid_delete(self, factoid, newvalue, channel, editor):
log_edit(factoid, channel, editor)
def factoid_delete(self, factoid, channel, editor):
edited = str(datetime.datetime.utcnow())
self.log_edit('delete', factoid, channel, editor, edited)
db = self.get_db(channel)
cs = db.cursor()
cs.execute("DELETE FROM facts WHERE name = ?", (factoid.name,))
@ -609,13 +593,15 @@ class Encyclopedia(callbacks.Plugin):
ret = self.check_aliases(channel, factoid)
if ret:
return ret
return 'delete', factoid, None, retmsg
return 'delete', factoid, retmsg
def factoid_edit(self, factoid, newvalue, channel, editor):
log_edit(factoid, channel, editor)
def factoid_edit(self, factoid, channel, editor):
edited = str(datetime.datetime.utcnow())
self.log_edit('edit', factoid, channel, editor, edited)
db = self.get_db(channel)
cs = db.cursor()
cs.execute("UPDATE facts SET value = ? WHERE name = ?", (newvalue, factoid.name))
cs.execute("UPDATE facts SET value = ?, editor = ?, edited = ? WHERE name = ?",
(factoid.value, editor, edited, factoid.name))
db.commit()
def factoid_edit_check(self, text, channel, editor):
@ -631,6 +617,7 @@ class Encyclopedia(callbacks.Plugin):
return "I know nothing about '%s' yet, %s" % (name, editor)
if value == factoid.value:
return "Nothing changed there"
factoid.value = value
retmsg = "I'll remember that, %s" % editor
elif text.lower().startswith('forget '):
@ -638,7 +625,7 @@ class Encyclopedia(callbacks.Plugin):
factoid = self.get_single_factoid(channel, name)
if not factoid:
return "I know nothing about '%s' yet, %s" % (name, editor)
value = '<deleted>%s' % factoid.value
factoid.value = '<deleted>%s' % factoid.value
retmsg = "I'll forget that, %s" % editor
elif text.lower().startswith('unforget '):
@ -648,7 +635,7 @@ class Encyclopedia(callbacks.Plugin):
return "I know nothing about '%s' at all, %s" % (name, editor)
if not factoid.value.startswith('<deleted>'):
return "Factoid '%s' is not deleted yet, %s" % (factoid.name, editor)
value = factoid.value[9:]
factoid.value = factoid.value[9:]
retmsg = "I suddenly remember '%s' again, %s" % (factoid.name, editor)
else:
@ -679,18 +666,21 @@ class Encyclopedia(callbacks.Plugin):
value = regex.sub(replace, factoid.value, 1)
if value == factoid.value:
return "Nothing changed there"
factoid.value = value
retmsg = "I'll remember that, %s" % editor
ret = self.check_aliases(channel, factoid)
if ret:
return ret
return 'edit', factoid, value, retmsg
return 'edit', factoid, retmsg
def factoid_add(self, factoid, newvalue, channel, editor):
def factoid_add(self, factoid, channel, editor):
edited = str(datetime.datetime.utcnow())
self.log_edit('add', factoid, channel, editor, edited)
db = self.get_db(channel)
cs = db.cursor()
cs.execute("INSERT INTO facts (name, value, author, added) VALUES (?, ?, ?, ?)",
(factoid.name, factoid.value, editor, str(datetime.datetime.utcnow())))
(factoid.name, factoid.value, editor, edited))
db.commit()
def factoid_add_check(self, text, channel, editor):
@ -711,7 +701,7 @@ class Encyclopedia(callbacks.Plugin):
ret = self.check_aliases(channel, factoid)
if ret:
return ret
return 'add', factoid, None, retmsg
return 'add', factoid, retmsg
def factoid_lookup(self, text, channel, nick, display=None):
def expand(value):
@ -753,30 +743,23 @@ class Encyclopedia(callbacks.Plugin):
break
return ret
def log_edit(self, factoid, channel, editor):
def log_edit(self, etype, factoid, channel, editor, edited):
db = self.get_db(channel)
cs = db.cursor()
cs.execute("INSERT INTO log (name, oldvalue, author, added) VALUES (?, ?, ?, ?)",
(factoid.name, factoid.value, editor, str(datetime.datetime.utcnow())))
cs.execute("INSERT INTO log (type, name, value, author, added) VALUES (?, ?, ?, ?, ?)",
(etype, factoid.name, factoid.value, editor, edited))
db.commit()
def log_request(self, rtype, factoid, newvalue, channel, requester):
db = self.get_log_db(channel)
if not db:
return
def log_request(self, rtype, factoid, channel, requester):
db = self.get_db(channel)
cur = db.cursor()
cur.execute("SELECT type, value FROM requests WHERE name = ?", (factoid.name,))
items = cur.fetchall()
if newvalue:
oldvalue = factoid.value
factoid.value = newvalue
else:
oldvalue = ''
for i in items:
if i[0] == rtype and i[1] == factoid.value:
return
cur.execute("INSERT INTO requests (type, name, value, oldvalue, requester, requested) VALUES (?, ?, ?, ?, ?, ?)",
(rtype, factoid.name, factoid.value, oldvalue, requester, str(datetime.datetime.utcnow())))
cur.execute("INSERT INTO requests (type, name, value, requester, requested) VALUES (?, ?, ?, ?, ?)",
(rtype, factoid.name, factoid.value, requester, str(datetime.datetime.utcnow())))
db.commit()
def search_factoid(self, text, channel):
@ -881,10 +864,6 @@ class Encyclopedia(callbacks.Plugin):
Looks up factoids created or edited by <author>,
<author> defaults to you.
"""
def isLastEdit(name, lid):
cur.execute("SELECT max(id) FROM log WHERE name = ?", (name,))
return int(cur.fetchall()[0][0]) == lid
if not capab(msg.prefix, "editfactoids"):
irc.errorNoCapability("editfactoids")
return
@ -894,23 +873,17 @@ class Encyclopedia(callbacks.Plugin):
author = author[:author.find('!')]
db = self.get_db(channel)
cur = db.cursor()
auth_ret, edit_ret, done = [], [], []
auth_ret, edit_ret = [], []
cur.execute("SELECT name, value FROM facts WHERE author LIKE ? ORDER BY popularity DESC", ('%s%%' % author,))
auth_res = cur.fetchall()
cur.execute("SELECT id, name, oldvalue FROM log WHERE author LIKE ? ORDER BY id DESC", ('%s%%' % author,))
log_res = cur.fetchall()
cur.execute("SELECT name, value FROM facts WHERE editor LIKE ? AND author NOT LIKE ? ORDER BY popularity DESC", ('%s%%' % author, '%s%%' % author))
edit_res = cur.fetchall()
for r in auth_res:
n, v = r[0], r[1]
auth_ret.append(get_factoid_label(n, v))
done.append(n)
auth_ret.append(get_factoid_label(r[0], r[1]))
for r in log_res:
i, n, v = r[0], r[1], r[2]
if n not in done:
if isLastEdit(n, i):
edit_ret.append(get_factoid_label(n, v))
done.append(n)
for r in edit_res:
edit_ret.append(get_factoid_label(r[0], r[1]))
if not auth_ret:
auth_rmsg = "Authored: None found"

103
Encyclopedia/sqlite2to3.py Executable file
View File

@ -0,0 +1,103 @@
#!/usr/bin/env python
# -*- Encoding: utf-8 -*-
###
# Copyright (c) 2018 Krytarik Raido
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
###
import sqlite, sqlite3
class Factoid:
def __init__(self, name, value, author, added, popularity,
editor=None, edited=None):
try:
self.name = name.decode('ascii')
except UnicodeDecodeError:
try:
self.name = name.decode('utf-8')
except UnicodeDecodeError:
try:
self.name = name.decode('cp1252')
except UnicodeDecodeError as e:
print '%s: %s' % (e, name)
return
try:
self.value = value.decode('ascii')
except UnicodeDecodeError:
try:
self.value = value.decode('utf-8')
except UnicodeDecodeError:
try:
self.value = value.decode('cp1252')
except UnicodeDecodeError as e:
print '%s: %s' % (e, value)
return
self.author = author; self.added = added
self.editor = editor; self.edited = edited
self.popularity = popularity
class Log:
def __init__(self, author, added):
self.author = author; self.added = added
# Get old data
con2 = sqlite.connect('ubuntu.db')
cur2 = con2.cursor()
cur2.execute("SELECT name, value, author, added, popularity FROM facts ORDER BY id")
factoids = [Factoid(*x) for x in cur2.fetchall()]
# Create new database
con3 = sqlite3.connect('ubuntu-new.db')
cur3 = con3.cursor()
cur3.execute("""CREATE TABLE facts (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
value TEXT NOT NULL,
author TEXT NOT NULL,
added TEXT NOT NULL,
editor TEXT,
edited TEXT,
popularity INTEGER NOT NULL DEFAULT 0
)""")
cur3.execute("""CREATE TABLE log (
id INTEGER PRIMARY KEY,
type TEXT NOT NULL,
name TEXT NOT NULL,
value TEXT NOT NULL,
author TEXT NOT NULL,
added TEXT NOT NULL
)""")
cur3.execute("""CREATE TABLE requests (
id INTEGER PRIMARY KEY,
type TEXT NOT NULL,
name TEXT NOT NULL,
value TEXT NOT NULL,
requester TEXT NOT NULL,
requested TEXT NOT NULL
)""")
# Write new data
for fact in factoids:
if not hasattr(fact, 'author'):
continue
# Get old last edit data
cur2.execute("SELECT author, added FROM log WHERE name = '%s' ORDER BY id DESC LIMIT 1" % fact.name.encode('utf-8').replace("'","''"))
edit = [Log(*x) for x in cur2.fetchall()]
if edit:
fact.editor, fact.edited = edit[0].author, edit[0].added
cur3.execute("INSERT INTO facts (name, value, author, added, editor, edited, popularity) VALUES (?, ?, ?, ?, ?, ?, ?)",
(fact.name, fact.value, fact.author, fact.added, fact.editor, fact.edited, fact.popularity))
con3.commit()
con3.close()
con2.close()

13
Encyclopedia/test.py
View File

@ -47,15 +47,26 @@ class EncyclopediaTestCase(ChannelPluginTestCase):
value TEXT NOT NULL,
author TEXT NOT NULL,
added TEXT NOT NULL,
editor TEXT,
edited TEXT,
popularity INTEGER NOT NULL DEFAULT 0
)""")
cur.execute("""CREATE TABLE log (
id INTEGER PRIMARY KEY,
type TEXT NOT NULL,
name TEXT NOT NULL,
oldvalue TEXT NOT NULL,
value TEXT NOT NULL,
author TEXT NOT NULL,
added TEXT NOT NULL
)""")
cur.execute("""CREATE TABLE requests (
id INTEGER PRIMARY KEY,
type TEXT NOT NULL,
name TEXT NOT NULL,
value TEXT NOT NULL,
requester TEXT NOT NULL,
requested TEXT NOT NULL
)""")
db.commit()
db.close()
self.getCallback().databases = {}