From 14370aa04e065719f11c8f39b1dd1d721e9eacc5 Mon Sep 17 00:00:00 2001 From: Xavier Ruppen Date: Thu, 31 Dec 2020 00:51:53 +0100 Subject: [PATCH 1/3] db: store sqlite schemaversion in options object This will be needed to deduce what unit timestamps use in next commit. --- quasselgrep/db.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/quasselgrep/db.py b/quasselgrep/db.py index 116584d..6562ef0 100644 --- a/quasselgrep/db.py +++ b/quasselgrep/db.py @@ -14,6 +14,13 @@ def connect(self, options): self.connection = dbmodule.connect(options.db_name, check_same_thread=False) cursor = self.connection.cursor() + + #Newer sqlite versions have timestamps in milliseconds. + cursor.execute('SELECT value FROM coreinfo WHERE key="schemaversion"') + results = cursor.fetchall() + if len(results) != 1: + raise ValueError('Incorrect sqlite schemaversion format') + options.sqlite_version = results[0][0] elif options.db_type == 'postgres': options.param_string = '%s' try: From ae01272edb562d86452d280efc178a91166a1b78 Mon Sep 17 00:00:00 2001 From: Xavier Ruppen Date: Thu, 31 Dec 2020 00:53:04 +0100 Subject: [PATCH 2/3] query: timestamps are in ms for sqlite3 schemaversion >= 31 Since sqlite3 schemaversion 31, timestamps are in milliseconds (see 56f686f7 in quassel). As a result, timestamps were broken on sqlite3 databases >= 31. This fixes issue #19. --- quasselgrep/query.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/quasselgrep/query.py b/quasselgrep/query.py index d48a7b1..71ee126 100644 --- a/quasselgrep/query.py +++ b/quasselgrep/query.py @@ -58,6 +58,11 @@ def __init__(self, cursor, options, text, timerange=None): self.fromtime = timerange[0].strftime('%s') self.totime = timerange[1].strftime('%s') + #Newer sqlite versions have timestamps in milliseconds. + if options.sqlite_version >= 31: + self.fromtime = int(self.fromtime) * 1000 + self.totime = int(self.totime) * 1000 + if options.inclusive: self.msg_types = (MSG, NOTICE, ACTION, NICK, MODE, JOIN, PART, QUIT, KICK, TOPIC, INVITE, SPLITJOIN, SPLITQUIT) else: @@ -136,7 +141,11 @@ def columns(self): if self.options.db_type == 'postgres': columns.append('backlog.time::timestamp(0)') elif self.options.db_type == 'sqlite': - columns.append("datetime(backlog.time, 'unixepoch') as time") + #Newer sqlite versions have timestamps in milliseconds. + if self.options.sqlite_version >= 31: + columns.append("datetime(backlog.time / 1000, 'unixepoch') as time") + else: + columns.append("datetime(backlog.time, 'unixepoch') as time") columns += ["backlog.type", "backlog.message", "sender.sender", "buffer.buffername", "network.networkname"] return columns From 308b949a543df64fd3d476683582623609cec6e7 Mon Sep 17 00:00:00 2001 From: Xavier Ruppen Date: Thu, 31 Dec 2020 11:56:12 +0100 Subject: [PATCH 3/3] db: fix python3 crash on string to int conversion Python3 requires a cast of options.sqlite_version to int, otherwise the following crash happens: File "$HOME/Desktop/fdsa/quasselgrep-sqlite3_ts_ms/quasselgrep/query.py", line 145, in columns if self.options.sqlite_version >= 31: TypeError: '>=' not supported between instances of 'str' and 'int' Also raise an exception with an explicit error message in case quassel ever decides to introduce characters in its schema version. --- quasselgrep/db.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/quasselgrep/db.py b/quasselgrep/db.py index 6562ef0..d8e5c68 100644 --- a/quasselgrep/db.py +++ b/quasselgrep/db.py @@ -20,7 +20,11 @@ def connect(self, options): results = cursor.fetchall() if len(results) != 1: raise ValueError('Incorrect sqlite schemaversion format') - options.sqlite_version = results[0][0] + try: + #Schema version should be an integer, but this isn't guaranteed + options.sqlite_version = int(results[0][0]) + except ValueError as e: + raise ValueError('Unexpected sqlite schemaversion %s, not an integer: %s' % (results[0][0], e)) elif options.db_type == 'postgres': options.param_string = '%s' try: