Merge branch 'logger' into 'master'

Refactor Logger

See merge request !108

gajim/chat_control.py

@@ -1411,14 +1411,14 @@ class ChatControl(ChatControlBase):
 
         # number of messages that are in queue and are already logged, we want
         # to avoid duplication
-        pending_how_many = len(gajim.events.get_events(self.account, jid,
+        pending = len(gajim.events.get_events(self.account, jid,
                 ['chat', 'pm']))
         if self.resource:
-            pending_how_many += len(gajim.events.get_events(self.account,
+            pending += len(gajim.events.get_events(self.account,
                     self.contact.get_full_jid(), ['chat', 'pm']))
 
         rows = gajim.logger.get_last_conversation_lines(
-            jid, pending_how_many, self.account)
+            self.account, jid, pending)
 
         local_old_kind = None
         self.conv_textview.just_cleared = True

gajim/command_system/implementation/standard.py

@@ -87,7 +87,7 @@ class StandardCommonCommands(CommandContainer):
     @command('lastlog', overlap=True)
     @doc(_("Show logged messages which mention given text"))
     def grep(self, text, limit=None):
-        results = gajim.logger.search_log(self.contact.jid, text, self.account)
+        results = gajim.logger.search_log(self.account, self.contact.jid, text)
 
         if not results:
             raise CommandError(_("%s: Nothing found") % text)

gajim/common/connection.py

@@ -2580,19 +2580,16 @@ class Connection(CommonConnection, ConnectionHandlers):
         # last date/time in history to avoid duplicate
         if room_jid not in self.last_history_time:
             # Not in memory, get it from DB
-            last_log = None
+            last_log = 0
-            # Do not check if we are not logging for this room
             if gajim.config.should_log(self.name, room_jid):
                 # Check time first in the FAST table
-                last_log = gajim.logger.get_room_last_message_time(room_jid)
+                last_log = gajim.logger.get_room_last_message_time(
-                if last_log is None:
+                    self.name, room_jid)
-                    # Not in special table, get it from messages DB
+                if not last_log:
-                    last_log = gajim.logger.get_last_date_that_has_logs(room_jid,
+                    last_log = 0
-                            is_room=True)
+
             # Create self.last_history_time[room_jid] even if not logging,
             # could be used in connection_handlers
-            if last_log is None:
-                last_log = 0
             self.last_history_time[room_jid] = last_log
 
         p = nbxmpp.Presence(to='%s/%s' % (room_jid, nick),

gajim/common/logger.py

@@ -32,6 +32,7 @@ import os
 import sys
 import time
 import datetime
+import calendar
 import json
 from collections import namedtuple
 from gzip import GzipFile
@@ -68,6 +69,9 @@ class KindConstant(IntEnum):
     CHAT_MSG_SENT = 6
     ERROR = 7
 
+    def __str__(self):
+        return str(self.value)
+
 @unique
 class ShowConstant(IntEnum):
     ONLINE = 0
@@ -160,6 +164,11 @@ class Logger:
                 isolation_level='IMMEDIATE')
         os.chdir(back)
         self.con.row_factory = self.namedtuple_factory
+
+        # DB functions
+        self.con.create_function("like", 1, self._like)
+        self.con.create_function("get_timeout", 0, self._get_timeout)
+
         self.cur = self.con.cursor()
         self.set_synchronous(False)
 
@@ -183,6 +192,22 @@ class Logger:
         self.open_db()
         self.get_jids_already_in_db()
 
+    @staticmethod
+    def _get_timeout():
+        """
+        returns the timeout in epoch
+        """
+        timeout = gajim.config.get('restore_timeout')
+
+        now = int(time.time())
+        if timeout > 0:
+            timeout = now - (timeout * 60)
+        return timeout
+
+    @staticmethod
+    def _like(search_str):
+        return '%{}%'.format(search_str)
+
     def commit(self):
         try:
             self.con.commit()
@@ -249,6 +274,22 @@ class Logger:
                 return True
             return False
 
+    @staticmethod
+    def _get_family_jids(account, jid):
+        """
+        Get all jids of the metacontacts family
+
+        :param account: The account
+
+        :param jid:     The JID
+
+        returns a list of JIDs'
+        """
+        family = gajim.contacts.get_metacontacts_family(account, jid)
+        if family:
+            return [user['jid'] for user in family]
+        return [jid]
+
     def get_jid_id(self, jid, typestr=None):
         """
         jids table has jid and jid_id logs table has log_id, jid_id,
@@ -593,44 +634,49 @@ class Logger:
                 exceptions.PysqliteOperationalError) as error:
             self.dispatch('DB_ERROR', error)
 
-    def get_last_conversation_lines(self, jid, pending_how_many, account):
+    def get_last_conversation_lines(self, account, jid, pending):
         """
-        Accept how many rows to restore and when to time them out (in minutes)
+        Get recent messages
-        (mark them as too old) and number of messages that are in queue and are
+
-        already logged but pending to be viewed, returns a list of tuples
+        Pending messages are already in queue to be printed when the
-        containg time, kind, message, subject list with empty tuple if nothing
+        ChatControl is opened, so we dont want to request those messages.
-        found to meet our demands
+        How many messages are requested depends on the 'restore_lines'
+        config value. How far back in time messages are requested depends on
+        _get_timeout().
+
+        :param account: The account
+
+        :param jid:     The jid from which we request the conversation lines
+
+        :param pending: How many messages are currently pending so we dont
+                        request those messages
+
+        returns a list of namedtuples
         """
-        try:
+
-            self.get_jid_id(jid)
+        restore = gajim.config.get('restore_lines')
-        except exceptions.PysqliteOperationalError:
+        if restore <= 0:
-            # Error trying to create a new jid_id. This means there is no log
             return []
-        where_sql, jid_tuple = self._build_contact_where(account, jid)
 
-        # How many lines to restore and when to time them out
+        kinds = map(str, [KindConstant.SINGLE_MSG_RECV,
-        restore_how_many = gajim.config.get('restore_lines')
+                          KindConstant.SINGLE_MSG_SENT,
-        if restore_how_many <= 0:
+                          KindConstant.CHAT_MSG_RECV,
-            return []
+                          KindConstant.CHAT_MSG_SENT,
-        timeout = gajim.config.get('restore_timeout')  # in minutes
+                          KindConstant.ERROR])
 
-        now = int(float(time.time()))
+        jids = self._get_family_jids(account, jid)
-        if timeout > 0:
+
-            timeout = now - (timeout * 60) # before that they are too old
+        sql = '''
+            SELECT time, kind, message, subject, additional_data
+            FROM logs NATURAL JOIN jids WHERE jid IN ({jids}) AND
+            kind IN ({kinds}) AND time > get_timeout()
+            ORDER BY time DESC, log_line_id DESC LIMIT ? OFFSET ?
+            '''.format(jids=', '.join('?' * len(jids)),
+                       kinds=', '.join(kinds))
 
-        # so if we ask last 5 lines and we have 2 pending we get
-        # 3 - 8 (we avoid the last 2 lines but we still return 5 asked)
         try:
-            self.cur.execute('''
+            messages = self.con.execute(
-                SELECT time, kind, message, subject, additional_data FROM logs
+                sql, (*jids, restore, pending)).fetchall()
-                WHERE (%s) AND kind IN (%d, %d, %d, %d, %d) AND time > %d
-                ORDER BY time DESC LIMIT %d OFFSET %d
-                ''' % (where_sql, KindConstant.SINGLE_MSG_RECV,
-                KindConstant.CHAT_MSG_RECV, KindConstant.SINGLE_MSG_SENT,
-                KindConstant.CHAT_MSG_SENT, KindConstant.ERROR, timeout,
-                restore_how_many, pending_how_many), jid_tuple)
-
-            messages = self.cur.fetchall()
         except sqlite.DatabaseError:
             self.dispatch('DB_ERROR',
                           exceptions.DatabaseMalformed(LOG_DB_PATH))
@@ -649,174 +695,163 @@ class Logger:
         start_of_day = int(time.mktime(local_time))
         return start_of_day
 
-    def get_conversation_for_date(self, jid, year, month, day, account):
+    def get_conversation_for_date(self, account, jid, date):
         """
         Load the complete conversation with a given jid on a specific date
 
-        The conversation contains all messages that were exchanged between
+        :param account: The account
-        `account` and `jid` on the day specified by `year`, `month` and `day`,
-        where `month` and `day` are 1-based.
 
-        The conversation will be returned as a list of single messages of type
+        :param jid:     The jid for which we request the conversation
-        `Logger.Message`. Messages in the list are sorted chronologically. An
+
-        empty list will be returned if there are no messages in the log database
+        :param date:    datetime.datetime instance
-        for the requested combination of `jid` and `account` on the given date.
+                        example: datetime.datetime(year, month, day)
+
+        returns a list of namedtuples
         """
-        try:
-            self.get_jid_id(jid)
-        except exceptions.PysqliteOperationalError:
-            # Error trying to create a new jid_id. This means there is no log
-            return []
-        where_sql, jid_tuple = self._build_contact_where(account, jid)
 
-        start_of_day = self.get_unix_time_from_date(year, month, day)
+        jids = self._get_family_jids(account, jid)
-        seconds_in_a_day = 86400 # 60 * 60 * 24
-        last_second_of_day = start_of_day + seconds_in_a_day - 1
 
-        self.cur.execute('''
+        delta = datetime.timedelta(
+            hours=23, minutes=59, seconds=59, microseconds=999999)
+
+        sql = '''
             SELECT contact_name, time, kind, show, message, subject,
                    additional_data, log_line_id
-            FROM logs
+            FROM logs NATURAL JOIN jids WHERE jid IN ({jids})
-            WHERE (%s)
+            AND time BETWEEN ? AND ?
-            AND time BETWEEN %d AND %d
+            ORDER BY time, log_line_id
-            ORDER BY time
+            '''.format(jids=', '.join('?' * len(jids)))
-            ''' % (where_sql, start_of_day, last_second_of_day), jid_tuple)
 
-        return self.cur.fetchall()
+        return self.con.execute(sql, (*jids, 
+                                      date.timestamp(),
+                                      (date + delta).timestamp())).fetchall()
 
-    def search_log(self, jid, query, account, year=None, month=None, day=None):
+    def search_log(self, account, jid, query, date=None):
         """
         Search the conversation log for messages containing the `query` string.
 
-        The search can either span the complete log for the given `account` and
+        The search can either span the complete log for the given
-        `jid` or be restriced to a single day by specifying `year`, `month` and
+        `account` and `jid` or be restriced to a single day by
-        `day`, where `month` and `day` are 1-based.
+        specifying `date`.
 
-        All messages matching the specified criteria will be returned in a list
+        :param account: The account
-        containing tuples of type `Logger.Message`. If no messages match the
+
-        criteria, an empty list will be returned.
+        :param jid:     The jid for which we request the conversation
+
+        :param query:   A search string
+
+        :param date:    datetime.datetime instance
+                        example: datetime.datetime(year, month, day)
+
+        returns a list of namedtuples
         """
-        try:
+        jids = self._get_family_jids(account, jid)
-            self.get_jid_id(jid)
-        except exceptions.PysqliteOperationalError:
-            # Error trying to create a new jid_id. This means there is no log
-            return []
 
-        where_sql, jid_tuple = self._build_contact_where(account, jid)
+        if date:
-        like_sql = '%' + query.replace("'", "''") + '%'
+            delta = datetime.timedelta(
-        if year and month and day:
+                hours=23, minutes=59, seconds=59, microseconds=999999)
-            start_of_day = self.get_unix_time_from_date(year, month, day)
+
-            seconds_in_a_day = 86400 # 60 * 60 * 24
+            between = '''
-            last_second_of_day = start_of_day + seconds_in_a_day - 1
+                AND time BETWEEN {start} AND {end}
-            self.cur.execute('''
+                '''.format(start=date.timestamp(),
-            SELECT contact_name, time, kind, show, message, subject,
+                           end=(date + delta).timestamp())
-                   additional_data, log_line_id
+
-            FROM logs
+        sql = '''
-            WHERE (%s) AND message LIKE '%s'
+        SELECT contact_name, time, kind, show, message, subject,
-            AND time BETWEEN %d AND %d
+               additional_data, log_line_id
+        FROM logs NATURAL JOIN jids WHERE jid IN ({jids})
+        AND message LIKE like(?) {date_search}
+        ORDER BY time, log_line_id
+        '''.format(jids=', '.join('?' * len(jids)),
+                   date_search=between if date else '')
+
+        return self.con.execute(sql, (*jids, query)).fetchall()
+
+    def get_days_with_logs(self, account, jid, year, month):
+        """
+        Request the days in a month where we received messages
+        for a given `jid`.
+
+        :param account: The account
+
+        :param jid:     The jid for which we request the days
+
+        :param year:    The year
+
+        :param month:   The month
+
+        returns a list of namedtuples
+        """
+        jids = self._get_family_jids(account, jid)
+
+        kinds = map(str, [KindConstant.STATUS,
+                          KindConstant.GCSTATUS])
+
+        # Calculate the start and end datetime of the month
+        date = datetime.datetime(year, month, 1)
+        days = calendar.monthrange(year, month)[1] - 1
+        delta = datetime.timedelta(
+            days=days, hours=23, minutes=59, seconds=59, microseconds=999999)
+
+        sql = """
+            SELECT DISTINCT 
+            CAST(strftime('%d', time, 'unixepoch', 'localtime') AS INTEGER)
+            AS day FROM logs NATURAL JOIN jids WHERE jid IN ({jids})
+            AND time BETWEEN ? AND ?
+            AND kind NOT IN ({kinds})
             ORDER BY time
-            ''' % (where_sql, like_sql, start_of_day, last_second_of_day),
+            """.format(jids=', '.join('?' * len(jids)),
-                jid_tuple)
+                       kinds=', '.join(kinds))
-        else:
-            self.cur.execute('''
-            SELECT contact_name, time, kind, show, message, subject,
-                   additional_data, log_line_id
-            FROM logs
-            WHERE (%s) AND message LIKE '%s'
-            ORDER BY time
-            ''' % (where_sql, like_sql), jid_tuple)
 
-        return self.cur.fetchall()
+        return self.con.execute(sql, (*jids,
+                                      date.timestamp(),
+                                      (date + delta).timestamp())).fetchall()
 
-    def get_days_with_logs(self, jid, year, month, max_day, account):
+    def get_last_date_that_has_logs(self, account, jid):
         """
-        Return the list of days that have logs (not status messages)
+        Get the timestamp of the last message we received for the jid.
+
+        :param account: The account
+
+        :param jid:     The jid for which we request the last timestamp
+
+        returns a timestamp or None
         """
-        try:
+        jids = self._get_family_jids(account, jid)
-            self.get_jid_id(jid)
-        except exceptions.PysqliteOperationalError:
-            # Error trying to create a new jid_id. This means there is no log
-            return []
-        days_with_logs = []
-        where_sql, jid_tuple = self._build_contact_where(account, jid)
 
-        # First select all date of month whith logs we want
+        kinds = map(str, [KindConstant.STATUS,
-        start_of_month = self.get_unix_time_from_date(year, month, 1)
+                          KindConstant.GCSTATUS])
-        seconds_in_a_day = 86400 # 60 * 60 * 24
-        last_second_of_month = start_of_month + (seconds_in_a_day * max_day) - 1
 
-        # Select times and 'floor' them to time 0:00
+        sql = '''
-        # (by dividing, they are integers)
-        # and take only one of the same values (distinct)
-        # Now we have timestamps of time 0:00 of every day with logs
-        self.cur.execute('''
-            SELECT DISTINCT time/(86400)*86400 as time FROM logs
-            WHERE (%s)
-            AND time BETWEEN %d AND %d
-            AND kind NOT IN (%d, %d)
-            ORDER BY time
-            ''' % (where_sql, start_of_month, last_second_of_month,
-            KindConstant.STATUS, KindConstant.GCSTATUS), jid_tuple)
-        result = self.cur.fetchall()
-
-        # convert timestamps to day of month
-        for line in result:
-            days_with_logs[0:0]=[time.gmtime(line.time)[2]]
-
-        return days_with_logs
-
-    def get_last_date_that_has_logs(self, jid, account=None, is_room=False):
-        """
-        Return last time (in seconds since EPOCH) for which we had logs
-        (excluding statuses)
-        """
-        where_sql = ''
-        if not is_room:
-            where_sql, jid_tuple = self._build_contact_where(account, jid)
-        else:
-            try:
-                jid_id = self.get_jid_id(jid, 'ROOM')
-            except exceptions.PysqliteOperationalError:
-                # Error trying to create a new jid_id. This means there is no log
-                return None
-            where_sql = 'jid_id = ?'
-            jid_tuple = (jid_id,)
-        self.cur.execute('''
             SELECT MAX(time) as time FROM logs
-            WHERE (%s)
+            NATURAL JOIN jids WHERE jid IN ({jids})
-            AND kind NOT IN (%d, %d)
+            AND kind NOT IN ({kinds})
-            ''' % (where_sql, KindConstant.STATUS, KindConstant.GCSTATUS),
+            '''.format(jids=', '.join('?' * len(jids)),
-            jid_tuple)
+                       kinds=', '.join(kinds))
 
-        results = self.cur.fetchone()
+        # fetchone() returns always at least one Row with all
-        if results is not None:
+        # attributes set to None because of the MAX() function
-            result = results.time
+        return self.con.execute(sql, (*jids,)).fetchone().time
-        else:
-            result = None
-        return result
 
-    def get_room_last_message_time(self, jid):
+    def get_room_last_message_time(self, account, jid):
         """
-        Return FASTLY last time (in seconds since EPOCH) for which we had logs
+        Get the timestamp of the last message we received in a room.
-        for that room from rooms_last_message_time table
-        """
-        try:
-            jid_id = self.get_jid_id(jid, 'ROOM')
-        except exceptions.PysqliteOperationalError:
-            # Error trying to create a new jid_id. This means there is no log
-            return None
-        where_sql = 'jid_id = %s' % jid_id
-        self.cur.execute('''
-                SELECT time FROM rooms_last_message_time
-                WHERE (%s)
-                ''' % (where_sql))
 
-        results = self.cur.fetchone()
+        :param account: The account
-        if results is not None:
+
-            result = results.time
+        :param jid:     The jid for which we request the last timestamp
-        else:
+
-            result = None
+        returns a timestamp or None
-        return result
+        """
+        sql = '''
+            SELECT time FROM rooms_last_message_time
+            NATURAL JOIN jids WHERE jid = ?
+            '''
+
+        row = self.con.execute(sql, (jid,)).fetchone()
+        if not row:
+            return self.get_last_date_that_has_logs(account, jid)
+        return row.time
 
     def set_room_last_message_time(self, jid, time):
         """
@@ -829,31 +864,6 @@ class Logger:
                 (jid_id, time)
         self.simple_commit(sql)
 
-    def _build_contact_where(self, account, jid):
-        """
-        Build the where clause for a jid, including metacontacts jid(s) if any
-        """
-        where_sql = ''
-        jid_tuple = ()
-        # will return empty list if jid is not associated with
-        # any metacontacts
-        family = gajim.contacts.get_metacontacts_family(account, jid)
-        if family:
-            for user in family:
-                try:
-                    jid_id = self.get_jid_id(user['jid'])
-                except exceptions.PysqliteOperationalError:
-                    continue
-                where_sql += 'jid_id = ?'
-                jid_tuple += (jid_id,)
-                if user != family[-1]:
-                    where_sql += ' OR '
-        else: # if jid was not associated with metacontacts
-            jid_id = self.get_jid_id(jid)
-            where_sql = 'jid_id = ?'
-            jid_tuple += (jid_id,)
-        return where_sql, jid_tuple
-
     def save_transport_type(self, jid, type_):
         """
         Save the type of the transport in DB

gajim/history_window.py

@@ -30,6 +30,7 @@ from gi.repository import Gdk
 from gi.repository import GLib
 import time
 import calendar
+import datetime
 
 from enum import IntEnum, unique
 
@@ -304,7 +305,7 @@ class HistoryWindow:
             # select logs for last date we have logs with contact
             self.calendar.set_sensitive(True)
             last_log = \
-                    gajim.logger.get_last_date_that_has_logs(self.jid, self.account)
+                    gajim.logger.get_last_date_that_has_logs(self.account, self.jid)
 
             date = time.localtime(last_log)
 
@@ -356,20 +357,18 @@ class HistoryWindow:
             widget.select_day(1)
             return
 
-        # in gtk January is 1, in python January is 0,
-        # I want the second
-        # first day of month is 1 not 0
         widget.clear_marks()
         month = gtkgui_helpers.make_gtk_month_python_month(month)
-        days_in_this_month = calendar.monthrange(year, month)[1]
+
         try:
-            log_days = gajim.logger.get_days_with_logs(self.jid, year, month,
+            log_days = gajim.logger.get_days_with_logs(
-                days_in_this_month, self.account)
+                self.account, self.jid, year, month)
         except exceptions.PysqliteOperationalError as e:
             dialogs.ErrorDialog(_('Disk Error'), str(e))
             return
-        for day in log_days:
+
-            widget.mark_day(day)
+        for date in log_days:
+            widget.mark_day(date.day)
 
     def _get_string_show_from_constant_int(self, show):
         if show == ShowConstant.ONLINE:
@@ -397,8 +396,11 @@ class HistoryWindow:
         self.last_time_printout = 0
         show_status = self.show_status_checkbutton.get_active()
 
+        date = datetime.datetime(year, month, day)
+
         conversation = gajim.logger.get_conversation_for_date(
-                self.jid, year, month, day, self.account)
+            self.account, self.jid, date)
+
         for message in conversation:
             if not show_status and message.kind in (KindConstant.GCSTATUS,
                                                     KindConstant.STATUS):
@@ -537,13 +539,15 @@ class HistoryWindow:
                 # This may leed to wrong self nick in the displayed history (Uggh!)
                 account = list(gajim.contacts.get_accounts())[0]
 
-            year, month, day = False, False, False
+            date = None
             if self.search_in_date.get_active():
                 year, month, day = self.calendar.get_date() # integers
                 month = gtkgui_helpers.make_gtk_month_python_month(month)
+                date = datetime.datetime(year, month, day)
 
             show_status = self.show_status_checkbutton.get_active()
-            results = gajim.logger.search_log(jid, text, account, year, month, day)
+
+            results = gajim.logger.search_log(account, jid, text, date)
             #FIXME:
             # add "subject:  | message: " in message column if kind is single
             # also do we need show at all? (we do not search on subject)