diff --git a/gajim/chat_control.py b/gajim/chat_control.py index 98d7022c7..9ef1aa1b7 100644 --- a/gajim/chat_control.py +++ b/gajim/chat_control.py @@ -1411,14 +1411,14 @@ class ChatControl(ChatControlBase): # number of messages that are in queue and are already logged, we want # to avoid duplication - pending_how_many = len(gajim.events.get_events(self.account, jid, + pending = len(gajim.events.get_events(self.account, jid, ['chat', 'pm'])) if self.resource: - pending_how_many += len(gajim.events.get_events(self.account, + pending += len(gajim.events.get_events(self.account, self.contact.get_full_jid(), ['chat', 'pm'])) rows = gajim.logger.get_last_conversation_lines( - jid, pending_how_many, self.account) + self.account, jid, pending) local_old_kind = None self.conv_textview.just_cleared = True diff --git a/gajim/command_system/implementation/standard.py b/gajim/command_system/implementation/standard.py index 4caba25d7..768d1906c 100644 --- a/gajim/command_system/implementation/standard.py +++ b/gajim/command_system/implementation/standard.py @@ -87,7 +87,7 @@ class StandardCommonCommands(CommandContainer): @command('lastlog', overlap=True) @doc(_("Show logged messages which mention given text")) def grep(self, text, limit=None): - results = gajim.logger.search_log(self.contact.jid, text, self.account) + results = gajim.logger.search_log(self.account, self.contact.jid, text) if not results: raise CommandError(_("%s: Nothing found") % text) diff --git a/gajim/common/connection.py b/gajim/common/connection.py index 127d7aa42..5065611b0 100644 --- a/gajim/common/connection.py +++ b/gajim/common/connection.py @@ -2580,19 +2580,16 @@ class Connection(CommonConnection, ConnectionHandlers): # last date/time in history to avoid duplicate if room_jid not in self.last_history_time: # Not in memory, get it from DB - last_log = None - # Do not check if we are not logging for this room + last_log = 0 if gajim.config.should_log(self.name, room_jid): # Check time first in the FAST table - last_log = gajim.logger.get_room_last_message_time(room_jid) - if last_log is None: - # Not in special table, get it from messages DB - last_log = gajim.logger.get_last_date_that_has_logs(room_jid, - is_room=True) + last_log = gajim.logger.get_room_last_message_time( + self.name, room_jid) + if not last_log: + last_log = 0 + # Create self.last_history_time[room_jid] even if not logging, # could be used in connection_handlers - if last_log is None: - last_log = 0 self.last_history_time[room_jid] = last_log p = nbxmpp.Presence(to='%s/%s' % (room_jid, nick), diff --git a/gajim/common/logger.py b/gajim/common/logger.py index 0b7e04796..9d07e65ca 100644 --- a/gajim/common/logger.py +++ b/gajim/common/logger.py @@ -32,6 +32,7 @@ import os import sys import time import datetime +import calendar import json from collections import namedtuple from gzip import GzipFile @@ -68,6 +69,9 @@ class KindConstant(IntEnum): CHAT_MSG_SENT = 6 ERROR = 7 + def __str__(self): + return str(self.value) + @unique class ShowConstant(IntEnum): ONLINE = 0 @@ -160,6 +164,11 @@ class Logger: isolation_level='IMMEDIATE') os.chdir(back) self.con.row_factory = self.namedtuple_factory + + # DB functions + self.con.create_function("like", 1, self._like) + self.con.create_function("get_timeout", 0, self._get_timeout) + self.cur = self.con.cursor() self.set_synchronous(False) @@ -183,6 +192,22 @@ class Logger: self.open_db() self.get_jids_already_in_db() + @staticmethod + def _get_timeout(): + """ + returns the timeout in epoch + """ + timeout = gajim.config.get('restore_timeout') + + now = int(time.time()) + if timeout > 0: + timeout = now - (timeout * 60) + return timeout + + @staticmethod + def _like(search_str): + return '%{}%'.format(search_str) + def commit(self): try: self.con.commit() @@ -249,6 +274,22 @@ class Logger: return True return False + @staticmethod + def _get_family_jids(account, jid): + """ + Get all jids of the metacontacts family + + :param account: The account + + :param jid: The JID + + returns a list of JIDs' + """ + family = gajim.contacts.get_metacontacts_family(account, jid) + if family: + return [user['jid'] for user in family] + return [jid] + def get_jid_id(self, jid, typestr=None): """ jids table has jid and jid_id logs table has log_id, jid_id, @@ -593,44 +634,49 @@ class Logger: exceptions.PysqliteOperationalError) as error: self.dispatch('DB_ERROR', error) - def get_last_conversation_lines(self, jid, pending_how_many, account): + def get_last_conversation_lines(self, account, jid, pending): """ - Accept how many rows to restore and when to time them out (in minutes) - (mark them as too old) and number of messages that are in queue and are - already logged but pending to be viewed, returns a list of tuples - containg time, kind, message, subject list with empty tuple if nothing - found to meet our demands + Get recent messages + + Pending messages are already in queue to be printed when the + ChatControl is opened, so we dont want to request those messages. + How many messages are requested depends on the 'restore_lines' + config value. How far back in time messages are requested depends on + _get_timeout(). + + :param account: The account + + :param jid: The jid from which we request the conversation lines + + :param pending: How many messages are currently pending so we dont + request those messages + + returns a list of namedtuples """ - try: - self.get_jid_id(jid) - except exceptions.PysqliteOperationalError: - # Error trying to create a new jid_id. This means there is no log + + restore = gajim.config.get('restore_lines') + if restore <= 0: return [] - where_sql, jid_tuple = self._build_contact_where(account, jid) - # How many lines to restore and when to time them out - restore_how_many = gajim.config.get('restore_lines') - if restore_how_many <= 0: - return [] - timeout = gajim.config.get('restore_timeout') # in minutes + kinds = map(str, [KindConstant.SINGLE_MSG_RECV, + KindConstant.SINGLE_MSG_SENT, + KindConstant.CHAT_MSG_RECV, + KindConstant.CHAT_MSG_SENT, + KindConstant.ERROR]) - now = int(float(time.time())) - if timeout > 0: - timeout = now - (timeout * 60) # before that they are too old + jids = self._get_family_jids(account, jid) + + sql = ''' + SELECT time, kind, message, subject, additional_data + FROM logs NATURAL JOIN jids WHERE jid IN ({jids}) AND + kind IN ({kinds}) AND time > get_timeout() + ORDER BY time DESC, log_line_id DESC LIMIT ? OFFSET ? + '''.format(jids=', '.join('?' * len(jids)), + kinds=', '.join(kinds)) - # so if we ask last 5 lines and we have 2 pending we get - # 3 - 8 (we avoid the last 2 lines but we still return 5 asked) try: - self.cur.execute(''' - SELECT time, kind, message, subject, additional_data FROM logs - WHERE (%s) AND kind IN (%d, %d, %d, %d, %d) AND time > %d - ORDER BY time DESC LIMIT %d OFFSET %d - ''' % (where_sql, KindConstant.SINGLE_MSG_RECV, - KindConstant.CHAT_MSG_RECV, KindConstant.SINGLE_MSG_SENT, - KindConstant.CHAT_MSG_SENT, KindConstant.ERROR, timeout, - restore_how_many, pending_how_many), jid_tuple) - - messages = self.cur.fetchall() + messages = self.con.execute( + sql, (*jids, restore, pending)).fetchall() except sqlite.DatabaseError: self.dispatch('DB_ERROR', exceptions.DatabaseMalformed(LOG_DB_PATH)) @@ -649,174 +695,163 @@ class Logger: start_of_day = int(time.mktime(local_time)) return start_of_day - def get_conversation_for_date(self, jid, year, month, day, account): + def get_conversation_for_date(self, account, jid, date): """ Load the complete conversation with a given jid on a specific date - The conversation contains all messages that were exchanged between - `account` and `jid` on the day specified by `year`, `month` and `day`, - where `month` and `day` are 1-based. + :param account: The account - The conversation will be returned as a list of single messages of type - `Logger.Message`. Messages in the list are sorted chronologically. An - empty list will be returned if there are no messages in the log database - for the requested combination of `jid` and `account` on the given date. + :param jid: The jid for which we request the conversation + + :param date: datetime.datetime instance + example: datetime.datetime(year, month, day) + + returns a list of namedtuples """ - try: - self.get_jid_id(jid) - except exceptions.PysqliteOperationalError: - # Error trying to create a new jid_id. This means there is no log - return [] - where_sql, jid_tuple = self._build_contact_where(account, jid) - start_of_day = self.get_unix_time_from_date(year, month, day) - seconds_in_a_day = 86400 # 60 * 60 * 24 - last_second_of_day = start_of_day + seconds_in_a_day - 1 + jids = self._get_family_jids(account, jid) - self.cur.execute(''' + delta = datetime.timedelta( + hours=23, minutes=59, seconds=59, microseconds=999999) + + sql = ''' SELECT contact_name, time, kind, show, message, subject, additional_data, log_line_id - FROM logs - WHERE (%s) - AND time BETWEEN %d AND %d - ORDER BY time - ''' % (where_sql, start_of_day, last_second_of_day), jid_tuple) + FROM logs NATURAL JOIN jids WHERE jid IN ({jids}) + AND time BETWEEN ? AND ? + ORDER BY time, log_line_id + '''.format(jids=', '.join('?' * len(jids))) - return self.cur.fetchall() + return self.con.execute(sql, (*jids, + date.timestamp(), + (date + delta).timestamp())).fetchall() - def search_log(self, jid, query, account, year=None, month=None, day=None): + def search_log(self, account, jid, query, date=None): """ Search the conversation log for messages containing the `query` string. - The search can either span the complete log for the given `account` and - `jid` or be restriced to a single day by specifying `year`, `month` and - `day`, where `month` and `day` are 1-based. + The search can either span the complete log for the given + `account` and `jid` or be restriced to a single day by + specifying `date`. - All messages matching the specified criteria will be returned in a list - containing tuples of type `Logger.Message`. If no messages match the - criteria, an empty list will be returned. + :param account: The account + + :param jid: The jid for which we request the conversation + + :param query: A search string + + :param date: datetime.datetime instance + example: datetime.datetime(year, month, day) + + returns a list of namedtuples """ - try: - self.get_jid_id(jid) - except exceptions.PysqliteOperationalError: - # Error trying to create a new jid_id. This means there is no log - return [] + jids = self._get_family_jids(account, jid) - where_sql, jid_tuple = self._build_contact_where(account, jid) - like_sql = '%' + query.replace("'", "''") + '%' - if year and month and day: - start_of_day = self.get_unix_time_from_date(year, month, day) - seconds_in_a_day = 86400 # 60 * 60 * 24 - last_second_of_day = start_of_day + seconds_in_a_day - 1 - self.cur.execute(''' - SELECT contact_name, time, kind, show, message, subject, - additional_data, log_line_id - FROM logs - WHERE (%s) AND message LIKE '%s' - AND time BETWEEN %d AND %d + if date: + delta = datetime.timedelta( + hours=23, minutes=59, seconds=59, microseconds=999999) + + between = ''' + AND time BETWEEN {start} AND {end} + '''.format(start=date.timestamp(), + end=(date + delta).timestamp()) + + sql = ''' + SELECT contact_name, time, kind, show, message, subject, + additional_data, log_line_id + FROM logs NATURAL JOIN jids WHERE jid IN ({jids}) + AND message LIKE like(?) {date_search} + ORDER BY time, log_line_id + '''.format(jids=', '.join('?' * len(jids)), + date_search=between if date else '') + + return self.con.execute(sql, (*jids, query)).fetchall() + + def get_days_with_logs(self, account, jid, year, month): + """ + Request the days in a month where we received messages + for a given `jid`. + + :param account: The account + + :param jid: The jid for which we request the days + + :param year: The year + + :param month: The month + + returns a list of namedtuples + """ + jids = self._get_family_jids(account, jid) + + kinds = map(str, [KindConstant.STATUS, + KindConstant.GCSTATUS]) + + # Calculate the start and end datetime of the month + date = datetime.datetime(year, month, 1) + days = calendar.monthrange(year, month)[1] - 1 + delta = datetime.timedelta( + days=days, hours=23, minutes=59, seconds=59, microseconds=999999) + + sql = """ + SELECT DISTINCT + CAST(strftime('%d', time, 'unixepoch', 'localtime') AS INTEGER) + AS day FROM logs NATURAL JOIN jids WHERE jid IN ({jids}) + AND time BETWEEN ? AND ? + AND kind NOT IN ({kinds}) ORDER BY time - ''' % (where_sql, like_sql, start_of_day, last_second_of_day), - jid_tuple) - else: - self.cur.execute(''' - SELECT contact_name, time, kind, show, message, subject, - additional_data, log_line_id - FROM logs - WHERE (%s) AND message LIKE '%s' - ORDER BY time - ''' % (where_sql, like_sql), jid_tuple) + """.format(jids=', '.join('?' * len(jids)), + kinds=', '.join(kinds)) - return self.cur.fetchall() + return self.con.execute(sql, (*jids, + date.timestamp(), + (date + delta).timestamp())).fetchall() - def get_days_with_logs(self, jid, year, month, max_day, account): + def get_last_date_that_has_logs(self, account, jid): """ - Return the list of days that have logs (not status messages) + Get the timestamp of the last message we received for the jid. + + :param account: The account + + :param jid: The jid for which we request the last timestamp + + returns a timestamp or None """ - try: - self.get_jid_id(jid) - except exceptions.PysqliteOperationalError: - # Error trying to create a new jid_id. This means there is no log - return [] - days_with_logs = [] - where_sql, jid_tuple = self._build_contact_where(account, jid) + jids = self._get_family_jids(account, jid) - # First select all date of month whith logs we want - start_of_month = self.get_unix_time_from_date(year, month, 1) - seconds_in_a_day = 86400 # 60 * 60 * 24 - last_second_of_month = start_of_month + (seconds_in_a_day * max_day) - 1 + kinds = map(str, [KindConstant.STATUS, + KindConstant.GCSTATUS]) - # Select times and 'floor' them to time 0:00 - # (by dividing, they are integers) - # and take only one of the same values (distinct) - # Now we have timestamps of time 0:00 of every day with logs - self.cur.execute(''' - SELECT DISTINCT time/(86400)*86400 as time FROM logs - WHERE (%s) - AND time BETWEEN %d AND %d - AND kind NOT IN (%d, %d) - ORDER BY time - ''' % (where_sql, start_of_month, last_second_of_month, - KindConstant.STATUS, KindConstant.GCSTATUS), jid_tuple) - result = self.cur.fetchall() - - # convert timestamps to day of month - for line in result: - days_with_logs[0:0]=[time.gmtime(line.time)[2]] - - return days_with_logs - - def get_last_date_that_has_logs(self, jid, account=None, is_room=False): - """ - Return last time (in seconds since EPOCH) for which we had logs - (excluding statuses) - """ - where_sql = '' - if not is_room: - where_sql, jid_tuple = self._build_contact_where(account, jid) - else: - try: - jid_id = self.get_jid_id(jid, 'ROOM') - except exceptions.PysqliteOperationalError: - # Error trying to create a new jid_id. This means there is no log - return None - where_sql = 'jid_id = ?' - jid_tuple = (jid_id,) - self.cur.execute(''' + sql = ''' SELECT MAX(time) as time FROM logs - WHERE (%s) - AND kind NOT IN (%d, %d) - ''' % (where_sql, KindConstant.STATUS, KindConstant.GCSTATUS), - jid_tuple) + NATURAL JOIN jids WHERE jid IN ({jids}) + AND kind NOT IN ({kinds}) + '''.format(jids=', '.join('?' * len(jids)), + kinds=', '.join(kinds)) - results = self.cur.fetchone() - if results is not None: - result = results.time - else: - result = None - return result + # fetchone() returns always at least one Row with all + # attributes set to None because of the MAX() function + return self.con.execute(sql, (*jids,)).fetchone().time - def get_room_last_message_time(self, jid): + def get_room_last_message_time(self, account, jid): """ - Return FASTLY last time (in seconds since EPOCH) for which we had logs - for that room from rooms_last_message_time table - """ - try: - jid_id = self.get_jid_id(jid, 'ROOM') - except exceptions.PysqliteOperationalError: - # Error trying to create a new jid_id. This means there is no log - return None - where_sql = 'jid_id = %s' % jid_id - self.cur.execute(''' - SELECT time FROM rooms_last_message_time - WHERE (%s) - ''' % (where_sql)) + Get the timestamp of the last message we received in a room. - results = self.cur.fetchone() - if results is not None: - result = results.time - else: - result = None - return result + :param account: The account + + :param jid: The jid for which we request the last timestamp + + returns a timestamp or None + """ + sql = ''' + SELECT time FROM rooms_last_message_time + NATURAL JOIN jids WHERE jid = ? + ''' + + row = self.con.execute(sql, (jid,)).fetchone() + if not row: + return self.get_last_date_that_has_logs(account, jid) + return row.time def set_room_last_message_time(self, jid, time): """ @@ -829,31 +864,6 @@ class Logger: (jid_id, time) self.simple_commit(sql) - def _build_contact_where(self, account, jid): - """ - Build the where clause for a jid, including metacontacts jid(s) if any - """ - where_sql = '' - jid_tuple = () - # will return empty list if jid is not associated with - # any metacontacts - family = gajim.contacts.get_metacontacts_family(account, jid) - if family: - for user in family: - try: - jid_id = self.get_jid_id(user['jid']) - except exceptions.PysqliteOperationalError: - continue - where_sql += 'jid_id = ?' - jid_tuple += (jid_id,) - if user != family[-1]: - where_sql += ' OR ' - else: # if jid was not associated with metacontacts - jid_id = self.get_jid_id(jid) - where_sql = 'jid_id = ?' - jid_tuple += (jid_id,) - return where_sql, jid_tuple - def save_transport_type(self, jid, type_): """ Save the type of the transport in DB diff --git a/gajim/history_window.py b/gajim/history_window.py index fb5190f65..a55f25373 100644 --- a/gajim/history_window.py +++ b/gajim/history_window.py @@ -30,6 +30,7 @@ from gi.repository import Gdk from gi.repository import GLib import time import calendar +import datetime from enum import IntEnum, unique @@ -304,7 +305,7 @@ class HistoryWindow: # select logs for last date we have logs with contact self.calendar.set_sensitive(True) last_log = \ - gajim.logger.get_last_date_that_has_logs(self.jid, self.account) + gajim.logger.get_last_date_that_has_logs(self.account, self.jid) date = time.localtime(last_log) @@ -356,20 +357,18 @@ class HistoryWindow: widget.select_day(1) return - # in gtk January is 1, in python January is 0, - # I want the second - # first day of month is 1 not 0 widget.clear_marks() month = gtkgui_helpers.make_gtk_month_python_month(month) - days_in_this_month = calendar.monthrange(year, month)[1] + try: - log_days = gajim.logger.get_days_with_logs(self.jid, year, month, - days_in_this_month, self.account) + log_days = gajim.logger.get_days_with_logs( + self.account, self.jid, year, month) except exceptions.PysqliteOperationalError as e: dialogs.ErrorDialog(_('Disk Error'), str(e)) return - for day in log_days: - widget.mark_day(day) + + for date in log_days: + widget.mark_day(date.day) def _get_string_show_from_constant_int(self, show): if show == ShowConstant.ONLINE: @@ -397,8 +396,11 @@ class HistoryWindow: self.last_time_printout = 0 show_status = self.show_status_checkbutton.get_active() + date = datetime.datetime(year, month, day) + conversation = gajim.logger.get_conversation_for_date( - self.jid, year, month, day, self.account) + self.account, self.jid, date) + for message in conversation: if not show_status and message.kind in (KindConstant.GCSTATUS, KindConstant.STATUS): @@ -537,13 +539,15 @@ class HistoryWindow: # This may leed to wrong self nick in the displayed history (Uggh!) account = list(gajim.contacts.get_accounts())[0] - year, month, day = False, False, False + date = None if self.search_in_date.get_active(): year, month, day = self.calendar.get_date() # integers month = gtkgui_helpers.make_gtk_month_python_month(month) + date = datetime.datetime(year, month, day) show_status = self.show_status_checkbutton.get_active() - results = gajim.logger.search_log(jid, text, account, year, month, day) + + results = gajim.logger.search_log(account, jid, text, date) #FIXME: # add "subject: | message: " in message column if kind is single # also do we need show at all? (we do not search on subject)