Add MAM for MUCs

gajim/common/caps_cache.py

@@ -33,10 +33,12 @@ through ClientCaps objects which are hold by contact instances.
 
 import base64
 import hashlib
+from collections import namedtuple
 
 import logging
 log = logging.getLogger('gajim.c.caps_cache')
 
+import nbxmpp
 from nbxmpp import (NS_XHTML_IM, NS_ESESSION, NS_CHATSTATES,
     NS_JINGLE_ICE_UDP, NS_JINGLE_RTP_AUDIO, NS_JINGLE_RTP_VIDEO,
     NS_JINGLE_FILE_TRANSFER_5)
@@ -44,7 +46,7 @@ from nbxmpp import (NS_XHTML_IM, NS_ESESSION, NS_CHATSTATES,
 FEATURE_BLACKLIST = [NS_CHATSTATES, NS_XHTML_IM, NS_ESESSION,
     NS_JINGLE_ICE_UDP, NS_JINGLE_RTP_AUDIO, NS_JINGLE_RTP_VIDEO,
     NS_JINGLE_FILE_TRANSFER_5]
-
+from gajim.common import app
 # Query entry status codes
 NEW = 0
 QUERIED = 1
@@ -56,12 +58,15 @@ FAKED = 3 # allow NullClientCaps to behave as it has a cached item
 ################################################################################
 
 capscache = None
+muc_caps_cache = None
 def initialize(logger):
     """
     Initialize this module
     """
     global capscache
+    global muc_caps_cache
     capscache = CapsCache(logger)
+    muc_caps_cache = MucCapsCache()
 
 def client_supports(client_caps, requested_feature):
     lookup_item = client_caps.get_cache_lookup_strategy()
@@ -438,3 +443,51 @@ class CapsCache(object):
         key = (hash_method, hash)
         if key in self.__cache:
             del self.__cache[key]
+
+
+class MucCapsCache:
+
+    DiscoInfo = namedtuple('DiscoInfo', ['identities', 'features', 'data'])
+
+    def __init__(self):
+        self.cache = {}
+
+    def append(self, stanza):
+        jid = stanza.getFrom()
+        identities, features, data = [], [], []
+        query_childs = stanza.getQueryChildren()
+        if not query_childs:
+            app.log('gajim.muc').warning('%s returned empty disco info', jid)
+            return
+
+        for child in query_childs:
+            if child.getName() == 'identity':
+                attr = {}
+                for key in child.getAttrs().keys():
+                    attr[key] = child.getAttr(key)
+                identities.append(attr)
+            elif child.getName() == 'feature':
+                features.append(child.getAttr('var'))
+            elif child.getName() == 'x':
+                if child.getNamespace() == nbxmpp.NS_DATA:
+                    data.append(nbxmpp.DataForm(node=child))
+
+        self.cache[jid] = self.DiscoInfo(identities, features, data)
+
+    def is_cached(self, jid):
+        return jid in self.cache
+
+    def supports(self, jid, feature):
+        if jid in self.cache:
+            if feature in self.cache[jid].features:
+                return True
+        return False
+
+    def has_mam(self, jid):
+        try:
+            if nbxmpp.NS_MAM_2 in self.cache[jid].features:
+                return True
+            if nbxmpp.NS_MAM_1 in self.cache[jid].features:
+                return True
+        except (KeyError, AttributeError):
+            return False

gajim/common/connection.py

@@ -42,6 +42,7 @@ import locale
 import hmac
 import hashlib
 import json
+from functools import partial
 
 try:
     randomsource = random.SystemRandom()
@@ -2583,6 +2584,11 @@ class Connection(CommonConnection, ConnectionHandlers):
             # Never join a room when invisible
             return
 
+        self.discoverMUC(
+            room_jid, partial(self._join_gc, nick, show, room_jid,
+                              password, change_nick, rejoin))
+
+    def _join_gc(self, nick, show, room_jid, password, change_nick, rejoin):
         # Check time first in the FAST table
         last_date = app.logger.get_room_last_message_time(
             self.name, room_jid)
@@ -2599,11 +2605,19 @@ class Connection(CommonConnection, ConnectionHandlers):
         if app.config.get('send_sha_in_gc_presence'):
             p = self.add_sha(p)
         self.add_lang(p)
-        if not change_nick:
-            t = p.setTag(nbxmpp.NS_MUC + ' x')
+        if change_nick:
+            self.connection.send(p)
+            return
+
+        t = p.setTag(nbxmpp.NS_MUC + ' x')
+        if muc_caps_cache.has_mam(room_jid):
+            # The room is MAM capable dont get MUC History
+            t.setTag('history', {'maxchars': '0'})
+        else:
+            # Request MUC History (not MAM)
             tags = {}
             timeout = app.config.get_per('rooms', room_jid,
-                'muc_restore_timeout')
+                                         'muc_restore_timeout')
             if timeout is None or timeout == -2:
                 timeout = app.config.get('muc_restore_timeout')
             if last_date == 0 and timeout >= 0:
@@ -2621,8 +2635,9 @@ class Connection(CommonConnection, ConnectionHandlers):
                 tags['maxstanzas'] = nb
             if tags:
                 t.setTag('history', tags)
-            if password:
-                t.setTagData('password', password)
+
+        if password:
+            t.setTagData('password', password)
         self.connection.send(p)
 
     def _nec_gc_message_outgoing(self, obj):

gajim/common/connection_handlers.py

@@ -47,6 +47,7 @@ from gajim.common import helpers
 from gajim.common import app
 from gajim.common import dataforms
 from gajim.common import jingle_xtls
+from gajim.common.caps_cache import muc_caps_cache
 from gajim.common.commands import ConnectionCommands
 from gajim.common.pubsub import ConnectionPubSub
 from gajim.common.protocol.caps import ConnectionCaps
@@ -96,6 +97,29 @@ class ConnectionDisco:
         id_ = self._discover(nbxmpp.NS_DISCO_INFO, jid, node, id_prefix)
         self.disco_info_ids.append(id_)
 
+    def discoverMUC(self, jid, callback):
+        disco_info = nbxmpp.Iq(typ='get', to=jid, queryNS=nbxmpp.NS_DISCO_INFO)
+        self.connection.SendAndCallForResponse(
+            disco_info, self.received_muc_info, {'callback': callback})
+
+    def received_muc_info(self, conn, stanza, callback):
+        if nbxmpp.isResultNode(stanza):
+            app.log('gajim.muc').info(
+                'Received MUC DiscoInfo for %s', stanza.getFrom())
+            muc_caps_cache.append(stanza)
+            callback()
+        else:
+            error = stanza.getError()
+            if error == 'item-not-found':
+                # Groupchat does not exist
+                callback()
+                return
+            app.nec.push_incoming_event(
+                InformationEvent(None, conn=self,
+                                 level='error',
+                                 pri_txt=_('Unable to join Groupchat'),
+                                 sec_txt=error))
+
     def request_register_agent_info(self, agent):
         if not self.connection or self.connected < 2:
             return None
@@ -760,6 +784,8 @@ class ConnectionHandlersBase:
             self._nec_message_received)
         app.ged.register_event_handler('mam-message-received', ged.CORE,
             self._nec_message_received)
+        app.ged.register_event_handler('mam-gc-message-received', ged.CORE,
+            self._nec_message_received)
         app.ged.register_event_handler('decrypted-message-received', ged.CORE,
             self._nec_decrypted_message_received)
         app.ged.register_event_handler('gc-message-received', ged.CORE,
@@ -776,6 +802,8 @@ class ConnectionHandlersBase:
             self._nec_message_received)
         app.ged.remove_event_handler('mam-message-received', ged.CORE,
             self._nec_message_received)
+        app.ged.remove_event_handler('mam-gc-message-received', ged.CORE,
+            self._nec_message_received)
         app.ged.remove_event_handler('decrypted-message-received', ged.CORE,
             self._nec_decrypted_message_received)
         app.ged.remove_event_handler('gc-message-received', ged.CORE,

gajim/common/connection_handlers_events.py

@@ -1052,12 +1052,12 @@ class MamMessageReceivedEvent(nec.NetworkIncomingEvent, HelperEvent):
         :stanza:        Complete stanza Node
         :forwarded:     Forwarded Node
         :result:        Result Node
-        :unique_id:     The unique stable id
         '''
         self._set_base_event_vars_as_attributes(base_event)
         self.additional_data = {}
         self.encrypted = False
         self.groupchat = False
+        self.nick = None
 
     def generate(self):
         archive_jid = self.stanza.getFrom()
@@ -1070,7 +1070,6 @@ class MamMessageReceivedEvent(nec.NetworkIncomingEvent, HelperEvent):
         self.msg_ = self.forwarded.getTag('message', protocol=True)
 
         if self.msg_.getType() == 'groupchat':
-            log.info('Received groupchat message from user archive')
             return False
 
         # use stanza-id as unique-id
@@ -1081,7 +1080,6 @@ class MamMessageReceivedEvent(nec.NetworkIncomingEvent, HelperEvent):
             return
 
         self.msgtxt = self.msg_.getTagData('body')
-        self.query_id = self.result.getAttr('queryid')
 
         frm = self.msg_.getFrom()
         # Some servers dont set the 'to' attribute when
@@ -1123,26 +1121,81 @@ class MamMessageReceivedEvent(nec.NetworkIncomingEvent, HelperEvent):
         return True
 
     def get_unique_id(self):
+        stanza_id = self.get_stanza_id(self.result, query=True)
         if self.conn.get_own_jid().bareMatch(self.msg_.getFrom()):
-            # On our own Messages we have to check for both
-            # stanza-id and origin-id, because other resources
-            # maybe not support origin-id
-            stanza_id = None
-            if self.result.getNamespace() == nbxmpp.NS_MAM_2:
-                # Only mam:2 ensures valid stanza-id
-                stanza_id = self.get_stanza_id(self.result, query=True)
-
-            # try always to get origin-id because its a message
-            # we sent.
+            # message we sent
             origin_id = self.msg_.getOriginID()
-
             return stanza_id, origin_id
 
         # A message we received
-        elif self.result.getNamespace() == nbxmpp.NS_MAM_2:
-            # Only mam:2 ensures valid stanza-id
-            return self.get_stanza_id(self.result, query=True), None
-        return None, None
+        return stanza_id, None
+
+class MamGcMessageReceivedEvent(nec.NetworkIncomingEvent, HelperEvent):
+    name = 'mam-gc-message-received'
+    base_network_events = ['raw-mam-message-received']
+
+    def __init__(self, name, base_event):
+        '''
+        Pre-Generated attributes on self:
+
+        :conn:          Connection instance
+        :stanza:        Complete stanza Node
+        :forwarded:     Forwarded Node
+        :result:        Result Node
+        '''
+        self._set_base_event_vars_as_attributes(base_event)
+        self.additional_data = {}
+        self.encrypted = False
+        self.groupchat = True
+        self.kind = KindConstant.GC_MSG
+
+    def generate(self):
+        self.room_jid = self.stanza.getFrom()
+        self.msg_ = self.forwarded.getTag('message', protocol=True)
+
+        if self.msg_.getType() != 'groupchat':
+            return False
+
+        self.unique_id = self.get_stanza_id(self.result, query=True)
+
+        # Check for duplicates
+        if app.logger.find_stanza_id(self.unique_id):
+            return
+
+        self.msgtxt = self.msg_.getTagData('body')
+        self.with_ = self.msg_.getFrom().getStripped()
+        self.nick = self.msg_.getFrom().getResource()
+
+        # Get the real jid if we have it
+        self.real_jid = None
+        muc_user = self.msg_.getTag('x', namespace=nbxmpp.NS_MUC_USER)
+        if muc_user is not None:
+            self.real_jid = muc_user.getTagAttr('item', 'jid')
+
+        delay = self.forwarded.getTagAttr(
+            'delay', 'stamp', namespace=nbxmpp.NS_DELAY2)
+        if delay is None:
+            log.error('Received MAM message without timestamp')
+            return
+
+        self.timestamp = helpers.parse_datetime(
+            delay, check_utc=True, epoch=True)
+        if self.timestamp is None:
+            log.error('Received MAM message with invalid timestamp: %s', delay)
+            return
+
+        # Save timestamp added by the user
+        user_delay = self.msg_.getTagAttr(
+            'delay', 'stamp', namespace=nbxmpp.NS_DELAY2)
+        if user_delay is not None:
+            self.user_timestamp = helpers.parse_datetime(
+                user_delay, check_utc=True, epoch=True)
+            if self.user_timestamp is None:
+                log.warning('Received MAM message with '
+                            'invalid user timestamp: %s', user_delay)
+
+        log.debug('Received mam-gc-message: unique id: %s', self.unique_id)
+        return True
 
 class MamDecryptedMessageReceivedEvent(nec.NetworkIncomingEvent, HelperEvent):
     name = 'mam-decrypted-message-received'
@@ -1156,6 +1209,9 @@ class MamDecryptedMessageReceivedEvent(nec.NetworkIncomingEvent, HelperEvent):
 
         self.get_oob_data(self.msg_)
 
+        if self.groupchat:
+            return True
+
         self.is_pm = app.logger.jid_is_room_jid(self.with_.getStripped())
         if self.is_pm is None:
             # Check if this event is triggered after a disco, so we dont
@@ -1288,6 +1344,12 @@ class MessageReceivedEvent(nec.NetworkIncomingEvent, HelperEvent):
         if result and result.getNamespace() in (nbxmpp.NS_MAM,
                                                 nbxmpp.NS_MAM_1,
                                                 nbxmpp.NS_MAM_2):
+
+            if result.getAttr('queryid') not in self.conn.mam_query_ids:
+                log.warning('Invalid MAM Message: unknown query id')
+                log.debug(self.stanza)
+                return
+
             forwarded = result.getTag('forwarded',
                                       namespace=nbxmpp.NS_FORWARD,
                                       protocol=True)
@@ -1300,8 +1362,7 @@ class MessageReceivedEvent(nec.NetworkIncomingEvent, HelperEvent):
                              conn=self.conn,
                              stanza=self.stanza,
                              forwarded=forwarded,
-                             result=result,
-                             stanza_id=self.unique_id))
+                             result=result))
             return
 
         # Mediated invitation?

gajim/common/message_archiving.py

@@ -36,13 +36,13 @@ class ConnectionArchive313:
         self.archiving_313_supported = False
         self.mam_awaiting_disco_result = {}
         self.iq_answer = []
-        self.mam_query_date = None
-        self.mam_query_id = None
+        self.mam_query_ids = []
         app.nec.register_incoming_event(ev.MamMessageReceivedEvent)
         app.ged.register_event_handler('archiving-finished-legacy', ged.CORE,
             self._nec_result_finished)
         app.ged.register_event_handler('archiving-finished', ged.CORE,
             self._nec_result_finished)
+        app.nec.register_incoming_event(ev.MamGcMessageReceivedEvent)
         app.ged.register_event_handler('agent-info-error-received', ged.CORE,
             self._nec_agent_info_error)
         app.ged.register_event_handler('agent-info-received', ged.CORE,
@@ -101,28 +101,43 @@ class ConnectionArchive313:
                     None, disco=True, **vars(msg_obj)))
         del self.mam_awaiting_disco_result[obj.jid]
 
-    def _nec_result_finished(self, obj):
-        if obj.conn.name != self.name:
+    def _result_finished(self, conn, stanza, query_id, start_date, groupchat):
+        if not nbxmpp.isResultNode(stanza):
+            log.error('Error on MAM query: %s', stanza.getError())
             return
 
-        if obj.query_id != self.mam_query_id:
+        fin = stanza.getTag('fin')
+        if fin is None:
+            log.error('Malformed MAM query result received: %s', stanza)
             return
 
-        set_ = obj.fin.getTag('set', namespace=nbxmpp.NS_RSM)
-        if set_:
-            last = set_.getTagData('last')
-            complete = obj.fin.getAttr('complete')
-            if last:
+        if fin.getAttr('queryid') != query_id:
+            log.error('Result with unknown query id received')
+            return
+
+        set_ = fin.getTag('set', namespace=nbxmpp.NS_RSM)
+        if set_ is None:
+            log.error(
+                'Malformed MAM query result received (no "set" Node): %s',
+                stanza)
+            return
+
+        last = set_.getTagData('last')
+        complete = fin.getAttr('complete')
+        if last is not None:
+            if not groupchat:
                 app.config.set_per('accounts', self.name, 'last_mam_id', last)
-                if complete != 'true':
-                    self.request_archive(self.get_query_id(), after=last)
-            if complete == 'true':
-                self.mam_query_id = None
-                if self.mam_query_date:
-                    app.config.set_per(
-                        'accounts', self.name,
-                        'mam_start_date', self.mam_query_date.timestamp())
-                    self.mam_query_date = None
+            if complete != 'true':
+                query_id = self.get_query_id()
+                query = self.get_archive_query(query_id, after=last)
+                self.send_archive_query(query, query_id, groupchat=groupchat)
+
+        if complete == 'true':
+            self.mam_query_ids.remove(query_id)
+            if not groupchat and start_date is not None:
+                app.config.set_per(
+                    'accounts', self.name,
+                    'mam_start_date', start_date.timestamp())
 
     def _nec_mam_decrypted_message_received(self, obj):
         if obj.conn.name != self.name:
@@ -132,33 +147,55 @@ class ConnectionArchive313:
         duplicate = app.logger.search_for_duplicate(
             obj.with_, obj.timestamp, obj.msgtxt)
         if duplicate:
-            return
-        app.logger.insert_into_logs(
-            obj.with_, obj.timestamp, obj.kind,
-            unread=False,
-            message=obj.msgtxt,
-            additional_data=obj.additional_data,
-            stanza_id=obj.unique_id)
+            # dont propagate the event further
+            return True
+        app.logger.insert_into_logs(obj.with_,
+                                    obj.timestamp,
+                                    obj.kind,
+                                    unread=False,
+                                    message=obj.msgtxt,
+                                    contact_name=obj.nick,
+                                    additional_data=obj.additional_data,
+                                    stanza_id=obj.unique_id)
 
     def get_query_id(self):
-        self.mam_query_id = self.connection.getAnID()
-        return self.mam_query_id
+        query_id = self.connection.getAnID()
+        self.mam_query_ids.append(query_id)
+        return query_id
 
     def request_archive_on_signin(self):
         mam_id = app.config.get_per('accounts', self.name, 'last_mam_id')
+        start_date = None
         query_id = self.get_query_id()
         if mam_id:
-            self.request_archive(query_id, after=mam_id)
+            log.info('MAM query after %s:', mam_id)
+            query = self.get_archive_query(query_id, after=mam_id)
         else:
             # First Start, we request the last week
-            self.mam_query_date = datetime.utcnow() - timedelta(days=7)
-            log.info('First start: query archive start: %s', self.mam_query_date)
-            self.request_archive(query_id, start=self.mam_query_date)
+            start_date = datetime.utcnow() - timedelta(days=7)
+            log.info('First start: query archive start: %s', start_date)
+            query = self.get_archive_query(query_id, start=start_date)
+        self.send_archive_query(query, query_id, start_date)
 
-    def request_archive(self, query_id, start=None, end=None, with_=None,
-                        after=None, max_=30):
+    def request_archive_on_muc_join(self, jid):
+        # First Start, we request one month
+        start_date = datetime.utcnow() - timedelta(days=30)
+        query_id = self.get_query_id()
+        log.info('First join: query archive start: %s', start_date)
+        query = self.get_archive_query(query_id, jid=jid, start=start_date)
+        self.send_archive_query(query, query_id, start_date, groupchat=True)
+
+    def send_archive_query(self, query, query_id, start_date=None,
+                           groupchat=False):
+        self.connection.SendAndCallForResponse(
+            query, self._result_finished, {'query_id': query_id,
+                                           'start_date': start_date,
+                                           'groupchat': groupchat})
+
+    def get_archive_query(self, query_id, jid=None, start=None, end=None, with_=None,
+                          after=None, max_=30):
         namespace = self.archiving_namespace
-        iq = nbxmpp.Iq('set')
+        iq = nbxmpp.Iq('set', to=jid)
         query = iq.addChild('query', namespace=namespace)
         form = query.addChild(node=nbxmpp.DataForm(typ='submit'))
         field = nbxmpp.DataField(typ='hidden',
@@ -184,9 +221,7 @@ class ConnectionArchive313:
         if after:
             set_.setTagData('after', after)
         query.setAttr('queryid', query_id)
-        id_ = self.connection.getAnID()
-        iq.setID(id_)
-        self.connection.send(iq)
+        return iq
 
     def request_archive_preferences(self):
         if not app.account_is_connected(self.name):

gajim/groupchat_control.py

@@ -47,6 +47,7 @@ from gajim import vcard
 from gajim import cell_renderer_image
 from gajim import dataforms_widget
 from gajim.common.const import AvatarSize
+from gajim.common.caps_cache import muc_caps_cache
 import nbxmpp
 
 from enum import IntEnum, unique
@@ -478,6 +479,8 @@ class GroupchatControl(ChatControlBase):
             self._nec_gc_presence_received)
         app.ged.register_event_handler('gc-message-received', ged.GUI1,
             self._nec_gc_message_received)
+        app.ged.register_event_handler('mam-decrypted-message-received',
+            ged.GUI1, self._nec_mam_decrypted_message_received)
         app.ged.register_event_handler('vcard-published', ged.GUI1,
             self._nec_vcard_published)
         app.ged.register_event_handler('update-gc-avatar', ged.GUI1,
@@ -1053,6 +1056,17 @@ class GroupchatControl(ChatControlBase):
                                 obj.contact.name, obj.contact.avatar_sha)
         self.draw_avatar(obj.contact)
 
+    def _nec_mam_decrypted_message_received(self, obj):
+        if not obj.groupchat:
+            return
+        if obj.room_jid != self.room_jid:
+            return
+        self.print_conversation(
+            obj.msgtxt, contact=obj.nick,
+            tim=obj.timestamp, encrypted=obj.encrypted,
+            msg_stanza_id=obj.unique_id,
+            additional_data=obj.additional_data)
+
     def _nec_gc_message_received(self, obj):
         if obj.room_jid != self.room_jid or obj.conn.name != self.account:
             return
@@ -1455,6 +1469,11 @@ class GroupchatControl(ChatControlBase):
             GLib.source_remove(self.autorejoin)
         self.autorejoin = None
 
+        if muc_caps_cache.has_mam(self.room_jid):
+            # Request MAM
+            app.connections[self.account].request_archive_on_muc_join(
+                self.room_jid)
+
         app.gc_connected[self.account][self.room_jid] = True
         ChatControlBase.got_connected(self)
         self.list_treeview.set_model(self.model)