Determine delay timestamp correctly

- Check the from attr on the delay node to determine if its a user timestamp or from the server
- Dont use user timestamp for sorting
- Record the user timestamp in additional data so its saved to the database

Fixes #9444

gajim/common/modules/mam.py

@@ -130,12 +130,13 @@ class MAM:
             raise nbxmpp.NodeProcessed
 
         # Timestamp parsing
-        timestamp = parse_delay(forwarded)
-        if timestamp is None:
+        # Most servers dont set the 'from' attr, so we cant check for it
+        delay_timestamp = parse_delay(forwarded)
+        if delay_timestamp is None:
+            log.warning('No timestamp on MAM message')
+            log.warning(stanza)
             raise nbxmpp.NodeProcessed
 
-        user_timestamp = parse_delay(message)
-
         # Fix for self messaging
         if not groupchat:
             to = message.getTo()
@@ -180,8 +181,7 @@ class MAM:
             {'conn': self._con,
              'additional_data': {},
              'encrypted': False,
-             'timestamp': timestamp,
-             'user_timestamp': user_timestamp,
+             'timestamp': delay_timestamp,
              'self_message': self_message,
              'groupchat': groupchat,
              'muc_pm': muc_pm,
@@ -248,7 +248,12 @@ class MAM:
             # For example Chatstates, Receipts, Chatmarkers
             log.debug(event.message.getProperties())
             return
-        log.debug(event.msgtxt)
+
+        user_timestamp = parse_delay(event.stanza)
+        if user_timestamp is not None:
+            # Record it as a user timestamp
+            event.additional_data.set_value(
+                'gajim', 'user_timestamp', user_timestamp)
 
         event.correct_id = parse_correction(event.message)
         parse_oob(event.message, event.additional_data)

gajim/common/modules/message.py

@@ -223,15 +223,40 @@ class Message:
         except nbxmpp.NodeProcessed:
             return
 
-        timestamp, delayed = parse_delay(event.stanza), True
-        if timestamp is None:
+        subject = event.stanza.getSubject()
+        groupchat = event.mtype == 'groupchat'
+        muc_subject = subject and groupchat
+
+        # Determine timestamps
+        if groupchat:
+            delay_entity_jid = event.jid
+        else:
+            delay_entity_jid = self._con.get_own_jid().getDomain()
+
+        if muc_subject:
+            # MUC Subjects can have a delay timestamp
+            # to indicate when the user has set the subject,
+            # the 'from' attr on these delays is the MUC server
+            # but we treat it as user timestamp
             timestamp = time.time()
-            delayed = False
+            user_timestamp = parse_delay(event.stanza, from_=delay_entity_jid)
+
+        else:
+            timestamp = parse_delay(event.stanza, from_=delay_entity_jid)
+            if timestamp is None:
+                timestamp = time.time()
+
+            user_timestamp = parse_delay(event.stanza,
+                                         not_from=[delay_entity_jid])
+
+        if user_timestamp is not None:
+            event.additional_data.set_value(
+                'gajim', 'user_timestamp', user_timestamp)
 
         event_attr = {
             'popup': False,
             'msg_log_id': None,
-            'subject': event.stanza.getSubject(),
+            'subject': subject,
             'displaymarking': parse_securitylabel(event.stanza),
             'attention': parse_attention(event.stanza),
             'correct_id': parse_correction(event.stanza),
@@ -239,7 +264,7 @@ class Message:
             'form_node': parse_form(event.stanza),
             'xhtml': parse_xhtml(event.stanza),
             'timestamp': timestamp,
-            'delayed': delayed,
+            'delayed': user_timestamp is not None,
         }
         parse_oob(event.stanza, event.additional_data)
 
@@ -257,7 +282,7 @@ class Message:
                     event.session, event.fjid, timestamp)
             return
 
-        if event.mtype == 'groupchat':
+        if groupchat:
             app.nec.push_incoming_event(GcMessageReceivedEvent(
                 None,
                 conn=self._con,

gajim/common/modules/misc.py

@@ -65,18 +65,45 @@ def parse_eme(stanza):
 
 # XEP-0203: Delayed Delivery
 
-def parse_delay(stanza, epoch=True, convert='utc'):
-    timestamp = None
-    delay = stanza.getTagAttr(
-        'delay', 'stamp', namespace=nbxmpp.NS_DELAY2)
-    if delay is not None:
-        timestamp = parse_datetime(delay, check_utc=True,
+def parse_delay(stanza, epoch=True, convert='utc', from_=None, not_from=None):
+    '''
+    Returns the first valid delay timestamp that matches
+
+    :param epoch:      Returns the timestamp as epoch
+
+    :param convert:    Converts the timestamp to either utc or local
+
+    :param from_:      Matches only delays that have the according
+                       from attr set
+
+    :param not_from:   Matches only delays that have the according
+                       from attr not set
+    '''
+    delays = stanza.getTags('delay', namespace=nbxmpp.NS_DELAY2)
+
+    for delay in delays:
+        stamp = delay.getAttr('stamp')
+        if stamp is None:
+            log.warning('Invalid timestamp received: %s', stamp)
+            log.warning(stanza)
+            continue
+
+        delay_from = delay.getAttr('from')
+        if from_ is not None:
+            if delay_from != from_:
+                continue
+        if not_from is not None:
+            if delay_from in not_from:
+                continue
+
+        timestamp = parse_datetime(stamp, check_utc=True,
                                    epoch=epoch, convert=convert)
         if timestamp is None:
-            log.warning('Invalid timestamp received: %s', delay)
+            log.warning('Invalid timestamp received: %s', stamp)
             log.warning(stanza)
+            continue
 
-    return timestamp
+        return timestamp
 
 
 # XEP-0066: Out of Band Data

test/no_gui/unit/test_helpers.py

@@ -0,0 +1,30 @@
+import unittest
+
+import nbxmpp
+
+from gajim.common.modules.misc import parse_delay
+
+class TestHelpers(unittest.TestCase):
+
+    def test_parse_delay(self):
+
+        node = """
+        <message>
+            <delay xmlns='urn:xmpp:delay' from='capulet.com' stamp='2002-09-10T23:08:25Z' />
+            <delay xmlns='urn:xmpp:delay' from='romeo.com' stamp='2010-09-10T23:08:25Z' />
+            <delay xmlns='urn:xmpp:delay' stamp='2015-09-10T23:08:25Z' />
+        </message>
+        """
+        message = nbxmpp.Node(node=node)
+
+        timestamp = parse_delay(message)
+        self.assertEqual(timestamp, 1031699305.0)
+
+        timestamp = parse_delay(message, from_='capulet.com')
+        self.assertEqual(timestamp, 1031699305.0)
+
+        timestamp = parse_delay(message, from_='romeo.com')
+        self.assertEqual(timestamp, 1284160105.0)
+
+        timestamp = parse_delay(message, not_from=['romeo.com'])
+        self.assertEqual(timestamp, 1031699305.0)