diff --git a/src/common/contacts.py b/src/common/contacts.py
index 1f2ad1127..b98a8906d 100644
--- a/src/common/contacts.py
+++ b/src/common/contacts.py
@@ -159,6 +159,8 @@ class Contacts:
 			return
 		if contact in self._contacts[account][contact.jid]:
 			self._contacts[account][contact.jid].remove(contact)
+		if len(self._contacts[account][contact.jid]) == 0:
+			del self._contacts[account][contact.jid]
 
 	def remove_jid(self, account, jid):
 		'''Removes all contacts for a given jid'''