Rework Emoji implementation

- Use emoji data from a generated dict based on the offical unicode docs,
this makes it easier to update in the future
- Rewrite the emoji chooser
- Add a search field to the emoji chooser
- The emoji chooser is loaded async
- Update to current Unicode 11 Noto theme

gajim/chat_control_base.py

@@ -42,7 +42,7 @@ from gajim.gtk.util import convert_rgb_to_hex
 from gajim import notify
 import re
 
-from gajim import emoticons
+from gajim.gtk.emoji_chooser import emoji_chooser
 from gajim.common import events
 from gajim.common import app
 from gajim.common import helpers
@@ -676,7 +676,7 @@ class ChatControlBase(MessageControl, ChatCommandProcessor, CommandTools):
         event.keyval == Gdk.KEY_KP_Enter:  # ENTER
             message_textview = widget
             message_buffer = message_textview.get_buffer()
-            emoticons.replace_with_codepoint(message_buffer)
+            message_textview.replace_emojis()
             start_iter, end_iter = message_buffer.get_bounds()
             message = message_buffer.get_text(start_iter, end_iter, False)
             xhtml = self.msg_textview.get_xhtml()
@@ -1055,10 +1055,9 @@ class ChatControlBase(MessageControl, ChatCommandProcessor, CommandTools):
         if not self.parent_win:
             return
 
-        popover = emoticons.get_popover()
+        emoji_chooser.text_widget = self.msg_textview
-        popover.set_callbacks(self.msg_textview)
         emoticons_button = self.xml.get_object('emoticons_button')
-        emoticons_button.set_popover(popover)
+        emoticons_button.set_popover(emoji_chooser)
 
     def on_color_menuitem_activate(self, widget):
         color_dialog = Gtk.ColorChooserDialog(None, self.parent_win.window)

gajim/common/configpaths.py

@@ -129,6 +129,7 @@ class ConfigPaths:
         source_paths = [
             ('DATA', os.path.join(basedir, 'data')),
             ('STYLE', os.path.join(basedir, 'data', 'style')),
+            ('EMOTICONS', os.path.join(basedir, 'data', 'emoticons')),
             ('GUI', os.path.join(basedir, 'data', 'gui')),
             ('ICONS', os.path.join(basedir, 'data', 'icons')),
             ('HOME', os.path.expanduser('~')),

gajim/common/helpers.py

@@ -1479,36 +1479,29 @@ def version_condition(current_version, required_version):
 
 def get_available_emoticon_themes():
     emoticons_themes = []
-    emoticons_data_path = os.path.join(configpaths.get('DATA'), 'emoticons')
+    if sys.platform not in ('win32', 'darwin'):
-    font_theme_path = os.path.join(
+        # Colored emoji fonts only supported on Linux
-        configpaths.get('DATA'), 'emoticons', 'font-emoticons', 'emoticons_theme.py')
+        emoticons_themes.append('font')
+
+    files = []
+    with os.scandir(configpaths.get('EMOTICONS')) as scan:
+        for entry in scan:
+            if not entry.is_dir():
+                continue
+            with os.scandir(entry.path) as scan_theme:
+                for theme in scan_theme:
+                    if theme.is_file():
+                        files.append(theme.name)
 
-    folders = os.listdir(emoticons_data_path)
     if os.path.isdir(configpaths.get('MY_EMOTS')):
-        folders += os.listdir(configpaths.get('MY_EMOTS'))
+        files += os.listdir(configpaths.get('MY_EMOTS'))
 
-    file = 'emoticons_theme.py'
+    for file in files:
-    if os.name == 'nt' and not os.path.exists(font_theme_path):
+        if file.endswith('.png'):
-        # When starting Gajim from source .py files are available
+            emoticons_themes.append(file[:-4])
-        # We test this with font-emoticons and fallback to .pyc files otherwise
-        file = 'emoticons_theme.pyc'
-
-    for theme in folders:
-        theme_path = os.path.join(emoticons_data_path, theme, file)
-        if os.path.exists(theme_path):
-            emoticons_themes.append(theme)
     emoticons_themes.sort()
     return emoticons_themes
 
-def get_emoticon_theme_path(theme):
-    emoticons_data_path = os.path.join(configpaths.get('DATA'), 'emoticons', theme)
-    if os.path.exists(emoticons_data_path):
-        return emoticons_data_path
-
-    emoticons_user_path = os.path.join(configpaths.get('MY_EMOTS'), theme)
-    if os.path.exists(emoticons_user_path):
-        return emoticons_user_path
-
 def call_counter(func):
     def helper(self, restart=False):
         if restart:

gajim/conversation_textview.py

@@ -34,7 +34,7 @@ from gi.repository import GObject
 from gi.repository import GLib
 import time
 import os
-from gajim import dialogs
+
 import queue
 import urllib
 
@@ -43,12 +43,14 @@ from gajim.gtk import util
 from gajim.gtk.util import load_icon
 from gajim.gtk.util import get_builder
 from gajim.gtk.util import get_cursor
+from gajim.gtk.emoji_data import emoji_pixbufs
+from gajim.gtk.emoji_data import is_emoji
+from gajim.gtk.emoji_data import get_emoji_pixbuf
 from gajim.common import app
 from gajim.common import helpers
 from gajim.common import i18n
 from calendar import timegm
 from gajim.common.fuzzyclock import FuzzyClock
-from gajim import emoticons
 from gajim.common.const import StyleAttr
 
 from gajim.htmltextview import HtmlTextView
@@ -197,7 +199,6 @@ class ConversationTextview(GObject.GObject):
         self.tv.set_left_margin(2)
         self.tv.set_right_margin(2)
         self.handlers = {}
-        self.images = []
         self.image_cache = {}
         self.xep0184_marks = {}
         # self.last_sent_message_id = msg_stanza_id
@@ -927,16 +928,32 @@ class ConversationTextview(GObject.GObject):
         else:
             end_iter = buffer_.get_end_iter()
 
-        pixbuf = emoticons.get_pixbuf(special_text)
+        theme = app.config.get('emoticons_theme')
-        if app.config.get('emoticons_theme') and pixbuf and graphics:
+        show_emojis = theme and theme != 'font'
+        if show_emojis and graphics and is_emoji(special_text):
             # it's an emoticon
-            anchor = buffer_.create_child_anchor(end_iter)
+            if emoji_pixbufs.complete:
-            img = TextViewImage(anchor,
+                # only search for the pixbuf if we are sure
-                GLib.markup_escape_text(special_text))
+                # that loading is completed
-            img.set_from_pixbuf(pixbuf)
+                pixbuf = get_emoji_pixbuf(special_text)
-            img.show()
+                if pixbuf is None:
-            self.images.append(img)
+                    buffer_.insert(end_iter, special_text)
-            self.tv.add_child_at_anchor(img, anchor)
+                else:
+                    pixbuf = pixbuf.copy()
+                    anchor = buffer_.create_child_anchor(end_iter)
+                    anchor.plaintext = special_text
+                    img = Gtk.Image.new_from_pixbuf(pixbuf)
+                    img.show()
+                    self.tv.add_child_at_anchor(img, anchor)
+            else:
+                # Set marks and save them so we can replace the emojis
+                # once the loading is complete
+                start_mark = buffer_.create_mark(None, end_iter, True)
+                buffer_.insert(end_iter, special_text)
+                end_mark = buffer_.create_mark(None, end_iter, True)
+                emoji_pixbufs.append_marks(
+                    self.tv, start_mark, end_mark, special_text)
+
         elif special_text.startswith('www.') or \
             special_text.startswith('ftp.') or \
             text_is_valid_uri and not is_xhtml_link:

gajim/data/gui/emoji_chooser.ui

@@ -0,0 +1,124 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Generated with glade 3.22.1 -->
+<interface>
+  <requires lib="gtk+" version="3.20"/>
+  <object class="GtkBox" id="box">
+    <property name="visible">True</property>
+    <property name="can_focus">False</property>
+    <property name="orientation">vertical</property>
+    <child>
+      <object class="GtkSearchEntry" id="search">
+        <property name="visible">True</property>
+        <property name="can_focus">True</property>
+        <property name="primary_icon_name">edit-find-symbolic</property>
+        <property name="primary_icon_activatable">False</property>
+        <property name="primary_icon_sensitive">False</property>
+        <signal name="search-changed" handler="_search_changed" swapped="no"/>
+        <signal name="stop-search" handler="_search_changed" swapped="no"/>
+      </object>
+      <packing>
+        <property name="expand">False</property>
+        <property name="fill">True</property>
+        <property name="position">0</property>
+      </packing>
+    </child>
+    <child>
+      <object class="GtkStack" id="stack">
+        <property name="visible">True</property>
+        <property name="can_focus">False</property>
+        <child>
+          <object class="GtkBox">
+            <property name="visible">True</property>
+            <property name="can_focus">False</property>
+            <property name="orientation">vertical</property>
+            <child>
+              <object class="GtkScrolledWindow">
+                <property name="visible">True</property>
+                <property name="can_focus">True</property>
+                <property name="shadow_type">in</property>
+                <property name="min_content_width">350</property>
+                <property name="min_content_height">300</property>
+                <child>
+                  <object class="GtkViewport">
+                    <property name="visible">True</property>
+                    <property name="can_focus">False</property>
+                    <child>
+                      <object class="GtkBox" id="section_box">
+                        <property name="visible">True</property>
+                        <property name="can_focus">False</property>
+                        <property name="orientation">vertical</property>
+                        <child>
+                          <placeholder/>
+                        </child>
+                      </object>
+                    </child>
+                  </object>
+                </child>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">0</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="name">list</property>
+          </packing>
+        </child>
+        <child>
+          <object class="GtkBox">
+            <property name="visible">True</property>
+            <property name="can_focus">False</property>
+            <property name="orientation">vertical</property>
+            <property name="homogeneous">True</property>
+            <child>
+              <object class="GtkImage">
+                <property name="visible">True</property>
+                <property name="can_focus">False</property>
+                <property name="valign">end</property>
+                <property name="pixel_size">72</property>
+                <property name="icon_name">edit-find-symbolic</property>
+                <style>
+                  <class name="dim-label"/>
+                </style>
+              </object>
+              <packing>
+                <property name="expand">True</property>
+                <property name="fill">True</property>
+                <property name="position">0</property>
+              </packing>
+            </child>
+            <child>
+              <object class="GtkLabel">
+                <property name="visible">True</property>
+                <property name="can_focus">False</property>
+                <property name="valign">start</property>
+                <property name="margin_top">12</property>
+                <property name="label" translatable="yes">No Results Found</property>
+                <style>
+                  <class name="dim-label"/>
+                  <class name="bold24"/>
+                </style>
+              </object>
+              <packing>
+                <property name="expand">False</property>
+                <property name="fill">True</property>
+                <property name="position">1</property>
+              </packing>
+            </child>
+          </object>
+          <packing>
+            <property name="name">not-found</property>
+            <property name="position">1</property>
+          </packing>
+        </child>
+      </object>
+      <packing>
+        <property name="expand">True</property>
+        <property name="fill">True</property>
+        <property name="position">1</property>
+      </packing>
+    </child>
+  </object>
+</interface>

gajim/data/style/gajim.css

@@ -64,9 +64,13 @@
 popover#EmoticonPopover button { background: none; border: none; box-shadow:none; padding: 0px;}
 popover#EmoticonPopover button > label { font-size: 24px; }
 popover#EmoticonPopover flowboxchild > label { font-size: 24px; }
-popover#EmoticonPopover notebook label { font-size: 24px; }
-popover#EmoticonPopover flowbox { padding-left: 5px; padding-right: 6px; }
 popover#EmoticonPopover flowboxchild { padding-top: 5px; padding-bottom: 5px; }
+popover#EmoticonPopover scrolledwindow { border: none; }
+popover#EmoticonPopover { padding: 5px; background-color: @theme_unfocused_base_color}
+
+.emoji-chooser-heading { font-size: 13px; font-weight: bold; padding: 5px;}
+.emoji-chooser-flowbox { padding-left: 5px; padding-right: 11px; }
+.emoji-modifier-chooser-flowbox { padding-left: 5px; }
 
 /* HistorySyncAssistant */
 #HistorySyncAssistant list { border: 1px solid; border-color: @borders; }
@@ -187,6 +191,7 @@ list.settings > row > box {
 /* Text style */
 
 .bold16 { font-size: 16px; font-weight: bold; }
+.bold24 { font-size: 24px; font-weight: bold; }
 .large-header { font-size: 20px; font-weight: bold; }
 .status-away { color: #ff8533;}
 .status-dnd { color: #e62e00;}

gajim/emoticons.py

@@ -1,321 +0,0 @@
-# -*- coding:utf-8 -*-
-#
-# Copyright (C) 2017  Philipp Hörist <philipp AT hoerist.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import os
-import sys
-import logging
-from collections import OrderedDict
-from importlib.machinery import SourceFileLoader
-
-from gi.repository import GdkPixbuf, Gtk, GLib
-
-MODIFIER_MAX_CHILDREN_PER_LINE = 6
-MAX_CHILDREN_PER_LINE = 10
-MIN_HEIGHT = 200
-
-pixbufs = dict()
-codepoints = dict()
-popover_instance = None
-
-log = logging.getLogger('gajim.emoticons')
-
-class SubPixbuf:
-
-    height = 24
-    width = 24
-    columns = 20
-
-    def __init__(self, path):
-        self.cur_column = 0
-        self.src_x = 0
-        self.src_y = 0
-        self.atlas = GdkPixbuf.Pixbuf.new_from_file(path)
-
-    def get_pixbuf(self):
-        self.src_x = self.cur_column * self.width
-
-        subpixbuf = self.atlas.new_subpixbuf(self.src_x, self.src_y, self.width, self.height)
-
-        if self.cur_column == self.columns - 1:
-            self.src_y += self.width
-            self.cur_column = 0
-        else:
-            self.cur_column += 1
-
-        return subpixbuf
-
-def load(path, ascii_emoticons):
-    module_name = 'emoticons_theme.py'
-    theme_path = os.path.join(path, module_name)
-    if sys.platform == 'win32' and not os.path.exists(theme_path):
-        module_name = 'emoticons_theme.pyc'
-        theme_path = os.path.join(path, module_name)
-
-    loader = SourceFileLoader(module_name, theme_path)
-    try:
-        theme = loader.load_module()
-    except FileNotFoundError:
-        log.exception('Emoticons theme not found')
-        return
-
-    if not theme.use_image:
-        # Use Font to display emoticons
-        set_popover(theme.emoticons, False)
-        return True
-
-    try:
-        sub = SubPixbuf(os.path.join(path, 'emoticons.png'))
-    except GLib.GError:
-        log.exception('Error while creating subpixbuf')
-        return False
-
-    def add_emoticon(codepoint_, sub, mod_list=None):
-        pix = sub.get_pixbuf()
-        for alternate in codepoint_:
-            if not ascii_emoticons:
-                try:
-                    alternate.encode('ascii')
-                    continue
-                except UnicodeEncodeError:
-                    pass
-            codepoints[alternate] = pix
-            if pix not in pixbufs:
-                pixbufs[pix] = alternate
-        if mod_list is not None:
-            mod_list.append(pix)
-        else:
-            pixbuf_list.append(pix)
-
-    popover_dict = OrderedDict()
-    try:
-        for category in theme.emoticons:
-            if not theme.emoticons[category]:
-                # Empty category
-                continue
-
-            pixbuf_list = []
-            for filename, codepoint_ in theme.emoticons[category]:
-                if codepoint_ is None:
-                    # Category image
-                    pixbuf_list.append(sub.get_pixbuf())
-                    continue
-                if not filename:
-                    # We have an emoticon with a modifier
-                    mod_list = []
-                    for _, mod_codepoint in codepoint_:
-                        add_emoticon(mod_codepoint, sub, mod_list)
-                    pixbuf_list.append(mod_list)
-                else:
-                    add_emoticon(codepoint_, sub)
-
-            popover_dict[category] = pixbuf_list
-
-    except Exception:
-        log.exception('Error while loading emoticon theme')
-        return
-
-    set_popover(popover_dict, True)
-
-    return True
-
-def set_popover(popover_dict, use_image):
-    global popover_instance
-    popover_instance = EmoticonPopover(popover_dict, use_image)
-
-def get_popover():
-    return popover_instance
-
-def get_pixbuf(codepoint_):
-    try:
-        return codepoints[codepoint_]
-    except KeyError:
-        return None
-
-def get_codepoint(pixbuf_):
-    try:
-        return pixbufs[pixbuf_]
-    except KeyError:
-        return None
-
-def replace_with_codepoint(buffer_):
-    if not pixbufs:
-        # We use font emoticons
-        return
-    iter_ = buffer_.get_start_iter()
-    pix = iter_.get_pixbuf()
-
-    def replace(pix):
-        if pix:
-            emote = get_codepoint(pix)
-            if not emote:
-                return
-            iter_2 = iter_.copy()
-            iter_2.forward_char()
-            buffer_.delete(iter_, iter_2)
-            buffer_.insert(iter_, emote)
-
-    replace(pix)
-    while iter_.forward_char():
-        pix = iter_.get_pixbuf()
-        replace(pix)
-
-class EmoticonPopover(Gtk.Popover):
-    def __init__(self, emoji_dict, use_image):
-        super().__init__()
-        self.set_name('EmoticonPopover')
-        self.text_widget = None
-        self.use_image = use_image
-
-        notebook = Gtk.Notebook()
-        self.add(notebook)
-        self.handler_id = self.connect('key_press_event', self.on_key_press)
-
-        for category in emoji_dict:
-            scrolled_window = Gtk.ScrolledWindow()
-            scrolled_window.set_min_content_height(MIN_HEIGHT)
-
-            flowbox = Gtk.FlowBox()
-            flowbox.set_max_children_per_line(MAX_CHILDREN_PER_LINE)
-            flowbox.connect('child_activated', self.on_emoticon_press)
-
-            scrolled_window.add(flowbox)
-
-            # Use first entry as a label for the notebook page
-            if self.use_image:
-                cat_image = Gtk.Image()
-                cat_image.set_from_pixbuf(emoji_dict[category][0])
-                notebook.append_page(scrolled_window, cat_image)
-            else:
-                notebook.append_page(scrolled_window, Gtk.Label(label=emoji_dict[category][0]))
-
-            # Populate the category with emojis
-            for pix in emoji_dict[category][1:]:
-                if isinstance(pix, list):
-                    widget = self.add_emoticon_modifier(pix)
-                else:
-                    if self.use_image:
-                        widget = Gtk.Image()
-                        widget.set_from_pixbuf(pix)
-                    else:
-                        widget = Gtk.Label(label=pix)
-                flowbox.add(widget)
-
-        notebook.show_all()
-
-    def add_emoticon_modifier(self, pixbuf_list):
-        button = Gtk.MenuButton()
-        button.set_relief(Gtk.ReliefStyle.NONE)
-
-        if self.use_image:
-            # We use the first item of the list as image for the button
-            button.get_child().set_from_pixbuf(pixbuf_list[0])
-        else:
-            button.remove(button.get_child())
-            label = Gtk.Label(label=pixbuf_list[0])
-            button.add(label)
-
-        button.connect('button-press-event', self.on_modifier_press)
-
-        popover = Gtk.Popover()
-        popover.set_name('EmoticonPopover')
-        popover.connect('key_press_event', self.on_key_press)
-
-        flowbox = Gtk.FlowBox()
-        flowbox.set_size_request(200, -1)
-        flowbox.set_max_children_per_line(MODIFIER_MAX_CHILDREN_PER_LINE)
-        flowbox.connect('child_activated', self.on_emoticon_press)
-
-        popover.add(flowbox)
-
-        for pix in pixbuf_list[1:]:
-            if self.use_image:
-                widget = Gtk.Image()
-                widget.set_from_pixbuf(pix)
-            else:
-                widget = Gtk.Label(label=pix)
-            flowbox.add(widget)
-
-        flowbox.show_all()
-        button.set_popover(popover)
-        return button
-
-    def set_callbacks(self, widget):
-        self.text_widget = widget
-        # Because the handlers getting disconnected when on_destroy() is called
-        # we connect them again
-        if self.handler_id:
-            self.disconnect(self.handler_id)
-        self.handler_id = self.connect('key_press_event', self.on_key_press)
-
-    def on_key_press(self, widget, event):
-        self.text_widget.grab_focus()
-
-    def on_modifier_press(self, button, event):
-        if event.button == 3:
-            button.get_popover().show()
-            button.get_popover().get_child().unselect_all()
-        if event.button == 1:
-            button.get_parent().emit('activate')
-            if self.use_image:
-                self.append_emoticon(button.get_child().get_pixbuf())
-            else:
-                self.append_emoticon(button.get_child().get_text())
-            return True
-
-    def on_emoticon_press(self, flowbox, child):
-        GLib.timeout_add(100, flowbox.unselect_all)
-        
-        if isinstance(child.get_child(), Gtk.MenuButton):
-            return
-
-        if self.use_image:
-            self.append_emoticon(child.get_child().get_pixbuf())
-        else:
-            self.append_emoticon(child.get_child().get_text())
-
-    def append_emoticon(self, pix):
-        self.text_widget.remove_placeholder()
-        buffer_ = self.text_widget.get_buffer()
-        if buffer_.get_char_count():
-            buffer_.insert_at_cursor(' ')
-            insert_mark = buffer_.get_insert()
-            insert_iter = buffer_.get_iter_at_mark(insert_mark)
-            if self.use_image:
-                buffer_.insert_pixbuf(insert_iter, pix)
-            else:
-                buffer_.insert(insert_iter, pix)
-            buffer_.insert_at_cursor(' ')
-        else: # we are the beginning of buffer
-            insert_mark = buffer_.get_insert()
-            insert_iter = buffer_.get_iter_at_mark(insert_mark)
-            if self.use_image:
-                buffer_.insert_pixbuf(insert_iter, pix)
-            else:
-                buffer_.insert(insert_iter, pix)
-            buffer_.insert_at_cursor(' ')
-
-    def do_destroy(self):
-        # Remove the references we hold to other objects
-        self.text_widget = None
-        # Even though we don't destroy the Popover, handlers are getting
-        # still disconnected, which makes the handler_id invalid
-        # FIXME: find out how we can prevent handlers getting disconnected
-        self.handler_id = None
-        # Never destroy, creating a new EmoticonPopover is expensive
-        return True
-
-

gajim/gtk/emoji_chooser.py

@@ -0,0 +1,382 @@
+# This file is part of Gajim.
+#
+# Gajim is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published
+# by the Free Software Foundation; version 3 only.
+#
+# Gajim is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Gajim. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+import logging
+import weakref
+from collections import OrderedDict
+
+from gi.repository import Gtk
+from gi.repository import GLib
+from gi.repository import GdkPixbuf
+
+from gajim.common import app
+from gajim.common import helpers
+from gajim.common import configpaths
+
+from gajim.gtk.util import get_builder
+from gajim.gtk.emoji_data import emoji_data
+from gajim.gtk.emoji_data import emoji_pixbufs
+from gajim.gtk.emoji_data import Emoji
+
+MODIFIER_MAX_CHILDREN_PER_LINE = 6
+MAX_CHILDREN_PER_LINE = 10
+
+log = logging.getLogger('gajim.emoji')
+
+
+class Section(Gtk.Box):
+    def __init__(self, name, search_entry, press_cb):
+        Gtk.Box.__init__(self, orientation=Gtk.Orientation.VERTICAL)
+        self._press_cb = press_cb
+        self.pixbuf_generator = None
+        self.heading = Gtk.Label(label=name)
+        self.heading.set_halign(Gtk.Align.START)
+        self.heading.get_style_context().add_class('emoji-chooser-heading')
+        self.add(self.heading)
+
+        self.flowbox = Gtk.FlowBox()
+        self.flowbox.get_style_context().add_class('emoji-chooser-flowbox')
+        self.flowbox.set_max_children_per_line(MAX_CHILDREN_PER_LINE)
+        self.flowbox.set_filter_func(self._filter_func, search_entry)
+        self.flowbox.connect('child-activated', press_cb)
+
+        self.add(self.flowbox)
+        self.show_all()
+
+    def _filter_func(self, child, search_entry):
+        name = search_entry.get_text()
+        if not name:
+            self.show()
+            return True
+
+        if name in child.desc:
+            self.show()
+            return True
+        return False
+
+    def add_emoji(self, codepoint, attrs):
+        # Return always True, this method is used for the emoji factory
+        # called by GLib.idle_add()
+        pixbuf = self._get_next_pixbuf()
+
+        variations = attrs.get('variations', None)
+        if variations is None:
+            if pixbuf is None:
+                return True
+            self.flowbox.add(EmojiChild(codepoint, pixbuf, attrs['desc']))
+            if pixbuf != 'font':
+                # We save the pixbuf for fast access if we need to
+                # replace a codepoint in the textview
+                emoji_pixbufs[codepoint] = pixbuf
+        else:
+            if pixbuf is not None:
+                chooser = self._get_emoji_modifier(
+                    codepoint, pixbuf, attrs)
+                chooser.flowbox.connect(
+                    'child-activated', self._press_cb)
+
+                if pixbuf != 'font':
+                    emoji_pixbufs[codepoint] = pixbuf
+
+                for codepoint, attrs in variations.items():
+                    pixbuf = self._get_next_pixbuf()
+                    if pixbuf is None:
+                        continue
+                    chooser.add_emoji(codepoint, pixbuf)
+
+                    if pixbuf != 'font':
+                        emoji_pixbufs[codepoint] = pixbuf
+            else:
+                # We dont have a image for the base codepoint
+                # so skip all modifiers of it
+                for codepoint, attrs in variations.items():
+                    pixbuf = self._get_next_pixbuf()
+        return True
+
+    def clear_emojis(self):
+        def _remove_emoji(emoji):
+            self.flowbox.remove(emoji)
+            emoji.destroy()
+        self.flowbox.foreach(_remove_emoji)
+
+    def _get_emoji_modifier(self, codepoint, pixbuf, attrs):
+        chooser = ModifierChooser()
+        modifier_button = EmojiModifierChild(codepoint, pixbuf, attrs['desc'])
+        modifier_button.button.set_popover(chooser)
+        self.flowbox.add(modifier_button)
+        return chooser
+
+    def _get_next_pixbuf(self):
+        if self.pixbuf_generator is None:
+            return 'font'
+        return next(self.pixbuf_generator, False)
+
+
+class EmojiChild(Gtk.FlowBoxChild):
+    def __init__(self, codepoint, pixbuf, desc):
+        Gtk.FlowBoxChild.__init__(self)
+        self.desc = desc
+        self.codepoint = codepoint
+        self.pixbuf = pixbuf
+        if pixbuf == 'font':
+            self.add(Gtk.Label(label=codepoint))
+        else:
+            self.add(Gtk.Image.new_from_pixbuf(pixbuf))
+        self.set_tooltip_text(desc)
+        self.show_all()
+
+    def get_emoji(self):
+        if self.pixbuf != 'font':
+            pixbuf = self.get_child().get_pixbuf()
+            pixbuf = pixbuf.scale_simple(Emoji.INPUT_SIZE,
+                                         Emoji.INPUT_SIZE,
+                                         GdkPixbuf.InterpType.HYPER)
+            return self.codepoint, pixbuf
+        return self.codepoint, None
+
+
+class EmojiModifierChild(Gtk.FlowBoxChild):
+    def __init__(self, codepoint, pixbuf, desc):
+        Gtk.FlowBoxChild.__init__(self)
+        self.desc = desc
+        self.codepoint = codepoint
+        self.pixbuf = pixbuf
+
+        self.button = Gtk.MenuButton()
+        self.button.set_relief(Gtk.ReliefStyle.NONE)
+        self.button.connect('button-press-event', self._button_press)
+
+        if pixbuf == 'font':
+            self.button.remove(self.button.get_child())
+            label = Gtk.Label(label=codepoint)
+            self.button.add(label)
+        else:
+            self.button.get_child().set_from_pixbuf(pixbuf)
+
+        self.set_tooltip_text(desc)
+        self.add(self.button)
+        self.show_all()
+
+    def _button_press(self, button, event):
+        if event.button == 3:
+            button.get_popover().show()
+            button.get_popover().get_child().unselect_all()
+            return True
+        if event.button == 1:
+            self.get_parent().emit('child-activated', self)
+            return True
+
+    def get_emoji(self):
+        if self.pixbuf != 'font':
+            pixbuf = self.button.get_child().get_pixbuf()
+            pixbuf = pixbuf.scale_simple(Emoji.INPUT_SIZE,
+                                         Emoji.INPUT_SIZE,
+                                         GdkPixbuf.InterpType.HYPER)
+            return self.codepoint, pixbuf
+        return self.codepoint, None
+
+
+class ModifierChooser(Gtk.Popover):
+    def __init__(self):
+        Gtk.Popover.__init__(self)
+        self.set_name('EmoticonPopover')
+
+        self.flowbox = Gtk.FlowBox()
+        self.flowbox.get_style_context().add_class(
+            'emoji-modifier-chooser-flowbox')
+        self.flowbox.set_size_request(200, -1)
+        self.flowbox.set_max_children_per_line(MODIFIER_MAX_CHILDREN_PER_LINE)
+        self.flowbox.show()
+        self.add(self.flowbox)
+
+    def add_emoji(self, codepoint, pixbuf):
+        self.flowbox.add(EmojiChild(codepoint, pixbuf, None))
+
+
+class EmojiChooser(Gtk.Popover):
+
+    _section_names = [
+        'Smileys & People',
+        'Animals & Nature',
+        'Food & Drink',
+        'Travel & Places',
+        'Activities',
+        'Objects',
+        'Symbols',
+        'Flags'
+    ]
+
+    def __init__(self):
+        super().__init__()
+        self.set_name('EmoticonPopover')
+        self._text_widget = None
+        self._load_source_id = None
+
+        self._builder = get_builder('emoji_chooser.ui')
+        self._search = self._builder.get_object('search')
+        self._stack = self._builder.get_object('stack')
+
+        self._sections = OrderedDict()
+        for name in self._section_names:
+            self._sections[name] = Section(
+                name, self._search, self._on_emoticon_press)
+
+        section_box = self._builder.get_object('section_box')
+        for section in self._sections.values():
+            section_box.add(section)
+
+        self.add(self._builder.get_object('box'))
+
+        self.connect('key-press-event', self._key_press)
+        self._builder.connect_signals(self)
+        self.show_all()
+
+    @property
+    def text_widget(self):
+        return self._text_widget
+
+    @text_widget.setter
+    def text_widget(self, value):
+        # Hold only a weak reference so if we can destroy
+        # the ChatControl
+        self._text_widget = weakref.ref(value)
+
+    def _key_press(self, widget, event):
+        return self._search.handle_event(event)
+
+    def _search_changed(self, entry):
+        for section in self._sections.values():
+            section.hide()
+            section.flowbox.invalidate_filter()
+        self._switch_stack()
+
+    def _clear_sections(self):
+        for section in self._sections.values():
+            section.clear_emojis()
+
+    def _switch_stack(self):
+        for section in self._sections.values():
+            if section.is_visible():
+                self._stack.set_visible_child_name('list')
+                return
+        self._stack.set_visible_child_name('not-found')
+
+    def _get_current_theme(self):
+        theme = app.config.get('emoticons_theme')
+        if not theme:
+            log.warning('No emoji theme set')
+            return
+
+        themes = helpers.get_available_emoticon_themes()
+        if theme not in themes:
+            if sys.platform not in ('win32', 'darwin'):
+                app.config.set('emoticons_theme', 'font')
+                theme = 'font'
+            else:
+                # Win/Mac fallback to noto
+                app.config.set('emoticons_theme', 'noto')
+                theme = 'noto'
+        return theme
+
+    @staticmethod
+    def _get_emoji_theme_path(theme):
+        if theme == 'font':
+            return 'font'
+        emoticons_data_path = os.path.join(configpaths.get('EMOTICONS'),
+                                           theme,
+                                           '%s.png' % theme)
+        if os.path.exists(emoticons_data_path):
+            return emoticons_data_path
+
+        emoticons_user_path = os.path.join(configpaths.get('MY_EMOTS'),
+                                           '%s.png' % theme)
+        if os.path.exists(emoticons_user_path):
+            return emoticons_user_path
+
+        log.warning('Could not find emoji theme: %s', theme)
+
+    def load(self):
+        theme = self._get_current_theme()
+        path = self._get_emoji_theme_path(theme)
+        if not theme or path is None:
+            self._clear_sections()
+            emoji_pixbufs.clear()
+            return
+
+        # Attach pixbuf generator
+        pixbuf_generator = None
+        if theme != 'font':
+            pixbuf_generator = self._get_next_pixbuf(path)
+        for section in self._sections.values():
+            section.pixbuf_generator = pixbuf_generator
+
+        if self._load_source_id is not None:
+            GLib.source_remove(self._load_source_id)
+
+        # When we change emoji theme
+        self._clear_sections()
+        emoji_pixbufs.clear()
+
+        factory = self._emoji_factory()
+        self._load_source_id = GLib.idle_add(lambda: next(factory, False),
+                                             priority=GLib.PRIORITY_LOW)
+
+    def _emoji_factory(self):
+        for codepoint, attrs in emoji_data.items():
+            if not attrs['fully-qualified']:
+                # We dont add these to the UI
+                continue
+
+            section = self._sections[attrs['group']]
+            yield section.add_emoji(codepoint, attrs)
+        self._load_source_id = None
+        emoji_pixbufs.complete = True
+
+    def _get_next_pixbuf(self, path):
+        src_x = src_y = cur_column = 0
+        atlas = GdkPixbuf.Pixbuf.new_from_file(path)
+
+        while True:
+            src_x = cur_column * Emoji.PARSE_WIDTH
+            subpixbuf = atlas.new_subpixbuf(src_x, src_y,
+                                            Emoji.PARSE_WIDTH,
+                                            Emoji.PARSE_HEIGHT)
+
+            if list(subpixbuf.get_pixels())[0:4] == [0, 0, 0, 255]:
+                # top left corner is a black pixel means image is missing
+                subpixbuf = None
+
+            if cur_column == Emoji.PARSE_COLUMNS - 1:
+                src_y += Emoji.PARSE_WIDTH
+                cur_column = 0
+            else:
+                cur_column += 1
+
+            yield subpixbuf
+
+    def _on_emoticon_press(self, flowbox, child):
+        GLib.timeout_add(100, flowbox.unselect_child, child)
+        codepoint, pixbuf = child.get_emoji()
+        self._text_widget().insert_emoji(codepoint, pixbuf)
+
+    def do_destroy(self):
+        # Remove the references we hold to other objects
+        self._text_widget = None
+        # Never destroy, creating a new EmoticonPopover is expensive
+        return True
+
+
+emoji_chooser = EmojiChooser()

gajim/gtk/preferences.py

@@ -557,8 +557,8 @@ class Preferences(Gtk.ApplicationWindow):
         else:
             app.config.set('emoticons_theme', emot_theme)
 
-        app.interface.init_emoticons()
+        from gajim.gtk.emoji_chooser import emoji_chooser
-        app.interface.make_regexps()
+        emoji_chooser.load()
         self.toggle_emoticons()
 
     def toggle_emoticons(self):

gajim/gui_interface.py

@@ -101,7 +101,6 @@ from gajim.common.connection_handlers_events import (
 from gajim.common.modules.httpupload import HTTPUploadProgressEvent
 from gajim.common.connection import Connection
 from gajim.common.file_props import FilesProp
-from gajim import emoticons
 from gajim.common.const import AvatarSize, SSLError, PEPEventType
 from gajim.common.const import ACTIVITIES, MOODS
 
@@ -111,6 +110,7 @@ from threading import Thread
 from gajim.common import ged
 from gajim.common.caps_cache import muc_caps_cache
 
+from gajim.gtk.emoji_data import emoji_data, emoji_ascii_data
 from gajim.gtk import JoinGroupchatWindow
 from gajim.gtk import ErrorDialog
 from gajim.gtk import WarningDialog
@@ -1794,8 +1794,8 @@ class Interface:
     @property
     def emot_and_basic_re(self):
         if not self._emot_and_basic_re:
-            self._emot_and_basic_re = re.compile(self.emot_and_basic,
+            self._emot_and_basic_re = re.compile(
-                    re.IGNORECASE + re.UNICODE)
+                self.emot_and_basic, re.IGNORECASE)
         return self._emot_and_basic_re
 
     @property
@@ -1867,43 +1867,13 @@ class Interface:
             basic_pattern += formatting
         self.basic_pattern = basic_pattern
 
-        emoticons_pattern = ''
-        if app.config.get('emoticons_theme'):
-            # When an emoticon is bordered by an alpha-numeric character it is
-            # NOT expanded.  e.g., foo:) NO, foo :) YES, (brb) NO, (:)) YES, etc
-            # We still allow multiple emoticons side-by-side like :P:P:P
-            # sort keys by length so :qwe emot is checked before :q
-            keys = sorted(emoticons.codepoints.keys(), key=len, reverse=True)
-            emoticons_pattern_prematch = ''
-            emoticons_pattern_postmatch = ''
-            emoticon_length = 0
-            for emoticon in keys: # travel thru emoticons list
-                emoticon_escaped = re.escape(emoticon) # escape regexp metachars
-                # | means or in regexp
-                emoticons_pattern += emoticon_escaped + '|'
-                if (emoticon_length != len(emoticon)):
-                    # Build up expressions to match emoticons next to others
-                    emoticons_pattern_prematch  = \
-                        emoticons_pattern_prematch[:-1]  + ')|(?<='
-                    emoticons_pattern_postmatch = \
-                        emoticons_pattern_postmatch[:-1] + ')|(?='
-                    emoticon_length = len(emoticon)
-                emoticons_pattern_prematch += emoticon_escaped  + '|'
-                emoticons_pattern_postmatch += emoticon_escaped + '|'
-            # We match from our list of emoticons, but they must either have
-            # whitespace, or another emoticon next to it to match successfully
-            # [\w.] alphanumeric and dot (for not matching 8) in (2.8))
-            emoticons_pattern = '|' + r'(?:(?<![\w.]' + \
-                emoticons_pattern_prematch[:-1] + '))' + '(?:' + \
-                emoticons_pattern[:-1] + ')' + r'(?:(?![\w]' + \
-                emoticons_pattern_postmatch[:-1] + '))'
-
         # because emoticons match later (in the string) they need to be after
         # basic matches that may occur earlier
-        self.emot_and_basic = basic_pattern + emoticons_pattern
+        emoticons = emoji_data.get_regex()
-
+        if app.config.get('ascii_emoticons'):
-        # needed for xhtml display
+            emoticons += '|%s' % emoji_ascii_data.get_regex()
-        self.emot_only = emoticons_pattern
+            pass
+        self.emot_and_basic = '%s|%s' % (basic_pattern, emoticons)
 
         # at least one character in 3 parts (before @, after @, after .)
         self.sth_at_sth_dot_sth = r'\S+@\S+\.\S*[^\s)?]'
@@ -1912,30 +1882,6 @@ class Interface:
         self.invalid_XML_chars = '[\x00-\x08]|[\x0b-\x0c]|[\x0e-\x1f]|'\
             '[\ud800-\udfff]|[\ufffe-\uffff]'
 
-    def init_emoticons(self):
-        emot_theme = app.config.get('emoticons_theme')
-        ascii_emoticons = app.config.get('ascii_emoticons')
-        if not emot_theme:
-            return
-
-        themes = helpers.get_available_emoticon_themes()
-        if emot_theme not in themes:
-            if 'font-emoticons' in themes:
-                emot_theme = 'font-emoticons'
-                app.config.set('emoticons_theme', 'font-emoticons')
-            else:
-                app.config.set('emoticons_theme', '')
-                return
-
-        path = helpers.get_emoticon_theme_path(emot_theme)
-        if not emoticons.load(path, ascii_emoticons):
-            WarningDialog(
-                    _('Emoticons disabled'),
-                    _('Your configured emoticons theme could not be loaded.'
-                      ' See the log for more details.'),
-                    transient_for=app.get_app_window('Preferences'))
-            app.config.set('emoticons_theme', '')
-            return
 
 ################################################################################
 ### Methods for opening new messages controls
@@ -2667,7 +2613,6 @@ class Interface:
         self.basic_pattern = None
         self.emot_and_basic = None
         self.sth_at_sth_dot_sth = None
-        self.emot_only = None
 
         cfg_was_read = parser.read()
 
@@ -2814,7 +2759,9 @@ class Interface:
             # set the icon to all windows
             Gtk.Window.set_default_icon_list(pixs)
 
-        self.init_emoticons()
+        # Init emoji_chooser
+        from gajim.gtk.emoji_chooser import emoji_chooser
+        emoji_chooser.load()
         self.make_regexps()
 
         # get transports type from DB

gajim/htmltextview.py

@@ -1081,6 +1081,20 @@ class HtmlTextView(Gtk.TextView):
                 search_iter.forward_char()
         return selection
 
+    def replace_emojis(self, start_mark, end_mark, pixbuf, codepoint):
+        buffer_ = self.get_buffer()
+        start_iter = buffer_.get_iter_at_mark(start_mark)
+        end_iter = buffer_.get_iter_at_mark(end_mark)
+        buffer_.delete(start_iter, end_iter)
+
+        anchor = buffer_.create_child_anchor(start_iter)
+        anchor.plaintext = codepoint
+        emoji = Gtk.Image.new_from_pixbuf(pixbuf)
+        emoji.show()
+        self.add_child_at_anchor(emoji, anchor)
+        buffer_.delete_mark(start_mark)
+        buffer_.delete_mark(end_mark)
+
 change_cursor = None
 
 if __name__ == '__main__':

gajim/message_textview.py

@@ -350,6 +350,53 @@ class MessageTextView(Gtk.TextView):
         else:
             return None
 
+    def replace_emojis(self):
+        theme = app.config.get('emoticons_theme')
+        if not theme or theme == 'font':
+            return
+
+        def replace(anchor):
+            if anchor is None:
+                return
+            image = anchor.get_widgets()[0]
+            if hasattr(image, 'codepoint'):
+                # found emoji
+                self.replace_char_at_iter(iter_, image.codepoint)
+                image.destroy()
+
+        iter_ = self.get_buffer().get_start_iter()
+        replace(iter_.get_child_anchor())
+
+        while iter_.forward_char():
+            replace(iter_.get_child_anchor())
+
+    def replace_char_at_iter(self, iter_, new_char):
+        buffer_ = self.get_buffer()
+        iter_2 = iter_.copy()
+        iter_2.forward_char()
+        buffer_.delete(iter_, iter_2)
+        buffer_.insert(iter_, new_char)
+
+    def insert_emoji(self, codepoint, pixbuf):
+        self.remove_placeholder()
+        buffer_ = self.get_buffer()
+        if buffer_.get_char_count():
+            # buffer contains text
+            buffer_.insert_at_cursor(' ')
+
+        insert_mark = buffer_.get_insert()
+        insert_iter = buffer_.get_iter_at_mark(insert_mark)
+
+        if pixbuf is None:
+            buffer_.insert(insert_iter, codepoint)
+        else:
+            anchor = buffer_.create_child_anchor(insert_iter)
+            image = Gtk.Image.new_from_pixbuf(pixbuf)
+            image.codepoint = codepoint
+            image.show()
+            self.add_child_at_anchor(image, anchor)
+        buffer_.insert_at_cursor(' ')
+
     def destroy(self):
         GLib.idle_add(gc.collect)
 

setup.py

@@ -214,8 +214,7 @@ class update_po(Command):
 
 
 package_data_activities = ['data/activities/*/*/*.png']
-package_data_emoticons = ['data/emoticons/*/emoticons_theme.py',
+package_data_emoticons = ['data/emoticons/*/*.png',
-                          'data/emoticons/*/*.png',
                           'data/emoticons/*/LICENSE']
 package_data_gui = ['data/gui/*.ui']
 package_data_icons = ['data/icons/hicolor/*/*/*.png',