diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 21c257c45..6e9948f65 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -10,7 +10,7 @@ stages:
 run-test:
   stage: test
   script:
-    - python3 setup.py test_nogui
+    - python3 setup.py test -s test.no_gui -q
 
 run-mypy:
   stage: test
diff --git a/setup.cfg b/setup.cfg
index 03445e3f9..bfec00edd 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -18,6 +18,7 @@ classifiers =
 [options]
 python_requires = >=3.5
 packages = find:
+test_suite = test
 install_requires =
     cssutils>=1.0.2
     keyring
diff --git a/setup.py b/setup.py
index 9830a5062..58b67d251 100644
--- a/setup.py
+++ b/setup.py
@@ -176,27 +176,6 @@ class install(_install):
         _install.run(self)
 
 
-class test(Command):
-    description = "Run all tests"
-    user_options = []
-
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        exit(subprocess.call("./test/runtests.py"))
-
-
-class test_nogui(test):
-    description = "Run tests without GUI"
-
-    def run(self):
-        exit(subprocess.call(["./test/runtests.py", "-n"]))
-
-
 class update_po(Command):
     description = "Update po files"
     user_options = []
@@ -229,8 +208,6 @@ setup(
     cmdclass={
         'build_py': build,
         'install': install,
-        'test': test,
-        'test_nogui': test_nogui,
         'update_po': update_po,
     },
     entry_points={
diff --git a/test/integration/test_gui_event_integration.py b/test/broken/integration/test_gui_event_integration.py
similarity index 100%
rename from test/integration/test_gui_event_integration.py
rename to test/broken/integration/test_gui_event_integration.py
diff --git a/test/integration/test_roster.py b/test/broken/integration/test_roster.py
similarity index 100%
rename from test/integration/test_roster.py
rename to test/broken/integration/test_roster.py
diff --git a/test/test_pluginmanager.py b/test/broken/test_pluginmanager.py
similarity index 100%
rename from test/test_pluginmanager.py
rename to test/broken/test_pluginmanager.py
diff --git a/test/unit/test_jingle.py b/test/broken/unit/test_jingle.py
similarity index 100%
rename from test/unit/test_jingle.py
rename to test/broken/unit/test_jingle.py
diff --git a/test/unit/test_sessions.py b/test/broken/unit/test_sessions.py
similarity index 100%
rename from test/unit/test_sessions.py
rename to test/broken/unit/test_sessions.py
diff --git a/test/unit/test_socks5.py b/test/broken/unit/test_socks5.py
similarity index 99%
rename from test/unit/test_socks5.py
rename to test/broken/unit/test_socks5.py
index c7f85aa0f..67d57ca97 100644
--- a/test/unit/test_socks5.py
+++ b/test/broken/unit/test_socks5.py
@@ -167,8 +167,6 @@ class TestSocks5(unittest.TestCase):
         self._check_inout()
 
 
-
-
 if __name__ == '__main__':
 
     unittest.main()
diff --git a/test/gtk/htmltextview.py b/test/gtk/htmltextview.py
index 4e0a48db9..a49bc6897 100644
--- a/test/gtk/htmltextview.py
+++ b/test/gtk/htmltextview.py
@@ -15,7 +15,6 @@ from gajim.gtk.util import get_cursor
 from gajim.conversation_textview import ConversationTextview
 from gajim.gui_interface import Interface
 
-
 caps_cache.capscache = MagicMock()
 app.plugin_manager = MagicMock()
 app.logger = MagicMock()
@@ -24,6 +23,7 @@ app.interface = Interface()
 change_cursor = None
 htmlview = ConversationTextview(None)
 
+
 def on_textview_motion_notify_event(widget, event):
     """
     Change the cursor to a hand when we are over a mail or an url
@@ -52,7 +52,6 @@ def handler(texttag, widget, event, iter_, kind):
     if event.type == Gdk.EventType.BUTTON_PRESS:
         pass
 
-htmlview.tv.hyperlink_handler = htmlview.hyperlink_handler
 
 htmlview.print_real_text(None, xhtml='<div>'
 '<span style="color: red; text-decoration:underline">Hello</span><br/>\n'
diff --git a/test/integration/test_resolver.py b/test/integration/test_resolver.py
index f33a33241..2354f70cf 100644
--- a/test/integration/test_resolver.py
+++ b/test/integration/test_resolver.py
@@ -1,8 +1,6 @@
 import unittest
 
-import time
-
-import lib
+from test import lib
 lib.setup_env()
 
 from gi.repository import GLib
@@ -14,7 +12,8 @@ JABBERCZ_TXT_NAME = '_xmppconnect.jabber.cz'
 JABBERCZ_SRV_NAME = '_xmpp-client._tcp.jabber.cz'
 
 TEST_LIST = [(NONSENSE_NAME, 'srv', False),
-        (JABBERCZ_SRV_NAME, 'srv', True)]
+             (JABBERCZ_SRV_NAME, 'srv', True)]
+
 
 class TestResolver(unittest.TestCase):
     '''
@@ -45,9 +44,9 @@ class TestResolver(unittest.TestCase):
 
     def _runGR(self, name, type_):
         self.resolver.resolve(
-                host = name,
-                type_ = type_,
-                on_ready = self._myonready)
+                host=name,
+                type_=type_,
+                on_ready=self._myonready)
 
         self.main_loop.run()
 
diff --git a/test/lib/__init__.py b/test/lib/__init__.py
index ba1367a1b..d3816c14d 100644
--- a/test/lib/__init__.py
+++ b/test/lib/__init__.py
@@ -1,26 +1,15 @@
-import sys
 import os
-import getopt
 
-use_x = True
-shortargs = 'hnv:'
-longargs = 'help no-x verbose='
-opts, args = getopt.getopt(sys.argv[1:], shortargs, longargs.split())
-for o, a in opts:
-    if o in ('-n', '--no-x'):
-        use_x = False
-
-gajim_root = os.path.join(os.path.abspath(os.path.dirname(__file__)), '../..')
-
-# look for modules in the CWD, then gajim/test/lib, then gajim/gajim,
-# then everywhere else
-sys.path.insert(1, gajim_root)
-sys.path.insert(1, gajim_root + '/test/lib')
+from tempfile import gettempdir
 
 # a temporary version of ~/.gajim for testing
-configdir = gajim_root + '/test/tmp'
+configdir = os.path.join(gettempdir(), 'gajim')
+os.makedirs(configdir, exist_ok=True)
+
 # plugins config dir
 pluginsconfigdir = configdir + '/pluginsconfig'
+# theme config directory
+themedir = configdir + '/theme'
 
 # define _ for i18n
 import builtins
@@ -28,7 +17,7 @@ builtins._ = lambda x: x
 
 from gajim.common.contacts import LegacyContactsAPI
 
-def setup_env():
+def setup_env(use_x=True):
     # wipe config directory
     if os.path.isdir(configdir):
         import shutil
@@ -36,6 +25,7 @@ def setup_env():
 
     os.mkdir(configdir)
     os.mkdir(pluginsconfigdir)
+    os.mkdir(themedir)
 
     from gajim.common import configpaths
     configpaths.set_config_root(configdir)
@@ -47,13 +37,10 @@ def setup_env():
     import logging
     logging.basicConfig()
 
-    configpaths.override_path('DATA', gajim_root + '/gajim/data')
     app.use_x = use_x
     app.contacts = LegacyContactsAPI()
     app.connections = {}
 
     if use_x:
-        from gajim import gtkgui_helpers
-        gtkgui_helpers.GUI_DIR = gajim_root + '/gajim/data/gui'
         from gajim.application import GajimApplication
         app.app = GajimApplication()
diff --git a/test/lib/gajim_mocks.py b/test/lib/gajim_mocks.py
index 5f03d835b..b5ffa9562 100644
--- a/test/lib/gajim_mocks.py
+++ b/test/lib/gajim_mocks.py
@@ -2,7 +2,7 @@
 Module with dummy classes for Gajim specific unit testing
 '''
 
-from mock import Mock
+from .mock import Mock
 from gajim.common import app
 from gajim.common import ged
 
diff --git a/test/no_gui/__init__.py b/test/no_gui/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/no_gui/unit/__init__.py b/test/no_gui/unit/__init__.py
new file mode 100644
index 000000000..0252a7b2f
--- /dev/null
+++ b/test/no_gui/unit/__init__.py
@@ -0,0 +1,5 @@
+'''
+
+This package just contains plain unit tests
+
+'''
diff --git a/test/unit/test_account.py b/test/no_gui/unit/test_account.py
similarity index 94%
rename from test/unit/test_account.py
rename to test/no_gui/unit/test_account.py
index 60c6eca96..75d94542e 100644
--- a/test/unit/test_account.py
+++ b/test/no_gui/unit/test_account.py
@@ -3,11 +3,9 @@ Tests for Account classes
 '''
 import unittest
 
-import lib
-lib.setup_env()
-
 from gajim.common.account import Account
 
+
 class Test(unittest.TestCase):
 
     def testInstantiate(self):
diff --git a/test/unit/test_caps_cache.py b/test/no_gui/unit/test_caps_cache.py
similarity index 99%
rename from test/unit/test_caps_cache.py
rename to test/no_gui/unit/test_caps_cache.py
index 58b2c4da7..0fb2204ae 100644
--- a/test/unit/test_caps_cache.py
+++ b/test/no_gui/unit/test_caps_cache.py
@@ -3,9 +3,6 @@ Tests for capabilities and the capabilities cache
 '''
 import unittest
 
-import lib
-lib.setup_env()
-
 from unittest.mock import MagicMock, Mock
 from nbxmpp import NS_MUC, NS_PING, NS_XHTML_IM, Iq
 from gajim.common import caps_cache as caps
diff --git a/test/unit/test_contacts.py b/test/no_gui/unit/test_contacts.py
similarity index 99%
rename from test/unit/test_contacts.py
rename to test/no_gui/unit/test_contacts.py
index 40127dabf..8ca9810cb 100644
--- a/test/unit/test_contacts.py
+++ b/test/no_gui/unit/test_contacts.py
@@ -2,12 +2,9 @@
 Test for Contact, GC_Contact and Contacts
 '''
 import unittest
-
-import lib
-lib.setup_env()
+from nbxmpp import NS_MUC
 
 from gajim.common.contacts import CommonContact, Contact, GC_Contact, LegacyContactsAPI
-from nbxmpp import NS_MUC
 
 from gajim.common import caps_cache
 
diff --git a/test/unit/test_protocol_caps.py b/test/no_gui/unit/test_protocol_caps.py
similarity index 98%
rename from test/unit/test_protocol_caps.py
rename to test/no_gui/unit/test_protocol_caps.py
index c1201ba00..e3e0bbece 100644
--- a/test/unit/test_protocol_caps.py
+++ b/test/no_gui/unit/test_protocol_caps.py
@@ -7,9 +7,6 @@ from unittest.mock import MagicMock
 
 import nbxmpp
 
-import lib
-lib.setup_env()
-
 from gajim.common import app
 from gajim.common import nec
 from gajim.common import ged
diff --git a/test/runtests.py b/test/runtests.py
deleted file mode 100755
index 857ba8e63..000000000
--- a/test/runtests.py
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/usr/bin/env python3
-
-
-'''
-Runs Gajim's Test Suite
-
-Unit tests tests will be run on each commit.
-'''
-
-import sys
-import unittest
-import getopt
-use_x = True
-verbose = 1
-
-try:
-    shortargs = 'hnv:'
-    longargs = 'help no-x verbose='
-    opts, args = getopt.getopt(sys.argv[1:], shortargs, longargs.split())
-except getopt.error as msg:
-    print(msg)
-    print('for help use --help')
-    sys.exit(2)
-for o, a in opts:
-    if o in ('-h', '--help'):
-        print('runtests [--help] [--no-x] [--verbose level]')
-        sys.exit()
-    elif o in ('-n', '--no-x'):
-        use_x = False
-    elif o in ('-v', '--verbose'):
-        try:
-            verbose = int(a)
-        except Exception:
-            print('verbose must be a number >= 0')
-            sys.exit(2)
-
-# new test modules need to be added manually
-modules = ( 'unit.test_protocol_caps',
-            'unit.test_caps_cache',
-            'unit.test_contacts',
-            'unit.test_account',
-          )
-
-if use_x:
-    modules += ( 'unit.test_sessions',
-                 #'integration.test_gui_event_integration',
-                 'integration.test_roster',
-                 'integration.test_resolver',
-                 'unit.test_gui_interface',
-               )
-
-nb_errors = 0
-nb_failures = 0
-
-for mod in modules:
-    print("Now running: %s" % mod)
-    suite = unittest.defaultTestLoader.loadTestsFromName(mod)
-    result = unittest.TextTestRunner(verbosity=verbose).run(suite)
-    if use_x:
-        # Wait 500ms to be sure all timeout_add will be called before we cleanup main loop
-        import time
-        time.sleep(0.5)
-        # Clean main loop
-        from gi.repository import GLib
-        mc = GLib.main_context_default()
-        while mc.pending():
-            mc.iteration()
-    nb_errors += len(result.errors)
-    nb_failures += len(result.failures)
-
-sys.exit(nb_errors + nb_failures)
diff --git a/test/unit/test_gui_interface.py b/test/unit/test_gui_interface.py
index 2055f7653..babb5ad42 100644
--- a/test/unit/test_gui_interface.py
+++ b/test/unit/test_gui_interface.py
@@ -3,7 +3,7 @@ Some diverse tests covering functionality in the GUI Interface class.
 '''
 import unittest
 
-import lib
+from test import lib
 lib.setup_env()
 
 from gajim.common import logging_helpers