diff --git a/src/common/check_paths.py b/src/common/check_paths.py index 459a106a6..e79332a3c 100644 --- a/src/common/check_paths.py +++ b/src/common/check_paths.py @@ -20,7 +20,7 @@ import os import sys import stat -import gajim +from common import gajim import logger import i18n diff --git a/src/common/config.py b/src/common/config.py index 2940cf853..a037ed890 100644 --- a/src/common/config.py +++ b/src/common/config.py @@ -22,7 +22,7 @@ import sre import copy -from common import i18n +import i18n _ = i18n._ OPT_TYPE = 0 diff --git a/src/common/gajim.py b/src/common/gajim.py index 588f11046..29c3b2c80 100644 --- a/src/common/gajim.py +++ b/src/common/gajim.py @@ -22,12 +22,12 @@ import sys import logging import mutex -import common.config +import config interface = None # The actual interface (the gtk one for the moment) version = '0.9' -config = common.config.Config() +config = config.Config() connections = {} verbose = False @@ -37,8 +37,8 @@ h.setFormatter(f) log = logging.getLogger('Gajim') log.addHandler(h) -import common.logger -logger = common.logger.Logger() # init the logger +import logger +logger = logger.Logger() # init the logger if os.name == 'nt': if '.svn' in os.listdir('.') or '_svn' in os.listdir('.'): diff --git a/src/common/helpers.py b/src/common/helpers.py index f66888e44..c4abced66 100644 --- a/src/common/helpers.py +++ b/src/common/helpers.py @@ -27,8 +27,8 @@ from pysqlite2 import dbapi2 as sqlite import gajim import logger -from common import i18n -from common.xmpp_stringprep import nodeprep, resourceprep, nameprep +import i18n +from xmpp_stringprep import nodeprep, resourceprep, nameprep try: import winsound # windows-only built-in module for playing wav diff --git a/src/common/logger.py b/src/common/logger.py index 68d7d2b79..2275a45d9 100644 --- a/src/common/logger.py +++ b/src/common/logger.py @@ -22,8 +22,8 @@ import sys import time import datetime -from common import exceptions -from common import i18n +import exceptions +import i18n _ = i18n._ try: diff --git a/src/common/migrate_logs_to_dot9_db.py b/src/common/migrate_logs_to_dot9_db.py index 7013d20a2..ee27f5172 100755 --- a/src/common/migrate_logs_to_dot9_db.py +++ b/src/common/migrate_logs_to_dot9_db.py @@ -13,8 +13,6 @@ signal.signal(signal.SIGINT, signal.SIG_DFL) # ^C exits the application from pysqlite2 import dbapi2 as sqlite -constants = logger.Constants() - if os.name == 'nt': try: PATH_TO_LOGS_BASE_DIR = os.path.join(os.environ['appdata'], 'Gajim', 'Logs') @@ -25,182 +23,198 @@ if os.name == 'nt': PATH_TO_DB = '../src/logs.db' else: PATH_TO_LOGS_BASE_DIR = os.path.expanduser('~/.gajim/logs') - PATH_TO_DB = os.path.expanduser('~/.gajim/logs.db') # database is called logs.db + PATH_TO_DB = os.path.expanduser('~/.gajim/logs2.db') # database is called logs.db -if os.path.exists(PATH_TO_DB): - print '%s already exists. Exiting..' % PATH_TO_DB - sys.exit() +class migration: + def __init__(self): + self.constants = logger.Constants() + self.DONE = False -jids_already_in = [] # jid we already put in DB -con = sqlite.connect(PATH_TO_DB) -os.chmod(PATH_TO_DB, 0600) # rw only for us -cur = con.cursor() -# create the tables -# kind can be -# status, gcstatus, gc_msg, (we only recv for those 3), -# single_msg_recv, chat_msg_recv, chat_msg_sent, single_msg_sent -# to meet all our needs -# logs.jid_id --> jids.jid_id but Sqlite doesn't do FK etc so it's done in python code -cur.executescript( - ''' - CREATE TABLE jids( - jid_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE, - jid TEXT UNIQUE, - type INTEGER - ); - - CREATE TABLE logs( - log_line_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE, - jid_id INTEGER, - contact_name TEXT, - time INTEGER, - kind INTEGER, - show INTEGER, - message TEXT, - subject TEXT - ); - ''' - ) + if os.path.exists(PATH_TO_DB): + print '%s already exists. Exiting..' % PATH_TO_DB + sys.exit() -con.commit() + self.jids_already_in = [] # jid we already put in DB -def get_jid(dirname, filename): - # jids.jid text column will be JID if TC-related, room_jid if GC-related, - # ROOM_JID/nick if pm-related. Here I get names from filenames - if dirname.endswith('logs') or dirname.endswith('Logs'): - # we have file (not dir) in logs base dir, so it's TC - jid = filename # file is JID - else: - # we are in a room folder (so it can be either pm or message in room) - if filename == os.path.basename(dirname): # room/room - jid = dirname # filename is ROOM_JID - else: #room/nick it's pm - jid = dirname + '/' + filename - - if jid.startswith('/'): - p = len(PATH_TO_LOGS_BASE_DIR) - jid = jid[p+1:] - jid = jid.lower() - return jid - -def decode_jid(string): - '''try to decode (to make it Unicode instance) given jid''' - string = decode_string(string) - if isinstance(string, str): - return None # decode failed - return string - -def visit(arg, dirname, filenames): - s = _('Visiting %s') % dirname - print s - for filename in filenames: - # Don't take this file into account, this is dup info - # notifications are also in contact log file - if filename in ('notify.log', 'readme'): - continue - path_to_text_file = os.path.join(dirname, filename) - if os.path.isdir(path_to_text_file): - continue - - jid = get_jid(dirname, filename) - - jid = decode_jid(jid) - if not jid: - continue - - if filename == os.path.basename(dirname): # gajim@conf/gajim@conf then gajim@conf is type room - jid_type = constants.JID_ROOM_TYPE - #Type of log - typ = 'room' + def get_jid(self, dirname, filename): + # jids.jid text column will be JID if TC-related, room_jid if GC-related, + # ROOM_JID/nick if pm-related. Here I get names from filenames + if dirname.endswith('logs') or dirname.endswith('Logs'): + # we have file (not dir) in logs base dir, so it's TC + jid = filename # file is JID else: - jid_type = constants.JID_NORMAL_TYPE - #Type of log - typ = _('normal') - s = _('Processing %s of type %s') % (jid.encode('utf-8'), typ) - print s + # we are in a room folder (so it can be either pm or message in room) + if filename == os.path.basename(dirname): # room/room + jid = dirname # filename is ROOM_JID + else: #room/nick it's pm + jid = dirname + '/' + filename - JID_ID = None - f = open(path_to_text_file, 'r') - lines = f.readlines() - for line in lines: - line = from_one_line(line) - splitted_line = line.split(':') - if len(splitted_line) > 2: - # type in logs is one of - # 'gc', 'gcstatus', 'recv', 'sent' and if nothing of those - # it is status - # new db has: - # status, gcstatus, gc_msg, (we only recv those 3), - # single_msg_recv, chat_msg_recv, chat_msg_sent, single_msg_sent - # to meet all our needs - # here I convert - # gc ==> gc_msg, gcstatus ==> gcstatus, recv ==> chat_msg_recv - # sent ==> chat_msg_sent, status ==> status - typ = splitted_line[1] # line[1] has type of logged message - message_data = splitted_line[2:] # line[2:] has message data - # line[0] is date, - # some lines can be fucked up, just drop them - try: - tim = int(float(splitted_line[0])) - except: - continue + if jid.startswith('/'): + p = len(PATH_TO_LOGS_BASE_DIR) + jid = jid[p+1:] + jid = jid.lower() + return jid - contact_name = None - show = None - if typ == 'gc': - contact_name = message_data[0] - message = ':'.join(message_data[1:]) - kind = constants.KIND_GC_MSG - elif typ == 'gcstatus': - contact_name = message_data[0] - show = message_data[1] - message = ':'.join(message_data[2:]) # status msg - kind = constants.KIND_GCSTATUS - elif typ == 'recv': - message = ':'.join(message_data[0:]) - kind = constants.KIND_CHAT_MSG_RECV - elif typ == 'sent': - message = ':'.join(message_data[0:]) - kind = constants.KIND_CHAT_MSG_SENT - else: # status - kind = constants.KIND_STATUS - show = message_data[0] - message = ':'.join(message_data[1:]) # status msg + def decode_jid(self, string): + '''try to decode (to make it Unicode instance) given jid''' + string = decode_string(string) + if isinstance(string, str): + return None # decode failed + return string -# message = decode_string(message) - message = message[:-1] # remove last \n - if not message: - continue + def visit(self, arg, dirname, filenames): + s = _('Visiting %s') % dirname + if self.queue: + self.queue.put(s) + else: + print s + for filename in filenames: + # Don't take this file into account, this is dup info + # notifications are also in contact log file + if filename in ('notify.log', 'readme'): + continue + path_to_text_file = os.path.join(dirname, filename) + if os.path.isdir(path_to_text_file): + continue - # jid is already in the DB, don't create a new row, just get his jid_id - if not JID_ID: - if jid in jids_already_in: - cur.execute('SELECT jid_id FROM jids WHERE jid = "%s"' % jid) - JID_ID = cur.fetchone()[0] - else: - jids_already_in.append(jid) - cur.execute('INSERT INTO jids (jid, type) VALUES (?, ?)', - (jid, jid_type)) - con.commit() - JID_ID = cur.lastrowid + jid = self.get_jid(dirname, filename) - sql = 'INSERT INTO logs (jid_id, contact_name, time, kind, show, message) '\ - 'VALUES (?, ?, ?, ?, ?, ?)' + jid = self.decode_jid(jid) + if not jid: + continue - values = (JID_ID, contact_name, tim, kind, show, message) - cur.execute(sql, values) - con.commit() + if filename == os.path.basename(dirname): # gajim@conf/gajim@conf then gajim@conf is type room + jid_type = self.constants.JID_ROOM_TYPE + #Type of log + typ = 'room' + else: + jid_type = self.constants.JID_NORMAL_TYPE + #Type of log + typ = _('normal') + s = _('Processing %s of type %s') % (jid.encode('utf-8'), typ) + if self.queue: + self.queue.put(s) + else: + print s -def migrate(): - os.path.walk(PATH_TO_LOGS_BASE_DIR, visit, None) - s = '''We do not use plain-text files anymore, because they do not scale. + JID_ID = None + f = open(path_to_text_file, 'r') + lines = f.readlines() + for line in lines: + line = from_one_line(line) + splitted_line = line.split(':') + if len(splitted_line) > 2: + # type in logs is one of + # 'gc', 'gcstatus', 'recv', 'sent' and if nothing of those + # it is status + # new db has: + # status, gcstatus, gc_msg, (we only recv those 3), + # single_msg_recv, chat_msg_recv, chat_msg_sent, single_msg_sent + # to meet all our needs + # here I convert + # gc ==> gc_msg, gcstatus ==> gcstatus, recv ==> chat_msg_recv + # sent ==> chat_msg_sent, status ==> status + typ = splitted_line[1] # line[1] has type of logged message + message_data = splitted_line[2:] # line[2:] has message data + # line[0] is date, + # some lines can be fucked up, just drop them + try: + tim = int(float(splitted_line[0])) + except: + continue + + contact_name = None + show = None + if typ == 'gc': + contact_name = message_data[0] + message = ':'.join(message_data[1:]) + kind = self.constants.KIND_GC_MSG + elif typ == 'gcstatus': + contact_name = message_data[0] + show = message_data[1] + message = ':'.join(message_data[2:]) # status msg + kind = self.constants.KIND_GCSTATUS + elif typ == 'recv': + message = ':'.join(message_data[0:]) + kind = self.constants.KIND_CHAT_MSG_RECV + elif typ == 'sent': + message = ':'.join(message_data[0:]) + kind = self.constants.KIND_CHAT_MSG_SENT + else: # status + kind = self.constants.KIND_STATUS + show = message_data[0] + message = ':'.join(message_data[1:]) # status msg + +# message = decode_string(message) + message = message[:-1] # remove last \n + if not message: + continue + + # jid is already in the DB, don't create a new row, just get his jid_id + if not JID_ID: + if jid in self.jids_already_in: + self.cur.execute('SELECT jid_id FROM jids WHERE jid = "%s"' % jid) + JID_ID = self.cur.fetchone()[0] + else: + self.jids_already_in.append(jid) + self.cur.execute('INSERT INTO jids (jid, type) VALUES (?, ?)', + (jid, jid_type)) + self.con.commit() + JID_ID = self.cur.lastrowid + + sql = 'INSERT INTO logs (jid_id, contact_name, time, kind, show, message) '\ + 'VALUES (?, ?, ?, ?, ?, ?)' + + values = (JID_ID, contact_name, tim, kind, show, message) + self.cur.execute(sql, values) + self.con.commit() + + def migrate(self, queue = None): + self.queue = queue + self.con = sqlite.connect(PATH_TO_DB) + os.chmod(PATH_TO_DB, 0600) # rw only for us + self.cur = self.con.cursor() + # create the tables + # kind can be + # status, gcstatus, gc_msg, (we only recv for those 3), + # single_msg_recv, chat_msg_recv, chat_msg_sent, single_msg_sent + # to meet all our needs + # logs.jid_id --> jids.jid_id but Sqlite doesn't do FK etc so it's done in python code + self.cur.executescript( + ''' + CREATE TABLE jids( + jid_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE, + jid TEXT UNIQUE, + type INTEGER + ); + + CREATE TABLE logs( + log_line_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE, + jid_id INTEGER, + contact_name TEXT, + time INTEGER, + kind INTEGER, + show INTEGER, + message TEXT, + subject TEXT + ); + ''' + ) + + self.con.commit() + + os.path.walk(PATH_TO_LOGS_BASE_DIR, self.visit, None) + s = '''We do not use plain-text files anymore, because they do not scale. Those files here are logs for Gajim up until 0.8.2 We now use an sqlite database called logs.db found in ~/.gajim You can now safly remove your %s folder Thank you''' % PATH_TO_LOGS_BASE_DIR - f = open(os.path.join(PATH_TO_LOGS_BASE_DIR, 'README'), 'w') - f.write(s) - f.close() + f = open(os.path.join(PATH_TO_LOGS_BASE_DIR, 'README'), 'w') + f.write(s) + f.close() + if queue: + queue.put(s) + self.DONE = True if __name__ == '__main__': print 'IMPORTNANT: PLEASE READ http://trac.gajim.org/wiki/MigrateLogToDot9DB' @@ -210,4 +224,5 @@ if __name__ == '__main__': print 'Starting Logs Migration' print '=======================' print 'Please do NOT run Gajim until this script is over' - migrate() + m = migration() + m.migrate() diff --git a/src/dialogs.py b/src/dialogs.py index 11c63fb57..70cc650af 100644 --- a/src/dialogs.py +++ b/src/dialogs.py @@ -1277,35 +1277,36 @@ class ProgressDialog: messages_queue has the message to show in the textview''' self.xml = gtk.glade.XML(GTKGUI_GLADE, 'progress_dialog', APP) - dialog = self.xml.get_widget('progress_dialog') + self.dialog = self.xml.get_widget('progress_dialog') + self.messages_queue = messages_queue self.label = self.xml.get_widget('label') self.label.set_markup('' + during_text + '') self.progressbar = self.xml.get_widget('progressbar') - self.textview_buffer = self.xml.get_widget('textview').get_buffer() + self.textview = self.xml.get_widget('textview') + self.textview_buffer = self.textview.get_buffer() + end_iter = self.textview_buffer.get_end_iter() + self.textview_buffer.create_mark('end', end_iter, False) - dialog.set_title(title_text) - dialog.show_all() + self.dialog.set_title(title_text) + self.dialog.show_all() self.xml.signal_autoconnect(self) self.update_progressbar_timeout_id = gobject.timeout_add(100, self.update_progressbar) - self.read_from_queue_id = gobject.timeout_add(1000, - self.read_from_queue_and_update_textview, messages_queue) + self.read_from_queue_id = gobject.timeout_add(200, + self.read_from_queue_and_update_textview) def update_progressbar(self): self.progressbar.pulse() return True # loop forever - def read_from_queue_and_update_textview(self, messages_queue): - try: - message = messages_queue.get_nowait() - print message - except Queue.Empty: - pass - else: + def read_from_queue_and_update_textview(self): + while not self.messages_queue.empty(): + message = self.messages_queue.get() end_iter = self.textview_buffer.get_end_iter() - self.textview_buffer.insert(end_iter, message) + self.textview_buffer.insert(end_iter, message + '\n') + self.textview.scroll_to_mark(self.textview_buffer.get_mark('end'), 0, True, 0, 1) return True # loop for ever @@ -1316,6 +1317,7 @@ class ProgressDialog: '''whatever we were doing is done (either we problems or not), make close button sensitive and show the done_text in label''' self.xml.get_widget('close_button').set_sensitive(True) - self.label.set_markup('' + done_text + '' + done_text + '') gobject.source_remove(self.update_progressbar_timeout_id) gobject.source_remove(self.read_from_queue_id) + self.read_from_queue_and_update_textview() diff --git a/src/gajim.py b/src/gajim.py index d2ce3f593..dd9b93e46 100755 --- a/src/gajim.py +++ b/src/gajim.py @@ -71,12 +71,6 @@ if pritext: dlg.destroy() sys.exit() -from common import logger -LOG_DB_PATH = logger.LOG_DB_PATH -NO_DB = False -if not os.path.isfile(LOG_DB_PATH): - NO_DB = True - path = os.getcwd() if '.svn' in os.listdir(path) or '_svn' in os.listdir(path): # import gtkexcepthook only for those that run svn @@ -94,6 +88,7 @@ import sre import signal import getopt import time +import threading import gtkgui_helpers import notify @@ -1399,6 +1394,14 @@ class Interface: gobject.timeout_add(200, self.process_connections) gobject.timeout_add(500, self.read_sleepy) +def wait_migration(migration): + if not migration.DONE: + return True + dialog.done(_('Logs have been successfully migrated to the database.')) + dialog.dialog.run() + dialog.dialog.destroy() + gtk.main_quit() + if __name__ == '__main__': signal.signal(signal.SIGINT, signal.SIG_DFL) # ^C exits the application @@ -1460,9 +1463,18 @@ if __name__ == '__main__': pass # Migrate old logs if user wants that - if NO_DB: - pass # launch migration script - del NO_DB + from common import logger + LOG_DB_PATH = logger.LOG_DB_PATH + if not os.path.isfile(LOG_DB_PATH): + import Queue + q = Queue.Queue(100) + from common import migrate_logs_to_dot9_db + m = migrate_logs_to_dot9_db.migration() + dialog = dialogs.ProgressDialog(_('Migrating logs...'), _('Please wait while logs are being migrated...'), q) + t = threading.Thread(target = m.migrate, args = (q,)) + t.start() + gobject.timeout_add(500, wait_migration, m) + gtk.main() check_paths.check_and_possibly_create_paths() Interface()