* when in common, no need to do from common import FOO, just import FOO

* we now migrate automaticaly at startup in a thread
 * progressDialog now scroll textvew to the end

src/common/check_paths.py

@@ -20,7 +20,7 @@ import os
 import sys
 import stat
 
-import gajim
+from common import gajim
 import logger
 import i18n
 

src/common/config.py

@@ -22,7 +22,7 @@
 
 import sre
 import copy
-from common import i18n
+import i18n
 _ = i18n._
 
 OPT_TYPE = 0

src/common/gajim.py

@@ -22,12 +22,12 @@ import sys
 import logging
 import mutex
 
-import common.config
+import config
 
 
 interface = None # The actual interface (the gtk one for the moment)
 version = '0.9'
-config = common.config.Config()
+config = config.Config()
 connections = {}
 verbose = False
 
@@ -37,8 +37,8 @@ h.setFormatter(f)
 log = logging.getLogger('Gajim')
 log.addHandler(h)
 
-import common.logger
+import logger
-logger = common.logger.Logger() # init the logger
+logger = logger.Logger() # init the logger
 
 if os.name == 'nt':
 	if '.svn' in os.listdir('.') or '_svn' in os.listdir('.'):

src/common/helpers.py

@@ -27,8 +27,8 @@ from pysqlite2 import dbapi2 as sqlite
 
 import gajim
 import logger
-from common import i18n
+import i18n
-from common.xmpp_stringprep import nodeprep, resourceprep, nameprep
+from xmpp_stringprep import nodeprep, resourceprep, nameprep
 
 try:
 	import winsound # windows-only built-in module for playing wav

src/common/logger.py

@@ -22,8 +22,8 @@ import sys
 import time
 import datetime
 
-from common import exceptions
+import exceptions
-from common import i18n
+import i18n
 _ = i18n._
 
 try:

src/common/migrate_logs_to_dot9_db.py

@@ -13,8 +13,6 @@ signal.signal(signal.SIGINT, signal.SIG_DFL) # ^C exits the application
 
 from pysqlite2 import dbapi2 as sqlite
 
-constants = logger.Constants()
-
 if os.name == 'nt':
 	try:
 		PATH_TO_LOGS_BASE_DIR = os.path.join(os.environ['appdata'], 'Gajim', 'Logs')
@@ -25,182 +23,198 @@ if os.name == 'nt':
 		PATH_TO_DB = '../src/logs.db'
 else:
 	PATH_TO_LOGS_BASE_DIR = os.path.expanduser('~/.gajim/logs')
-	PATH_TO_DB = os.path.expanduser('~/.gajim/logs.db') # database is called logs.db
+	PATH_TO_DB = os.path.expanduser('~/.gajim/logs2.db') # database is called logs.db
 
-if os.path.exists(PATH_TO_DB):
+class migration:
-	print '%s already exists. Exiting..' % PATH_TO_DB
+	def __init__(self):
-	sys.exit()
+		self.constants = logger.Constants()
+		self.DONE = False
 
-jids_already_in = [] # jid we already put in DB
+		if os.path.exists(PATH_TO_DB):
-con = sqlite.connect(PATH_TO_DB) 
+			print '%s already exists. Exiting..' % PATH_TO_DB
-os.chmod(PATH_TO_DB, 0600) # rw only for us
+			sys.exit()
-cur = con.cursor()
-# create the tables
-# kind can be
-# status, gcstatus, gc_msg, (we only recv for those 3),
-# single_msg_recv, chat_msg_recv, chat_msg_sent, single_msg_sent
-# to meet all our needs
-# logs.jid_id --> jids.jid_id but Sqlite doesn't do FK etc so it's done in python code
-cur.executescript(
-	'''
-	CREATE TABLE jids(
-		jid_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE,
-		jid TEXT UNIQUE,
-		type INTEGER
-	);
-	
-	CREATE TABLE logs(
-		log_line_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE,
-		jid_id INTEGER,
-		contact_name TEXT,
-		time INTEGER,
-		kind INTEGER,
-		show INTEGER,
-		message TEXT,
-		subject TEXT
-	);
-	'''
-	)
 
-con.commit()
+		self.jids_already_in = [] # jid we already put in DB
 
-def get_jid(dirname, filename):
+	def get_jid(self, dirname, filename):
-	# jids.jid text column will be JID if TC-related, room_jid if GC-related,
+		# jids.jid text column will be JID if TC-related, room_jid if GC-related,
-	# ROOM_JID/nick if pm-related. Here I get names from filenames
+		# ROOM_JID/nick if pm-related. Here I get names from filenames
-	if dirname.endswith('logs') or dirname.endswith('Logs'):
+		if dirname.endswith('logs') or dirname.endswith('Logs'):
-		# we have file (not dir) in logs base dir, so it's TC
+			# we have file (not dir) in logs base dir, so it's TC
-		jid = filename # file is JID
+			jid = filename # file is JID
-	else:
-		# we are in a room folder (so it can be either pm or message in room)
-		if filename == os.path.basename(dirname): # room/room
-			jid = dirname # filename is ROOM_JID
-		else: #room/nick it's pm
-			jid = dirname + '/' + filename
-
-	if jid.startswith('/'):
-		p = len(PATH_TO_LOGS_BASE_DIR)
-		jid = jid[p+1:]
-	jid = jid.lower()
-	return jid
-
-def decode_jid(string):
-	'''try to decode (to make it Unicode instance) given jid'''
-	string = decode_string(string)
-	if isinstance(string, str):
-		return None # decode failed
-	return string
-
-def visit(arg, dirname, filenames):
-	s = _('Visiting %s') % dirname
-	print s
-	for filename in filenames:
-		# Don't take this file into account, this is dup info
-		# notifications are also in contact log file
-		if filename in ('notify.log', 'readme'):
-			continue
-		path_to_text_file = os.path.join(dirname, filename)
-		if os.path.isdir(path_to_text_file):
-			continue
-
-		jid = get_jid(dirname, filename)
-
-		jid = decode_jid(jid)
-		if not jid:
-			continue
-
-		if filename == os.path.basename(dirname): # gajim@conf/gajim@conf then gajim@conf is type room
-			jid_type = constants.JID_ROOM_TYPE
-			#Type of log
-			typ = 'room'
 		else:
-			jid_type = constants.JID_NORMAL_TYPE
+			# we are in a room folder (so it can be either pm or message in room)
-			#Type of log
+			if filename == os.path.basename(dirname): # room/room
-			typ = _('normal')
+				jid = dirname # filename is ROOM_JID
-		s = _('Processing %s of type %s') % (jid.encode('utf-8'), typ)
+			else: #room/nick it's pm
-		print s
+				jid = dirname + '/' + filename
 
-		JID_ID = None
+		if jid.startswith('/'):
-		f = open(path_to_text_file, 'r')
+			p = len(PATH_TO_LOGS_BASE_DIR)
-		lines = f.readlines()
+			jid = jid[p+1:]
-		for line in lines:
+		jid = jid.lower()
-			line = from_one_line(line)
+		return jid
-			splitted_line = line.split(':')
-			if len(splitted_line) > 2:
-				# type in logs is one of 
-				# 'gc', 'gcstatus', 'recv', 'sent' and if nothing of those
-				# it is status
-				# new db has:
-				# status, gcstatus, gc_msg, (we only recv those 3),
-				# single_msg_recv, chat_msg_recv, chat_msg_sent, single_msg_sent
-				# to meet all our needs
-				# here I convert
-				# gc ==> gc_msg, gcstatus ==> gcstatus, recv ==> chat_msg_recv
-				# sent ==> chat_msg_sent, status ==> status
-				typ = splitted_line[1] # line[1] has type of logged message
-				message_data = splitted_line[2:] # line[2:] has message data
-				# line[0] is date,
-				# some lines can be fucked up, just drop them
-				try:
-					tim = int(float(splitted_line[0]))
-				except:
-					continue
 
-				contact_name = None
+	def decode_jid(self, string):
-				show = None
+		'''try to decode (to make it Unicode instance) given jid'''
-				if typ == 'gc':
+		string = decode_string(string)
-					contact_name = message_data[0]
+		if isinstance(string, str):
-					message = ':'.join(message_data[1:])
+			return None # decode failed
-					kind = constants.KIND_GC_MSG
+		return string
-				elif typ == 'gcstatus':
-					contact_name = message_data[0]
-					show = message_data[1]
-					message = ':'.join(message_data[2:]) # status msg
-					kind = constants.KIND_GCSTATUS
-				elif typ == 'recv':
-					message = ':'.join(message_data[0:])
-					kind = constants.KIND_CHAT_MSG_RECV
-				elif typ == 'sent':
-					message = ':'.join(message_data[0:])
-					kind = constants.KIND_CHAT_MSG_SENT
-				else: # status
-					kind = constants.KIND_STATUS
-					show = message_data[0]
-					message = ':'.join(message_data[1:]) # status msg
 
-#				message = decode_string(message)
+	def visit(self, arg, dirname, filenames):
-				message = message[:-1] # remove last \n
+		s = _('Visiting %s') % dirname
-				if not message:
+		if self.queue:
-					continue
+			self.queue.put(s)
+		else:
+			print s
+		for filename in filenames:
+			# Don't take this file into account, this is dup info
+			# notifications are also in contact log file
+			if filename in ('notify.log', 'readme'):
+				continue
+			path_to_text_file = os.path.join(dirname, filename)
+			if os.path.isdir(path_to_text_file):
+				continue
 
-				# jid is already in the DB, don't create a new row, just get his jid_id
+			jid = self.get_jid(dirname, filename)
-				if not JID_ID:
-					if jid in jids_already_in:
-						cur.execute('SELECT jid_id FROM jids WHERE jid = "%s"' % jid)
-						JID_ID = cur.fetchone()[0]
-					else:
-						jids_already_in.append(jid)
-						cur.execute('INSERT INTO jids (jid, type) VALUES (?, ?)',
-							(jid, jid_type))
-						con.commit()
-						JID_ID = cur.lastrowid
 
-				sql = 'INSERT INTO logs (jid_id, contact_name, time, kind, show, message) '\
+			jid = self.decode_jid(jid)
-					'VALUES (?, ?, ?, ?, ?, ?)'
+			if not jid:
+				continue
 
-				values = (JID_ID, contact_name, tim, kind, show, message)
+			if filename == os.path.basename(dirname): # gajim@conf/gajim@conf then gajim@conf is type room
-				cur.execute(sql, values)
+				jid_type = self.constants.JID_ROOM_TYPE
-		con.commit()
+				#Type of log
+				typ = 'room'
+			else:
+				jid_type = self.constants.JID_NORMAL_TYPE
+				#Type of log
+				typ = _('normal')
+			s = _('Processing %s of type %s') % (jid.encode('utf-8'), typ)
+			if self.queue:
+				self.queue.put(s)
+			else:
+				print s
 
-def migrate():
+			JID_ID = None
-	os.path.walk(PATH_TO_LOGS_BASE_DIR, visit, None)
+			f = open(path_to_text_file, 'r')
-	s = '''We do not use plain-text files anymore, because they do not scale.
+			lines = f.readlines()
+			for line in lines:
+				line = from_one_line(line)
+				splitted_line = line.split(':')
+				if len(splitted_line) > 2:
+					# type in logs is one of 
+					# 'gc', 'gcstatus', 'recv', 'sent' and if nothing of those
+					# it is status
+					# new db has:
+					# status, gcstatus, gc_msg, (we only recv those 3),
+					# single_msg_recv, chat_msg_recv, chat_msg_sent, single_msg_sent
+					# to meet all our needs
+					# here I convert
+					# gc ==> gc_msg, gcstatus ==> gcstatus, recv ==> chat_msg_recv
+					# sent ==> chat_msg_sent, status ==> status
+					typ = splitted_line[1] # line[1] has type of logged message
+					message_data = splitted_line[2:] # line[2:] has message data
+					# line[0] is date,
+					# some lines can be fucked up, just drop them
+					try:
+						tim = int(float(splitted_line[0]))
+					except:
+						continue
+
+					contact_name = None
+					show = None
+					if typ == 'gc':
+						contact_name = message_data[0]
+						message = ':'.join(message_data[1:])
+						kind = self.constants.KIND_GC_MSG
+					elif typ == 'gcstatus':
+						contact_name = message_data[0]
+						show = message_data[1]
+						message = ':'.join(message_data[2:]) # status msg
+						kind = self.constants.KIND_GCSTATUS
+					elif typ == 'recv':
+						message = ':'.join(message_data[0:])
+						kind = self.constants.KIND_CHAT_MSG_RECV
+					elif typ == 'sent':
+						message = ':'.join(message_data[0:])
+						kind = self.constants.KIND_CHAT_MSG_SENT
+					else: # status
+						kind = self.constants.KIND_STATUS
+						show = message_data[0]
+						message = ':'.join(message_data[1:]) # status msg
+
+#					message = decode_string(message)
+					message = message[:-1] # remove last \n
+					if not message:
+						continue
+
+					# jid is already in the DB, don't create a new row, just get his jid_id
+					if not JID_ID:
+						if jid in self.jids_already_in:
+							self.cur.execute('SELECT jid_id FROM jids WHERE jid = "%s"' % jid)
+							JID_ID = self.cur.fetchone()[0]
+						else:
+							self.jids_already_in.append(jid)
+							self.cur.execute('INSERT INTO jids (jid, type) VALUES (?, ?)',
+								(jid, jid_type))
+							self.con.commit()
+							JID_ID = self.cur.lastrowid
+
+					sql = 'INSERT INTO logs (jid_id, contact_name, time, kind, show, message) '\
+						'VALUES (?, ?, ?, ?, ?, ?)'
+
+					values = (JID_ID, contact_name, tim, kind, show, message)
+					self.cur.execute(sql, values)
+			self.con.commit()
+
+	def migrate(self, queue = None):
+		self.queue = queue
+		self.con = sqlite.connect(PATH_TO_DB) 
+		os.chmod(PATH_TO_DB, 0600) # rw only for us
+		self.cur = self.con.cursor()
+		# create the tables
+		# kind can be
+		# status, gcstatus, gc_msg, (we only recv for those 3),
+		# single_msg_recv, chat_msg_recv, chat_msg_sent, single_msg_sent
+		# to meet all our needs
+		# logs.jid_id --> jids.jid_id but Sqlite doesn't do FK etc so it's done in python code
+		self.cur.executescript(
+			'''
+			CREATE TABLE jids(
+				jid_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE,
+				jid TEXT UNIQUE,
+				type INTEGER
+			);
+	
+			CREATE TABLE logs(
+				log_line_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE,
+				jid_id INTEGER,
+				contact_name TEXT,
+				time INTEGER,
+				kind INTEGER,
+				show INTEGER,
+				message TEXT,
+				subject TEXT
+			);
+			'''
+		)
+
+		self.con.commit()
+
+		os.path.walk(PATH_TO_LOGS_BASE_DIR, self.visit, None)
+		s = '''We do not use plain-text files anymore, because they do not scale.
 Those files here are logs for Gajim up until 0.8.2
 We now use an sqlite database called logs.db found in ~/.gajim
 You can now safly remove your %s folder
 Thank you''' % PATH_TO_LOGS_BASE_DIR
-	f = open(os.path.join(PATH_TO_LOGS_BASE_DIR, 'README'), 'w')
+		f = open(os.path.join(PATH_TO_LOGS_BASE_DIR, 'README'), 'w')
-	f.write(s)
+		f.write(s)
-	f.close()
+		f.close()
+		if queue:
+			queue.put(s)
+		self.DONE = True
 
 if __name__ == '__main__':
 	print 'IMPORTNANT: PLEASE READ http://trac.gajim.org/wiki/MigrateLogToDot9DB'
@@ -210,4 +224,5 @@ if __name__ == '__main__':
 	print 'Starting Logs Migration'
 	print '======================='
 	print 'Please do NOT run Gajim until this script is over'
-	migrate()
+	m = migration()
+	m.migrate()

src/dialogs.py

@@ -1277,35 +1277,36 @@ class ProgressDialog:
 		messages_queue has the message to show
 		in the textview'''
 		self.xml = gtk.glade.XML(GTKGUI_GLADE, 'progress_dialog', APP)
-		dialog = self.xml.get_widget('progress_dialog')
+		self.dialog = self.xml.get_widget('progress_dialog')
+		self.messages_queue = messages_queue
 		self.label = self.xml.get_widget('label')
 		self.label.set_markup('<big>' + during_text + '</big>')
 		self.progressbar = self.xml.get_widget('progressbar')
-		self.textview_buffer = self.xml.get_widget('textview').get_buffer()
+		self.textview = self.xml.get_widget('textview')
+		self.textview_buffer = self.textview.get_buffer()
+		end_iter = self.textview_buffer.get_end_iter()
+		self.textview_buffer.create_mark('end', end_iter, False)
 		
-		dialog.set_title(title_text)
+		self.dialog.set_title(title_text)
-		dialog.show_all()
+		self.dialog.show_all()
 		self.xml.signal_autoconnect(self)
 		
 		self.update_progressbar_timeout_id = gobject.timeout_add(100,
 					self.update_progressbar)
 		
-		self.read_from_queue_id = gobject.timeout_add(1000,
+		self.read_from_queue_id = gobject.timeout_add(200,
-			self.read_from_queue_and_update_textview, messages_queue)
+			self.read_from_queue_and_update_textview)
 	
 	def update_progressbar(self):
 		self.progressbar.pulse()
 		return True # loop forever
 	
-	def read_from_queue_and_update_textview(self, messages_queue):
+	def read_from_queue_and_update_textview(self):
-		try:
+		while not self.messages_queue.empty():
-			message = messages_queue.get_nowait()
+			message = self.messages_queue.get()
-			print message
-		except Queue.Empty:
-			pass
-		else:
 			end_iter = self.textview_buffer.get_end_iter()
-			self.textview_buffer.insert(end_iter, message)
+			self.textview_buffer.insert(end_iter, message + '\n')
+			self.textview.scroll_to_mark(self.textview_buffer.get_mark('end'), 0, True, 0, 1)
 
 		return True # loop for ever
 	
@@ -1316,6 +1317,7 @@ class ProgressDialog:
 		'''whatever we were doing is done (either we problems or not),
 		make close button sensitive and show the done_text in label'''
 		self.xml.get_widget('close_button').set_sensitive(True)
-		self.label.set_markup('<big>' + done_text + '</big')
+		self.label.set_markup('<big>' + done_text + '</big>')
 		gobject.source_remove(self.update_progressbar_timeout_id)
 		gobject.source_remove(self.read_from_queue_id)
+		self.read_from_queue_and_update_textview()

src/gajim.py

@@ -71,12 +71,6 @@ if pritext:
 	dlg.destroy()
 	sys.exit()
 
-from common import logger
-LOG_DB_PATH = logger.LOG_DB_PATH
-NO_DB = False
-if not os.path.isfile(LOG_DB_PATH):
-	NO_DB = True
-
 path = os.getcwd()
 if '.svn' in os.listdir(path) or '_svn' in os.listdir(path):
 	# import gtkexcepthook only for those that run svn
@@ -94,6 +88,7 @@ import sre
 import signal
 import getopt
 import time
+import threading
 
 import gtkgui_helpers
 import notify
@@ -1399,6 +1394,14 @@ class Interface:
 		gobject.timeout_add(200, self.process_connections)
 		gobject.timeout_add(500, self.read_sleepy)
 
+def wait_migration(migration):
+	if not migration.DONE:
+		return True
+	dialog.done(_('Logs have been successfully migrated to the database.'))
+	dialog.dialog.run()
+	dialog.dialog.destroy()
+	gtk.main_quit()
+
 if __name__ == '__main__':
 	signal.signal(signal.SIGINT, signal.SIG_DFL) # ^C exits the application
 
@@ -1460,9 +1463,18 @@ if __name__ == '__main__':
 			pass
 	
 	# Migrate old logs if user wants that
-	if NO_DB:
+	from common import logger
-		pass # launch migration script
+	LOG_DB_PATH = logger.LOG_DB_PATH
-	del NO_DB
+	if not os.path.isfile(LOG_DB_PATH):
+		import Queue
+		q = Queue.Queue(100)
+		from common import migrate_logs_to_dot9_db
+		m = migrate_logs_to_dot9_db.migration()
+		dialog = dialogs.ProgressDialog(_('Migrating logs...'), _('Please wait while logs are being migrated...'), q)
+		t = threading.Thread(target = m.migrate, args = (q,))
+		t.start()
+		gobject.timeout_add(500, wait_migration, m)
+		gtk.main()
 	check_paths.check_and_possibly_create_paths()
 
 	Interface()