new db design and improved migration script. if you ran before the migration please remove you old logs.db as it is incompatible with the new one and rerun migration if you care about old logs. a print on show is on purpose, bonus of the first to traceback and give us the value of show that causes the tb. yea svn got unstable these days. sorry about that but big changes

scripts/migrate_logs_to_dot9_db.py

@@ -9,6 +9,35 @@ signal.signal(signal.SIGINT, signal.SIG_DFL) # ^C exits the application
 
 from pysqlite2 import dbapi2 as sqlite
 
+
+class Constants:
+	def __init__(self):
+		(
+			self.JID_NORMAL_TYPE,
+			self.JID_ROOM_TYPE # image to show state (online, new message etc)
+		) = range(2)
+		
+		(
+			self.KIND_STATUS,
+			self.KIND_GCSTATUS,
+			self.KIND_GC_MSG,
+			self.KIND_SINGLE_MSG_RECV,
+			self.KIND_CHAT_MSG_RECV,
+			self.KIND_SINGLE_MSG_SENT,
+			self.KIND_CHAT_MSG_SENT
+		) = range(7)
+		
+		(
+			self.SHOW_ONLINE,
+			self.SHOW_CHAT,
+			self.SHOW_AWAY,
+			self.SHOW_XA,
+			self.SHOW_DND,
+			self.SHOW_OFFLINE
+		) = range(6)
+
+constants = Constants()
+
 if os.name == 'nt':
 	try:
 		PATH_TO_LOGS_BASE_DIR = os.path.join(os.environ['appdata'], 'Gajim', 'Logs')
@@ -27,6 +56,7 @@ if os.path.exists(PATH_TO_DB):
 
 jids_already_in = [] # jid we already put in DB
 con = sqlite.connect(PATH_TO_DB) 
+os.chmod(PATH_TO_DB, 0600) # rw only for us
 cur = con.cursor()
 # create the tables
 # kind can be
@@ -38,7 +68,8 @@ cur.executescript(
 	'''
 	CREATE TABLE jids(
 		jid_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE,
-		jid TEXT UNIQUE
+		jid TEXT UNIQUE,
+		type INTEGER
 	);
 	
 	CREATE TABLE logs(
@@ -46,9 +77,10 @@ cur.executescript(
 		jid_id INTEGER,
 		contact_name TEXT,
 		time INTEGER,
-		kind TEXT,
+		kind INTEGER,
-		show TEXT,
+		show INTEGER,
-		message TEXT
+		message TEXT,
+		subject TEXT
 	);
 	'''
 	)
@@ -86,7 +118,7 @@ def get_jid(dirname, filename):
 
 	if jid.startswith('/'):
 		p = len(PATH_TO_LOGS_BASE_DIR)
-		jid = jid[p:]
+		jid = jid[p+1:]
 	jid = jid.lower()
 	return jid
 
@@ -118,14 +150,21 @@ def visit(arg, dirname, filenames):
 			continue
 
 		jid = get_jid(dirname, filename)
+		if filename == os.path.basename(dirname): # gajim@conf/gajim@conf then gajim@conf is type room
+			type = constants.JID_ROOM_TYPE
+			print 'marking jid as of type room'
+		else:
+			type = constants.JID_NORMAL_TYPE
+			print 'marking jid as of type normal'
+
 		print 'Processing', jid
-		# jid is already in the DB, don't create the table, just get his jid_id
+		# jid is already in the DB, don't create a new row, just get his jid_id
 		if jid in jids_already_in:
 			cur.execute('SELECT jid_id FROM jids WHERE jid = "%s"' % jid)
 			JID_ID = cur.fetchone()[0]
 		else:
 			jids_already_in.append(jid)
-			cur.execute('INSERT INTO jids (jid) VALUES (?)', (jid,))
+			cur.execute('INSERT INTO jids (jid, type) VALUES (?, ?)', (jid, type))
 			con.commit()
 
 			JID_ID = cur.lastrowid
@@ -150,7 +189,6 @@ def visit(arg, dirname, filenames):
 				type = splitted_line[1] # line[1] has type of logged message
 				message_data = splitted_line[2:] # line[2:] has message data
 				# line[0] is date,
-
 				# some lines can be fucked up, just drop them
 				try:
 					tim = int(float(splitted_line[0]))
@@ -165,24 +203,25 @@ def visit(arg, dirname, filenames):
 				if type == 'gc':
 					contact_name = message_data[0]
 					message = ':'.join(message_data[1:])
-					kind = 'gc_msg'
+					kind = constants.KIND_GC_MSG
 				elif type == 'gcstatus':
 					contact_name = message_data[0]
 					show = message_data[1]
 					message = ':'.join(message_data[2:]) # status msg
-					kind = type
+					kind = constants.KIND_GCSTATUS
 				elif type == 'recv':
 					message = ':'.join(message_data[0:])
-					kind = 'chat_msg_recv'
+					kind = constants.KIND_CHAT_MSG_RECV
 				elif type == 'sent':
 					message = ':'.join(message_data[0:])
-					kind = 'chat_msg_sent'
+					kind = constants.KIND_CHAT_MSG_SENT
 				else: # status
-					kind = 'status'
+					kind = constants.KIND_STATUS
 					show = message_data[0]
 					message = ':'.join(message_data[1:]) # status msg
 
 				message = decode_string(message)
+				message = message[:-1] # remove last \n
 				if not message:
 					continue
 				values = (JID_ID, contact_name, tim, kind, show, message)
@@ -192,7 +231,7 @@ def visit(arg, dirname, filenames):
 if __name__ == '__main__':
 	print 'IMPORTNANT: PLEASE READ http://trac.gajim.org/wiki/MigrateLogToDot9DB'
 	print 'Migration will start in 40 seconds unless you press Ctrl+C'
-	time.sleep(40) # give him time to act
+	time.sleep(40) # give the user time to act
 	print
 	print 'Starting Logs Migration'
 	print '======================='

src/common/check_paths.py

@@ -32,6 +32,7 @@ from pysqlite2 import dbapi2 as sqlite # DO NOT MOVE ABOVE OF import gajim
 def create_log_db():
 	print _('creating logs database')
 	con = sqlite.connect(logger.LOG_DB_PATH) 
+	os.chmod(logger.LOG_DB_PATH, 0600) # rw only for us
 	cur = con.cursor()
 	# create the tables
 	# kind can be
@@ -45,7 +46,8 @@ def create_log_db():
 		'''
 		CREATE TABLE jids(
 			jid_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE,
-			jid TEXT UNIQUE
+			jid TEXT UNIQUE,
+			type INTEGER
 		);
 		
 		CREATE TABLE logs(
@@ -53,9 +55,10 @@ def create_log_db():
 			jid_id INTEGER,
 			contact_name TEXT,
 			time INTEGER,
-			kind TEXT,
+			kind INTEGER,
-			show TEXT,
+			show INTEGER,
-			message TEXT
+			message TEXT,
+			subject TEXT
 		);
 		'''
 		)

src/common/connection.py

@@ -366,10 +366,7 @@ class Connection:
 				self.dispatch('GC_MSG', (frm, msgtxt, tim))
 				gajim.logger.write('gc_msg', frm, msgtxt, tim = tim)
 		elif mtype == 'normal': # it's single message
-			log_msgtxt = msgtxt
+			gajim.logger.write('single_msg_recv', frm, msgtxt, tim = tim, subject = subject)
-			if subject:
-				log_msgtxt = _('Subject: %s\n%s') % (subject, msgtxt)
-			gajim.logger.write('single_msg_recv', frm, log_msgtxt, tim = tim)
 			if invite is not None:
 				item = invite.getTag('invite')
 				jid_from = item.getAttr('from')
@@ -383,11 +380,8 @@ class Connection:
 		else: # it's type 'chat'
 			if not msg.getTag('body') and chatstate is None: #no <body>
 				return
-			log_msgtxt = msgtxt
-			if subject:
-				log_msgtxt = _('Subject: %s\n%s') % (subject, msgtxt)
 			if msg.getTag('body'):
-				gajim.logger.write('chat_msg_recv', frm, log_msgtxt, tim = tim)
+				gajim.logger.write('chat_msg_recv', frm, msgtxt, tim = tim, subject = subject)
 			self.dispatch('MSG', (frm, msgtxt, tim, encrypted, mtype, subject,
 				chatstate))
 	# END messageCB

src/common/logger.py

@@ -30,7 +30,7 @@ try:
 except ImportError:
 	error = _('pysqlite2 (aka python-pysqlite2) dependency is missing. '\
 		'After you install pysqlite3, if you want to migrate your logs '\
-		'to the new database, please read: http://trac.gajim.org/wiki/MigrateLogToDot9DB '
+		'to the new database, please read: http://trac.gajim.org/wiki/MigrateLogToDot9DB '\
 		'Exiting...'
 		)
 	print >> sys.stderr, error
@@ -53,16 +53,45 @@ try:
 except:
 	pass
 
-con = sqlite.connect(LOG_DB_PATH)
+class Constants:
-cur = con.cursor()
+	def __init__(self):
+		(
+			self.JID_NORMAL_TYPE,
+			self.JID_ROOM_TYPE # image to show state (online, new message etc)
+		) = range(2)
+		
+		(
+			self.KIND_STATUS,
+			self.KIND_GCSTATUS,
+			self.KIND_GC_MSG,
+			self.KIND_SINGLE_MSG_RECV,
+			self.KIND_CHAT_MSG_RECV,
+			self.KIND_SINGLE_MSG_SENT,
+			self.KIND_CHAT_MSG_SENT
+		) = range(7)
+		
+		(
+			self.SHOW_ONLINE,
+			self.SHOW_CHAT,
+			self.SHOW_AWAY,
+			self.SHOW_XA,
+			self.SHOW_DND,
+			self.SHOW_OFFLINE
+		) = range(6)
+
+constants = Constants()
 
 class Logger:
 	def __init__(self):
 		if not os.path.exists(LOG_DB_PATH):
 			# this can happen only the first time (the time we create the db)
-			# db is created in src/common/checks_paths.py
+			# db is not created here but in src/common/checks_paths.py
 			return
 		
+		global con, cur
+		con = sqlite.connect(LOG_DB_PATH)
+		cur = con.cursor()
+		
 		self.get_jids_already_in_db()
 
 	def get_jids_already_in_db(self):
@@ -74,21 +103,26 @@ class Logger:
 			self.jids_already_in.append(row[0])
 		GOT_JIDS_ALREADY_IN_DB = True
 
-	def jid_is_from_pm(cur, jid):
+	def jid_is_from_pm(self, jid):
 		'''if jid is gajim@conf/nkour it's likely a pm one, how we know
 		gajim@conf is not a normal guy and nkour is not his resource?
-		we ask if gajim@conf is already in jids (as room)
+		we ask if gajim@conf is already in jids (with type room jid)
-		this fails if user disable logging for room and only enables for
+		this fails if user disables logging for room and only enables for
-		pm (so higly unlikely) and if we fail we do not force chaos
+		pm (so higly unlikely) and if we fail we do not go chaos
-		(user will see the first pm as if it was message in room's public chat)'''
+		(user will see the first pm as if it was message in room's public chat)
+		and after that all okay'''
 		
 		possible_room_jid, possible_nick = jid.split('/', 1)
+		print possible_room_jid
 		
-		cur.execute('SELECT jid_id FROM jids WHERE jid="%s"' % possible_room_jid)
+		cur.execute('SELECT jid_id FROM jids WHERE jid="%s" AND type=%d' %\
-		jid_id = cur.fetchone()[0]
+			(possible_room_jid, constants.JID_ROOM_TYPE))
-		if jid_id:
+		row = cur.fetchone()
+		if row is not None:
+			print 'PM!!!!!!!!!!!!!'
 			return True
 		else:
+			print 'NOT PM!!!'
 			return False
 	
 	def get_jid_id(self, jid):
@@ -98,29 +132,113 @@ class Logger:
 		this method asks jid and returns the jid_id for later sql-ing on logs
 		'''
 		if jid.find('/') != -1: # if it has a /
+			is_pm = self.jid_is_from_pm(jid)
+			if not is_pm: # it's normal jid with resource
 				jid = jid.split('/', 1)[0] # remove the resource
 		if jid in self.jids_already_in: # we already have jids in DB
 			cur.execute('SELECT jid_id FROM jids WHERE jid="%s"' % jid)
 			jid_id = cur.fetchone()[0]
-		else: # oh! a new jid :), we add him now
+		else: # oh! a new jid :), we add it now
 			cur.execute('INSERT INTO jids (jid) VALUES (?)', (jid,))
 			con.commit()
 			jid_id = cur.lastrowid
 			self.jids_already_in.append(jid)
 		return jid_id
 	
-	def write(self, kind, jid, message = None, show = None, tim = None):
+	def convert_human_values_to_db_api_values(self, kind, show):
+		'''coverts from string style to constant ints for db'''
+		if kind == 'status':
+			kind_col = constants.KIND_STATUS
+		elif kind == 'gcstatus':
+			kind_col = constants.KIND_GCSTATUS
+		elif kind == 'gc_msg':
+			kind_col = constants.KIND_GC_MSG
+		elif kind == 'single_msg_recv':
+			kind_col = constants.KIND_SINGLE_MSG_RECV
+		elif kind == 'single_msg_sent':
+			kind_col = constants.KIND_SINGLE_MSG_SENT
+		elif kind == 'chat_msg_recv':
+			kind_col = constants.KIND_CHAT_MSG_RECV
+		elif kind == 'chat_msg_sent':
+			kind_col = constants.KIND_CHAT_MSG_SENT
+			
+		print 'show', show
+		
+		if show == 'online':
+			show_col = constants.SHOW_ONLINE
+		elif show == 'chat':
+			show_col = constants.SHOW_CHAT
+		elif show == 'away':
+			show_col = constants.SHOW_AWAY
+		elif show == 'xa':
+			show_col = constants.SHOW_XA
+		elif show == 'dnd':
+			show_col = constants.SHOW_DND
+		elif show == 'offline':
+			show_col = constants.SHOW_OFFLINE
+		elif show is None:
+			show_col = None
+		
+		return kind_col, show_col
+
+	def convert_db_api_values_to_human_values(self, kind_col, show_col):
+		'''coverts from db constant ints to string style'''
+		print 'kind_col', kind_col, 'show_col', show_col
+		if kind_col == constants.KIND_STATUS:
+			kind = 'status'
+		elif kind_col == constants.KIND_GCSTATUS:
+			kind = 'gcstatus'
+		elif kind_col == constants.KIND_GC_MSG:
+			kind = 'gc_msg'
+		elif kind_col == constants.KIND_SINGLE_MSG_RECV:
+			kind = 'single_msg_recv'
+		elif kind_col == constants.KIND_SINGLE_MSG_SENT:
+			kind = 'single_msg_sent'
+		elif kind_col == constants.KIND_CHAT_MSG_RECV:
+			kind = 'chat_msg_recv'
+		elif kind_col == constants.KIND_CHAT_MSG_SENT:
+			kind = 'chat_msg_sent'
+			
+		if show_col == constants.SHOW_ONLINE:
+			show = 'online'
+		elif show_col == constants.SHOW_CHAT:
+			show = 'chat'
+		elif show_col == constants.SHOW_AWAY:
+			show = 'away'
+		elif show_col == constants.SHOW_XA:
+			show = 'xa'
+		elif show_col == constants.SHOW_DND:
+			show = 'dnd'
+		elif show_col == constants.SHOW_OFFLINE:
+			show = 'offline'
+		elif show_col is None:
+			show = None
+		
+		return kind, show
+	
+	def commit_to_db(self, values):
+		#print 'saving', values
+		sql = 'INSERT INTO logs (jid_id, contact_name, time, kind, show, message, subject) '\
+				'VALUES (?, ?, ?, ?, ?, ?, ?)'
+		cur.execute(sql, values)
+		con.commit()
+		
+	
+	def write(self, kind, jid, message = None, show = None, tim = None, subject = None):
 		'''write a row (status, gcstatus, message etc) to logs database
 		kind can be status, gcstatus, gc_msg, (we only recv for those 3),
 		single_msg_recv, chat_msg_recv, chat_msg_sent, single_msg_sent
 		we cannot know if it is pm or normal chat message, we try to guess
-		see jid_is_from_pm()
+		see jid_is_from_pm() which is called by get_jid_id()
 		
 		we analyze jid and store it as follows:
 		jids.jid text column will hold JID if TC-related, room_jid if GC-related,
 		ROOM_JID/nick if pm-related.'''
 
 		if not GOT_JIDS_ALREADY_IN_DB:
+			global con, cur
+			con = sqlite.connect(LOG_DB_PATH)
+			cur = con.cursor()
 			self.get_jids_already_in_db()
 			
 		jid = jid.lower()
@@ -128,40 +246,31 @@ class Logger:
 		# message holds the message unless kind is status or gcstatus,
 		# then it holds status message
 		message_col = message
-		show_col = show
+		subject_col = subject
 		if tim:
 			time_col = int(float(time.mktime(tim)))
 		else:
 			time_col = int(float(time.time()))
 		
-		def commit_to_db(values):
-			sql = 'INSERT INTO logs (jid_id, contact_name, time, kind, show, message) '\
-					'VALUES (?, ?, ?, ?, ?, ?)'
-			cur.execute(sql, values)
-			con.commit()
-			#print 'saved', values
-		
 		jid_id = self.get_jid_id(jid)
-		#print 'jid', jid, 'gets jid_id', jid_id
 		
+		kind_col, show_col = self.convert_human_values_to_db_api_values(kind, show)
+					
+		# now we may have need to do extra care for some values in columns
 		if kind == 'status': # we store (not None) time, jid, show, msg
 			# status for roster items
 			if show is None:
-				show_col = 'online'
+				show_col = constants.SHOW_ONLINE
 
-			values = (jid_id, contact_name_col, time_col, kind, show_col, message_col)
-			commit_to_db(values)
 		elif kind == 'gcstatus':
 			# status in ROOM (for pm status see status)
 			if show is None:
-				show_col = 'online'
+				show_col = constants.SHOW_ONLINE
 			
 			jid, nick = jid.split('/', 1)
-			
 			jid_id = self.get_jid_id(jid) # re-get jid_id for the new jid
 			contact_name_col = nick
-			values = (jid_id, contact_name_col, time_col, kind, show_col, message_col)
+
-			commit_to_db(values)
 		elif kind == 'gc_msg':
 			if jid.find('/') != -1: # if it has a /
 				jid, nick = jid.split('/', 1)
@@ -172,11 +281,9 @@ class Logger:
 			jid_id = self.get_jid_id(jid) # re-get jid_id for the new jid
 			contact_name_col = nick
 		
-			values = (jid_id, contact_name_col, time_col, kind, show_col, message_col)
+		values = (jid_id, contact_name_col, time_col, kind_col, show_col,
-			commit_to_db(values)
+			message_col, subject_col)
-		elif kind in ('single_msg_recv', 'chat_msg_recv', 'chat_msg_sent', 'single_msg_sent'):
+		self.commit_to_db(values)
-			values = (jid_id, contact_name_col, time_col, kind, show_col, message_col)
-			commit_to_db(values)
 
 	def get_last_conversation_lines(self, jid, restore_how_many_rows,
 		pending_how_many, timeout):
@@ -192,10 +299,11 @@ class Logger:
 		# 3 - 8 (we avoid the last 2 lines but we still return 5 asked)
 		cur.execute('''
 			SELECT time, kind, message FROM logs
-			WHERE jid_id = %d AND kind IN
+			WHERE jid_id = %d AND kind IN	(%d, %d, %d, %d)
-			('single_msg_recv', 'chat_msg_recv', 'chat_msg_sent', 'single_msg_sent')
 			ORDER BY time DESC LIMIT %d OFFSET %d
-			''' % (jid_id, restore_how_many_rows, pending_how_many)
+			''' % (constants.KIND_SINGLE_MSG_RECV, constants.KIND_CHAT_MSG_RECV,
+				constants.KIND_SINGLE_MSG_SENT, constants.KIND_CHAT_MSG_SENT,
+				jid_id, restore_how_many_rows, pending_how_many)
 			)
 
 		results = cur.fetchall()

src/history_window.py

@@ -29,6 +29,10 @@ from common import gajim
 from common import helpers
 from common import i18n
 
+from common.logger import Constants
+
+constants = Constants()
+
 _ = i18n._
 APP = i18n.APP
 gtk.glade.bindtextdomain(APP, i18n.DIR)
@@ -97,6 +101,22 @@ class HistoryWindow:
 			else:
 				widget.unmark_day(day)
 
+	def get_string_show_from_constant_int(self, show):
+		if show == constants.SHOW_ONLINE:
+			show = 'online'
+		elif show == constants.SHOW_CHAT:
+			show = 'chat'
+		elif show == constants.SHOW_AWAY:
+			show = 'away'
+		elif show == constants.SHOW_XA:
+			show = 'xa'
+		elif show == constants.SHOW_DND:
+			show = 'dnd'
+		elif show == constants.SHOW_OFFLINE:
+			show = 'offline'
+
+		return show
+
 	def add_lines_for_date(self, year, month, day):
 		'''adds all the lines for given date in textbuffer'''
 		self.history_buffer.set_text('') # clear the buffer first
@@ -117,18 +137,20 @@ class HistoryWindow:
 		tag_name = ''
 		tag_msg = ''
 		
-		if kind == 'gc_msg':
+		show = self.get_string_show_from_constant_int(show)
+		
+		if kind == constants.KIND_GC_MSG:
 			tag_name = 'incoming'
-		elif kind in ('single_msg_recv', 'chat_msg_recv'):
+		elif kind in (constants.KIND_SINGLE_MSG_RECV, constants.KIND_CHAT_MSG_RECV):
 			try:
 				contact_name = gajim.contacts[self.account][self.jid][0].name
 			except:
 				contact_name = self.jid.split('@')[0]
 			tag_name = 'incoming'
-		elif kind in ('single_msg_sent', 'chat_msg_sent'):
+		elif kind in (constants.KIND_SINGLE_MSG_SENT, constants.KIND_CHAT_MSG_SENT):
 			contact_name = gajim.nicks[self.account]
 			tag_name = 'outgoing'
-		elif kind == 'gcstatus':
+		elif kind == constants.KIND_GCSTATUS:
 			# message here (if not None) is status message
 			if message:
 				message = _('%(nick)s is now %(status)s: %(status_msg)s') %\
@@ -150,7 +172,7 @@ class HistoryWindow:
 
 		# do not do this if gcstats, avoid dupping contact_name
 		# eg. nkour: nkour is now Offline
-		if contact_name and kind != 'gcstatus':
+		if contact_name and kind != constants.KIND_GCSTATUS:
 			# add stuff before and after contact name
 			before_str = gajim.config.get('before_nickname')
 			after_str = gajim.config.get('after_nickname')

src/roster_window.py

@@ -46,7 +46,7 @@ APP = i18n.APP
 gtk.glade.bindtextdomain(APP, i18n.DIR)
 gtk.glade.textdomain(APP)
 
-#(icon, name, type, jid, account, editable, s)
+#(icon, name, type, jid, account, editable, second pixbuf)
 (
 C_IMG, # image to show state (online, new message etc)
 C_NAME, # cellrenderer text that holds contact nickame

src/tabbed_chat_window.py

@@ -31,6 +31,9 @@ import gtkgui_helpers
 
 from common import gajim
 from common import helpers
+from common.logger import Constants
+constants = Constants()
+
 from common import i18n
 
 _ = i18n._
@@ -910,10 +913,10 @@ class TabbedChatWindow(chat.Chat):
 			pending_how_many, timeout)
 		
 		for row in rows: # row[0] time, row[1] has kind, row[2] the message
-			if row[1] in ('chat_msg_sent', 'single_msg_sent'):
+			if row[1] in (constants.KIND_CHAT_MSG_SENT, constants.KIND_SINGLE_MSG_SENT):
 				kind = 'outgoing'
 				name = gajim.nicks[self.account]
-			elif row[1] in ('single_msg_recv', 'chat_msg_recv'):
+			elif row[1] in (constants.KIND_SINGLE_MSG_RECV, constants.KIND_CHAT_MSG_RECV):
 				kind = 'incoming'
 				name = self.contacts[jid].name