milis/bin/smem

#!/usr/bin/env python
#
# smem - a tool for meaningful memory reporting
#
# Copyright 2008-2009 Matt Mackall <mpm@selenic.com>
#
# This software may be used and distributed according to the terms of
# the GNU General Public License version 2 or later, incorporated
# herein by reference.

import re, os, sys, pwd, optparse, errno, tarfile

warned = False

class procdata(object):
    def __init__(self, source):
        self._ucache = {}
        self._gcache = {}
        self.source = source and source or ""
        self._memdata = None
    def _list(self):
        return os.listdir(self.source + "/proc")
    def _read(self, f):
        return file(self.source + '/proc/' + f).read()
    def _readlines(self, f):
        return self._read(f).splitlines(True)
    def _stat(self, f):
        return os.stat(self.source + "/proc/" + f)

    def pids(self):
        '''get a list of processes'''
        return [int(e) for e in self._list()
                if e.isdigit() and not iskernel(e)]
    def mapdata(self, pid):
        return self._readlines('%s/smaps' % pid)
    def memdata(self):
        if self._memdata is None:
            self._memdata = self._readlines('meminfo')
        return self._memdata
    def version(self):
        return self._readlines('version')[0]
    def pidname(self, pid):
        try:
            l = self._read('%d/stat' % pid)
            return l[l.find('(') + 1: l.find(')')]
        except:
            return '?'
    def pidcmd(self, pid):
        try:
            c = self._read('%s/cmdline' % pid)[:-1]
            return c.replace('\0', ' ')
        except:
            return '?'
    def piduser(self, pid):
        try:
            return self._stat('%d' % pid).st_uid
        except:
            return -1
    def pidgroup(self, pid):
        try:
            return self._stat('%d' % pid).st_gid
        except:
            return -1
    def username(self, uid):
        if uid == -1:
            return '?'
        if uid not in self._ucache:
            try:
                self._ucache[uid] = pwd.getpwuid(uid)[0]
            except KeyError:
                self._ucache[uid] = str(uid)
        return self._ucache[uid]
    def groupname(self, gid):
        if gid == -1:
            return '?'
        if gid not in self._gcache:
            try:
                self._gcache[gid] = pwd.getgrgid(gid)[0]
            except KeyError:
                self._gcache[gid] = str(gid)
        return self._gcache[gid]

class tardata(procdata):
    def __init__(self, source):
        procdata.__init__(self, source)
        self.tar = tarfile.open(source)
    def _list(self):
        for ti in self.tar:
            if ti.name.endswith('/smaps'):
                d,f = ti.name.split('/')
                yield d
    def _read(self, f):
        return self.tar.extractfile(f).read()
    def _readlines(self, f):
        return self.tar.extractfile(f).readlines()
    def piduser(self, p):
        t = self.tar.getmember("%d" % p)
        if t.uname:
            self._ucache[t.uid] = t.uname
        return t.uid
    def pidgroup(self, p):
        t = self.tar.getmember("%d" % p)
        if t.gname:
            self._gcache[t.gid] = t.gname
        return t.gid
    def username(self, u):
        return self._ucache.get(u, str(u))
    def groupname(self, g):
        return self._gcache.get(g, str(g))

_totalmem = 0
def totalmem():
    global _totalmem
    if not _totalmem:
        if options.realmem:
            _totalmem = fromunits(options.realmem) / 1024
        else:
            _totalmem = memory()['memtotal']
    return _totalmem

_kernelsize = 0
def kernelsize():
    global _kernelsize
    if not _kernelsize and options.kernel:
        try:
            d = os.popen("size %s" % options.kernel).readlines()[1]
            _kernelsize = int(d.split()[3]) / 1024
        except:
            try:
                # try some heuristic to find gzipped part in kernel image
                packedkernel = open(options.kernel).read()
                pos = packedkernel.find('\x1F\x8B')
                if pos >= 0 and pos < 25000:
                    sys.stderr.write("Maybe uncompressed kernel can be extracted by the command:\n"
                            "  dd if=%s bs=1 skip=%d | gzip -d >%s.unpacked\n\n" % (options.kernel, pos, options.kernel))
            except:
                pass
            sys.stderr.write("Parameter '%s' should be an original uncompressed compiled kernel file.\n\n" % options.kernel)
    return _kernelsize

def pidmaps(pid):
    global warned
    maps = {}
    start = None
    seen = False
    empty = True
    for l in src.mapdata(pid):
        empty = False
        f = l.split()
        if f[-1] == 'kB':
            if f[0].startswith('Pss'):
                seen = True
            maps[start][f[0][:-1].lower()] = int(f[1])
        elif '-' in f[0] and ':' not in f[0]: # looks like a mapping range
            start, end = f[0].split('-')
            start = int(start, 16)
            name = "<anonymous>"
            if len(f) > 5:
                name = f[5]
            maps[start] = dict(end=int(end, 16), mode=f[1],
                               offset=int(f[2], 16),
                               device=f[3], inode=f[4], name=name)

    if not empty and not seen and not warned:
        sys.stderr.write('warning: kernel does not appear to support PSS measurement\n')
        warned = True
        if not options.sort:
            options.sort = 'rss'

    if options.mapfilter:
        f = {}
        for m in maps:
            if not filter(options.mapfilter, m, lambda x: maps[x]['name']):
                f[m] = maps[m]
        return f

    return maps

def sortmaps(totals, key):
    l = []
    for pid in totals:
        l.append((totals[pid][key], pid))
    l.sort()
    return [pid for pid,key in l]

def iskernel(pid):
    return src.pidcmd(pid) == ""

def memory():
    t = {}
    f = re.compile('(\\S+):\\s+(\\d+) kB')
    for l in src.memdata():
        m = f.match(l)
        if m:
            t[m.group(1).lower()] = int(m.group(2))
    return t

def units(x):
    s = ''
    if x == 0:
        return '0'
    for s in ('', 'K', 'M', 'G', 'T'):
        if x < 1024:
            break
        x /= 1024.0
    return "%.1f%s" % (x, s)

def fromunits(x):
    s = dict(k=2**10, K=2**10, kB=2**10, KB=2**10,
             M=2**20, MB=2**20, G=2**30, GB=2**30,
             T=2**40, TB=2**40)
    for k,v in s.items():
        if x.endswith(k):
            return int(float(x[:-len(k)])*v)
    sys.stderr.write("Memory size should be written with units, for example 1024M\n")
    sys.exit(-1)

def pidusername(pid):
    return src.username(src.piduser(pid))

def showamount(a, total):
    if options.abbreviate:
        return units(a * 1024)
    elif options.percent:
        if total == 0:
            return 'N/A'
        return "%.2f%%" % (100.0 * a / total)
    return a

def filter(opt, arg, *sources):
    if not opt:
        return False

    for f in sources:
        if re.search(opt, f(arg)):
            return False
    return True

def pidtotals(pid):
    maps = pidmaps(pid)
    t = dict(size=0, rss=0, pss=0, shared_clean=0, shared_dirty=0,
             private_clean=0, private_dirty=0, referenced=0, swap=0)
    for m in maps.iterkeys():
        for k in t:
            t[k] += maps[m].get(k, 0)

    t['uss'] = t['private_clean'] + t['private_dirty']
    t['maps'] = len(maps)

    return t

def processtotals(pids):
    totals = {}
    for pid in pids:
        if (filter(options.processfilter, pid, src.pidname, src.pidcmd) or
                filter(options.userfilter, pid, pidusername)):
            continue
        try:
            p = pidtotals(pid)
            if p['maps'] != 0:
                totals[pid] = p
        except:
            continue
    return totals

def showpids():
    p = src.pids()
    pt = processtotals(p)

    def showuser(p):
        if options.numeric:
            return src.piduser(p)
        return pidusername(p)

    fields = dict(
        pid=('PID', lambda n: n, '% 5s', lambda x: len(pt),
             'process ID'),
        user=('User', showuser, '%-8s', lambda x: len(dict.fromkeys(x)),
              'owner of process'),
        name=('Name', src.pidname, '%-24.24s', None,
              'name of process'),
        command=('Command', src.pidcmd, '%-27.27s', None,
                 'process command line'),
        maps=('Maps',lambda n: pt[n]['maps'], '% 5s', sum,
              'total number of mappings'),
        swap=('Swap',lambda n: pt[n]['swap'], '% 8a', sum,
              'amount of swap space consumed (ignoring sharing)'),
        uss=('USS', lambda n: pt[n]['uss'], '% 8a', sum,
             'unique set size'),
        rss=('RSS', lambda n: pt[n]['rss'], '% 8a', sum,
             'resident set size (ignoring sharing)'),
        pss=('PSS', lambda n: pt[n]['pss'], '% 8a', sum,
             'proportional set size (including sharing)'),
        vss=('VSS', lambda n: pt[n]['size'], '% 8a', sum,
             'virtual set size (total virtual memory mapped)'),
        )
    columns = options.columns or 'pid user command swap uss pss rss'

    showtable(pt.keys(), fields, columns.split(), options.sort or 'pss')

def maptotals(pids):
    totals = {}
    for pid in pids:
        if (filter(options.processfilter, pid, src.pidname, src.pidcmd) or
                filter(options.userfilter, pid, pidusername)):
            continue
        try:
            maps = pidmaps(pid)
            seen = {}
            for m in maps.iterkeys():
                name = maps[m]['name']
                if name not in totals:
                    t = dict(size=0, rss=0, pss=0, shared_clean=0,
                             shared_dirty=0, private_clean=0, count=0,
                             private_dirty=0, referenced=0, swap=0, pids=0)
                else:
                    t = totals[name]

                for k in t:
                    t[k] += maps[m].get(k, 0)
                t['count'] += 1
                if name not in seen:
                    t['pids'] += 1
                    seen[name] = 1
                totals[name] = t
        except EnvironmentError:
            continue
    return totals

def showmaps():
    p = src.pids()
    pt = maptotals(p)

    fields = dict(
        map=('Map', lambda n: n, '%-40.40s', len,
             'mapping name'),
        count=('Count', lambda n: pt[n]['count'], '% 5s', sum,
               'number of mappings found'),
        pids=('PIDs', lambda n: pt[n]['pids'], '% 5s', sum,
              'number of PIDs using mapping'),
        swap=('Swap',lambda n: pt[n]['swap'], '% 8a', sum,
              'amount of swap space consumed (ignoring sharing)'),
        uss=('USS', lambda n: pt[n]['private_clean']
             + pt[n]['private_dirty'], '% 8a', sum,
             'unique set size'),
        rss=('RSS', lambda n: pt[n]['rss'], '% 8a', sum,
             'resident set size (ignoring sharing)'),
        pss=('PSS', lambda n: pt[n]['pss'], '% 8a', sum,
             'proportional set size (including sharing)'),
        vss=('VSS', lambda n: pt[n]['size'], '% 8a', sum,
             'virtual set size (total virtual address space mapped)'),
        avgpss=('AVGPSS', lambda n: int(1.0 * pt[n]['pss']/pt[n]['pids']),
                '% 8a', sum,
                'average PSS per PID'),
        avguss=('AVGUSS', lambda n: int(1.0 * pt[n]['uss']/pt[n]['pids']),
                '% 8a', sum,
                'average USS per PID'),
        avgrss=('AVGRSS', lambda n: int(1.0 * pt[n]['rss']/pt[n]['pids']),
                '% 8a', sum,
                'average RSS per PID'),
        )
    columns = options.columns or 'map pids avgpss pss'

    showtable(pt.keys(), fields, columns.split(), options.sort or 'pss')

def usertotals(pids):
    totals = {}
    for pid in pids:
        if (filter(options.processfilter, pid, src.pidname, src.pidcmd) or
                filter(options.userfilter, pid, pidusername)):
            continue
        try:
            maps = pidmaps(pid)
            if len(maps) == 0:
                continue
        except EnvironmentError:
            continue
        user = src.piduser(pid)
        if user not in totals:
            t = dict(size=0, rss=0, pss=0, shared_clean=0,
                     shared_dirty=0, private_clean=0, count=0,
                     private_dirty=0, referenced=0, swap=0)
        else:
            t = totals[user]

        for m in maps.iterkeys():
            for k in t:
                t[k] += maps[m].get(k, 0)

        t['count'] += 1
        totals[user] = t
    return totals

def showusers():
    p = src.pids()
    pt = usertotals(p)

    def showuser(u):
        if options.numeric:
            return u
        return src.username(u)

    fields = dict(
        user=('User', showuser, '%-8s', None,
              'user name or ID'),
        count=('Count', lambda n: pt[n]['count'], '% 5s', sum,
               'number of processes'),
        swap=('Swap',lambda n: pt[n]['swap'], '% 8a', sum,
              'amount of swapspace consumed (ignoring sharing)'),
        uss=('USS', lambda n: pt[n]['private_clean']
             + pt[n]['private_dirty'], '% 8a', sum,
             'unique set size'),
        rss=('RSS', lambda n: pt[n]['rss'], '% 8a', sum,
             'resident set size (ignoring sharing)'),
        pss=('PSS', lambda n: pt[n]['pss'], '% 8a', sum,
             'proportional set size (including sharing)'),
        vss=('VSS', lambda n: pt[n]['pss'], '% 8a', sum,
             'virtual set size (total virtual memory mapped)'),
        )
    columns = options.columns or 'user count swap uss pss rss'

    showtable(pt.keys(), fields, columns.split(), options.sort or 'pss')

def showsystem():
    t = totalmem()
    ki = kernelsize()
    m = memory()

    mt = m['memtotal']
    f = m['memfree']

    # total amount used by hardware
    fh = max(t - mt - ki, 0)

    # total amount mapped into userspace (ie mapped an unmapped pages)
    u = m['anonpages'] + m['mapped']

    # total amount allocated by kernel not for userspace
    kd = mt - f - u

    # total amount in kernel caches
    kdc = m['buffers'] + m['sreclaimable'] + (m['cached'] - m['mapped'])

    l = [("firmware/hardware", fh, 0),
         ("kernel image", ki, 0),
         ("kernel dynamic memory", kd, kdc),
         ("userspace memory", u, m['mapped']),
         ("free memory", f, f)]

    fields = dict(
        order=('Order', lambda n: n, '% 1s', lambda x: '',
             'hierarchical order'),
        area=('Area', lambda n: l[n][0], '%-24s', lambda x: '',
             'memory area'),
        used=('Used', lambda n: l[n][1], '%10a', sum,
              'area in use'),
        cache=('Cache', lambda n: l[n][2], '%10a', sum,
              'area used as reclaimable cache'),
        noncache=('Noncache', lambda n: l[n][1] - l[n][2], '%10a', sum,
              'area not reclaimable'))

    columns = options.columns or 'area used cache noncache'
    showtable(range(len(l)), fields, columns.split(), options.sort or 'order')

def showfields(fields, f):
    if f != list:
        print "unknown field", f
    print "known fields:"
    for l in sorted(fields.keys()):
        print "%-8s %s" % (l, fields[l][-1])

def showtable(rows, fields, columns, sort):
    header = ""
    format = ""
    formatter = []

    if sort not in fields:
            showfields(fields, sort)
            sys.exit(-1)

    if options.pie:
        columns.append(options.pie)
    if options.bar:
        columns.append(options.bar)

    mt = totalmem()
    st = memory()['swaptotal']

    for n in columns:
        if n not in fields:
            showfields(fields, n)
            sys.exit(-1)

        f = fields[n][2]
        if 'a' in f:
            if n == 'swap':
                formatter.append(lambda x: showamount(x, st))
            else:
                formatter.append(lambda x: showamount(x, mt))
            f = f.replace('a', 's')
        else:
            formatter.append(lambda x: x)
        format += f + " "
        header += f % fields[n][0] + " "

    l = []
    for n in rows:
        r = [fields[c][1](n) for c in columns]
        l.append((fields[sort][1](n), r))

    l.sort(reverse=bool(options.reverse))

    if options.pie:
        showpie(l, sort)
        return
    elif options.bar:
        showbar(l, columns, sort)
        return

    if not options.no_header:
        print header

    for k,r in l:
        print format % tuple([f(v) for f,v in zip(formatter, r)])

    if options.totals:
        # totals
        t = []
        for c in columns:
            f = fields[c][3]
            if f:
                t.append(f([fields[c][1](n) for n in rows]))
            else:
                t.append("")

        print "-" * len(header)
        print format % tuple([f(v) for f,v in zip(formatter, t)])

def showpie(l, sort):
    try:
        import pylab
    except ImportError:
        sys.stderr.write("pie chart requires matplotlib\n")
        sys.exit(-1)

    if (l[0][0] < l[-1][0]):
        l.reverse()

    labels = [r[1][-1] for r in l]
    values = [r[0] for r in l] # sort field

    tm = totalmem()
    s = sum(values)
    unused = tm - s
    t = 0
    while values and (t + values[-1] < (tm * .02) or
                      values[-1] < (tm * .005)):
        t += values.pop()
        labels.pop()

    if t:
        values.append(t)
        labels.append('other')

    explode = [0] * len(values)
    if unused > 0:
        values.insert(0, unused)
        labels.insert(0, 'unused')
        explode.insert(0, .05)

    pylab.figure(1, figsize=(6,6))
    ax = pylab.axes([0.1, 0.1, 0.8, 0.8])
    pylab.pie(values, explode = explode, labels=labels,
              autopct="%.2f%%", shadow=True)
    pylab.title('%s by %s' % (options.pie, sort))
    pylab.show()

def showbar(l, columns, sort):
    try:
        import pylab, numpy
    except ImportError:
        sys.stderr.write("bar chart requires matplotlib\n")
        sys.exit(-1)

    if (l[0][0] < l[-1][0]):
        l.reverse()

    rc = []
    key = []
    for n in range(len(columns) - 1):
        try:
            if columns[n] in 'pid user group'.split():
                continue
            float(l[0][1][n])
            rc.append(n)
            key.append(columns[n])
        except:
            pass

    width = 1.0 / (len(rc) + 1)
    offset = width / 2

    def gc(n):
        return 'bgrcmyw'[n % 7]

    pl = []
    ind = numpy.arange(len(l))
    for n in xrange(len(rc)):
        pl.append(pylab.bar(ind + offset + width * n,
                             [x[1][rc[n]] for x in l], width, color=gc(n)))

    #plt.xticks(ind + .5, )
    pylab.gca().set_xticks(ind + .5)
    pylab.gca().set_xticklabels([x[1][-1] for x in l], rotation=45)
    pylab.legend([p[0] for p in pl], key)
    pylab.show()


parser = optparse.OptionParser("%prog [options]")
parser.add_option("-H", "--no-header", action="store_true",
                  help="disable header line")
parser.add_option("-c", "--columns", type="str",
                  help="columns to show")
parser.add_option("-t", "--totals", action="store_true",
                  help="show totals")

parser.add_option("-R", "--realmem", type="str",
                  help="amount of physical RAM")
parser.add_option("-K", "--kernel", type="str",
                  help="path to kernel image")

parser.add_option("-m", "--mappings", action="store_true",
                  help="show mappings")
parser.add_option("-u", "--users", action="store_true",
                  help="show users")
parser.add_option("-w", "--system", action="store_true",
                  help="show whole system")

parser.add_option("-P", "--processfilter", type="str",
                  help="process filter regex")
parser.add_option("-M", "--mapfilter", type="str",
                  help="map filter regex")
parser.add_option("-U", "--userfilter", type="str",
                  help="user filter regex")

parser.add_option("-n", "--numeric", action="store_true",
                  help="numeric output")
parser.add_option("-s", "--sort", type="str",
                  help="field to sort on")
parser.add_option("-r", "--reverse", action="store_true",
                  help="reverse sort")

parser.add_option("-p", "--percent", action="store_true",
                  help="show percentage")
parser.add_option("-k", "--abbreviate", action="store_true",
                  help="show unit suffixes")

parser.add_option("", "--pie", type='str',
                  help="show pie graph")
parser.add_option("", "--bar", type='str',
                  help="show bar graph")

parser.add_option("-S", "--source", type="str",
                  help="/proc data source")


defaults = {}
parser.set_defaults(**defaults)
(options, args) = parser.parse_args()

try:
    src = tardata(options.source)
except:
    src = procdata(options.source)

try:
    if options.mappings:
        showmaps()
    elif options.users:
        showusers()
    elif options.system:
        showsystem()
    else:
        showpids()
except IOError, e:
    if e.errno == errno.EPIPE:
        pass
except KeyboardInterrupt:
    pass