smem-eklendi

bin/smem

@@ -0,0 +1,689 @@
+#!/usr/bin/env python
+#
+# smem - a tool for meaningful memory reporting
+#
+# Copyright 2008-2009 Matt Mackall <mpm@selenic.com>
+#
+# This software may be used and distributed according to the terms of
+# the GNU General Public License version 2 or later, incorporated
+# herein by reference.
+
+import re, os, sys, pwd, optparse, errno, tarfile
+
+warned = False
+
+class procdata(object):
+    def __init__(self, source):
+        self._ucache = {}
+        self._gcache = {}
+        self.source = source and source or ""
+        self._memdata = None
+    def _list(self):
+        return os.listdir(self.source + "/proc")
+    def _read(self, f):
+        return file(self.source + '/proc/' + f).read()
+    def _readlines(self, f):
+        return self._read(f).splitlines(True)
+    def _stat(self, f):
+        return os.stat(self.source + "/proc/" + f)
+
+    def pids(self):
+        '''get a list of processes'''
+        return [int(e) for e in self._list()
+                if e.isdigit() and not iskernel(e)]
+    def mapdata(self, pid):
+        return self._readlines('%s/smaps' % pid)
+    def memdata(self):
+        if self._memdata is None:
+            self._memdata = self._readlines('meminfo')
+        return self._memdata
+    def version(self):
+        return self._readlines('version')[0]
+    def pidname(self, pid):
+        try:
+            l = self._read('%d/stat' % pid)
+            return l[l.find('(') + 1: l.find(')')]
+        except:
+            return '?'
+    def pidcmd(self, pid):
+        try:
+            c = self._read('%s/cmdline' % pid)[:-1]
+            return c.replace('\0', ' ')
+        except:
+            return '?'
+    def piduser(self, pid):
+        try:
+            return self._stat('%d' % pid).st_uid
+        except:
+            return -1
+    def pidgroup(self, pid):
+        try:
+            return self._stat('%d' % pid).st_gid
+        except:
+            return -1
+    def username(self, uid):
+        if uid == -1:
+            return '?'
+        if uid not in self._ucache:
+            try:
+                self._ucache[uid] = pwd.getpwuid(uid)[0]
+            except KeyError:
+                self._ucache[uid] = str(uid)
+        return self._ucache[uid]
+    def groupname(self, gid):
+        if gid == -1:
+            return '?'
+        if gid not in self._gcache:
+            try:
+                self._gcache[gid] = pwd.getgrgid(gid)[0]
+            except KeyError:
+                self._gcache[gid] = str(gid)
+        return self._gcache[gid]
+
+class tardata(procdata):
+    def __init__(self, source):
+        procdata.__init__(self, source)
+        self.tar = tarfile.open(source)
+    def _list(self):
+        for ti in self.tar:
+            if ti.name.endswith('/smaps'):
+                d,f = ti.name.split('/')
+                yield d
+    def _read(self, f):
+        return self.tar.extractfile(f).read()
+    def _readlines(self, f):
+        return self.tar.extractfile(f).readlines()
+    def piduser(self, p):
+        t = self.tar.getmember("%d" % p)
+        if t.uname:
+            self._ucache[t.uid] = t.uname
+        return t.uid
+    def pidgroup(self, p):
+        t = self.tar.getmember("%d" % p)
+        if t.gname:
+            self._gcache[t.gid] = t.gname
+        return t.gid
+    def username(self, u):
+        return self._ucache.get(u, str(u))
+    def groupname(self, g):
+        return self._gcache.get(g, str(g))
+
+_totalmem = 0
+def totalmem():
+    global _totalmem
+    if not _totalmem:
+        if options.realmem:
+            _totalmem = fromunits(options.realmem) / 1024
+        else:
+            _totalmem = memory()['memtotal']
+    return _totalmem
+
+_kernelsize = 0
+def kernelsize():
+    global _kernelsize
+    if not _kernelsize and options.kernel:
+        try:
+            d = os.popen("size %s" % options.kernel).readlines()[1]
+            _kernelsize = int(d.split()[3]) / 1024
+        except:
+            try:
+                # try some heuristic to find gzipped part in kernel image
+                packedkernel = open(options.kernel).read()
+                pos = packedkernel.find('\x1F\x8B')
+                if pos >= 0 and pos < 25000:
+                    sys.stderr.write("Maybe uncompressed kernel can be extracted by the command:\n"
+                            "  dd if=%s bs=1 skip=%d | gzip -d >%s.unpacked\n\n" % (options.kernel, pos, options.kernel))
+            except:
+                pass
+            sys.stderr.write("Parameter '%s' should be an original uncompressed compiled kernel file.\n\n" % options.kernel)
+    return _kernelsize
+
+def pidmaps(pid):
+    global warned
+    maps = {}
+    start = None
+    seen = False
+    empty = True
+    for l in src.mapdata(pid):
+        empty = False
+        f = l.split()
+        if f[-1] == 'kB':
+            if f[0].startswith('Pss'):
+                seen = True
+            maps[start][f[0][:-1].lower()] = int(f[1])
+        elif '-' in f[0] and ':' not in f[0]: # looks like a mapping range
+            start, end = f[0].split('-')
+            start = int(start, 16)
+            name = "<anonymous>"
+            if len(f) > 5:
+                name = f[5]
+            maps[start] = dict(end=int(end, 16), mode=f[1],
+                               offset=int(f[2], 16),
+                               device=f[3], inode=f[4], name=name)
+
+    if not empty and not seen and not warned:
+        sys.stderr.write('warning: kernel does not appear to support PSS measurement\n')
+        warned = True
+        if not options.sort:
+            options.sort = 'rss'
+
+    if options.mapfilter:
+        f = {}
+        for m in maps:
+            if not filter(options.mapfilter, m, lambda x: maps[x]['name']):
+                f[m] = maps[m]
+        return f
+
+    return maps
+
+def sortmaps(totals, key):
+    l = []
+    for pid in totals:
+        l.append((totals[pid][key], pid))
+    l.sort()
+    return [pid for pid,key in l]
+
+def iskernel(pid):
+    return src.pidcmd(pid) == ""
+
+def memory():
+    t = {}
+    f = re.compile('(\\S+):\\s+(\\d+) kB')
+    for l in src.memdata():
+        m = f.match(l)
+        if m:
+            t[m.group(1).lower()] = int(m.group(2))
+    return t
+
+def units(x):
+    s = ''
+    if x == 0:
+        return '0'
+    for s in ('', 'K', 'M', 'G', 'T'):
+        if x < 1024:
+            break
+        x /= 1024.0
+    return "%.1f%s" % (x, s)
+
+def fromunits(x):
+    s = dict(k=2**10, K=2**10, kB=2**10, KB=2**10,
+             M=2**20, MB=2**20, G=2**30, GB=2**30,
+             T=2**40, TB=2**40)
+    for k,v in s.items():
+        if x.endswith(k):
+            return int(float(x[:-len(k)])*v)
+    sys.stderr.write("Memory size should be written with units, for example 1024M\n")
+    sys.exit(-1)
+
+def pidusername(pid):
+    return src.username(src.piduser(pid))
+
+def showamount(a, total):
+    if options.abbreviate:
+        return units(a * 1024)
+    elif options.percent:
+        if total == 0:
+            return 'N/A'
+        return "%.2f%%" % (100.0 * a / total)
+    return a
+
+def filter(opt, arg, *sources):
+    if not opt:
+        return False
+
+    for f in sources:
+        if re.search(opt, f(arg)):
+            return False
+    return True
+
+def pidtotals(pid):
+    maps = pidmaps(pid)
+    t = dict(size=0, rss=0, pss=0, shared_clean=0, shared_dirty=0,
+             private_clean=0, private_dirty=0, referenced=0, swap=0)
+    for m in maps.iterkeys():
+        for k in t:
+            t[k] += maps[m].get(k, 0)
+
+    t['uss'] = t['private_clean'] + t['private_dirty']
+    t['maps'] = len(maps)
+
+    return t
+
+def processtotals(pids):
+    totals = {}
+    for pid in pids:
+        if (filter(options.processfilter, pid, src.pidname, src.pidcmd) or
+                filter(options.userfilter, pid, pidusername)):
+            continue
+        try:
+            p = pidtotals(pid)
+            if p['maps'] != 0:
+                totals[pid] = p
+        except:
+            continue
+    return totals
+
+def showpids():
+    p = src.pids()
+    pt = processtotals(p)
+
+    def showuser(p):
+        if options.numeric:
+            return src.piduser(p)
+        return pidusername(p)
+
+    fields = dict(
+        pid=('PID', lambda n: n, '% 5s', lambda x: len(pt),
+             'process ID'),
+        user=('User', showuser, '%-8s', lambda x: len(dict.fromkeys(x)),
+              'owner of process'),
+        name=('Name', src.pidname, '%-24.24s', None,
+              'name of process'),
+        command=('Command', src.pidcmd, '%-27.27s', None,
+                 'process command line'),
+        maps=('Maps',lambda n: pt[n]['maps'], '% 5s', sum,
+              'total number of mappings'),
+        swap=('Swap',lambda n: pt[n]['swap'], '% 8a', sum,
+              'amount of swap space consumed (ignoring sharing)'),
+        uss=('USS', lambda n: pt[n]['uss'], '% 8a', sum,
+             'unique set size'),
+        rss=('RSS', lambda n: pt[n]['rss'], '% 8a', sum,
+             'resident set size (ignoring sharing)'),
+        pss=('PSS', lambda n: pt[n]['pss'], '% 8a', sum,
+             'proportional set size (including sharing)'),
+        vss=('VSS', lambda n: pt[n]['size'], '% 8a', sum,
+             'virtual set size (total virtual memory mapped)'),
+        )
+    columns = options.columns or 'pid user command swap uss pss rss'
+
+    showtable(pt.keys(), fields, columns.split(), options.sort or 'pss')
+
+def maptotals(pids):
+    totals = {}
+    for pid in pids:
+        if (filter(options.processfilter, pid, src.pidname, src.pidcmd) or
+                filter(options.userfilter, pid, pidusername)):
+            continue
+        try:
+            maps = pidmaps(pid)
+            seen = {}
+            for m in maps.iterkeys():
+                name = maps[m]['name']
+                if name not in totals:
+                    t = dict(size=0, rss=0, pss=0, shared_clean=0,
+                             shared_dirty=0, private_clean=0, count=0,
+                             private_dirty=0, referenced=0, swap=0, pids=0)
+                else:
+                    t = totals[name]
+
+                for k in t:
+                    t[k] += maps[m].get(k, 0)
+                t['count'] += 1
+                if name not in seen:
+                    t['pids'] += 1
+                    seen[name] = 1
+                totals[name] = t
+        except EnvironmentError:
+            continue
+    return totals
+
+def showmaps():
+    p = src.pids()
+    pt = maptotals(p)
+
+    fields = dict(
+        map=('Map', lambda n: n, '%-40.40s', len,
+             'mapping name'),
+        count=('Count', lambda n: pt[n]['count'], '% 5s', sum,
+               'number of mappings found'),
+        pids=('PIDs', lambda n: pt[n]['pids'], '% 5s', sum,
+              'number of PIDs using mapping'),
+        swap=('Swap',lambda n: pt[n]['swap'], '% 8a', sum,
+              'amount of swap space consumed (ignoring sharing)'),
+        uss=('USS', lambda n: pt[n]['private_clean']
+             + pt[n]['private_dirty'], '% 8a', sum,
+             'unique set size'),
+        rss=('RSS', lambda n: pt[n]['rss'], '% 8a', sum,
+             'resident set size (ignoring sharing)'),
+        pss=('PSS', lambda n: pt[n]['pss'], '% 8a', sum,
+             'proportional set size (including sharing)'),
+        vss=('VSS', lambda n: pt[n]['size'], '% 8a', sum,
+             'virtual set size (total virtual address space mapped)'),
+        avgpss=('AVGPSS', lambda n: int(1.0 * pt[n]['pss']/pt[n]['pids']),
+                '% 8a', sum,
+                'average PSS per PID'),
+        avguss=('AVGUSS', lambda n: int(1.0 * pt[n]['uss']/pt[n]['pids']),
+                '% 8a', sum,
+                'average USS per PID'),
+        avgrss=('AVGRSS', lambda n: int(1.0 * pt[n]['rss']/pt[n]['pids']),
+                '% 8a', sum,
+                'average RSS per PID'),
+        )
+    columns = options.columns or 'map pids avgpss pss'
+
+    showtable(pt.keys(), fields, columns.split(), options.sort or 'pss')
+
+def usertotals(pids):
+    totals = {}
+    for pid in pids:
+        if (filter(options.processfilter, pid, src.pidname, src.pidcmd) or
+                filter(options.userfilter, pid, pidusername)):
+            continue
+        try:
+            maps = pidmaps(pid)
+            if len(maps) == 0:
+                continue
+        except EnvironmentError:
+            continue
+        user = src.piduser(pid)
+        if user not in totals:
+            t = dict(size=0, rss=0, pss=0, shared_clean=0,
+                     shared_dirty=0, private_clean=0, count=0,
+                     private_dirty=0, referenced=0, swap=0)
+        else:
+            t = totals[user]
+
+        for m in maps.iterkeys():
+            for k in t:
+                t[k] += maps[m].get(k, 0)
+
+        t['count'] += 1
+        totals[user] = t
+    return totals
+
+def showusers():
+    p = src.pids()
+    pt = usertotals(p)
+
+    def showuser(u):
+        if options.numeric:
+            return u
+        return src.username(u)
+
+    fields = dict(
+        user=('User', showuser, '%-8s', None,
+              'user name or ID'),
+        count=('Count', lambda n: pt[n]['count'], '% 5s', sum,
+               'number of processes'),
+        swap=('Swap',lambda n: pt[n]['swap'], '% 8a', sum,
+              'amount of swapspace consumed (ignoring sharing)'),
+        uss=('USS', lambda n: pt[n]['private_clean']
+             + pt[n]['private_dirty'], '% 8a', sum,
+             'unique set size'),
+        rss=('RSS', lambda n: pt[n]['rss'], '% 8a', sum,
+             'resident set size (ignoring sharing)'),
+        pss=('PSS', lambda n: pt[n]['pss'], '% 8a', sum,
+             'proportional set size (including sharing)'),
+        vss=('VSS', lambda n: pt[n]['pss'], '% 8a', sum,
+             'virtual set size (total virtual memory mapped)'),
+        )
+    columns = options.columns or 'user count swap uss pss rss'
+
+    showtable(pt.keys(), fields, columns.split(), options.sort or 'pss')
+
+def showsystem():
+    t = totalmem()
+    ki = kernelsize()
+    m = memory()
+
+    mt = m['memtotal']
+    f = m['memfree']
+
+    # total amount used by hardware
+    fh = max(t - mt - ki, 0)
+
+    # total amount mapped into userspace (ie mapped an unmapped pages)
+    u = m['anonpages'] + m['mapped']
+
+    # total amount allocated by kernel not for userspace
+    kd = mt - f - u
+
+    # total amount in kernel caches
+    kdc = m['buffers'] + m['sreclaimable'] + (m['cached'] - m['mapped'])
+
+    l = [("firmware/hardware", fh, 0),
+         ("kernel image", ki, 0),
+         ("kernel dynamic memory", kd, kdc),
+         ("userspace memory", u, m['mapped']),
+         ("free memory", f, f)]
+
+    fields = dict(
+        order=('Order', lambda n: n, '% 1s', lambda x: '',
+             'hierarchical order'),
+        area=('Area', lambda n: l[n][0], '%-24s', lambda x: '',
+             'memory area'),
+        used=('Used', lambda n: l[n][1], '%10a', sum,
+              'area in use'),
+        cache=('Cache', lambda n: l[n][2], '%10a', sum,
+              'area used as reclaimable cache'),
+        noncache=('Noncache', lambda n: l[n][1] - l[n][2], '%10a', sum,
+              'area not reclaimable'))
+
+    columns = options.columns or 'area used cache noncache'
+    showtable(range(len(l)), fields, columns.split(), options.sort or 'order')
+
+def showfields(fields, f):
+    if f != list:
+        print "unknown field", f
+    print "known fields:"
+    for l in sorted(fields.keys()):
+        print "%-8s %s" % (l, fields[l][-1])
+
+def showtable(rows, fields, columns, sort):
+    header = ""
+    format = ""
+    formatter = []
+
+    if sort not in fields:
+            showfields(fields, sort)
+            sys.exit(-1)
+
+    if options.pie:
+        columns.append(options.pie)
+    if options.bar:
+        columns.append(options.bar)
+
+    mt = totalmem()
+    st = memory()['swaptotal']
+
+    for n in columns:
+        if n not in fields:
+            showfields(fields, n)
+            sys.exit(-1)
+
+        f = fields[n][2]
+        if 'a' in f:
+            if n == 'swap':
+                formatter.append(lambda x: showamount(x, st))
+            else:
+                formatter.append(lambda x: showamount(x, mt))
+            f = f.replace('a', 's')
+        else:
+            formatter.append(lambda x: x)
+        format += f + " "
+        header += f % fields[n][0] + " "
+
+    l = []
+    for n in rows:
+        r = [fields[c][1](n) for c in columns]
+        l.append((fields[sort][1](n), r))
+
+    l.sort(reverse=bool(options.reverse))
+
+    if options.pie:
+        showpie(l, sort)
+        return
+    elif options.bar:
+        showbar(l, columns, sort)
+        return
+
+    if not options.no_header:
+        print header
+
+    for k,r in l:
+        print format % tuple([f(v) for f,v in zip(formatter, r)])
+
+    if options.totals:
+        # totals
+        t = []
+        for c in columns:
+            f = fields[c][3]
+            if f:
+                t.append(f([fields[c][1](n) for n in rows]))
+            else:
+                t.append("")
+
+        print "-" * len(header)
+        print format % tuple([f(v) for f,v in zip(formatter, t)])
+
+def showpie(l, sort):
+    try:
+        import pylab
+    except ImportError:
+        sys.stderr.write("pie chart requires matplotlib\n")
+        sys.exit(-1)
+
+    if (l[0][0] < l[-1][0]):
+        l.reverse()
+
+    labels = [r[1][-1] for r in l]
+    values = [r[0] for r in l] # sort field
+
+    tm = totalmem()
+    s = sum(values)
+    unused = tm - s
+    t = 0
+    while values and (t + values[-1] < (tm * .02) or
+                      values[-1] < (tm * .005)):
+        t += values.pop()
+        labels.pop()
+
+    if t:
+        values.append(t)
+        labels.append('other')
+
+    explode = [0] * len(values)
+    if unused > 0:
+        values.insert(0, unused)
+        labels.insert(0, 'unused')
+        explode.insert(0, .05)
+
+    pylab.figure(1, figsize=(6,6))
+    ax = pylab.axes([0.1, 0.1, 0.8, 0.8])
+    pylab.pie(values, explode = explode, labels=labels,
+              autopct="%.2f%%", shadow=True)
+    pylab.title('%s by %s' % (options.pie, sort))
+    pylab.show()
+
+def showbar(l, columns, sort):
+    try:
+        import pylab, numpy
+    except ImportError:
+        sys.stderr.write("bar chart requires matplotlib\n")
+        sys.exit(-1)
+
+    if (l[0][0] < l[-1][0]):
+        l.reverse()
+
+    rc = []
+    key = []
+    for n in range(len(columns) - 1):
+        try:
+            if columns[n] in 'pid user group'.split():
+                continue
+            float(l[0][1][n])
+            rc.append(n)
+            key.append(columns[n])
+        except:
+            pass
+
+    width = 1.0 / (len(rc) + 1)
+    offset = width / 2
+
+    def gc(n):
+        return 'bgrcmyw'[n % 7]
+
+    pl = []
+    ind = numpy.arange(len(l))
+    for n in xrange(len(rc)):
+        pl.append(pylab.bar(ind + offset + width * n,
+                             [x[1][rc[n]] for x in l], width, color=gc(n)))
+
+    #plt.xticks(ind + .5, )
+    pylab.gca().set_xticks(ind + .5)
+    pylab.gca().set_xticklabels([x[1][-1] for x in l], rotation=45)
+    pylab.legend([p[0] for p in pl], key)
+    pylab.show()
+
+
+parser = optparse.OptionParser("%prog [options]")
+parser.add_option("-H", "--no-header", action="store_true",
+                  help="disable header line")
+parser.add_option("-c", "--columns", type="str",
+                  help="columns to show")
+parser.add_option("-t", "--totals", action="store_true",
+                  help="show totals")
+
+parser.add_option("-R", "--realmem", type="str",
+                  help="amount of physical RAM")
+parser.add_option("-K", "--kernel", type="str",
+                  help="path to kernel image")
+
+parser.add_option("-m", "--mappings", action="store_true",
+                  help="show mappings")
+parser.add_option("-u", "--users", action="store_true",
+                  help="show users")
+parser.add_option("-w", "--system", action="store_true",
+                  help="show whole system")
+
+parser.add_option("-P", "--processfilter", type="str",
+                  help="process filter regex")
+parser.add_option("-M", "--mapfilter", type="str",
+                  help="map filter regex")
+parser.add_option("-U", "--userfilter", type="str",
+                  help="user filter regex")
+
+parser.add_option("-n", "--numeric", action="store_true",
+                  help="numeric output")
+parser.add_option("-s", "--sort", type="str",
+                  help="field to sort on")
+parser.add_option("-r", "--reverse", action="store_true",
+                  help="reverse sort")
+
+parser.add_option("-p", "--percent", action="store_true",
+                  help="show percentage")
+parser.add_option("-k", "--abbreviate", action="store_true",
+                  help="show unit suffixes")
+
+parser.add_option("", "--pie", type='str',
+                  help="show pie graph")
+parser.add_option("", "--bar", type='str',
+                  help="show bar graph")
+
+parser.add_option("-S", "--source", type="str",
+                  help="/proc data source")
+
+
+defaults = {}
+parser.set_defaults(**defaults)
+(options, args) = parser.parse_args()
+
+try:
+    src = tardata(options.source)
+except:
+    src = procdata(options.source)
+
+try:
+    if options.mappings:
+        showmaps()
+    elif options.users:
+        showusers()
+    elif options.system:
+        showsystem()
+    else:
+        showpids()
+except IOError, e:
+    if e.errno == errno.EPIPE:
+        pass
+except KeyboardInterrupt:
+    pass