regression.py

#!/usr/bin/python
"""
Program that parses standard format results,
compute and check regression bug.

:copyright: Red Hat 2011-2012
:author: Amos Kong <akong@redhat.com>
"""
import os
import sys
import re
import commands
import warnings
import ConfigParser
import MySQLdb


def exec_sql(cmd, conf="../../global_config.ini"):
    config = ConfigParser.ConfigParser()
    config.read(conf)
    user = config.get("AUTOTEST_WEB", "user")
    passwd = config.get("AUTOTEST_WEB", "password")
    db = config.get("AUTOTEST_WEB", "database")
    db_type = config.get("AUTOTEST_WEB", "db_type")
    if db_type != 'mysql':
        print "regression.py: only support mysql database!"
        sys.exit(1)

    conn = MySQLdb.connect(host="localhost", user=user,
                           passwd=passwd, db=db)
    cursor = conn.cursor()
    cursor.execute(cmd)
    rows = cursor.fetchall()
    lines = []
    for row in rows:
        line = []
        for c in row:
            line.append(str(c))
        lines.append(" ".join(line))

    cursor.close()
    conn.close()
    return lines


def get_test_keyval(jobid, keyname, default=''):
    idx = exec_sql("select job_idx from tko_jobs where afe_job_id=%s"
                   % jobid)[-1]
    test_idx = exec_sql('select test_idx from tko_tests where job_idx=%s'
                        % idx)[3]
    try:
        return exec_sql('select value from tko_test_attributes'
                        ' where test_idx=%s and attribute="%s"'
                        % (test_idx, keyname))[-1]
    except:
        return default


class Sample(object):

    """ Collect test results in same environment to a sample """

    def __init__(self, sample_type, arg):
        def generate_raw_table(test_dict):
            ret_dict = []
            tmp = []
            sample_type = category = None
            for i in test_dict:
                line = i.split('|')[1:]
                if not sample_type:
                    sample_type = line[0:2]
                if sample_type != line[0:2]:
                    ret_dict.append('|'.join(sample_type + tmp))
                    sample_type = line[0:2]
                    tmp = []
                if "e+" in line[-1]:
                    tmp.append("%f" % float(line[-1]))
                elif 'e-' in line[-1]:
                    tmp.append("%f" % float(line[-1]))
                elif not (re.findall("[a-zA-Z]", line[-1]) or is_int(line[-1])):
                    tmp.append("%f" % float(line[-1]))
                else:
                    tmp.append(line[-1])

                if category != i.split('|')[0]:
                    category = i.split('|')[0]
                    ret_dict.append("Category:" + category.strip())
                    ret_dict.append(self.categories)
            ret_dict.append('|'.join(sample_type + tmp))
            return ret_dict

        if sample_type == 'filepath':
            files = arg.split()
            self.files_dict = []
            for i in range(len(files)):
                fd = open(files[i], "r")
                f = []
                for l in fd.readlines():
                    l = l.strip()
                    if re.findall("^### ", l):
                        if "kvm-userspace-ver" in l:
                            self.kvmver = l.split(':')[-1]
                        elif "kvm_version" in l:
                            self.hostkernel = l.split(':')[-1]
                        elif "guest-kernel-ver" in l:
                            self.guestkernel = l.split(':')[-1]
                        elif "session-length" in l:
                            self.len = l.split(':')[-1]
                    else:
                        f.append(l.strip())
                self.files_dict.append(f)
                fd.close()

            sysinfodir = os.path.join(os.path.dirname(files[0]), "../../sysinfo/")
            sysinfodir = os.path.realpath(sysinfodir)
            cpuinfo = commands.getoutput("cat %s/cpuinfo" % sysinfodir)
            lscpu = commands.getoutput("cat %s/lscpu" % sysinfodir)
            meminfo = commands.getoutput("cat %s/meminfo" % sysinfodir)
            lspci = commands.getoutput("cat %s/lspci_-vvnn" % sysinfodir)
            partitions = commands.getoutput("cat %s/partitions" % sysinfodir)
            fdisk = commands.getoutput("cat %s/fdisk_-l" % sysinfodir)

            status_path = os.path.join(os.path.dirname(files[0]), "../status")
            status_file = open(status_path, 'r')
            content = status_file.readlines()
            self.testdata = re.findall("localtime=(.*)\t", content[-1])[-1]

            cpunum = len(re.findall("processor\s+: \d", cpuinfo))
            cpumodel = re.findall("Model name:\s+(.*)", lscpu)
            socketnum = int(re.findall("Socket\(s\):\s+(\d+)", lscpu)[0])
            corenum = int(re.findall("Core\(s\) per socket:\s+(\d+)", lscpu)[0]) * socketnum
            threadnum = int(re.findall("Thread\(s\) per core:\s+(\d+)", lscpu)[0]) * corenum
            numanodenum = int(re.findall("NUMA node\(s\):\s+(\d+)", lscpu)[0])
            memnum = float(re.findall("MemTotal:\s+(\d+)", meminfo)[0]) / 1024 / 1024
            nicnum = len(re.findall("\d+:\d+\.0 Ethernet", lspci))
            disknum = re.findall("sd\w+\S", partitions)
            fdiskinfo = re.findall("Disk\s+(/dev/sd.*\s+GiB),", fdisk)
        elif sample_type == 'database':
            jobid = arg
            self.kvmver = get_test_keyval(jobid, "kvm-userspace-ver")
            self.hostkernel = get_test_keyval(jobid, "kvm_version")
            self.guestkernel = get_test_keyval(jobid, "guest-kernel-ver")
            self.len = get_test_keyval(jobid, "session-length")
            self.categories = get_test_keyval(jobid, "category")

            idx = exec_sql("select job_idx from tko_jobs where afe_job_id=%s"
                           % jobid)[-1]
            data = exec_sql("select test_idx,iteration_key,iteration_value"
                            " from tko_perf_view where job_idx=%s" % idx)
            testidx = None
            job_dict = []
            test_dict = []
            for l in data:
                s = l.split()
                if not testidx:
                    testidx = s[0]
                if testidx != s[0]:
                    job_dict.append(generate_raw_table(test_dict))
                    test_dict = []
                    testidx = s[0]
                test_dict.append(' | '.join(s[1].split('--')[0:] + s[-1:]))

            job_dict.append(generate_raw_table(test_dict))
            self.files_dict = job_dict

        self.version = " userspace: %s\n host kernel: %s\n guest kernel: %s" % (
            self.kvmver, self.hostkernel, self.guestkernel)
        nrepeat = len(self.files_dict)
        if nrepeat < 2:
            print "`nrepeat' should be larger than 1!"
            sys.exit(1)

        self.desc = """<hr>Machine Info:
o CPUs(%s * %s), Cores(%s), Threads(%s), Sockets(%s),
o NumaNodes(%s), Memory(%.1fG), NICs(%s)
o Disks(%s | %s)

Please check sysinfo directory in autotest result to get more details.
(eg: http://autotest-server.com/results/5057-autotest/host1/sysinfo/)
<hr>""" % (cpunum, cpumodel, corenum, threadnum, socketnum, numanodenum, memnum, nicnum, fdiskinfo, disknum)

        self.desc += """ - Every Avg line represents the average value based on *%d* repetitions of the same test,
   and the following SD line represents the Standard Deviation between the *%d* repetitions.
 - The Standard deviation is displayed as a percentage of the average.
 - The significance of the differences between the two averages is calculated using unpaired T-test that
   takes into account the SD of the averages.
 - The paired t-test is computed for the averages of same category.

""" % (nrepeat, nrepeat)

    def getAvg(self, avg_update=None):
        return self._process_files(self.files_dict, self._get_list_avg,
                                   avg_update=avg_update)

    def getAvgPercent(self, avgs_dict):
        return self._process_files(avgs_dict, self._get_augment_rate)

    def getSD(self):
        return self._process_files(self.files_dict, self._get_list_sd)

    def getSDRate(self, sds_dict):
        return self._process_files(sds_dict, self._get_rate)

    def getTtestPvalue(self, fs_dict1, fs_dict2, paired=None, ratio=None):
        """
        scipy lib is used to compute p-value of Ttest
        scipy: http://www.scipy.org/
        t-test: http://en.wikipedia.org/wiki/Student's_t-test
        """
        try:
            from scipy import stats
            import numpy as np
        except ImportError:
            print "No python scipy/numpy library installed!"
            return None

        ret = []
        s1 = self._process_files(fs_dict1, self._get_list_self, merge=False)
        s2 = self._process_files(fs_dict2, self._get_list_self, merge=False)
        # s*[line][col] contians items (line*col) of all sample files

        for line in range(len(s1)):
            tmp = []
            if type(s1[line]) != list:
                tmp = s1[line]
            else:
                if len(s1[line][0]) < 2:
                    continue
                for col in range(len(s1[line])):
                    avg1 = self._get_list_avg(s1[line][col])
                    avg2 = self._get_list_avg(s2[line][col])
                    sample1 = np.array(s1[line][col])
                    sample2 = np.array(s2[line][col])
                    warnings.simplefilter("ignore", RuntimeWarning)
                    if (paired):
                        if (ratio):
                            (_, p) = stats.ttest_rel(np.log(sample1), np.log(sample2))
                        else:
                            (_, p) = stats.ttest_rel(sample1, sample2)
                    else:
                        (_, p) = stats.ttest_ind(sample1, sample2)
                    flag = "+"
                    if float(avg1) > float(avg2):
                        flag = "-"
                    tmp.append(flag + "%f" % (1 - p))
                tmp = "|".join(tmp)
            ret.append(tmp)
        return ret

    def _get_rate(self, data):
        """ num2 / num1 * 100 """
        result = "0.0"
        if len(data) == 2 and float(data[0]) != 0:
            result = float(data[1]) / float(data[0]) * 100
            if result > 100:
                result = "%.2f%%" % result
            else:
                result = "%.4f%%" % result
        return result

    def _get_augment_rate(self, data):
        """ (num2 - num1) / num1 * 100 """
        result = "+0.0"
        if len(data) == 2 and float(data[0]) != 0:
            result = (float(data[1]) - float(data[0])) / float(data[0]) * 100
            if result > 100:
                result = "%+.2f%%" % result
            else:
                result = "%+.4f%%" % result
        return result

    def _get_list_sd(self, data):
        """
        sumX = x1 + x2 + ... + xn
        avgX = sumX / n
        sumSquareX = x1^2 + ... + xn^2
        SD = sqrt([sumSquareX - (n * (avgX ^ 2))] / (n - 1))
        """
        o_sum = sqsum = 0.0
        n = len(data)
        for i in data:
            o_sum += float(i)
            sqsum += float(i) ** 2
        avg = o_sum / n
        if avg == 0 or n == 1 or sqsum - (n * avg ** 2) <= 0:
            return "0.0"
        return "%f" % (((sqsum - (n * avg ** 2)) / (n - 1)) ** 0.5)

    def _get_list_avg(self, data):
        """ Compute the average of list entries """
        o_sum = 0.0
        for i in data:
            o_sum += float(i)
        return "%f" % (o_sum / len(data))

    def _get_list_self(self, data):
        """ Use this to convert sample dicts """
        return data

    def _process_lines(self, files_dict, row, func, avg_update, merge):
        """ Use unified function to process same lines of different samples """
        lines = []
        ret = []

        for i in range(len(files_dict)):
            lines.append(files_dict[i][row].split("|"))

        for col in range(len(lines[0])):
            data_list = []
            for i in range(len(lines)):
                tmp = lines[i][col].strip()
                if is_int(tmp):
                    data_list.append(int(tmp))
                else:
                    data_list.append(float(tmp))
            ret.append(func(data_list))

        if avg_update:
            for i in avg_update.split('|'):
                l = i.split(',')
                ret[int(l[0])] = "%f" % (float(ret[int(l[1])]) /
                                         float(ret[int(l[2])]))
        if merge:
            return "|".join(ret)
        return ret

    def _process_files(self, files_dict, func, avg_update=None, merge=True):
        """
        Process dicts of sample files with assigned function,
        func has one list augment.
        """
        ret_lines = []
        for i in range(len(files_dict[0])):
            if re.findall("[a-zA-Z]", files_dict[0][i]):
                ret_lines.append(files_dict[0][i].strip())
            else:
                line = self._process_lines(files_dict, i, func, avg_update,
                                           merge)
                ret_lines.append(line)
        return ret_lines


def display(lists, rates, allpvalues, f, ignore_col, o_sum="Augment Rate",
            prefix0=None, prefix1=None, prefix2=None, prefix3=None):
    """
    Display lists data to standard format

    param lists: row data lists
    param rates: augment rates lists
    param f: result output filepath
    param ignore_col: do not display some columns
    param o_sum: compare result summary
    param prefix0: output prefix in head lines
    param prefix1: output prefix in Avg/SD lines
    param prefix2: output prefix in Diff Avg/P-value lines
    param prefix3: output prefix in total Sign line
    """
    def str_ignore(out, split=False):
        out = out.split("|")
        for i in range(ignore_col):
            out[i] = " "
        if split:
            return "|".join(out[ignore_col:])
        return "|".join(out)

    def tee_line(content, filepath, n=None):
        fd = open(filepath, "a")
        print content
        out = ""
        out += "<TR ALIGN=CENTER>"
        content = content.split("|")
        for i in range(len(content)):
            if not is_int(content[i]) and is_float(content[i]):
                if "+" in content[i] or "-" in content[i]:
                    if float(content[i]) > 100:
                        content[i] = "%+.2f" % float(content[i])
                    else:
                        content[i] = "%+.4f" % float(content[i])
                elif float(content[i]) > 100:
                    content[i] = "%.2f" % float(content[i])
                else:
                    content[i] = "%.4f" % float(content[i])
            if n and i >= 2 and i < ignore_col + 2:
                out += "<TD ROWSPAN=%d WIDTH=1%% >%.0f</TD>" % (n, float(content[i]))
            else:
                out += "<TD WIDTH=1%% >%s</TD>" % content[i]
        out += "</TR>"
        fd.write(out + "\n")
        fd.close()

    for l in range(len(lists[0])):
        if not re.findall("[a-zA-Z]", lists[0][l]):
            break
    tee("<TABLE BORDER=1 CELLSPACING=1 CELLPADDING=1 width=10%><TBODY>",
        f)
    tee("<h3>== %s " % o_sum + "==</h3>", f)
    category = 0
    for i in range(len(lists[0])):
        for n in range(len(lists)):
            is_diff = False
            for j in range(len(lists)):
                if lists[0][i] != lists[j][i]:
                    is_diff = True
                if len(lists) == 1 and not re.findall("[a-zA-Z]", lists[j][i]):
                    is_diff = True

            pfix = prefix1[0]
            if len(prefix1) != 1:
                pfix = prefix1[n]
            if is_diff:
                if n == 0:
                    tee_line(pfix + lists[n][i], f, n=len(lists) + len(rates))
                else:
                    tee_line(pfix + str_ignore(lists[n][i], True), f)
            if not is_diff and n == 0:
                if '|' in lists[n][i]:
                    tee_line(prefix0 + lists[n][i], f)
                elif "Category:" in lists[n][i]:
                    if category != 0 and prefix3:
                        if len(allpvalues[category - 1]) > 0:
                            tee_line(prefix3 + str_ignore(
                                     allpvalues[category - 1][0]), f)
                        tee("</TBODY></TABLE>", f)
                        tee("<br>", f)
                        tee("<TABLE BORDER=1 CELLSPACING=1 CELLPADDING=1 "
                            "width=10%><TBODY>", f)
                    category += 1
                    tee("<TH colspan=3 >%s</TH>" % lists[n][i], f)
                else:
                    tee("<TH colspan=3 >%s</TH>" % lists[n][i], f)
        for n in range(len(rates)):
            if lists[0][i] != rates[n][i] and (not re.findall("[a-zA-Z]",
                                                              rates[n][i]) or "nan" in rates[n][i]):
                tee_line(prefix2[n] + str_ignore(rates[n][i], True), f)
    if prefix3 and len(allpvalues[-1]) > 0:
        tee_line(prefix3 + str_ignore(allpvalues[category - 1][0]), f)
    tee("</TBODY></TABLE>", f)


def analyze(test, sample_type, arg1, arg2, configfile):
    """ Compute averages/p-vales of two samples, print results nicely """
    config = ConfigParser.ConfigParser()
    config.read(configfile)
    ignore_col = int(config.get(test, "ignore_col"))
    avg_update = config.get(test, "avg_update")
    desc = config.get(test, "desc")

    def get_list(directory):
        result_file_pattern = config.get(test, "result_file_pattern")
        cmd = 'find %s|grep "%s.*/%s"' % (directory, test, result_file_pattern)
        print cmd
        return commands.getoutput(cmd)

    if sample_type == 'filepath':
        arg1 = get_list(arg1)
        arg2 = get_list(arg2)

    commands.getoutput("rm -f %s.*html" % test)
    s1 = Sample(sample_type, arg1)
    avg1 = s1.getAvg(avg_update=avg_update)
    sd1 = s1.getSD()

    s2 = Sample(sample_type, arg2)
    avg2 = s2.getAvg(avg_update=avg_update)
    sd2 = s2.getSD()

    sd1 = s1.getSDRate([avg1, sd1])
    sd2 = s1.getSDRate([avg2, sd2])
    avgs_rate = s1.getAvgPercent([avg1, avg2])

    navg1 = []
    navg2 = []
    allpvalues = []
    tmp1 = []
    tmp2 = []
    for i in range(len(avg1)):
        if not re.findall("[a-zA-Z]", avg1[i]):
            tmp1.append([avg1[i]])
            tmp2.append([avg2[i]])
        elif 'Category' in avg1[i] and i != 0:
            navg1.append(tmp1)
            navg2.append(tmp2)
            tmp1 = []
            tmp2 = []
    navg1.append(tmp1)
    navg2.append(tmp2)

    for i in range(len(navg1)):
        allpvalues.append(s1.getTtestPvalue(navg1[i], navg2[i], True, True))

    pvalues = s1.getTtestPvalue(s1.files_dict, s2.files_dict, False)
    rlist = [avgs_rate]
    if pvalues:
        # p-value list isn't null
        rlist.append(pvalues)

    desc = desc % s1.len

    tee("<pre>####1. Description of setup#1\n%s\n test data:  %s</pre>"
        % (s1.version, s1.testdata), "%s.html" % test)
    tee("<pre>####2. Description of setup#2\n%s\n test data:  %s</pre>"
        % (s2.version, s2.testdata), "%s.html" % test)
    tee("<pre>" + '\n'.join(desc.split('\\n')) + "</pre>", test + ".html")
    tee("<pre>" + s1.desc + "</pre>", test + ".html")

    display([avg1, sd1, avg2, sd2], rlist, allpvalues, test + ".html",
            ignore_col, o_sum="Regression Testing: %s" % test, prefix0="#|Tile|",
            prefix1=["1|Avg|", " |%SD|", "2|Avg|", " |%SD|"],
            prefix2=["-|%Diff between Avg|", "-|Significance|"],
            prefix3="-|Total Significance|")

    display(s1.files_dict, [avg1], [], test + ".avg.html", ignore_col,
            o_sum="Raw data of sample 1", prefix0="#|Tile|",
            prefix1=[" |    |"],
            prefix2=["-|Avg |"], prefix3="")

    display(s2.files_dict, [avg2], [], test + ".avg.html", ignore_col,
            o_sum="Raw data of sample 2", prefix0="#|Tile|",
            prefix1=[" |    |"],
            prefix2=["-|Avg |"], prefix3="")


def is_int(n):
    try:
        int(n)
        return True
    except ValueError:
        return False


def is_float(n):
    try:
        float(n)
        return True
    except ValueError:
        return False


def tee(content, filepath):
    """ Write content to standard output and filepath """
    fd = open(filepath, "a")
    fd.write(content + "\n")
    fd.close()
    print content


if __name__ == "__main__":
    if len(sys.argv) != 5:
        this = os.path.basename(sys.argv[0])
        print 'Usage: %s <testname> filepath <dir1> <dir2>' % this
        print '    or %s <testname> db <jobid1> <jobid2>' % this
        sys.exit(1)
    analyze(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], 'perf.conf')