1#!/usr/bin/python
2
3import optparse
4import sys
5import sqlite3
6import scipy.stats
7import numpy
8from math import log10, floor
9import matplotlib
10
11matplotlib.use("Agg")
12
13import matplotlib.pyplot as plt
14import pylab
15
16import adbutil
17from devices import DEVICES
18
19DB_PATH="/data/data/com.android.benchmark/databases/BenchmarkResults"
20OUT_PATH = "db/"
21
22QUERY_BAD_FRAME = ("select run_id, name, iteration, total_duration from ui_results "
23                   "where total_duration >= 16 order by run_id, name, iteration")
24QUERY_PERCENT_JANK = ("select run_id, name, iteration, sum(jank_frame) as jank_count, count (*) as total "
25                      "from ui_results group by run_id, name, iteration")
26
27SKIP_TESTS = [
28    # "BMUpload",
29    # "Low-hitrate text render",
30    # "High-hitrate text render",
31    # "Edit Text Input",
32    # "List View Fling"
33]
34
35INCLUDE_TESTS = [
36    #"BMUpload"
37    #"Shadow Grid Fling"
38    #"Image List View Fling"
39    #"Edit Text Input"
40]
41
42class IterationResult:
43    def __init__(self):
44        self.durations = []
45        self.jank_count = 0
46        self.total_count = 0
47
48
49def get_scoremap(dbpath):
50    db = sqlite3.connect(dbpath)
51    rows = db.execute(QUERY_BAD_FRAME)
52
53    scoremap = {}
54    for row in rows:
55        run_id = row[0]
56        name = row[1]
57        iteration = row[2]
58        total_duration = row[3]
59
60        if not run_id in scoremap:
61            scoremap[run_id] = {}
62
63        if not name in scoremap[run_id]:
64            scoremap[run_id][name] = {}
65
66        if not iteration in scoremap[run_id][name]:
67            scoremap[run_id][name][iteration] = IterationResult()
68
69        scoremap[run_id][name][iteration].durations.append(float(total_duration))
70
71    for row in db.execute(QUERY_PERCENT_JANK):
72        run_id = row[0]
73        name = row[1]
74        iteration = row[2]
75        jank_count = row[3]
76        total_count = row[4]
77
78        if run_id in scoremap.keys() and name in scoremap[run_id].keys() and iteration in scoremap[run_id][name].keys():
79            scoremap[run_id][name][iteration].jank_count = long(jank_count)
80            scoremap[run_id][name][iteration].total_count = long(total_count)
81
82    db.close()
83    return scoremap
84
85def round_to_2(val):
86    return val
87    if val == 0:
88        return val
89    return round(val , -int(floor(log10(abs(val)))) + 1)
90
91def score_device(name, serial, pull = False, verbose = False):
92    dbpath = OUT_PATH + name + ".db"
93
94    if pull:
95        adbutil.root(serial)
96        adbutil.pull(serial, DB_PATH, dbpath)
97
98    scoremap = None
99    try:
100        scoremap = get_scoremap(dbpath)
101    except sqlite3.DatabaseError:
102        print "Database corrupt, fetching..."
103        adbutil.root(serial)
104        adbutil.pull(serial, DB_PATH, dbpath)
105        scoremap = get_scoremap(dbpath)
106
107    per_test_score = {}
108    per_test_sample_count = {}
109    global_overall = {}
110
111    for run_id in iter(scoremap):
112        overall = []
113        if len(scoremap[run_id]) < 1:
114            if verbose:
115                print "Skipping short run %s" % run_id
116            continue
117        print "Run: %s" % run_id
118        for test in iter(scoremap[run_id]):
119            if test in SKIP_TESTS:
120                continue
121            if INCLUDE_TESTS and test not in INCLUDE_TESTS:
122                continue
123            if verbose:
124                print "\t%s" % test
125            scores = []
126            means = []
127            stddevs = []
128            pjs = []
129            sample_count = 0
130            hit_min_count = 0
131            # try pooling together all iterations
132            for iteration in iter(scoremap[run_id][test]):
133                res = scoremap[run_id][test][iteration]
134                stddev = round_to_2(numpy.std(res.durations))
135                mean = round_to_2(numpy.mean(res.durations))
136                sample_count += len(res.durations)
137                pj = round_to_2(100 * res.jank_count / float(res.total_count))
138                score = stddev * mean * pj
139                score = 100 * len(res.durations) / float(res.total_count)
140                if score == 0:
141                    score = 1
142                scores.append(score)
143                means.append(mean)
144                stddevs.append(stddev)
145                pjs.append(pj)
146                if verbose:
147                    print "\t%s: Score = %f x %f x %f = %f (%d samples)" % (iteration, stddev, mean, pj, score, len(res.durations))
148
149            if verbose:
150                print "\tHit min: %d" % hit_min_count
151                print "\tMean Variation: %0.2f%%" % (100 * scipy.stats.variation(means))
152                print "\tStdDev Variation: %0.2f%%" % (100 * scipy.stats.variation(stddevs))
153                print "\tPJ Variation: %0.2f%%" % (100 * scipy.stats.variation(pjs))
154
155            geo_run = numpy.mean(scores)
156            if test not in per_test_score:
157                per_test_score[test] = []
158
159            if test not in per_test_sample_count:
160                per_test_sample_count[test] = []
161
162            sample_count /= len(scoremap[run_id][test])
163
164            per_test_score[test].append(geo_run)
165            per_test_sample_count[test].append(int(sample_count))
166            overall.append(geo_run)
167
168            if not verbose:
169                print "\t%s:\t%0.2f (%0.2f avg. sample count)" % (test, geo_run, sample_count)
170            else:
171                print "\tOverall:\t%0.2f (%0.2f avg. sample count)" % (geo_run, sample_count)
172                print ""
173
174        global_overall[run_id] = scipy.stats.gmean(overall)
175        print "Run Overall: %f" % global_overall[run_id]
176        print ""
177
178    print ""
179    print "Variability (CV) - %s:" % name
180
181    worst_offender_test = None
182    worst_offender_variation = 0
183    for test in per_test_score:
184        variation = 100 * scipy.stats.variation(per_test_score[test])
185        if worst_offender_variation < variation:
186            worst_offender_test = test
187            worst_offender_variation = variation
188        print "\t%s:\t%0.2f%% (%0.2f avg sample count)" % (test, variation, numpy.mean(per_test_sample_count[test]))
189
190    print "\tOverall: %0.2f%%" % (100 * scipy.stats.variation([x for x in global_overall.values()]))
191    print ""
192
193    return {
194            "overall": global_overall.values(),
195            "worst_offender_test": (name, worst_offender_test, worst_offender_variation)
196            }
197
198def parse_options(argv):
199    usage = 'Usage: %prog [options]'
200    desc = 'Example: %prog'
201    parser = optparse.OptionParser(usage=usage, description=desc)
202    parser.add_option("-p", dest='pull', action="store_true")
203    parser.add_option("-d", dest='device', action="store")
204    parser.add_option("-v", dest='verbose', action="store_true")
205    options, categories = parser.parse_args(argv[1:])
206    return options
207
208def main():
209    options = parse_options(sys.argv)
210    if options.device != None:
211        score_device(options.device, DEVICES[options.device], options.pull, options.verbose)
212    else:
213        device_scores = []
214        worst_offenders = []
215        for name, serial in DEVICES.iteritems():
216            print "======== %s =========" % name
217            result = score_device(name, serial, options.pull, options.verbose)
218            device_scores.append((name, result["overall"]))
219            worst_offenders.append(result["worst_offender_test"])
220
221
222        device_scores.sort(cmp=(lambda x, y: cmp(x[1], y[1])))
223        print "Ranking by max overall score:"
224        for name, score in device_scores:
225            plt.plot([0, 1, 2, 3, 4, 5], score, label=name)
226            print "\t%s: %s" % (name, score)
227
228        plt.ylabel("Jank %")
229        plt.xlabel("Iteration")
230        plt.title("Jank Percentage")
231        plt.legend()
232        pylab.savefig("holy.png", bbox_inches="tight")
233
234        print "Worst offender tests:"
235        for device, test, variation in worst_offenders:
236            print "\t%s: %s %.2f%%" % (device, test, variation)
237
238if __name__ == "__main__":
239    main()
240
241