1 /*
2  * Copyright (C) 2016 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.perftests.utils;
18 
19 import android.app.Activity;
20 import android.app.Instrumentation;
21 import android.os.Bundle;
22 import android.os.Debug;
23 import android.support.test.InstrumentationRegistry;
24 import android.util.Log;
25 
26 import java.io.File;
27 import java.util.ArrayList;
28 import java.util.concurrent.TimeUnit;
29 
30 /**
31  * Provides a benchmark framework.
32  *
33  * Example usage:
34  * // Executes the code while keepRunning returning true.
35  *
36  * public void sampleMethod() {
37  *     BenchmarkState state = new BenchmarkState();
38  *
39  *     int[] src = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
40  *     while (state.keepRunning()) {
41  *         int[] dest = new int[src.length];
42  *         System.arraycopy(src, 0, dest, 0, src.length);
43  *     }
44  *     System.out.println(state.summaryLine());
45  * }
46  */
47 public final class BenchmarkState {
48 
49     private static final String TAG = "BenchmarkState";
50     private static final boolean ENABLE_PROFILING = false;
51 
52     private static final int NOT_STARTED = 0;  // The benchmark has not started yet.
53     private static final int WARMUP = 1; // The benchmark is warming up.
54     private static final int RUNNING = 2;  // The benchmark is running.
55     private static final int FINISHED = 3;  // The benchmark has stopped.
56 
57     private int mState = NOT_STARTED;  // Current benchmark state.
58 
59     private static final long WARMUP_DURATION_NS = ms2ns(250); // warm-up for at least 250ms
60     private static final int WARMUP_MIN_ITERATIONS = 16; // minimum iterations to warm-up for
61 
62     // TODO: Tune these values.
63     private static final long TARGET_TEST_DURATION_NS = ms2ns(500); // target testing for 500 ms
64     private static final int MAX_TEST_ITERATIONS = 1000000;
65     private static final int MIN_TEST_ITERATIONS = 10;
66     private static final int REPEAT_COUNT = 5;
67 
68     private long mStartTimeNs = 0;  // Previously captured System.nanoTime().
69     private boolean mPaused;
70     private long mPausedTimeNs = 0; // The System.nanoTime() when the pauseTiming() is called.
71     private long mPausedDurationNs = 0;  // The duration of paused state in nano sec.
72 
73     private int mIteration = 0;
74     private int mMaxIterations = 0;
75 
76     private int mRepeatCount = 0;
77 
78     // Statistics. These values will be filled when the benchmark has finished.
79     // The computation needs double precision, but long int is fine for final reporting.
80     private Stats mStats;
81 
82     // Individual duration in nano seconds.
83     private ArrayList<Long> mResults = new ArrayList<>();
84 
ms2ns(long ms)85     private static final long ms2ns(long ms) {
86         return TimeUnit.MILLISECONDS.toNanos(ms);
87     }
88 
89     // Stops the benchmark timer.
90     // This method can be called only when the timer is running.
pauseTiming()91     public void pauseTiming() {
92         if (mPaused) {
93             throw new IllegalStateException(
94                     "Unable to pause the benchmark. The benchmark has already paused.");
95         }
96         mPausedTimeNs = System.nanoTime();
97         mPaused = true;
98     }
99 
100     // Starts the benchmark timer.
101     // This method can be called only when the timer is stopped.
resumeTiming()102     public void resumeTiming() {
103         if (!mPaused) {
104             throw new IllegalStateException(
105                     "Unable to resume the benchmark. The benchmark is already running.");
106         }
107         mPausedDurationNs += System.nanoTime() - mPausedTimeNs;
108         mPausedTimeNs = 0;
109         mPaused = false;
110     }
111 
beginWarmup()112     private void beginWarmup() {
113         mStartTimeNs = System.nanoTime();
114         mIteration = 0;
115         mState = WARMUP;
116     }
117 
beginBenchmark(long warmupDuration, int iterations)118     private void beginBenchmark(long warmupDuration, int iterations) {
119         if (ENABLE_PROFILING) {
120             File f = new File(InstrumentationRegistry.getContext().getDataDir(), "benchprof");
121             Log.d(TAG, "Tracing to: " + f.getAbsolutePath());
122             Debug.startMethodTracingSampling(f.getAbsolutePath(), 16 * 1024 * 1024, 100);
123         }
124         mMaxIterations = (int) (TARGET_TEST_DURATION_NS / (warmupDuration / iterations));
125         mMaxIterations = Math.min(MAX_TEST_ITERATIONS,
126                 Math.max(mMaxIterations, MIN_TEST_ITERATIONS));
127         mPausedDurationNs = 0;
128         mIteration = 0;
129         mRepeatCount = 0;
130         mState = RUNNING;
131         mStartTimeNs = System.nanoTime();
132     }
133 
startNextTestRun()134     private boolean startNextTestRun() {
135         final long currentTime = System.nanoTime();
136         mResults.add((currentTime - mStartTimeNs - mPausedDurationNs) / mMaxIterations);
137         mRepeatCount++;
138         if (mRepeatCount >= REPEAT_COUNT) {
139             if (ENABLE_PROFILING) {
140                 Debug.stopMethodTracing();
141             }
142             mStats = new Stats(mResults);
143             mState = FINISHED;
144             return false;
145         }
146         mPausedDurationNs = 0;
147         mIteration = 0;
148         mStartTimeNs = System.nanoTime();
149         return true;
150     }
151 
152     /**
153      * Judges whether the benchmark needs more samples.
154      *
155      * For the usage, see class comment.
156      */
keepRunning()157     public boolean keepRunning() {
158         switch (mState) {
159             case NOT_STARTED:
160                 beginWarmup();
161                 return true;
162             case WARMUP:
163                 mIteration++;
164                 // Only check nanoTime on every iteration in WARMUP since we
165                 // don't yet have a target iteration count.
166                 final long duration = System.nanoTime() - mStartTimeNs;
167                 if (mIteration >= WARMUP_MIN_ITERATIONS && duration >= WARMUP_DURATION_NS) {
168                     beginBenchmark(duration, mIteration);
169                 }
170                 return true;
171             case RUNNING:
172                 mIteration++;
173                 if (mIteration >= mMaxIterations) {
174                     return startNextTestRun();
175                 }
176                 if (mPaused) {
177                     throw new IllegalStateException(
178                             "Benchmark step finished with paused state. " +
179                             "Resume the benchmark before finishing each step.");
180                 }
181                 return true;
182             case FINISHED:
183                 throw new IllegalStateException("The benchmark has finished.");
184             default:
185                 throw new IllegalStateException("The benchmark is in unknown state.");
186         }
187     }
188 
mean()189     private long mean() {
190         if (mState != FINISHED) {
191             throw new IllegalStateException("The benchmark hasn't finished");
192         }
193         return (long) mStats.getMean();
194     }
195 
median()196     private long median() {
197         if (mState != FINISHED) {
198             throw new IllegalStateException("The benchmark hasn't finished");
199         }
200         return mStats.getMedian();
201     }
202 
min()203     private long min() {
204         if (mState != FINISHED) {
205             throw new IllegalStateException("The benchmark hasn't finished");
206         }
207         return mStats.getMin();
208     }
209 
standardDeviation()210     private long standardDeviation() {
211         if (mState != FINISHED) {
212             throw new IllegalStateException("The benchmark hasn't finished");
213         }
214         return (long) mStats.getStandardDeviation();
215     }
216 
summaryLine()217     private String summaryLine() {
218         StringBuilder sb = new StringBuilder();
219         sb.append("Summary: ");
220         sb.append("median=").append(median()).append("ns, ");
221         sb.append("mean=").append(mean()).append("ns, ");
222         sb.append("min=").append(min()).append("ns, ");
223         sb.append("sigma=").append(standardDeviation()).append(", ");
224         sb.append("iteration=").append(mResults.size()).append(", ");
225         // print out the first few iterations' number for double checking.
226         int sampleNumber = Math.min(mResults.size(), 16);
227         for (int i = 0; i < sampleNumber; i++) {
228             sb.append("No ").append(i).append(" result is ").append(mResults.get(i)).append(", ");
229         }
230         return sb.toString();
231     }
232 
sendFullStatusReport(Instrumentation instrumentation, String key)233     public void sendFullStatusReport(Instrumentation instrumentation, String key) {
234         Log.i(TAG, key + summaryLine());
235         Bundle status = new Bundle();
236         status.putLong(key + "_median", median());
237         status.putLong(key + "_mean", mean());
238         status.putLong(key + "_min", min());
239         status.putLong(key + "_standardDeviation", standardDeviation());
240         instrumentation.sendStatus(Activity.RESULT_OK, status);
241     }
242 }
243