1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.videoencodingquality.cts; 18 19 import android.cts.host.utils.DeviceJUnit4ClassRunnerWithParameters; 20 import android.cts.host.utils.DeviceJUnit4Parameterized; 21 import android.platform.test.annotations.AppModeFull; 22 23 import com.android.compatibility.common.util.CddTest; 24 import com.android.ddmlib.IDevice; 25 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; 26 import com.android.ddmlib.testrunner.TestResult.TestStatus; 27 import com.android.tradefed.config.Option; 28 import com.android.tradefed.config.OptionClass; 29 import com.android.tradefed.device.DeviceNotAvailableException; 30 import com.android.tradefed.device.ITestDevice; 31 import com.android.tradefed.log.LogUtil; 32 import com.android.tradefed.result.CollectingTestListener; 33 import com.android.tradefed.result.TestDescription; 34 import com.android.tradefed.result.TestResult; 35 import com.android.tradefed.result.TestRunResult; 36 import com.android.tradefed.testtype.IDeviceTest; 37 38 import org.json.JSONArray; 39 import org.json.JSONObject; 40 import org.junit.Assert; 41 import org.junit.Assume; 42 import org.junit.Test; 43 import org.junit.runner.RunWith; 44 import org.junit.runners.Parameterized; 45 import org.junit.runners.Parameterized.UseParametersRunnerFactory; 46 47 import java.io.BufferedReader; 48 import java.io.File; 49 import java.io.FileReader; 50 import java.io.FileWriter; 51 import java.io.IOException; 52 import java.io.InputStreamReader; 53 import java.nio.charset.StandardCharsets; 54 import java.nio.file.Files; 55 import java.nio.file.Paths; 56 import java.util.ArrayList; 57 import java.util.Arrays; 58 import java.util.List; 59 import java.util.Map; 60 import java.util.concurrent.TimeUnit; 61 import java.util.concurrent.locks.Condition; 62 import java.util.concurrent.locks.Lock; 63 import java.util.concurrent.locks.ReentrantLock; 64 65 import javax.annotation.Nullable; 66 67 /** 68 * This class constitutes host-part of video encoding quality test (go/pc14-veq). This test is 69 * aimed towards benchmarking encoders on the target device. 70 * <p> 71 * Video encoding quality test quantifies encoders on the test device by encoding a set of clips 72 * at various configurations. The encoded output is analysed for vmaf and compared against 73 * reference. This entire process is not carried on the device. The host side of the test 74 * prepares the test environment by installing a VideoEncodingApp on the device. It also pushes 75 * the test vectors and test configurations on to the device. The VideoEncodingApp transcodes the 76 * input clips basing on the configurations shared. The host side of the test then pulls output 77 * files from the device and analyses for vmaf. These values are compared against reference using 78 * Bjontegaard metric. 79 **/ 80 @AppModeFull(reason = "Instant apps cannot access the SD card") 81 @RunWith(DeviceJUnit4Parameterized.class) 82 @UseParametersRunnerFactory(DeviceJUnit4ClassRunnerWithParameters.RunnerFactory.class) 83 @OptionClass(alias = "pc-veq-test") 84 public class CtsVideoEncodingQualityHostTest implements IDeviceTest { 85 private static final String RES_URL = 86 "https://storage.googleapis.com/android_media/cts/hostsidetests/pc14_veq/veqtests-1_2.tar.gz"; 87 88 // variables related to host-side of the test 89 private static final int MEDIA_PERFORMANCE_CLASS_14 = 34; 90 private static final int MINIMUM_VALID_SDK = 31; 91 // test is not valid before sdk 31, aka Android 12, aka Android S 92 93 private static final Lock sLock = new ReentrantLock(); 94 private static final Condition sCondition = sLock.newCondition(); 95 private static boolean sIsTestSetUpDone = false; 96 // install apk, push necessary resources to device to run the test. lock/condition 97 // pair is to keep setupTestEnv() thread safe 98 private static File sHostWorkDir; 99 100 // Variables related to device-side of the test. These need to kept in sync with definitions of 101 // VideoEncodingApp.apk 102 private static final String DEVICE_SIDE_TEST_PACKAGE = "android.videoencoding.app"; 103 private static final String DEVICE_SIDE_TEST_CLASS = 104 "android.videoencoding.app.VideoTranscoderTest"; 105 private static final String RUNNER = "androidx.test.runner.AndroidJUnitRunner"; 106 private static final String TEST_CONFIG_INST_ARGS_KEY = "conf-json"; 107 private static final long DEFAULT_SHELL_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(5); 108 private static final String TEST_TIMEOUT_INST_ARGS_KEY = "timeout_msec"; 109 private static final long DEFAULT_TEST_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(3); 110 111 // local variables related to host-side of the test 112 private final String mJsonName; 113 private ITestDevice mDevice; 114 115 @Option(name = "force-to-run", description = "Force to run the test even if the device is not" 116 + " a right performance class device.") 117 private boolean mForceToRun = false; 118 119 @Option(name = "skip-avc", description = "Skip avc encoder testing") 120 private boolean mSkipAvc = false; 121 122 @Option(name = "skip-hevc", description = "Skip hevc encoder testing") 123 private boolean mSkipHevc = false; 124 125 @Option(name = "skip-p", description = "Skip P only testing") 126 private boolean mSkipP = false; 127 128 @Option(name = "skip-b", description = "Skip B frame testing") 129 private boolean mSkipB = false; 130 131 @Option(name = "reset", description = "Start with a fresh directory.") 132 private boolean mReset = false; 133 134 @Option(name = "quick-check", description = "Run a quick check.") 135 private boolean mQuickCheck = false; 136 CtsVideoEncodingQualityHostTest(String jsonName, @SuppressWarnings("unused") String testLabel)137 public CtsVideoEncodingQualityHostTest(String jsonName, 138 @SuppressWarnings("unused") String testLabel) { 139 mJsonName = jsonName; 140 } 141 142 private static final List<Object[]> AVC_VBR_B0_PARAMS = Arrays.asList(new Object[][]{ 143 {"AVICON-MOBILE-Beach-SO04-CRW02-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json", 144 "Beach_SO04_CRW02_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0"}, 145 {"AVICON-MOBILE-BirthdayHalfway-SI17-CRUW03-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0" 146 + ".json", 147 "BirthdayHalfway_SI17_CRUW03_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0"}, 148 {"AVICON-MOBILE-SelfieTeenKitchenSocialMedia-SS01-CF01-P-420-8bit-SDR-1080p" 149 + "-30fps_hw_avc_vbr_b0.json", 150 "SelfieTeenKitchenSocialMedia_SS01_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_" 151 + "vbr_b0"}, 152 {"AVICON-MOBILE-Waterfall-SO05-CRW01-P-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json", 153 "Waterfall_SO05_CRW01_P_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0"}, 154 {"AVICON-MOBILE-SelfieFamily-SF14-CF01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json" 155 , "SelfieFamily_SF14_CF01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0"}, 156 {"AVICON-MOBILE-River-SO03-CRW01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json", 157 "River_SO03_CRW01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0"}, 158 {"AVICON-MOBILE-SelfieGroupGarden-SF15-CF01-P-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0" 159 + ".json", 160 "SelfieGroupGarden_SF15_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0"}, 161 {"AVICON-MOBILE-ConcertNear-SI10-CRW01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json" 162 , "ConcertNear_SI10_CRW01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0"}, 163 {"AVICON-MOBILE-SelfieCoupleCitySocialMedia-SS02-CF01-P-420-8bit-SDR" 164 + "-1080p-30fps_hw_avc_vbr_b0.json", 165 "SelfieCoupleCitySocialMedia_SS02_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_" 166 + "vbr_b0"}}); 167 168 private static final List<Object[]> AVC_VBR_B3_PARAMS = Arrays.asList(new Object[][]{ 169 {"AVICON-MOBILE-Beach-SO04-CRW02-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b3.json", 170 "Beach_SO04_CRW02_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b3"}, 171 {"AVICON-MOBILE-BirthdayHalfway-SI17-CRUW03-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b3" 172 + ".json", 173 "BirthdayHalfway_SI17_CRUW03_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b3"}, 174 {"AVICON-MOBILE-SelfieTeenKitchenSocialMedia-SS01-CF01-P-420-8bit-SDR-1080p" 175 + "-30fps_hw_avc_vbr_b3.json", 176 "SelfieTeenKitchenSocialMedia_SS01_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_" 177 + "vbr_b3"}, 178 {"AVICON-MOBILE-Waterfall-SO05-CRW01-P-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b3.json", 179 "Waterfall_SO05_CRW01_P_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b3"}, 180 {"AVICON-MOBILE-SelfieFamily-SF14-CF01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b3.json" 181 , "SelfieFamily_SF14_CF01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b3"}, 182 {"AVICON-MOBILE-River-SO03-CRW01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b3.json", 183 "River_SO03_CRW01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b3"}, 184 {"AVICON-MOBILE-SelfieGroupGarden-SF15-CF01-P-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b3" 185 + ".json", 186 "SelfieGroupGarden_SF15_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b3"}, 187 {"AVICON-MOBILE-ConcertNear-SI10-CRW01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b3.json" 188 , "ConcertNear_SI10_CRW01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b3"}, 189 {"AVICON-MOBILE-SelfieCoupleCitySocialMedia-SS02-CF01-P-420-8bit-SDR" 190 + "-1080p-30fps_hw_avc_vbr_b3.json", 191 "SelfieCoupleCitySocialMedia_SS02_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_" 192 + "vbr_b3"}}); 193 194 private static final List<Object[]> HEVC_VBR_B0_PARAMS = Arrays.asList(new Object[][]{ 195 {"AVICON-MOBILE-Beach-SO04-CRW02-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json", 196 "Beach_SO04_CRW02_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0"}, 197 {"AVICON-MOBILE-BirthdayHalfway-SI17-CRUW03-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0" 198 + ".json", 199 "BirthdayHalfway_SI17_CRUW03_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0"}, 200 {"AVICON-MOBILE-SelfieTeenKitchenSocialMedia-SS01-CF01-P-420-8bit-SDR-1080p" 201 + "-30fps_hw_hevc_vbr_b0.json", 202 "SelfieTeenKitchenSocialMedia_SS01_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_" 203 + "vbr_b0"}, 204 {"AVICON-MOBILE-Waterfall-SO05-CRW01-P-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json", 205 "Waterfall_SO05_CRW01_P_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0"}, 206 {"AVICON-MOBILE-SelfieFamily-SF14-CF01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json" 207 , "SelfieFamily_SF14_CF01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0"}, 208 {"AVICON-MOBILE-River-SO03-CRW01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json", 209 "River_SO03_CRW01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0"}, 210 {"AVICON-MOBILE-SelfieGroupGarden-SF15-CF01-P-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0" 211 + ".json", 212 "SelfieGroupGarden_SF15_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0"}, 213 {"AVICON-MOBILE-ConcertNear-SI10-CRW01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json" 214 , "ConcertNear_SI10_CRW01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0"}, 215 {"AVICON-MOBILE-SelfieCoupleCitySocialMedia-SS02-CF01-P-420-8bit-SDR" 216 + "-1080p-30fps_hw_hevc_vbr_b0.json", 217 "SelfieCoupleCitySocialMedia_SS02_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_" 218 + "vbr_b0"}}); 219 220 private static final List<Object[]> HEVC_VBR_B3_PARAMS = Arrays.asList(new Object[][]{ 221 {"AVICON-MOBILE-Beach-SO04-CRW02-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b3.json", 222 "Beach_SO04_CRW02_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b3"}, 223 {"AVICON-MOBILE-BirthdayHalfway-SI17-CRUW03-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b3" 224 + ".json", 225 "BirthdayHalfway_SI17_CRUW03_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b3"}, 226 {"AVICON-MOBILE-SelfieTeenKitchenSocialMedia-SS01-CF01-P-420-8bit-SDR-1080p" 227 + "-30fps_hw_hevc_vbr_b3.json", 228 "SelfieTeenKitchenSocialMedia_SS01_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_" 229 + "vbr_b3"}, 230 {"AVICON-MOBILE-Waterfall-SO05-CRW01-P-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b3.json", 231 "Waterfall_SO05_CRW01_P_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b3"}, 232 {"AVICON-MOBILE-SelfieFamily-SF14-CF01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b3.json" 233 , "SelfieFamily_SF14_CF01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b3"}, 234 {"AVICON-MOBILE-River-SO03-CRW01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b3.json", 235 "River_SO03_CRW01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b3"}, 236 // Abnormal curve, not monotonically increasing. 237 /*{"AVICON-MOBILE-SelfieGroupGarden-SF15-CF01-P-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b3" 238 + ".json", 239 "SelfieGroupGarden_SF15_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b3"},*/ 240 {"AVICON-MOBILE-ConcertNear-SI10-CRW01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b3.json" 241 , "ConcertNear_SI10_CRW01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b3"}, 242 {"AVICON-MOBILE-SelfieCoupleCitySocialMedia-SS02-CF01-P-420-8bit-SDR" 243 + "-1080p-30fps_hw_hevc_vbr_b3.json", 244 "SelfieCoupleCitySocialMedia_SS02_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_" 245 + "vbr_b3"}}); 246 247 private static final List<Object[]> QUICK_RUN_PARAMS = Arrays.asList(new Object[][]{ 248 {"AVICON-MOBILE-SelfieTeenKitchenSocialMedia-SS01-CF01-P-420-8bit-SDR-1080p" 249 + "-30fps_hw_avc_vbr_b0.json", 250 "SelfieTeenKitchenSocialMedia_SS01_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_" + 251 "vbr_b0"}, 252 {"AVICON-MOBILE-SelfieTeenKitchenSocialMedia-SS01-CF01-P-420-8bit-SDR-1080p" 253 + "-30fps_hw_hevc_vbr_b0.json", 254 "SelfieTeenKitchenSocialMedia_SS01_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_" 255 + "vbr_b0"}}); 256 257 @Parameterized.Parameters(name = "{index}_{1}") input()258 public static List<Object[]> input() { 259 final List<Object[]> args = new ArrayList<>(); 260 args.addAll(AVC_VBR_B0_PARAMS); 261 args.addAll(AVC_VBR_B3_PARAMS); 262 args.addAll(HEVC_VBR_B0_PARAMS); 263 args.addAll(HEVC_VBR_B3_PARAMS); 264 return args; 265 } 266 267 @Override setDevice(ITestDevice device)268 public void setDevice(ITestDevice device) { 269 mDevice = device; 270 } 271 272 @Override getDevice()273 public ITestDevice getDevice() { 274 return mDevice; 275 } 276 277 /** 278 * Sets up the necessary environment for the video encoding quality test. 279 */ setupTestEnv()280 public void setupTestEnv() throws Exception { 281 String sdkAsString = getDevice().getProperty("ro.build.version.sdk"); 282 int sdk = Integer.parseInt(sdkAsString); 283 Assume.assumeTrue("Test requires sdk >= " + MINIMUM_VALID_SDK 284 + " test device has sdk = " + sdk, sdk >= MINIMUM_VALID_SDK); 285 286 String pcAsString = getDevice().getProperty("ro.odm.build.media_performance_class"); 287 int mpc = 0; 288 try { 289 mpc = Integer.parseInt("0" + pcAsString); 290 } catch (Exception e) { 291 LogUtil.CLog.i("Invalid pcAsString: " + pcAsString + ", exception: " + e); 292 } 293 Assume.assumeTrue("Test device does not advertise performance class", 294 mForceToRun || (mpc >= MEDIA_PERFORMANCE_CLASS_14)); 295 296 Assert.assertTrue("Failed to install package on device : " + DEVICE_SIDE_TEST_PACKAGE, 297 getDevice().isPackageInstalled(DEVICE_SIDE_TEST_PACKAGE)); 298 299 // set up host-side working directory 300 String tmpBase = System.getProperty("java.io.tmpdir"); 301 String dirName = "CtsVideoEncodingQualityHostTest_" + getDevice().getSerialNumber(); 302 String tmpDir = tmpBase + "/" + dirName; 303 LogUtil.CLog.i("tmpBase= " + tmpBase + " tmpDir =" + tmpDir); 304 sHostWorkDir = new File(tmpDir); 305 if (mReset || sHostWorkDir.isFile()) { 306 File cwd = new File("."); 307 runCommand("rm -rf " + tmpDir, cwd); 308 } 309 try { 310 if (!sHostWorkDir.isDirectory()) { 311 Assert.assertTrue("Failed to create directory : " + sHostWorkDir.getAbsolutePath(), 312 sHostWorkDir.mkdirs()); 313 } 314 } catch (SecurityException e) { 315 LogUtil.CLog.e("Unable to establish temp directory " + sHostWorkDir.getPath()); 316 } 317 318 // Clean up output folders before starting the test 319 runCommand("rm -rf " + "output_*", sHostWorkDir); 320 321 // Download the test suite tar file. 322 downloadFile(RES_URL, sHostWorkDir); 323 324 // Unpack the test suite tar file. 325 String fileName = RES_URL.substring(RES_URL.lastIndexOf('/') + 1); 326 int result = runCommand("tar xvzf " + fileName, sHostWorkDir); 327 Assert.assertEquals("Failed to untar " + fileName, 0, result); 328 329 // Push input files to device 330 String deviceInDir = getDevice().getMountPoint(IDevice.MNT_EXTERNAL_STORAGE) 331 + "/veq/input/"; 332 String deviceJsonDir = deviceInDir + "json/"; 333 String deviceSamplesDir = deviceInDir + "samples/"; 334 Assert.assertNotNull("Failed to create directory " + deviceJsonDir + " on device ", 335 getDevice().executeAdbCommand("shell", "mkdir", "-p", deviceJsonDir)); 336 Assert.assertNotNull("Failed to create directory " + deviceSamplesDir + " on device ", 337 getDevice().executeAdbCommand("shell", "mkdir", "-p", deviceSamplesDir)); 338 Assert.assertTrue("Failed to push json files to " + deviceJsonDir + " on device ", 339 getDevice().pushDir(new File(sHostWorkDir.getPath() + "/json/"), deviceJsonDir)); 340 Assert.assertTrue("Failed to push mp4 files to " + deviceSamplesDir + " on device ", 341 getDevice().pushDir(new File(sHostWorkDir.getPath() + "/samples/"), 342 deviceSamplesDir)); 343 344 sIsTestSetUpDone = true; 345 } 346 containsJson(String jsonName, List<Object[]> params)347 public static boolean containsJson(String jsonName, List<Object[]> params) { 348 for (Object[] param : params) { 349 if (param[0].equals(jsonName)) { 350 return true; 351 } 352 } 353 return false; 354 } 355 356 /** 357 * Verify the video encoding quality requirements for the performance class 14 devices. 358 */ 359 @CddTest(requirements = {"2.2.7.1/5.8/H-1-1"}) 360 @Test testEncoding()361 public void testEncoding() throws Exception { 362 Assume.assumeFalse("Skipping due to quick run mode", 363 mQuickCheck && !containsJson(mJsonName, QUICK_RUN_PARAMS)); 364 Assume.assumeFalse("Skipping avc encoder tests", 365 mSkipAvc && (containsJson(mJsonName, AVC_VBR_B0_PARAMS) || containsJson(mJsonName, 366 AVC_VBR_B3_PARAMS))); 367 Assume.assumeFalse("Skipping hevc encoder tests", 368 mSkipHevc && (containsJson(mJsonName, HEVC_VBR_B0_PARAMS) || containsJson(mJsonName, 369 HEVC_VBR_B3_PARAMS))); 370 Assume.assumeFalse("Skipping b-frame tests", 371 mSkipB && (containsJson(mJsonName, AVC_VBR_B3_PARAMS) || containsJson(mJsonName, 372 HEVC_VBR_B3_PARAMS))); 373 Assume.assumeFalse("Skipping non b-frame tests", 374 mSkipP && (containsJson(mJsonName, AVC_VBR_B0_PARAMS) || containsJson(mJsonName, 375 HEVC_VBR_B0_PARAMS))); 376 377 // set up test environment 378 sLock.lock(); 379 try { 380 if (!sIsTestSetUpDone) setupTestEnv(); 381 sCondition.signalAll(); 382 } finally { 383 sLock.unlock(); 384 } 385 386 // transcode input 387 runDeviceTests(DEVICE_SIDE_TEST_PACKAGE, DEVICE_SIDE_TEST_CLASS, "testTranscode"); 388 389 // copy the encoded output from the device to the host. 390 String outDir = "output_" + mJsonName.substring(0, mJsonName.indexOf('.')); 391 File outHostPath = new File(sHostWorkDir, outDir); 392 try { 393 if (!outHostPath.isDirectory()) { 394 Assert.assertTrue("Failed to create directory : " + outHostPath.getAbsolutePath(), 395 outHostPath.mkdirs()); 396 } 397 } catch (SecurityException e) { 398 LogUtil.CLog.e("Unable to establish output host directory : " + outHostPath.getPath()); 399 } 400 String outDevPath = getDevice().getMountPoint(IDevice.MNT_EXTERNAL_STORAGE) + "/veq/output/" 401 + outDir; 402 Assert.assertTrue("Failed to pull mp4 files from " + outDevPath 403 + " to " + outHostPath.getPath(), getDevice().pullDir(outDevPath, outHostPath)); 404 getDevice().deleteFile(outDevPath); 405 406 // Parse json file 407 String jsonPath = sHostWorkDir.getPath() + "/json/" + mJsonName; 408 String jsonString = 409 new String(Files.readAllBytes(Paths.get(jsonPath)), StandardCharsets.UTF_8); 410 JSONArray jsonArray = new JSONArray(jsonString); 411 JSONObject obj = jsonArray.getJSONObject(0); 412 String refFileName = obj.getString("RefFileName"); 413 int fps = obj.getInt("FrameRate"); 414 int frameCount = obj.getInt("FrameCount"); 415 int clipDuration = frameCount / fps; 416 417 // Compute Vmaf 418 try (FileWriter writer = new FileWriter(outHostPath.getPath() + "/" + "all_vmafs.txt")) { 419 JSONArray codecConfigs = obj.getJSONArray("CodecConfigs"); 420 int th = Runtime.getRuntime().availableProcessors() / 2; 421 th = Math.min(Math.max(1, th), 8); 422 String filter = "libvmaf=feature=name=psnr:model=version=vmaf_v0.6.1:n_threads=" + th; 423 for (int i = 0; i < codecConfigs.length(); i++) { 424 JSONObject codecConfig = codecConfigs.getJSONObject(i); 425 String outputName = codecConfig.getString("EncodedFileName"); 426 outputName = outputName.substring(0, outputName.lastIndexOf(".")); 427 String outputVmafPath = outDir + "/" + outputName + ".txt"; 428 String cmd = "./bin/ffmpeg"; 429 cmd += " -hide_banner"; 430 cmd += " -i " + outDir + "/" + outputName + ".mp4" + " -an"; 431 cmd += " -i " + "samples/" + refFileName + " -an"; 432 cmd += " -filter_complex " + "\"" + filter + "\""; 433 cmd += " -f null -"; 434 cmd += " > " + outputVmafPath + " 2>&1"; 435 LogUtil.CLog.i("ffmpeg command : " + cmd); 436 int result = runCommand(cmd, sHostWorkDir); 437 Assert.assertEquals("Encountered error during vmaf computation.", 0, result); 438 439 String vmafLine = ""; 440 try (BufferedReader reader = new BufferedReader( 441 new FileReader(sHostWorkDir.getPath() + "/" + outputVmafPath))) { 442 String token = "VMAF score: "; 443 String line; 444 while ((line = reader.readLine()) != null) { 445 if (line.contains(token)) { 446 line = line.substring(line.indexOf(token)); 447 vmafLine = "VMAF score = " + line.substring(token.length()); 448 LogUtil.CLog.i(vmafLine); 449 break; 450 } 451 } 452 } catch (IOException e) { 453 throw new AssertionError("Unexpected IOException: " + e.getMessage()); 454 } 455 456 writer.write(vmafLine + "\n"); 457 writer.write("Y4M file = " + refFileName + "\n"); 458 writer.write("MP4 file = " + refFileName + "\n"); 459 File file = new File(outHostPath + "/" + outputName + ".mp4"); 460 Assert.assertTrue("output file from device missing", file.exists()); 461 long fileSize = file.length(); 462 writer.write("Filesize = " + fileSize + "\n"); 463 writer.write("FPS = " + fps + "\n"); 464 writer.write("FRAME_COUNT = " + frameCount + "\n"); 465 writer.write("CLIP_DURATION = " + clipDuration + "\n"); 466 long totalBits = fileSize * 8; 467 long totalBits_kbps = totalBits / 1000; 468 long bitrate_kbps = totalBits_kbps / clipDuration; 469 writer.write("Bitrate kbps = " + bitrate_kbps + "\n"); 470 } 471 } catch (IOException e) { 472 throw new AssertionError("Unexpected IOException: " + e.getMessage()); 473 } 474 475 // bd rate verification 476 String jarCmd = "java -jar " + "./bin/cts-media-videoquality-bdrate.jar " 477 + "--uid= --gid= --chroot= " 478 + "--REF_JSON_FILE=" + "json/" + mJsonName + " " 479 + "--TEST_VMAF_FILE=" + outDir + "/" + "all_vmafs.txt " 480 + "> " + outDir + "/result.txt"; 481 LogUtil.CLog.i("bdrate command : " + jarCmd); 482 int result = runCommand(jarCmd, sHostWorkDir); 483 Assert.assertEquals("bd rate validation failed.", 0, result); 484 485 LogUtil.CLog.i("Finished executing the process."); 486 } 487 runCommand(String command, File dir)488 private int runCommand(String command, File dir) throws IOException, InterruptedException { 489 Process p = new ProcessBuilder("/bin/sh", "-c", command) 490 .directory(dir) 491 .redirectErrorStream(true) 492 .redirectOutput(ProcessBuilder.Redirect.INHERIT) 493 .start(); 494 495 BufferedReader stdInput = new BufferedReader(new InputStreamReader(p.getInputStream())); 496 BufferedReader stdError = new BufferedReader(new InputStreamReader(p.getErrorStream())); 497 String line; 498 while ((line = stdInput.readLine()) != null || (line = stdError.readLine()) != null) { 499 LogUtil.CLog.i(line + "\n"); 500 } 501 return p.waitFor(); 502 } 503 504 // Download the indicated file (within the base_url folder) to our desired destination 505 // simple caching -- if file exists, we do not re-download downloadFile(String url, File destDir)506 private void downloadFile(String url, File destDir) { 507 String fileName = url.substring(RES_URL.lastIndexOf('/') + 1); 508 File destination = new File(destDir, fileName); 509 510 // save bandwidth, also allows a user to manually preload files 511 LogUtil.CLog.i("Do we already have a copy of file " + destination.getPath()); 512 if (destination.isFile()) { 513 LogUtil.CLog.i("Skipping re-download of file " + destination.getPath()); 514 return; 515 } 516 517 String cmd = "wget -O " + destination.getPath() + " " + url; 518 LogUtil.CLog.i("wget_cmd = " + cmd); 519 520 int result = 0; 521 try { 522 result = runCommand(cmd, destDir); 523 } catch (IOException e) { 524 result = -2; 525 } catch (InterruptedException e) { 526 result = -3; 527 } 528 Assert.assertEquals("download file failed.\n", 0, result); 529 } 530 runDeviceTests(String pkgName, @Nullable String testClassName, @Nullable String testMethodName)531 private void runDeviceTests(String pkgName, @Nullable String testClassName, 532 @Nullable String testMethodName) throws DeviceNotAvailableException { 533 RemoteAndroidTestRunner testRunner = getTestRunner(pkgName, testClassName, testMethodName); 534 CollectingTestListener listener = new CollectingTestListener(); 535 Assert.assertTrue(getDevice().runInstrumentationTests(testRunner, listener)); 536 assertTestsPassed(listener.getCurrentRunResults()); 537 } 538 getTestRunner(String pkgName, String testClassName, String testMethodName)539 private RemoteAndroidTestRunner getTestRunner(String pkgName, String testClassName, 540 String testMethodName) { 541 if (testClassName != null && testClassName.startsWith(".")) { 542 testClassName = pkgName + testClassName; 543 } 544 RemoteAndroidTestRunner testRunner = 545 new RemoteAndroidTestRunner(pkgName, RUNNER, getDevice().getIDevice()); 546 testRunner.setMaxTimeToOutputResponse(DEFAULT_SHELL_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); 547 testRunner.addInstrumentationArg(TEST_TIMEOUT_INST_ARGS_KEY, 548 Long.toString(DEFAULT_TEST_TIMEOUT_MILLIS)); 549 testRunner.addInstrumentationArg(TEST_CONFIG_INST_ARGS_KEY, mJsonName); 550 if (testClassName != null && testMethodName != null) { 551 testRunner.setMethodName(testClassName, testMethodName); 552 } else if (testClassName != null) { 553 testRunner.setClassName(testClassName); 554 } 555 return testRunner; 556 } 557 assertTestsPassed(TestRunResult testRunResult)558 private void assertTestsPassed(TestRunResult testRunResult) { 559 if (testRunResult.isRunFailure()) { 560 throw new AssertionError("Failed to successfully run device tests for " 561 + testRunResult.getName() + ": " + testRunResult.getRunFailureMessage()); 562 } 563 if (testRunResult.getNumTests() != testRunResult.getPassedTests().size()) { 564 for (Map.Entry<TestDescription, TestResult> resultEntry : 565 testRunResult.getTestResults().entrySet()) { 566 if (resultEntry.getValue().getStatus().equals(TestStatus.FAILURE)) { 567 StringBuilder errorBuilder = new StringBuilder("On-device tests failed:\n"); 568 errorBuilder.append(resultEntry.getKey().toString()); 569 errorBuilder.append(":\n"); 570 errorBuilder.append(resultEntry.getValue().getStackTrace()); 571 throw new AssertionError(errorBuilder.toString()); 572 } 573 if (resultEntry.getValue().getStatus().equals(TestStatus.ASSUMPTION_FAILURE)) { 574 StringBuilder errorBuilder = 575 new StringBuilder("On-device tests assumption failed:\n"); 576 errorBuilder.append(resultEntry.getKey().toString()); 577 errorBuilder.append(":\n"); 578 errorBuilder.append(resultEntry.getValue().getStackTrace()); 579 Assume.assumeTrue(errorBuilder.toString(), false); 580 } 581 } 582 } 583 } 584 } 585