1#!/usr/bin/python
2
3# Copyright (C) 2009 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#       http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Module for generating CTS test descriptions and test plans."""
18
19import glob
20import os
21import re
22import shutil
23import subprocess
24import sys
25import xml.dom.minidom as dom
26from cts import tools
27from multiprocessing import Pool
28
29def GetSubDirectories(root):
30  """Return all directories under the given root directory."""
31  return [x for x in os.listdir(root) if os.path.isdir(os.path.join(root, x))]
32
33def ReadFileLines(filePath):
34  """Reads a file and returns its contents as a line list."""
35  f = open(filePath, 'r');
36  lines = [line.strip() for line in f.readlines()]
37  f.close()
38  return lines
39
40def GetMakeFileVars(makefile_path):
41  """Extracts variable definitions from the given make file.
42
43  Args:
44    makefile_path: Path to the make file.
45
46  Returns:
47    A dictionary mapping variable names to their assigned value.
48  """
49  result = {}
50  pattern = re.compile(r'^\s*([^:#=\s]+)\s*:=\s*(.*?[^\\])$', re.MULTILINE + re.DOTALL)
51  stream = open(makefile_path, 'r')
52  content = stream.read()
53  for match in pattern.finditer(content):
54    result[match.group(1)] = match.group(2)
55  stream.close()
56  return result
57
58
59class CtsBuilder(object):
60  """Main class for generating test descriptions and test plans."""
61
62  def __init__(self, argv):
63    """Initialize the CtsBuilder from command line arguments."""
64    if len(argv) != 6:
65      print 'Usage: %s <testRoot> <ctsOutputDir> <tempDir> <androidRootDir> <docletPath>' % argv[0]
66      print ''
67      print 'testRoot:       Directory under which to search for CTS tests.'
68      print 'ctsOutputDir:   Directory in which the CTS repository should be created.'
69      print 'tempDir:        Directory to use for storing temporary files.'
70      print 'androidRootDir: Root directory of the Android source tree.'
71      print 'docletPath:     Class path where the DescriptionGenerator doclet can be found.'
72      sys.exit(1)
73    self.test_root = sys.argv[1]
74    self.out_dir = sys.argv[2]
75    self.temp_dir = sys.argv[3]
76    self.android_root = sys.argv[4]
77    self.doclet_path = sys.argv[5]
78
79    self.test_repository = os.path.join(self.out_dir, 'repository/testcases')
80    self.plan_repository = os.path.join(self.out_dir, 'repository/plans')
81    self.definedplans_repository = os.path.join(self.android_root, 'cts/tests/plans')
82
83  def GenerateTestDescriptions(self):
84    """Generate test descriptions for all packages."""
85    pool = Pool(processes=2)
86
87    # generate test descriptions for android tests
88    results = []
89    pool.close()
90    pool.join()
91    return sum(map(lambda result: result.get(), results))
92
93  def __WritePlan(self, plan, plan_name):
94    print 'Generating test plan %s' % plan_name
95    plan.Write(os.path.join(self.plan_repository, plan_name + '.xml'))
96
97  def GenerateTestPlans(self):
98    """Generate default test plans."""
99    # TODO: Instead of hard-coding the plans here, use a configuration file,
100    # such as test_defs.xml
101    packages = []
102    descriptions = sorted(glob.glob(os.path.join(self.test_repository, '*.xml')))
103    for description in descriptions:
104      doc = tools.XmlFile(description)
105      packages.append(doc.GetAttr('TestPackage', 'appPackageName'))
106    # sort the list to give the same sequence based on name
107    packages.sort()
108
109    plan = tools.TestPlan(packages)
110    plan.Exclude('android\.performance.*')
111    self.__WritePlan(plan, 'CTS')
112    self.__WritePlan(plan, 'CTS-TF')
113
114    plan = tools.TestPlan(packages)
115    plan.Exclude('android\.performance.*')
116    plan.Exclude('android\.media\.cts\.StreamingMediaPlayerTest.*')
117    # Test plan to not include media streaming tests
118    self.__WritePlan(plan, 'CTS-No-Media-Stream')
119
120    plan = tools.TestPlan(packages)
121    plan.Exclude('android\.performance.*')
122    self.__WritePlan(plan, 'SDK')
123
124    plan.Exclude(r'android\.signature')
125    plan.Exclude(r'android\.core.*')
126    self.__WritePlan(plan, 'Android')
127
128    plan = tools.TestPlan(packages)
129    plan.Include(r'android\.core\.tests.*')
130    plan.Exclude(r'android\.core\.tests\.libcore.\package.\harmony*')
131    self.__WritePlan(plan, 'Java')
132
133    # TODO: remove this once the tests are fixed and merged into Java plan above.
134    plan = tools.TestPlan(packages)
135    plan.Include(r'android\.core\.tests\.libcore.\package.\harmony*')
136    self.__WritePlan(plan, 'Harmony')
137
138    plan = tools.TestPlan(packages)
139    plan.Include(r'android\.core\.vm-tests-tf')
140    self.__WritePlan(plan, 'VM-TF')
141
142    plan = tools.TestPlan(packages)
143    plan.Include(r'android\.tests\.appsecurity')
144    self.__WritePlan(plan, 'AppSecurity')
145
146    # hard-coded white list for PDK plan
147    plan.Exclude('.*')
148    plan.Include('android\.aadb')
149    plan.Include('android\.bluetooth')
150    plan.Include('android\.graphics.*')
151    plan.Include('android\.hardware')
152    plan.Include('android\.media')
153    plan.Exclude('android\.mediastress')
154    plan.Include('android\.net')
155    plan.Include('android\.opengl.*')
156    plan.Include('android\.renderscript')
157    plan.Include('android\.telephony')
158    plan.Include('android\.nativemedia.*')
159    plan.Include('com\.android\.cts\..*')#TODO(stuartscott): Should PDK have all these?
160    self.__WritePlan(plan, 'PDK')
161
162    temporarily_known_failure_tests = BuildCtsTemporarilyKnownFailureList();
163    flaky_tests = BuildCtsFlakyTestList()
164    releasekey_tests = BuildListForReleaseBuildTest()
165
166    # CTS Stable plan
167    plan = tools.TestPlan(packages)
168    plan.Exclude(r'com\.android\.cts\.browserbench')
169    for package, test_list in flaky_tests.iteritems():
170      plan.ExcludeTests(package, test_list)
171    for package, test_list in releasekey_tests.iteritems():
172      plan.ExcludeTests(package, test_list)
173    self.__WritePlan(plan, 'CTS-stable')
174
175    # CTS Flaky plan - list of tests known to be flaky in lab environment
176    plan = tools.TestPlan(packages)
177    plan.Exclude('.*')
178    plan.Include(r'com\.android\.cts\.browserbench')
179    for package, test_list in flaky_tests.iteritems():
180      plan.Include(package+'$')
181      plan.IncludeTests(package, test_list)
182    self.__WritePlan(plan, 'CTS-flaky')
183
184    small_tests = BuildAospSmallSizeTestList()
185    medium_tests = BuildAospMediumSizeTestList()
186    new_test_packages = BuildCtsVettedNewPackagesList()
187
188    # CTS - sub plan for public, small size tests
189    plan = tools.TestPlan(packages)
190    plan.Exclude('.*')
191    for package, test_list in small_tests.iteritems():
192      plan.Include(package+'$')
193    plan.Exclude(r'com\.android\.cts\.browserbench')
194    for package, test_list in flaky_tests.iteritems():
195      plan.ExcludeTests(package, test_list)
196    for package, test_list in releasekey_tests.iteritems():
197      plan.ExcludeTests(package, test_list)
198    self.__WritePlan(plan, 'CTS-kitkat-small')
199    self.__WritePlan(plan, 'CTS-public-small')
200
201    # CTS - sub plan for public, medium size tests
202    plan = tools.TestPlan(packages)
203    plan.Exclude('.*')
204    for package, test_list in medium_tests.iteritems():
205      plan.Include(package+'$')
206    plan.Exclude(r'com\.android\.cts\.browserbench')
207    for package, test_list in flaky_tests.iteritems():
208      plan.ExcludeTests(package, test_list)
209    for package, test_list in releasekey_tests.iteritems():
210      plan.ExcludeTests(package, test_list)
211    self.__WritePlan(plan, 'CTS-kitkat-medium')
212    self.__WritePlan(plan, 'CTS-public-medium')
213
214    # CTS - sub plan for hardware tests which is public, large
215    plan = tools.TestPlan(packages)
216    plan.Exclude('.*')
217    plan.Include(r'android\.hardware$')
218    plan.Exclude(r'com\.android\.cts\.browserbench')
219    for package, test_list in flaky_tests.iteritems():
220      plan.ExcludeTests(package, test_list)
221    for package, test_list in releasekey_tests.iteritems():
222      plan.ExcludeTests(package, test_list)
223    self.__WritePlan(plan, 'CTS-hardware')
224
225    # CTS - sub plan for media tests which is public, large
226    plan = tools.TestPlan(packages)
227    plan.Exclude('.*')
228    plan.Include(r'android\.media$')
229    plan.Include(r'android\.view$')
230    plan.Exclude(r'com\.android\.cts\.browserbench')
231    for package, test_list in flaky_tests.iteritems():
232      plan.ExcludeTests(package, test_list)
233    for package, test_list in releasekey_tests.iteritems():
234      plan.ExcludeTests(package, test_list)
235    self.__WritePlan(plan, 'CTS-media')
236
237    # CTS - sub plan for mediastress tests which is public, large
238    plan = tools.TestPlan(packages)
239    plan.Exclude('.*')
240    plan.Include(r'android\.mediastress$')
241    plan.Exclude(r'com\.android\.cts\.browserbench')
242    for package, test_list in flaky_tests.iteritems():
243      plan.ExcludeTests(package, test_list)
244    for package, test_list in releasekey_tests.iteritems():
245      plan.ExcludeTests(package, test_list)
246    self.__WritePlan(plan, 'CTS-mediastress')
247
248    # CTS - sub plan for new tests that is vetted for L launch
249    plan = tools.TestPlan(packages)
250    plan.Exclude('.*')
251    for package, test_list in new_test_packages.iteritems():
252      plan.Include(package+'$')
253    plan.Exclude(r'com\.android\.cts\.browserbench')
254    for package, test_list in flaky_tests.iteritems():
255      plan.ExcludeTests(package, test_list)
256    for package, test_list in releasekey_tests.iteritems():
257      plan.ExcludeTests(package, test_list)
258    self.__WritePlan(plan, 'CTS-l-tests')
259
260    # CTS - sub plan for tests in drawelement packages
261    plan = tools.TestPlan(packages)
262    plan.Exclude('.*')
263    plan.Include(r'com\.drawelements\.')
264    self.__WritePlan(plan, 'CTS-DEQP')
265
266    # CTS - sub plan for new test packages added for staging
267    plan = tools.TestPlan(packages)
268    for package, test_list in small_tests.iteritems():
269      plan.Exclude(package+'$')
270    for package, test_list in medium_tests.iteritems():
271      plan.Exclude(package+'$')
272    for package, tests_list in new_test_packages.iteritems():
273      plan.Exclude(package+'$')
274    plan.Exclude(r'com\.drawelements\.')
275    plan.Exclude(r'android\.hardware$')
276    plan.Exclude(r'android\.media$')
277    plan.Exclude(r'android\.view$')
278    plan.Exclude(r'android\.mediastress$')
279    plan.Exclude(r'com\.android\.cts\.browserbench')
280    for package, test_list in flaky_tests.iteritems():
281      plan.ExcludeTests(package, test_list)
282    for package, test_list in releasekey_tests.iteritems():
283      plan.ExcludeTests(package, test_list)
284    self.__WritePlan(plan, 'CTS-m-tests')
285
286
287    # CTS - sub plan for new test packages added for staging
288    plan = tools.TestPlan(packages)
289    plan.Exclude('.*')
290    for package, test_list in temporarily_known_failure_tests.iteritems():
291      plan.Include(package+'$')
292      plan.IncludeTests(package, test_list)
293    self.__WritePlan(plan, 'CTS-staging')
294
295    plan = tools.TestPlan(packages)
296    plan.Exclude('.*')
297    self.__WritePlan(plan, 'CTS-webview')
298
299
300def BuildAospMediumSizeTestList():
301  """ Construct a defaultdic that lists package names of medium tests
302      already published to aosp. """
303  return {
304      'android.app' : [],
305      'android.core.tests.libcore.package.libcore' : [],
306      'android.core.tests.libcore.package.org' : [],
307      'android.core.vm-tests-tf' : [],
308      'android.dpi' : [],
309      'android.host.security' : [],
310      'android.net' : [],
311      'android.os' : [],
312      'android.permission2' : [],
313      'android.security' : [],
314      'android.telephony' : [],
315      'android.webkit' : [],
316      'android.widget' : [],
317      'com.android.cts.browserbench' : []}
318
319def BuildAospSmallSizeTestList():
320  """ Construct a default dict that lists packages names of small tests
321      already published to aosp. """
322  return {
323      'android.aadb' : [],
324      'android.acceleration' : [],
325      'android.accessibility' : [],
326      'android.accessibilityservice' : [],
327      'android.accounts' : [],
328      'android.admin' : [],
329      'android.animation' : [],
330      'android.bionic' : [],
331      'android.bluetooth' : [],
332      'android.calendarcommon' : [],
333      'android.content' : [],
334      'android.core.tests.libcore.package.com' : [],
335      'android.core.tests.libcore.package.conscrypt' : [],
336      'android.core.tests.libcore.package.dalvik' : [],
337      'android.core.tests.libcore.package.sun' : [],
338      'android.core.tests.libcore.package.tests' : [],
339      'android.database' : [],
340      'android.dreams' : [],
341      'android.drm' : [],
342      'android.effect' : [],
343      'android.gesture' : [],
344      'android.graphics' : [],
345      'android.graphics2' : [],
346      'android.jni' : [],
347      'android.keystore' : [],
348      'android.location' : [],
349      'android.nativemedia.sl' : [],
350      'android.nativemedia.xa' : [],
351      'android.nativeopengl' : [],
352      'android.ndef' : [],
353      'android.opengl' : [],
354      'android.openglperf' : [],
355      'android.permission' : [],
356      'android.preference' : [],
357      'android.preference2' : [],
358      'android.provider' : [],
359      'android.renderscript' : [],
360      'android.rscpp' : [],
361      'android.rsg' : [],
362      'android.sax' : [],
363      'android.signature' : [],
364      'android.speech' : [],
365      'android.tests.appsecurity' : [],
366      'android.text' : [],
367      'android.textureview' : [],
368      'android.theme' : [],
369      'android.usb' : [],
370      'android.util' : [],
371      'com.android.cts.dram' : [],
372      'com.android.cts.filesystemperf' : [],
373      'com.android.cts.jank' : [],
374      'com.android.cts.jank2' : [],
375      'com.android.cts.opengl' : [],
376      'com.android.cts.simplecpu' : [],
377      'com.android.cts.ui' : [],
378      'com.android.cts.uihost' : [],
379      'com.android.cts.videoperf' : [],
380      'zzz.android.monkey' : []}
381
382def BuildCtsVettedNewPackagesList():
383  """ Construct a defaultdict that maps package names that is vetted for L. """
384  return {
385      'android.JobScheduler' : [],
386      'android.core.tests.libcore.package.harmony_annotation' : [],
387      'android.core.tests.libcore.package.harmony_beans' : [],
388      'android.core.tests.libcore.package.harmony_java_io' : [],
389      'android.core.tests.libcore.package.harmony_java_lang' : [],
390      'android.core.tests.libcore.package.harmony_java_math' : [],
391      'android.core.tests.libcore.package.harmony_java_net' : [],
392      'android.core.tests.libcore.package.harmony_java_nio' : [],
393      'android.core.tests.libcore.package.harmony_java_util' : [],
394      'android.core.tests.libcore.package.harmony_java_text' : [],
395      'android.core.tests.libcore.package.harmony_javax_security' : [],
396      'android.core.tests.libcore.package.harmony_logging' : [],
397      'android.core.tests.libcore.package.harmony_prefs' : [],
398      'android.core.tests.libcore.package.harmony_sql' : [],
399      'android.core.tests.libcore.package.jsr166' : [],
400      'android.core.tests.libcore.package.okhttp' : [],
401      'android.display' : [],
402      'android.host.theme' : [],
403      'android.jdwp' : [],
404      'android.location2' : [],
405      'android.print' : [],
406      'android.renderscriptlegacy' : [],
407      'android.signature' : [],
408      'android.tv' : [],
409      'android.uiautomation' : [],
410      'android.uirendering' : []}
411
412def BuildListForReleaseBuildTest():
413  """ Construct a defaultdict that maps package name to a list of tests
414      that are expected to pass only when running against a user/release-key build. """
415  return {
416      'android.app' : [
417          'android.app.cts.ActivityManagerTest#testIsRunningInTestHarness',],
418      'android.dpi' : [
419          'android.dpi.cts.DefaultManifestAttributesSdkTest#testPackageHasExpectedSdkVersion',],
420      'android.host.security' : [
421          'android.cts.security.SELinuxHostTest#testAllEnforcing',
422          'android.cts.security.SELinuxHostTest#testSuDomain',],
423      'android.os' : [
424          'android.os.cts.BuildVersionTest#testReleaseVersion',
425          'android.os.cts.BuildTest#testIsSecureUserBuild',],
426      'android.security' : [
427          'android.security.cts.BannedFilesTest#testNoSu',
428          'android.security.cts.BannedFilesTest#testNoSuInPath',
429          'android.security.cts.PackageSignatureTest#testPackageSignatures',
430          'android.security.cts.SELinuxDomainTest#testSuDomain',],
431      '' : []}
432
433def BuildCtsFlakyTestList():
434  """ Construct a defaultdict that maps package name to a list of tests
435      that flaky during dev cycle and cause other subsequent tests to fail. """
436  return {
437      'android.hardware' : [
438          'android.hardware.cts.CameraTest#testVideoSnapshot',
439          'android.hardware.cts.CameraGLTest#testCameraToSurfaceTextureMetadata',
440          'android.hardware.cts.CameraGLTest#testSetPreviewTextureBothCallbacks',
441          'android.hardware.cts.CameraGLTest#testSetPreviewTexturePreviewCallback',],
442      'android.media' : [
443          'android.media.cts.DecoderTest#testCodecResetsH264WithSurface',
444          'android.media.cts.StreamingMediaPlayerTest#testHLS',],
445      'android.net' : [
446          'android.net.cts.ConnectivityManagerTest#testStartUsingNetworkFeature_enableHipri',
447          'android.net.cts.DnsTest#testDnsWorks',
448          'android.net.cts.SSLCertificateSocketFactoryTest#testCreateSocket',
449          'android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_bind',
450          'android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_simple',
451          'android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_wrapping',
452          'android.net.cts.TrafficStatsTest#testTrafficStatsForLocalhost',
453          'android.net.wifi.cts.NsdManagerTest#testAndroidTestCaseSetupProperly',],
454      'android.security' : [
455          'android.security.cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdp6Ports',
456          'android.security.cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdpPorts',],
457      'android.webkit' : [
458          'android.webkit.cts.WebViewClientTest#testOnUnhandledKeyEvent',],
459      'com.android.cts.filesystemperf' : [
460          'com.android.cts.filesystemperf.RandomRWTest#testRandomRead',
461          'com.android.cts.filesystemperf.RandomRWTest#testRandomUpdate',],
462      '' : []}
463
464def BuildCtsTemporarilyKnownFailureList():
465  """ Construct a defaultdict that maps package name to a list of tests
466      that are known failures during dev cycle but expected to be fixed before launch """
467  return {
468      'android.alarmclock' : [
469          'android.alarmclock.cts.DismissAlarmTest#testAll',
470          'android.alarmclock.cts.SetAlarmTest#testAll',
471          'android.alarmclock.cts.SnoozeAlarmTest#testAll',
472      ],
473      'android.assist' : [
474          'android.assist.cts.ExtraAssistDataTest',
475          'android.assist.cts.AssistantContentViewTest',
476          'android.assist.cts.ScreenshotTest',
477      ],
478      'android.calllog' : [
479          'android.calllog.cts.CallLogBackupTest#testSingleCallBackup',
480      ],
481      'android.dumpsys' : [
482          'android.dumpsys.cts.DumpsysHostTest#testBatterystatsOutput',
483          'android.dumpsys.cts.DumpsysHostTest#testGfxinfoFramestats',
484      ],
485      'android.telecom' : [
486          'android.telecom.cts.ExtendedInCallServiceTest#testAddNewOutgoingCallAndThenDisconnect',
487          'android.telecom.cts.RemoteConferenceTest#testRemoteConferenceCallbacks_ConferenceableConnections',
488      ],
489      'android.transition' : [
490          'android.transition.cts.ChangeScrollTest#testChangeScroll',
491      ],
492      'android.voicesettings' : [
493          'android.voicesettings.cts.ZenModeTest#testAll',
494      ],
495      'com.android.cts.app.os' : [
496          'com.android.cts.app.os.OsHostTests#testNonExportedActivities',
497      ],
498      'com.android.cts.devicepolicy' : [
499          'com.android.cts.devicepolicy.MixedDeviceOwnerTest#testPackageInstallUserRestrictions',
500          'com.android.cts.devicepolicy.MixedProfileOwnerTest#testPackageInstallUserRestrictions',
501      ],
502      '' : []}
503
504def LogGenerateDescription(name):
505  print 'Generating test description for package %s' % name
506
507if __name__ == '__main__':
508  builder = CtsBuilder(sys.argv)
509  result = builder.GenerateTestDescriptions()
510  if result != 0:
511    sys.exit(result)
512  builder.GenerateTestPlans()
513
514