1#!/usr/bin/python -B
2
3# Copyright 2017 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Utility methods associated with ICU source and builds."""
18
19from __future__ import print_function
20
21import filecmp
22import glob
23import os
24import shutil
25import subprocess
26import sys
27
28import i18nutil
29import ziputil
30
31def cldrDir():
32  """Returns the location of CLDR in the Android source tree."""
33  android_build_top = i18nutil.GetAndroidRootOrDie()
34  cldr_dir = os.path.realpath('%s/external/cldr' % android_build_top)
35  i18nutil.CheckDirExists(cldr_dir, 'external/cldr')
36  return cldr_dir
37
38
39def icuDir():
40  """Returns the location of ICU in the Android source tree."""
41  android_build_top = i18nutil.GetAndroidRootOrDie()
42  icu_dir = os.path.realpath('%s/external/icu' % android_build_top)
43  i18nutil.CheckDirExists(icu_dir, 'external/icu')
44  return icu_dir
45
46
47def icu4cDir():
48  """Returns the location of ICU4C in the Android source tree."""
49  icu4c_dir = os.path.realpath('%s/icu4c/source' % icuDir())
50  i18nutil.CheckDirExists(icu4c_dir, 'external/icu/icu4c/source')
51  return icu4c_dir
52
53
54def icu4jDir():
55  """Returns the location of ICU4J in the Android source tree."""
56  icu4j_dir = os.path.realpath('%s/icu4j' % icuDir())
57  i18nutil.CheckDirExists(icu4j_dir, 'external/icu/icu4j')
58  return icu4j_dir
59
60
61def datFile(icu_build_dir):
62  """Returns the location of the ICU .dat file in the specified ICU build dir."""
63  dat_file_pattern = '%s/data/out/tmp/icudt??l.dat' % icu_build_dir
64  dat_files = glob.glob(dat_file_pattern)
65  if len(dat_files) != 1:
66    print('ERROR: Unexpectedly found %d .dat files (%s). Halting.' % (len(datfiles), datfiles))
67    sys.exit(1)
68  dat_file = dat_files[0]
69  return dat_file
70
71
72def PrepareIcuBuild(icu_build_dir):
73  """Sets up an ICU build in the specified (non-existent) directory.
74
75  Creates the directory and runs "runConfigureICU Linux"
76  """
77  # Keep track of the original cwd so we can go back to it at the end.
78  original_working_dir = os.getcwd()
79
80  # Create a directory to run 'make' from.
81  os.mkdir(icu_build_dir)
82  os.chdir(icu_build_dir)
83
84  # Build the ICU tools.
85  print('Configuring ICU tools...')
86  subprocess.check_call(['env', 'ICU_DATA_BUILDTOOL_OPTS=--include_uni_core_data', '%s/runConfigureICU' % icu4cDir(), 'Linux'])
87
88  os.chdir(original_working_dir)
89
90
91def MakeTzDataFiles(icu_build_dir, iana_tar_file):
92  """Builds and runs the ICU tools in ${icu_Build_dir}/tools/tzcode.
93
94  The tools are run against the specified IANA tzdata .tar.gz.
95  The resulting zoneinfo64.txt is copied into the src directories.
96  """
97  tzcode_working_dir = '%s/tools/tzcode' % icu_build_dir
98
99  # Fix missing files.
100  # The tz2icu tool only picks up icuregions and icuzones if they are in the CWD
101  for icu_data_file in [ 'icuregions', 'icuzones']:
102    icu_data_file_source = '%s/tools/tzcode/%s' % (icu4cDir(), icu_data_file)
103    icu_data_file_symlink = '%s/%s' % (tzcode_working_dir, icu_data_file)
104    os.symlink(icu_data_file_source, icu_data_file_symlink)
105
106  iana_tar_filename = os.path.basename(iana_tar_file)
107  working_iana_tar_file = '%s/%s' % (tzcode_working_dir, iana_tar_filename)
108  shutil.copyfile(iana_tar_file, working_iana_tar_file)
109
110  print('Making ICU tz data files...')
111  # The Makefile assumes the existence of the bin directory.
112  os.mkdir('%s/bin' % icu_build_dir)
113
114  # -j1 is needed because the build is not parallelizable. http://b/109641429
115  subprocess.check_call(['make', '-j1', '-C', tzcode_working_dir])
116
117  # Copy the source file to its ultimate destination.
118  zoneinfo_file = '%s/zoneinfo64.txt' % tzcode_working_dir
119  icu_txt_data_dir = '%s/data/misc' % icu4cDir()
120  print('Copying zoneinfo64.txt to %s ...' % icu_txt_data_dir)
121  shutil.copy(zoneinfo_file, icu_txt_data_dir)
122
123
124def MakeAndCopyIcuDataFiles(icu_build_dir):
125  """Builds the ICU .dat and .jar files using the current src data.
126
127  The files are copied back into the expected locations in the src tree.
128  """
129  # Keep track of the original cwd so we can go back to it at the end.
130  original_working_dir = os.getcwd()
131
132  # Regenerate the .dat file.
133  os.chdir(icu_build_dir)
134  subprocess.check_call(['make', '-j32'])
135
136  # Copy the .dat file to its ultimate destination.
137  icu_dat_data_dir = '%s/stubdata' % icu4cDir()
138  dat_file = datFile(icu_build_dir)
139
140  print('Copying %s to %s ...' % (dat_file, icu_dat_data_dir))
141  shutil.copy(dat_file, icu_dat_data_dir)
142
143  # Generate the ICU4J .jar files
144  subprocess.check_call(['make', '-j32', 'icu4j-data'])
145
146  # Generate the test data in icu4c/source/test/testdata/out
147  subprocess.check_call(['make', '-j32', 'tests'])
148
149  # Copy the ICU4J .jar files to their ultimate destination.
150  icu_jar_data_dir = '%s/main/shared/data' % icu4jDir()
151  jarfiles = glob.glob('data/out/icu4j/*.jar')
152  if len(jarfiles) != 3:
153    print('ERROR: Unexpectedly found %d .jar files (%s). Halting.' % (len(jarfiles), jarfiles))
154    sys.exit(1)
155  for jarfile in jarfiles:
156    icu_jarfile = os.path.join(icu_jar_data_dir, os.path.basename(jarfile))
157    if ziputil.ZipCompare(jarfile, icu_jarfile):
158      print('Ignoring %s which is identical to %s ...' % (jarfile, icu_jarfile))
159    else:
160      print('Copying %s to %s ...' % (jarfile, icu_jar_data_dir))
161      shutil.copy(jarfile, icu_jar_data_dir)
162
163  testdata_out_dir = '%s/test/testdata/out' % icu4cDir()
164  print('Copying test data to %s ' % testdata_out_dir)
165  if os.path.exists(testdata_out_dir):
166    shutil.rmtree(testdata_out_dir)
167  shutil.copytree('test/testdata/out', testdata_out_dir)
168
169  # Switch back to the original working cwd.
170  os.chdir(original_working_dir)
171
172
173def MakeAndCopyOverlayTzIcuData(icu_build_dir, dest_file):
174  """Makes a .dat file containing just time-zone data.
175
176  The overlay file can be used as an overlay of a full ICU .dat file
177  to provide newer time zone data. Some strings like translated
178  time zone names will be missing, but rules will be correct."""
179  # Keep track of the original cwd so we can go back to it at the end.
180  original_working_dir = os.getcwd()
181
182  # Regenerate the .res files.
183  os.chdir(icu_build_dir)
184  subprocess.check_call(['make', '-j32'])
185
186  # The list of ICU resources needed for time zone data overlays.
187  tz_res_names = [
188          'metaZones.res',
189          'timezoneTypes.res',
190          'windowsZones.res',
191          'zoneinfo64.res',
192  ]
193
194  dat_file = datFile(icu_build_dir)
195  icu_package_dat = os.path.basename(dat_file)
196  if not icu_package_dat.endswith('.dat'):
197      print('%s does not end with .dat' % icu_package_dat)
198      sys.exit(1)
199  icu_package = icu_package_dat[:-4]
200
201  # Create a staging directory to hold the files to go into the overlay .dat
202  res_staging_dir = '%s/overlay_res' % icu_build_dir
203  os.mkdir(res_staging_dir)
204
205  # Copy all the .res files we need from, e.g. ./data/out/build/icudt55l, to the staging directory
206  res_src_dir = '%s/data/out/build/%s' % (icu_build_dir, icu_package)
207  for tz_res_name in tz_res_names:
208    shutil.copy('%s/%s' % (res_src_dir, tz_res_name), res_staging_dir)
209
210  # Create a .lst file to pass to pkgdata.
211  tz_files_file = '%s/tzdata.lst' % res_staging_dir
212  with open(tz_files_file, "a") as tz_files:
213    for tz_res_name in tz_res_names:
214      tz_files.write('%s\n' % tz_res_name)
215
216  icu_lib_dir = '%s/lib' % icu_build_dir
217  pkg_data_bin = '%s/bin/pkgdata' % icu_build_dir
218
219  # Run pkgdata to create a .dat file.
220  icu_env = os.environ.copy()
221  icu_env["LD_LIBRARY_PATH"] = icu_lib_dir
222
223  # pkgdata treats the .lst file it is given as relative to CWD, and the path also affects the
224  # resource names in the .dat file produced so we change the CWD.
225  os.chdir(res_staging_dir)
226
227  # -F : force rebuilding all data
228  # -m common : create a .dat
229  # -v : verbose
230  # -T . : use "." as a temp dir
231  # -d . : use "." as the dest dir
232  # -p <name> : Set the "data name"
233  p = subprocess.Popen(
234      [pkg_data_bin, '-F', '-m', 'common', '-v', '-T', '.', '-d', '.', '-p',
235          icu_package, tz_files_file],
236      env=icu_env)
237  p.wait()
238  if p.returncode != 0:
239    print('pkgdata failed with status code: %s' % p.returncode)
240
241  # Copy the .dat to the chosen place / name.
242  generated_dat_file = '%s/%s' % (res_staging_dir, icu_package_dat)
243  shutil.copyfile(generated_dat_file, dest_file)
244  print('ICU overlay .dat can be found here: %s' % dest_file)
245
246  # Switch back to the original working cwd.
247  os.chdir(original_working_dir)
248
249def RequiredToMakeLangInfo():
250  """ Returns true if icu4c/source/data/misc/langInfo.txt has been re-generated.
251  Returns false if re-generation is not needed.
252  """
253
254  # Generate icu4c/source/data/misc/langInfo.txt by a ICU4J tool
255  langInfo_dst_path = os.path.join(icu4cDir(), 'data/misc/langInfo.txt')
256  print('Building %s' % langInfo_dst_path)
257  langInfo_out_path = '/tmp/langInfo.txt'  # path hardcoded in the LocaleDistanceBuilder tool
258  if os.path.exists(langInfo_out_path):
259    os.remove(langInfo_out_path)
260
261  icu4j_dir = icu4jDir()
262  os.chdir(icu4j_dir)
263  subprocess.check_call(['ant', 'icu4jJar'])
264  os.chdir(os.path.join(icu4j_dir, 'tools', 'misc'))
265  subprocess.check_call(['ant', 'jar'])
266  subprocess.check_call([
267    'java',
268    '-cp',
269    'out/lib/icu4j-tools.jar:../../icu4j.jar',
270    'com.ibm.icu.dev.tool.locale.LocaleDistanceBuilder',
271  ])
272  if (filecmp.cmp(langInfo_dst_path, langInfo_out_path)):
273    print('The files {src} and {dst} are the same'.format(src=langInfo_out_path, dst=langInfo_dst_path))
274    return False
275
276  print('Copying {src} to {dst}'.format(src=langInfo_out_path, dst=langInfo_dst_path))
277  shutil.copyfile(langInfo_out_path, langInfo_dst_path)
278  return True
279
280def CopyLicenseFiles(target_dir):
281  """Copies ICU license files to the target_dir"""
282
283  license_file = '%s/main/shared/licenses/LICENSE' % icu4jDir()
284  print('Copying %s to %s ...' % (license_file, target_dir))
285  shutil.copy(license_file, target_dir)
286
287