1#
2# Copyright (C) 2016 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16"""Class to fetch artifacts from internal build server.
17"""
18
19import googleapiclient
20import httplib2
21import io
22import json
23import logging
24import re
25import time
26from googleapiclient.discovery import build
27from oauth2client import client as oauth2_client
28from oauth2client.service_account import ServiceAccountCredentials
29
30logger = logging.getLogger('artifact_fetcher')
31
32
33class DriverError(Exception):
34    """Base Android GCE driver exception."""
35
36
37class AndroidBuildClient(object):
38    """Client that manages Android Build.
39
40    Attributes:
41        service: object, initialized and authorized service object for the
42                 androidbuildinternal API.
43        API_NAME: string, name of internal API accessed by the client.
44        API_VERSION: string, version of the internal API accessed by the client.
45        SCOPE: string, URL for which to request access via oauth2.
46        DEFAULT_RESOURCE_ID: string, default artifact name to request.
47        DEFAULT_ATTEMPT_ID: string, default attempt to request for the artifact.
48        DEFAULT_CHUNK_SIZE: int, number of bytes to download at a time.
49        RETRY_COUNT: int, max number of retries.
50        RETRY_DELAY_IN_SECS: int, time delays between retries in seconds.
51    """
52
53    API_NAME = "androidbuildinternal"
54    API_VERSION = "v2beta1"
55    SCOPE = "https://www.googleapis.com/auth/androidbuild.internal"
56
57    # other variables.
58    BUILDS_KEY = "builds"
59    BUILD_ID_KEY = "buildId"
60    DEFAULT_ATTEMPT_ID = "latest"
61    DEFAULT_BUILD_ATTEMPT_STATUS = "complete"
62    DEFAULT_BUILD_TYPE = "submitted"
63    DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024
64    DEFAULT_RESOURCE_ID = "0"
65
66    # Defaults for retry.
67    RETRY_COUNT = 5
68    RETRY_DELAY_IN_SECS = 3
69
70    def __init__(self, oauth2_service_json):
71        """Initialize.
72
73        Args:
74          oauth2_service_json: Path to service account json file.
75        """
76        authToken = ServiceAccountCredentials.from_json_keyfile_name(
77            oauth2_service_json, [self.SCOPE])
78        http_auth = authToken.authorize(httplib2.Http())
79        for _ in xrange(self.RETRY_COUNT):
80            try:
81                self.service = build(
82                    serviceName=self.API_NAME,
83                    version=self.API_VERSION,
84                    http=http_auth)
85                break
86            except oauth2_client.AccessTokenRefreshError as e:
87                # The following HTTP code typically indicates transient errors:
88                #    500  (Internal Server Error)
89                #    502  (Bad Gateway)
90                #    503  (Service Unavailable)
91                logging.exception(e)
92                logging.info("Retrying to connect to %s", self.API_NAME)
93                time.sleep(self.RETRY_DELAY_IN_SECS)
94
95    def DownloadArtifactToFile(self,
96                               branch,
97                               build_target,
98                               build_id,
99                               resource_id,
100                               dest_filepath,
101                               attempt_id=None):
102        """Get artifact from android build server.
103
104        Args:
105            branch: Branch from which the code was built, e.g. "master"
106            build_target: Target name, e.g. "gce_x86-userdebug"
107            build_id: Build id, a string, e.g. "2263051", "P2804227"
108            resource_id: Name of resource to be downloaded, a string.
109            attempt_id: string, attempt id, will default to DEFAULT_ATTEMPT_ID.
110            dest_filepath: string, set a file path to store to a file.
111
112        Returns:
113            Contents of the requested resource as a string if dest_filepath is None;
114            None otherwise.
115        """
116        return self.GetArtifact(branch, build_target, build_id, resource_id,
117                                attempt_id=attempt_id, dest_filepath=dest_filepath)
118
119    def GetArtifact(self,
120                    branch,
121                    build_target,
122                    build_id,
123                    resource_id,
124                    attempt_id=None,
125                    dest_filepath=None):
126        """Get artifact from android build server.
127
128        Args:
129            branch: Branch from which the code was built, e.g. "master"
130            build_target: Target name, e.g. "gce_x86-userdebug"
131            build_id: Build id, a string, e.g. "2263051", "P2804227"
132            resource_id: Name of resource to be downloaded, a string.
133            attempt_id: string, attempt id, will default to DEFAULT_ATTEMPT_ID.
134            dest_filepath: string, set a file path to store to a file.
135
136        Returns:
137            Contents of the requested resource as a string if dest_filepath is None;
138            None otherwise.
139        """
140        attempt_id = attempt_id or self.DEFAULT_ATTEMPT_ID
141        api = self.service.buildartifact().get_media(
142            buildId=build_id,
143            target=build_target,
144            attemptId=attempt_id,
145            resourceId=resource_id)
146        logger.info("Downloading artifact: target: %s, build_id: %s, "
147                    "resource_id: %s", build_target, build_id, resource_id)
148        fh = None
149        try:
150            if dest_filepath:
151                fh = io.FileIO(dest_filepath, mode='wb')
152            else:
153                fh = io.BytesIO()
154
155            downloader = googleapiclient.http.MediaIoBaseDownload(
156                fh, api, chunksize=self.DEFAULT_CHUNK_SIZE)
157            done = False
158            while not done:
159                _, done = downloader.next_chunk()
160            logger.info("Downloaded artifact %s" % resource_id)
161
162            if not dest_filepath:
163                return fh.getvalue()
164        except OSError as e:
165            logger.error("Downloading artifact failed: %s", str(e))
166            raise DriverError(str(e))
167        finally:
168            if fh:
169                fh.close()
170
171    def GetManifest(self, branch, build_target, build_id, attempt_id=None):
172        """Get Android build manifest XML file.
173
174        Args:
175            branch: Branch from which the code was built, e.g. "master"
176            build_target: Target name, e.g. "gce_x86-userdebug"
177            build_id: Build id, a string, e.g. "2263051", "P2804227"
178            attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID.
179
180
181        Returns:
182            Contents of the requested XML file as a string.
183        """
184        resource_id = "manifest_%s.xml" % build_id
185        return self.GetArtifact(branch, build_target, build_id, resource_id,
186                                attempt_id)
187
188    def GetRepoDictionary(self,
189                          branch,
190                          build_target,
191                          build_id,
192                          attempt_id=None):
193        """Get dictionary of repositories and git revision IDs
194
195        Args:
196            branch: Branch from which the code was built, e.g. "master"
197            build_target: Target name, e.g. "gce_x86-userdebug"
198            build_id: Build id, a string, e.g. "2263051", "P2804227"
199            attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID.
200
201
202        Returns:
203            Dictionary of project names (string) to commit ID (string)
204        """
205        resource_id = "BUILD_INFO"
206        build_info = self.GetArtifact(branch, build_target, build_id,
207                                      resource_id, attempt_id)
208        try:
209            return json.loads(build_info)["repo-dict"]
210        except (ValueError, KeyError):
211            logger.warn("Could not find repo dictionary.")
212            return {}
213
214    def GetCoverage(self,
215                    branch,
216                    build_target,
217                    build_id,
218                    product,
219                    attempt_id=None):
220        """Get Android build coverage zip file.
221
222        Args:
223            branch: Branch from which the code was built, e.g. "master"
224            build_target: Target name, e.g. "gce_x86-userdebug"
225            build_id: Build id, a string, e.g. "2263051", "P2804227"
226            product: Name of product for build target, e.g. "bullhead", "angler"
227            attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID.
228
229
230        Returns:
231            Contents of the requested zip file as a string.
232        """
233        resource_id = ("%s-coverage-%s.zip" % (product, build_id))
234        return self.GetArtifact(branch, build_target, build_id, resource_id,
235                                attempt_id)
236
237    def ListBuildIds(self,
238                     branch,
239                     build_target,
240                     limit=1,
241                     build_type=DEFAULT_BUILD_TYPE,
242                     build_attempt_status=DEFAULT_BUILD_ATTEMPT_STATUS):
243        """Get a list of most recent build IDs.
244
245        Args:
246            branch: Branch from which the code was built, e.g. "master"
247            build_target: Target name, e.g. "gce_x86-userdebug"
248            limit: (optional) an int, max number of build IDs to fetch,
249                default of 1
250            build_type: (optional) a string, the build type to filter, default
251                of "submitted"
252            build_attempt_status: (optional) a string, the build attempt status
253                to filter, default of "completed"
254
255        Returns:
256            A list of build ID strings in reverse time order.
257        """
258        builds = self.service.build().list(
259            branch=branch,
260            target=build_target,
261            maxResults=limit,
262            buildType=build_type,
263            buildAttemptStatus=build_attempt_status).execute()
264        return [str(build.get(self.BUILD_ID_KEY))
265                for build in builds.get(self.BUILDS_KEY)]
266