1#!/usr/bin/env python
2#
3# Copyright 2016 - The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16"""Base Cloud API Client.
17
18BasicCloudApiCliend does basic setup for a cloud API.
19"""
20import httplib
21import logging
22import socket
23import ssl
24
25import six
26
27# pylint: disable=import-error
28from apiclient import errors as gerrors
29from apiclient.discovery import build
30import apiclient.http
31import httplib2
32from oauth2client import client
33
34from acloud import errors
35from acloud.internal.lib import utils
36
37
38logger = logging.getLogger(__name__)
39
40
41class BaseCloudApiClient(object):
42    """A class that does basic setup for a cloud API."""
43
44    # To be overriden by subclasses.
45    API_NAME = ""
46    API_VERSION = "v1"
47    SCOPE = ""
48
49    # Defaults for retry.
50    RETRY_COUNT = 5
51    RETRY_BACKOFF_FACTOR = 1.5
52    RETRY_SLEEP_MULTIPLIER = 2
53    RETRY_HTTP_CODES = [
54        # 403 is to retry the "Rate Limit Exceeded" error.
55        # We could retry on a finer-grained error message later if necessary.
56        403,
57        500,  # Internal Server Error
58        502,  # Bad Gateway
59        503,  # Service Unavailable
60    ]
61    RETRIABLE_ERRORS = (httplib.HTTPException, httplib2.HttpLib2Error,
62                        socket.error, ssl.SSLError)
63    RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, )
64
65    def __init__(self, oauth2_credentials):
66        """Initialize.
67
68        Args:
69            oauth2_credentials: An oauth2client.OAuth2Credentials instance.
70        """
71        self._service = self.InitResourceHandle(oauth2_credentials)
72
73    @classmethod
74    def InitResourceHandle(cls, oauth2_credentials):
75        """Authenticate and initialize a Resource object.
76
77        Authenticate http and create a Resource object with methods
78        for interacting with the service.
79
80        Args:
81            oauth2_credentials: An oauth2client.OAuth2Credentials instance.
82
83        Returns:
84            An apiclient.discovery.Resource object
85        """
86        http_auth = oauth2_credentials.authorize(httplib2.Http())
87        return utils.RetryExceptionType(
88            exception_types=cls.RETRIABLE_AUTH_ERRORS,
89            max_retries=cls.RETRY_COUNT,
90            functor=build,
91            sleep_multiplier=cls.RETRY_SLEEP_MULTIPLIER,
92            retry_backoff_factor=cls.RETRY_BACKOFF_FACTOR,
93            serviceName=cls.API_NAME,
94            version=cls.API_VERSION,
95            # This is workaround for a known issue of some veriosn
96            # of api client.
97            # https://github.com/google/google-api-python-client/issues/435
98            cache_discovery=False,
99            http=http_auth)
100
101    @staticmethod
102    def _ShouldRetry(exception, retry_http_codes,
103                     other_retriable_errors):
104        """Check if exception is retriable.
105
106        Args:
107            exception: An instance of Exception.
108            retry_http_codes: a list of integers, retriable HTTP codes of
109                              HttpError
110            other_retriable_errors: a tuple of error types to retry other than
111                                    HttpError.
112
113        Returns:
114            Boolean, True if retriable, False otherwise.
115        """
116        if isinstance(exception, other_retriable_errors):
117            return True
118
119        if isinstance(exception, errors.HttpError):
120            if exception.code in retry_http_codes:
121                return True
122            logger.debug("_ShouldRetry: Exception code %s not in %s: %s",
123                         exception.code, retry_http_codes, str(exception))
124
125        logger.debug("_ShouldRetry: Exception %s is not one of %s: %s",
126                     type(exception),
127                     list(other_retriable_errors) + [errors.HttpError],
128                     str(exception))
129        return False
130
131    @staticmethod
132    def _TranslateError(exception):
133        """Translate the exception to a desired type.
134
135        Args:
136            exception: An instance of Exception.
137
138        Returns:
139            gerrors.HttpError will be translated to errors.HttpError.
140            If the error code is errors.HTTP_NOT_FOUND_CODE, it will
141            be translated to errors.ResourceNotFoundError.
142            Unrecognized error type will not be translated and will
143            be returned as is.
144        """
145        if isinstance(exception, gerrors.HttpError):
146            exception = errors.HttpError.CreateFromHttpError(exception)
147            if exception.code == errors.HTTP_NOT_FOUND_CODE:
148                exception = errors.ResourceNotFoundError(
149                    exception.code, str(exception))
150        return exception
151
152    def ExecuteOnce(self, api):
153        """Execute an api and parse the errors.
154
155        Args:
156            api: An apiclient.http.HttpRequest, representing the api to execute.
157
158        Returns:
159            Execution result of the api.
160
161        Raises:
162            errors.ResourceNotFoundError: For 404 error.
163            errors.HttpError: For other types of http error.
164        """
165        try:
166            return api.execute()
167        except gerrors.HttpError as e:
168            raise self._TranslateError(e)
169
170    def Execute(self,
171                api,
172                retry_http_codes=None,
173                max_retry=None,
174                sleep=None,
175                backoff_factor=None,
176                other_retriable_errors=None):
177        """Execute an api with retry.
178
179        Call ExecuteOnce and retry on http error with given codes.
180
181        Args:
182            api: An apiclient.http.HttpRequest, representing the api to execute:
183            retry_http_codes: A list of http codes to retry.
184            max_retry: See utils.Retry.
185            sleep: See utils.Retry.
186            backoff_factor: See utils.Retry.
187            other_retriable_errors: A tuple of error types that should be retried
188                                    other than errors.HttpError.
189
190        Returns:
191          Execution result of the api.
192
193        Raises:
194          See ExecuteOnce.
195        """
196        retry_http_codes = (self.RETRY_HTTP_CODES
197                            if retry_http_codes is None else retry_http_codes)
198        max_retry = (self.RETRY_COUNT if max_retry is None else max_retry)
199        sleep = (self.RETRY_SLEEP_MULTIPLIER if sleep is None else sleep)
200        backoff_factor = (self.RETRY_BACKOFF_FACTOR
201                          if backoff_factor is None else backoff_factor)
202        other_retriable_errors = (self.RETRIABLE_ERRORS
203                                  if other_retriable_errors is None else
204                                  other_retriable_errors)
205
206        def _Handler(exc):
207            """Check if |exc| is a retriable exception.
208
209            Args:
210                exc: An exception.
211
212            Returns:
213                True if exc is an errors.HttpError and code exists in |retry_http_codes|
214                False otherwise.
215            """
216            if self._ShouldRetry(exc, retry_http_codes,
217                                 other_retriable_errors):
218                logger.debug("Will retry error: %s", str(exc))
219                return True
220            return False
221
222        return utils.Retry(
223            _Handler,
224            max_retries=max_retry,
225            functor=self.ExecuteOnce,
226            sleep_multiplier=sleep,
227            retry_backoff_factor=backoff_factor,
228            api=api)
229
230    def BatchExecuteOnce(self, requests):
231        """Execute requests in a batch.
232
233        Args:
234            requests: A dictionary where key is request id and value
235                      is an http request.
236
237        Returns:
238            results, a dictionary in the following format
239            {request_id: (response, exception)}
240            request_ids are those from requests; response
241            is the http response for the request or None on error;
242            exception is an instance of DriverError or None if no error.
243        """
244        results = {}
245
246        def _CallBack(request_id, response, exception):
247            results[request_id] = (response, self._TranslateError(exception))
248
249        batch = apiclient.http.BatchHttpRequest()
250        for request_id, request in six.iteritems(requests):
251            batch.add(
252                request=request, callback=_CallBack, request_id=request_id)
253        batch.execute()
254        return results
255
256    def BatchExecute(self,
257                     requests,
258                     retry_http_codes=None,
259                     max_retry=None,
260                     sleep=None,
261                     backoff_factor=None,
262                     other_retriable_errors=None):
263        """Batch execute multiple requests with retry.
264
265        Call BatchExecuteOnce and retry on http error with given codes.
266
267        Args:
268            requests: A dictionary where key is request id picked by caller,
269                      and value is a apiclient.http.HttpRequest.
270            retry_http_codes: A list of http codes to retry.
271            max_retry: See utils.Retry.
272            sleep: See utils.Retry.
273            backoff_factor: See utils.Retry.
274            other_retriable_errors: A tuple of error types that should be retried
275                                    other than errors.HttpError.
276
277        Returns:
278            results, a dictionary in the following format
279            {request_id: (response, exception)}
280            request_ids are those from requests; response
281            is the http response for the request or None on error;
282            exception is an instance of DriverError or None if no error.
283        """
284        executor = utils.BatchHttpRequestExecutor(
285            self.BatchExecuteOnce,
286            requests=requests,
287            retry_http_codes=retry_http_codes or self.RETRY_HTTP_CODES,
288            max_retry=max_retry or self.RETRY_COUNT,
289            sleep=sleep or self.RETRY_SLEEP_MULTIPLIER,
290            backoff_factor=backoff_factor or self.RETRY_BACKOFF_FACTOR,
291            other_retriable_errors=other_retriable_errors
292            or self.RETRIABLE_ERRORS)
293        executor.Execute()
294        return executor.GetResults()
295
296    def ListWithMultiPages(self, api_resource, *args, **kwargs):
297        """Call an api that list a type of resource.
298
299        Multiple google services support listing a type of
300        resource (e.g list gce instances, list storage objects).
301        The querying pattern is similar --
302        Step 1: execute the api and get a response object like,
303        {
304            "items": [..list of resource..],
305            # The continuation token that can be used
306            # to get the next page.
307            "nextPageToken": "A String",
308        }
309        Step 2: execute the api again with the nextPageToken to
310        retrieve more pages and get a response object.
311
312        Step 3: Repeat Step 2 until no more page.
313
314        This method encapsulates the generic logic of
315        calling such listing api.
316
317        Args:
318            api_resource: An apiclient.discovery.Resource object
319                used to create an http request for the listing api.
320            *args: Arguments used to create the http request.
321            **kwargs: Keyword based arguments to create the http
322                      request.
323
324        Returns:
325            A list of items.
326        """
327        items = []
328        next_page_token = None
329        while True:
330            api = api_resource(pageToken=next_page_token, *args, **kwargs)
331            response = self.Execute(api)
332            items.extend(response.get("items", []))
333            next_page_token = response.get("nextPageToken")
334            if not next_page_token:
335                break
336        return items
337
338    @property
339    def service(self):
340        """Return self._service as a property."""
341        return self._service
342