1#!/usr/bin/env python
2#
3# Copyright 2016 - The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Base Cloud API Client.
18
19BasicCloudApiCliend does basic setup for a cloud API.
20"""
21import httplib
22import logging
23import os
24import socket
25import ssl
26
27from apiclient import errors as gerrors
28from apiclient.discovery import build
29import apiclient.http
30import httplib2
31from oauth2client import client
32
33from acloud.internal.lib import utils
34from acloud.public import errors
35
36logger = logging.getLogger(__name__)
37
38
39class BaseCloudApiClient(object):
40    """A class that does basic setup for a cloud API."""
41
42    # To be overriden by subclasses.
43    API_NAME = ""
44    API_VERSION = "v1"
45    SCOPE = ""
46
47    # Defaults for retry.
48    RETRY_COUNT = 5
49    RETRY_BACKOFF_FACTOR = 1.5
50    RETRY_SLEEP_MULTIPLIER = 2
51    RETRY_HTTP_CODES = [
52        # 403 is to retry the "Rate Limit Exceeded" error.
53        # We could retry on a finer-grained error message later if necessary.
54        403,
55        500,  # Internal Server Error
56        502,  # Bad Gateway
57        503,  # Service Unavailable
58    ]
59    RETRIABLE_ERRORS = (httplib.HTTPException, httplib2.HttpLib2Error,
60                        socket.error, ssl.SSLError)
61    RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, )
62
63    def __init__(self, oauth2_credentials):
64        """Initialize.
65
66        Args:
67            oauth2_credentials: An oauth2client.OAuth2Credentials instance.
68        """
69        self._service = self.InitResourceHandle(oauth2_credentials)
70
71    @classmethod
72    def InitResourceHandle(cls, oauth2_credentials):
73        """Authenticate and initialize a Resource object.
74
75        Authenticate http and create a Resource object with methods
76        for interacting with the service.
77
78        Args:
79            oauth2_credentials: An oauth2client.OAuth2Credentials instance.
80
81        Returns:
82            An apiclient.discovery.Resource object
83        """
84        http_auth = oauth2_credentials.authorize(httplib2.Http())
85        return utils.RetryExceptionType(
86                exception_types=cls.RETRIABLE_AUTH_ERRORS,
87                max_retries=cls.RETRY_COUNT,
88                functor=build,
89                sleep_multiplier=cls.RETRY_SLEEP_MULTIPLIER,
90                retry_backoff_factor=cls.RETRY_BACKOFF_FACTOR,
91                serviceName=cls.API_NAME,
92                version=cls.API_VERSION,
93                http=http_auth)
94
95    def _ShouldRetry(self, exception, retry_http_codes,
96                     other_retriable_errors):
97        """Check if exception is retriable.
98
99        Args:
100            exception: An instance of Exception.
101            retry_http_codes: a list of integers, retriable HTTP codes of
102                              HttpError
103            other_retriable_errors: a tuple of error types to retry other than
104                                    HttpError.
105
106        Returns:
107            Boolean, True if retriable, False otherwise.
108        """
109        if isinstance(exception, other_retriable_errors):
110            return True
111
112        if isinstance(exception, errors.HttpError):
113            if exception.code in retry_http_codes:
114                return True
115            else:
116                logger.debug("_ShouldRetry: Exception code %s not in %s: %s",
117                             exception.code, retry_http_codes, str(exception))
118
119        logger.debug(
120            "_ShouldRetry: Exception %s is not one of %s: %s", type(exception),
121            list(other_retriable_errors) + [errors.HttpError], str(exception))
122        return False
123
124    def _TranslateError(self, exception):
125        """Translate the exception to a desired type.
126
127        Args:
128            exception: An instance of Exception.
129
130        Returns:
131            gerrors.HttpError will be translated to errors.HttpError.
132            If the error code is errors.HTTP_NOT_FOUND_CODE, it will
133            be translated to errors.ResourceNotFoundError.
134            Unrecognized error type will not be translated and will
135            be returned as is.
136        """
137        if isinstance(exception, gerrors.HttpError):
138            exception = errors.HttpError.CreateFromHttpError(exception)
139            if exception.code == errors.HTTP_NOT_FOUND_CODE:
140                exception = errors.ResourceNotFoundError(exception.code,
141                                                         str(exception))
142        return exception
143
144    def ExecuteOnce(self, api):
145        """Execute an api and parse the errors.
146
147        Args:
148            api: An apiclient.http.HttpRequest, representing the api to execute.
149
150        Returns:
151            Execution result of the api.
152
153        Raises:
154            errors.ResourceNotFoundError: For 404 error.
155            errors.HttpError: For other types of http error.
156        """
157        try:
158            return api.execute()
159        except gerrors.HttpError as e:
160            raise self._TranslateError(e)
161
162    def Execute(self,
163                api,
164                retry_http_codes=None,
165                max_retry=None,
166                sleep=None,
167                backoff_factor=None,
168                other_retriable_errors=None):
169        """Execute an api with retry.
170
171        Call ExecuteOnce and retry on http error with given codes.
172
173        Args:
174            api: An apiclient.http.HttpRequest, representing the api to execute:
175            retry_http_codes: A list of http codes to retry.
176            max_retry: See utils.Retry.
177            sleep: See utils.Retry.
178            backoff_factor: See utils.Retry.
179            other_retriable_errors: A tuple of error types that should be retried
180                                    other than errors.HttpError.
181
182        Returns:
183          Execution result of the api.
184
185        Raises:
186          See ExecuteOnce.
187        """
188        retry_http_codes = (self.RETRY_HTTP_CODES if retry_http_codes is None
189                            else retry_http_codes)
190        max_retry = (self.RETRY_COUNT if max_retry is None else max_retry)
191        sleep = (self.RETRY_SLEEP_MULTIPLIER if sleep is None else sleep)
192        backoff_factor = (self.RETRY_BACKOFF_FACTOR if backoff_factor is None
193                          else backoff_factor)
194        other_retriable_errors = (self.RETRIABLE_ERRORS
195                                  if other_retriable_errors is None else
196                                  other_retriable_errors)
197
198        def _Handler(exc):
199            """Check if |exc| is a retriable exception.
200
201            Args:
202                exc: An exception.
203
204            Returns:
205                True if exc is an errors.HttpError and code exists in |retry_http_codes|
206                False otherwise.
207            """
208            if self._ShouldRetry(exc, retry_http_codes,
209                                 other_retriable_errors):
210                logger.debug("Will retry error: %s", str(exc))
211                return True
212            return False
213
214        return utils.Retry(
215             _Handler, max_retries=max_retry, functor=self.ExecuteOnce,
216             sleep_multiplier=sleep, retry_backoff_factor=backoff_factor,
217             api=api)
218
219    def BatchExecuteOnce(self, requests):
220        """Execute requests in a batch.
221
222        Args:
223            requests: A dictionary where key is request id and value
224                      is an http request.
225
226        Returns:
227            results, a dictionary in the following format
228            {request_id: (response, exception)}
229            request_ids are those from requests; response
230            is the http response for the request or None on error;
231            exception is an instance of DriverError or None if no error.
232        """
233        results = {}
234
235        def _CallBack(request_id, response, exception):
236            results[request_id] = (response, self._TranslateError(exception))
237
238        batch = apiclient.http.BatchHttpRequest()
239        for request_id, request in requests.iteritems():
240            batch.add(request=request,
241                      callback=_CallBack,
242                      request_id=request_id)
243        batch.execute()
244        return results
245
246    def BatchExecute(self,
247                     requests,
248                     retry_http_codes=None,
249                     max_retry=None,
250                     sleep=None,
251                     backoff_factor=None,
252                     other_retriable_errors=None):
253        """Batch execute multiple requests with retry.
254
255        Call BatchExecuteOnce and retry on http error with given codes.
256
257        Args:
258            requests: A dictionary where key is request id picked by caller,
259                      and value is a apiclient.http.HttpRequest.
260            retry_http_codes: A list of http codes to retry.
261            max_retry: See utils.Retry.
262            sleep: See utils.Retry.
263            backoff_factor: See utils.Retry.
264            other_retriable_errors: A tuple of error types that should be retried
265                                    other than errors.HttpError.
266
267        Returns:
268            results, a dictionary in the following format
269            {request_id: (response, exception)}
270            request_ids are those from requests; response
271            is the http response for the request or None on error;
272            exception is an instance of DriverError or None if no error.
273        """
274        executor = utils.BatchHttpRequestExecutor(
275            self.BatchExecuteOnce,
276            requests=requests,
277            retry_http_codes=retry_http_codes or self.RETRY_HTTP_CODES,
278            max_retry=max_retry or self.RETRY_COUNT,
279            sleep=sleep or self.RETRY_SLEEP_MULTIPLIER,
280            backoff_factor=backoff_factor or self.RETRY_BACKOFF_FACTOR,
281            other_retriable_errors=other_retriable_errors or
282            self.RETRIABLE_ERRORS)
283        executor.Execute()
284        return executor.GetResults()
285
286    def ListWithMultiPages(self, api_resource, *args, **kwargs):
287        """Call an api that list a type of resource.
288
289        Multiple google services support listing a type of
290        resource (e.g list gce instances, list storage objects).
291        The querying pattern is similar --
292        Step 1: execute the api and get a response object like,
293        {
294            "items": [..list of resource..],
295            # The continuation token that can be used
296            # to get the next page.
297            "nextPageToken": "A String",
298        }
299        Step 2: execute the api again with the nextPageToken to
300        retrieve more pages and get a response object.
301
302        Step 3: Repeat Step 2 until no more page.
303
304        This method encapsulates the generic logic of
305        calling such listing api.
306
307        Args:
308            api_resource: An apiclient.discovery.Resource object
309                used to create an http request for the listing api.
310            *args: Arguments used to create the http request.
311            **kwargs: Keyword based arguments to create the http
312                      request.
313
314        Returns:
315            A list of items.
316        """
317        items = []
318        next_page_token = None
319        while True:
320            api = api_resource(pageToken=next_page_token, *args, **kwargs)
321            response = self.Execute(api)
322            items.extend(response.get("items", []))
323            next_page_token = response.get("nextPageToken")
324            if not next_page_token:
325                break
326        return items
327
328    @property
329    def service(self):
330        """Return self._service as a property."""
331        return self._service
332