1#!/usr/bin/env python
2#
3# Copyright 2016 - The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""A client that talks to Google Cloud Storage APIs."""
18
19import io
20import logging
21import os
22
23import apiclient
24
25from acloud.internal.lib import base_cloud_client
26from acloud.internal.lib import utils
27from acloud.public import errors
28
29logger = logging.getLogger(__name__)
30
31
32class StorageClient(base_cloud_client.BaseCloudApiClient):
33    """Client that talks to Google Cloud Storages."""
34
35    # API settings, used by BaseCloudApiClient.
36    API_NAME = "storage"
37    API_VERSION = "v1"
38    SCOPE = "https://www.googleapis.com/auth/devstorage.read_write"
39    GET_OBJ_MAX_RETRY = 3
40    GET_OBJ_RETRY_SLEEP = 5
41
42    # Other class variables.
43    OBJECT_URL_FMT = "https://storage.googleapis.com/%s/%s"
44
45    def Get(self, bucket_name, object_name):
46        """Get object in a bucket.
47
48        Args:
49            bucket_name: String, google cloud storage bucket name.
50            object_name: String, full path to the object within the bucket.
51
52        Returns:
53            A dictronary representing an object resource.
54        """
55        request = self.service.objects().get(bucket=bucket_name,
56                                             object=object_name)
57        return self.Execute(request)
58
59    def List(self, bucket_name, prefix=None):
60        """Lists objects in a bucket.
61
62        Args:
63            bucket_name: String, google cloud storage bucket name.
64            prefix: String, Filter results to objects whose names begin with
65                    this prefix.
66
67        Returns:
68            A list of google storage objects whose names match the prefix.
69            Each element is dictionary that wraps all the information about an object.
70        """
71        logger.debug("Listing storage bucket: %s, prefix: %s", bucket_name,
72                     prefix)
73        items = self.ListWithMultiPages(
74            api_resource=self.service.objects().list,
75            bucket=bucket_name,
76            prefix=prefix)
77        return items
78
79    def Upload(self, local_src, bucket_name, object_name, mime_type):
80        """Uploads a file.
81
82        Args:
83            local_src: string, a local path to a file to be uploaded.
84            bucket_name: string, google cloud storage bucket name.
85            object_name: string, the name of the remote file in storage.
86            mime_type: string, mime-type of the file.
87
88        Returns:
89            URL to the inserted artifact in storage.
90        """
91        logger.info("Uploading file: src: %s, bucket: %s, object: %s",
92                    local_src, bucket_name, object_name)
93        try:
94            with io.FileIO(local_src, mode="rb") as fh:
95                media = apiclient.http.MediaIoBaseUpload(fh, mime_type)
96                request = self.service.objects().insert(bucket=bucket_name,
97                                                        name=object_name,
98                                                        media_body=media)
99                response = self.Execute(request)
100            logger.info("Uploaded artifact: %s", response["selfLink"])
101            return response
102        except OSError as e:
103            logger.error("Uploading artifact fails: %s", str(e))
104            raise errors.DriverError(str(e))
105
106    def Delete(self, bucket_name, object_name):
107        """Deletes a file.
108
109        Args:
110            bucket_name: string, google cloud storage bucket name.
111            object_name: string, the name of the remote file in storage.
112        """
113        logger.info("Deleting file: bucket: %s, object: %s", bucket_name,
114                    object_name)
115        request = self.service.objects().delete(bucket=bucket_name,
116                                                object=object_name)
117        self.Execute(request)
118        logger.info("Deleted file: bucket: %s, object: %s", bucket_name,
119                    object_name)
120
121    def DeleteFiles(self, bucket_name, object_names):
122        """Deletes multiple files.
123
124        Args:
125            bucket_name: string, google cloud storage bucket name.
126            object_names: A list of strings, each of which is a name of a remote file.
127
128        Returns:
129            A tuple, (deleted, failed, error_msgs)
130            deleted: A list of names of objects that have been deleted.
131            faild: A list of names of objects that we fail to delete.
132            error_msgs: A list of failure messages.
133        """
134        deleted = []
135        failed = []
136        error_msgs = []
137        for object_name in object_names:
138            try:
139                self.Delete(bucket_name, object_name)
140                deleted.append(object_name)
141            except errors.DriverError as e:
142                failed.append(object_name)
143                error_msgs.append(str(e))
144        return deleted, failed, error_msgs
145
146    def GetUrl(self, bucket_name, object_name):
147        """Get information about a file object.
148
149        Args:
150            bucket_name: string, google cloud storage bucket name.
151            object_name: string, name of the file to look for.
152
153        Returns:
154            Value of "selfLink" field from the response, which represents
155            a url to the file.
156
157        Raises:
158            errors.ResourceNotFoundError: when file is not found.
159        """
160        item = utils.RetryExceptionType(
161                errors.ResourceNotFoundError,
162                max_retries=self.GET_OBJ_MAX_RETRY, functor=self.Get,
163                sleep_multiplier=self.GET_OBJ_RETRY_SLEEP,
164                bucket_name=bucket_name, object_name=object_name)
165        return item["selfLink"]
166