1# Copyright 2019 - The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""RemoteInstanceDeviceFactory provides basic interface to create a cuttlefish
16device factory."""
17
18import glob
19import logging
20import os
21import shutil
22import tempfile
23
24from acloud import errors
25from acloud.create import create_common
26from acloud.internal import constants
27from acloud.internal.lib import auth
28from acloud.internal.lib import cvd_compute_client_multi_stage
29from acloud.internal.lib import utils
30from acloud.internal.lib import ssh
31from acloud.public.actions import base_device_factory
32
33
34logger = logging.getLogger(__name__)
35
36_USER_BUILD = "userbuild"
37
38
39class RemoteInstanceDeviceFactory(base_device_factory.BaseDeviceFactory):
40    """A class that can produce a cuttlefish device.
41
42    Attributes:
43        avd_spec: AVDSpec object that tells us what we're going to create.
44        cfg: An AcloudConfig instance.
45        local_image_artifact: A string, path to local image.
46        cvd_host_package_artifact: A string, path to cvd host package.
47        report_internal_ip: Boolean, True for the internal ip is used when
48                            connecting from another GCE instance.
49        credentials: An oauth2client.OAuth2Credentials instance.
50        compute_client: An object of cvd_compute_client.CvdComputeClient.
51        ssh: An Ssh object.
52    """
53    def __init__(self, avd_spec, local_image_artifact=None,
54                 cvd_host_package_artifact=None):
55        """Constructs a new remote instance device factory."""
56        self._avd_spec = avd_spec
57        self._cfg = avd_spec.cfg
58        self._local_image_artifact = local_image_artifact
59        self._cvd_host_package_artifact = cvd_host_package_artifact
60        self._report_internal_ip = avd_spec.report_internal_ip
61        self.credentials = auth.CreateCredentials(avd_spec.cfg)
62        # Control compute_client with enable_multi_stage
63        compute_client = cvd_compute_client_multi_stage.CvdComputeClient(
64            acloud_config=avd_spec.cfg,
65            oauth2_credentials=self.credentials,
66            ins_timeout_secs=avd_spec.ins_timeout_secs,
67            report_internal_ip=avd_spec.report_internal_ip,
68            gpu=avd_spec.gpu)
69        super(RemoteInstanceDeviceFactory, self).__init__(compute_client)
70        self._ssh = None
71
72    def CreateInstance(self):
73        """Create a single configured cuttlefish device.
74
75        GCE:
76        1. Create gcp instance.
77        2. Upload local built artifacts to remote instance or fetch build on
78           remote instance.
79        3. Launch CVD.
80
81        Remote host:
82        1. Init remote host.
83        2. Download the artifacts to local and upload the artifacts to host
84        3. Launch CVD.
85
86        Returns:
87            A string, representing instance name.
88        """
89        if self._avd_spec.instance_type == constants.INSTANCE_TYPE_HOST:
90            instance = self._InitRemotehost()
91            self._ProcessRemoteHostArtifacts()
92            self._LaunchCvd(instance=instance,
93                            decompress_kernel=True,
94                            boot_timeout_secs=self._avd_spec.boot_timeout_secs)
95        else:
96            instance = self._CreateGceInstance()
97            # If instance is failed, no need to go next step.
98            if instance in self.GetFailures():
99                return instance
100            try:
101                self._ProcessArtifacts(self._avd_spec.image_source)
102                self._LaunchCvd(instance=instance,
103                                boot_timeout_secs=self._avd_spec.boot_timeout_secs)
104            except errors.DeviceConnectionError as e:
105                self._SetFailures(instance, e)
106
107        return instance
108
109    def _InitRemotehost(self):
110        """Initialize remote host.
111
112        Determine the remote host instance name, and activate ssh. It need to
113        get the IP address in the common_operation. So need to pass the IP and
114        ssh to compute_client.
115
116        build_target: The format is like "aosp_cf_x86_phone". We only get info
117                      from the user build image file name. If the file name is
118                      not custom format (no "-"), we will use $TARGET_PRODUCT
119                      from environment variable as build_target.
120
121        Returns:
122            A string, representing instance name.
123        """
124        image_name = os.path.basename(
125            self._local_image_artifact) if self._local_image_artifact else ""
126        build_target = (os.environ.get(constants.ENV_BUILD_TARGET) if "-" not
127                        in image_name else image_name.split("-")[0])
128        build_id = _USER_BUILD
129        if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE:
130            build_id = self._avd_spec.remote_image[constants.BUILD_ID]
131
132        instance = "%s-%s-%s-%s" % (constants.INSTANCE_TYPE_HOST,
133                                    self._avd_spec.remote_host,
134                                    build_id, build_target)
135        ip = ssh.IP(ip=self._avd_spec.remote_host)
136        self._ssh = ssh.Ssh(
137            ip=ip,
138            user=self._avd_spec.host_user,
139            ssh_private_key_path=(self._avd_spec.host_ssh_private_key_path or
140                                  self._cfg.ssh_private_key_path),
141            extra_args_ssh_tunnel=self._cfg.extra_args_ssh_tunnel,
142            report_internal_ip=self._report_internal_ip)
143        self._compute_client.InitRemoteHost(
144            self._ssh, ip, self._avd_spec.host_user)
145        return instance
146
147    @utils.TimeExecute(function_description="Downloading Android Build artifact")
148    def _DownloadArtifacts(self, extract_path):
149        """Download the CF image artifacts and process them.
150
151        - Download image from the Android Build system, then decompress it.
152        - Download cvd host package from the Android Build system.
153
154        Args:
155            extract_path: String, a path include extracted files.
156        """
157        cfg = self._avd_spec.cfg
158        build_id = self._avd_spec.remote_image[constants.BUILD_ID]
159        build_target = self._avd_spec.remote_image[constants.BUILD_TARGET]
160
161        # Image zip
162        remote_image = "%s-img-%s.zip" % (build_target.split('-')[0], build_id)
163        create_common.DownloadRemoteArtifact(
164            cfg, build_target, build_id, remote_image, extract_path, decompress=True)
165
166        # Cvd host package
167        create_common.DownloadRemoteArtifact(
168            cfg, build_target, build_id, constants.CVD_HOST_PACKAGE,
169            extract_path)
170
171    def _ProcessRemoteHostArtifacts(self):
172        """Process remote host artifacts.
173
174        - If images source is local, tool will upload images from local site to
175          remote host.
176        - If images source is remote, tool will download images from android
177          build to local and unzip it then upload to remote host, because there
178          is no permission to fetch build rom on the remote host.
179        """
180        if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL:
181            self._UploadArtifacts(
182                self._local_image_artifact, self._cvd_host_package_artifact,
183                self._avd_spec.local_image_dir)
184        else:
185            try:
186                artifacts_path = tempfile.mkdtemp()
187                logger.debug("Extracted path of artifacts: %s", artifacts_path)
188                self._DownloadArtifacts(artifacts_path)
189                self._UploadArtifacts(
190                    None,
191                    os.path.join(artifacts_path, constants.CVD_HOST_PACKAGE),
192                    artifacts_path)
193            finally:
194                shutil.rmtree(artifacts_path)
195
196    def _ProcessArtifacts(self, image_source):
197        """Process artifacts.
198
199        - If images source is local, tool will upload images from local site to
200          remote instance.
201        - If images source is remote, tool will download images from android
202          build to remote instance. Before download images, we have to update
203          fetch_cvd to remote instance.
204
205        Args:
206            image_source: String, the type of image source is remote or local.
207        """
208        if image_source == constants.IMAGE_SRC_LOCAL:
209            self._UploadArtifacts(self._local_image_artifact,
210                                  self._cvd_host_package_artifact,
211                                  self._avd_spec.local_image_dir)
212        elif image_source == constants.IMAGE_SRC_REMOTE:
213            self._compute_client.UpdateFetchCvd()
214            self._FetchBuild(
215                self._avd_spec.remote_image[constants.BUILD_ID],
216                self._avd_spec.remote_image[constants.BUILD_BRANCH],
217                self._avd_spec.remote_image[constants.BUILD_TARGET],
218                self._avd_spec.system_build_info[constants.BUILD_ID],
219                self._avd_spec.system_build_info[constants.BUILD_BRANCH],
220                self._avd_spec.system_build_info[constants.BUILD_TARGET],
221                self._avd_spec.kernel_build_info[constants.BUILD_ID],
222                self._avd_spec.kernel_build_info[constants.BUILD_BRANCH],
223                self._avd_spec.kernel_build_info[constants.BUILD_TARGET])
224
225    def _FetchBuild(self, build_id, branch, build_target, system_build_id,
226                    system_branch, system_build_target, kernel_build_id,
227                    kernel_branch, kernel_build_target):
228        """Download CF artifacts from android build.
229
230        Args:
231            build_branch: String, git branch name. e.g. "aosp-master"
232            build_target: String, the build target, e.g. cf_x86_phone-userdebug
233            build_id: String, build id, e.g. "2263051", "P2804227"
234            kernel_branch: Kernel branch name, e.g. "kernel-common-android-4.14"
235            kernel_build_id: Kernel build id, a string, e.g. "223051", "P280427"
236            kernel_build_target: String, Kernel build target name.
237            system_build_target: Target name for the system image,
238                                 e.g. "cf_x86_phone-userdebug"
239            system_branch: A String, branch name for the system image.
240            system_build_id: A string, build id for the system image.
241
242        """
243        self._compute_client.FetchBuild(
244            build_id, branch, build_target, system_build_id,
245            system_branch, system_build_target, kernel_build_id,
246            kernel_branch, kernel_build_target)
247
248    def _CreateGceInstance(self):
249        """Create a single configured cuttlefish device.
250
251        Override method from parent class.
252        build_target: The format is like "aosp_cf_x86_phone". We only get info
253                      from the user build image file name. If the file name is
254                      not custom format (no "-"), we will use $TARGET_PRODUCT
255                      from environment variable as build_target.
256
257        Returns:
258            A string, representing instance name.
259        """
260        image_name = os.path.basename(
261            self._local_image_artifact) if self._local_image_artifact else ""
262        build_target = (os.environ.get(constants.ENV_BUILD_TARGET) if "-" not
263                        in image_name else image_name.split("-")[0])
264        build_id = _USER_BUILD
265        if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE:
266            build_id = self._avd_spec.remote_image[constants.BUILD_ID]
267            build_target = self._avd_spec.remote_image[constants.BUILD_TARGET]
268
269        if self._avd_spec.instance_name_to_reuse:
270            instance = self._avd_spec.instance_name_to_reuse
271        else:
272            instance = self._compute_client.GenerateInstanceName(
273                build_target=build_target, build_id=build_id)
274
275        # Create an instance from Stable Host Image
276        self._compute_client.CreateInstance(
277            instance=instance,
278            image_name=self._cfg.stable_host_image_name,
279            image_project=self._cfg.stable_host_image_project,
280            blank_data_disk_size_gb=self._cfg.extra_data_disk_size_gb,
281            avd_spec=self._avd_spec)
282        ip = self._compute_client.GetInstanceIP(instance)
283        self._ssh = ssh.Ssh(ip=ip,
284                            user=constants.GCE_USER,
285                            ssh_private_key_path=self._cfg.ssh_private_key_path,
286                            extra_args_ssh_tunnel=self._cfg.extra_args_ssh_tunnel,
287                            report_internal_ip=self._report_internal_ip)
288        return instance
289
290    @utils.TimeExecute(function_description="Processing and uploading local images")
291    def _UploadArtifacts(self,
292                         local_image_zip,
293                         cvd_host_package_artifact,
294                         images_dir):
295        """Upload local images and avd local host package to instance.
296
297        There are two ways to upload local images.
298        1. Using local image zip, it would be decompressed by install_zip.sh.
299        2. Using local image directory, this directory contains all images.
300           Images are compressed/decompressed by lzop during upload process.
301
302        Args:
303            local_image_zip: String, path to zip of local images which
304                             build from 'm dist'.
305            cvd_host_package_artifact: String, path to cvd host package.
306            images_dir: String, directory of local images which build
307                        from 'm'.
308        """
309        if local_image_zip:
310            remote_cmd = ("/usr/bin/install_zip.sh . < %s" % local_image_zip)
311            logger.debug("remote_cmd:\n %s", remote_cmd)
312            self._ssh.Run(remote_cmd)
313        else:
314            # Compress image files for faster upload.
315            artifact_files = [os.path.basename(image) for image in glob.glob(
316                os.path.join(images_dir, "*.img"))]
317            cmd = ("tar -cf - --lzop -S -C {images_dir} {artifact_files} | "
318                   "{ssh_cmd} -- tar -xf - --lzop -S".format(
319                       images_dir=images_dir,
320                       artifact_files=" ".join(artifact_files),
321                       ssh_cmd=self._ssh.GetBaseCmd(constants.SSH_BIN)))
322            logger.debug("cmd:\n %s", cmd)
323            ssh.ShellCmdWithRetry(cmd)
324
325        # host_package
326        remote_cmd = ("tar -x -z -f - < %s" % cvd_host_package_artifact)
327        logger.debug("remote_cmd:\n %s", remote_cmd)
328        self._ssh.Run(remote_cmd)
329
330    def _LaunchCvd(self, instance, decompress_kernel=None,
331                   boot_timeout_secs=None):
332        """Launch CVD.
333
334        Args:
335            instance: String, instance name.
336            boot_timeout_secs: Integer, the maximum time to wait for the
337                               command to respond.
338        """
339        kernel_build = None
340        # TODO(b/140076771) Support kernel image for local image mode.
341        if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE:
342            kernel_build = self._compute_client.GetKernelBuild(
343                self._avd_spec.kernel_build_info[constants.BUILD_ID],
344                self._avd_spec.kernel_build_info[constants.BUILD_BRANCH],
345                self._avd_spec.kernel_build_info[constants.BUILD_TARGET])
346        self._compute_client.LaunchCvd(
347            instance,
348            self._avd_spec,
349            self._cfg.extra_data_disk_size_gb,
350            kernel_build,
351            decompress_kernel,
352            boot_timeout_secs)
353
354    def GetFailures(self):
355        """Get failures from all devices.
356
357        Returns:
358            A dictionary that contains all the failures.
359            The key is the name of the instance that fails to boot,
360            and the value is an errors.DeviceBootError object.
361        """
362        return self._compute_client.all_failures
363
364    def _SetFailures(self, instance, error_msg):
365        """Set failures from this device.
366
367        Record the failures for any steps in AVD creation.
368
369        Args:
370            instance: String of instance name.
371            error_msg: String of error message.
372        """
373        self._compute_client.all_failures[instance] = error_msg
374
375    def GetBuildInfoDict(self):
376        """Get build info dictionary.
377
378        Returns:
379            A build info dictionary. None for local image case.
380        """
381        if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL:
382            return None
383        build_info_dict = {
384            key: val for key, val in self._avd_spec.remote_image.items() if val}
385
386        # kernel_target have default value "kernel". If user provide kernel_build_id
387        # or kernel_branch, then start to process kernel image.
388        if (self._avd_spec.kernel_build_info[constants.BUILD_ID]
389                or self._avd_spec.kernel_build_info[constants.BUILD_BRANCH]):
390            build_info_dict.update(
391                {"kernel_%s" % key: val
392                 for key, val in self._avd_spec.kernel_build_info.items() if val}
393            )
394        build_info_dict.update(
395            {"system_%s" % key: val
396             for key, val in self._avd_spec.system_build_info.items() if val}
397        )
398        return build_info_dict
399