1#!/bin/bash
2# Copyright 2024 Google Inc. All rights reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16color_cyan="\033[0;36m"
17color_plain="\033[0m"
18color_yellow="\033[0;33m"
19
20# validate number of arguments
21if [ "$#" -lt 1 ] || [ "$#" -gt 4 ]; then
22  echo "This script requires 1 mandatory and 3 optional parameters,"
23  echo "server address and optionally cvd instances per docker, and number of " \
24       "docker instances to invoke, and vendor_boot image to replace."
25  exit 1
26fi
27
28# map arguments to variables
29# $1: ARM server address
30# $2: CVD Instance number per docker (Optional, default is 1)
31# $3: Docker Instance number (Optional, default is 1)
32# $4: Vendor Boot Image path (Optional, default is "")
33server=$1
34
35if [ "$#" -lt 2 ]; then
36 num_instances_per_docker=1
37else
38 num_instances_per_docker=$2
39fi
40
41if [ "$#" -lt 3 ]; then
42 num_dockers=1
43else
44 num_dockers=$3
45fi
46
47if [ "$#" -lt 4 ]; then
48 vendor_boot_image=""
49else
50 vendor_boot_image=$4
51fi
52
53# set img_dir and cvd_host_tool_dir
54img_dir=${ANDROID_PRODUCT_OUT:-$PWD}
55cvd_host_tool_dir=${ANDROID_HOST_OUT:+"$ANDROID_HOST_OUT/../linux_musl-arm64"}
56cvd_host_tool_dir=${cvd_host_tool_dir:-$PWD}
57
58# upload artifacts into ARM server
59cvd_home_dir=cvd_home
60ssh $server -t "mkdir -p ~/.cvd_artifact; mkdir -p ~/$cvd_home_dir"
61if [ -f $img_dir/required_images ]; then
62  rsync -aSvch --recursive $img_dir --files-from=$img_dir/required_images $server:~/$cvd_home_dir --info=progress2
63  cvd_home_files=($(rsync -rzan --recursive $img_dir --out-format="%n" --files-from=$img_dir/required_images $server:~/$cvd_home_dir --info=name2 | awk '{print $1}'))
64else
65  rsync -aSvch --recursive $img_dir/bootloader $img_dir/*.img $server:~/$cvd_home_dir --info=progress2
66  cvd_home_files=($(rsync -rzan --recursive $img_dir/bootloader --out-format="%n" $img_dir/*.img $server:~/$cvd_home_dir --info=name2 | awk '{print $1}'))
67fi
68
69if [[ $vendor_boot_image != "" ]]; then
70  scp $vendor_boot_image $server:~/$cvd_home_dir/vendor_boot.img
71fi
72
73# upload cvd-host_package.tar.gz into ARM server
74temp_dir=/tmp/cvd_dist
75rm -rf $temp_dir
76mkdir -p $temp_dir
77if [ -d $cvd_host_tool_dir/cvd-host_package ]; then
78  echo "Use contents in cvd-host_package dir"
79  pushd $cvd_host_tool_dir/cvd-host_package > /dev/null
80  tar -cf $temp_dir/cvd-host_package.tar ./*
81  popd > /dev/null
82  pigz -R $temp_dir/cvd-host_package.tar
83elif [ -f $cvd_host_tool_dir/cvd-host_package.tar.gz ]; then
84  echo "Use contents in cvd-host_package.tar.gz"
85  # re-compress with rsyncable option
86  # TODO(b/275312073): remove this if toxbox supports rsyncable
87  pigz -d -c $cvd_host_tool_dir/cvd-host_package.tar.gz | pigz -R > $temp_dir/cvd-host_package.tar.gz
88else
89  echo "There is neither cvd-host_package dir nor cvd-host_package.tar.gz"
90  exit 1
91fi
92rsync -avch $temp_dir/cvd-host_package.tar.gz $server:~/$cvd_home_dir --info=progress2
93cvd_home_files+=("cvd-host_package.tar.gz")
94
95# run root docker instance
96root_container_id=$(ssh $server -t "docker run --privileged -p 2443 -d cuttlefish")
97root_container_id=${root_container_id//$'\r'} # to remove trailing ^M
98echo -e "${color_cyan}Booting root container $root_container_id${color_plain}"
99
100# set trap to stop docker instance
101trap cleanup SIGINT
102cleanup() {
103  echo -e "${color_yellow}SIGINT: stopping the launch instances${color_plain}"
104  ssh $server "docker rm -f $root_container_id ${container_ids[*]} && \
105               docker rmi -f cvd_root_image:$root_container_id && \
106               docker system prune -f"
107  exit 0
108}
109
110# extract Host Orchestrator Port
111docker_inspect=$(ssh $server "docker inspect --format='{{json .NetworkSettings.Ports }}' $root_container_id")
112docker_host_orchestrator_port_parser_script='
113import sys, json;
114json_raw=input()
115data = json.loads(json_raw)
116for k in data:
117  if not data[k]:
118    continue
119
120  original_port = int(k.split("/")[0])
121  assigned_port = int(data[k][0]["HostPort"])
122  if original_port == 2443:
123    print(assigned_port)
124    break
125'
126docker_host_orchestrator_port=$(echo $docker_inspect | python -c "$docker_host_orchestrator_port_parser_script")
127host_orchestrator_url=https://localhost:$docker_host_orchestrator_port
128echo -e "Extracting host orchestrator port in root docker instance"
129
130# create user artifact directory
131create_user_artifacts_dir_script="ssh $server curl -s -k -X POST ${host_orchestrator_url}/userartifacts | jq -r '.name'"
132user_artifacts_dir=$($create_user_artifacts_dir_script)
133while [ -z "$user_artifacts_dir" ]; do
134  echo -e "Failed to create user_artifacts_dir, retrying"
135  sleep 1
136  user_artifacts_dir=$($create_user_artifacts_dir_script)
137done
138echo -e "Succeeded to create user_artifacts_dir"
139
140# upload artifacts and cvd-host_pachage.tar.gz into docker instance
141ssh $server \
142  "for filename in ${cvd_home_files[*]}; do \
143     absolute_path=\$HOME/$cvd_home_dir/\$filename && \
144     size=\$(stat -c%s \$absolute_path) && \
145     echo Uploading \$filename\\(size:\$size\\) ... && \
146     curl -s -k --location -X PUT $host_orchestrator_url/userartifacts/$user_artifacts_dir \
147       -H 'Content-Type: multipart/form-data' \
148       -F chunk_number=1 \
149       -F chunk_total=1 \
150       -F chunk_size_bytes=\$size \
151       -F file=@\$absolute_path; \
152   done"
153echo -e "Done"
154
155# extract cvd-host_package.tar.gz with /:extract API if the API exists
156ssh $server \
157  "job_id=\$(curl -s -k -X POST ${host_orchestrator_url}/userartifacts/$user_artifacts_dir/cvd-host_package.tar.gz/:extract | jq -r '.name' 2>/dev/null) && \
158   if [[ \$job_id != \"\" ]]; then \
159     echo Extracting cvd-host_package.tar.gz ... && \
160     job_done=\"false\" && \
161     while [[ \$job_done == \"false\" ]]; do \
162       sleep 1 && \
163       job_done=\$(curl -s -k ${host_orchestrator_url}/operations/\$job_id | jq -r '.done'); \
164     done && \
165     echo Done; \
166   fi"
167
168echo -e "Creating image from root docker container"
169root_image_id=$(ssh $server -t "docker commit $root_container_id cvd_root_image:$root_container_id")
170root_image_id=${root_image_id//$'\r'} # to remove trailing ^M
171echo -e "${color_cyan}Root image $root_image_id${color_plain}"
172
173echo -e "${color_cyan}Booting containers ... ${color_plain}"
174container_ids=$(ssh $server \
175  "container_ids=() && \
176  for docker_num in \$(seq 1 $num_dockers); do \
177    if [ \$docker_num -eq 1 ]; then \
178      web_port_forward=\"-p 1443 -p 15550-15560 \"; \
179    else \
180      web_port_forward=\"\"; \
181    fi && \
182    adb_port_forward=\"\" && \
183    for instance_num in \$(seq 1 $num_instances_per_docker); do
184      adb_port_forward+=\"-p \$((instance_num + 6520 - 1)) \";
185    done && \
186    container_id=\$(docker run --rm --privileged \$web_port_forward -p 2443 \$adb_port_forward -d $root_image_id) && \
187    container_id=\${container_id//\$'\\r'} && \
188    container_ids+=(\${container_id}); \
189  done && \
190  echo \${container_ids[*]}
191")
192
193echo -e "Extracting host orchestrator ports in docker instances"
194docker_inspects=$(ssh $server \
195  "docker_inspects=() && container_ids=(${container_ids[*]}) &&
196  for container_id in \${container_ids[*]}; do \
197    docker_inspect=\$(docker inspect --format='{{json .NetworkSettings.Ports }}' \$container_id) && \
198    docker_inspects+=(\${docker_inspect}); \
199  done && \
200  echo \${docker_inspects[*]}
201")
202host_orchestrator_ports=()
203for docker_inspect in ${docker_inspects[*]}; do
204  port=$(echo $docker_inspect | python -c "$docker_host_orchestrator_port_parser_script")
205  host_orchestrator_ports+=($port)
206done
207
208# start Cuttlefish instance on top of docker instance
209# TODO(b/317942272): support starting the instance with an optional vendor boot debug image.
210echo -e "Starting Cuttlefish"
211ssh $server "for port in ${host_orchestrator_ports[*]}; do \
212  host_orchestrator_url=https://localhost:\$port && \
213  curl -s -k -X POST \$host_orchestrator_url/cvds \
214  -H 'Content-Type: application/json' \
215  -d '{\"cvd\": {\"build_source\": {\"user_build_source\": {\"artifacts_dir\": \"$user_artifacts_dir\"}}}, \
216       \"additional_instances_num\": $((num_instances_per_docker - 1))}'; \
217done
218"
219
220# Web UI port is 3443 instead 1443 because there could be a running operator or host orchestrator in this machine as well.
221web_ui_port=3443
222echo -e "Web UI port: $web_ui_port. ${color_cyan}Please point your browser to https://localhost:$web_ui_port for the UI${color_plain}"
223
224# sets up SSH port forwarding to the remote server for various ports and launch cvd instance
225adb_port=6520
226for docker_num in $(seq 1 $num_dockers); do
227  for instance_num in $(seq 1 $num_instances_per_docker); do
228    device_name="cvd_$instance_num"
229    device_adb_port=$((adb_port + ( (docker_num - 1) * num_instances_per_docker) + instance_num - 1))
230    echo -e "$device_name of docker $docker_num is using adb port $device_adb_port. Try ${color_cyan}adb connect 127.0.0.1:${device_adb_port}${color_plain} if you want to connect to this device"
231  done
232done
233
234docker_port_parser_script='
235import sys, json;
236web_ui_port = int(sys.argv[1])
237adb_port = int(sys.argv[2])
238max_instances = 100
239num_instance = int(sys.argv[3])
240json_raw=input()
241data = json.loads(json_raw)
242for k in data:
243  if not data[k]:
244    continue
245
246  original_port = int(k.split("/")[0])
247  assigned_port = int(data[k][0]["HostPort"])
248  if original_port == 1443:
249    original_port = web_ui_port
250  elif original_port in (1080, 2080, 2443): # Do not expose other operator or host orchestrator port beyond ARM server
251    continue
252  elif original_port >= 6520 and original_port <= 6520 + max_instances:
253    if original_port - 6520 >= num_instance:
254      continue
255    original_port = adb_port + original_port - 6520
256  print(f"-L {original_port}:127.0.0.1:{assigned_port}", end=" ")
257'
258
259ports_forwarding=""
260current_adb_port=$adb_port
261
262for docker_inspect in ${docker_inspects[*]}; do
263  ports_forwarding+=$(echo $docker_inspect | python -c "$docker_port_parser_script" $web_ui_port $current_adb_port $num_instances_per_docker)
264  current_adb_port=$((current_adb_port + num_instances_per_docker))
265done
266
267echo "Set up ssh ports forwarding: $ports_forwarding"
268echo -e "${color_yellow}Please stop the running instances by ctrl+c${color_plain}"
269ssh $server $ports_forwarding "tail -f /dev/null"
270