1#!/bin/bash
2
3# Common code to build a host image on GCE
4
5# INTERNAL_extra_source may be set to a directory containing the source for
6# extra package to build.
7
8# INTERNAL_IP can be set to --internal-ip run on a GCE instance
9# The instance will need --scope compute-rw
10
11if [ -z "${ANDROID_BUILD_TOP}" ]; then
12  echo "ANDROID_BUILD_TOP is not set, did you forget to lunch?" && exit 1
13fi
14
15source "${ANDROID_BUILD_TOP}/external/shflags/shflags"
16DIR="${ANDROID_BUILD_TOP}/device/google/cuttlefish_vmm"
17
18# ARM-board options
19
20DEFINE_boolean arm false "Build on an ARM board"
21DEFINE_string arm_instance "" "IP address or DNS name of an ARM system to do the secondary build"
22DEFINE_string arm_user "vsoc-01" "User to invoke on the ARM system"
23
24# Docker options
25
26DEFINE_boolean docker false "Build inside docker"
27DEFINE_boolean docker_persistent true "Build inside a privileged, persistent container (faster for iterative development)"
28DEFINE_string docker_arch "$(uname -m)" "Target architectre"
29DEFINE_boolean docker_build_image true "When --noreuse is specified, this flag controls building the docker image (else we assume it was built and reuse it)"
30DEFINE_string docker_image "docker_vmm" "Name of docker image to build"
31DEFINE_string docker_container "docker_vmm" "Name of docker container to create"
32DEFINE_string docker_source "" "Path to sources checked out using manifest"
33DEFINE_string docker_working "" "Path to working directory"
34DEFINE_string docker_output "" "Output directory (when --docker is specified)"
35DEFINE_string docker_user "${USER}" "Docker-container user"
36DEFINE_string docker_uid "${UID}" "Docker-container user ID"
37
38# GCE options
39
40DEFINE_boolean gce false "Build on a GCE instance"
41DEFINE_string gce_arch "$(uname -m)" "Target architecture"
42DEFINE_string gce_project "$(gcloud config get-value project)" "Project to use" "p"
43DEFINE_string gce_instance "${USER}-build" "Instance name to create for the build" "i"
44DEFINE_string gce_user cuttlefish_crosvm_builder "User name to use on GCE when doing the build"
45DEFINE_integer gce_vcpus 4 "Instance size (vcpus) to create"
46DEFINE_string gce_zone "$(gcloud config get-value compute/zone)" "Zone to use" "z"
47
48# Common options
49
50DEFINE_string manifest "" "Path to custom manifest to use for the build"
51DEFINE_boolean reuse false "Set to true to reuse a previously-set-up instance."
52DEFINE_boolean reuse_resync false "Reuse a previously-set-up instance, but clean and re-sync the sources. Overrides --reuse if both are specified."
53
54set -e
55
56SSH_FLAGS=(${INTERNAL_IP})
57
58wait_for_instance() {
59  alive=""
60  while [[ -z "${alive}" ]]; do
61    sleep 5
62    alive="$(gcloud compute ssh "${SSH_FLAGS[@]}" "$@" -- uptime || true)"
63  done
64}
65
66check_common_docker_options() {
67  if [[ -z "${FLAGS_docker_image}" ]]; then
68    echo Option --docker_image must not be empty 1>&1
69    fail=1
70  fi
71  if [[ -z "${FLAGS_docker_container}" ]]; then
72    echo Options --docker_container must not be empty 1>&2
73    fail=1
74  fi
75  if [[ -z "${FLAGS_docker_user}" ]]; then
76    echo Options --docker_user must not be empty 1>&2
77    fail=1
78  fi
79  if [[ -z "${FLAGS_docker_uid}" ]]; then
80    echo Options --docker_uid must not be empty 1>&2
81    fail=1
82  fi
83  # Volume mapping are specified only when a container is created.  With
84  # --reuse, an already-created persistent container is reused, which implies
85  # that we cannot change the volume maps.  For non-persistent containers, we
86  # use docker run, which creates and runs the continer in one step; in that
87  # case, we must pass the same values for --docker_source and --docker_output
88  # that we passed when we ran the non-persistent continer the first time.
89  if [[ ${_reuse} -eq 1 && ${FLAGS_docker_persistent} -eq ${FLAGS_TRUE} ]]; then
90    if [ -n "${FLAGS_docker_source}" ]; then
91      echo Option --docker_source may not be specified with --reuse and --docker_persistent 1>&2
92      fail=1
93    fi
94    if [ -n "${FLAGS_docker_working}" ]; then
95      echo Option --docker_working may not be specified with --reuse and --docker_persistent 1>&2
96      fail=1
97    fi
98    if [ -n "${FLAGS_docker_output}" ]; then
99      echo Option --docker_output may not be specified with --reuse and --docker_persistent 1>&2
100      fail=1
101    fi
102  fi
103  if [[ "${fail}" -ne 0 ]]; then
104    exit "${fail}"
105  fi
106}
107
108build_locally_using_docker() {
109  check_common_docker_options
110  case "${FLAGS_docker_arch}" in
111    aarch64) ;;
112    x86_64) ;;
113    *) echo Invalid value ${FLAGS_docker_arch} for --docker_arch 1>&2
114      fail=1
115      ;;
116  esac
117  if [[ "${fail}" -ne 0 ]]; then
118    exit "${fail}"
119  fi
120  local -i _persistent=0
121  if [[ ${FLAGS_docker_persistent} -eq ${FLAGS_TRUE} ]]; then
122    _persistent=1
123  fi
124
125  local -i _build_image=0
126  if [[ ${FLAGS_docker_build_image} -eq ${FLAGS_TRUE} ]]; then
127    _build_image=1
128  fi
129
130  local _docker_output=""
131  if [ -z "${FLAGS_docker_output}" ]; then
132    _docker_output="${ANDROID_BUILD_TOP}/device/google/cuttlefish_vmm/${FLAGS_docker_arch}-linux-gnu"
133  else
134    _docker_output="${FLAGS_docker_output}"
135  fi
136
137  local _temp="$(mktemp -d)"
138  rsync -avR "${relative_source_files[@]/#/${DIR}/./}" "${_temp}"
139  if [ -n "${custom_manifest}" ]; then
140    cp "${custom_manifest}" "${_temp}"/custom.xml
141  else
142    touch "${_temp}"/custom.xml
143  fi
144
145  ${DIR}/rebuild-docker.sh "${FLAGS_docker_image}" \
146                     "${FLAGS_docker_container}" \
147                     "${FLAGS_docker_arch}" \
148                     "${FLAGS_docker_user}" \
149                     "${FLAGS_docker_uid}" \
150                     "${_persistent}" \
151                     "x${FLAGS_docker_source}" \
152                     "x${FLAGS_docker_working}" \
153                     "x${_docker_output}" \
154                     "${_reuse}" \
155                     "${_build_image}" \
156                     "${_temp}/Dockerfile" \
157                     "${_temp}" \
158                     "${#docker_flags[@]}" "${docker_flags[@]}" \
159                     "${#_prepare_source[@]}" "${_prepare_source[@]}"
160
161  rm -rf "${_temp}"
162}
163
164function build_on_gce() {
165  check_common_docker_options
166  if [[ "${FLAGS_gce_arch}" != "${FLAGS_docker_arch}" ]]; then
167    echo Docker arch must match gce arch 1>&2
168    fail=1
169  fi
170  local _image_family=""
171  local _machine_type=""
172  case "${FLAGS_gce_arch}" in
173    aarch64)
174      _image_family=debian-11-arm64
175      _machine_type=t2a-standard
176      ;;
177    x86_64)
178      _image_family=debian-11
179      _machine_type=n1-standard
180      ;;
181    *) echo Invalid value ${FLAGS_gce_arch} for --gce_arch 1>&2
182      fail=1
183      ;;
184  esac
185  if [[ -z "${FLAGS_gce_instance}" ]]; then
186    echo Must specify instance 1>&2
187    fail=1
188  fi
189  if [[ -z "${FLAGS_gce_project}" ]]; then
190    echo Must specify project 1>&2
191    fail=1
192  fi
193  if [[ -z "${FLAGS_gce_zone}" ]]; then
194    echo Must specify zone 1>&2
195    fail=1
196  fi
197  if [[ "${fail}" -ne 0 ]]; then
198    exit "${fail}"
199  fi
200  project_zone_flags=(--project="${FLAGS_gce_project}" --zone="${FLAGS_gce_zone}")
201  if [ ${_reuse} -eq 0 ]; then
202    delete_instances=("${FLAGS_gce_instance}")
203    gcloud compute instances delete -q \
204      "${delete_instances[@]}" \
205      "${project_zone_flags[@]}" || \
206        echo Instance does not exist
207    gcloud compute images delete -q \
208      "${delete_instances[@]/%/-image}" \
209      --project "${FLAGS_gce_project}" || \
210        echo Image does not exist
211    gcloud compute disks delete -q \
212      "${delete_instances[@]/%/-disk}" \
213      "${project_zone_flags[@]}" || \
214        echo Disk does not exist
215
216    gcloud compute disks create \
217      "${delete_instances[@]/%/-disk}" \
218      "${project_zone_flags[@]}" \
219      --image-project="debian-cloud" \
220      --image-family="${_image_family}"
221    gcloud compute images create \
222      "${delete_instances[@]/%/-image}" \
223      --source-disk "${delete_instances[@]/%/-disk}" \
224      --project "${FLAGS_gce_project}" --source-disk-zone "${FLAGS_gce_zone}"
225    gcloud compute instances create \
226      "${delete_instances[@]}" \
227      "${project_zone_flags[@]}" \
228      --image "${delete_instances[@]/%/-image}" \
229      --boot-disk-size=200GB \
230      --machine-type="${_machine_type}-${FLAGS_gce_vcpus}" \
231      --network-interface=nic-type=GVNIC
232
233    wait_for_instance "${FLAGS_gce_instance}" "${project_zone_flags[@]}"
234
235    # install docker
236    gcloud beta compute ssh "${SSH_FLAGS[@]}" \
237        "${project_zone_flags[@]}" \
238        "${FLAGS_gce_user}@${FLAGS_gce_instance}" -- \
239        'curl --retry 10 --retry-all-errors -fsSL https://get.docker.com | /bin/bash'
240    gcloud beta compute ssh "${SSH_FLAGS[@]}" \
241        "${project_zone_flags[@]}" \
242        "${FLAGS_gce_user}@${FLAGS_gce_instance}" -- \
243      sudo usermod -aG docker "${FLAGS_gce_user}"
244
245    # beta for the --internal-ip flag that may be passed via SSH_FLAGS
246
247    gcloud beta compute ssh "${SSH_FLAGS[@]}" \
248        "${project_zone_flags[@]}" \
249        "${FLAGS_gce_user}@${FLAGS_gce_instance}" -- \
250        mkdir -p '$PWD/docker $PWD/docker/source $PWD/docker/working $PWD/docker/output'
251
252    tar czv -C "${DIR}" -f - "${relative_source_files[@]}" | \
253      gcloud beta compute ssh "${SSH_FLAGS[@]}" \
254          "${project_zone_flags[@]}" \
255          "${FLAGS_gce_user}@${FLAGS_gce_instance}" -- \
256          'tar xzv -C ~/docker -f -'
257    if [ -n "${custom_manifest}" ]; then
258      gcloud beta compute scp "${SSH_FLAGS[@]}" \
259        "${project_zone_flags[@]}" \
260        "${custom_manifest}" \
261        "${FLAGS_gce_user}@${FLAGS_gce_instance}:~/docker/custom.xml"
262    else
263      gcloud beta compute ssh "${SSH_FLAGS[@]}" \
264        "${project_zone_flags[@]}" \
265        "${FLAGS_gce_user}@${FLAGS_gce_instance}" -- \
266        "touch ~/docker/custom.xml"
267    fi
268  fi
269
270  local _status=$(gcloud compute instances list \
271                  --project="${FLAGS_gce_project}" \
272                  --zones="${FLAGS_gce_zone}" \
273                  --filter="name=('${FLAGS_gce_instance}')" \
274                  --format=flattened | awk '/status:/ {print $2}')
275  if [ "${_status}" != "RUNNING" ] ; then
276    echo "Instance ${FLAGS_gce_instance} is not running."
277    exit 1;
278  fi
279
280  local -i _persistent=0
281  if [[ ${FLAGS_docker_persistent} -eq ${FLAGS_TRUE} ]]; then
282    _persistent=1
283  fi
284  local -i _build_image=0
285  if [[ ${FLAGS_docker_build_image} -eq ${FLAGS_TRUE} ]]; then
286    _build_image=1
287  fi
288  gcloud beta compute ssh "${SSH_FLAGS[@]}" \
289      "${project_zone_flags[@]}" \
290      "${FLAGS_gce_user}@${FLAGS_gce_instance}" -- \
291      ./docker/rebuild-docker.sh "${FLAGS_docker_image}" \
292                       "${FLAGS_docker_container}" \
293                       "${FLAGS_docker_arch}" \
294                       '${USER}' \
295                       '${UID}' \
296                       "${_persistent}" \
297                       'x$PWD/docker/source' \
298                       'x$PWD/docker/working' \
299                       'x$PWD/docker/output' \
300                       "${_reuse}" \
301                       "${_build_image}" \
302                       '~/docker/Dockerfile' \
303                       '~/docker/' \
304                       "${#docker_flags[@]}" "${docker_flags[@]}" \
305                       "${#_prepare_source[@]}" "${_prepare_source[@]}"
306
307  gcloud beta compute ssh "${SSH_FLAGS[@]}" \
308      "${project_zone_flags[@]}" \
309      "${FLAGS_gce_user}@${FLAGS_gce_instance}" --command \
310      'tar czv -C $PWD/docker/output -f - $(find $PWD/docker/output -printf "%P\n")' | \
311    tar xzv -C ${DIR}/${FLAGS_docker_arch}-linux-gnu -f -
312
313  gcloud compute disks describe \
314    "${project_zone_flags[@]}" "${FLAGS_gce_instance}" | \
315      grep ^sourceImage: > "${DIR}"/x86_64-linux-gnu/builder_image.txt
316}
317
318function build_on_arm_board() {
319  check_common_docker_options
320  if [[ "${FLAGS_docker_arch}" != "aarch64" ]]; then
321    echo ARM board supports building only aarch64 1>&2
322    fail=1
323  fi
324  if [[ -z "${FLAGS_arm_instance}" ]]; then
325    echo Must specify IP address of ARM board 1>&2
326    fail=1
327  fi
328  if [[ -z "${FLAGS_arm_user}" ]]; then
329    echo Must specify a user account on ARM board 1>&2
330    fail=1
331  fi
332  if [[ "${fail}" -ne 0 ]]; then
333    exit "${fail}"
334  fi
335  if [[ "${_reuse}" -eq 0 ]]; then
336    ssh -t "${FLAGS_arm_user}@${FLAGS_arm_instance}" -- \
337      rm -rf '$PWD/docker'
338  fi
339  rsync -avR -e ssh \
340    "${relative_source_files[@]/#/${DIR}/./}" \
341    "${FLAGS_arm_user}@${FLAGS_arm_instance}:~/docker/"
342
343  if [ -n "${custom_manifest}" ]; then
344    scp "${custom_manifest}" "${FLAGS_arm_user}@${FLAGS_arm_instance}":~/docker/custom.xml
345  else
346    ssh -t "${FLAGS_arm_user}@${FLAGS_arm_instance}" -- \
347      "touch ~/docker/custom.xml"
348  fi
349
350  local -i _persistent=0
351  if [[ ${FLAGS_docker_persistent} -eq ${FLAGS_TRUE} ]]; then
352    _persistent=1
353  fi
354  local -i _build_image=0
355  if [[ ${FLAGS_docker_build_image} -eq ${FLAGS_TRUE} ]]; then
356    _build_image=1
357  fi
358  ssh -t "${FLAGS_arm_user}@${FLAGS_arm_instance}" -- \
359    mkdir -p '$PWD/docker/source' '$PWD/docker/working' '$PWD/docker/output'
360  ssh -t "${FLAGS_arm_user}@${FLAGS_arm_instance}" -- \
361    ./docker/rebuild-docker.sh "${FLAGS_docker_image}" \
362                     "${FLAGS_docker_container}" \
363                     "${FLAGS_docker_arch}" \
364                     '${USER}' \
365                     '${UID}' \
366                     "${_persistent}" \
367                     'x$PWD/docker/source' \
368                     'x$PWD/docker/working' \
369                     'x$PWD/docker/output' \
370                     "${_reuse}" \
371                     "${_build_image}" \
372                     '~/docker/Dockerfile' \
373                     '~/docker/' \
374                     "${#docker_flags[@]}" "${docker_flags[@]}" \
375                     "${#_prepare_source[@]}" "${_prepare_source[@]}"
376
377  rsync -avR -e ssh "${FLAGS_arm_user}@${FLAGS_arm_instance}":docker/output/./ \
378    "${ANDROID_BUILD_TOP}/device/google/cuttlefish_vmm/${FLAGS_docker_arch}-linux-gnu"
379}
380
381main() {
382  set -o errexit
383  set -x
384  fail=0
385  relative_source_files=("rebuild-docker.sh"
386     "rebuild-internal.sh"
387     "Dockerfile"
388     "manifest.xml"
389     ".dockerignore")
390  # These must match the definitions in the Dockerfile
391  docker_flags=("-eSOURCE_DIR=/source" "-eWORKING_DIR=/working" "-eOUTPUT_DIR=/output" "-eTOOLS_DIR=/static/tools")
392
393  if [[ $(( $((${FLAGS_gce}==${FLAGS_TRUE})) + $((${FLAGS_arm}==${FLAGS_TRUE})) + $((${FLAGS_docker}==${FLAGS_TRUE})) )) > 1 ]]; then
394    echo You may specify only one of --gce, --docker, or --arm 1>&2
395    exit 2
396  fi
397
398  if [[ -n "${FLAGS_manifest}" ]]; then
399    if [[ ! -f "${FLAGS_manifest}" ]]; then
400      echo custom manifest not found: ${FLAGS_manifest} 1>&1
401      exit 2
402    fi
403    custom_manifest="${FLAGS_manifest}"
404    docker_flags+=("-eCUSTOM_MANIFEST=/static/custom.xml")
405  else
406    custom_manifest="${DIR}/manifest.xml"
407    docker_flags+=("-eCUSTOM_MANIFEST=/static/manifest.xml")
408  fi
409  local -a _prepare_source=(setup_env fetch_source);
410  local -i _reuse=0
411  if [[ ${FLAGS_reuse} -eq ${FLAGS_TRUE} ]]; then
412    # neither install packages, nor sync sources; skip to building them
413    _prepare_source=(setup_env)
414    # unless you're setting up a non-persistent container and --docker_source is
415    # the empty string; in this case, --reuse implies --reuse_resync
416    if [[ "${FLAGS_docker_persistent}" -eq ${FLAGS_FALSE} && \
417          -z "${FLAGS_docker_source}" ]]; then
418      _prepare_source+=(resync_source)
419    fi
420    _reuse=1
421  fi
422  if [[ ${FLAGS_reuse_resync} -eq ${FLAGS_TRUE} ]]; then
423    # do not install packages but clean and sync sources afresh
424    _prepare_source=(setup_env resync_source);
425    _reuse=1
426  fi
427  if [[ ${FLAGS_gce} -eq ${FLAGS_TRUE} ]]; then
428    build_on_gce
429    exit 0
430    gcloud compute instances delete -q \
431      "${project_zone_flags[@]}" \
432      "${FLAGS_gce_instance}"
433  fi
434  if [[ ${FLAGS_arm} -eq ${FLAGS_TRUE} ]]; then
435    build_on_arm_board
436    exit 0
437  fi
438  if [[ ${FLAGS_docker} -eq ${FLAGS_TRUE} ]]; then
439    build_locally_using_docker
440    exit 0
441  fi
442}
443
444FLAGS "$@" || exit 1
445main "${FLAGS_ARGV[@]}"
446