1#!/bin/bash 2# Copyright (C) 2019 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16set -eux -o pipefail 17 18# num-workers is set at VM creation time in the Makefile. 19URL='http://metadata.google.internal/computeMetadata/v1/instance/attributes/num-workers' 20NUM_WORKERS=$(curl --silent --fail -H'Metadata-Flavor:Google' $URL || echo 1) 21 22for SSD in /dev/nvme0n*; do 23mkswap $SSD 24swapon -p -1 $SSD 25done 26 27# This is used by the sandbox containers, NOT needed by the workers. 28# Rationale for size=100G: by default tmpfs mount are set to RAM/2, which makes 29# the CI depend too much on the underlying VM. Here and below, we pick an 30# arbitrary fixed size (we use local scratch NVME as a swap device). 31export SHARED_WORKER_CACHE=/mnt/disks/shared_worker_cache 32rm -rf $SHARED_WORKER_CACHE 33mkdir -p $SHARED_WORKER_CACHE 34mount -t tmpfs tmpfs $SHARED_WORKER_CACHE -o mode=777,size=100G 35 36# This is used to queue build artifacts that are uploaded to GCS. 37export ARTIFACTS_DIR=/mnt/disks/artifacts 38rm -rf $ARTIFACTS_DIR 39mkdir -p $ARTIFACTS_DIR 40mount -t tmpfs tmpfs $ARTIFACTS_DIR -o mode=777,size=100G 41 42# Pull the latest images from the registry. 43docker pull eu.gcr.io/perfetto-ci/worker 44docker pull eu.gcr.io/perfetto-ci/sandbox 45 46# Create the restricted bridge for the sandbox container. 47# Prevent access to the metadata server and impersonation of service accounts. 48docker network rm sandbox 2>/dev/null || true # Handles the reboot case. 49docker network create sandbox -o com.docker.network.bridge.name=sandbox 50sudo iptables -I DOCKER-USER -i sandbox -d 169.254.0.0/16 -j REJECT 51 52# These args will be appended to the docker run invocation for the sandbox. 53export SANDBOX_NETWORK_ARGS="--network sandbox --dns 8.8.8.8" 54 55# The worker_main_loop.py script creates one docker sandbox container for 56# each job invocation. It needs to talk back to the host docker to do so. 57# This implies that the worker container is trusted and should never run code 58# from the repo, as opposite to the sandbox container that is isolated. 59for i in $(seq $NUM_WORKERS); do 60 61# We manually mount a tmpfs mount ourselves because Docker doesn't allow to 62# both override tmpfs-size AND "-o exec" (see also 63# https://github.com/moby/moby/issues/32131) 64SANDBOX_TMP=/mnt/disks/sandbox-$i-tmp 65rm -rf $SANDBOX_TMP 66mkdir -p $SANDBOX_TMP 67mount -t tmpfs tmpfs $SANDBOX_TMP -o mode=777,size=100G 68 69docker rm -f worker-$i 2>/dev/null || true 70docker run -d \ 71 -v /var/run/docker.sock:/var/run/docker.sock \ 72 -v $ARTIFACTS_DIR:$ARTIFACTS_DIR \ 73 --env SHARED_WORKER_CACHE="$SHARED_WORKER_CACHE" \ 74 --env SANDBOX_NETWORK_ARGS="$SANDBOX_NETWORK_ARGS" \ 75 --env ARTIFACTS_DIR="$ARTIFACTS_DIR" \ 76 --env SANDBOX_TMP="$SANDBOX_TMP" \ 77 --env WORKER_HOST="$(hostname)" \ 78 --name worker-$i \ 79 --hostname worker-$i \ 80 --log-driver gcplogs \ 81 eu.gcr.io/perfetto-ci/worker 82done 83 84 85# Register a systemd service to stop worker containers gracefully on shutdown. 86cat > /etc/systemd/system/graceful_shutdown.sh <<EOF 87#!/bin/sh 88logger 'Shutting down worker containers' 89docker ps -q -f 'name=worker-\d+$' | xargs docker stop -t 30 90exit 0 91EOF 92 93chmod 755 /etc/systemd/system/graceful_shutdown.sh 94 95# This service will cause the graceful_shutdown.sh to be invoked before stopping 96# docker, hence before tearing down any other container. 97cat > /etc/systemd/system/graceful_shutdown.service <<EOF 98[Unit] 99Description=Worker container lifecycle 100Wants=gcr-online.target docker.service 101After=gcr-online.target docker.service 102Requires=docker.service 103 104[Service] 105Type=oneshot 106RemainAfterExit=yes 107ExecStop=/etc/systemd/system/graceful_shutdown.sh 108EOF 109 110systemctl daemon-reload 111systemctl start graceful_shutdown.service