1 /*
2 * Copyright (C) 2021 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "host/libs/vm_manager/gem5_manager.h"
18
19 #include <sys/socket.h>
20 #include <sys/stat.h>
21 #include <sys/types.h>
22 #include <sys/un.h>
23 #include <sys/wait.h>
24 #include <unistd.h>
25
26 #include <fstream>
27 #include <string>
28 #include <unordered_map>
29 #include <utility>
30 #include <vector>
31
32 #include <android-base/strings.h>
33 #include <android-base/logging.h>
34 #include <vulkan/vulkan.h>
35
36 #include "common/libs/utils/result.h"
37 #include "common/libs/utils/subprocess.h"
38 #include "host/libs/config/command_source.h"
39 #include "host/libs/config/cuttlefish_config.h"
40
41 using cuttlefish::StringFromEnv;
42
43 namespace cuttlefish {
44 namespace vm_manager {
45 namespace {
46
47 static constexpr char kFsHeader[] = R"CPP_STR_END(import argparse
48 import devices
49 import os
50 import shutil
51 import m5
52 from m5.util import addToPath
53 from m5.objects import *
54 from m5.options import *
55 from m5.objects.Ethernet import NSGigE, IGbE_igb, IGbE_e1000, EtherTap
56 from common import SysPaths
57 from common import ObjectList
58 from common import MemConfig
59 from common.cores.arm import HPI
60 m5.util.addToPath('../..')
61 )CPP_STR_END";
62
63 static constexpr char kFsMemPci[] = R"CPP_STR_END(
64 MemConfig.config_mem(args, root.system)
65
66 pci_devices = []
67 pci_devices.append(PciVirtIO(vio=VirtIOConsole(device=Terminal(number=0))))
68 pci_devices.append(PciVirtIO(vio=VirtIOConsole(device=Terminal(number=1, outfile="none"))))
69 pci_devices.append(PciVirtIO(vio=VirtIOConsole(device=Terminal(number=2))))
70 pci_devices.append(PciVirtIO(vio=VirtIOConsole(device=Terminal(number=3, outfile="none"))))
71 pci_devices.append(PciVirtIO(vio=VirtIOConsole(device=Terminal(number=4, outfile="none"))))
72 pci_devices.append(PciVirtIO(vio=VirtIOConsole(device=Terminal(number=5, outfile="none"))))
73 pci_devices.append(PciVirtIO(vio=VirtIOConsole(device=Terminal(number=6, outfile="none"))))
74 pci_devices.append(PciVirtIO(vio=VirtIOConsole(device=Terminal(number=7, outfile="none"))))
75 pci_devices.append(PciVirtIO(vio=VirtIOConsole(device=Terminal(number=8, outfile="none"))))
76 pci_devices.append(PciVirtIO(vio=VirtIOConsole(device=Terminal(number=9, outfile="none"))))
77
78 for each_item in args.disk_image:
79 disk_image = CowDiskImage()
80 disk_image.child.image_file = SysPaths.disk(each_item)
81 pci_devices.append(PciVirtIO(vio=VirtIOBlock(image=disk_image)))
82
83 nic = IGbE_e1000(pci_bus=0, pci_dev=0, pci_func=0, InterruptLine=1, InterruptPin=1)
84 pci_devices.append(nic)
85 root.system.pci_devices = pci_devices
86 for pci_device in root.system.pci_devices:
87 root.system.attach_pci(pci_device)
88
89 root.tap = EtherTap(tun_clone_device='/dev/net/tun', tap_device_name='cvd-mtap-01')
90 root.tap.tap = nic.interface
91 root.system.connect()
92 )CPP_STR_END";
93
94 static constexpr char kFsKernelCmd[] = R"CPP_STR_END(
95 kernel_cmd = [
96 "lpj=19988480",
97 "norandmaps",
98 "mem=%s" % args.mem_size,
99 "console=hvc0",
100 "panic=-1",
101 "earlycon=pl011,mmio32,0x1c090000",
102 "audit=1",
103 "printk.devkmsg=on",
104 "firmware_class.path=/vendor/etc/",
105 "kfence.sample_interval=500",
106 "loop.max_part=7",
107 "bootconfig",
108 "androidboot.force_normal_boot=1",
109 ]
110 root.system.workload.command_line = " ".join(kernel_cmd)
111 if args.restore is not None:
112 m5.instantiate(args.restore)
113 else:
114 m5.instantiate()
115
116 while True:
117 event = m5.simulate()
118 msg = event.getCause()
119 cur_tick = m5.curTick()
120 if msg == "checkpoint":
121 backup_path = backup_path = os.path.join(root_dir, "gem5_checkpoint")
122 if not os.path.isdir(backup_path):
123 os.mkdir(backup_path)
124
125 print("Checkpoint @", cur_tick)
126 src_dir = os.path.join(m5.options.outdir, "cpt.%d" % cur_tick)
127 backup_path = os.path.join(backup_path, "cpt.%d" % cur_tick)
128 m5.checkpoint(src_dir)
129 shutil.copytree(src_dir, backup_path)
130 print("Checkpoint done.")
131 else:
132 print("Exit msg: " + msg + " @", cur_tick)
133 break
134 sys.exit(event.getCode())
135 )CPP_STR_END";
136
137 static constexpr char kFsExeMain[] = R"CPP_STR_END(
138 if __name__ == "__m5_main__":
139 main()
140 )CPP_STR_END";
141
GenerateGem5File(const CuttlefishConfig & config,const CuttlefishConfig::InstanceSpecific & instance)142 void GenerateGem5File(const CuttlefishConfig& config,
143 const CuttlefishConfig::InstanceSpecific& instance) {
144 // Gem5 specific config, currently users have to change these config locally (without through launch_cvd input flag) to meet their design
145 // TODO: Add these config into launch_cvd input flag or parse from one json file
146 std::string cpu_class = "AtomicSimpleCPU";
147 std::string l1_icache_class = "None";
148 std::string l1_dcache_class = "None";
149 std::string walk_cache_class = "None";
150 std::string l2_Cache_class = "None";
151 std::string cpu_freq = "4GHz";
152 int num_cores = 1;
153 std::string mem_type = "DDR3_1600_8x8";
154 int mem_channels = 1;
155 std::string mem_ranks = "None";
156
157 // start generating starter_fs.py
158 std::string fs_path = instance.gem5_binary_dir() +
159 "/configs/example/arm/starter_fs.py";
160 std::ofstream starter_fs_ofstream(fs_path.c_str());
161 starter_fs_ofstream << kFsHeader << "\n";
162
163 // global vars in python
164 starter_fs_ofstream << "default_disk = 'linaro-minimal-aarch64.img'\n";
165
166 // main function
167 starter_fs_ofstream << "def main():\n";
168
169 // args
170 starter_fs_ofstream << " parser = argparse.ArgumentParser(epilog=__doc__)\n";
171 starter_fs_ofstream << " parser.add_argument(\"--disk-image\", action=\"append\", type=str, default=[])\n";
172 starter_fs_ofstream << " parser.add_argument(\"--mem-type\", default=\"" << mem_type << "\", choices=ObjectList.mem_list.get_names())\n";
173 starter_fs_ofstream << " parser.add_argument(\"--mem-channels\", type=int, default=" << mem_channels << ")\n";
174 starter_fs_ofstream << " parser.add_argument(\"--mem-ranks\", type=int, default=" << mem_ranks << ")\n";
175 starter_fs_ofstream << " parser.add_argument(\"--mem-size\", action=\"store\", type=str, default=\"" << instance.memory_mb() << "MB\")\n";
176 starter_fs_ofstream << " parser.add_argument(\"--restore\", type=str, default=None)\n";
177 starter_fs_ofstream << " args = parser.parse_args()\n";
178
179 // instantiate system
180 starter_fs_ofstream << " root = Root(full_system=True)\n";
181 starter_fs_ofstream << " mem_mode = " << cpu_class << ".memory_mode()\n";
182 starter_fs_ofstream << " has_caches = True if mem_mode == \"timing\" else False\n";
183 starter_fs_ofstream << " root.system = devices.SimpleSystem(has_caches, args.mem_size, mem_mode=mem_mode, workload=ArmFsLinux(object_file=SysPaths.binary(\"" << config.assembly_dir() << "/kernel\")))\n";
184
185 // mem config and pci instantiate
186 starter_fs_ofstream << kFsMemPci;
187
188 // system settings
189 starter_fs_ofstream << " root.system.cpu_cluster = [devices.CpuCluster(root.system, " << num_cores << ", \"" << cpu_freq << "\", \"1.0V\", " << cpu_class << ", " << l1_icache_class << ", " << l1_dcache_class << ", " << walk_cache_class << ", " << l2_Cache_class << ")]\n";
190 starter_fs_ofstream << " root.system.addCaches(has_caches, last_cache_level=2)\n";
191 starter_fs_ofstream << " root.system.realview.setupBootLoader(root.system, SysPaths.binary)\n";
192 starter_fs_ofstream << " root.system.workload.dtb_filename = os.path.join(m5.options.outdir, 'system.dtb')\n";
193 starter_fs_ofstream << " root.system.generateDtb(root.system.workload.dtb_filename)\n";
194 starter_fs_ofstream << " root.system.workload.initrd_filename = \"" << instance.PerInstancePath("initrd.img") << "\"\n";
195 starter_fs_ofstream << " root_dir = \"" << StringFromEnv("HOME", ".") << "\"\n";
196
197 //kernel cmd
198 starter_fs_ofstream << kFsKernelCmd << "\n";
199
200 // execute main
201 starter_fs_ofstream << kFsExeMain << "\n";
202 }
203
204 } // namespace
205
Gem5Manager(Arch arch)206 Gem5Manager::Gem5Manager(Arch arch) : arch_(arch) {}
207
IsSupported()208 bool Gem5Manager::IsSupported() {
209 return HostSupportsQemuCli();
210 }
211
212 Result<std::unordered_map<std::string, std::string>>
ConfigureGraphics(const CuttlefishConfig::InstanceSpecific & instance)213 Gem5Manager::ConfigureGraphics(
214 const CuttlefishConfig::InstanceSpecific& instance) {
215 // TODO: Add support for the gem5 gpu models
216
217 // Override the default HAL search paths in all cases. We do this because
218 // the HAL search path allows for fallbacks, and fallbacks in conjunction
219 // with properties lead to non-deterministic behavior while loading the
220 // HALs.
221
222 std::unordered_map<std::string, std::string> bootconfig_args;
223
224 if (instance.gpu_mode() == kGpuModeGuestSwiftshader) {
225 LOG(INFO) << "We are in SwiftShader mode";
226 bootconfig_args = {
227 {"androidboot.cpuvulkan.version", std::to_string(VK_API_VERSION_1_1)},
228 {"androidboot.hardware.gralloc", "minigbm"},
229 {"androidboot.hardware.hwcomposer", "ranchu"},
230 {"androidboot.hardware.hwcomposer.mode", "noop"},
231 {"androidboot.hardware.hwcomposer.display_finder_mode", "gem5"},
232 {"androidboot.hardware.hwcomposer.display_framebuffer_format",
233 instance.guest_uses_bgra_framebuffers() ? "bgra" : "rgba"},
234 {"androidboot.hardware.egl", "angle"},
235 {"androidboot.hardware.vulkan", "pastel"},
236 {"androidboot.opengles.version", "196609"}, // OpenGL ES 3.1
237 };
238 } else if (instance.gpu_mode() == kGpuModeGfxstream) {
239 LOG(INFO) << "We are in Gfxstream mode";
240 bootconfig_args = {
241 {"androidboot.cpuvulkan.version", "0"},
242 {"androidboot.hardware.gralloc", "minigbm"},
243 {"androidboot.hardware.hwcomposer", "ranchu"},
244 {"androidboot.hardware.hwcomposer.display_finder_mode", "gem5"},
245 {"androidboot.hardware.hwcomposer.display_framebuffer_format",
246 instance.guest_uses_bgra_framebuffers() ? "bgra" : "rgba"},
247 {"androidboot.hardware.egl", "emulation"},
248 {"androidboot.hardware.vulkan", "ranchu"},
249 {"androidboot.hardware.gltransport", "virtio-gpu-pipe"},
250 {"androidboot.opengles.version", "196609"}, // OpenGL ES 3.1
251 };
252 } else if (instance.gpu_mode() == kGpuModeNone) {
253 return {};
254 } else {
255 return CF_ERR("Unknown GPU mode " << instance.gpu_mode());
256 }
257
258 if (!instance.gpu_angle_feature_overrides_enabled().empty()) {
259 bootconfig_args["androidboot.hardware.angle_feature_overrides_enabled"] =
260 instance.gpu_angle_feature_overrides_enabled();
261 }
262 if (!instance.gpu_angle_feature_overrides_disabled().empty()) {
263 bootconfig_args["androidboot.hardware.angle_feature_overrides_disabled"] =
264 instance.gpu_angle_feature_overrides_disabled();
265 }
266
267 return bootconfig_args;
268 }
269
270 Result<std::unordered_map<std::string, std::string>>
ConfigureBootDevices(const CuttlefishConfig::InstanceSpecific &)271 Gem5Manager::ConfigureBootDevices(
272 const CuttlefishConfig::InstanceSpecific& /*instance*/) {
273 switch (arch_) {
274 case Arch::Arm:
275 case Arch::Arm64:
276 return {{{"androidboot.boot_devices", "30000000.pci"}}};
277 // TODO: Add x86 support
278 default:
279 return CF_ERR("Unhandled arch");
280 }
281 }
282
StartCommands(const CuttlefishConfig & config,std::vector<VmmDependencyCommand * > &)283 Result<std::vector<MonitorCommand>> Gem5Manager::StartCommands(
284 const CuttlefishConfig& config, std::vector<VmmDependencyCommand*>&) {
285 auto instance = config.ForDefaultInstance();
286
287 std::string gem5_binary = instance.gem5_binary_dir();
288 switch (arch_) {
289 case Arch::Arm:
290 case Arch::Arm64:
291 gem5_binary += "/build/ARM/gem5.opt";
292 break;
293 case Arch::RiscV64:
294 gem5_binary += "/build/RISCV/gem5.opt";
295 break;
296 case Arch::X86:
297 case Arch::X86_64:
298 gem5_binary += "/build/X86/gem5.opt";
299 break;
300 }
301 // generate Gem5 starter_fs.py before we execute it
302 GenerateGem5File(config, instance);
303
304 Command gem5_cmd(gem5_binary);
305
306 // Always enable listeners, because auto mode will disable once it detects
307 // gem5 is not run interactively
308 gem5_cmd.AddParameter("--listener-mode=on");
309
310 // Add debug-flags and debug-file before the script (i.e. starter_fs.py).
311 // We check the flags are not empty first since they are optional
312 if(!config.gem5_debug_flags().empty()) {
313 gem5_cmd.AddParameter("--debug-flags=", config.gem5_debug_flags());
314 if(!instance.gem5_debug_file().empty()) {
315 gem5_cmd.AddParameter("--debug-file=", instance.gem5_debug_file());
316 }
317 }
318
319 gem5_cmd.AddParameter(instance.gem5_binary_dir(),
320 "/configs/example/arm/starter_fs.py");
321
322 // restore checkpoint case
323 if (instance.gem5_checkpoint_dir() != "") {
324 gem5_cmd.AddParameter("--restore=",
325 instance.gem5_checkpoint_dir());
326 }
327
328 gem5_cmd.AddParameter("--mem-size=", instance.memory_mb() * 1024ULL * 1024ULL);
329 for (const auto& disk : instance.virtual_disk_paths()) {
330 gem5_cmd.AddParameter("--disk-image=", disk);
331 }
332
333 gem5_cmd.AddEnvironmentVariable("M5_PATH", config.assembly_dir());
334
335 std::vector<MonitorCommand> commands;
336 commands.emplace_back(std::move(gem5_cmd), true);
337 return commands;
338 }
339
340 } // namespace vm_manager
341 } // namespace cuttlefish
342