use std::env;
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use std::process::Command;
use std::sync::Once;
use std::time::Duration;
use anyhow::anyhow;
use anyhow::bail;
use anyhow::Context;
use anyhow::Result;
use base::syslog;
use base::test_utils::check_can_sudo;
use crc32fast::hash;
use delegate::wire_format::DelegateMessage;
use delegate::wire_format::ExitStatus;
use delegate::wire_format::GuestToHostMessage;
use delegate::wire_format::HostToGuestMessage;
use delegate::wire_format::ProgramExit;
use log::info;
use log::Level;
use prebuilts::download_file;
use readclock::ClockValues;
use url::Url;
use crate::sys::SerialArgs;
use crate::sys::TestVmSys;
use crate::utils::run_with_timeout;
const PREBUILT_URL: &str = "https://storage.googleapis.com/crosvm/integration_tests";
#[cfg(target_arch = "x86_64")]
const ARCH: &str = "x86_64";
#[cfg(target_arch = "arm")]
const ARCH: &str = "arm";
#[cfg(target_arch = "aarch64")]
const ARCH: &str = "aarch64";
#[cfg(target_arch = "riscv64")]
const ARCH: &str = "riscv64";
const COMMUNICATION_TIMEOUT: Duration = Duration::from_secs(5);
const BOOT_TIMEOUT: Duration = Duration::from_secs(60);
const DEFAULT_COMMAND_TIMEOUT: Duration = Duration::from_secs(10);
fn prebuilt_version() -> &'static str {
include_str!("../../guest_under_test/PREBUILT_VERSION").trim()
}
fn kernel_prebuilt_url_string() -> Url {
Url::parse(&format!(
"{}/guest-bzimage-{}-{}",
PREBUILT_URL,
ARCH,
prebuilt_version()
))
.unwrap()
}
fn rootfs_prebuilt_url_string() -> Url {
Url::parse(&format!(
"{}/guest-rootfs-{}-{}",
PREBUILT_URL,
ARCH,
prebuilt_version()
))
.unwrap()
}
pub(super) fn local_path_from_url(url: &Url) -> PathBuf {
if url.scheme() == "file" {
return url.to_file_path().unwrap();
}
if url.scheme() != "http" && url.scheme() != "https" {
panic!("Only file, http, https URLs are supported for artifacts")
}
env::current_exe().unwrap().parent().unwrap().join(format!(
"e2e_prebuilt-{:x}-{:x}",
hash(url.as_str().as_bytes()),
hash(url.path().as_bytes())
))
}
#[must_use]
pub struct GuestProcess {
command: String,
timeout: Duration,
}
impl GuestProcess {
pub fn with_timeout(self, duration: Duration) -> Self {
Self {
timeout: duration,
..self
}
}
pub fn wait_ok(self, vm: &mut TestVm) -> Result<ProgramExit> {
let command = self.command.clone();
let result = self.wait_result(vm)?;
match &result.exit_status {
ExitStatus::Code(0) => Ok(result),
ExitStatus::Code(code) => {
bail!("Command `{}` terminated with exit code {}", command, code)
}
ExitStatus::Signal(code) => bail!("Command `{}` stopped with signal {}", command, code),
ExitStatus::None => bail!("Command `{}` stopped for unknown reason", command),
}
}
pub fn wait_result(self, vm: &mut TestVm) -> Result<ProgramExit> {
let message = vm.read_message_from_guest(self.timeout).with_context(|| {
format!(
"Command `{}`: Failed to read response from guest",
self.command
)
})?;
match message {
GuestToHostMessage::ProgramExit(exit_info) => Ok(exit_info),
_ => bail!("Receive other message when anticipating ProgramExit"),
}
}
}
pub struct Config {
pub(super) extra_args: Vec<String>,
pub(super) o_direct: bool,
pub(super) log_level: Level,
pub(super) log_file: Option<String>,
pub(super) wrapper_cmd: Option<String>,
pub(super) kernel_url: Url,
pub(super) initrd_url: Option<Url>,
pub(super) rootfs_url: Option<Url>,
pub(super) rootfs_rw: bool,
pub(super) rootfs_compressed: bool,
pub(super) console_hardware: String,
}
impl Default for Config {
fn default() -> Self {
Self {
log_level: Level::Info,
extra_args: Default::default(),
o_direct: Default::default(),
log_file: None,
wrapper_cmd: None,
kernel_url: kernel_prebuilt_url_string(),
initrd_url: None,
rootfs_url: Some(rootfs_prebuilt_url_string()),
rootfs_rw: false,
rootfs_compressed: false,
console_hardware: "virtio-console".to_owned(),
}
}
}
impl Config {
pub fn new() -> Self {
Self::from_env()
}
pub fn extra_args(mut self, args: Vec<String>) -> Self {
let mut args = args;
self.extra_args.append(&mut args);
self
}
pub fn o_direct(mut self) -> Self {
self.o_direct = true;
self
}
pub fn disable_sandbox(mut self) -> Self {
self.extra_args.push("--disable-sandbox".to_string());
self
}
pub fn from_env() -> Self {
let mut cfg: Config = Default::default();
if let Ok(wrapper_cmd) = env::var("CROSVM_CARGO_TEST_E2E_WRAPPER_CMD") {
cfg.wrapper_cmd = Some(wrapper_cmd);
}
if let Ok(log_file) = env::var("CROSVM_CARGO_TEST_LOG_FILE") {
cfg.log_file = Some(log_file);
}
if env::var("CROSVM_CARGO_TEST_LOG_LEVEL_DEBUG").is_ok() {
cfg.log_level = Level::Debug;
}
if let Ok(kernel_url) = env::var("CROSVM_CARGO_TEST_KERNEL_IMAGE") {
info!("Using overrided kernel from env CROSVM_CARGO_TEST_KERNEL_IMAGE={kernel_url}");
cfg.kernel_url = Url::from_file_path(kernel_url).unwrap();
}
if let Ok(initrd_url) = env::var("CROSVM_CARGO_TEST_INITRD_IMAGE") {
info!("Using overrided kernel from env CROSVM_CARGO_TEST_INITRD_IMAGE={initrd_url}");
cfg.initrd_url = Some(Url::from_file_path(initrd_url).unwrap());
}
if let Ok(rootfs_url) = env::var("CROSVM_CARGO_TEST_ROOTFS_IMAGE") {
info!("Using overrided kernel from env CROSVM_CARGO_TEST_ROOTFS_IMAGE={rootfs_url}");
cfg.rootfs_url = Some(Url::from_file_path(rootfs_url).unwrap());
}
cfg
}
pub fn with_kernel(mut self, url: &str) -> Self {
self.kernel_url = Url::parse(url).unwrap();
self
}
pub fn with_initrd(mut self, url: &str) -> Self {
self.initrd_url = Some(Url::parse(url).unwrap());
self
}
pub fn with_rootfs(mut self, url: &str) -> Self {
self.rootfs_url = Some(Url::parse(url).unwrap());
self
}
pub fn rootfs_is_rw(mut self) -> Self {
self.rootfs_rw = true;
self
}
pub fn rootfs_is_compressed(mut self) -> Self {
self.rootfs_compressed = true;
self
}
pub fn with_stdout_hardware(mut self, hw_type: &str) -> Self {
self.console_hardware = hw_type.into();
self
}
pub fn with_vhost_user(mut self, device_type: &str, socket_path: &Path) -> Self {
self.extra_args.push("--vhost-user".to_string());
self.extra_args.push(format!(
"{},socket={}",
device_type,
socket_path.to_str().unwrap()
));
self
}
#[cfg(any(target_os = "android", target_os = "linux"))]
pub fn with_vhost_user_fs(mut self, socket_path: &Path, tag: &str) -> Self {
self.extra_args.push("--vhost-user-fs".to_string());
self.extra_args
.push(format!("{},tag={}", socket_path.to_str().unwrap(), tag));
self
}
}
static PREP_ONCE: Once = Once::new();
pub struct TestVm {
sys: TestVmSys,
ready: bool,
sudo: bool,
}
impl TestVm {
fn initialize_once() {
if let Err(e) = syslog::init() {
panic!("failed to initiailize syslog: {}", e);
}
info!("Prebuilt version to be used: {}", prebuilt_version());
if let Ok(value) = env::var("CROSVM_CARGO_TEST_PREBUILT_VERSION") {
if value != prebuilt_version() {
panic!(
"Environment provided prebuilts are version {}, but crosvm was compiled \
for prebuilt version {}. Did you update PREBUILT_VERSION everywhere?",
value,
prebuilt_version()
);
}
}
}
fn initiailize_artifacts(cfg: &Config) {
let kernel_path = local_path_from_url(&cfg.kernel_url);
if !kernel_path.exists() && cfg.kernel_url.scheme() != "file" {
download_file(cfg.kernel_url.as_str(), &kernel_path).unwrap();
}
assert!(kernel_path.exists(), "{:?} does not exist", kernel_path);
if let Some(initrd_url) = &cfg.initrd_url {
let initrd_path = local_path_from_url(initrd_url);
if !initrd_path.exists() && initrd_url.scheme() != "file" {
download_file(initrd_url.as_str(), &initrd_path).unwrap();
}
assert!(initrd_path.exists(), "{:?} does not exist", initrd_path);
}
if let Some(rootfs_url) = &cfg.rootfs_url {
let rootfs_download_path = local_path_from_url(rootfs_url);
if !rootfs_download_path.exists() && rootfs_url.scheme() != "file" {
download_file(rootfs_url.as_str(), &rootfs_download_path).unwrap();
}
assert!(
rootfs_download_path.exists(),
"{:?} does not exist",
rootfs_download_path
);
if cfg.rootfs_compressed {
let rootfs_raw_path = rootfs_download_path.with_extension("raw");
Command::new("zstd")
.arg("-d")
.arg(&rootfs_download_path)
.arg("-o")
.arg(&rootfs_raw_path)
.arg("-f")
.output()
.expect("Failed to decompress rootfs");
TestVmSys::check_rootfs_file(&rootfs_raw_path);
} else {
TestVmSys::check_rootfs_file(&rootfs_download_path);
}
}
}
pub fn new_generic<F>(f: F, cfg: Config, sudo: bool) -> Result<TestVm>
where
F: FnOnce(&mut Command, &SerialArgs, &Config) -> Result<()>,
{
PREP_ONCE.call_once(TestVm::initialize_once);
TestVm::initiailize_artifacts(&cfg);
let mut vm = TestVm {
sys: TestVmSys::new_generic(f, cfg, sudo).with_context(|| "Could not start crosvm")?,
ready: false,
sudo,
};
vm.wait_for_guest_ready(BOOT_TIMEOUT)
.with_context(|| "Guest did not become ready after boot")?;
Ok(vm)
}
pub fn new_generic_restore<F>(f: F, cfg: Config, sudo: bool) -> Result<TestVm>
where
F: FnOnce(&mut Command, &SerialArgs, &Config) -> Result<()>,
{
PREP_ONCE.call_once(TestVm::initialize_once);
let mut vm = TestVm {
sys: TestVmSys::new_generic(f, cfg, sudo).with_context(|| "Could not start crosvm")?,
ready: false,
sudo,
};
vm.ready = true;
Ok(vm)
}
pub fn new(cfg: Config) -> Result<TestVm> {
TestVm::new_generic(TestVmSys::append_config_args, cfg, false)
}
pub fn new_restore(cfg: Config) -> Result<TestVm> {
let mut vm = TestVm::new_generic_restore(TestVmSys::append_config_args, cfg, false)?;
let vm = run_with_timeout(
move || {
vm.resume_full().expect("failed to resume after VM restore");
vm
},
Duration::from_secs(60),
)
.expect("VM restore timeout");
Ok(vm)
}
pub fn new_restore_suspended(cfg: Config) -> Result<TestVm> {
TestVm::new_generic_restore(TestVmSys::append_config_args, cfg, false)
}
pub fn new_sudo(cfg: Config) -> Result<TestVm> {
check_can_sudo();
TestVm::new_generic(TestVmSys::append_config_args, cfg, true)
}
pub fn new_with_config_file(cfg: Config) -> Result<TestVm> {
TestVm::new_generic(TestVmSys::append_config_file_arg, cfg, false)
}
pub fn exec_in_guest(&mut self, command: &str) -> Result<ProgramExit> {
self.exec_in_guest_async(command)?.wait_ok(self)
}
pub fn exec_in_guest_unchecked(&mut self, command: &str) -> Result<ProgramExit> {
self.exec_in_guest_async(command)?.wait_result(self)
}
pub fn exec_in_guest_async(&mut self, command: &str) -> Result<GuestProcess> {
assert!(self.ready);
self.ready = false;
self.write_message_to_guest(
&HostToGuestMessage::RunCommand {
command: command.to_owned(),
},
COMMUNICATION_TIMEOUT,
)
.with_context(|| format!("Command `{}`: Failed to write to guest pipe", command))?;
Ok(GuestProcess {
command: command.to_owned(),
timeout: DEFAULT_COMMAND_TIMEOUT,
})
}
fn wait_for_guest_ready(&mut self, timeout: Duration) -> Result<()> {
assert!(!self.ready);
let message: GuestToHostMessage = self.read_message_from_guest(timeout)?;
match message {
GuestToHostMessage::Ready => {
self.ready = true;
Ok(())
}
_ => Err(anyhow!("Recevied unexpected data from delegate")),
}
}
fn read_message_from_guest(&mut self, timeout: Duration) -> Result<GuestToHostMessage> {
let reader = self.sys.from_guest_reader.clone();
let result = run_with_timeout(
move || loop {
let message = { reader.lock().unwrap().next() };
if let Some(message_result) = message {
if let Ok(msg) = message_result {
match msg {
DelegateMessage::GuestToHost(guest_to_host) => {
return Ok(guest_to_host);
}
DelegateMessage::HostToGuest(_) => {
continue;
}
}
} else {
bail!(format!(
"Failed to receive message from guest: {:?}",
message_result.unwrap_err()
))
};
};
},
timeout,
);
match result {
Ok(x) => {
self.ready = true;
x
}
Err(x) => Err(x),
}
}
fn write_message_to_guest(
&mut self,
data: &HostToGuestMessage,
timeout: Duration,
) -> Result<()> {
let writer = self.sys.to_guest.clone();
let data_str = serde_json::to_string_pretty(&DelegateMessage::HostToGuest(data.clone()))?;
run_with_timeout(
move || -> Result<()> {
println!("-> {}", &data_str);
{
writeln!(writer.lock().unwrap(), "{}", &data_str)?;
}
Ok(())
},
timeout,
)?
}
pub fn hotplug_tap(&mut self, tap_name: &str) -> Result<()> {
self.sys
.crosvm_command(
"virtio-net",
vec!["add".to_owned(), tap_name.to_owned()],
self.sudo,
)
.map(|_| ())
}
pub fn remove_pci_device(&mut self, bus_num: u8) -> Result<()> {
self.sys
.crosvm_command(
"virtio-net",
vec!["remove".to_owned(), bus_num.to_string()],
self.sudo,
)
.map(|_| ())
}
pub fn stop(&mut self) -> Result<()> {
self.sys
.crosvm_command("stop", vec![], self.sudo)
.map(|_| ())
}
pub fn suspend(&mut self) -> Result<()> {
self.sys
.crosvm_command("suspend", vec![], self.sudo)
.map(|_| ())
}
pub fn suspend_full(&mut self) -> Result<()> {
self.sys
.crosvm_command("suspend", vec!["--full".to_string()], self.sudo)
.map(|_| ())
}
pub fn resume(&mut self) -> Result<()> {
self.sys
.crosvm_command("resume", vec![], self.sudo)
.map(|_| ())
}
pub fn resume_full(&mut self) -> Result<()> {
self.sys
.crosvm_command("resume", vec!["--full".to_string()], self.sudo)
.map(|_| ())
}
pub fn disk(&mut self, args: Vec<String>) -> Result<()> {
self.sys.crosvm_command("disk", args, self.sudo).map(|_| ())
}
pub fn snapshot(&mut self, filename: &std::path::Path) -> Result<()> {
self.sys
.crosvm_command(
"snapshot",
vec!["take".to_string(), String::from(filename.to_str().unwrap())],
self.sudo,
)
.map(|_| ())
}
pub fn restore(&mut self, filename: &std::path::Path) -> Result<()> {
self.sys
.crosvm_command(
"snapshot",
vec![
"restore".to_string(),
String::from(filename.to_str().unwrap()),
],
self.sudo,
)
.map(|_| ())
}
pub fn swap_command(&mut self, command: &str) -> Result<Vec<u8>> {
self.sys
.crosvm_command("swap", vec![command.to_string()], self.sudo)
}
pub fn guest_clock_values(&mut self) -> Result<ClockValues> {
let output = self
.exec_in_guest("readclock")
.context("Failed to execute readclock binary")?;
serde_json::from_str(&output.stdout).context("Failed to parse result")
}
}
impl Drop for TestVm {
fn drop(&mut self) {
self.stop().unwrap();
let status = self.sys.process.take().unwrap().wait().unwrap();
if !status.success() {
panic!("VM exited illegally: {}", status);
}
}
}