1use std::collections::BTreeMap;
10use std::convert::TryFrom;
11use std::fs;
12use std::sync::Arc;
13use std::time::Duration;
14
15use anyhow::Context;
16use base::custom_serde::serialize_arc_mutex;
17use base::debug;
18use base::error;
19use base::warn;
20use base::AsRawDescriptor;
21use base::Descriptor;
22use base::Error as SysError;
23use base::Event;
24use base::EventToken;
25use base::SendTube;
26use base::Timer;
27use base::TimerTrait;
28use base::Tube;
29use base::VmEventType;
30use base::WaitContext;
31use base::WorkerThread;
32use serde::Deserialize;
33use serde::Serialize;
34use snapshot::AnySnapshot;
35use sync::Mutex;
36use vm_control::DeviceId;
37use vm_control::PlatformDeviceId;
38use vm_control::VmResponse;
39
40use crate::BusAccessInfo;
41use crate::BusDevice;
42use crate::IrqEdgeEvent;
43use crate::Suspendable;
44
45const VMWDT_REG_STATUS: u32 = 0x00;
47const VMWDT_REG_LOAD_CNT: u32 = 0x04;
48const VMWDT_REG_CURRENT_CNT: u32 = 0x08;
49const VMWDT_REG_CLOCK_FREQ_HZ: u32 = 0x0C;
50
51const VMWDT_REG_LEN: u64 = 0x10;
53
54pub const VMWDT_DEFAULT_TIMEOUT_SEC: u32 = 10;
55pub const VMWDT_DEFAULT_CLOCK_HZ: u32 = 2;
56
57const PROCSTAT_GUEST_TIME_INDX: usize = 42;
59
60#[derive(Serialize)]
61pub struct VmwdtPerCpu {
62 is_enabled: bool,
64 #[serde(skip_serializing)]
66 timer: Timer,
67 timer_freq_hz: u64,
69 last_guest_time_ms: i64,
71 thread_id: u32,
73 process_id: u32,
75 next_expiration_interval_ms: i64,
78 stall_evt_ppi_triggered: bool,
80 repeating_interval: Option<Duration>,
82}
83
84#[derive(Deserialize)]
85struct VmwdtPerCpuRestore {
86 is_enabled: bool,
87 timer_freq_hz: u64,
88 last_guest_time_ms: i64,
89 next_expiration_interval_ms: i64,
90 repeating_interval: Option<Duration>,
91}
92
93pub struct Vmwdt {
94 vm_wdts: Arc<Mutex<Vec<VmwdtPerCpu>>>,
95 worker_thread: Option<WorkerThread<Tube>>,
97 reset_evt_wrtube: SendTube,
100 activated: bool,
101 stall_evt: IrqEdgeEvent,
103 vm_ctrl_tube: Option<Tube>,
104}
105
106#[derive(Serialize)]
107struct VmwdtSnapshot {
108 #[serde(serialize_with = "serialize_arc_mutex")]
109 vm_wdts: Arc<Mutex<Vec<VmwdtPerCpu>>>,
110 activated: bool,
111}
112
113#[derive(Deserialize)]
114struct VmwdtRestore {
115 vm_wdts: Vec<VmwdtPerCpuRestore>,
116 activated: bool,
117}
118
119impl Vmwdt {
120 pub fn new(
121 cpu_count: usize,
122 reset_evt_wrtube: SendTube,
123 evt: IrqEdgeEvent,
124 vm_ctrl_tube: Tube,
125 ) -> anyhow::Result<Vmwdt> {
126 let mut vec = Vec::new();
127 for _ in 0..cpu_count {
128 vec.push(VmwdtPerCpu {
129 last_guest_time_ms: 0,
130 thread_id: 0,
131 process_id: 0,
132 is_enabled: false,
133 stall_evt_ppi_triggered: false,
134 timer: Timer::new().context("failed to create Timer")?,
135 timer_freq_hz: 0,
136 next_expiration_interval_ms: 0,
137 repeating_interval: None,
138 });
139 }
140 let vm_wdts = Arc::new(Mutex::new(vec));
141
142 Ok(Vmwdt {
143 vm_wdts,
144 worker_thread: None,
145 reset_evt_wrtube,
146 activated: false,
147 stall_evt: evt,
148 vm_ctrl_tube: Some(vm_ctrl_tube),
149 })
150 }
151
152 pub fn vmwdt_worker_thread(
153 vm_wdts: Arc<Mutex<Vec<VmwdtPerCpu>>>,
154 kill_evt: Event,
155 reset_evt_wrtube: SendTube,
156 stall_evt: IrqEdgeEvent,
157 vm_ctrl_tube: Tube,
158 worker_started_send: Option<SendTube>,
159 ) -> anyhow::Result<Tube> {
160 let msg = vm_control::VmRequest::VcpuPidTid;
161 vm_ctrl_tube
162 .send(&msg)
163 .context("failed to send request to fetch Vcpus PID and TID")?;
164 let vcpus_pid_tid: BTreeMap<usize, (u32, u32)> = match vm_ctrl_tube
165 .recv()
166 .context("failed to receive vmwdt pids and tids")?
167 {
168 VmResponse::VcpuPidTidResponse { pid_tid_map } => pid_tid_map,
169 _ => {
170 return Err(anyhow::anyhow!(
171 "Receive incorrect message type when trying to get vcpu pid tid map"
172 ));
173 }
174 };
175 {
176 let mut vm_wdts = vm_wdts.lock();
177 for (i, vmwdt) in (*vm_wdts).iter_mut().enumerate() {
178 let pid_tid = vcpus_pid_tid
179 .get(&i)
180 .context("vmwdts empty, which could indicate no vcpus are initialized")?;
181 vmwdt.process_id = pid_tid.0;
182 vmwdt.thread_id = pid_tid.1;
183 }
184 }
185 if let Some(worker_started_send) = worker_started_send {
186 worker_started_send
187 .send(&())
188 .context("failed to send vmwdt worker started")?;
189 }
190 #[derive(EventToken)]
191 enum Token {
192 Kill,
193 Timer(usize),
194 }
195
196 let wait_ctx: WaitContext<Token> =
197 WaitContext::new().context("Failed to create wait_ctx")?;
198 wait_ctx
199 .add(&kill_evt, Token::Kill)
200 .context("Failed to add Tokens to wait_ctx")?;
201
202 let len = vm_wdts.lock().len();
203 for clock_id in 0..len {
204 let timer_fd = vm_wdts.lock()[clock_id].timer.as_raw_descriptor();
205 wait_ctx
206 .add(&Descriptor(timer_fd), Token::Timer(clock_id))
207 .context("Failed to link FDs to Tokens")?;
208 }
209
210 loop {
211 let events = wait_ctx.wait().context("Failed to wait for events")?;
212 for event in events.iter().filter(|e| e.is_readable) {
213 match event.token {
214 Token::Kill => {
215 return Ok(vm_ctrl_tube);
216 }
217 Token::Timer(cpu_id) => {
218 let mut wdts_locked = vm_wdts.lock();
219 let watchdog = &mut wdts_locked[cpu_id];
220 match watchdog.timer.mark_waited() {
221 Ok(true) => continue, Ok(false) => {}
223 Err(e) => {
224 error!("error waiting for timer event on vcpu {cpu_id}: {e:#}");
225 continue;
226 }
227 }
228
229 let current_guest_time_ms =
230 match Vmwdt::get_guest_time_ms(watchdog.process_id, watchdog.thread_id)
231 {
232 Ok(value) => value,
233 Err(e) => {
234 error!("get_guest_time_ms returned error: {}", e);
235 if e.errno() == libc::ENOENT {
239 return Ok(vm_ctrl_tube);
240 } else {
241 watchdog.last_guest_time_ms
242 }
243 }
244 };
245 let remaining_time_ms = watchdog.next_expiration_interval_ms
246 - (current_guest_time_ms - watchdog.last_guest_time_ms);
247
248 if remaining_time_ms > 0 {
249 watchdog.next_expiration_interval_ms = remaining_time_ms;
250 if let Err(e) = watchdog
251 .timer
252 .reset_oneshot(Duration::from_millis(remaining_time_ms as u64))
253 {
254 error!(
255 "failed to reset internal timer on vcpu {}: {:#}",
256 cpu_id, e
257 );
258 }
259 watchdog.repeating_interval = None;
260 } else {
261 if watchdog.stall_evt_ppi_triggered {
262 if let Err(e) = reset_evt_wrtube
263 .send::<VmEventType>(&VmEventType::WatchdogReset)
264 {
265 error!("{} failed to send reset event from vcpu {}", e, cpu_id)
266 }
267 }
268
269 stall_evt
270 .trigger()
271 .context("Failed to trigger stall event")?;
272 watchdog.stall_evt_ppi_triggered = true;
273 watchdog.last_guest_time_ms = current_guest_time_ms;
274 }
275 }
276 }
277 }
278 }
279 }
280
281 fn start(&mut self, worker_started_send: Option<SendTube>) -> anyhow::Result<()> {
282 let vm_wdts = self.vm_wdts.clone();
283 let reset_evt_wrtube = self.reset_evt_wrtube.try_clone().unwrap();
284 let stall_event = self.stall_evt.try_clone().unwrap();
285 let vm_ctrl_tube = self
286 .vm_ctrl_tube
287 .take()
288 .context("missing vm control tube")?;
289
290 self.activated = true;
291 self.worker_thread = Some(WorkerThread::start("vmwdt worker", |kill_evt| {
292 Vmwdt::vmwdt_worker_thread(
293 vm_wdts,
294 kill_evt,
295 reset_evt_wrtube,
296 stall_event,
297 vm_ctrl_tube,
298 worker_started_send,
299 )
300 .expect("failed to start vmwdt worker thread")
301 }));
302 Ok(())
303 }
304
305 fn ensure_started(&mut self) {
306 if self.worker_thread.is_some() {
307 return;
308 }
309
310 let (worker_started_send, worker_started_recv) =
311 Tube::directional_pair().expect("failed to create vmwdt worker started tubes");
312 self.start(Some(worker_started_send))
313 .expect("failed to start Vmwdt");
314 worker_started_recv
315 .recv::<()>()
316 .expect("failed to receive vmwdt worker started");
317 }
318
319 #[cfg(any(target_os = "linux", target_os = "android"))]
320 pub fn get_guest_time_ms(process_id: u32, thread_id: u32) -> Result<i64, SysError> {
321 let stat_path = format!("/proc/{process_id}/task/{thread_id}/stat");
323 let contents = fs::read_to_string(stat_path)?;
324
325 let gtime_ticks = contents
326 .split_whitespace()
327 .nth(PROCSTAT_GUEST_TIME_INDX)
328 .and_then(|guest_time| guest_time.parse::<u64>().ok())
329 .unwrap_or(0);
330
331 let ticks_per_sec = unsafe { libc::sysconf(libc::_SC_CLK_TCK) } as u64;
334 Ok((gtime_ticks * 1000 / ticks_per_sec) as i64)
335 }
336
337 #[cfg(not(any(target_os = "linux", target_os = "android")))]
338 pub fn get_guest_time_ms(process_id: u32, thread_id: u32) -> Result<i64, SysError> {
339 Ok(0)
340 }
341}
342
343impl BusDevice for Vmwdt {
344 fn debug_label(&self) -> String {
345 "Vmwdt".to_owned()
346 }
347
348 fn device_id(&self) -> DeviceId {
349 PlatformDeviceId::VmWatchdog.into()
350 }
351
352 fn read(&mut self, _offset: BusAccessInfo, _data: &mut [u8]) {}
353
354 fn write(&mut self, info: BusAccessInfo, data: &[u8]) {
355 let data_array = match <&[u8; 4]>::try_from(data) {
356 Ok(array) => array,
357 _ => {
358 error!("Bad write size: {} for vmwdt", data.len());
359 return;
360 }
361 };
362
363 let reg_val = u32::from_ne_bytes(*data_array);
364 let cpu_index: usize = (info.offset / VMWDT_REG_LEN) as usize;
365 let reg_offset = (info.offset % VMWDT_REG_LEN) as u32;
366
367 if cpu_index > self.vm_wdts.lock().len() {
368 error!("Bad write cpu_index {}", cpu_index);
369 return;
370 }
371
372 match reg_offset {
373 VMWDT_REG_STATUS => {
374 self.ensure_started();
375 let mut wdts_locked = self.vm_wdts.lock();
376 let cpu_watchdog = &mut wdts_locked[cpu_index];
377
378 cpu_watchdog.is_enabled = reg_val != 0;
379
380 if reg_val != 0 {
381 let interval = Duration::from_millis(1000 / cpu_watchdog.timer_freq_hz);
382 cpu_watchdog.repeating_interval = Some(interval);
383 cpu_watchdog
384 .timer
385 .reset_repeating(interval)
386 .expect("Failed to reset timer repeating interval");
387 } else {
388 cpu_watchdog.repeating_interval = None;
389 cpu_watchdog
390 .timer
391 .clear()
392 .expect("Failed to clear cpu watchdog timer");
393 }
394 }
395 VMWDT_REG_LOAD_CNT => {
396 self.ensure_started();
397 let mut wdts_locked = self.vm_wdts.lock();
398 let cpu_watchdog = &mut wdts_locked[cpu_index];
399 let process_id = cpu_watchdog.process_id;
400 let thread_id = cpu_watchdog.thread_id;
401 let guest_time_ms = Vmwdt::get_guest_time_ms(process_id, thread_id)
402 .expect("get_guest_time_ms failed");
403 let next_expiration_interval_ms =
404 reg_val as u64 * 1000 / cpu_watchdog.timer_freq_hz;
405
406 cpu_watchdog.last_guest_time_ms = guest_time_ms;
407 cpu_watchdog.stall_evt_ppi_triggered = false;
408 cpu_watchdog.next_expiration_interval_ms = next_expiration_interval_ms as i64;
409
410 if cpu_watchdog.is_enabled {
411 if let Err(_e) = cpu_watchdog
412 .timer
413 .reset_oneshot(Duration::from_millis(next_expiration_interval_ms))
414 {
415 error!("failed to reset one-shot vcpu time {}", cpu_index);
416 }
417 cpu_watchdog.repeating_interval = None;
418 }
419 }
420 VMWDT_REG_CURRENT_CNT => {
421 warn!("invalid write to read-only VMWDT_REG_CURRENT_CNT register");
422 }
423 VMWDT_REG_CLOCK_FREQ_HZ => {
424 let mut wdts_locked = self.vm_wdts.lock();
425 let cpu_watchdog = &mut wdts_locked[cpu_index];
426
427 debug!(
428 "CPU:{:x} wrote VMWDT_REG_CLOCK_FREQ_HZ {:x}",
429 cpu_index, reg_val
430 );
431 cpu_watchdog.timer_freq_hz = reg_val as u64;
432 }
433 _ => unreachable!(),
434 }
435 }
436}
437
438impl Suspendable for Vmwdt {
439 fn sleep(&mut self) -> anyhow::Result<()> {
440 if let Some(worker) = self.worker_thread.take() {
441 self.vm_ctrl_tube = Some(worker.stop());
442 }
443 Ok(())
444 }
445
446 fn wake(&mut self) -> anyhow::Result<()> {
447 if self.activated {
448 self.start(None)?;
455 let mut vm_wdts = self.vm_wdts.lock();
456 for vmwdt in vm_wdts.iter_mut() {
457 if let Some(interval) = &vmwdt.repeating_interval {
458 vmwdt
459 .timer
460 .reset_repeating(*interval)
461 .context("failed to write repeating interval")?;
462 } else if vmwdt.is_enabled {
463 vmwdt
464 .timer
465 .reset_oneshot(Duration::from_millis(
466 vmwdt.next_expiration_interval_ms as u64,
467 ))
468 .context("failed to write oneshot interval")?;
469 }
470 }
471 }
472 Ok(())
473 }
474
475 fn snapshot(&mut self) -> anyhow::Result<AnySnapshot> {
476 AnySnapshot::to_any(&VmwdtSnapshot {
477 vm_wdts: self.vm_wdts.clone(),
478 activated: self.activated,
479 })
480 .context("failed to snapshot Vmwdt")
481 }
482
483 fn restore(&mut self, data: AnySnapshot) -> anyhow::Result<()> {
484 let deser: VmwdtRestore =
485 AnySnapshot::from_any(data).context("failed to deserialize Vmwdt")?;
486 let mut vm_wdts = self.vm_wdts.lock();
487 for (vmwdt_restore, vmwdt) in deser.vm_wdts.iter().zip(vm_wdts.iter_mut()) {
488 vmwdt.is_enabled = vmwdt_restore.is_enabled;
489 vmwdt.timer_freq_hz = vmwdt_restore.timer_freq_hz;
490 vmwdt.last_guest_time_ms = vmwdt_restore.last_guest_time_ms;
491 vmwdt.next_expiration_interval_ms = vmwdt_restore.next_expiration_interval_ms;
492 vmwdt.repeating_interval = vmwdt_restore.repeating_interval;
493 }
494 self.activated = deser.activated;
495 Ok(())
496 }
497}
498
499#[cfg(test)]
500mod tests {
501 use std::process;
502 use std::thread::sleep;
503
504 #[cfg(any(target_os = "linux", target_os = "android"))]
505 use base::gettid;
506 use base::poll_assert;
507 use base::Tube;
508
509 use super::*;
510
511 const AARCH64_VMWDT_ADDR: u64 = 0x3000;
512 const TEST_VMWDT_CPU_NO: usize = 0x1;
513
514 fn vmwdt_bus_address(offset: u64) -> BusAccessInfo {
515 BusAccessInfo {
516 offset,
517 address: AARCH64_VMWDT_ADDR,
518 id: 0,
519 }
520 }
521
522 #[test]
523 fn test_watchdog_internal_timer() {
524 let (vm_evt_wrtube, _vm_evt_rdtube) = Tube::directional_pair().unwrap();
525 let (vm_ctrl_wrtube, vm_ctrl_rdtube) = Tube::pair().unwrap();
526 let irq = IrqEdgeEvent::new().unwrap();
527 #[cfg(any(target_os = "linux", target_os = "android"))]
528 {
529 vm_ctrl_wrtube
530 .send(&VmResponse::VcpuPidTidResponse {
531 pid_tid_map: BTreeMap::from([(0, (process::id(), gettid() as u32))]),
532 })
533 .unwrap();
534 }
535 let mut device = Vmwdt::new(TEST_VMWDT_CPU_NO, vm_evt_wrtube, irq, vm_ctrl_rdtube).unwrap();
536
537 device.write(
539 vmwdt_bus_address(VMWDT_REG_CLOCK_FREQ_HZ as u64),
540 &[10, 0, 0, 0],
541 );
542 device.write(vmwdt_bus_address(VMWDT_REG_LOAD_CNT as u64), &[1, 0, 0, 0]);
543 device.write(vmwdt_bus_address(VMWDT_REG_STATUS as u64), &[1, 0, 0, 0]);
544 let next_expiration_ms = {
545 let mut vmwdt_locked = device.vm_wdts.lock();
546 vmwdt_locked[0].last_guest_time_ms = 10;
549 vmwdt_locked[0].next_expiration_interval_ms
550 };
551
552 poll_assert!(10, || {
554 sleep(Duration::from_millis(50));
555 let vmwdt_locked = device.vm_wdts.lock();
556 vmwdt_locked[0].next_expiration_interval_ms != next_expiration_ms
558 });
559 }
560
561 #[test]
562 fn test_watchdog_expiration() {
563 let (vm_evt_wrtube, vm_evt_rdtube) = Tube::directional_pair().unwrap();
564 let (vm_ctrl_wrtube, vm_ctrl_rdtube) = Tube::pair().unwrap();
565 let irq = IrqEdgeEvent::new().unwrap();
566 #[cfg(any(target_os = "linux", target_os = "android"))]
567 {
568 vm_ctrl_wrtube
569 .send(&VmResponse::VcpuPidTidResponse {
570 pid_tid_map: BTreeMap::from([(0, (process::id(), gettid() as u32))]),
571 })
572 .unwrap();
573 }
574 let mut device = Vmwdt::new(TEST_VMWDT_CPU_NO, vm_evt_wrtube, irq, vm_ctrl_rdtube).unwrap();
575
576 device.write(
578 vmwdt_bus_address(VMWDT_REG_CLOCK_FREQ_HZ as u64),
579 &[10, 0, 0, 0],
580 );
581 device.write(vmwdt_bus_address(VMWDT_REG_LOAD_CNT as u64), &[1, 0, 0, 0]);
582 device.write(vmwdt_bus_address(VMWDT_REG_STATUS as u64), &[1, 0, 0, 0]);
583 device.vm_wdts.lock()[0].last_guest_time_ms = -100;
586
587 poll_assert!(10, || {
589 sleep(Duration::from_millis(50));
590 let vmwdt_locked = device.vm_wdts.lock();
591 vmwdt_locked[0].stall_evt_ppi_triggered
592 });
593
594 device.vm_wdts.lock()[0].last_guest_time_ms = -100;
596
597 poll_assert!(10, || {
599 sleep(Duration::from_millis(50));
600 match vm_evt_rdtube.recv::<VmEventType>() {
601 Ok(vm_event) => vm_event == VmEventType::WatchdogReset,
602 Err(_e) => false,
603 }
604 });
605 }
606
607 #[test]
608 fn test_watchdog_vcpu_death() {
611 let (vm_evt_wrtube, _vm_evt_rdtube) = Tube::directional_pair().unwrap();
612 let (vm_ctrl_wrtube, vm_ctrl_rdtube) = Tube::pair().unwrap();
613 let irq = IrqEdgeEvent::new().unwrap();
614
615 let (tid_tx, tid_rx) = std::sync::mpsc::channel();
617 let (exit_tx, exit_rx) = std::sync::mpsc::channel();
618 let helper_thread = std::thread::spawn(move || {
619 let tid = base::gettid() as u32;
620 tid_tx.send(tid).unwrap();
621 exit_rx.recv().unwrap(); });
623 let helper_tid = tid_rx.recv().unwrap();
624
625 #[cfg(any(target_os = "linux", target_os = "android"))]
626 {
627 vm_ctrl_wrtube
628 .send(&VmResponse::VcpuPidTidResponse {
629 pid_tid_map: BTreeMap::from([(0, (process::id(), helper_tid))]),
630 })
631 .unwrap();
632 }
633 let mut device = Vmwdt::new(TEST_VMWDT_CPU_NO, vm_evt_wrtube, irq, vm_ctrl_rdtube).unwrap();
634
635 device.write(
637 vmwdt_bus_address(VMWDT_REG_CLOCK_FREQ_HZ as u64),
638 &[10, 0, 0, 0],
639 );
640
641 device.write(vmwdt_bus_address(VMWDT_REG_LOAD_CNT as u64), &[1, 0, 0, 0]);
644
645 exit_tx.send(()).unwrap();
647 helper_thread.join().unwrap();
648
649 device.write(vmwdt_bus_address(VMWDT_REG_STATUS as u64), &[1, 0, 0, 0]);
651
652 sleep(Duration::from_millis(200));
656
657 std::mem::drop(device);
659 }
660}