devices/
vmwdt.rs

1// Copyright 2022 The ChromiumOS Authors
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5//! vmwdt is a virtual watchdog memory mapped device which detects stalls
6//! on the vCPUs and resets the guest when no 'pet' events are received.
7//! <https://docs.google.com/document/d/1DYmk2roxlwHZsOfcJi8xDMdWOHAmomvs2SDh7KPud3Y/edit?usp=sharing&resourcekey=0-oSNabc-t040a1q0K4cyI8Q>
8
9use std::collections::BTreeMap;
10use std::convert::TryFrom;
11use std::fs;
12use std::sync::Arc;
13use std::time::Duration;
14
15use anyhow::Context;
16use base::custom_serde::serialize_arc_mutex;
17use base::debug;
18use base::error;
19use base::warn;
20use base::AsRawDescriptor;
21use base::Descriptor;
22use base::Error as SysError;
23use base::Event;
24use base::EventToken;
25use base::SendTube;
26use base::Timer;
27use base::TimerTrait;
28use base::Tube;
29use base::VmEventType;
30use base::WaitContext;
31use base::WorkerThread;
32use serde::Deserialize;
33use serde::Serialize;
34use snapshot::AnySnapshot;
35use sync::Mutex;
36use vm_control::DeviceId;
37use vm_control::PlatformDeviceId;
38use vm_control::VmResponse;
39
40use crate::BusAccessInfo;
41use crate::BusDevice;
42use crate::IrqEdgeEvent;
43use crate::Suspendable;
44
45// Registers offsets
46const VMWDT_REG_STATUS: u32 = 0x00;
47const VMWDT_REG_LOAD_CNT: u32 = 0x04;
48const VMWDT_REG_CURRENT_CNT: u32 = 0x08;
49const VMWDT_REG_CLOCK_FREQ_HZ: u32 = 0x0C;
50
51// Length of the registers
52const VMWDT_REG_LEN: u64 = 0x10;
53
54pub const VMWDT_DEFAULT_TIMEOUT_SEC: u32 = 10;
55pub const VMWDT_DEFAULT_CLOCK_HZ: u32 = 2;
56
57// Proc stat indexes
58const PROCSTAT_GUEST_TIME_INDX: usize = 42;
59
60#[derive(Serialize)]
61pub struct VmwdtPerCpu {
62    // Flag which indicated if the watchdog is started
63    is_enabled: bool,
64    // Timer used to generate periodic events at `timer_freq_hz` frequency
65    #[serde(skip_serializing)]
66    timer: Timer,
67    // The frequency of the `timer`
68    timer_freq_hz: u64,
69    // Timestamp measured in miliseconds of the last guest activity
70    last_guest_time_ms: i64,
71    // The thread_id of the thread this vcpu belongs to
72    thread_id: u32,
73    // The process id of the task this vcpu belongs to
74    process_id: u32,
75    // The pre-programmed one-shot expiration interval. If the guest runs in this
76    // interval but we don't receive a periodic event, the guest is stalled.
77    next_expiration_interval_ms: i64,
78    // Keep track if the watchdog PPI raised.
79    stall_evt_ppi_triggered: bool,
80    // Keep track if the time was armed with oneshot mode or with repeating interval
81    repeating_interval: Option<Duration>,
82}
83
84#[derive(Deserialize)]
85struct VmwdtPerCpuRestore {
86    is_enabled: bool,
87    timer_freq_hz: u64,
88    last_guest_time_ms: i64,
89    next_expiration_interval_ms: i64,
90    repeating_interval: Option<Duration>,
91}
92
93pub struct Vmwdt {
94    vm_wdts: Arc<Mutex<Vec<VmwdtPerCpu>>>,
95    // The worker thread that waits on the timer fd
96    worker_thread: Option<WorkerThread<Tube>>,
97    // TODO: @sebastianene add separate reset event for the watchdog
98    // Reset source if the device is not responding
99    reset_evt_wrtube: SendTube,
100    activated: bool,
101    // Event to be used to interrupt the guest on detected stalls
102    stall_evt: IrqEdgeEvent,
103    vm_ctrl_tube: Option<Tube>,
104}
105
106#[derive(Serialize)]
107struct VmwdtSnapshot {
108    #[serde(serialize_with = "serialize_arc_mutex")]
109    vm_wdts: Arc<Mutex<Vec<VmwdtPerCpu>>>,
110    activated: bool,
111}
112
113#[derive(Deserialize)]
114struct VmwdtRestore {
115    vm_wdts: Vec<VmwdtPerCpuRestore>,
116    activated: bool,
117}
118
119impl Vmwdt {
120    pub fn new(
121        cpu_count: usize,
122        reset_evt_wrtube: SendTube,
123        evt: IrqEdgeEvent,
124        vm_ctrl_tube: Tube,
125    ) -> anyhow::Result<Vmwdt> {
126        let mut vec = Vec::new();
127        for _ in 0..cpu_count {
128            vec.push(VmwdtPerCpu {
129                last_guest_time_ms: 0,
130                thread_id: 0,
131                process_id: 0,
132                is_enabled: false,
133                stall_evt_ppi_triggered: false,
134                timer: Timer::new().context("failed to create Timer")?,
135                timer_freq_hz: 0,
136                next_expiration_interval_ms: 0,
137                repeating_interval: None,
138            });
139        }
140        let vm_wdts = Arc::new(Mutex::new(vec));
141
142        Ok(Vmwdt {
143            vm_wdts,
144            worker_thread: None,
145            reset_evt_wrtube,
146            activated: false,
147            stall_evt: evt,
148            vm_ctrl_tube: Some(vm_ctrl_tube),
149        })
150    }
151
152    pub fn vmwdt_worker_thread(
153        vm_wdts: Arc<Mutex<Vec<VmwdtPerCpu>>>,
154        kill_evt: Event,
155        reset_evt_wrtube: SendTube,
156        stall_evt: IrqEdgeEvent,
157        vm_ctrl_tube: Tube,
158        worker_started_send: Option<SendTube>,
159    ) -> anyhow::Result<Tube> {
160        let msg = vm_control::VmRequest::VcpuPidTid;
161        vm_ctrl_tube
162            .send(&msg)
163            .context("failed to send request to fetch Vcpus PID and TID")?;
164        let vcpus_pid_tid: BTreeMap<usize, (u32, u32)> = match vm_ctrl_tube
165            .recv()
166            .context("failed to receive vmwdt pids and tids")?
167        {
168            VmResponse::VcpuPidTidResponse { pid_tid_map } => pid_tid_map,
169            _ => {
170                return Err(anyhow::anyhow!(
171                    "Receive incorrect message type when trying to get vcpu pid tid map"
172                ));
173            }
174        };
175        {
176            let mut vm_wdts = vm_wdts.lock();
177            for (i, vmwdt) in (*vm_wdts).iter_mut().enumerate() {
178                let pid_tid = vcpus_pid_tid
179                    .get(&i)
180                    .context("vmwdts empty, which could indicate no vcpus are initialized")?;
181                vmwdt.process_id = pid_tid.0;
182                vmwdt.thread_id = pid_tid.1;
183            }
184        }
185        if let Some(worker_started_send) = worker_started_send {
186            worker_started_send
187                .send(&())
188                .context("failed to send vmwdt worker started")?;
189        }
190        #[derive(EventToken)]
191        enum Token {
192            Kill,
193            Timer(usize),
194        }
195
196        let wait_ctx: WaitContext<Token> =
197            WaitContext::new().context("Failed to create wait_ctx")?;
198        wait_ctx
199            .add(&kill_evt, Token::Kill)
200            .context("Failed to add Tokens to wait_ctx")?;
201
202        let len = vm_wdts.lock().len();
203        for clock_id in 0..len {
204            let timer_fd = vm_wdts.lock()[clock_id].timer.as_raw_descriptor();
205            wait_ctx
206                .add(&Descriptor(timer_fd), Token::Timer(clock_id))
207                .context("Failed to link FDs to Tokens")?;
208        }
209
210        loop {
211            let events = wait_ctx.wait().context("Failed to wait for events")?;
212            for event in events.iter().filter(|e| e.is_readable) {
213                match event.token {
214                    Token::Kill => {
215                        return Ok(vm_ctrl_tube);
216                    }
217                    Token::Timer(cpu_id) => {
218                        let mut wdts_locked = vm_wdts.lock();
219                        let watchdog = &mut wdts_locked[cpu_id];
220                        match watchdog.timer.mark_waited() {
221                            Ok(true) => continue, // timer not actually ready
222                            Ok(false) => {}
223                            Err(e) => {
224                                error!("error waiting for timer event on vcpu {cpu_id}: {e:#}");
225                                continue;
226                            }
227                        }
228
229                        let current_guest_time_ms =
230                            match Vmwdt::get_guest_time_ms(watchdog.process_id, watchdog.thread_id)
231                            {
232                                Ok(value) => value,
233                                Err(e) => {
234                                    error!("get_guest_time_ms returned error: {}", e);
235                                    // Return VM control tube on ENOENT. ENOENT signals the FD does
236                                    // not exist, which means the Vcpu has shut down or crashed.
237                                    // Return the control tube to gracefully shut down
238                                    if e.errno() == libc::ENOENT {
239                                        return Ok(vm_ctrl_tube);
240                                    } else {
241                                        watchdog.last_guest_time_ms
242                                    }
243                                }
244                            };
245                        let remaining_time_ms = watchdog.next_expiration_interval_ms
246                            - (current_guest_time_ms - watchdog.last_guest_time_ms);
247
248                        if remaining_time_ms > 0 {
249                            watchdog.next_expiration_interval_ms = remaining_time_ms;
250                            if let Err(e) = watchdog
251                                .timer
252                                .reset_oneshot(Duration::from_millis(remaining_time_ms as u64))
253                            {
254                                error!(
255                                    "failed to reset internal timer on vcpu {}: {:#}",
256                                    cpu_id, e
257                                );
258                            }
259                            watchdog.repeating_interval = None;
260                        } else {
261                            if watchdog.stall_evt_ppi_triggered {
262                                if let Err(e) = reset_evt_wrtube
263                                    .send::<VmEventType>(&VmEventType::WatchdogReset)
264                                {
265                                    error!("{} failed to send reset event from vcpu {}", e, cpu_id)
266                                }
267                            }
268
269                            stall_evt
270                                .trigger()
271                                .context("Failed to trigger stall event")?;
272                            watchdog.stall_evt_ppi_triggered = true;
273                            watchdog.last_guest_time_ms = current_guest_time_ms;
274                        }
275                    }
276                }
277            }
278        }
279    }
280
281    fn start(&mut self, worker_started_send: Option<SendTube>) -> anyhow::Result<()> {
282        let vm_wdts = self.vm_wdts.clone();
283        let reset_evt_wrtube = self.reset_evt_wrtube.try_clone().unwrap();
284        let stall_event = self.stall_evt.try_clone().unwrap();
285        let vm_ctrl_tube = self
286            .vm_ctrl_tube
287            .take()
288            .context("missing vm control tube")?;
289
290        self.activated = true;
291        self.worker_thread = Some(WorkerThread::start("vmwdt worker", |kill_evt| {
292            Vmwdt::vmwdt_worker_thread(
293                vm_wdts,
294                kill_evt,
295                reset_evt_wrtube,
296                stall_event,
297                vm_ctrl_tube,
298                worker_started_send,
299            )
300            .expect("failed to start vmwdt worker thread")
301        }));
302        Ok(())
303    }
304
305    fn ensure_started(&mut self) {
306        if self.worker_thread.is_some() {
307            return;
308        }
309
310        let (worker_started_send, worker_started_recv) =
311            Tube::directional_pair().expect("failed to create vmwdt worker started tubes");
312        self.start(Some(worker_started_send))
313            .expect("failed to start Vmwdt");
314        worker_started_recv
315            .recv::<()>()
316            .expect("failed to receive vmwdt worker started");
317    }
318
319    #[cfg(any(target_os = "linux", target_os = "android"))]
320    pub fn get_guest_time_ms(process_id: u32, thread_id: u32) -> Result<i64, SysError> {
321        // TODO: @sebastianene check if we can avoid open-read-close on each call
322        let stat_path = format!("/proc/{process_id}/task/{thread_id}/stat");
323        let contents = fs::read_to_string(stat_path)?;
324
325        let gtime_ticks = contents
326            .split_whitespace()
327            .nth(PROCSTAT_GUEST_TIME_INDX)
328            .and_then(|guest_time| guest_time.parse::<u64>().ok())
329            .unwrap_or(0);
330
331        // SAFETY:
332        // Safe because this just returns an integer
333        let ticks_per_sec = unsafe { libc::sysconf(libc::_SC_CLK_TCK) } as u64;
334        Ok((gtime_ticks * 1000 / ticks_per_sec) as i64)
335    }
336
337    #[cfg(not(any(target_os = "linux", target_os = "android")))]
338    pub fn get_guest_time_ms(process_id: u32, thread_id: u32) -> Result<i64, SysError> {
339        Ok(0)
340    }
341}
342
343impl BusDevice for Vmwdt {
344    fn debug_label(&self) -> String {
345        "Vmwdt".to_owned()
346    }
347
348    fn device_id(&self) -> DeviceId {
349        PlatformDeviceId::VmWatchdog.into()
350    }
351
352    fn read(&mut self, _offset: BusAccessInfo, _data: &mut [u8]) {}
353
354    fn write(&mut self, info: BusAccessInfo, data: &[u8]) {
355        let data_array = match <&[u8; 4]>::try_from(data) {
356            Ok(array) => array,
357            _ => {
358                error!("Bad write size: {} for vmwdt", data.len());
359                return;
360            }
361        };
362
363        let reg_val = u32::from_ne_bytes(*data_array);
364        let cpu_index: usize = (info.offset / VMWDT_REG_LEN) as usize;
365        let reg_offset = (info.offset % VMWDT_REG_LEN) as u32;
366
367        if cpu_index > self.vm_wdts.lock().len() {
368            error!("Bad write cpu_index {}", cpu_index);
369            return;
370        }
371
372        match reg_offset {
373            VMWDT_REG_STATUS => {
374                self.ensure_started();
375                let mut wdts_locked = self.vm_wdts.lock();
376                let cpu_watchdog = &mut wdts_locked[cpu_index];
377
378                cpu_watchdog.is_enabled = reg_val != 0;
379
380                if reg_val != 0 {
381                    let interval = Duration::from_millis(1000 / cpu_watchdog.timer_freq_hz);
382                    cpu_watchdog.repeating_interval = Some(interval);
383                    cpu_watchdog
384                        .timer
385                        .reset_repeating(interval)
386                        .expect("Failed to reset timer repeating interval");
387                } else {
388                    cpu_watchdog.repeating_interval = None;
389                    cpu_watchdog
390                        .timer
391                        .clear()
392                        .expect("Failed to clear cpu watchdog timer");
393                }
394            }
395            VMWDT_REG_LOAD_CNT => {
396                self.ensure_started();
397                let mut wdts_locked = self.vm_wdts.lock();
398                let cpu_watchdog = &mut wdts_locked[cpu_index];
399                let process_id = cpu_watchdog.process_id;
400                let thread_id = cpu_watchdog.thread_id;
401                let guest_time_ms = Vmwdt::get_guest_time_ms(process_id, thread_id)
402                    .expect("get_guest_time_ms failed");
403                let next_expiration_interval_ms =
404                    reg_val as u64 * 1000 / cpu_watchdog.timer_freq_hz;
405
406                cpu_watchdog.last_guest_time_ms = guest_time_ms;
407                cpu_watchdog.stall_evt_ppi_triggered = false;
408                cpu_watchdog.next_expiration_interval_ms = next_expiration_interval_ms as i64;
409
410                if cpu_watchdog.is_enabled {
411                    if let Err(_e) = cpu_watchdog
412                        .timer
413                        .reset_oneshot(Duration::from_millis(next_expiration_interval_ms))
414                    {
415                        error!("failed to reset one-shot vcpu time {}", cpu_index);
416                    }
417                    cpu_watchdog.repeating_interval = None;
418                }
419            }
420            VMWDT_REG_CURRENT_CNT => {
421                warn!("invalid write to read-only VMWDT_REG_CURRENT_CNT register");
422            }
423            VMWDT_REG_CLOCK_FREQ_HZ => {
424                let mut wdts_locked = self.vm_wdts.lock();
425                let cpu_watchdog = &mut wdts_locked[cpu_index];
426
427                debug!(
428                    "CPU:{:x} wrote VMWDT_REG_CLOCK_FREQ_HZ {:x}",
429                    cpu_index, reg_val
430                );
431                cpu_watchdog.timer_freq_hz = reg_val as u64;
432            }
433            _ => unreachable!(),
434        }
435    }
436}
437
438impl Suspendable for Vmwdt {
439    fn sleep(&mut self) -> anyhow::Result<()> {
440        if let Some(worker) = self.worker_thread.take() {
441            self.vm_ctrl_tube = Some(worker.stop());
442        }
443        Ok(())
444    }
445
446    fn wake(&mut self) -> anyhow::Result<()> {
447        if self.activated {
448            // We do not pass a tube to notify that the worker thread has started on wake.
449            // At this stage, vm_control is blocked on resuming devices and cannot provide the vcpu
450            // PIDs/TIDs yet.
451            // At the same time, the Vcpus are still frozen, which means no MMIO will get
452            // processed, and write will not get triggered.
453            // The request to get PIDs/TIDs should get processed before any MMIO request occurs.
454            self.start(None)?;
455            let mut vm_wdts = self.vm_wdts.lock();
456            for vmwdt in vm_wdts.iter_mut() {
457                if let Some(interval) = &vmwdt.repeating_interval {
458                    vmwdt
459                        .timer
460                        .reset_repeating(*interval)
461                        .context("failed to write repeating interval")?;
462                } else if vmwdt.is_enabled {
463                    vmwdt
464                        .timer
465                        .reset_oneshot(Duration::from_millis(
466                            vmwdt.next_expiration_interval_ms as u64,
467                        ))
468                        .context("failed to write oneshot interval")?;
469                }
470            }
471        }
472        Ok(())
473    }
474
475    fn snapshot(&mut self) -> anyhow::Result<AnySnapshot> {
476        AnySnapshot::to_any(&VmwdtSnapshot {
477            vm_wdts: self.vm_wdts.clone(),
478            activated: self.activated,
479        })
480        .context("failed to snapshot Vmwdt")
481    }
482
483    fn restore(&mut self, data: AnySnapshot) -> anyhow::Result<()> {
484        let deser: VmwdtRestore =
485            AnySnapshot::from_any(data).context("failed to deserialize Vmwdt")?;
486        let mut vm_wdts = self.vm_wdts.lock();
487        for (vmwdt_restore, vmwdt) in deser.vm_wdts.iter().zip(vm_wdts.iter_mut()) {
488            vmwdt.is_enabled = vmwdt_restore.is_enabled;
489            vmwdt.timer_freq_hz = vmwdt_restore.timer_freq_hz;
490            vmwdt.last_guest_time_ms = vmwdt_restore.last_guest_time_ms;
491            vmwdt.next_expiration_interval_ms = vmwdt_restore.next_expiration_interval_ms;
492            vmwdt.repeating_interval = vmwdt_restore.repeating_interval;
493        }
494        self.activated = deser.activated;
495        Ok(())
496    }
497}
498
499#[cfg(test)]
500mod tests {
501    use std::process;
502    use std::thread::sleep;
503
504    #[cfg(any(target_os = "linux", target_os = "android"))]
505    use base::gettid;
506    use base::poll_assert;
507    use base::Tube;
508
509    use super::*;
510
511    const AARCH64_VMWDT_ADDR: u64 = 0x3000;
512    const TEST_VMWDT_CPU_NO: usize = 0x1;
513
514    fn vmwdt_bus_address(offset: u64) -> BusAccessInfo {
515        BusAccessInfo {
516            offset,
517            address: AARCH64_VMWDT_ADDR,
518            id: 0,
519        }
520    }
521
522    #[test]
523    fn test_watchdog_internal_timer() {
524        let (vm_evt_wrtube, _vm_evt_rdtube) = Tube::directional_pair().unwrap();
525        let (vm_ctrl_wrtube, vm_ctrl_rdtube) = Tube::pair().unwrap();
526        let irq = IrqEdgeEvent::new().unwrap();
527        #[cfg(any(target_os = "linux", target_os = "android"))]
528        {
529            vm_ctrl_wrtube
530                .send(&VmResponse::VcpuPidTidResponse {
531                    pid_tid_map: BTreeMap::from([(0, (process::id(), gettid() as u32))]),
532                })
533                .unwrap();
534        }
535        let mut device = Vmwdt::new(TEST_VMWDT_CPU_NO, vm_evt_wrtube, irq, vm_ctrl_rdtube).unwrap();
536
537        // Configure the watchdog device, 2Hz internal clock
538        device.write(
539            vmwdt_bus_address(VMWDT_REG_CLOCK_FREQ_HZ as u64),
540            &[10, 0, 0, 0],
541        );
542        device.write(vmwdt_bus_address(VMWDT_REG_LOAD_CNT as u64), &[1, 0, 0, 0]);
543        device.write(vmwdt_bus_address(VMWDT_REG_STATUS as u64), &[1, 0, 0, 0]);
544        let next_expiration_ms = {
545            let mut vmwdt_locked = device.vm_wdts.lock();
546            // In the test scenario the guest does not interpret the /proc/stat::guest_time, thus
547            // the function get_guest_time() returns 0
548            vmwdt_locked[0].last_guest_time_ms = 10;
549            vmwdt_locked[0].next_expiration_interval_ms
550        };
551
552        // Poll multiple times as we don't get a signal when the watchdog thread has run.
553        poll_assert!(10, || {
554            sleep(Duration::from_millis(50));
555            let vmwdt_locked = device.vm_wdts.lock();
556            // Verify that our timer expired and the next_expiration_interval_ms changed
557            vmwdt_locked[0].next_expiration_interval_ms != next_expiration_ms
558        });
559    }
560
561    #[test]
562    fn test_watchdog_expiration() {
563        let (vm_evt_wrtube, vm_evt_rdtube) = Tube::directional_pair().unwrap();
564        let (vm_ctrl_wrtube, vm_ctrl_rdtube) = Tube::pair().unwrap();
565        let irq = IrqEdgeEvent::new().unwrap();
566        #[cfg(any(target_os = "linux", target_os = "android"))]
567        {
568            vm_ctrl_wrtube
569                .send(&VmResponse::VcpuPidTidResponse {
570                    pid_tid_map: BTreeMap::from([(0, (process::id(), gettid() as u32))]),
571                })
572                .unwrap();
573        }
574        let mut device = Vmwdt::new(TEST_VMWDT_CPU_NO, vm_evt_wrtube, irq, vm_ctrl_rdtube).unwrap();
575
576        // Configure the watchdog device, 2Hz internal clock
577        device.write(
578            vmwdt_bus_address(VMWDT_REG_CLOCK_FREQ_HZ as u64),
579            &[10, 0, 0, 0],
580        );
581        device.write(vmwdt_bus_address(VMWDT_REG_LOAD_CNT as u64), &[1, 0, 0, 0]);
582        device.write(vmwdt_bus_address(VMWDT_REG_STATUS as u64), &[1, 0, 0, 0]);
583        // In the test scenario the guest does not interpret the /proc/stat::guest_time, thus
584        // the function get_guest_time() returns 0
585        device.vm_wdts.lock()[0].last_guest_time_ms = -100;
586
587        // Check that the interrupt has raised
588        poll_assert!(10, || {
589            sleep(Duration::from_millis(50));
590            let vmwdt_locked = device.vm_wdts.lock();
591            vmwdt_locked[0].stall_evt_ppi_triggered
592        });
593
594        // Simulate that the time has passed since the last expiration
595        device.vm_wdts.lock()[0].last_guest_time_ms = -100;
596
597        // Poll multiple times as we don't get a signal when the watchdog thread has run.
598        poll_assert!(10, || {
599            sleep(Duration::from_millis(50));
600            match vm_evt_rdtube.recv::<VmEventType>() {
601                Ok(vm_event) => vm_event == VmEventType::WatchdogReset,
602                Err(_e) => false,
603            }
604        });
605    }
606
607    #[test]
608    // Testing with invalid vcpu tid which would simulate the same behavior as the Vcpu dying and
609    // the watchdog trying to read from the FD that no longer exists.
610    fn test_watchdog_vcpu_death() {
611        let (vm_evt_wrtube, _vm_evt_rdtube) = Tube::directional_pair().unwrap();
612        let (vm_ctrl_wrtube, vm_ctrl_rdtube) = Tube::pair().unwrap();
613        let irq = IrqEdgeEvent::new().unwrap();
614
615        // Spawn a helper thread that we can kill to simulate vCPU death.
616        let (tid_tx, tid_rx) = std::sync::mpsc::channel();
617        let (exit_tx, exit_rx) = std::sync::mpsc::channel();
618        let helper_thread = std::thread::spawn(move || {
619            let tid = base::gettid() as u32;
620            tid_tx.send(tid).unwrap();
621            exit_rx.recv().unwrap(); // Block until told to exit
622        });
623        let helper_tid = tid_rx.recv().unwrap();
624
625        #[cfg(any(target_os = "linux", target_os = "android"))]
626        {
627            vm_ctrl_wrtube
628                .send(&VmResponse::VcpuPidTidResponse {
629                    pid_tid_map: BTreeMap::from([(0, (process::id(), helper_tid))]),
630                })
631                .unwrap();
632        }
633        let mut device = Vmwdt::new(TEST_VMWDT_CPU_NO, vm_evt_wrtube, irq, vm_ctrl_rdtube).unwrap();
634
635        // Configure the watchdog device, 10Hz clock
636        device.write(
637            vmwdt_bus_address(VMWDT_REG_CLOCK_FREQ_HZ as u64),
638            &[10, 0, 0, 0],
639        );
640
641        // Write to LOAD_CNT.
642        // This should succeed because the helper thread is still alive.
643        device.write(vmwdt_bus_address(VMWDT_REG_LOAD_CNT as u64), &[1, 0, 0, 0]);
644
645        // Now tell the helper thread to exit and join it to ensure it is dead and reaped.
646        exit_tx.send(()).unwrap();
647        helper_thread.join().unwrap();
648
649        // Enable the watchdog, which starts the worker thread.
650        device.write(vmwdt_bus_address(VMWDT_REG_STATUS as u64), &[1, 0, 0, 0]);
651
652        // Wait for the timer to expire (load count is 1, freq is 10Hz -> 100ms).
653        // The worker thread will handle the timer, call get_guest_time_ms which fails (since the
654        // thread is dead), log the error, and exit gracefully (returns Ok(vm_ctrl_tube)).
655        sleep(Duration::from_millis(200));
656
657        // Stop the device.
658        std::mem::drop(device);
659    }
660}