devices/usb/xhci/
event_ring.rs

1// Copyright 2018 The ChromiumOS Authors
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use std::mem::size_of;
6use std::sync::atomic::fence;
7use std::sync::atomic::Ordering;
8
9use remain::sorted;
10use thiserror::Error;
11use vm_memory::GuestAddress;
12use vm_memory::GuestMemory;
13use vm_memory::GuestMemoryError;
14use zerocopy::IntoBytes;
15
16use super::xhci_abi::EventRingSegmentTableEntry;
17use super::xhci_abi::Trb;
18
19#[sorted]
20#[derive(Error, Debug)]
21pub enum Error {
22    #[error("event ring has a bad enqueue pointer: {0}")]
23    BadEnqueuePointer(GuestAddress),
24    #[error("event ring has a bad seg table addr: {0}")]
25    BadSegTableAddress(GuestAddress),
26    #[error("event ring has a bad seg table index: {0}")]
27    BadSegTableIndex(u16),
28    #[error("event ring is full")]
29    EventRingFull,
30    #[error("event ring cannot read from guest memory: {0}")]
31    MemoryRead(GuestMemoryError),
32    #[error("event ring cannot write to guest memory: {0}")]
33    MemoryWrite(GuestMemoryError),
34    #[error("event ring is uninitialized")]
35    Uninitialized,
36}
37
38type Result<T> = std::result::Result<T, Error>;
39
40/// Event rings are segmented circular buffers used to pass event TRBs from the xHCI device back to
41/// the guest.  Each event ring is associated with a single interrupter.  See section 4.9.4 of the
42/// xHCI specification for more details.
43/// This implementation is only for primary interrupter. Please review xhci spec before using it
44/// for secondary.
45pub struct EventRing {
46    mem: GuestMemory,
47    segment_table_size: u16,
48    segment_table_base_address: GuestAddress,
49    current_segment_index: u16,
50    trb_count: u16,
51    enqueue_pointer: GuestAddress,
52    dequeue_pointer: GuestAddress,
53    producer_cycle_state: bool,
54}
55
56impl EventRing {
57    /// Create an empty, uninitialized event ring.
58    pub fn new(mem: GuestMemory) -> Self {
59        EventRing {
60            mem,
61            segment_table_size: 0,
62            segment_table_base_address: GuestAddress(0),
63            current_segment_index: 0,
64            enqueue_pointer: GuestAddress(0),
65            dequeue_pointer: GuestAddress(0),
66            trb_count: 0,
67            // As specified in xHCI spec 4.9.4, cycle state should be initialized to 1.
68            producer_cycle_state: true,
69        }
70    }
71
72    /// This function implements left side of xHCI spec, Figure 4-12.
73    pub fn add_event(&mut self, mut trb: Trb) -> Result<()> {
74        self.check_inited()?;
75        if self.is_full()? {
76            return Err(Error::EventRingFull);
77        }
78        // Event is write twice to avoid race condition.
79        // Guest kernel use cycle bit to check ownership, thus we should write cycle last.
80        trb.set_cycle(!self.producer_cycle_state);
81        self.mem
82            .write_obj_at_addr(trb, self.enqueue_pointer)
83            .map_err(Error::MemoryWrite)?;
84
85        // Updating the cycle state bit should always happen after updating other parts.
86        fence(Ordering::SeqCst);
87
88        trb.set_cycle(self.producer_cycle_state);
89
90        // Offset of cycle state byte.
91        const CYCLE_STATE_OFFSET: usize = 12usize;
92        let data = trb.as_bytes();
93        // Trb contains 4 dwords, the last one contains cycle bit.
94        let cycle_bit_dword = &data[CYCLE_STATE_OFFSET..];
95        let address = self.enqueue_pointer;
96        let address = address
97            .checked_add(CYCLE_STATE_OFFSET as u64)
98            .ok_or(Error::BadEnqueuePointer(self.enqueue_pointer))?;
99        self.mem
100            .write_all_at_addr(cycle_bit_dword, address)
101            .map_err(Error::MemoryWrite)?;
102
103        xhci_trace!(
104            "event write to pointer {:#x}, trb_count {}, {}",
105            self.enqueue_pointer.0,
106            self.trb_count,
107            trb
108        );
109        self.enqueue_pointer = match self.enqueue_pointer.checked_add(size_of::<Trb>() as u64) {
110            Some(addr) => addr,
111            None => return Err(Error::BadEnqueuePointer(self.enqueue_pointer)),
112        };
113        self.trb_count -= 1;
114        if self.trb_count == 0 {
115            self.current_segment_index += 1;
116            if self.current_segment_index == self.segment_table_size {
117                self.producer_cycle_state ^= true;
118                self.current_segment_index = 0;
119            }
120            self.load_current_seg_table_entry()?;
121        }
122        Ok(())
123    }
124
125    /// Set segment table size.
126    pub fn set_seg_table_size(&mut self, size: u16) -> Result<()> {
127        xhci_trace!("set_seg_table_size({:#x})", size);
128        self.segment_table_size = size;
129        self.try_reconfigure_event_ring()
130    }
131
132    /// Set segment table base addr.
133    pub fn set_seg_table_base_addr(&mut self, addr: GuestAddress) -> Result<()> {
134        xhci_trace!("set_seg_table_base_addr({:#x})", addr.0);
135        self.segment_table_base_address = addr;
136        self.try_reconfigure_event_ring()
137    }
138
139    /// Set dequeue pointer.
140    pub fn set_dequeue_pointer(&mut self, addr: GuestAddress) {
141        xhci_trace!("set_dequeue_pointer({:#x})", addr.0);
142        self.dequeue_pointer = addr;
143    }
144
145    /// Check if event ring is empty.
146    pub fn is_empty(&self) -> bool {
147        self.enqueue_pointer == self.dequeue_pointer
148    }
149
150    /// Event ring is considered full when there is only space for one last TRB. In this case, xHC
151    /// should write an error Trb and do a bunch of handlings. See spec, figure 4-12 for more
152    /// details.
153    /// For now, we just check event ring full and fail (as it's unlikely to happen).
154    pub fn is_full(&self) -> Result<bool> {
155        if self.trb_count == 1 {
156            // erst == event ring segment table
157            let next_erst_idx = (self.current_segment_index + 1) % self.segment_table_size;
158            let erst_entry = self.read_seg_table_entry(next_erst_idx)?;
159            Ok(self.dequeue_pointer.0 == erst_entry.get_ring_segment_base_address())
160        } else {
161            Ok(self.dequeue_pointer.0 == self.enqueue_pointer.0 + size_of::<Trb>() as u64)
162        }
163    }
164
165    /// Try to init event ring. Will fail if seg table size/address are invalid.
166    fn try_reconfigure_event_ring(&mut self) -> Result<()> {
167        if self.segment_table_size == 0 || self.segment_table_base_address.0 == 0 {
168            return Ok(());
169        }
170        self.load_current_seg_table_entry()
171    }
172
173    // Check if this event ring is inited.
174    fn check_inited(&self) -> Result<()> {
175        if self.segment_table_size == 0
176            || self.segment_table_base_address == GuestAddress(0)
177            || self.enqueue_pointer == GuestAddress(0)
178        {
179            return Err(Error::Uninitialized);
180        }
181        Ok(())
182    }
183
184    // Load entry of current seg table.
185    fn load_current_seg_table_entry(&mut self) -> Result<()> {
186        let entry = self.read_seg_table_entry(self.current_segment_index)?;
187        self.enqueue_pointer = GuestAddress(entry.get_ring_segment_base_address());
188        self.trb_count = entry.get_ring_segment_size();
189        Ok(())
190    }
191
192    // Get seg table entry at index.
193    fn read_seg_table_entry(&self, index: u16) -> Result<EventRingSegmentTableEntry> {
194        let seg_table_addr = self.get_seg_table_addr(index)?;
195        // TODO(jkwang) We can refactor GuestMemory to allow in-place memory operation.
196        self.mem
197            .read_obj_from_addr(seg_table_addr)
198            .map_err(Error::MemoryRead)
199    }
200
201    // Get seg table addr at index.
202    fn get_seg_table_addr(&self, index: u16) -> Result<GuestAddress> {
203        if index > self.segment_table_size {
204            return Err(Error::BadSegTableIndex(index));
205        }
206        self.segment_table_base_address
207            .checked_add(((size_of::<EventRingSegmentTableEntry>() as u16) * index) as u64)
208            .ok_or(Error::BadSegTableAddress(self.segment_table_base_address))
209    }
210}
211
212#[cfg(test)]
213mod test {
214    use std::mem::size_of;
215
216    use base::pagesize;
217
218    use super::*;
219
220    #[test]
221    fn test_uninited() {
222        let gm = GuestMemory::new(&[(GuestAddress(0), pagesize() as u64)]).unwrap();
223        let mut er = EventRing::new(gm);
224        let trb = Trb::new();
225        match er.add_event(trb).err().unwrap() {
226            Error::Uninitialized => {}
227            _ => panic!("unexpected error"),
228        }
229        assert_eq!(er.is_empty(), true);
230        assert_eq!(er.is_full().unwrap(), false);
231    }
232
233    #[test]
234    fn test_event_ring() {
235        let trb_size = size_of::<Trb>() as u64;
236        let gm = GuestMemory::new(&[(GuestAddress(0), pagesize() as u64)]).unwrap();
237        let mut er = EventRing::new(gm.clone());
238        let mut st_entries = [EventRingSegmentTableEntry::new(); 3];
239        st_entries[0].set_ring_segment_base_address(0x100);
240        st_entries[0].set_ring_segment_size(3);
241        st_entries[1].set_ring_segment_base_address(0x200);
242        st_entries[1].set_ring_segment_size(3);
243        st_entries[2].set_ring_segment_base_address(0x300);
244        st_entries[2].set_ring_segment_size(3);
245        gm.write_obj_at_addr(st_entries[0], GuestAddress(0x8))
246            .unwrap();
247        gm.write_obj_at_addr(
248            st_entries[1],
249            GuestAddress(0x8 + size_of::<EventRingSegmentTableEntry>() as u64),
250        )
251        .unwrap();
252        gm.write_obj_at_addr(
253            st_entries[2],
254            GuestAddress(0x8 + 2 * size_of::<EventRingSegmentTableEntry>() as u64),
255        )
256        .unwrap();
257        // Init event ring. Must init after segment tables writting.
258        er.set_seg_table_size(3).unwrap();
259        er.set_seg_table_base_addr(GuestAddress(0x8)).unwrap();
260        er.set_dequeue_pointer(GuestAddress(0x100));
261
262        let mut trb = Trb::new();
263
264        // Fill first table.
265        trb.set_control(1);
266        assert_eq!(er.is_empty(), true);
267        assert_eq!(er.is_full().unwrap(), false);
268        assert!(er.add_event(trb).is_ok());
269        assert_eq!(er.is_full().unwrap(), false);
270        assert_eq!(er.is_empty(), false);
271        let t: Trb = gm.read_obj_from_addr(GuestAddress(0x100)).unwrap();
272        assert_eq!(t.get_control(), 1);
273        assert_eq!(t.get_cycle(), true);
274
275        trb.set_control(2);
276        assert!(er.add_event(trb).is_ok());
277        assert_eq!(er.is_full().unwrap(), false);
278        assert_eq!(er.is_empty(), false);
279        let t: Trb = gm
280            .read_obj_from_addr(GuestAddress(0x100 + trb_size))
281            .unwrap();
282        assert_eq!(t.get_control(), 2);
283        assert_eq!(t.get_cycle(), true);
284
285        trb.set_control(3);
286        assert!(er.add_event(trb).is_ok());
287        assert_eq!(er.is_full().unwrap(), false);
288        assert_eq!(er.is_empty(), false);
289        let t: Trb = gm
290            .read_obj_from_addr(GuestAddress(0x100 + 2 * trb_size))
291            .unwrap();
292        assert_eq!(t.get_control(), 3);
293        assert_eq!(t.get_cycle(), true);
294
295        // Fill second table.
296        trb.set_control(4);
297        assert!(er.add_event(trb).is_ok());
298        assert_eq!(er.is_full().unwrap(), false);
299        assert_eq!(er.is_empty(), false);
300        let t: Trb = gm.read_obj_from_addr(GuestAddress(0x200)).unwrap();
301        assert_eq!(t.get_control(), 4);
302        assert_eq!(t.get_cycle(), true);
303
304        trb.set_control(5);
305        assert!(er.add_event(trb).is_ok());
306        assert_eq!(er.is_full().unwrap(), false);
307        assert_eq!(er.is_empty(), false);
308        let t: Trb = gm
309            .read_obj_from_addr(GuestAddress(0x200 + trb_size))
310            .unwrap();
311        assert_eq!(t.get_control(), 5);
312        assert_eq!(t.get_cycle(), true);
313
314        trb.set_control(6);
315        assert!(er.add_event(trb).is_ok());
316        assert_eq!(er.is_full().unwrap(), false);
317        assert_eq!(er.is_empty(), false);
318        let t: Trb = gm
319            .read_obj_from_addr(GuestAddress(0x200 + 2 * trb_size))
320            .unwrap();
321        assert_eq!(t.get_control(), 6);
322        assert_eq!(t.get_cycle(), true);
323
324        // Fill third table.
325        trb.set_control(7);
326        assert!(er.add_event(trb).is_ok());
327        assert_eq!(er.is_full().unwrap(), false);
328        assert_eq!(er.is_empty(), false);
329        let t: Trb = gm.read_obj_from_addr(GuestAddress(0x300)).unwrap();
330        assert_eq!(t.get_control(), 7);
331        assert_eq!(t.get_cycle(), true);
332
333        trb.set_control(8);
334        assert!(er.add_event(trb).is_ok());
335        // There is only one last trb. Considered full.
336        assert_eq!(er.is_full().unwrap(), true);
337        assert_eq!(er.is_empty(), false);
338        let t: Trb = gm
339            .read_obj_from_addr(GuestAddress(0x300 + trb_size))
340            .unwrap();
341        assert_eq!(t.get_control(), 8);
342        assert_eq!(t.get_cycle(), true);
343
344        // Add the last trb will result in error.
345        match er.add_event(trb) {
346            Err(Error::EventRingFull) => {}
347            _ => panic!("er should be full"),
348        };
349
350        // Dequeue one trb.
351        er.set_dequeue_pointer(GuestAddress(0x100 + trb_size));
352        assert_eq!(er.is_full().unwrap(), false);
353        assert_eq!(er.is_empty(), false);
354
355        // Fill the last trb of the third table.
356        trb.set_control(9);
357        assert!(er.add_event(trb).is_ok());
358        // There is only one last trb. Considered full.
359        assert_eq!(er.is_full().unwrap(), true);
360        assert_eq!(er.is_empty(), false);
361        let t: Trb = gm
362            .read_obj_from_addr(GuestAddress(0x300 + trb_size))
363            .unwrap();
364        assert_eq!(t.get_control(), 8);
365        assert_eq!(t.get_cycle(), true);
366
367        // Add the last trb will result in error.
368        match er.add_event(trb) {
369            Err(Error::EventRingFull) => {}
370            _ => panic!("er should be full"),
371        };
372
373        // Dequeue until empty.
374        er.set_dequeue_pointer(GuestAddress(0x100));
375        assert_eq!(er.is_full().unwrap(), false);
376        assert_eq!(er.is_empty(), true);
377
378        // Fill first table again.
379        trb.set_control(10);
380        assert!(er.add_event(trb).is_ok());
381        assert_eq!(er.is_full().unwrap(), false);
382        assert_eq!(er.is_empty(), false);
383        let t: Trb = gm.read_obj_from_addr(GuestAddress(0x100)).unwrap();
384        assert_eq!(t.get_control(), 10);
385        // cycle bit should be reversed.
386        assert_eq!(t.get_cycle(), false);
387
388        trb.set_control(11);
389        assert!(er.add_event(trb).is_ok());
390        assert_eq!(er.is_full().unwrap(), false);
391        assert_eq!(er.is_empty(), false);
392        let t: Trb = gm
393            .read_obj_from_addr(GuestAddress(0x100 + trb_size))
394            .unwrap();
395        assert_eq!(t.get_control(), 11);
396        assert_eq!(t.get_cycle(), false);
397
398        trb.set_control(12);
399        assert!(er.add_event(trb).is_ok());
400        assert_eq!(er.is_full().unwrap(), false);
401        assert_eq!(er.is_empty(), false);
402        let t: Trb = gm
403            .read_obj_from_addr(GuestAddress(0x100 + 2 * trb_size))
404            .unwrap();
405        assert_eq!(t.get_control(), 12);
406        assert_eq!(t.get_cycle(), false);
407    }
408}
devices/usb/xhci/event_ring.rs

devices/usb/xhci/
event_ring.rs
