devices/usb/xhci/
xhci_transfer.rs

1// Copyright 2019 The ChromiumOS Authors
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use std::cmp::min;
6use std::collections::VecDeque;
7use std::fmt;
8use std::fmt::Display;
9use std::mem;
10use std::sync::Arc;
11use std::sync::Weak;
12
13use base::debug;
14use base::error;
15use base::info;
16use base::warn;
17use base::Error as SysError;
18use base::Event;
19use bit_field::Error as BitFieldError;
20use remain::sorted;
21use sync::Mutex;
22use thiserror::Error;
23use usb_util::TransferStatus;
24use usb_util::UsbRequestSetup;
25use vm_memory::GuestMemory;
26use vm_memory::GuestMemoryError;
27
28use super::device_slot::DeviceSlot;
29use super::interrupter::Error as InterrupterError;
30use super::interrupter::Interrupter;
31use super::ring_buffer_stop_cb::RingBufferStopCallback;
32use super::scatter_gather_buffer::Error as BufferError;
33use super::scatter_gather_buffer::ScatterGatherBuffer;
34use super::usb_hub::Error as HubError;
35use super::usb_hub::UsbPort;
36use super::xhci_abi::AddressedTrb;
37use super::xhci_abi::Error as TrbError;
38use super::xhci_abi::EventDataTrb;
39use super::xhci_abi::SetupStageTrb;
40use super::xhci_abi::TransferDescriptor;
41use super::xhci_abi::TrbCast;
42use super::xhci_abi::TrbCompletionCode;
43use super::xhci_abi::TrbType;
44use super::xhci_regs::MAX_INTERRUPTER;
45
46#[sorted]
47#[derive(Error, Debug)]
48pub enum Error {
49    #[error("unexpected trb type: {0:?}")]
50    BadTrbType(TrbType),
51    #[error("failed to cancel transfer")]
52    CancelTransfer,
53    #[error("cannot cast trb: {0}")]
54    CastTrb(TrbError),
55    #[error("cannot create transfer buffer: {0}")]
56    CreateBuffer(BufferError),
57    #[error("cannot detach from port: {0}")]
58    DetachPort(HubError),
59    #[error("failed to get max payload length for ep: {0}")]
60    GetMaxPayload(u8),
61    #[error("failed to halt the endpoint: {0}")]
62    HaltEndpoint(u8),
63    #[error("failed to read guest memory: {0}")]
64    ReadGuestMemory(GuestMemoryError),
65    #[error("cannot send interrupt: {0}")]
66    SendInterrupt(InterrupterError),
67    #[error("failed to submit transfer to backend")]
68    SubmitTransfer,
69    #[error("cannot get transfer length: {0}")]
70    TransferLength(TrbError),
71    #[error("cannot get trb type: {0}")]
72    TrbType(BitFieldError),
73    #[error("cannot write completion event: {0}")]
74    WriteCompletionEvent(SysError),
75    #[error("failed to write guest memory: {0}")]
76    WriteGuestMemory(GuestMemoryError),
77}
78
79type Result<T> = std::result::Result<T, Error>;
80
81/// Type of usb endpoints.
82#[derive(PartialEq, Eq, Clone, Copy, Debug)]
83pub enum TransferDirection {
84    In,
85    Out,
86    Control,
87}
88
89type CancelCallback = Box<dyn FnOnce() -> Result<()> + Send>;
90
91/// Current state of xhci transfer. The transfer in a Submitted or Cancelling state is owned by the
92/// host and should always be reaped to prevent memory leak.
93pub enum XhciTransferState {
94    Created,
95    /// When transfer is submitted, it will contain a transfer callback, which should be invoked
96    /// when the transfer is cancelled.
97    Submitted {
98        cancel_callback: CancelCallback,
99    },
100    Cancelling,
101    Cancelled,
102    Completed,
103}
104
105impl XhciTransferState {
106    /// Try to cancel this transfer, if it's possible.
107    pub fn try_cancel(&mut self, force: bool) -> bool {
108        let mut cancelled = true;
109        match mem::replace(self, XhciTransferState::Created) {
110            XhciTransferState::Submitted { cancel_callback } => {
111                // If we fail to cancel, there are two cases: the URB has already completed
112                // (EINVAL) or the device is gone (ENODEV). For both cases, we put back the state
113                // to Submitted and check the URB status in the completion handler, to report the
114                // already completed one properly to the guest. However, we can't do that once we
115                // have cancelled a preceding request, because the request must be processed in the
116                // order of submission.
117                match cancel_callback() {
118                    Ok(()) => {
119                        *self = XhciTransferState::Cancelling;
120                    }
121                    Err(_e) => {
122                        if force {
123                            *self = XhciTransferState::Cancelling;
124                        } else {
125                            let error_callback = Box::new(move || Err(Error::CancelTransfer));
126                            *self = XhciTransferState::Submitted {
127                                cancel_callback: error_callback,
128                            };
129                            cancelled = false;
130                        }
131                    }
132                }
133            }
134            XhciTransferState::Cancelling => {
135                error!("Another cancellation is already issued.");
136                *self = XhciTransferState::Cancelling;
137            }
138            _ => {
139                *self = XhciTransferState::Cancelled;
140            }
141        }
142        cancelled
143    }
144}
145
146/// Type of a transfer received handled by transfer ring.
147pub enum XhciTransferType {
148    // Normal means bulk transfer or interrupt transfer, depending on endpoint type.
149    // See spec 4.11.2.1.
150    Normal,
151    // See usb spec for setup stage, data stage and status stage,
152    // see xHCI spec 4.11.2.2 for corresponding trbs.
153    SetupStage,
154    DataStage,
155    StatusStage,
156    // See xHCI spec 4.11.2.3.
157    Isochronous,
158    // See xHCI spec 6.4.1.4.
159    Noop,
160}
161
162impl Display for XhciTransferType {
163    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
164        use self::XhciTransferType::*;
165
166        match self {
167            Normal => write!(f, "Normal"),
168            SetupStage => write!(f, "SetupStage"),
169            DataStage => write!(f, "DataStage"),
170            StatusStage => write!(f, "StatusStage"),
171            Isochronous => write!(f, "Isochronous"),
172            Noop => write!(f, "Noop"),
173        }
174    }
175}
176
177/// Xhci Transfer manager holds reference to all ongoing transfers. Can cancel them all if
178/// needed.
179#[derive(Clone)]
180pub struct XhciTransferManager {
181    transfers: Arc<Mutex<VecDeque<Weak<Mutex<XhciTransferState>>>>>,
182    device_slot: Weak<DeviceSlot>,
183    stop_callback: Arc<Mutex<Vec<RingBufferStopCallback>>>,
184}
185
186impl XhciTransferManager {
187    /// Create a new manager.
188    pub fn new(device_slot: Weak<DeviceSlot>) -> XhciTransferManager {
189        XhciTransferManager {
190            transfers: Arc::new(Mutex::new(VecDeque::new())),
191            device_slot,
192            stop_callback: Arc::new(Mutex::new(Vec::new())),
193        }
194    }
195
196    /// Build a new XhciTransfer. Endpoint id is the id in xHCI device slot.
197    pub fn create_transfer(
198        &self,
199        mem: GuestMemory,
200        port: Arc<UsbPort>,
201        interrupter: Arc<Mutex<Interrupter>>,
202        slot_id: u8,
203        endpoint_id: u8,
204        transfer_descriptor: TransferDescriptor,
205        trigger_event: Event,
206        stream_id: Option<u16>,
207    ) -> XhciTransfer {
208        let transfer_dir = {
209            if endpoint_id == 1 {
210                TransferDirection::Control
211            } else if (endpoint_id % 2) == 0 {
212                TransferDirection::Out
213            } else {
214                TransferDirection::In
215            }
216        };
217        let t = XhciTransfer {
218            manager: self.clone(),
219            state: Arc::new(Mutex::new(XhciTransferState::Created)),
220            mem,
221            port,
222            interrupter,
223            slot_id,
224            endpoint_id,
225            transfer_dir,
226            transfer_descriptor,
227            trigger_event,
228            device_slot: self.device_slot.clone(),
229            stream_id,
230        };
231        self.transfers.lock().push_back(Arc::downgrade(&t.state));
232        t
233    }
234
235    /// Cancel all current transfers and execute the callback once completed.
236    pub fn cancel_all(&self, callback: RingBufferStopCallback) {
237        let locked_transfers = self.transfers.lock();
238        if !locked_transfers.is_empty() {
239            self.stop_callback.lock().push(callback);
240        }
241
242        let mut force_cancel = false;
243        locked_transfers.iter().for_each(|t| {
244            let state = match t.upgrade() {
245                Some(state) => state,
246                None => {
247                    error!("transfer is already cancelled or finished");
248                    return;
249                }
250            };
251            force_cancel |= state.lock().try_cancel(force_cancel);
252        });
253    }
254
255    fn remove_transfer(&self, t: &Arc<Mutex<XhciTransferState>>) {
256        let mut transfers = self.transfers.lock();
257        match transfers.iter().position(|wt| match wt.upgrade() {
258            Some(wt) => Arc::ptr_eq(&wt, t),
259            None => false,
260        }) {
261            None => error!("attempted to remove unknown transfer"),
262            Some(i) => {
263                transfers.remove(i);
264            }
265        }
266        if transfers.is_empty() {
267            self.stop_callback.lock().clear();
268        }
269    }
270}
271
272impl Default for XhciTransferManager {
273    fn default() -> Self {
274        Self::new(Weak::new())
275    }
276}
277
278/// Xhci transfer denotes a transfer initiated by guest os driver. It will be submitted to a
279/// XhciBackendDevice.
280pub struct XhciTransfer {
281    manager: XhciTransferManager,
282    state: Arc<Mutex<XhciTransferState>>,
283    mem: GuestMemory,
284    port: Arc<UsbPort>,
285    interrupter: Arc<Mutex<Interrupter>>,
286    slot_id: u8,
287    // id of endpoint in device slot.
288    endpoint_id: u8,
289    transfer_dir: TransferDirection,
290    transfer_descriptor: TransferDescriptor,
291    trigger_event: Event,
292    device_slot: Weak<DeviceSlot>,
293    stream_id: Option<u16>,
294}
295
296impl Drop for XhciTransfer {
297    fn drop(&mut self) {
298        self.manager.remove_transfer(&self.state);
299    }
300}
301
302impl fmt::Debug for XhciTransfer {
303    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
304        write!(
305            f,
306            "xhci_transfer slot id: {}, endpoint id {}, transfer_dir {:?}, transfer_descriptor {:?}",
307            self.slot_id, self.endpoint_id, self.transfer_dir, self.transfer_descriptor
308        )
309    }
310}
311
312#[derive(Debug, PartialEq, Clone, Copy)]
313enum TransferAction {
314    HaltEndpoint,
315    SendEvent {
316        code: TrbCompletionCode,
317        gpa: u64,
318        residual_or_edtla: u32,
319        event_data: bool,
320    },
321}
322
323impl XhciTransfer {
324    /// Get state of this transfer.
325    pub fn state(&self) -> &Arc<Mutex<XhciTransferState>> {
326        &self.state
327    }
328
329    /// Get transfer type.
330    pub fn get_transfer_type(&self) -> Result<XhciTransferType> {
331        // We can figure out transfer type from the first trb.
332        // See transfer descriptor description in xhci spec for more details.
333        match self
334            .transfer_descriptor
335            .first_atrb()
336            .trb
337            .get_trb_type()
338            .map_err(Error::TrbType)?
339        {
340            TrbType::Normal => Ok(XhciTransferType::Normal),
341            TrbType::SetupStage => Ok(XhciTransferType::SetupStage),
342            TrbType::DataStage => Ok(XhciTransferType::DataStage),
343            TrbType::StatusStage => Ok(XhciTransferType::StatusStage),
344            TrbType::Isoch => Ok(XhciTransferType::Isochronous),
345            TrbType::Noop => Ok(XhciTransferType::Noop),
346            t => Err(Error::BadTrbType(t)),
347        }
348    }
349
350    /// Create a scatter gather buffer for the given xhci transfer
351    pub fn create_buffer(&self) -> Result<ScatterGatherBuffer> {
352        ScatterGatherBuffer::new(self.mem.clone(), self.transfer_descriptor.clone())
353            .map_err(Error::CreateBuffer)
354    }
355
356    /// Create a usb request setup for the control transfer buffer
357    pub fn create_usb_request_setup(&self) -> Result<UsbRequestSetup> {
358        let first_atrb = self.transfer_descriptor.first_atrb();
359        let trb = first_atrb
360            .trb
361            .checked_cast::<SetupStageTrb>()
362            .map_err(Error::CastTrb)?;
363        Ok(UsbRequestSetup::new(
364            trb.get_request_type(),
365            trb.get_request(),
366            trb.get_value(),
367            trb.get_index(),
368            trb.get_length(),
369        ))
370    }
371
372    /// Get endpoint number.
373    pub fn get_endpoint_number(&self) -> u8 {
374        // See spec 4.5.1 for dci.
375        self.endpoint_id / 2
376    }
377
378    /// get transfer direction.
379    pub fn get_transfer_dir(&self) -> TransferDirection {
380        self.transfer_dir
381    }
382
383    /// get stream id.
384    pub fn get_stream_id(&self) -> Option<u16> {
385        self.stream_id
386    }
387
388    /// get max payload length for Isochronous transfer.
389    pub fn get_max_payload(&self) -> Result<u32> {
390        let Some(device_slot) = self.device_slot.upgrade() else {
391            return Err(Error::GetMaxPayload(self.endpoint_id));
392        };
393        device_slot
394            .get_max_esit_payload(self.endpoint_id)
395            .map_err(|_| Error::GetMaxPayload(self.endpoint_id))
396    }
397
398    fn process_td_results(
399        &self,
400        status: &TransferStatus,
401        bytes_transferred: u32,
402    ) -> Result<Vec<TransferAction>> {
403        let mut actions = Vec::new();
404        // This can be processed first, because xHCI specification does not guarantee the order
405        // between the context update and the transfer completion event.
406        if *status == TransferStatus::Stalled {
407            warn!("xhci: endpoint is stalled. set state to Halted");
408            actions.push(TransferAction::HaltEndpoint);
409        }
410
411        let mut edtla: u32 = 0;
412        let mut remaining_transferred = bytes_transferred;
413        let mut retiring_on_short: bool = false;
414        let mut residual_on_short: u32 = 0;
415        let last_atrb_gpa = self.transfer_descriptor.last_atrb().gpa;
416
417        // As noted in xHCI spec 4.11.3.1
418        // Transfer Event TRB only occurs under the following conditions:
419        //   1. If the Interrupt On Completion flag is set.
420        //   2. When a short transfer occurs during the execution of a Transfer TRB and the
421        //      Interrupt-on-Short Packet flag is set.
422        //   3. If an error occurs during the execution of a Transfer TRB.
423        for atrb in &self.transfer_descriptor {
424            // For details about event data trb and EDTLA, see spec 4.11.5.2.
425            if atrb.trb.get_trb_type().map_err(Error::TrbType)? == TrbType::EventData {
426                let code = if retiring_on_short {
427                    TrbCompletionCode::ShortPacket
428                } else {
429                    TrbCompletionCode::Success
430                };
431                actions.push(TransferAction::SendEvent {
432                    code,
433                    gpa: atrb
434                        .trb
435                        .cast::<EventDataTrb>()
436                        .map_err(Error::CastTrb)?
437                        .get_event_data(),
438                    residual_or_edtla: edtla,
439                    event_data: true,
440                });
441                edtla = 0;
442                self.report_completion(atrb);
443                continue;
444            }
445
446            let length = atrb.trb.transfer_length().map_err(Error::TransferLength)?;
447            let transferred = min(length, remaining_transferred);
448            remaining_transferred -= transferred;
449
450            let residual = length - transferred;
451            edtla += transferred;
452
453            // Report StallError for the TRB with residual data or the last TRB of the TD.
454            // The latter condition covers a stall on the Status Stage. The TRB that caused the
455            // stall is considered as not completed.
456            if *status == TransferStatus::Stalled && (residual > 0 || atrb.gpa == last_atrb_gpa) {
457                debug!("xhci: on transfer complete stalled");
458                actions.push(TransferAction::SendEvent {
459                    code: TrbCompletionCode::StallError,
460                    gpa: atrb.gpa,
461                    residual_or_edtla: residual,
462                    event_data: false,
463                });
464                break;
465            }
466
467            // If Short Packet is detected, the rest of the TRBs in the same TD are not executed.
468            // However, events are still generated for the EventData TRBs (handled above) or other
469            // TRBs with IOC (handled below).
470            if retiring_on_short {
471                if atrb.trb.interrupt_on_completion() {
472                    actions.push(TransferAction::SendEvent {
473                        code: TrbCompletionCode::ShortPacket,
474                        gpa: atrb.gpa,
475                        residual_or_edtla: residual_on_short,
476                        event_data: false,
477                    });
478                }
479            } else if residual > 0 {
480                retiring_on_short = true;
481                residual_on_short = residual;
482                if atrb.trb.interrupt_on_completion() || atrb.trb.interrupt_on_short_packet() {
483                    debug!("xhci: on transfer complete short packet");
484                    actions.push(TransferAction::SendEvent {
485                        code: TrbCompletionCode::ShortPacket,
486                        gpa: atrb.gpa,
487                        residual_or_edtla: residual,
488                        event_data: false,
489                    });
490                }
491            } else if atrb.trb.interrupt_on_completion() {
492                debug!("xhci: on transfer complete success");
493                actions.push(TransferAction::SendEvent {
494                    code: TrbCompletionCode::Success,
495                    gpa: atrb.gpa,
496                    residual_or_edtla: 0,
497                    event_data: false,
498                });
499            }
500
501            // The dequeue pointer still needs to be advanced after a Short Packet.
502            self.report_completion(atrb);
503        }
504        Ok(actions)
505    }
506
507    fn report_completion(&self, trb: &AddressedTrb) {
508        if let Some(device_slot) = self.device_slot.upgrade() {
509            device_slot.report_trb_completion(self.endpoint_id, self.stream_id, trb);
510        }
511    }
512
513    /// This functions should be invoked when transfer is completed (or failed).
514    pub fn on_transfer_complete(
515        &self,
516        status: &TransferStatus,
517        bytes_transferred: u32,
518    ) -> Result<()> {
519        match status {
520            TransferStatus::NoDevice => {
521                info!("xhci: device disconnected, detaching from port");
522                // Actual port detachment is handled by the UsbUtilEventHandler.
523                return self.send_late_trigger();
524            }
525            TransferStatus::Cancelled => {
526                return self.send_late_trigger();
527            }
528            TransferStatus::Completed => {}
529            TransferStatus::Stalled => {
530                // This is not a critical error, especially during the enumeration. Some devices
531                // takes time to become ready and may return StallError until then. A mass storage
532                // may also return StallError on checking write-protection.
533            }
534            TransferStatus::Error => {
535                // Transfer failed, we are not handling this correctly yet. Guest kernel might see
536                // short packets for in transfer and might think control transfer is successful. It
537                // will eventually find out device is in a wrong state.
538            }
539        }
540
541        let mut halted = false;
542        let actions = self.process_td_results(status, bytes_transferred)?;
543        for action in actions {
544            match action {
545                TransferAction::SendEvent {
546                    code,
547                    gpa,
548                    residual_or_edtla,
549                    event_data,
550                } => {
551                    self.interrupter
552                        .lock()
553                        .send_transfer_event_trb(
554                            code,
555                            gpa,
556                            residual_or_edtla,
557                            event_data,
558                            self.slot_id,
559                            self.endpoint_id,
560                        )
561                        .map_err(Error::SendInterrupt)?;
562                }
563                TransferAction::HaltEndpoint => {
564                    if let Some(device_slot) = self.device_slot.upgrade() {
565                        device_slot
566                            .halt_endpoint(self.endpoint_id)
567                            .map_err(|_| Error::HaltEndpoint(self.endpoint_id))?;
568                        halted = true;
569                    }
570                }
571            }
572        }
573
574        // Since the event loop is single threaded, there's no need to trigger it early. We delay
575        // it to the end so that its error, if it ever occurs, won't cause the above transfer
576        // events to be omitted.
577        if !halted {
578            self.send_late_trigger()
579        } else {
580            Ok(())
581        }
582    }
583
584    /// Send this transfer to backend if it's a valid transfer.
585    pub fn send_to_backend_if_valid(self) -> Result<()> {
586        if self.validate_transfer()? {
587            // Backend should call on_transfer_complete() when the transfer is completed to generate
588            // events. The ring controller is also triggered there for bulk & interrupt transfers.
589            // For an isochronous transfer, the ring controller is triggered here without waiting
590            // for the completion.
591            let port = self.port.clone();
592            let mut backend = port.backend_device();
593            match &mut *backend {
594                Some(backend) => {
595                    self.send_early_trigger()?;
596                    backend
597                        .lock()
598                        .submit_xhci_transfer(self)
599                        .map_err(|_| Error::SubmitTransfer)?
600                }
601                None => {
602                    error!("backend is already disconnected");
603                    self.send_trigger()?;
604                }
605            }
606        } else {
607            error!("invalid td on transfer ring");
608            self.send_trigger()?;
609        }
610        Ok(())
611    }
612
613    // Check each trb in the transfer descriptor for invalid or out of bounds
614    // parameters. Returns true iff the transfer descriptor is valid.
615    fn validate_transfer(&self) -> Result<bool> {
616        let mut valid = true;
617        for atrb in &self.transfer_descriptor {
618            if !trb_is_valid(atrb) {
619                self.interrupter
620                    .lock()
621                    .send_transfer_event_trb(
622                        TrbCompletionCode::TrbError,
623                        atrb.gpa,
624                        0,
625                        false,
626                        self.slot_id,
627                        self.endpoint_id,
628                    )
629                    .map_err(Error::SendInterrupt)?;
630                valid = false;
631            }
632        }
633        Ok(valid)
634    }
635
636    // ISOC transfer posts many small descriptors at once. We give an early signal to the ring
637    // controller so that we can keep up with the timing requirements.
638    fn needs_early_trigger(&self) -> bool {
639        matches!(self.get_transfer_type(), Ok(XhciTransferType::Isochronous))
640    }
641
642    fn send_early_trigger(&self) -> Result<()> {
643        if self.needs_early_trigger() {
644            self.send_trigger()?;
645        }
646        Ok(())
647    }
648
649    fn send_late_trigger(&self) -> Result<()> {
650        if !self.needs_early_trigger() {
651            self.send_trigger()?;
652        }
653        Ok(())
654    }
655
656    fn send_trigger(&self) -> Result<()> {
657        self.trigger_event
658            .signal()
659            .map_err(Error::WriteCompletionEvent)
660    }
661}
662
663fn trb_is_valid(atrb: &AddressedTrb) -> bool {
664    let can_be_in_transfer_ring = match atrb.trb.can_be_in_transfer_ring() {
665        Ok(v) => v,
666        Err(e) => {
667            error!("unknown error {:?}", e);
668            return false;
669        }
670    };
671    can_be_in_transfer_ring && (atrb.trb.interrupter_target() < MAX_INTERRUPTER)
672}
673
674#[cfg(test)]
675mod tests {
676    use base::pagesize;
677    use vm_memory::GuestAddress;
678
679    use super::*;
680    use crate::usb::xhci::xhci_abi::NormalTrb;
681    use crate::usb::xhci::xhci_abi::StatusStageTrb;
682    use crate::usb::xhci::xhci_abi::Trb;
683    use crate::usb::xhci::xhci_backend_device::BackendType;
684    use crate::usb::xhci::XhciRegs;
685
686    fn create_test_transfer(trbs: Vec<Trb>) -> XhciTransfer {
687        let mem = GuestMemory::new(&[(GuestAddress(0), pagesize() as u64)]).unwrap();
688        let mut gpa = 0x100;
689        let mut atrbs = Vec::new();
690        for trb in trbs {
691            mem.write_obj_at_addr(trb, GuestAddress(gpa)).unwrap();
692            atrbs.push(AddressedTrb { trb, gpa });
693            gpa += 16;
694        }
695
696        let td = TransferDescriptor::new(atrbs).unwrap();
697        let manager = XhciTransferManager::new(Weak::new());
698
699        let test_reg32 = register!(
700            name: "test",
701            ty: u32,
702            offset: 0x0,
703            reset_value: 0,
704            guest_writeable_mask: 0x0,
705            guest_write_1_to_clear_mask: 0,
706        );
707        let test_reg64 = register!(
708            name: "test",
709            ty: u64,
710            offset: 0x0,
711            reset_value: 0,
712            guest_writeable_mask: 0x0,
713            guest_write_1_to_clear_mask: 0,
714        );
715        let xhci_regs = XhciRegs {
716            usbcmd: test_reg32.clone(),
717            usbsts: test_reg32.clone(),
718            dnctrl: test_reg32.clone(),
719            crcr: test_reg64.clone(),
720            dcbaap: test_reg64.clone(),
721            config: test_reg64.clone(),
722            portsc: vec![test_reg32.clone(); 16],
723            doorbells: Vec::new(),
724            iman: test_reg32.clone(),
725            imod: test_reg32.clone(),
726            erstsz: test_reg32.clone(),
727            erstba: test_reg64.clone(),
728            erdp: test_reg64.clone(),
729        };
730
731        XhciTransfer {
732            manager,
733            state: Arc::new(Mutex::new(XhciTransferState::Created)),
734            mem,
735            port: Arc::new(UsbPort::new(
736                BackendType::Usb2,
737                1,
738                test_reg32.clone(),
739                test_reg32.clone(),
740                Arc::new(Mutex::new(Interrupter::new(
741                    GuestMemory::new(&[]).unwrap(),
742                    Event::new().unwrap(),
743                    &xhci_regs,
744                ))),
745            )),
746            interrupter: Arc::new(Mutex::new(Interrupter::new(
747                GuestMemory::new(&[]).unwrap(),
748                Event::new().unwrap(),
749                &xhci_regs,
750            ))),
751            trigger_event: Event::new().unwrap(),
752            slot_id: 1,
753            endpoint_id: 2,
754            transfer_dir: TransferDirection::Out,
755            transfer_descriptor: td,
756            device_slot: Weak::new(),
757            stream_id: None,
758        }
759    }
760
761    #[test]
762    fn test_bulk_success() {
763        let mut trb = Trb::new();
764        let normal_trb = trb.cast_mut::<NormalTrb>().unwrap();
765        normal_trb.set_trb_type(TrbType::Normal);
766        normal_trb.set_trb_transfer_length(100);
767        normal_trb.set_interrupt_on_completion(1);
768        let transfer = create_test_transfer(vec![trb]);
769
770        let actions = transfer
771            .process_td_results(&TransferStatus::Completed, 100)
772            .unwrap();
773        assert_eq!(
774            actions,
775            vec![TransferAction::SendEvent {
776                code: TrbCompletionCode::Success,
777                gpa: 0x100,
778                residual_or_edtla: 0,
779                event_data: false,
780            }]
781        );
782    }
783
784    #[test]
785    fn test_bulk_short_with_isp() {
786        // xHCI 4.10.1.1 Short Transfers states that an event should be generated if ISP or IOC is
787        // set to 1.
788        let mut trb = Trb::new();
789        let normal_trb = trb.cast_mut::<NormalTrb>().unwrap();
790        normal_trb.set_trb_type(TrbType::Normal);
791        normal_trb.set_trb_transfer_length(100);
792        normal_trb.set_interrupt_on_short_packet(1);
793        let transfer = create_test_transfer(vec![trb]);
794
795        let actions = transfer
796            .process_td_results(&TransferStatus::Completed, 40)
797            .unwrap();
798        assert_eq!(
799            actions,
800            vec![TransferAction::SendEvent {
801                code: TrbCompletionCode::ShortPacket,
802                gpa: 0x100,
803                residual_or_edtla: 60,
804                event_data: false,
805            }]
806        );
807    }
808
809    #[test]
810    fn test_bulk_short_with_ioc() {
811        // xHCI 4.10.1.1 Short Transfers states that an event should be generated if ISP or IOC is
812        // set to 1.
813        let mut trb = Trb::new();
814        let normal_trb = trb.cast_mut::<NormalTrb>().unwrap();
815        normal_trb.set_trb_type(TrbType::Normal);
816        normal_trb.set_trb_transfer_length(100);
817        normal_trb.set_interrupt_on_completion(1);
818        let transfer = create_test_transfer(vec![trb]);
819
820        let actions = transfer
821            .process_td_results(&TransferStatus::Completed, 40)
822            .unwrap();
823        assert_eq!(
824            actions,
825            vec![TransferAction::SendEvent {
826                code: TrbCompletionCode::ShortPacket,
827                gpa: 0x100,
828                residual_or_edtla: 60,
829                event_data: false,
830            }]
831        );
832    }
833
834    #[test]
835    fn test_bulk_without_evendata_retiring_after_short() {
836        // xHCI 4.9.1 Transfer Descriptors states that the TD should retire after detecting a short
837        // packet condition, but xHCI 4.10.1.1 Short Transfers states it should still generate an
838        // event for a TRB with IOC.
839        let mut trb1 = Trb::new();
840        let normal_trb1 = trb1.cast_mut::<NormalTrb>().unwrap();
841        normal_trb1.set_trb_type(TrbType::Normal);
842        normal_trb1.set_trb_transfer_length(100);
843        normal_trb1.set_interrupt_on_short_packet(1);
844
845        let mut trb2 = Trb::new();
846        let normal_trb2 = trb2.cast_mut::<NormalTrb>().unwrap();
847        normal_trb2.set_trb_type(TrbType::Normal);
848        normal_trb2.set_trb_transfer_length(100);
849        normal_trb2.set_interrupt_on_completion(1);
850
851        let mut trb3 = Trb::new();
852        let normal_trb3 = trb3.cast_mut::<NormalTrb>().unwrap();
853        normal_trb3.set_trb_type(TrbType::Normal);
854        normal_trb3.set_trb_transfer_length(100);
855        normal_trb3.set_interrupt_on_completion(1);
856
857        let transfer = create_test_transfer(vec![trb1, trb2, trb3]);
858
859        let actions = transfer
860            .process_td_results(&TransferStatus::Completed, 40)
861            .unwrap();
862        assert_eq!(
863            actions,
864            vec![
865                TransferAction::SendEvent {
866                    code: TrbCompletionCode::ShortPacket,
867                    gpa: 0x100,
868                    residual_or_edtla: 60,
869                    event_data: false,
870                },
871                TransferAction::SendEvent {
872                    code: TrbCompletionCode::ShortPacket,
873                    gpa: 0x110,
874                    residual_or_edtla: 60,
875                    event_data: false,
876                },
877                TransferAction::SendEvent {
878                    code: TrbCompletionCode::ShortPacket,
879                    gpa: 0x120,
880                    residual_or_edtla: 60,
881                    event_data: false,
882                },
883            ]
884        );
885    }
886
887    #[test]
888    fn test_bulk_with_evendata_retiring_after_short() {
889        // xHCI 4.9.1 Transfer Descriptors states that the TD should retire after detecting a short
890        // packet condition, but xHCI 4.10.1.1 Short Transfers states it should still generate an
891        // event for EventData TRB. This test assumes that we have PAE=1 in HCCPARAMS1 that forces
892        // all the EventData TRBs to generate an event even after the Short Packet.
893        let mut trb1 = Trb::new();
894        let normal_trb1 = trb1.cast_mut::<NormalTrb>().unwrap();
895        normal_trb1.set_trb_type(TrbType::Normal);
896        normal_trb1.set_trb_transfer_length(100);
897        normal_trb1.set_interrupt_on_short_packet(1);
898
899        let mut trb2 = Trb::new();
900        let event_trb = trb2.cast_mut::<EventDataTrb>().unwrap();
901        event_trb.set_trb_type(TrbType::EventData);
902        event_trb.set_event_data(0x12345678abcdef0);
903        event_trb.set_interrupt_on_completion(1);
904
905        let mut trb3 = Trb::new();
906        let event_trb = trb3.cast_mut::<EventDataTrb>().unwrap();
907        event_trb.set_trb_type(TrbType::EventData);
908        event_trb.set_event_data(0x12345678abcdef1);
909        event_trb.set_interrupt_on_completion(1);
910
911        let transfer = create_test_transfer(vec![trb1, trb2, trb3]);
912
913        let actions = transfer
914            .process_td_results(&TransferStatus::Completed, 40)
915            .unwrap();
916        assert_eq!(
917            actions,
918            vec![
919                TransferAction::SendEvent {
920                    code: TrbCompletionCode::ShortPacket,
921                    gpa: 0x100,
922                    residual_or_edtla: 60,
923                    event_data: false,
924                },
925                TransferAction::SendEvent {
926                    code: TrbCompletionCode::ShortPacket,
927                    gpa: 0x12345678abcdef0,
928                    residual_or_edtla: 40, // EDTLA
929                    event_data: true,
930                },
931                TransferAction::SendEvent {
932                    code: TrbCompletionCode::ShortPacket,
933                    gpa: 0x12345678abcdef1,
934                    residual_or_edtla: 0, // EDTLA
935                    event_data: true,
936                },
937            ]
938        );
939    }
940
941    #[test]
942    fn test_bulk_stall_partial() {
943        // xHCI 4.10.2 Errors state that an error during a transfer shall always generate an event,
944        // irrespective of whether ISP or IOC is set to 1. Also, 4.10.2.1 Stall Error states that
945        // the endpoint state should transition to Halted.
946        let mut trb = Trb::new();
947        let normal_trb = trb.cast_mut::<NormalTrb>().unwrap();
948        normal_trb.set_trb_type(TrbType::Normal);
949        normal_trb.set_trb_transfer_length(100);
950        let transfer = create_test_transfer(vec![trb]);
951
952        // Stall at 40 bytes.
953        let actions = transfer
954            .process_td_results(&TransferStatus::Stalled, 40)
955            .unwrap();
956        assert_eq!(
957            actions,
958            vec![
959                TransferAction::HaltEndpoint,
960                TransferAction::SendEvent {
961                    code: TrbCompletionCode::StallError,
962                    gpa: 0x100,
963                    residual_or_edtla: 60,
964                    event_data: false,
965                }
966            ]
967        );
968    }
969
970    #[test]
971    fn test_control_stall_no_data_stage() {
972        let mut trb = Trb::new();
973        let status_trb = trb.cast_mut::<StatusStageTrb>().unwrap();
974        status_trb.set_trb_type(TrbType::StatusStage);
975        status_trb.set_interrupt_on_completion(1);
976
977        let transfer = create_test_transfer(vec![trb]);
978
979        // Stall at Status Stage (length 0). bytes_transferred = 0.
980        let actions = transfer
981            .process_td_results(&TransferStatus::Stalled, 0)
982            .unwrap();
983
984        assert_eq!(
985            actions,
986            vec![
987                TransferAction::HaltEndpoint,
988                TransferAction::SendEvent {
989                    code: TrbCompletionCode::StallError,
990                    gpa: 0x100,
991                    residual_or_edtla: 0,
992                    event_data: false,
993                }
994            ]
995        );
996    }
997
998    #[test]
999    fn test_bulk_stall_at_trb_start() {
1000        // xHCI 6.4.2.1 Transfer Event TRB states that if an error occurs during a transfer TRB,
1001        // the event TRB shall point to the offending TRB.
1002        let mut trb1 = Trb::new();
1003        let normal_trb1 = trb1.cast_mut::<NormalTrb>().unwrap();
1004        normal_trb1.set_trb_type(TrbType::Normal);
1005        normal_trb1.set_trb_transfer_length(100);
1006
1007        let mut trb2 = Trb::new();
1008        let normal_trb2 = trb2.cast_mut::<NormalTrb>().unwrap();
1009        normal_trb2.set_trb_type(TrbType::Normal);
1010        normal_trb2.set_trb_transfer_length(100);
1011
1012        let transfer = create_test_transfer(vec![trb1, trb2]);
1013
1014        // Stall after 100 bytes (immediately when the second TRB is started).
1015        let actions = transfer
1016            .process_td_results(&TransferStatus::Stalled, 100)
1017            .unwrap();
1018
1019        assert_eq!(
1020            actions,
1021            vec![
1022                TransferAction::HaltEndpoint,
1023                TransferAction::SendEvent {
1024                    code: TrbCompletionCode::StallError,
1025                    gpa: 0x110, // Second TRB GPA
1026                    residual_or_edtla: 100,
1027                    event_data: false,
1028                }
1029            ]
1030        );
1031    }
1032
1033    #[test]
1034    fn test_event_data_single() {
1035        // xHCI 4.11.5.2 Event Data TRB states that the event should report EDTLA and set Event
1036        // Data (ED) field to 1.
1037        let mut trb1 = Trb::new();
1038        let normal_trb = trb1.cast_mut::<NormalTrb>().unwrap();
1039        normal_trb.set_trb_type(TrbType::Normal);
1040        normal_trb.set_trb_transfer_length(100);
1041
1042        let mut trb2 = Trb::new();
1043        let event_trb = trb2.cast_mut::<EventDataTrb>().unwrap();
1044        event_trb.set_trb_type(TrbType::EventData);
1045        event_trb.set_event_data(0x12345678abcdef0);
1046        event_trb.set_interrupt_on_completion(1);
1047
1048        let transfer = create_test_transfer(vec![trb1, trb2]);
1049
1050        // Successful transfer.
1051        let actions = transfer
1052            .process_td_results(&TransferStatus::Completed, 100)
1053            .unwrap();
1054        assert_eq!(
1055            actions,
1056            vec![TransferAction::SendEvent {
1057                code: TrbCompletionCode::Success,
1058                gpa: 0x12345678abcdef0,
1059                residual_or_edtla: 100,
1060                event_data: true,
1061            }]
1062        );
1063
1064        // Short packet.
1065        let actions = transfer
1066            .process_td_results(&TransferStatus::Completed, 40)
1067            .unwrap();
1068        assert_eq!(
1069            actions,
1070            vec![TransferAction::SendEvent {
1071                code: TrbCompletionCode::ShortPacket,
1072                gpa: 0x12345678abcdef0,
1073                residual_or_edtla: 40,
1074                event_data: true,
1075            }]
1076        );
1077    }
1078
1079    #[test]
1080    fn test_event_data_multiple() {
1081        // xHCI 4.11.5.2 Event Data TRB states that EDTLA should be cleared to 0 when an EventData
1082        // TRB is encountered.
1083        let mut trb1 = Trb::new();
1084        let normal_trb = trb1.cast_mut::<NormalTrb>().unwrap();
1085        normal_trb.set_trb_type(TrbType::Normal);
1086        normal_trb.set_trb_transfer_length(100);
1087        normal_trb.set_interrupt_on_short_packet(1);
1088
1089        let mut trb2 = Trb::new();
1090        let event_trb = trb2.cast_mut::<EventDataTrb>().unwrap();
1091        event_trb.set_trb_type(TrbType::EventData);
1092        event_trb.set_event_data(0x12345678abcdef0);
1093        event_trb.set_interrupt_on_completion(1);
1094
1095        let transfer = create_test_transfer(vec![trb1, trb2, trb1, trb2]);
1096
1097        // Successful transfer.
1098        let actions = transfer
1099            .process_td_results(&TransferStatus::Completed, 200)
1100            .unwrap();
1101        assert_eq!(
1102            actions,
1103            vec![
1104                TransferAction::SendEvent {
1105                    code: TrbCompletionCode::Success,
1106                    gpa: 0x12345678abcdef0,
1107                    residual_or_edtla: 100, // EDTLA
1108                    event_data: true,
1109                },
1110                TransferAction::SendEvent {
1111                    code: TrbCompletionCode::Success,
1112                    gpa: 0x12345678abcdef0,
1113                    residual_or_edtla: 100, // EDTLA
1114                    event_data: true,
1115                },
1116            ]
1117        );
1118
1119        // Short packet. xHCI 5.3.6 Capability Parameters 1 (HCCPARAMS1) states that if Parse All
1120        // Event Data (PAE) field is 1, then it should parse all the EventData TRBs while advancing
1121        // to the next TD after a Short Packet. See also 4.10.1.1 Short Transfers.
1122        let actions = transfer
1123            .process_td_results(&TransferStatus::Completed, 40)
1124            .unwrap();
1125        assert_eq!(
1126            actions,
1127            vec![
1128                TransferAction::SendEvent {
1129                    code: TrbCompletionCode::ShortPacket,
1130                    gpa: 0x100,
1131                    residual_or_edtla: 60, // residual
1132                    event_data: false,
1133                },
1134                TransferAction::SendEvent {
1135                    code: TrbCompletionCode::ShortPacket,
1136                    gpa: 0x12345678abcdef0,
1137                    residual_or_edtla: 40, // EDTLA
1138                    event_data: true,
1139                },
1140                TransferAction::SendEvent {
1141                    code: TrbCompletionCode::ShortPacket,
1142                    gpa: 0x12345678abcdef0,
1143                    residual_or_edtla: 0, // EDTLA (from last Event)
1144                    event_data: true,
1145                },
1146            ]
1147        );
1148    }
1149}
devices/usb/xhci/xhci_transfer.rs

devices/usb/xhci/
xhci_transfer.rs
