devices/
virtcpufreq_v2.rs

1// Copyright 2024 The ChromiumOS Authors
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use std::fs::File;
6use std::path::PathBuf;
7use std::sync::atomic::AtomicU32;
8use std::sync::atomic::Ordering;
9use std::sync::Arc;
10use std::time::Duration;
11
12use anyhow::Context;
13use base::sched_attr;
14use base::sched_setattr;
15use base::set_cpu_affinity;
16use base::warn;
17use base::Error;
18use base::Event;
19use base::EventToken;
20use base::Timer;
21use base::TimerTrait;
22use base::Tube;
23use base::WaitContext;
24use base::WorkerThread;
25use sync::Mutex;
26use vm_control::DeviceId;
27use vm_control::PlatformDeviceId;
28
29use crate::BusAccessInfo;
30use crate::BusDevice;
31use crate::Suspendable;
32
33const CPUFREQ_GOV_SCALE_FACTOR_DEFAULT: u32 = 100;
34const CPUFREQ_GOV_SCALE_FACTOR_SCHEDUTIL: u32 = 80;
35
36const SCHED_FLAG_RESET_ON_FORK: u64 = 0x1;
37const SCHED_FLAG_KEEP_POLICY: u64 = 0x08;
38const SCHED_FLAG_KEEP_PARAMS: u64 = 0x10;
39const SCHED_FLAG_UTIL_CLAMP_MIN: u64 = 0x20;
40const SCHED_FLAG_UTIL_CLAMP_MAX: u64 = 0x40;
41
42const VCPUFREQ_CUR_PERF: u32 = 0x0;
43const VCPUFREQ_SET_PERF: u32 = 0x4;
44const VCPUFREQ_FREQTBL_LEN: u32 = 0x8;
45const VCPUFREQ_FREQTBL_SEL: u32 = 0xc;
46const VCPUFREQ_FREQTBL_RD: u32 = 0x10;
47const VCPUFREQ_PERF_DOMAIN: u32 = 0x14;
48
49const SCHED_FLAG_KEEP_ALL: u64 = SCHED_FLAG_KEEP_POLICY | SCHED_FLAG_KEEP_PARAMS;
50const SCHED_CAPACITY_SCALE: u32 = 1024;
51
52// Timer values in microseconds
53const MIN_TIMER_US: u32 = 75;
54const TIMER_OVERHEAD_US: u32 = 15;
55
56/// Upstream linux compatible version of the virtual cpufreq interface
57pub struct VirtCpufreqV2 {
58    vcpu_freq_table: Vec<u32>,
59    pcpu_fmax: u32,
60    pcpu_capacity: u32,
61    pcpu: u32,
62    util_factor: u32,
63    freqtbl_sel: u32,
64    vcpu_domain: u32,
65    domain_uclamp_min: Option<File>,
66    domain_uclamp_max: Option<File>,
67    vcpu_fmax: u32,
68    vcpu_capacity: u32,
69    vcpu_relative_capacity: u32,
70    worker: Option<WorkerThread<()>>,
71    timer: Arc<Mutex<Timer>>,
72    vm_ctrl: Arc<Mutex<Tube>>,
73    pcpu_min_cap: u32,
74    /// The largest(or the last) pCPU index to be used by all the vCPUs. This index is used to
75    /// figure out the proper placement of the throttle workers which are placed on pCPUs right
76    /// after the last pCPU being used the vCPUs. Throttle workers require their own exclusive
77    /// pCPU allocation and this ensure that the workers are placed contiguously and makes it
78    /// easier for user to manage pCPU allocations when running multiple instances on a large
79    /// server.
80    largest_pcpu_idx: usize,
81    //TODO: Put the shared_domain_members in a struct
82    shared_domain_vcpus: Vec<usize>,
83    shared_domain_perf: Arc<AtomicU32>,
84}
85
86fn get_cpu_info(cpu_id: u32, property: &str) -> Result<u32, Error> {
87    let path = format!("/sys/devices/system/cpu/cpu{cpu_id}/{property}");
88    std::fs::read_to_string(path)?
89        .trim()
90        .parse()
91        .map_err(|_| Error::new(libc::EINVAL))
92}
93
94fn get_cpu_info_str(cpu_id: u32, property: &str) -> Result<String, Error> {
95    let path = format!("/sys/devices/system/cpu/cpu{cpu_id}/{property}");
96    Ok(std::fs::read_to_string(path)?)
97}
98
99fn get_cpu_capacity(cpu_id: u32) -> Result<u32, Error> {
100    get_cpu_info(cpu_id, "cpu_capacity")
101}
102
103fn get_cpu_maxfreq_khz(cpu_id: u32) -> Result<u32, Error> {
104    get_cpu_info(cpu_id, "cpufreq/cpuinfo_max_freq")
105}
106
107fn get_cpu_minfreq_khz(cpu_id: u32) -> Result<u32, Error> {
108    get_cpu_info(cpu_id, "cpufreq/cpuinfo_min_freq")
109}
110
111fn get_cpu_curfreq_khz(cpu_id: u32) -> Result<u32, Error> {
112    get_cpu_info(cpu_id, "cpufreq/scaling_cur_freq")
113}
114
115fn get_cpu_util_factor(cpu_id: u32) -> u32 {
116    match get_cpu_info_str(cpu_id, "cpufreq/scaling_governor") {
117        Ok(gov) if gov.trim() == "schedutil" => CPUFREQ_GOV_SCALE_FACTOR_SCHEDUTIL,
118        Ok(_) => CPUFREQ_GOV_SCALE_FACTOR_DEFAULT,
119        Err(err) => {
120            warn!(
121                "Unable to get cpufreq governor for cpu{cpu_id}, using 100% default util factor: {err:#}"
122            );
123            CPUFREQ_GOV_SCALE_FACTOR_DEFAULT
124        }
125    }
126}
127
128impl VirtCpufreqV2 {
129    pub fn new(
130        pcpu: u32,
131        vcpu_freq_table: Vec<u32>,
132        vcpu_domain_path: Option<PathBuf>,
133        vcpu_domain: u32,
134        vcpu_capacity: u32,
135        largest_pcpu_idx: usize,
136        vm_ctrl: Arc<Mutex<Tube>>,
137        shared_domain_vcpus: Vec<usize>,
138        shared_domain_perf: Arc<AtomicU32>,
139    ) -> Self {
140        let pcpu_capacity = get_cpu_capacity(pcpu).expect("Error reading capacity");
141        let pcpu_fmax = get_cpu_maxfreq_khz(pcpu).expect("Error reading max freq");
142        let util_factor = get_cpu_util_factor(pcpu);
143        let freqtbl_sel = 0;
144        let mut domain_uclamp_min = None;
145        let mut domain_uclamp_max = None;
146        // The vcpu_capacity passed in is normalized for frequency, reverse the normalization to
147        // get the performance per clock ratio between the vCPU and the pCPU its running on. This
148        // "relative capacity" is an approximation of the delta in IPC (Instructions per Cycle)
149        // between the pCPU vs vCPU running a usecase containing a mix of instruction types.
150        let vcpu_fmax = vcpu_freq_table.clone().into_iter().max().unwrap();
151        let vcpu_relative_capacity =
152            u32::try_from(u64::from(vcpu_capacity) * u64::from(pcpu_fmax) / u64::from(vcpu_fmax))
153                .unwrap();
154        let pcpu_min_cap =
155            get_cpu_minfreq_khz(pcpu).expect("Error reading min freq") * pcpu_capacity / pcpu_fmax;
156
157        if let Some(cgroup_path) = &vcpu_domain_path {
158            domain_uclamp_min = Some(
159                File::create(cgroup_path.join("cpu.uclamp.min")).unwrap_or_else(|err| {
160                    panic!(
161                        "Err: {}, Unable to open: {}",
162                        err,
163                        cgroup_path.join("cpu.uclamp.min").display()
164                    )
165                }),
166            );
167            domain_uclamp_max = Some(
168                File::create(cgroup_path.join("cpu.uclamp.max")).unwrap_or_else(|err| {
169                    panic!(
170                        "Err: {}, Unable to open: {}",
171                        err,
172                        cgroup_path.join("cpu.uclamp.max").display()
173                    )
174                }),
175            );
176        }
177
178        VirtCpufreqV2 {
179            vcpu_freq_table,
180            pcpu_fmax,
181            pcpu_capacity,
182            pcpu,
183            util_factor,
184            freqtbl_sel,
185            vcpu_domain,
186            domain_uclamp_min,
187            domain_uclamp_max,
188            vcpu_fmax,
189            vcpu_capacity,
190            vcpu_relative_capacity,
191            worker: None,
192            timer: Arc::new(Mutex::new(Timer::new().expect("failed to create Timer"))),
193            vm_ctrl,
194            pcpu_min_cap,
195            largest_pcpu_idx,
196            shared_domain_vcpus,
197            shared_domain_perf,
198        }
199    }
200}
201
202impl BusDevice for VirtCpufreqV2 {
203    fn device_id(&self) -> DeviceId {
204        PlatformDeviceId::VirtCpufreq.into()
205    }
206
207    fn debug_label(&self) -> String {
208        "VirtCpufreq Device".to_owned()
209    }
210
211    fn read(&mut self, info: BusAccessInfo, data: &mut [u8]) {
212        if data.len() != std::mem::size_of::<u32>() {
213            warn!(
214                "{}: unsupported read length {}, only support 4bytes read",
215                self.debug_label(),
216                data.len()
217            );
218            return;
219        }
220
221        let val = match info.offset as u32 {
222            VCPUFREQ_CUR_PERF => {
223                let shared_util = self.shared_domain_perf.load(Ordering::SeqCst);
224                if shared_util != 0 && shared_util < self.pcpu_min_cap {
225                    shared_util * self.vcpu_fmax / self.vcpu_capacity
226                } else {
227                    match get_cpu_curfreq_khz(self.pcpu) {
228                        Ok(freq) => u32::try_from(
229                            u64::from(freq) * u64::from(self.pcpu_capacity)
230                                / u64::from(self.vcpu_relative_capacity),
231                        )
232                        .unwrap(),
233                        Err(_) => 0,
234                    }
235                }
236            }
237            VCPUFREQ_FREQTBL_LEN => self.vcpu_freq_table.len() as u32,
238            VCPUFREQ_PERF_DOMAIN => self.vcpu_domain,
239            VCPUFREQ_FREQTBL_RD => *self
240                .vcpu_freq_table
241                .get(self.freqtbl_sel as usize)
242                .unwrap_or(&0),
243            _ => {
244                warn!("{}: unsupported read address {}", self.debug_label(), info);
245                return;
246            }
247        };
248
249        let val_arr = val.to_ne_bytes();
250        data.copy_from_slice(&val_arr);
251    }
252
253    fn write(&mut self, info: BusAccessInfo, data: &[u8]) {
254        let val: u32 = match data.try_into().map(u32::from_ne_bytes) {
255            Ok(v) => v,
256            Err(e) => {
257                warn!(
258                    "{}: unsupported write length {:#}, only support 4bytes write",
259                    self.debug_label(),
260                    e
261                );
262                return;
263            }
264        };
265
266        match info.offset as u32 {
267            VCPUFREQ_SET_PERF => {
268                // Util margin depends on the cpufreq governor on the host
269                let util_raw = match u32::try_from(
270                    u64::from(self.vcpu_capacity) * u64::from(val) / u64::from(self.vcpu_fmax),
271                ) {
272                    Ok(util) => util,
273                    Err(e) => {
274                        warn!("Potential overflow {:#}", e);
275                        SCHED_CAPACITY_SCALE
276                    }
277                };
278
279                let util = util_raw * self.util_factor / CPUFREQ_GOV_SCALE_FACTOR_DEFAULT;
280
281                if let (Some(domain_uclamp_min), Some(domain_uclamp_max)) =
282                    (&mut self.domain_uclamp_min, &mut self.domain_uclamp_max)
283                {
284                    use std::io::Write;
285                    let val = util as f32 * 100.0 / SCHED_CAPACITY_SCALE as f32;
286                    let val_formatted = format!("{val:4}").into_bytes();
287
288                    if self.vcpu_fmax != self.pcpu_fmax {
289                        if let Err(e) = domain_uclamp_max.write(&val_formatted) {
290                            warn!("Error setting uclamp_max: {:#}", e);
291                        }
292                    }
293                    if let Err(e) = domain_uclamp_min.write(&val_formatted) {
294                        warn!("Error setting uclamp_min: {:#}", e);
295                    }
296                } else {
297                    let mut sched_attr = sched_attr {
298                        sched_flags: SCHED_FLAG_KEEP_ALL
299                            | SCHED_FLAG_UTIL_CLAMP_MIN
300                            | SCHED_FLAG_UTIL_CLAMP_MAX
301                            | SCHED_FLAG_RESET_ON_FORK,
302                        sched_util_min: util,
303                        ..Default::default()
304                    };
305
306                    if self.vcpu_fmax != self.pcpu_fmax {
307                        sched_attr.sched_util_max = util;
308                    } else {
309                        sched_attr.sched_util_max = SCHED_CAPACITY_SCALE;
310                    }
311
312                    if let Err(e) = sched_setattr(0, &mut sched_attr, 0) {
313                        panic!("{}: Error setting util value: {:#}", self.debug_label(), e);
314                    }
315                }
316
317                // Return early if vcpu_fmax matches pcpu_fmax as that denotes no vCPU throttling
318                // is required.
319                if self.vcpu_fmax == self.pcpu_fmax {
320                    return;
321                }
322
323                self.shared_domain_perf.store(util_raw, Ordering::SeqCst);
324                let timer = self.timer.clone();
325                if self.worker.is_none() {
326                    let vcpu_id = info.id;
327                    let vm_ctrl = self.vm_ctrl.clone();
328                    let worker_cpu_affinity = self.largest_pcpu_idx + self.vcpu_domain as usize + 1;
329                    let shared_domain_vcpus = self.shared_domain_vcpus.clone();
330
331                    self.worker = Some(WorkerThread::start(
332                        format!("vcpu_throttle{vcpu_id}"),
333                        move |kill_evt| {
334                            vcpufreq_worker_thread(
335                                shared_domain_vcpus,
336                                kill_evt,
337                                timer,
338                                vm_ctrl,
339                                worker_cpu_affinity,
340                            )
341                            .expect("error running vpucfreq_worker")
342                        },
343                    ));
344                } else if util_raw < self.pcpu_min_cap {
345                    // The period is porportional to the performance requested by the vCPU, we
346                    // reduce the timeout period to increase the amount of throttling applied to
347                    // the vCPU as the performance decreases. Ex. If vCPU requests half of the
348                    // performance relatively to its pCPU@FMin, the vCPU will spend 50% of its
349                    // cycles being throttled to increase time for the same workload that otherwise
350                    // would've taken 1/2 of the time if ran at pCPU@FMin. We could've
351                    // alternatively adjusted the workload and used some fixed period (such as
352                    // 250us), but there's a floor for the minimum delay we add (cost of handling
353                    // the userspace exit) and limits the range of performance we can emulate.
354                    let timeout_period = (MIN_TIMER_US + TIMER_OVERHEAD_US) as f32
355                        / (1.0 - (util_raw as f32 / self.pcpu_min_cap as f32));
356                    let _ = timer
357                        .lock()
358                        .reset_repeating(Duration::from_micros(timeout_period as u64));
359                } else {
360                    let _ = timer.lock().clear();
361                }
362            }
363            VCPUFREQ_FREQTBL_SEL => self.freqtbl_sel = val,
364            _ => {
365                warn!("{}: unsupported read address {}", self.debug_label(), info);
366            }
367        }
368    }
369}
370
371pub fn vcpufreq_worker_thread(
372    shared_domain_vcpus: Vec<usize>,
373    kill_evt: Event,
374    timer: Arc<Mutex<Timer>>,
375    vm_ctrl: Arc<Mutex<Tube>>,
376    cpu_affinity: usize,
377) -> anyhow::Result<()> {
378    #[derive(EventToken)]
379    enum Token {
380        // The timer expired.
381        TimerExpire,
382        // The parent thread requested an exit.
383        Kill,
384    }
385
386    let wait_ctx = WaitContext::build_with(&[
387        (&*timer.lock(), Token::TimerExpire),
388        (&kill_evt, Token::Kill),
389    ])
390    .context("Failed to create wait_ctx")?;
391
392    // The vcpufreq thread has strict scheduling requirements, let's affine it away from the vCPU
393    // threads and clamp its util to high value.
394    let cpu_set: Vec<usize> = vec![cpu_affinity];
395    set_cpu_affinity(cpu_set)?;
396
397    let mut sched_attr = sched_attr {
398        sched_flags: SCHED_FLAG_KEEP_ALL
399            | SCHED_FLAG_UTIL_CLAMP_MIN
400            | SCHED_FLAG_UTIL_CLAMP_MAX
401            | SCHED_FLAG_RESET_ON_FORK,
402        sched_util_min: SCHED_CAPACITY_SCALE,
403        sched_util_max: SCHED_CAPACITY_SCALE,
404        ..Default::default()
405    };
406    if let Err(e) = sched_setattr(0, &mut sched_attr, 0) {
407        warn!("Error setting util value: {}", e);
408    }
409
410    loop {
411        let events = wait_ctx.wait().context("Failed to wait for events")?;
412        for event in events.iter().filter(|e| e.is_readable) {
413            match event.token {
414                Token::TimerExpire => {
415                    timer
416                        .lock()
417                        .mark_waited()
418                        .context("failed to reset timer")?;
419                    let vm_ctrl_unlocked = vm_ctrl.lock();
420                    for vcpu_id in &shared_domain_vcpus {
421                        let msg = vm_control::VmRequest::Throttle(*vcpu_id, MIN_TIMER_US);
422                        vm_ctrl_unlocked
423                            .send(&msg)
424                            .context("failed to stall vCPUs")?;
425                    }
426                }
427                Token::Kill => {
428                    return Ok(());
429                }
430            }
431        }
432    }
433}
434
435impl Suspendable for VirtCpufreqV2 {}
devices/virtcpufreq_v2.rs

devices/
virtcpufreq_v2.rs
