| // Copyright 2020 The ChromiumOS Authors |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| // We have u32 constants from bindings that are passed into archiitecture-dependent functions |
| // taking u32/64 parameters. So on 32 bit platforms we may have needless casts. |
| #![allow(clippy::useless_conversion)] |
| |
| use std::collections::BTreeMap; |
| use std::convert::TryFrom; |
| use std::mem::offset_of; |
| |
| use aarch64_sys_reg::AArch64SysRegId; |
| use anyhow::Context; |
| use base::errno_result; |
| use base::error; |
| use base::ioctl_with_mut_ref; |
| use base::ioctl_with_ref; |
| use base::ioctl_with_val; |
| use base::warn; |
| use base::Error; |
| use base::Result; |
| use cros_fdt::Fdt; |
| use kvm_sys::*; |
| use libc::EINVAL; |
| use libc::ENOBUFS; |
| use libc::ENOENT; |
| use libc::ENOTSUP; |
| use libc::ENXIO; |
| use serde::Deserialize; |
| use serde::Serialize; |
| use snapshot::AnySnapshot; |
| use vm_memory::GuestAddress; |
| use zerocopy::FromZeros; |
| |
| use super::Config; |
| use super::Kvm; |
| use super::KvmCap; |
| use super::KvmVcpu; |
| use super::KvmVm; |
| use crate::ClockState; |
| use crate::DeviceKind; |
| use crate::HypercallAbi; |
| use crate::Hypervisor; |
| use crate::IrqSourceChip; |
| use crate::ProtectionType; |
| use crate::PsciVersion; |
| use crate::VcpuAArch64; |
| use crate::VcpuExit; |
| use crate::VcpuFeature; |
| use crate::VcpuRegAArch64; |
| use crate::VmAArch64; |
| use crate::VmCap; |
| use crate::AARCH64_MAX_REG_COUNT; |
| use crate::PSCI_0_2; |
| |
| impl Kvm { |
| // Compute the machine type, which should be the IPA range for the VM |
| // Ideally, this would take a description of the memory map and return |
| // the closest machine type for this VM. Here, we just return the maximum |
| // the kernel support. |
| pub fn get_vm_type(&self, protection_type: ProtectionType) -> Result<u32> { |
| // SAFETY: |
| // Safe because we know self is a real kvm fd |
| let ipa_size = match unsafe { |
| ioctl_with_val(self, KVM_CHECK_EXTENSION, KVM_CAP_ARM_VM_IPA_SIZE.into()) |
| } { |
| // Not supported? Use 0 as the machine type, which implies 40bit IPA |
| ret if ret < 0 => 0, |
| ipa => ipa as u32, |
| }; |
| let protection_flag = if protection_type.isolates_memory() { |
| KVM_VM_TYPE_ARM_PROTECTED |
| } else { |
| 0 |
| }; |
| // Use the lower 8 bits representing the IPA space as the machine type |
| Ok((ipa_size & KVM_VM_TYPE_ARM_IPA_SIZE_MASK) | protection_flag) |
| } |
| |
| /// Get the size of guest physical addresses (IPA) in bits. |
| pub fn get_guest_phys_addr_bits(&self) -> u8 { |
| // SAFETY: |
| // Safe because we know self is a real kvm fd |
| match unsafe { ioctl_with_val(self, KVM_CHECK_EXTENSION, KVM_CAP_ARM_VM_IPA_SIZE.into()) } { |
| // Default physical address size is 40 bits if the extension is not supported. |
| ret if ret <= 0 => 40, |
| ipa => ipa as u8, |
| } |
| } |
| } |
| |
| impl KvmVm { |
| /// Does platform specific initialization for the KvmVm. |
| pub fn init_arch(&self, cfg: &Config) -> Result<()> { |
| if cfg.mte { |
| // SAFETY: |
| // Safe because it does not take pointer arguments. |
| unsafe { self.enable_raw_capability(KvmCap::ArmMte, 0, &[0, 0, 0, 0])? } |
| } |
| #[cfg(target_os = "android")] |
| if cfg.ffa { |
| self.set_enable_ffa(true)?; |
| } |
| |
| Ok(()) |
| } |
| |
| /// Checks if a particular `VmCap` is available, or returns None if arch-independent |
| /// Vm.check_capability() should handle the check. |
| pub fn check_capability_arch(&self, _c: VmCap) -> Option<bool> { |
| None |
| } |
| |
| /// Returns the params to pass to KVM_CREATE_DEVICE for a `kind` device on this arch, or None to |
| /// let the arch-independent `KvmVm::create_device` handle it. |
| pub fn get_device_params_arch(&self, kind: DeviceKind) -> Option<kvm_create_device> { |
| match kind { |
| DeviceKind::ArmVgicV2 => Some(kvm_create_device { |
| type_: kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V2, |
| fd: 0, |
| flags: 0, |
| }), |
| DeviceKind::ArmVgicV3 => Some(kvm_create_device { |
| type_: kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V3, |
| fd: 0, |
| flags: 0, |
| }), |
| DeviceKind::ArmVgicIts => Some(kvm_create_device { |
| type_: kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_ITS, |
| fd: 0, |
| flags: 0, |
| }), |
| _ => None, |
| } |
| } |
| |
| pub(super) fn enable_smccc_forwarding(&mut self, base: u32, nr_functions: u32) -> Result<()> { |
| let smccc_filter = kvm_smccc_filter { |
| base, |
| nr_functions, |
| action: kvm_smccc_filter_action_KVM_SMCCC_FILTER_FWD_TO_USER as u8, |
| pad: [0; 15], |
| }; |
| let dev_attr = kvm_device_attr { |
| group: KVM_ARM_VM_SMCCC_CTRL, |
| attr: KVM_ARM_VM_SMCCC_FILTER as u64, |
| addr: &smccc_filter as *const _ as u64, |
| flags: 0, |
| }; |
| |
| // SAFETY: |
| // Safe because we know that our file is a VM fd, we know the kernel will only read the |
| // correct amount of memory from our pointer, and we verify the return result. |
| match unsafe { ioctl_with_ref(self, KVM_SET_DEVICE_ATTR, &dev_attr) } { |
| 0 => Ok(()), |
| _ => errno_result(), |
| } |
| } |
| |
| /// Arch-specific implementation of `Vm::get_pvclock`. Always returns an error on AArch64. |
| pub fn get_pvclock_arch(&self) -> Result<ClockState> { |
| Err(Error::new(ENXIO)) |
| } |
| |
| /// Arch-specific implementation of `Vm::set_pvclock`. Always returns an error on AArch64. |
| pub fn set_pvclock_arch(&self, _state: &ClockState) -> Result<()> { |
| Err(Error::new(ENXIO)) |
| } |
| |
| /// Get pKVM hypervisor details, e.g. the firmware size. |
| /// |
| /// Returns `Err` if not running under pKVM. |
| /// |
| /// Uses `KVM_ENABLE_CAP` internally, but it is only a getter, there should be no side effects |
| /// in KVM. |
| fn get_protected_vm_info(&self) -> Result<KvmProtectedVmInfo> { |
| let mut info = KvmProtectedVmInfo { |
| firmware_size: 0, |
| reserved: [0; 7], |
| }; |
| // SAFETY: |
| // Safe because we allocated the struct and we know the kernel won't write beyond the end of |
| // the struct or keep a pointer to it. |
| unsafe { |
| self.enable_raw_capability( |
| KvmCap::ArmProtectedVm, |
| KVM_CAP_ARM_PROTECTED_VM_FLAGS_INFO, |
| &[&mut info as *mut KvmProtectedVmInfo as u64, 0, 0, 0], |
| ) |
| }?; |
| Ok(info) |
| } |
| |
| fn set_protected_vm_firmware_ipa(&self, fw_addr: GuestAddress) -> Result<()> { |
| // SAFETY: |
| // Safe because none of the args are pointers. |
| unsafe { |
| self.enable_raw_capability( |
| KvmCap::ArmProtectedVm, |
| KVM_CAP_ARM_PROTECTED_VM_FLAGS_SET_FW_IPA, |
| &[fw_addr.0, 0, 0, 0], |
| ) |
| } |
| } |
| |
| #[cfg(all(target_os = "android", target_arch = "aarch64"))] |
| fn set_enable_ffa(&self, ffa_support: bool) -> Result<()> { |
| // SAFETY: |
| // Safe because none of the args are pointers. |
| unsafe { |
| self.enable_raw_capability( |
| KvmCap::ArmProtectedVm, |
| KVM_CAP_ARM_PROTECTED_VM_FLAGS_SET_FFA, |
| &[ffa_support.into(), 0, 0, 0], |
| ) |
| } |
| } |
| } |
| |
| #[repr(C)] |
| struct KvmProtectedVmInfo { |
| firmware_size: u64, |
| reserved: [u64; 7], |
| } |
| |
| impl VmAArch64 for KvmVm { |
| fn get_hypervisor(&self) -> &dyn Hypervisor { |
| &self.kvm |
| } |
| |
| fn load_protected_vm_firmware( |
| &mut self, |
| fw_addr: GuestAddress, |
| fw_max_size: u64, |
| ) -> Result<()> { |
| let info = self.get_protected_vm_info()?; |
| if info.firmware_size == 0 { |
| Err(Error::new(ENOENT)) |
| } else { |
| if info.firmware_size > fw_max_size { |
| return Err(Error::new(ENOBUFS)); |
| } |
| self.set_protected_vm_firmware_ipa(fw_addr) |
| } |
| } |
| |
| fn create_vcpu(&self, id: usize) -> Result<Box<dyn VcpuAArch64>> { |
| // create_vcpu is declared separately in VmAArch64 and VmX86, so it can return VcpuAArch64 |
| // or VcpuX86. But both use the same implementation in KvmVm::create_kvm_vcpu. |
| Ok(Box::new(self.create_kvm_vcpu(id)?)) |
| } |
| |
| fn create_fdt(&self, _fdt: &mut Fdt, _phandles: &BTreeMap<&str, u32>) -> cros_fdt::Result<()> { |
| Ok(()) |
| } |
| |
| fn init_arch( |
| &self, |
| _payload_entry_address: GuestAddress, |
| _fdt_address: GuestAddress, |
| _fdt_size: usize, |
| ) -> anyhow::Result<()> { |
| Ok(()) |
| } |
| |
| fn set_counter_offset(&self, offset: u64) -> Result<()> { |
| let off = kvm_arm_counter_offset { |
| counter_offset: offset, |
| reserved: 0, |
| }; |
| // SAFETY: self.vm is a valid KVM fd |
| let ret = unsafe { ioctl_with_ref(&self.vm, KVM_ARM_SET_COUNTER_OFFSET, &off) }; |
| if ret != 0 { |
| return errno_result(); |
| } |
| Ok(()) |
| } |
| } |
| |
| impl KvmVcpu { |
| /// Handles a `KVM_EXIT_SYSTEM_EVENT` with event type `KVM_SYSTEM_EVENT_RESET` with the given |
| /// event flags and returns the appropriate `VcpuExit` value for the run loop to handle. |
| /// |
| /// `event_flags` should be one or more of the `KVM_SYSTEM_EVENT_RESET_FLAG_*` values defined by |
| /// KVM. |
| pub fn system_event_reset(&self, event_flags: u64) -> Result<VcpuExit> { |
| if event_flags & u64::from(KVM_SYSTEM_EVENT_RESET_FLAG_PSCI_RESET2) != 0 { |
| // Read reset_type and cookie from x1 and x2. |
| let reset_type = self.get_one_reg(VcpuRegAArch64::X(1))?; |
| let cookie = self.get_one_reg(VcpuRegAArch64::X(2))?; |
| warn!( |
| "PSCI SYSTEM_RESET2 with reset_type={:#x}, cookie={:#x}", |
| reset_type, cookie |
| ); |
| } |
| Ok(VcpuExit::SystemEventReset) |
| } |
| |
| fn kvm_reg_id(&self, reg: VcpuRegAArch64) -> Result<KvmVcpuRegister> { |
| match reg { |
| VcpuRegAArch64::X(n @ 0..=30) => Ok(KvmVcpuRegister::X(n)), |
| VcpuRegAArch64::Sp => Ok(KvmVcpuRegister::Sp), |
| VcpuRegAArch64::Pc => Ok(KvmVcpuRegister::Pc), |
| VcpuRegAArch64::Pstate => Ok(KvmVcpuRegister::Pstate), |
| // Special case for multiplexed KVM registers |
| VcpuRegAArch64::System(aarch64_sys_reg::CCSIDR_EL1) => { |
| let csselr = |
| self.get_one_reg(VcpuRegAArch64::System(aarch64_sys_reg::CSSELR_EL1))?; |
| Ok(KvmVcpuRegister::Ccsidr(csselr as u8)) |
| } |
| VcpuRegAArch64::System(sysreg) => Ok(KvmVcpuRegister::System(sysreg)), |
| _ => Err(Error::new(EINVAL)), |
| } |
| } |
| |
| fn set_one_kvm_reg_u32(&self, kvm_reg_id: KvmVcpuRegister, data: u32) -> Result<()> { |
| self.set_one_kvm_reg(kvm_reg_id, data.to_ne_bytes().as_slice()) |
| } |
| |
| fn set_one_kvm_reg_u64(&self, kvm_reg_id: KvmVcpuRegister, data: u64) -> Result<()> { |
| self.set_one_kvm_reg(kvm_reg_id, data.to_ne_bytes().as_slice()) |
| } |
| |
| fn set_one_kvm_reg_u128(&self, kvm_reg_id: KvmVcpuRegister, data: u128) -> Result<()> { |
| self.set_one_kvm_reg(kvm_reg_id, data.to_ne_bytes().as_slice()) |
| } |
| |
| fn set_one_kvm_reg(&self, kvm_reg_id: KvmVcpuRegister, data: &[u8]) -> Result<()> { |
| assert_eq!(kvm_reg_id.size(), data.len()); |
| let id: u64 = kvm_reg_id.into(); |
| let onereg = kvm_one_reg { |
| id, |
| addr: (data.as_ptr() as usize) |
| .try_into() |
| .expect("can't represent usize as u64"), |
| }; |
| // SAFETY: |
| // Safe because we allocated the struct and we know the kernel will read exactly the size of |
| // the struct. |
| let ret = unsafe { ioctl_with_ref(self, KVM_SET_ONE_REG, &onereg) }; |
| if ret == 0 { |
| Ok(()) |
| } else { |
| errno_result() |
| } |
| } |
| |
| fn get_one_kvm_reg_u32(&self, kvm_reg_id: KvmVcpuRegister) -> Result<u32> { |
| let mut bytes = 0u32.to_ne_bytes(); |
| self.get_one_kvm_reg(kvm_reg_id, bytes.as_mut_slice())?; |
| Ok(u32::from_ne_bytes(bytes)) |
| } |
| |
| fn get_one_kvm_reg_u64(&self, kvm_reg_id: KvmVcpuRegister) -> Result<u64> { |
| let mut bytes = 0u64.to_ne_bytes(); |
| self.get_one_kvm_reg(kvm_reg_id, bytes.as_mut_slice())?; |
| Ok(u64::from_ne_bytes(bytes)) |
| } |
| |
| fn get_one_kvm_reg_u128(&self, kvm_reg_id: KvmVcpuRegister) -> Result<u128> { |
| let mut bytes = 0u128.to_ne_bytes(); |
| self.get_one_kvm_reg(kvm_reg_id, bytes.as_mut_slice())?; |
| Ok(u128::from_ne_bytes(bytes)) |
| } |
| |
| fn get_one_kvm_reg(&self, kvm_reg_id: KvmVcpuRegister, data: &mut [u8]) -> Result<()> { |
| assert_eq!(kvm_reg_id.size(), data.len()); |
| let id: u64 = kvm_reg_id.into(); |
| let onereg = kvm_one_reg { |
| id, |
| addr: (data.as_mut_ptr() as usize) |
| .try_into() |
| .expect("can't represent usize as u64"), |
| }; |
| |
| // SAFETY: |
| // Safe because we allocated the struct and we know the kernel will read exactly the size of |
| // the struct. |
| let ret = unsafe { ioctl_with_ref(self, KVM_GET_ONE_REG, &onereg) }; |
| if ret == 0 { |
| Ok(()) |
| } else { |
| errno_result() |
| } |
| } |
| |
| pub(super) fn handle_smccc_call( |
| &self, |
| handle_fn: &mut dyn FnMut(&mut HypercallAbi) -> anyhow::Result<()>, |
| ) -> anyhow::Result<()> { |
| const SMCCC_NOT_SUPPORTED: usize = u64::MAX as usize; |
| // SMCCC defines FID as the value in W0 with most significant bits ignored. |
| let function_id = (self.get_one_reg(VcpuRegAArch64::X(0))? as u32) |
| .try_into() |
| .unwrap(); |
| let args = &[ |
| self.get_one_reg(VcpuRegAArch64::X(1))?.try_into().unwrap(), |
| self.get_one_reg(VcpuRegAArch64::X(2))?.try_into().unwrap(), |
| self.get_one_reg(VcpuRegAArch64::X(3))?.try_into().unwrap(), |
| ]; |
| let default_res = &[SMCCC_NOT_SUPPORTED, 0, 0, 0]; |
| |
| let mut smccc_abi = HypercallAbi::new(function_id, args, default_res); |
| |
| let err_or_ok = handle_fn(&mut smccc_abi); |
| |
| for (i, value) in smccc_abi.get_results().iter().enumerate() { |
| self.set_one_reg(VcpuRegAArch64::X(i as _), (*value).try_into().unwrap())?; |
| } |
| |
| err_or_ok |
| } |
| |
| #[inline] |
| pub(crate) fn handle_vm_exit_arch(&self, _run: &mut kvm_run) -> Option<VcpuExit> { |
| // No aarch64-specific exits (for now) |
| None |
| } |
| |
| fn get_reg_list(&self) -> Result<Vec<u64>> { |
| let mut kvm_reg_list = kvm_reg_list::<[u64; AARCH64_MAX_REG_COUNT]>::new_zeroed(); |
| kvm_reg_list.n = AARCH64_MAX_REG_COUNT as u64; |
| let ret = |
| // SAFETY: |
| // We trust the kernel not to read/write past the end of kvm_reg_list struct. |
| unsafe { ioctl_with_mut_ref(self, KVM_GET_REG_LIST, &mut kvm_reg_list) }; |
| if ret < 0 { |
| return errno_result(); |
| } |
| let n = kvm_reg_list.n; |
| assert!( |
| n <= AARCH64_MAX_REG_COUNT as u64, |
| "Get reg list returned more registers than possible" |
| ); |
| Ok(kvm_reg_list.reg[..n as usize].to_vec()) |
| } |
| |
| fn get_features_bitmap(&self, features: &[VcpuFeature]) -> Result<u32> { |
| let mut all_features = 0; |
| let check_extension = |ext: u32| -> bool { |
| // SAFETY: |
| // Safe because we know self.vm is a real kvm fd |
| unsafe { ioctl_with_val(&self.vm, KVM_CHECK_EXTENSION, ext.into()) == 1 } |
| }; |
| |
| for f in features { |
| let shift = match f { |
| VcpuFeature::PsciV0_2 => KVM_ARM_VCPU_PSCI_0_2, |
| VcpuFeature::PmuV3 => KVM_ARM_VCPU_PMU_V3, |
| VcpuFeature::PowerOff => KVM_ARM_VCPU_POWER_OFF, |
| VcpuFeature::Sve => { |
| if !check_extension(KVM_CAP_ARM_SVE) { |
| return Err(Error::new(ENOTSUP)); |
| } |
| KVM_ARM_VCPU_SVE |
| } |
| }; |
| all_features |= 1 << shift; |
| } |
| |
| if check_extension(KVM_CAP_ARM_PTRAUTH_ADDRESS) |
| && check_extension(KVM_CAP_ARM_PTRAUTH_GENERIC) |
| { |
| all_features |= 1 << KVM_ARM_VCPU_PTRAUTH_ADDRESS; |
| all_features |= 1 << KVM_ARM_VCPU_PTRAUTH_GENERIC; |
| } |
| |
| Ok(all_features) |
| } |
| |
| /// Finalize VCPU features setup. This does not affect features that do not make use of |
| /// finalize. |
| fn finalize(&self, features: u32) -> Result<()> { |
| if (features & 1 << KVM_ARM_VCPU_SVE) != 0 { |
| // SAFETY: |
| // Safe because we know that our file is a Vcpu fd and we verify the return result. |
| let ret = unsafe { |
| ioctl_with_ref( |
| self, |
| KVM_ARM_VCPU_FINALIZE, |
| &std::os::raw::c_int::try_from(KVM_ARM_VCPU_SVE) |
| .map_err(|_| Error::new(EINVAL))?, |
| ) |
| }; |
| if ret != 0 { |
| return errno_result(); |
| } |
| } |
| |
| Ok(()) |
| } |
| } |
| |
| /// KVM registers as used by the `GET_ONE_REG`/`SET_ONE_REG` ioctl API |
| /// |
| /// These variants represent the registers as exposed by KVM which must be different from |
| /// `VcpuRegAArch64` to support registers which don't have an architectural definition such as |
| /// pseudo-registers (`Firmware`) and multiplexed registers (`Ccsidr`). |
| /// |
| /// See https://docs.kernel.org/virt/kvm/api.html for more details. |
| #[derive(Debug, Copy, Clone, Serialize, Deserialize)] |
| pub enum KvmVcpuRegister { |
| /// General Purpose Registers X0-X30 |
| X(u8), |
| /// Stack Pointer |
| Sp, |
| /// Program Counter |
| Pc, |
| /// Processor State |
| Pstate, |
| /// FP & SIMD Registers V0-V31 |
| V(u8), |
| /// KVM Firmware Pseudo-Registers |
| Firmware(u16), |
| /// System Registers |
| System(AArch64SysRegId), |
| /// CCSIDR_EL1 Demultiplexed by CSSELR_EL1 |
| Ccsidr(u8), |
| } |
| |
| impl KvmVcpuRegister { |
| // Firmware pseudo-registers are part of the ARM KVM interface: |
| // https://docs.kernel.org/virt/kvm/arm/hypercalls.html |
| pub const PSCI_VERSION: Self = Self::Firmware(0); |
| pub const SMCCC_ARCH_WORKAROUND_1: Self = Self::Firmware(1); |
| pub const SMCCC_ARCH_WORKAROUND_2: Self = Self::Firmware(2); |
| pub const SMCCC_ARCH_WORKAROUND_3: Self = Self::Firmware(3); |
| |
| /// Size of this register in bytes. |
| pub fn size(&self) -> usize { |
| let kvm_reg = u64::from(*self); |
| let size_field = kvm_reg & KVM_REG_SIZE_MASK; |
| const REG_SIZE_U8: u64 = KVM_REG_SIZE_U8 as u64; // cast from bindgen's u32 to u64 |
| match size_field { |
| REG_SIZE_U8 => 1, |
| KVM_REG_SIZE_U16 => 2, |
| KVM_REG_SIZE_U32 => 4, |
| KVM_REG_SIZE_U64 => 8, |
| KVM_REG_SIZE_U128 => 16, |
| KVM_REG_SIZE_U256 => 32, |
| KVM_REG_SIZE_U512 => 64, |
| KVM_REG_SIZE_U1024 => 128, |
| KVM_REG_SIZE_U2048 => 256, |
| // `From<KvmVcpuRegister> for u64` should always include a valid size. |
| _ => panic!("invalid size field {size_field}"), |
| } |
| } |
| } |
| |
| /// Gives the `u64` register ID expected by the `GET_ONE_REG`/`SET_ONE_REG` ioctl API. |
| /// |
| /// See the KVM documentation of those ioctls for details about the format of the register ID. |
| impl From<KvmVcpuRegister> for u64 { |
| fn from(register: KvmVcpuRegister) -> Self { |
| const fn reg(size: u64, kind: u64, fields: u64) -> u64 { |
| KVM_REG_ARM64 | size | kind | fields |
| } |
| |
| const fn kvm_regs_reg(size: u64, offset: usize) -> u64 { |
| let offset = offset / std::mem::size_of::<u32>(); |
| |
| reg(size, KVM_REG_ARM_CORE as u64, offset as u64) |
| } |
| |
| const fn kvm_reg(offset: usize) -> u64 { |
| kvm_regs_reg(KVM_REG_SIZE_U64, offset) |
| } |
| |
| fn spsr_reg(spsr_reg: u32) -> u64 { |
| let n = std::mem::size_of::<u64>() * (spsr_reg as usize); |
| kvm_reg(offset_of!(kvm_regs, spsr) + n) |
| } |
| |
| fn user_pt_reg(offset: usize) -> u64 { |
| kvm_regs_reg(KVM_REG_SIZE_U64, offset_of!(kvm_regs, regs) + offset) |
| } |
| |
| fn user_fpsimd_state_reg(size: u64, offset: usize) -> u64 { |
| kvm_regs_reg(size, offset_of!(kvm_regs, fp_regs) + offset) |
| } |
| |
| const fn reg_u64(kind: u64, fields: u64) -> u64 { |
| reg(KVM_REG_SIZE_U64, kind, fields) |
| } |
| |
| const fn demux_reg(size: u64, index: u64, value: u64) -> u64 { |
| let index = (index << KVM_REG_ARM_DEMUX_ID_SHIFT) & (KVM_REG_ARM_DEMUX_ID_MASK as u64); |
| let value = |
| (value << KVM_REG_ARM_DEMUX_VAL_SHIFT) & (KVM_REG_ARM_DEMUX_VAL_MASK as u64); |
| |
| reg(size, KVM_REG_ARM_DEMUX as u64, index | value) |
| } |
| |
| match register { |
| KvmVcpuRegister::X(n @ 0..=30) => { |
| let n = std::mem::size_of::<u64>() * (n as usize); |
| |
| user_pt_reg(offset_of!(user_pt_regs, regs) + n) |
| } |
| KvmVcpuRegister::X(n) => unreachable!("invalid KvmVcpuRegister Xn index: {n}"), |
| KvmVcpuRegister::Sp => user_pt_reg(offset_of!(user_pt_regs, sp)), |
| KvmVcpuRegister::Pc => user_pt_reg(offset_of!(user_pt_regs, pc)), |
| KvmVcpuRegister::Pstate => user_pt_reg(offset_of!(user_pt_regs, pstate)), |
| KvmVcpuRegister::V(n @ 0..=31) => { |
| let n = std::mem::size_of::<u128>() * (n as usize); |
| |
| user_fpsimd_state_reg(KVM_REG_SIZE_U128, offset_of!(user_fpsimd_state, vregs) + n) |
| } |
| KvmVcpuRegister::V(n) => unreachable!("invalid KvmVcpuRegister Vn index: {n}"), |
| KvmVcpuRegister::System(aarch64_sys_reg::FPSR) => { |
| user_fpsimd_state_reg(KVM_REG_SIZE_U32, offset_of!(user_fpsimd_state, fpsr)) |
| } |
| KvmVcpuRegister::System(aarch64_sys_reg::FPCR) => { |
| user_fpsimd_state_reg(KVM_REG_SIZE_U32, offset_of!(user_fpsimd_state, fpcr)) |
| } |
| KvmVcpuRegister::System(aarch64_sys_reg::SPSR_EL1) => spsr_reg(KVM_SPSR_EL1), |
| KvmVcpuRegister::System(aarch64_sys_reg::SPSR_abt) => spsr_reg(KVM_SPSR_ABT), |
| KvmVcpuRegister::System(aarch64_sys_reg::SPSR_und) => spsr_reg(KVM_SPSR_UND), |
| KvmVcpuRegister::System(aarch64_sys_reg::SPSR_irq) => spsr_reg(KVM_SPSR_IRQ), |
| KvmVcpuRegister::System(aarch64_sys_reg::SPSR_fiq) => spsr_reg(KVM_SPSR_FIQ), |
| KvmVcpuRegister::System(aarch64_sys_reg::SP_EL1) => { |
| kvm_reg(offset_of!(kvm_regs, sp_el1)) |
| } |
| KvmVcpuRegister::System(aarch64_sys_reg::ELR_EL1) => { |
| kvm_reg(offset_of!(kvm_regs, elr_el1)) |
| } |
| // The KVM API accidentally swapped CNTV_CVAL_EL0 and CNTVCT_EL0. |
| KvmVcpuRegister::System(aarch64_sys_reg::CNTV_CVAL_EL0) => reg_u64( |
| KVM_REG_ARM64_SYSREG.into(), |
| aarch64_sys_reg::CNTVCT_EL0.encoded().into(), |
| ), |
| KvmVcpuRegister::System(aarch64_sys_reg::CNTVCT_EL0) => reg_u64( |
| KVM_REG_ARM64_SYSREG.into(), |
| aarch64_sys_reg::CNTV_CVAL_EL0.encoded().into(), |
| ), |
| KvmVcpuRegister::System(sysreg) => { |
| reg_u64(KVM_REG_ARM64_SYSREG.into(), sysreg.encoded().into()) |
| } |
| KvmVcpuRegister::Firmware(n) => reg_u64(KVM_REG_ARM_FW.into(), n.into()), |
| KvmVcpuRegister::Ccsidr(n) => demux_reg(KVM_REG_SIZE_U32, 0, n.into()), |
| } |
| } |
| } |
| |
| impl VcpuAArch64 for KvmVcpu { |
| fn init(&self, features: &[VcpuFeature]) -> Result<()> { |
| let mut kvi = kvm_vcpu_init { |
| target: KVM_ARM_TARGET_GENERIC_V8, |
| features: [0; 7], |
| }; |
| // SAFETY: |
| // Safe because we allocated the struct and we know the kernel will write exactly the size |
| // of the struct. |
| let ret = unsafe { ioctl_with_mut_ref(&self.vm, KVM_ARM_PREFERRED_TARGET, &mut kvi) }; |
| if ret != 0 { |
| return errno_result(); |
| } |
| |
| kvi.features[0] = self.get_features_bitmap(features)?; |
| // SAFETY: |
| // Safe because we allocated the struct and we know the kernel will read exactly the size of |
| // the struct. |
| let ret = unsafe { ioctl_with_ref(self, KVM_ARM_VCPU_INIT, &kvi) }; |
| if ret != 0 { |
| return errno_result(); |
| } |
| |
| self.finalize(kvi.features[0])?; |
| Ok(()) |
| } |
| |
| fn init_pmu(&self, irq: u64) -> Result<()> { |
| let irq_addr = &irq as *const u64; |
| |
| // The in-kernel PMU virtualization is initialized by setting the irq |
| // with KVM_ARM_VCPU_PMU_V3_IRQ and then by KVM_ARM_VCPU_PMU_V3_INIT. |
| |
| let irq_attr = kvm_device_attr { |
| group: KVM_ARM_VCPU_PMU_V3_CTRL, |
| attr: KVM_ARM_VCPU_PMU_V3_IRQ as u64, |
| addr: irq_addr as u64, |
| flags: 0, |
| }; |
| // SAFETY: |
| // Safe because we allocated the struct and we know the kernel will read exactly the size of |
| // the struct. |
| let ret = unsafe { ioctl_with_ref(self, kvm_sys::KVM_HAS_DEVICE_ATTR, &irq_attr) }; |
| if ret < 0 { |
| return errno_result(); |
| } |
| |
| // SAFETY: |
| // Safe because we allocated the struct and we know the kernel will read exactly the size of |
| // the struct. |
| let ret = unsafe { ioctl_with_ref(self, kvm_sys::KVM_SET_DEVICE_ATTR, &irq_attr) }; |
| if ret < 0 { |
| return errno_result(); |
| } |
| |
| let init_attr = kvm_device_attr { |
| group: KVM_ARM_VCPU_PMU_V3_CTRL, |
| attr: KVM_ARM_VCPU_PMU_V3_INIT as u64, |
| addr: 0, |
| flags: 0, |
| }; |
| // SAFETY: |
| // Safe because we allocated the struct and we know the kernel will read exactly the size of |
| // the struct. |
| let ret = unsafe { ioctl_with_ref(self, kvm_sys::KVM_SET_DEVICE_ATTR, &init_attr) }; |
| if ret < 0 { |
| return errno_result(); |
| } |
| |
| Ok(()) |
| } |
| |
| fn has_pvtime_support(&self) -> bool { |
| // The in-kernel PV time structure is initialized by setting the base |
| // address with KVM_ARM_VCPU_PVTIME_IPA |
| let pvtime_attr = kvm_device_attr { |
| group: KVM_ARM_VCPU_PVTIME_CTRL, |
| attr: KVM_ARM_VCPU_PVTIME_IPA as u64, |
| addr: 0, |
| flags: 0, |
| }; |
| // SAFETY: |
| // Safe because we allocated the struct and we know the kernel will read exactly the size of |
| // the struct. |
| let ret = unsafe { ioctl_with_ref(self, kvm_sys::KVM_HAS_DEVICE_ATTR, &pvtime_attr) }; |
| ret >= 0 |
| } |
| |
| fn init_pvtime(&self, pvtime_ipa: u64) -> Result<()> { |
| let pvtime_ipa_addr = &pvtime_ipa as *const u64; |
| |
| // The in-kernel PV time structure is initialized by setting the base |
| // address with KVM_ARM_VCPU_PVTIME_IPA |
| let pvtime_attr = kvm_device_attr { |
| group: KVM_ARM_VCPU_PVTIME_CTRL, |
| attr: KVM_ARM_VCPU_PVTIME_IPA as u64, |
| addr: pvtime_ipa_addr as u64, |
| flags: 0, |
| }; |
| |
| // SAFETY: |
| // Safe because we allocated the struct and we know the kernel will read exactly the size of |
| // the struct. |
| let ret = unsafe { ioctl_with_ref(self, kvm_sys::KVM_SET_DEVICE_ATTR, &pvtime_attr) }; |
| if ret < 0 { |
| return errno_result(); |
| } |
| |
| Ok(()) |
| } |
| |
| fn set_one_reg(&self, reg_id: VcpuRegAArch64, data: u64) -> Result<()> { |
| let kvm_reg = self.kvm_reg_id(reg_id)?; |
| match kvm_reg.size() { |
| 4 => self.set_one_kvm_reg_u32(kvm_reg, data as u32), |
| 8 => self.set_one_kvm_reg_u64(kvm_reg, data), |
| size => panic!("bad reg size {size}"), |
| } |
| } |
| |
| fn get_one_reg(&self, reg_id: VcpuRegAArch64) -> Result<u64> { |
| let kvm_reg = self.kvm_reg_id(reg_id)?; |
| match kvm_reg.size() { |
| 4 => self.get_one_kvm_reg_u32(kvm_reg).map(u64::from), |
| 8 => self.get_one_kvm_reg_u64(kvm_reg), |
| size => panic!("bad reg size {size}"), |
| } |
| } |
| |
| fn set_vector_reg(&self, reg_num: u8, data: u128) -> Result<()> { |
| if reg_num > 31 { |
| return Err(Error::new(EINVAL)); |
| } |
| self.set_one_kvm_reg_u128(KvmVcpuRegister::V(reg_num), data) |
| } |
| |
| fn get_vector_reg(&self, reg_num: u8) -> Result<u128> { |
| if reg_num > 31 { |
| return Err(Error::new(EINVAL)); |
| } |
| self.get_one_kvm_reg_u128(KvmVcpuRegister::V(reg_num)) |
| } |
| |
| fn get_mpidr(&self) -> Result<u64> { |
| self.get_one_reg(VcpuRegAArch64::System(aarch64_sys_reg::MPIDR_EL1)) |
| } |
| |
| fn get_psci_version(&self) -> Result<PsciVersion> { |
| let version = if let Ok(v) = self.get_one_kvm_reg_u64(KvmVcpuRegister::PSCI_VERSION) { |
| let v = u32::try_from(v).map_err(|_| Error::new(EINVAL))?; |
| PsciVersion::try_from(v)? |
| } else { |
| // When `KVM_REG_ARM_PSCI_VERSION` is not supported, we can return PSCI 0.2, as vCPU |
| // has been initialized with `KVM_ARM_VCPU_PSCI_0_2` successfully. |
| PSCI_0_2 |
| }; |
| |
| if version < PSCI_0_2 { |
| // PSCI v0.1 isn't currently supported for guests |
| Err(Error::new(ENOTSUP)) |
| } else { |
| Ok(version) |
| } |
| } |
| |
| fn get_max_hw_bps(&self) -> Result<usize> { |
| // SAFETY: |
| // Safe because the kernel will only return the result of the ioctl. |
| let max_hw_bps = unsafe { |
| ioctl_with_val( |
| &self.vm, |
| KVM_CHECK_EXTENSION, |
| KVM_CAP_GUEST_DEBUG_HW_BPS.into(), |
| ) |
| }; |
| |
| if max_hw_bps < 0 { |
| errno_result() |
| } else { |
| Ok(max_hw_bps.try_into().expect("can't represent u64 as usize")) |
| } |
| } |
| |
| fn get_system_regs(&self) -> Result<BTreeMap<AArch64SysRegId, u64>> { |
| let reg_list = self.get_reg_list()?; |
| let cntvct_el0: u16 = aarch64_sys_reg::CNTVCT_EL0.encoded(); |
| let cntv_cval_el0: u16 = aarch64_sys_reg::CNTV_CVAL_EL0.encoded(); |
| let mut sys_regs = BTreeMap::new(); |
| for reg in reg_list { |
| if (reg as u32) & KVM_REG_ARM_COPROC_MASK == KVM_REG_ARM64_SYSREG { |
| let r = if reg as u16 == cntvct_el0 { |
| aarch64_sys_reg::CNTV_CVAL_EL0 |
| } else if reg as u16 == cntv_cval_el0 { |
| aarch64_sys_reg::CNTVCT_EL0 |
| } else { |
| AArch64SysRegId::from_encoded((reg & 0xFFFF) as u16) |
| }; |
| sys_regs.insert(r, self.get_one_reg(VcpuRegAArch64::System(r))?); |
| // The register representations are tricky. Double check they round trip correctly. |
| assert_eq!( |
| Ok(reg), |
| self.kvm_reg_id(VcpuRegAArch64::System(r)).map(u64::from), |
| ); |
| } |
| } |
| |
| // The list of system registers below do not follow the convention of system registers |
| // for bits 31-16. |
| // System registers bits 31-16 == 0x0013, but not for those registers. Add them |
| // explicitly |
| let extra_sys_regs = [ |
| aarch64_sys_reg::ELR_EL1, |
| aarch64_sys_reg::FPCR, |
| aarch64_sys_reg::FPSR, |
| aarch64_sys_reg::SP_EL1, |
| aarch64_sys_reg::SPSR_EL1, |
| aarch64_sys_reg::SPSR_abt, |
| aarch64_sys_reg::SPSR_und, |
| aarch64_sys_reg::SPSR_irq, |
| aarch64_sys_reg::SPSR_fiq, |
| ]; |
| for reg in extra_sys_regs { |
| sys_regs.insert(reg, self.get_one_reg(VcpuRegAArch64::System(reg))?); |
| } |
| Ok(sys_regs) |
| } |
| |
| fn get_cache_info(&self) -> Result<BTreeMap<u8, u64>> { |
| const KVM_REG_CCSIDR: u64 = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | (KVM_REG_ARM_DEMUX as u64); |
| const CCSIDR_INDEX_MASK: u64 = 0xFF; |
| let reg_list = self.get_reg_list()?; |
| let mut cache_info = BTreeMap::new(); |
| for reg in reg_list { |
| if (reg & !CCSIDR_INDEX_MASK) == KVM_REG_CCSIDR { |
| let idx = reg as u8; |
| cache_info.insert( |
| idx, |
| self.get_one_kvm_reg_u32(KvmVcpuRegister::Ccsidr(idx))? |
| .into(), |
| ); |
| } |
| } |
| Ok(cache_info) |
| } |
| |
| fn set_cache_info(&self, cache_info: BTreeMap<u8, u64>) -> Result<()> { |
| for (idx, val) in cache_info { |
| self.set_one_kvm_reg_u32( |
| KvmVcpuRegister::Ccsidr(idx), |
| val.try_into() |
| .expect("trying to set a u32 register with a u64 value"), |
| )?; |
| } |
| Ok(()) |
| } |
| |
| fn hypervisor_specific_snapshot(&self) -> anyhow::Result<AnySnapshot> { |
| let reg_list = self.get_reg_list()?; |
| let mut firmware_regs = BTreeMap::new(); |
| for reg in reg_list { |
| if (reg as u32) & KVM_REG_ARM_COPROC_MASK == KVM_REG_ARM_FW { |
| firmware_regs.insert( |
| reg as u16, |
| self.get_one_kvm_reg_u64(KvmVcpuRegister::Firmware(reg as u16))?, |
| ); |
| } |
| } |
| |
| AnySnapshot::to_any(KvmSnapshot { firmware_regs }) |
| .context("Failed to serialize KVM specific data") |
| } |
| |
| fn hypervisor_specific_restore(&self, data: AnySnapshot) -> anyhow::Result<()> { |
| let deser: KvmSnapshot = |
| AnySnapshot::from_any(data).context("Failed to deserialize KVM specific data")?; |
| // TODO: need to set firmware registers before "create_fdt" is called, earlier in the |
| // stack. |
| for (id, val) in &deser.firmware_regs { |
| self.set_one_kvm_reg_u64(KvmVcpuRegister::Firmware(*id), *val)?; |
| } |
| Ok(()) |
| } |
| |
| #[allow(clippy::unusual_byte_groupings)] |
| fn set_guest_debug(&self, addrs: &[GuestAddress], enable_singlestep: bool) -> Result<()> { |
| let mut dbg = kvm_guest_debug { |
| control: KVM_GUESTDBG_ENABLE, |
| ..Default::default() |
| }; |
| |
| if enable_singlestep { |
| dbg.control |= KVM_GUESTDBG_SINGLESTEP; |
| } |
| if !addrs.is_empty() { |
| dbg.control |= KVM_GUESTDBG_USE_HW; |
| } |
| |
| for (i, guest_addr) in addrs.iter().enumerate() { |
| // From the ARMv8 Architecture Reference Manual (DDI0487H.a) D31.3.{2,3}: |
| // When DBGBCR<n>_EL1.BT == 0b000x: |
| // DBGBVR<n>_EL1, Bits [1:0]: Reserved, RES0 |
| if guest_addr.0 & 0b11 != 0 { |
| return Err(Error::new(EINVAL)); |
| } |
| let sign_ext = 15; |
| // DBGBVR<n>_EL1.RESS[14:0], bits [63:49]: Reserved, Sign extended |
| dbg.arch.dbg_bvr[i] = (((guest_addr.0 << sign_ext) as i64) >> sign_ext) as u64; |
| // DBGBCR<n>_EL1.BT, bits [23:20]: Breakpoint Type |
| // 0b0000: Unlinked instruction address match. |
| // DBGBVR<n>_EL1 is the address of an instruction. |
| // DBGBCR<n>_EL1.BAS, bits [8:5]: Byte address select |
| // 0b1111: Use for A64 and A32 instructions |
| // DBGBCR<n>_EL1.PMC, bits [2:1]: Privilege mode control |
| // 0b11: EL1 & EL0 |
| // DBGBCR<n>_EL1.E, bit [0]: Enable breakpoint |
| // 0b1: Enabled |
| dbg.arch.dbg_bcr[i] = 0b1111_11_1; |
| } |
| |
| // SAFETY: |
| // Safe because the kernel won't read past the end of the kvm_guest_debug struct. |
| let ret = unsafe { ioctl_with_ref(self, KVM_SET_GUEST_DEBUG, &dbg) }; |
| if ret == 0 { |
| Ok(()) |
| } else { |
| errno_result() |
| } |
| } |
| } |
| |
| #[derive(Debug, Serialize, Deserialize)] |
| struct KvmSnapshot { |
| firmware_regs: BTreeMap<u16, u64>, |
| } |
| |
| // This function translates an IrqSrouceChip to the kvm u32 equivalent. It has a different |
| // implementation between x86_64 and aarch64 because the irqchip KVM constants are not defined on |
| // all architectures. |
| pub(super) fn chip_to_kvm_chip(chip: IrqSourceChip) -> u32 { |
| match chip { |
| // ARM does not have a constant for this, but the default routing |
| // setup seems to set this to 0 |
| IrqSourceChip::Gic => 0, |
| _ => { |
| error!("Invalid IrqChipSource for ARM {:?}", chip); |
| 0 |
| } |
| } |
| } |
| |
| #[cfg(test)] |
| mod tests { |
| use super::*; |
| |
| #[test] |
| fn system_timer_register_mixup() { |
| // Per https://docs.kernel.org/virt/kvm/api.html ARM64 system register encoding docs, |
| // KVM_REG_ARM_TIMER_CVAL and KVM_REG_ARM_TIMER_CNT were accidentally defined backwards. |
| // Ensure the AArch64SysRegId to KvmVcpuRegister encoding maps these to the expected |
| // values. |
| |
| const KVM_REG_ARM_TIMER_CVAL: u64 = 0x6030_0000_0013_DF02; |
| let cntv_cval_el0_kvm = KvmVcpuRegister::System(aarch64_sys_reg::CNTV_CVAL_EL0); |
| assert_eq!(u64::from(cntv_cval_el0_kvm), KVM_REG_ARM_TIMER_CVAL); |
| |
| const KVM_REG_ARM_TIMER_CNT: u64 = 0x6030_0000_0013_DF1A; |
| let cntvct_el0_kvm = KvmVcpuRegister::System(aarch64_sys_reg::CNTVCT_EL0); |
| assert_eq!(u64::from(cntvct_el0_kvm), KVM_REG_ARM_TIMER_CNT); |
| } |
| } |
