From c211e7a4d0a1a3a0deb1879e7b1ce52b570f60a8 Mon Sep 17 00:00:00 2001 From: Harshavardhan Unnibhavi Date: Thu, 9 Sep 2021 23:55:41 +0200 Subject: [PATCH] Inflight I/O: Implement missing traits This commit implements the get and set inflight fd members of the VhostUserSlaveReqHandlerMut trait, which is used to pass inflight I/O queue tracking regions as memfds. Fixes #14. Signed-off-by: Harshavardhan Unnibhavi --- src/handler.rs | 345 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 334 insertions(+), 11 deletions(-) diff --git a/src/handler.rs b/src/handler.rs index e7f84b5..69e376e 100644 --- a/src/handler.rs +++ b/src/handler.rs @@ -6,12 +6,17 @@ use std::error; use std::fs::File; use std::io; -use std::os::unix::io::AsRawFd; +use std::mem; +use std::os::unix::io::RawFd; +use std::os::unix::io::{AsRawFd, FromRawFd}; use std::sync::Arc; use std::thread; +use libc::c_void; + use vhost::vhost_user::message::{ - VhostUserConfigFlags, VhostUserMemoryRegion, VhostUserProtocolFeatures, + DescStatePacked, DescStateSplit, QueueRegionPacked, QueueRegionSplit, VhostUserConfigFlags, + VhostUserInflight, VhostUserMemoryRegion, VhostUserProtocolFeatures, VhostUserSingleMemoryRegion, VhostUserVirtioFeatures, VhostUserVringAddrFlags, VhostUserVringState, }; @@ -19,11 +24,13 @@ use vhost::vhost_user::{ Error as VhostUserError, Result as VhostUserResult, SlaveFsCacheReq, VhostUserSlaveReqHandlerMut, }; +use virtio_bindings::bindings::virtio_net::VIRTIO_F_RING_PACKED; use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX; use vm_memory::bitmap::Bitmap; use vm_memory::mmap::NewBitmap; use vm_memory::{ - FileOffset, GuestAddress, GuestAddressSpace, GuestMemoryMmap, GuestRegionMmap, MmapRegion, + Address, FileOffset, GuestAddress, GuestAddressSpace, GuestMemoryMmap, GuestRegionMmap, + MmapRegion, }; use vmm_sys_util::epoll::EventSet; @@ -71,6 +78,41 @@ struct AddrMapping { gpa_base: u64, } +struct InflightFdState { + inflight_file: Option, + inflight_mapping_addr: Option, + inflight_mmap_size: usize, +} + +impl InflightFdState { + fn new() -> Self { + InflightFdState { + inflight_file: None, + inflight_mapping_addr: None, + inflight_mmap_size: 0, + } + } + + fn set_inflight_state( + &mut self, + inflight_file: Option, + inflight_mapping_addr: Option, + inflight_mmap_size: usize, + ) { + self.inflight_file = inflight_file; + self.inflight_mapping_addr = inflight_mapping_addr; + self.inflight_mmap_size = inflight_mmap_size; + } + + fn get_inflight_mapping_addr(&self) -> Option { + self.inflight_mapping_addr + } + + fn get_inflight_mmap_size(&self) -> usize { + self.inflight_mmap_size + } +} + pub struct VhostUserHandler where S: VhostUserBackend, @@ -90,6 +132,7 @@ where atomic_mem: GM, vrings: Vec, worker_threads: Vec>>, + inflight_state: InflightFdState, } impl VhostUserHandler @@ -147,6 +190,7 @@ where atomic_mem, vrings, worker_threads, + inflight_state: InflightFdState::new(), }) } } @@ -564,20 +608,206 @@ where fn get_inflight_fd( &mut self, - _inflight: &vhost::vhost_user::message::VhostUserInflight, + inflight: &vhost::vhost_user::message::VhostUserInflight, ) -> VhostUserResult<(vhost::vhost_user::message::VhostUserInflight, File)> { - // Assume the backend hasn't negotiated the inflight feature; it - // wouldn't be correct for the backend to do so, as we don't (yet) - // provide a way for it to handle such requests. - Err(VhostUserError::InvalidOperation) + let ret_val = -1; + // Total size of the inflight queue region + let mut total_mmap_size = + self.get_inflight_queue_size(inflight.queue_size) * inflight.num_queues as usize; + + // Create a memfd region to hold the queues for inflight I/O tracking + let (mmap_ptr, inflight_fd, dup_inflight_fd) = self.memfd_alloc(total_mmap_size)?; + + if mmap_ptr == ret_val as *mut c_void { + self.inflight_state.set_inflight_state(None, None, 0); + total_mmap_size = 0; + } else { + unsafe { libc::memset(mmap_ptr, 0, total_mmap_size) }; + self.inflight_state.set_inflight_state( + Some(unsafe { File::from_raw_fd(inflight_fd as i32) }), + Some(GuestAddress::new(mmap_ptr as u64)), + total_mmap_size, + ) + } + + Ok(( + VhostUserInflight { + mmap_size: total_mmap_size as u64, + mmap_offset: 0, + num_queues: inflight.num_queues, + queue_size: inflight.queue_size, + }, + unsafe { File::from_raw_fd(dup_inflight_fd as RawFd) }, + )) } fn set_inflight_fd( &mut self, - _inflight: &vhost::vhost_user::message::VhostUserInflight, - _file: File, + inflight: &vhost::vhost_user::message::VhostUserInflight, + file: File, ) -> VhostUserResult<()> { - Err(VhostUserError::InvalidOperation) + let ret_val = -1; + + // Need to unmap any previously mmaped regions as closing the + // associated file doesn't unmap it automatically + if let Some(inflight_addr) = self.inflight_state.get_inflight_mapping_addr() { + unsafe { + libc::munmap( + inflight_addr.raw_value() as *mut c_void, + self.inflight_state.get_inflight_mmap_size(), + ) + }; + } + + let mmap_size = inflight.mmap_size; + let mmap_offset = inflight.mmap_offset; + + let mmap_ptr = unsafe { + libc::mmap( + std::ptr::null_mut::(), + mmap_size as usize, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + file.as_raw_fd(), + mmap_offset as i64, + ) + }; + + if mmap_ptr == ret_val as *mut c_void { + self.inflight_state.set_inflight_state(None, None, 0); + } else { + self.inflight_state.set_inflight_state( + Some(file), + Some(GuestAddress::new(mmap_ptr as u64)), + mmap_size as usize, + ); + } + + self.set_inflight_region_desc_num(inflight.num_queues, inflight.queue_size); + + Ok(()) + } +} + +impl VhostUserHandler +where + S: VhostUserBackend, + V: VringT>, + B: NewBitmap + Clone, +{ + fn get_inflight_queue_size(&mut self, queue_size: u16) -> usize { + let queue_region_size; + let descr_state_size; + let virtio_features = self.get_features().unwrap(); + + if virtio_features & (1 << VIRTIO_F_RING_PACKED) == 0 { + // Use descriptor and queue states for split virtqueues + queue_region_size = mem::size_of::(); + descr_state_size = mem::size_of::(); + } else { + // Use descriptor and queue states for packed virtqueues + queue_region_size = mem::size_of::(); + descr_state_size = mem::size_of::(); + } + queue_region_size + descr_state_size * queue_size as usize + } + + fn memfd_alloc(&self, mmap_size: usize) -> VhostUserResult<(*mut c_void, i64, i64)> { + let mut ret_val; + let inflight_file; + let dup_inflight_file; + + ret_val = unsafe { + libc::syscall( + libc::SYS_memfd_create, + &std::ffi::CString::new("inflight-region").unwrap(), + libc::MFD_ALLOW_SEALING, + ) + }; + + if ret_val == -1 { + return Err(VhostUserError::MemFdCreateError); + } + + inflight_file = ret_val; + + ret_val = unsafe { libc::ftruncate(inflight_file as RawFd, mmap_size as i64) } as i64; + + if ret_val == -1 { + return Err(VhostUserError::FileTrucateError); + } + + ret_val = unsafe { + libc::fcntl( + inflight_file as RawFd, + libc::F_ADD_SEALS, + libc::F_SEAL_GROW | libc::F_SEAL_SHRINK | libc::F_SEAL_SEAL, + ) + } as i64; + + if ret_val == -1 { + return Err(VhostUserError::MemFdSealError); + } + + ret_val = unsafe { libc::dup(inflight_file as RawFd).into() }; + + if ret_val == -1 { + return Err(VhostUserError::MemFdCreateError); + } + + dup_inflight_file = ret_val; + + Ok(( + unsafe { + libc::mmap( + std::ptr::null_mut(), + mmap_size, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + inflight_file as RawFd, + 0, + ) + }, + inflight_file, + dup_inflight_file, + )) + } + + fn set_desc_num_packed(&mut self, inflight_region: u64, num_queues: u16, queue_size: u16) { + let raw_ptr = inflight_region as *mut QueueRegionPacked; + + for i in 0..num_queues { + unsafe { + let queue_region = raw_ptr.offset(i as isize).as_mut().unwrap(); + queue_region.desc_num = queue_size; + } + } + } + + fn set_desc_num_split(&mut self, inflight_region: u64, num_queues: u16, queue_size: u16) { + let raw_ptr = inflight_region as *mut QueueRegionSplit; + + for i in 0..num_queues { + unsafe { + let queue_region = raw_ptr.offset(i as isize).as_mut().unwrap(); + queue_region.desc_num = queue_size; + } + } + } + + fn set_inflight_region_desc_num(&mut self, num_queues: u16, queue_size: u16) { + let inflight_region = self + .inflight_state + .get_inflight_mapping_addr() + .unwrap() + .raw_value(); + + let virtio_features = self.get_features().unwrap(); + + match virtio_features & (1 << VIRTIO_F_RING_PACKED) { + 0 => self.set_desc_num_split(inflight_region, num_queues, queue_size), + _ => self.set_desc_num_packed(inflight_region, num_queues, queue_size), + }; } } @@ -598,3 +828,96 @@ where } } } + +#[cfg(test)] +pub mod tests { + use super::*; + use crate::VringRwLock; + use std::result; + use vm_memory::GuestMemoryAtomic; + use vmm_sys_util::eventfd::EventFd; + + #[derive(Clone)] + pub struct TestVhostBackend { + packed: bool, + } + + impl TestVhostBackend { + fn new(packed: bool) -> Self { + TestVhostBackend { packed } + } + } + + impl VhostUserBackend for TestVhostBackend { + fn num_queues(&self) -> usize { + 2 + } + + fn max_queue_size(&self) -> usize { + 256 + } + + fn features(&self) -> u64 { + if !self.packed { + println!("Packed bitch"); + return 0xffff_ffff_ffff_ffff & (!(1 << VIRTIO_F_RING_PACKED)); + } + return 0xffff_ffff_ffff_ffff; + } + + fn protocol_features(&self) -> VhostUserProtocolFeatures { + VhostUserProtocolFeatures::all() + } + + fn set_event_idx(&self, _enabled: bool) {} + + fn update_memory( + &self, + _mem: GuestMemoryAtomic, + ) -> result::Result<(), io::Error> { + Ok(()) + } + + fn exit_event(&self, _thread_index: usize) -> Option { + let event_fd = EventFd::new(0).unwrap(); + + Some(event_fd) + } + + fn handle_event( + &self, + _device_event: u16, + _evset: EventSet, + _vrings: &[VringRwLock], + _thread_id: usize, + ) -> result::Result { + Ok(true) + } + } + + #[test] + fn test_get_inflight_queue_size() { + let mut vhost_user_handler_packed = VhostUserHandler::new( + TestVhostBackend::new(true), + GuestMemoryAtomic::new( + GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0x100000), 0x10000)]).unwrap(), + ), + ) + .unwrap(); + let packed_queue_size = vhost_user_handler_packed + .get_inflight_queue_size(vhost_user_handler_packed.backend.max_queue_size() as u16); + + assert_eq!(packed_queue_size, 8229); + + let mut vhost_user_handler_split = VhostUserHandler::new( + TestVhostBackend::new(false), + GuestMemoryAtomic::new( + GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0x100000), 0x10000)]).unwrap(), + ), + ) + .unwrap(); + let split_queue_size = vhost_user_handler_split + .get_inflight_queue_size(vhost_user_handler_split.backend.max_queue_size() as u16); + assert_eq!(split_queue_size, 4120); + } +}