NIC queues and AF_XDP sockets

[luoffei]

Problem:

s2n-quic/quic/s2n-quic-qns/src/xdp.rs

Lines 111 to 195 in e7d53ef

	let max_queues = syscall::max_queues(&self.interface);
	let umem_size = (rx_queue_len + tx_queue_len) * max_queues;
	// create a UMEM
	let umem = umem::Builder {
	frame_count: umem_size,
	frame_size,
	..Default::default()
	}
	.build()?;
	// setup the address we're going to bind to
	let mut address = if_xdp::Address {
	flags: XdpFlags::USE_NEED_WAKEUP,
	..Default::default()
	};
	address.set_if_name(&CString::new(self.interface.clone())?)?;
	let mut shared_umem_fd = None;
	let mut tx_channels = vec![];
	let mut rx_channels = vec![];
	let mut rx_fds = vec![];
	let mut desc = umem.frames();
	// iterate over all of the queues and create sockets for each one
	for queue_id in 0..max_queues {
	let socket = socket::Fd::open()?;
	// if we've already attached a socket to the UMEM, then reuse the first FD
	if let Some(fd) = shared_umem_fd {
	address.set_shared_umem(&fd);
	} else {
	socket.attach_umem(&umem)?;
	shared_umem_fd = Some(socket.as_raw_fd());
	}
	// set the queue id to the current value
	address.queue_id = queue_id;
	// file descriptors can only be added once so wrap it in an Arc
	let async_fd = Arc::new(AsyncFd::new(socket.clone())?);
	// get the offsets for each of the rings
	let offsets = syscall::offsets(&socket)?;
	{
	// create a pair of rings for receiving packets
	let mut fill = ring::Fill::new(socket.clone(), &offsets, fill_ring_len)?;
	let rx = ring::Rx::new(socket.clone(), &offsets, rx_queue_len)?;
	// remember the FD so we can add it to the XSK map later
	rx_fds.push((queue_id, socket.clone()));
	// put descriptors in the Fill queue
	fill.init((&mut desc).take(rx_queue_len as _));
	let cooldown = Cooldown::new(self.rx_cooldown);
	rx_channels.push(xdp_io::rx::Channel {
	rx,
	fill,
	driver: async_fd.clone().with_cooldown(cooldown),
	});
	};
	{
	// create a pair of rings for transmitting packets
	let mut completion =
	ring::Completion::new(socket.clone(), &offsets, completion_ring_len)?;
	let tx = ring::Tx::new(socket.clone(), &offsets, tx_queue_len)?;
	// put descriptors in the completion queue
	completion.init((&mut desc).take(tx_queue_len as _));
	tx_channels.push(xdp_io::tx::Channel {
	tx,
	completion,
	driver: xdp_io::tx::BusyPoll,
	});
	};
	// finally bind the socket to the configured address
	syscall::bind(&socket, &mut address)?;
	}

s2n-quic/tools/xdp/s2n-quic-xdp/src/syscall.rs

Lines 309 to 312 in e7d53ef

	let queues = channels
	.max_rx
	.max(channels.max_tx)
	.max(channels.max_combined);

The CPU has 10 cores and 20 threads, and the maximum number of queues of the network card is 32. When binding the 21st AF_XDP socket, the prompt Os { code: 22, kind: InvalidInput, message: "Invalid argument" }

Solution:

let queues = channels
        .rx_count
        .max(channels.tx_count)
        .max(channels.combined_count);