Skip to content

Commit

Permalink
issue: 4219010 XLIO support for kernel 6.10
Browse files Browse the repository at this point in the history 
Kernel 6.10 netlink has breaked XLIO functionality.
Transitioned to libnl - an abstraction that wraps netlink.

This both solves the issue and makes us more robust.

Signed-off-by: Tomer Cabouly <[email protected]>
  • Loading branch information
@tomerdbz
tomerdbz committed Jan 5, 2025
1 parent ea2d6e2 commit 224e851
Show file tree
Hide file tree
Showing 7 changed files with 188 additions and 286 deletions.
173 changes: 36 additions & 137 deletions src/core/proto/netlink_socket_mgr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,173 +32,72 @@
* SOFTWARE.
*/

#include "core/sock/sock-redirect.h"
#include "core/util/utils.h"
#include "vlogger/vlogger.h"
#include "utils/bullseye.h"
#include "netlink_socket_mgr.h"

#include <linux/rtnetlink.h>
#include <linux/netlink.h>
#include <stdlib.h>
#include <unistd.h> // getpid()

#include <netlink/route/route.h>
#include <netlink/route/rule.h>
#include <netlink/route/link.h>

#ifndef MODULE_NAME
#define MODULE_NAME "netlink_socket_mgr:"
#endif

#define MSG_BUFF_SIZE 81920

// This function builds Netlink request to retrieve data (Rule, Route) from kernel.
// Parameters :
// data_type : either RULE_DATA_TYPE or ROUTE_DATA_TYPE
// pid : opaque pid for netlink request
// seq : opaque seq for netlink request
// buf : buffer for the request
// nl_msg : [out] pointer to request
void netlink_socket_mgr::build_request(nl_data_t data_type, uint32_t pid, uint32_t seq, char *buf,
struct nlmsghdr **nl_msg)
{
struct rtmsg *rt_msg;

assert(MSG_BUFF_SIZE >= NLMSG_SPACE(sizeof(struct rtmsg)));
memset(buf, 0, NLMSG_SPACE(sizeof(struct rtmsg)));

// point the header and the msg structure pointers into the buffer
*nl_msg = (struct nlmsghdr *)buf;
rt_msg = (struct rtmsg *)NLMSG_DATA(*nl_msg);

// Fill in the nlmsg header
(*nl_msg)->nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg));
(*nl_msg)->nlmsg_seq = seq;
(*nl_msg)->nlmsg_pid = pid;
(*nl_msg)->nlmsg_type = data_type == RULE_DATA_TYPE ? RTM_GETRULE : RTM_GETROUTE;
(*nl_msg)->nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST;

rt_msg->rtm_family = AF_UNSPEC;
}

// Query built request and receive requested data (Rule, Route)
// Parameters:
// nl_msg : request that is built previously.
// buf : [out] buffer for the reply
// len : [out] length of received data.
bool netlink_socket_mgr::query(const struct nlmsghdr *nl_msg, char *buf, int &len)
// Update data in a table
void netlink_socket_mgr::update_tbl(nl_data_t data_type)
{
int sockfd;

// Opaque information in the request. To track expected reply.
uint32_t nl_pid = nl_msg->nlmsg_pid;
uint32_t nl_seq = nl_msg->nlmsg_seq;
nl_sock *sockfd = nullptr;

BULLSEYE_EXCLUDE_BLOCK_START
if ((sockfd = SYSCALL(socket, PF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE)) < 0) {
__log_err("NL socket creation failed, errno = %d", errno);
return false;
}
if (SYSCALL(fcntl, sockfd, F_SETFD, FD_CLOEXEC) != 0) {
__log_warn("Fail in fcntl, errno = %d", errno);
}
if ((len = SYSCALL(send, sockfd, nl_msg, nl_msg->nlmsg_len, 0)) < 0) {
__log_err("Write to NL socket failed, errno = %d", errno);
}
if (len > 0 && (len = recv_info(sockfd, nl_pid, nl_seq, buf)) < 0) {
__log_err("Read from NL socket failed...");
}
BULLSEYE_EXCLUDE_BLOCK_END

close(sockfd);
return len > 0;
}
sockfd = nl_socket_alloc();
if (sockfd == nullptr) {
__log_err("NL socket Creation: ");
throw_xlio_exception("Failed nl_socket_alloc");
}

// Receive requested data and save it to buffer.
// Return length of received data.
// Parameters:
// sockfd : netlink socket
// pid : expected opaque pid value
// seq : expected opaque seq value
// buf : [out] read reply
int netlink_socket_mgr::recv_info(int sockfd, uint32_t pid, uint32_t seq, char *buf)
{
struct nlmsghdr *nlHdr;
int readLen;
int msgLen = 0;
char *buf_ptr = buf;

do {
// Receive response from the kernel
BULLSEYE_EXCLUDE_BLOCK_START
if ((readLen = SYSCALL(recv, sockfd, buf_ptr, MSG_BUFF_SIZE - msgLen, 0)) < 0) {
__log_err("NL socket read failed, errno = %d", errno);
return -1;
}

nlHdr = (struct nlmsghdr *)buf_ptr;

// Check if the header is valid
if ((NLMSG_OK(nlHdr, (u_int)readLen) == 0) || (nlHdr->nlmsg_type == NLMSG_ERROR)) {
__log_err("Error in received packet, readLen = %d, msgLen = %d, type=%d, bufLen = %d",
readLen, nlHdr->nlmsg_len, nlHdr->nlmsg_type, MSG_BUFF_SIZE);
if ((int)nlHdr->nlmsg_len >= MSG_BUFF_SIZE - msgLen) {
__log_err("The buffer we pass to netlink is too small for reading the whole table");
}
return -1;
}
BULLSEYE_EXCLUDE_BLOCK_END

if ((nlHdr->nlmsg_seq != seq) || (nlHdr->nlmsg_pid != pid)) {
// Skip not expected messages
continue;
}

buf_ptr += readLen;
msgLen += readLen;

// Loop until this is the last message of expected reply
} while (nlHdr->nlmsg_type != NLMSG_DONE && (nlHdr->nlmsg_flags & NLM_F_MULTI));

return msgLen;
}
if (nl_connect(sockfd, NETLINK_ROUTE) < 0) {
__log_err("NL socket Connection: ");
nl_socket_free(sockfd);
throw_xlio_exception("Failed nl_connect");
}

// Update data in a table
void netlink_socket_mgr::update_tbl(nl_data_t data_type)
{
struct nlmsghdr *nl_msg = nullptr;
char *buf;
int len = 0;
struct nl_cache *cache_state = {0};
int err = 0;

// Opaque netlink information
uint32_t nl_pid = getpid();
uint32_t nl_seq = static_cast<uint32_t>(data_type);
// cache allocation fetches the latest existing rules/routes
if (data_type == RULE_DATA_TYPE) {

__log_dbg("");
err = rtnl_rule_alloc_cache(sockfd, AF_INET, &cache_state);
} else if (data_type == ROUTE_DATA_TYPE) {

buf = new char[MSG_BUFF_SIZE];
if (!buf) {
__log_err("NL message buffer allocation failed");
return;
err = rtnl_route_alloc_cache(sockfd, AF_INET, 0, &cache_state);
}

build_request(data_type, nl_pid, nl_seq, buf, &nl_msg);
if (query(nl_msg, buf, len)) {
parse_tbl(buf, len);
if (err < 0) {
throw_xlio_exception("Failed to allocate route cache");
}

delete[] buf;
__log_dbg("Done");
parse_tbl(cache_state);
}

// Parse received data in a table
// Parameters:
// buf : buffer with netlink reply.
// len : length of received data.
void netlink_socket_mgr::parse_tbl(char *buf, int len)
void netlink_socket_mgr::parse_tbl(struct nl_cache *cache_state)
{
struct nlmsghdr *nl_header = (struct nlmsghdr *)buf;

for (; NLMSG_OK(nl_header, (u_int)len); nl_header = NLMSG_NEXT(nl_header, len)) {
parse_entry(nl_header);
}
// a lambda can't be casted to a c-fptr with ref captures - so we provide context ourselves
nl_cache_foreach(
cache_state,
[](struct nl_object *nl_obj, void *context) {
netlink_socket_mgr *this_ptr = reinterpret_cast<netlink_socket_mgr *>(context);
this_ptr->parse_entry(nl_obj);
},
this);
}

#undef MODULE_NAME
13 changes: 3 additions & 10 deletions src/core/proto/netlink_socket_mgr.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,25 +34,18 @@

#ifndef NETLINK_SOCKET_MGR_H
#define NETLINK_SOCKET_MGR_H

// Forward declarations
struct nlmsghdr;

#include <netlink/netlink.h>
// This enum specify the type of data to be retrieve using netlink socket.
enum nl_data_t { RULE_DATA_TYPE, ROUTE_DATA_TYPE };

// This class manages retrieving data (Rule, Route) from kernel using netlink socket.
class netlink_socket_mgr {
protected:
virtual void parse_entry(struct nlmsghdr *nl_header) = 0;
virtual void parse_entry(struct nl_object *nl_obj) = 0;
virtual void update_tbl(nl_data_t data_type);

private:
void build_request(nl_data_t data_type, uint32_t pid, uint32_t seq, char *buf,
struct nlmsghdr **nl_msg);
bool query(const struct nlmsghdr *nl_msg, char *buf, int &len);
int recv_info(int sockfd, uint32_t pid, uint32_t seq, char *buf);
void parse_tbl(char *buf, int len);
void parse_tbl(struct nl_cache *cache_state);
};

#endif /* NETLINK_SOCKET_MGR_H */
Loading

0 comments on commit 224e851

Please sign in to comment.