Support KNI ratelimit.

This commit is contained in:
fengbojiang 2024-10-10 17:48:40 +08:00
parent 5c144b199a
commit f069dcdcb7
6 changed files with 150 additions and 21 deletions

View File

@ -219,6 +219,14 @@ gateway=192.168.1.1
# The format is same as port_list
#tcp_port=80,443
#udp_port=53
# KNI ratelimit value, default: 0, means disable ratelimit.
# example:
# The total speed limit for a single process entering the kni ring is 10,000 QPS,
# 1000 QPS for general packets, 9000 QPS for console packets (ospf/arp, etc.)
# The total speed limit for kni forwarding to the kernel is 20,000 QPS.
#console_packets_ratelimit=0
#general_packets_ratelimit=0
#kernel_packets_ratelimit=0
# FreeBSD network performance tuning configurations.
# Most native FreeBSD configurations are supported.

View File

@ -909,6 +909,12 @@ ini_parse_handler(void* user, const char* section, const char* name,
pconfig->kni.enable= atoi(value);
} else if (MATCH("kni", "type")) {
pconfig->kni.type= atoi(value);
} else if (MATCH("kni", "console_packets_ratelimit")) {
pconfig->kni.console_packets_ratelimit= atoi(value);
} else if (MATCH("kni", "general_packets_ratelimit")) {
pconfig->kni.general_packets_ratelimit= atoi(value);
} else if (MATCH("kni", "kernel_packets_ratelimit")) {
pconfig->kni.kernel_packets_ratelimit= atoi(value);
} else if (MATCH("kni", "kni_action")) {
pconfig->kni.kni_action= strdup(value);
} else if (MATCH("kni", "method")) {
@ -1247,6 +1253,11 @@ ff_default_config(struct ff_config *cfg)
cfg->dpdk.promiscuous = 1;
cfg->dpdk.pkt_tx_delay = BURST_TX_DRAIN_US;
/* KNI ratelimit default disabled */
//cfg->kni.console_packets_ratelimit = KNI_RATELIMT_CONSOLE;
//cfg->kni.general_packets_ratelimit = KNI_RATELIMT_GENERAL;
//cfg->kni.kernel_packets_ratelimit = KNI_RATELIMT_KERNEL;
cfg->freebsd.hz = 100;
cfg->freebsd.physmem = 1048576*256;
cfg->freebsd.fd_reserve = 0;

View File

@ -38,6 +38,16 @@ extern "C" {
#define DPDK_MAX_VLAN_FILTER 128
#define PCAP_SNAP_MINLEN 94
#define PCAP_SAVE_MINLEN (2<<22)
/*
* KNI ratelimit default value.
* The total speed limit for a single process entering the kni ring is 10,000 QPS,
* 1000 QPS for general packets, 9000 QPS for console packets (ospf/arp, etc.)
* The total speed limit for kni forwarding to the kernel is 20,000 QPS.
*/
#define KNI_RATELIMT_PROCESS (10000)
#define KNI_RATELIMT_GENERAL (1000)
#define KNI_RATELIMT_CONSOLE (KNI_RATELIMT_PROCESS - KNI_RATELIMT_GENERAL)
#define KNI_RATELIMT_KERNEL (KNI_RATELIMT_PROCESS * 2)
extern int dpdk_argc;
extern char *dpdk_argv[DPDK_CONFIG_NUM + 1];
@ -241,6 +251,9 @@ struct ff_config {
struct {
int enable;
int type;
int console_packets_ratelimit;
int general_packets_ratelimit;
int kernel_packets_ratelimit;
char *kni_action;
char *method;
char *tcp_port;

View File

@ -1327,8 +1327,14 @@ protocol_filter(const void *data, uint16_t len)
len -= sizeof(struct rte_vlan_hdr);
}
if(ether_type == RTE_ETHER_TYPE_ARP)
if(ether_type == RTE_ETHER_TYPE_ARP) {
return FILTER_ARP;
}
/* Multicast protocol, such as stp(used by zebra), is forwarded to kni and has a separate speed limit */
if (rte_is_multicast_ether_addr(&hdr->dst_addr)) {
return FILTER_MULTI;
}
#if (!defined(__FreeBSD__) && defined(INET6) ) || \
( defined(__FreeBSD__) && defined(INET6) && defined(FF_KNI))
@ -1520,7 +1526,7 @@ process_packets(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **bufs,
mbuf_clone = pktmbuf_deep_clone(rtem, mbuf_pool);
if(mbuf_clone) {
ff_add_vlan_tag(mbuf_clone);
ff_kni_enqueue(port_id, mbuf_clone);
ff_kni_enqueue(filter, port_id, mbuf_clone);
}
}
#endif
@ -1529,15 +1535,15 @@ process_packets(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **bufs,
} else if (enable_kni) {
if (knictl_action == FF_KNICTL_ACTION_ALL_TO_KNI){
ff_add_vlan_tag(rtem);
ff_kni_enqueue(port_id, rtem);
ff_kni_enqueue(filter, port_id, rtem);
} else if (knictl_action == FF_KNICTL_ACTION_ALL_TO_FF){
ff_veth_input(ctx, rtem);
} else if (knictl_action == FF_KNICTL_ACTION_DEFAULT){
if (enable_kni &&
((filter == FILTER_KNI && kni_accept) ||
(filter == FILTER_UNKNOWN && !kni_accept)) ) {
((filter == FILTER_UNKNOWN || filter >= FILTER_OSPF) && !kni_accept)) ) {
ff_add_vlan_tag(rtem);
ff_kni_enqueue(port_id, rtem);
ff_kni_enqueue(filter, port_id, rtem);
} else {
ff_veth_input(ctx, rtem);
}
@ -2056,6 +2062,34 @@ main_loop(void *arg)
cur_tsc = rte_rdtsc();
if (unlikely(freebsd_clock.expire < cur_tsc)) {
rte_timer_manage();
#ifdef FF_KNI
/* reset kni ratelimt */
if (enable_kni &&
(ff_global_cfg.kni.console_packets_ratelimit ||
ff_global_cfg.kni.general_packets_ratelimit ||
ff_global_cfg.kni.kernel_packets_ratelimit)) {
static time_t last_sec = 0;
time_t sec;
long nsec;
ff_get_current_time(&sec, &nsec);
if (sec > last_sec) {
if (kni_rate_limt.gerneal_packets > ff_global_cfg.kni.general_packets_ratelimit ||
kni_rate_limt.console_packets > ff_global_cfg.kni.console_packets_ratelimit ||
kni_rate_limt.kernel_packets > ff_global_cfg.kni.kernel_packets_ratelimit) {
printf("kni ratelimit, general:%lu/%d, console:%lu/%d, kernel:%lu/%d, last sec:%ld, sec:%ld\n",
kni_rate_limt.gerneal_packets, ff_global_cfg.kni.general_packets_ratelimit,
kni_rate_limt.console_packets, ff_global_cfg.kni.console_packets_ratelimit,
kni_rate_limt.kernel_packets, ff_global_cfg.kni.kernel_packets_ratelimit, last_sec, sec);
}
last_sec = sec;
kni_rate_limt.gerneal_packets = 0;
kni_rate_limt.console_packets = 0;
kni_rate_limt.kernel_packets = 0;
}
}
#endif
}
idle = 1;

View File

@ -44,6 +44,10 @@
#include "ff_dpdk_kni.h"
#include "ff_config.h"
#ifndef IPPROTO_OSPFIGP
#define IPPROTO_OSPFIGP 89 /**< OSPFIGP */
#endif
/* Callback for request of changing MTU */
/* Total octets in ethernet header */
#define KNI_ENET_HEADER_SIZE 14
@ -92,6 +96,8 @@ struct kni_interface_stats {
struct rte_ring **kni_rp;
struct kni_interface_stats **kni_stat;
struct kni_ratelimit kni_rate_limt = {0, 0, 0};
static void
set_bitmap(uint16_t port, unsigned char *bitmap)
{
@ -219,9 +225,24 @@ kni_process_tx(uint16_t port_id, uint16_t queue_id,
struct rte_mbuf **pkts_burst, unsigned count)
{
/* read packet from kni ring(phy port) and transmit to kni */
uint16_t nb_tx, nb_kni_tx = 0;
uint16_t nb_tx, nb_to_tx, nb_kni_tx;
nb_tx = rte_ring_dequeue_burst(kni_rp[port_id], (void **)pkts_burst, count, NULL);
/*
* The total ratelimit forwarded to the kernel, may a few more packets being sent, but it doesnt matter,
* If there are too many processes, there is also the possibility that the control packet will be ratelimited.
*/
if (ff_global_cfg.kni.kernel_packets_ratelimit) {
if (likely(kni_rate_limt.kernel_packets < ff_global_cfg.kni.kernel_packets_ratelimit)) {
nb_to_tx = nb_tx;
} else {
nb_to_tx = 0;
}
kni_rate_limt.kernel_packets += nb_tx;
} else {
nb_to_tx = nb_tx;
}
#ifdef FF_KNI_KNI
if (ff_global_cfg.kni.type == KNI_TYPE_KNI) {
/* NB.
@ -229,14 +250,13 @@ kni_process_tx(uint16_t port_id, uint16_t queue_id,
* must Call regularly rte_kni_tx_burst(kni, NULL, 0).
* detail https://embedded.communities.intel.com/thread/6668
*/
nb_kni_tx = rte_kni_tx_burst(kni_stat[port_id]->kni, pkts_burst, nb_tx);
nb_kni_tx = rte_kni_tx_burst(kni_stat[port_id]->kni, pkts_burst, nb_to_tx);
rte_kni_handle_request(kni_stat[port_id]->kni);
} else if (ff_global_cfg.kni.type == KNI_TYPE_VIRTIO)
#endif
{
nb_kni_tx = rte_eth_tx_burst(kni_stat[port_id]->port_id, 0, pkts_burst, nb_tx);
nb_kni_tx = rte_eth_tx_burst(kni_stat[port_id]->port_id, 0, pkts_burst, nb_to_tx);
}
if(nb_kni_tx < nb_tx) {
uint16_t i;
for(i = nb_kni_tx; i < nb_tx; ++i)
@ -419,22 +439,28 @@ protocol_filter_ip(const void *data, uint16_t len, uint16_t eth_frame_type)
next_len = len - hdr_len;
switch (proto) {
#ifdef FF_KNI
/* The opsf protocol is forwarded to kni and the ratelimited separately */
case IPPROTO_OSPFIGP:
return FILTER_OSPF;
#endif
case IPPROTO_TCP:
#ifdef FF_KNI
if (!enable_kni)
break;
#else
break;
#endif
break;
return protocol_filter_tcp(next, next_len);
case IPPROTO_UDP:
#ifdef FF_KNI
if (!enable_kni)
break;
#else
break;
#endif
break;
return protocol_filter_udp(next, next_len);
case IPPROTO_IPIP:
return protocol_filter_ip(next, next_len, RTE_ETHER_TYPE_IPV4);
#ifdef INET6
@ -628,12 +654,34 @@ ff_kni_process(uint16_t port_id, uint16_t queue_id,
/* enqueue the packet, and own it */
int
ff_kni_enqueue(uint16_t port_id, struct rte_mbuf *pkt)
ff_kni_enqueue(enum FilterReturn filter, uint16_t port_id, struct rte_mbuf *pkt)
{
int ret = rte_ring_enqueue(kni_rp[port_id], pkt);
if (ret < 0)
rte_pktmbuf_free(pkt);
return 0;
if (filter >= FILTER_ARP) {
if (ff_global_cfg.kni.console_packets_ratelimit) {
kni_rate_limt.console_packets++;
if (kni_rate_limt.console_packets > ff_global_cfg.kni.console_packets_ratelimit) {
goto error;
}
}
} else {
if (ff_global_cfg.kni.general_packets_ratelimit) {
kni_rate_limt.gerneal_packets++;
if (kni_rate_limt.gerneal_packets > ff_global_cfg.kni.general_packets_ratelimit) {
goto error;
}
}
}
int ret = rte_ring_enqueue(kni_rp[port_id], pkt);
if (ret < 0) {
goto error;
}
return 0;
error:
rte_pktmbuf_free(pkt);
return -1;
}

View File

@ -34,13 +34,28 @@
extern int enable_kni;
extern int nb_dev_ports;
struct kni_ratelimit {
/* Important control plane packets enqueue to kni ring, such as arp, stp, ospf, etc. statistics for each process. */
uint64_t console_packets;
/* gerneal packets enqueue to kni ring, such ICMP pkts, statistics for each process. */
uint64_t gerneal_packets;
/* All packets forwarded to the kernel, statistics for primary process. */
uint64_t kernel_packets;
};
extern struct kni_ratelimit kni_rate_limt;
enum FilterReturn {
FILTER_UNKNOWN = -1,
FILTER_ARP = 1,
FILTER_KNI = 2,
FILTER_KNI = 1,
FILTER_ARP = 2,
#ifdef INET6
FILTER_NDP = 3, // Neighbor Solicitation/Advertisement, Router Solicitation/Advertisement/Redirect
#endif
FILTER_OSPF = 4,
FILTER_MULTI = 5,
};
void ff_kni_init(uint16_t nb_ports, int type, const char *tcp_ports,
@ -54,7 +69,7 @@ void ff_kni_process(uint16_t port_id, uint16_t queue_id,
enum FilterReturn ff_kni_proto_filter(const void *data, uint16_t len, uint16_t eth_frame_type);
int ff_kni_enqueue(uint16_t port_id, struct rte_mbuf *pkt);
int ff_kni_enqueue(enum FilterReturn filter, uint16_t port_id, struct rte_mbuf *pkt);
#endif /* ifndef _FSTACK_DPDK_KNI_H */