cmhi
/
f-stack
mirror of https://github.com/F-Stack/f-stack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support KNI ratelimit. 

KNI ratelimit default value.
The total speed limit for a single process entering the kni ring is 10,000 QPS,
1000 QPS for general packets, 9000 QPS for console packets (ospf/arp, etc.)
The total speed limit for kni forwarding to the kernel is 20,000 QPS.
This commit is contained in:
fengbojiang 2024-10-10 17:48:40 +08:00
parent 68e1f5aa90
commit 65299c5f75
6 changed files with 139 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
config.ini
View File

@ -207,6 +207,13 @@ gateway=192.168.1.1
# The format is same as port_list
#tcp_port=80,443
#udp_port=53
# KNI ratelimit default value.
# The total speed limit for a single process entering the kni ring is 10,000 QPS,
# 1000 QPS for general packets, 9000 QPS for console packets (ospf/arp, etc.)
# The total speed limit for kni forwarding to the kernel is 20,000 QPS.
#console_packets_ratelimit=1000
#general_packets_ratelimit=9000
#kernel_packets_ratelimit=20000
# FreeBSD network performance tuning configurations.
# Most native FreeBSD configurations are supported.

10
lib/ff_config.c
View File

@ -907,6 +907,12 @@ ini_parse_handler(void* user, const char* section, const char* name,
pconfig->dpdk.symmetric_rss = atoi(value);
} else if (MATCH("kni", "enable")) {
pconfig->kni.enable= atoi(value);
} else if (MATCH("kni", "console_packets_ratelimit")) {
pconfig->kni.console_packets_ratelimit= atoi(value);
} else if (MATCH("kni", "general_packets_ratelimit")) {
pconfig->kni.general_packets_ratelimit= atoi(value);
} else if (MATCH("kni", "kernel_packets_ratelimit")) {
pconfig->kni.kernel_packets_ratelimit= atoi(value);
} else if (MATCH("kni", "kni_action")) {
pconfig->kni.kni_action= strdup(value);
} else if (MATCH("kni", "method")) {
@ -1239,6 +1245,10 @@ ff_default_config(struct ff_config *cfg)
cfg->dpdk.promiscuous = 1;
cfg->dpdk.pkt_tx_delay = BURST_TX_DRAIN_US;
cfg->kni.console_packets_ratelimit = KNI_RATELIMT_CONSOLE;
cfg->kni.general_packets_ratelimit = KNI_RATELIMT_GENERAL;
cfg->kni.kernel_packets_ratelimit = KNI_RATELIMT_KERNEL;
cfg->freebsd.hz = 100;
cfg->freebsd.physmem = 1048576*256;
cfg->freebsd.fd_reserve = 0;

13
lib/ff_config.h
View File

@ -38,6 +38,16 @@ extern "C" {
#define DPDK_MAX_VLAN_FILTER 128
#define PCAP_SNAP_MINLEN 94
#define PCAP_SAVE_MINLEN (2<<22)
/*
* KNI ratelimit default value.
* The total speed limit for a single process entering the kni ring is 10,000 QPS,
* 1000 QPS for general packets, 9000 QPS for console packets (ospf/arp, etc.)
* The total speed limit for kni forwarding to the kernel is 20,000 QPS.
*/
#define KNI_RATELIMT_PROCESS (10000)
#define KNI_RATELIMT_GENERAL (1000)
#define KNI_RATELIMT_CONSOLE (KNI_RATELIMT_PROCESS - KNI_RATELIMT_GENERAL)
#define KNI_RATELIMT_KERNEL (KNI_RATELIMT_PROCESS * 2)
extern int dpdk_argc;
extern char *dpdk_argv[DPDK_CONFIG_NUM + 1];
@ -236,6 +246,9 @@ struct ff_config {
struct {
int enable;
int console_packets_ratelimit;
int general_packets_ratelimit;
int kernel_packets_ratelimit;
char *kni_action;
char *method;
char *tcp_port;

41
lib/ff_dpdk_if.c
View File

@ -1343,8 +1343,14 @@ protocol_filter(const void *data, uint16_t len)
len -= sizeof(struct rte_vlan_hdr);
}
if(ether_type == RTE_ETHER_TYPE_ARP)
if(ether_type == RTE_ETHER_TYPE_ARP) {
return FILTER_ARP;
}
/* Multicast protocol, such as stp(used by zebra), is forwarded to kni and has a separate speed limit */
if (rte_is_multicast_ether_addr(&hdr->d_addr)) {
return FILTER_MULTI;
}
#if (!defined(__FreeBSD__) && defined(INET6) ) || \
( defined(__FreeBSD__) && defined(INET6) && defined(FF_KNI))
@ -1536,7 +1542,7 @@ process_packets(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **bufs,
mbuf_clone = pktmbuf_deep_clone(rtem, mbuf_pool);
if(mbuf_clone) {
ff_add_vlan_tag(mbuf_clone);
ff_kni_enqueue(port_id, mbuf_clone);
ff_kni_enqueue(filter, port_id, mbuf_clone);
}
}
#endif
@ -1545,15 +1551,15 @@ process_packets(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **bufs,
} else if (enable_kni) {
if (knictl_action == FF_KNICTL_ACTION_ALL_TO_KNI){
ff_add_vlan_tag(rtem);
ff_kni_enqueue(port_id, rtem);
ff_kni_enqueue(filter, port_id, rtem);
} else if (knictl_action == FF_KNICTL_ACTION_ALL_TO_FF){
ff_veth_input(ctx, rtem);
} else if (knictl_action == FF_KNICTL_ACTION_DEFAULT){
if (enable_kni &&
((filter == FILTER_KNI && kni_accept) ||
(filter == FILTER_UNKNOWN && !kni_accept)) ) {
((filter == FILTER_UNKNOWN || filter >= FILTER_OSPF) && !kni_accept)) ) {
ff_add_vlan_tag(rtem);
ff_kni_enqueue(port_id, rtem);
ff_kni_enqueue(filter, port_id, rtem);
} else {
ff_veth_input(ctx, rtem);
}
@ -2067,6 +2073,31 @@ main_loop(void *arg)
cur_tsc = rte_rdtsc();
if (unlikely(freebsd_clock.expire < cur_tsc)) {
rte_timer_manage();
#ifdef FF_KNI
/* reset kni ratelimt */
if (enable_kni) {
static time_t last_sec = 0;
time_t sec;
long nsec;
ff_get_current_time(&sec, &nsec);
if (sec > last_sec) {
if (kni_rate_limt.gerneal_packets > ff_global_cfg.kni.general_packets_ratelimit ||
kni_rate_limt.console_packets > ff_global_cfg.kni.console_packets_ratelimit ||
kni_rate_limt.kernel_packets > ff_global_cfg.kni.kernel_packets_ratelimit) {
printf("kni ratelimit, general:%lu/%d, console:%lu/%d, kernel:%lu/%d, last sec:%ld, sec:%ld\n",
kni_rate_limt.gerneal_packets, ff_global_cfg.kni.general_packets_ratelimit,
kni_rate_limt.console_packets, ff_global_cfg.kni.console_packets_ratelimit,
kni_rate_limt.kernel_packets, ff_global_cfg.kni.kernel_packets_ratelimit, last_sec, sec);
}
last_sec = sec;
kni_rate_limt.gerneal_packets = 0;
kni_rate_limt.console_packets = 0;
kni_rate_limt.kernel_packets = 0;
}
}
#endif
}
idle = 1;

69
lib/ff_dpdk_kni.c
View File

@ -42,6 +42,10 @@
#include "ff_dpdk_kni.h"
#include "ff_config.h"
#ifndef IPPROTO_OSPFIGP
#define IPPROTO_OSPFIGP 89 /**< OSPFIGP */
#endif
/* Callback for request of changing MTU */
/* Total octets in ethernet header */
#define KNI_ENET_HEADER_SIZE 14
@ -81,6 +85,8 @@ struct kni_interface_stats {
struct rte_ring **kni_rp;
struct kni_interface_stats **kni_stat;
struct kni_ratelimit kni_rate_limt = {0, 0, 0};
static void
set_bitmap(uint16_t port, unsigned char *bitmap)
{
@ -160,7 +166,7 @@ kni_config_network_interface(uint16_t port_id, uint8_t if_up)
}
if (ret < 0)
printf("Failed to Configure network interface of %d %s\n",
printf("Failed to Configure network interface of %d %s\n",
port_id, if_up ? "up" : "down");
return ret;
@ -201,15 +207,26 @@ kni_process_tx(uint16_t port_id, uint16_t queue_id,
struct rte_mbuf **pkts_burst, unsigned count)
{
/* read packet from kni ring(phy port) and transmit to kni */
uint16_t nb_tx, nb_kni_tx;
uint16_t nb_tx, nb_to_tx, nb_kni_tx;
nb_tx = rte_ring_dequeue_burst(kni_rp[port_id], (void **)pkts_burst, count, NULL);
/*
* The total ratelimit forwarded to the kernel, may a few more packets being sent, but it doesn’t matter,
* If there are too many processes, there is also the possibility that the control packet will be ratelimited.
*/
if (likely(kni_rate_limt.kernel_packets < ff_global_cfg.kni.kernel_packets_ratelimit)) {
nb_to_tx = nb_tx;
} else {
nb_to_tx = 0;
}
kni_rate_limt.kernel_packets += nb_tx;
/* NB.
* if nb_tx is 0,it must call rte_kni_tx_burst
* must Call regularly rte_kni_tx_burst(kni, NULL, 0).
* detail https://embedded.communities.intel.com/thread/6668
*/
nb_kni_tx = rte_kni_tx_burst(kni_stat[port_id]->kni, pkts_burst, nb_tx);
nb_kni_tx = rte_kni_tx_burst(kni_stat[port_id]->kni, pkts_burst, nb_to_tx);
rte_kni_handle_request(kni_stat[port_id]->kni);
if(nb_kni_tx < nb_tx) {
uint16_t i;
@ -385,22 +402,28 @@ protocol_filter_ip(const void *data, uint16_t len, uint16_t eth_frame_type)
next_len = len - hdr_len;
switch (proto) {
#ifdef FF_KNI
/* The opsf protocol is forwarded to kni and the ratelimited separately */
case IPPROTO_OSPFIGP:
return FILTER_OSPF;
#endif
case IPPROTO_TCP:
#ifdef FF_KNI
if (!enable_kni)
break;
#else
break;
#endif
break;
return protocol_filter_tcp(next, next_len);
case IPPROTO_UDP:
#ifdef FF_KNI
if (!enable_kni)
break;
#else
break;
#endif
break;
return protocol_filter_udp(next, next_len);
case IPPROTO_IPIP:
return protocol_filter_ip(next, next_len, RTE_ETHER_TYPE_IPV4);
#ifdef INET6
@ -507,7 +530,7 @@ ff_kni_alloc(uint16_t port_id, unsigned socket_id,
conf.addr = pci_dev->addr;
conf.id = pci_dev->id;
}
/* Get the interface default mac address */
rte_eth_macaddr_get(port_id,
(struct rte_ether_addr *)&conf.mac_addr);
@ -534,7 +557,7 @@ ff_kni_alloc(uint16_t port_id, unsigned socket_id,
snprintf((char*)ring_name, RTE_KNI_NAMESIZE, "kni_ring_%u", port_id);
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
kni_rp[port_id] = rte_ring_create(ring_name, ring_queue_size,
kni_rp[port_id] = rte_ring_create(ring_name, ring_queue_size,
socket_id, RING_F_SC_DEQ);
if (rte_ring_lookup(ring_name) != kni_rp[port_id])
@ -560,12 +583,30 @@ ff_kni_process(uint16_t port_id, uint16_t queue_id,
/* enqueue the packet, and own it */
int
ff_kni_enqueue(uint16_t port_id, struct rte_mbuf *pkt)
ff_kni_enqueue(enum FilterReturn filter, uint16_t port_id, struct rte_mbuf *pkt)
{
if (filter >= FILTER_ARP) {
kni_rate_limt.console_packets++;
if (kni_rate_limt.console_packets > ff_global_cfg.kni.console_packets_ratelimit) {
goto error;
}
} else {
kni_rate_limt.gerneal_packets++;
if (kni_rate_limt.gerneal_packets > ff_global_cfg.kni.general_packets_ratelimit) {
goto error;
}
}
int ret = rte_ring_enqueue(kni_rp[port_id], pkt);
if (ret < 0)
rte_pktmbuf_free(pkt);
if (ret < 0) {
goto error;
}
return 0;
error:
rte_pktmbuf_free(pkt);
return -1;
}

21
lib/ff_dpdk_kni.h
View File

@ -33,13 +33,28 @@
extern int enable_kni;
struct kni_ratelimit {
/* Important control plane packets enqueue to kni ring, such as arp, stp, ospf, etc. statistics for each process. */
uint64_t console_packets;
/* gerneal packets enqueue to kni ring, such ICMP pkts, statistics for each process. */
uint64_t gerneal_packets;
/* All packets forwarded to the kernel, statistics for primary process. */
uint64_t kernel_packets;
};
extern struct kni_ratelimit kni_rate_limt;
enum FilterReturn {
FILTER_UNKNOWN = -1,
FILTER_ARP = 1,
FILTER_KNI = 2,
FILTER_KNI = 1,
FILTER_ARP = 2,
#ifdef INET6
FILTER_NDP = 3, // Neighbor Solicitation/Advertisement, Router Solicitation/Advertisement/Redirect
#endif
FILTER_OSPF = 4,
FILTER_MULTI = 5,
};
void ff_kni_init(uint16_t nb_ports, const char *tcp_ports,
@ -53,7 +68,7 @@ void ff_kni_process(uint16_t port_id, uint16_t queue_id,
enum FilterReturn ff_kni_proto_filter(const void *data, uint16_t len, uint16_t eth_frame_type);
int ff_kni_enqueue(uint16_t port_id, struct rte_mbuf *pkt);
int ff_kni_enqueue(enum FilterReturn filter, uint16_t port_id, struct rte_mbuf *pkt);
#endif /* ifndef _FSTACK_DPDK_KNI_H */