From e592cbbfec1ed8ac7fe4cbec5e99ea21d4654a59 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Tue, 14 Feb 2023 17:01:34 +0800 Subject: [PATCH 01/11] While use bbr, the `hz` should be set to 1000000, match the bintime and timer of F-Stack. Ref #701 #702. --- config.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config.ini b/config.ini index e374262c4..2003e9b33 100644 --- a/config.ini +++ b/config.ini @@ -161,7 +161,7 @@ gateway=192.168.1.1 # FreeBSD network performance tuning configurations. # Most native FreeBSD configurations are supported. [freebsd.boot] -# If use rack/bbr which depend HPTS, you should set a greater value of hz, such as 100000 means a tick is 10us. +# If use rack/bbr which depend HPTS, you should set a greater value of hz, such as 1000000 means a tick is 1us. hz=100 # Block out a range of descriptors to avoid overlap @@ -215,3 +215,7 @@ net.inet.ip.forwarding=0 net.inet.tcp.functions_default=freebsd # need by bbr, should enable it. net.inet.tcp.hpts.skip_swi=1 +# Interval between calls to hpts_timeout_dir. default min 250us, max 256-512ms, default 512ms. +net.inet.tcp.hpts.minsleep=250 +# [25600-51200] +net.inet.tcp.hpts.maxsleep=51200 From a47b73462562627da805c9a577de459cf9c52f31 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Tue, 14 Feb 2023 17:09:00 +0800 Subject: [PATCH 02/11] Fix the issue that setting `net.inet6.ip6.forwarding=1` does not take effect. --- config.ini | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/config.ini b/config.ini index 2003e9b33..85986291b 100644 --- a/config.ini +++ b/config.ini @@ -179,10 +179,6 @@ net.inet.tcp.tcbhashsize=65536 kern.ncallout=262144 kern.features.inet6=1 -net.inet6.ip6.auto_linklocal=1 -net.inet6.ip6.accept_rtadv=2 -net.inet6.icmp6.rediraccept=1 -net.inet6.ip6.forwarding=0 [freebsd.sysctl] kern.ipc.somaxconn=32768 @@ -211,6 +207,11 @@ net.inet.udp.blackhole=1 net.inet.ip.redirect=0 net.inet.ip.forwarding=0 +net.inet6.ip6.auto_linklocal=1 +net.inet6.ip6.accept_rtadv=2 +net.inet6.icmp6.rediraccept=1 +net.inet6.ip6.forwarding=0 + # set default stacks:freebsd, rack or bbr, may be you need increase the value of parameter 'freebsd.boot.hz' while use rack or bbr. net.inet.tcp.functions_default=freebsd # need by bbr, should enable it. From e14457fdc5c245c185bab40465f53507e9f86b5a Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 15 Feb 2023 18:18:09 +0800 Subject: [PATCH 03/11] Redis can listen IPv6 address. --- README.md | 2 +- app/redis-6.2.6/src/anet_ff.c | 2 +- doc/F-Stack_Quick_Start_Guide.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f7748350f..595c9cea7 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,7 @@ for more details, see [nginx guide](https://github.com/F-Stack/f-stack/blob/mast cd app/redis-6.2.6/deps/jemalloc ./autogen.sh - cd app/redis-6.2.6/ + cd ../.. make make install diff --git a/app/redis-6.2.6/src/anet_ff.c b/app/redis-6.2.6/src/anet_ff.c index b9e5172a1..d7fd6fdba 100644 --- a/app/redis-6.2.6/src/anet_ff.c +++ b/app/redis-6.2.6/src/anet_ff.c @@ -160,7 +160,7 @@ socket(int domain, int type, int protocol) return real_socket(domain, type, protocol); } - if ((AF_INET != domain) || (SOCK_STREAM != type && SOCK_DGRAM != type)) { + if ((AF_INET != domain && AF_INET6 != domain) || (SOCK_STREAM != type && SOCK_DGRAM != type)) { rc = real_socket(domain, type, protocol); return rc; } diff --git a/doc/F-Stack_Quick_Start_Guide.md b/doc/F-Stack_Quick_Start_Guide.md index fd2e2b43b..392f0d972 100644 --- a/doc/F-Stack_Quick_Start_Guide.md +++ b/doc/F-Stack_Quick_Start_Guide.md @@ -88,7 +88,7 @@ The mount point can be made permanent across reboots, by adding the following li cd app/redis-6.2.6/deps/jemalloc ./autogen.sh - cd app/redis-6.2.6/ + cd ../.. make # run with start.sh ./start.sh -b ./redis-server -o /path/to/redis.conf From 761dee801da2f90469337b5d26d2f7656e45987d Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 15 Feb 2023 19:21:58 +0800 Subject: [PATCH 04/11] Add VERSION file in root dir directory and upgrade version to 1.23. --- VERSION | 1 + lib/Makefile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 VERSION diff --git a/VERSION b/VERSION new file mode 100644 index 000000000..a1b6e17d6 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.23 diff --git a/lib/Makefile b/lib/Makefile index 3939db144..f204cd657 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -20,7 +20,7 @@ PREFIX_LIB=${PREFIX}/lib PREFIX_INCLUDE=/usr/local/include PREFIX_BIN=/usr/local/bin F-STACK_CONF=/etc/f-stack.conf -F-STACK_VERSION=1.22 +F-STACK_VERSION=1.23 TGT_OS=$(shell uname) ifeq ($(TGT_OS),FreeBSD) CC=gcc From c35e5fda179a1501fe5b51338182c1f8d219664b Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 16 Feb 2023 15:14:27 +0800 Subject: [PATCH 05/11] Fix Compile Error with gcc 11.3.0(in Ubuntu 22.04). Close #736. --- app/nginx-1.16.1/src/event/modules/ngx_ff_module.c | 5 +++++ freebsd/kern/kern_linker.c | 6 +++++- freebsd/netinet/in_mcast.c | 8 ++++++++ freebsd/netinet6/in6_mcast.c | 8 ++++++++ 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/app/nginx-1.16.1/src/event/modules/ngx_ff_module.c b/app/nginx-1.16.1/src/event/modules/ngx_ff_module.c index c74afff68..4ddd79e95 100644 --- a/app/nginx-1.16.1/src/event/modules/ngx_ff_module.c +++ b/app/nginx-1.16.1/src/event/modules/ngx_ff_module.c @@ -545,6 +545,11 @@ kevent(int kq, const struct kevent *changelist, int nchanges, return ff_kevent(kq, changelist, nchanges, eventlist, nevents, timeout); } +/* + * It is need to modify the definition, such as Ubuntu 22.04 or later. + * + * int(struct timeval * restrict, void * restrict) + */ int gettimeofday(struct timeval *tv, struct timezone *tz) { diff --git a/freebsd/kern/kern_linker.c b/freebsd/kern/kern_linker.c index e54546eec..2fb07a4f6 100644 --- a/freebsd/kern/kern_linker.c +++ b/freebsd/kern/kern_linker.c @@ -771,7 +771,7 @@ linker_file_add_dependency(linker_file_t file, linker_file_t dep) file->deps[file->ndeps] = dep; file->ndeps++; KLD_DPF(FILE, ("linker_file_add_dependency:" - " adding %s as dependency for %s\n", + " adding %s as dependency for %s\n", dep->filename, file->filename)); return (0); } @@ -1862,7 +1862,11 @@ linker_hints_lookup(const char *path, int pathlen, const char *modname, goto bad; best = cp = NULL; error = VOP_GETATTR(nd.ni_vp, &vattr, cred); +#ifdef FSTACK + if (error || vattr.va_size == 0) +#else if (error) +#endif goto bad; /* * XXX: we need to limit this number to some reasonable value diff --git a/freebsd/netinet/in_mcast.c b/freebsd/netinet/in_mcast.c index 6d390a0b2..97214d6d7 100644 --- a/freebsd/netinet/in_mcast.c +++ b/freebsd/netinet/in_mcast.c @@ -760,7 +760,11 @@ imf_get_source(struct in_mfilter *imf, const struct sockaddr_in *psin, if (lims == NULL) { if (imf->imf_nsrc == in_mcast_maxsocksrc) return (ENOSPC); +#ifdef FSTACK + nims = malloc(sizeof(struct ip_msource), M_INMFILTER, +#else nims = malloc(sizeof(struct in_msource), M_INMFILTER, +#endif M_NOWAIT | M_ZERO); if (nims == NULL) return (ENOMEM); @@ -791,7 +795,11 @@ imf_graft(struct in_mfilter *imf, const uint8_t st1, struct ip_msource *nims; struct in_msource *lims; +#ifdef FSTACK + nims = malloc(sizeof(struct ip_msource), M_INMFILTER, +#else nims = malloc(sizeof(struct in_msource), M_INMFILTER, +#endif M_NOWAIT | M_ZERO); if (nims == NULL) return (NULL); diff --git a/freebsd/netinet6/in6_mcast.c b/freebsd/netinet6/in6_mcast.c index 9b04c87d2..75dba103f 100644 --- a/freebsd/netinet6/in6_mcast.c +++ b/freebsd/netinet6/in6_mcast.c @@ -729,7 +729,11 @@ im6f_get_source(struct in6_mfilter *imf, const struct sockaddr_in6 *psin, if (lims == NULL) { if (imf->im6f_nsrc == in6_mcast_maxsocksrc) return (ENOSPC); +#ifdef FSTACK + nims = malloc(sizeof(struct ip6_msource), M_IN6MFILTER, +#else nims = malloc(sizeof(struct in6_msource), M_IN6MFILTER, +#endif M_NOWAIT | M_ZERO); if (nims == NULL) return (ENOMEM); @@ -760,7 +764,11 @@ im6f_graft(struct in6_mfilter *imf, const uint8_t st1, struct ip6_msource *nims; struct in6_msource *lims; +#ifdef FSTACK + nims = malloc(sizeof(struct ip6_msource), M_IN6MFILTER, +#else nims = malloc(sizeof(struct in6_msource), M_IN6MFILTER, +#endif M_NOWAIT | M_ZERO); if (nims == NULL) return (NULL); From c0a9f536ca427714a66e7423d96e73c287c5e586 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 16 Feb 2023 17:17:10 +0800 Subject: [PATCH 06/11] Fixed #705. While Adding -DNDEBUG flag will cause the helloworld example to crash. Because `assert((kq = ff_kqueue()) > 0);` was be ignored and nevents used usigned type. --- example/main.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/example/main.c b/example/main.c index 78ba53b37..0f9a2400e 100644 --- a/example/main.c +++ b/example/main.c @@ -25,7 +25,7 @@ int sockfd; int sockfd6; #endif -char html[] = +char html[] = "HTTP/1.1 200 OK\r\n" "Server: F-Stack\r\n" "Date: Sat, 25 Feb 2017 09:26:33 GMT\r\n" @@ -60,8 +60,14 @@ char html[] = int loop(void *arg) { /* Wait for events to happen */ - unsigned nevents = ff_kevent(kq, NULL, 0, events, MAX_EVENTS, NULL); - unsigned i; + int nevents = ff_kevent(kq, NULL, 0, events, MAX_EVENTS, NULL); + int i; + + if (nevents < 0) { + printf("ff_kevent failed:%d, %s\n", errno, + strerror(errno)); + return -1; + } for (i = 0; i < nevents; ++i) { struct kevent event = events[i]; @@ -111,7 +117,11 @@ int main(int argc, char * argv[]) { ff_init(argc, argv); - assert((kq = ff_kqueue()) > 0); + kq = ff_kqueue(); + if (kq < 0) { + printf("ff_kqueue failed, errno:%d, %s\n", errno, strerror(errno)); + exit(1); + } sockfd = ff_socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { @@ -167,7 +177,11 @@ int main(int argc, char * argv[]) } EV_SET(&kevSet, sockfd6, EVFILT_READ, EV_ADD, 0, MAX_EVENTS, NULL); - ff_kevent(kq, &kevSet, 1, NULL, 0, NULL); + ret = ff_kevent(kq, &kevSet, 1, NULL, 0, NULL); + if (ret < 0) { + printf("ff_kevent failed:%d, %s\n", errno, strerror(errno)); + exit(1); + } #endif ff_run(loop, NULL); From 06553fe07af448946316940e9c45baa9a2bf0086 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Fri, 17 Feb 2023 19:09:53 +0800 Subject: [PATCH 07/11] Code formatting fine-tuning for `example/main_zc`. --- example/main_zc.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/example/main_zc.c b/example/main_zc.c index 30dc422b3..1c7b413dd 100644 --- a/example/main_zc.c +++ b/example/main_zc.c @@ -25,7 +25,7 @@ int sockfd; int sockfd6; #endif -char html[] = +char html[] = "HTTP/1.1 200 OK\r\n" "Server: F-Stack\r\n" "Date: Sat, 25 Feb 2017 09:26:33 GMT\r\n" @@ -58,7 +58,7 @@ char html[] = ""; -char html1[] = +char html1[] = "HTTP/1.1 200 OK\r\n" "Server: F-Stack\r\n" "Date: Sat, 25 Feb 2017 09:26:33 GMT\r\n" @@ -94,7 +94,7 @@ char html1[] = "\r\n" ""; -char html2[] = +char html2[] = "HTTP/1.1 200 OK\r\n" "Server: F-Stack\r\n" "Date: Sat, 25 Feb 2017 09:26:33 GMT\r\n" @@ -195,7 +195,7 @@ int loop(void *arg) printf("ff_zc_mbuf_write failed, len:%d, errno:%d, %s\n", to_write_len, errno, strerror(errno)); exit(1); } - off += to_write_len; + off += to_write_len; } /* Or call ff_zc_mbuf_write one time */ From 7b98ccb479d33129421255fc0d45c4fb95eddf2c Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Fri, 17 Feb 2023 19:10:41 +0800 Subject: [PATCH 08/11] 1. Set non blocking in `example/main.c`. 2. Add some description of `ff_socket()` and `ff_write()`. 3. Ref #709. --- example/main.c | 16 +++++++++++++--- lib/ff_api.h | 24 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/example/main.c b/example/main.c index 0f9a2400e..c898ced5b 100644 --- a/example/main.c +++ b/example/main.c @@ -104,13 +104,19 @@ int loop(void *arg) } while (available); } else if (event.filter == EVFILT_READ) { char buf[256]; - size_t readlen = ff_read(clientfd, buf, sizeof(buf)); - - ff_write(clientfd, html, sizeof(html) - 1); + ssize_t readlen = ff_read(clientfd, buf, sizeof(buf)); + ssize_t writelen = ff_write(clientfd, html, sizeof(html) - 1); + if (writelen < 0){ + printf("ff_write failed:%d, %s\n", errno, + strerror(errno)); + ff_close(clientfd); + } } else { printf("unknown event: %8.8X\n", event.flags); } } + + return 0; } int main(int argc, char * argv[]) @@ -129,6 +135,10 @@ int main(int argc, char * argv[]) exit(1); } + /* Set non blocking */ + int on = 1; + ff_ioctl(sockfd, FIONBIO, &on); + struct sockaddr_in my_addr; bzero(&my_addr, sizeof(my_addr)); my_addr.sin_family = AF_INET; diff --git a/lib/ff_api.h b/lib/ff_api.h index 4e430011e..3cb57de86 100644 --- a/lib/ff_api.h +++ b/lib/ff_api.h @@ -64,6 +64,15 @@ int ff_sysctl(const int *name, u_int namelen, void *oldp, size_t *oldlenp, int ff_ioctl(int fd, unsigned long request, ...); +/* + * While get sockfd from this API, and then need set it to non-blocking mode like this, + * Otherwise, sometimes the socket interface will not work properly, such as `ff_write()` + * + * int on = 1; + * ff_ioctl(sockfd, FIONBIO, &on); + * + * See also `example/main.c` + */ int ff_socket(int domain, int type, int protocol); int ff_setsockopt(int s, int level, int optname, const void *optval, @@ -87,6 +96,21 @@ int ff_getsockname(int s, struct linux_sockaddr *name, ssize_t ff_read(int d, void *buf, size_t nbytes); ssize_t ff_readv(int fd, const struct iovec *iov, int iovcnt); + +/* + * Write data to the socket sendspace buf. + * + * Note: + * The `fd` parameter need set non-blocking mode in advance if F-Stack's APP. + * Otherwise if the `nbytes` parameter is greater than + * `net.inet.tcp.sendspace + net.inet.tcp.sendbuf_inc`, + * the API will return -1, but not the length that has been sent. + * + * You also can modify the value of `net.inet.tcp.sendspace`(default 16384 bytes) + * and `net.inet.tcp.sendbuf_inc`(default 16384 bytes) with `config.ini`. + * But it should be noted that not all parameters can take effect, such as 32768 and 32768. + * `ff_sysctl` can see there values while APP is running. + */ ssize_t ff_write(int fd, const void *buf, size_t nbytes); ssize_t ff_writev(int fd, const struct iovec *iov, int iovcnt); From 9e149018e36c7098f873419d396daac96fdffcd8 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Fri, 17 Feb 2023 19:13:42 +0800 Subject: [PATCH 09/11] Add header file `#include ` in `example/mai.c`. --- example/main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/example/main.c b/example/main.c index c898ced5b..a27b06e57 100644 --- a/example/main.c +++ b/example/main.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "ff_config.h" #include "ff_api.h" From 8020e2669c702fe08d19dc9bc8bf88b3a6da49ae Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 2 Mar 2023 10:59:55 +0800 Subject: [PATCH 10/11] Modify `pci_whitelist` to `allow` that from DPDK 20.11. Close #745. --- config.ini | 4 ++-- lib/ff_config.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config.ini b/config.ini index 85986291b..dcdc5adce 100644 --- a/config.ini +++ b/config.ini @@ -40,9 +40,9 @@ symmetric_rss=0 # PCI device enable list. # And driver options -#pci_whitelist=02:00.0 +#allow=02:00.0 # for multiple PCI devices -#pci_whitelist=02:00.0,03:00.0 +#allow=02:00.0,03:00.0 # enabled port list # diff --git a/lib/ff_config.c b/lib/ff_config.c index 79d88faad..fdff3f1ff 100644 --- a/lib/ff_config.c +++ b/lib/ff_config.c @@ -783,7 +783,7 @@ dpdk_args_setup(struct ff_config *cfg) char* rest = cfg->dpdk.pci_whitelist; while ((token = strtok_r(rest, ",", &rest))){ - sprintf(temp, "--pci-whitelist=%s", token); + sprintf(temp, "--allow=%s", token); dpdk_argv[n++] = strdup(temp); } From 5ed6baeedbf9750e3a14c2bbd4f9aa2481f16d0f Mon Sep 17 00:00:00 2001 From: zhutian Date: Fri, 3 Mar 2023 21:01:53 +0800 Subject: [PATCH 11/11] fix that vtoslab doesn't return the correct slab --- lib/include/vm/uma_int.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/include/vm/uma_int.h b/lib/include/vm/uma_int.h index a85d0ef74..2805f96b0 100644 --- a/lib/include/vm/uma_int.h +++ b/lib/include/vm/uma_int.h @@ -76,7 +76,7 @@ vtoslab(vm_offset_t va) hash_list = &uma_page_slab_hash[UMA_PAGE_HASH(va)]; LIST_FOREACH(up, hash_list, list_entry) - if (up->up_va == va) + if (up->up_va == (va & (~(PAGE_SIZE - 1)))) return (up->up_slab); return (NULL); } @@ -89,7 +89,7 @@ vtozoneslab(vm_offset_t va, uma_zone_t *zone, uma_slab_t *slab) hash_list = &uma_page_slab_hash[UMA_PAGE_HASH(va)]; LIST_FOREACH(up, hash_list, list_entry) - if (up->up_va == va) + if (up->up_va == (va & (~(PAGE_SIZE - 1)))) break; *slab = up->up_slab;