Merge pull request #667 from guhaoyu2005/dev

Added F-Stack FreeBSD support

README.md

@@ -38,6 +38,9 @@ Currently, besides authorized DNS server of DNSPod, there are various products i
     yum install numactl-devel          # on Centos
     #sudo apt-get install libnuma-dev  # on Ubuntu
 
+    # Install dependencies (FreeBSD only)
+    #pkg install meson pkgconf py38-pyelftools
+
     cd f-stack
     # Compile DPDK
     cd dpdk/
@@ -45,25 +48,27 @@ Currently, besides authorized DNS server of DNSPod, there are various products i
     ninja -C build
     ninja -C build install
 
-    # Set hugepage
+    # Set hugepage (Linux only)
     # single-node system
     echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
 
-    # or NUMA
+    # or NUMA (Linux only)
     echo 1024 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
     echo 1024 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages
 
-    # Using Hugepage with the DPDK
+    # Using Hugepage with the DPDK (Linux only)
     mkdir /mnt/huge
     mount -t hugetlbfs nodev /mnt/huge
 
-    # Close ASLR; it is necessary in multiple process
+    # Close ASLR; it is necessary in multiple process (Linux only)
     echo 0 > /proc/sys/kernel/randomize_va_space
 
     # Install python for running DPDK python scripts
     sudo apt install python # On ubuntu
+    #sudo pkg install python # On FreeBSD
 
     # Offload NIC
+    # For Linux:
     modprobe uio
     insmod /data/f-stack/dpdk/build/kernel/linux/igb_uio/igb_uio.ko
     insmod /data/f-stack/dpdk/build/kernel/linux/kni/rte_kni.ko carrier=on # carrier=on is necessary, otherwise need to be up `veth0` via `echo 1 > /sys/class/net/veth0/carrier`
@@ -71,11 +76,21 @@ Currently, besides authorized DNS server of DNSPod, there are various products i
     ifconfig eth0 down
     python dpdk-devbind.py --bind=igb_uio eth0 # assuming that use 10GE NIC and eth0
 
+    # For FreeBSD:
+    # Refer DPDK FreeBSD guide to set tunables in /boot/loader.conf
+    # Below is an example used for our testing machine
+    #echo "hw.nic_uio.bdfs=\"2:0:0\"" >> /boot/loader.conf
+    #echo "hw.contigmem.num_buffers=1" >> /boot/loader.conf
+    #echo "hw.contigmem.buffer_size=1073741824" >> /boot/loader.conf
+    #kldload contigmem
+    #kldload nic_uio
+
     # On Ubuntu, use gawk instead of the default mawk.
     #sudo apt-get install gawk  # or execute `sudo update-alternatives --config awk` to choose gawk.
 
     # Install dependencies for F-Stack
-    sudo apt install gcc make libssl-dev # On ubuntu
+    sudo apt install gcc make libssl-dev                            # On ubuntu
+    #sudo pkg install gcc gmake openssl pkgconf libepoll-shim       # On FreeBSD
 
     # Upgrade pkg-config while version < 0.28
     #cd /data
@@ -92,14 +107,16 @@ Currently, besides authorized DNS server of DNSPod, there are various products i
     export FF_PATH=/data/f-stack
     export PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig:/usr/lib/pkgconfig
     cd /data/f-stack/lib/
-    make
+    make    # On Linux
+    #gmake   # On FreeBSD
 
     # Install F-STACK
     # libfstack.a will be installed to /usr/local/lib
     # ff_*.h will be installed to /usr/local/include
     # start.sh will be installed to /usr/local/bin/ff_start
     # config.ini will be installed to /etc/f-stack.conf
-    make install
+    make install    # On Linux
+    #gmake install   # On FreeBSD
 
 #### Nginx
 

dpdk/lib/librte_eal/freebsd/include/rte_os.h

@@ -14,6 +14,22 @@
 #include <pthread_np.h>
 
 typedef cpuset_t rte_cpuset_t;
+#if __FreeBSD_version >= 1301000
+#define RTE_CPU_AND(dst, src1, src2) do \
+{ \
+	cpuset_t tmp; \
+	CPU_COPY(src1, &tmp); \
+	CPU_AND(&tmp, &tmp, src2); \
+	CPU_COPY(&tmp, dst); \
+} while (0)
+#define RTE_CPU_OR(dst, src1, src2) do \
+{ \
+	cpuset_t tmp; \
+	CPU_COPY(src1, &tmp); \
+	CPU_OR(&tmp, &tmp, src2); \
+	CPU_COPY(&tmp, dst); \
+} while (0)
+#else
 #define RTE_CPU_AND(dst, src1, src2) do \
 { \
 	cpuset_t tmp; \
@@ -28,6 +44,7 @@ typedef cpuset_t rte_cpuset_t;
 	CPU_OR(&tmp, src2); \
 	CPU_COPY(&tmp, dst); \
 } while (0)
+#endif
 #define RTE_CPU_FILL(set) CPU_FILL(set)
 
 /* In FreeBSD 13 CPU_NAND macro is CPU_ANDNOT */
@@ -40,6 +57,15 @@ typedef cpuset_t rte_cpuset_t;
 	CPU_COPY(&tmp, dst); \
 } while (0)
 #else
+#if __FreeBSD_version >= 1301000
+#define RTE_CPU_NOT(dst, src) do \
+{ \
+	cpuset_t tmp; \
+	CPU_FILL(&tmp); \
+	CPU_ANDNOT(&tmp, &tmp, src); \
+	CPU_COPY(&tmp, dst); \
+} while (0)
+#else
 #define RTE_CPU_NOT(dst, src) do \
 { \
 	cpuset_t tmp; \
@@ -47,6 +73,8 @@ typedef cpuset_t rte_cpuset_t;
 	CPU_ANDNOT(&tmp, src); \
 	CPU_COPY(&tmp, dst); \
 } while (0)
+
+#endif
 #endif
 
 #endif /* _RTE_OS_H_ */

lib/Makefile

@@ -21,12 +21,19 @@ PREFIX_INCLUDE=/usr/local/include
 PREFIX_BIN=/usr/local/bin
 F-STACK_CONF=/etc/f-stack.conf
 F-STACK_VERSION=1.22
+TGT_OS=$(shell uname)
+ifeq ($(TGT_OS),FreeBSD)
+CC=gcc
+endif
 
 HOST_OS:=$(shell uname -s)
 
 DEBUG=-O0 -gdwarf-2 -g3 -Wno-format-truncation
 
+# No DPDK KNI support on FreeBSD
+ifneq ($(TGT_OS),FreeBSD)
 FF_KNI=1
+endif
 
 #FF_FLOW_ISOLATE=1
 
@@ -59,6 +66,11 @@ INCLUDES+= -I./opt
 
 # Include search path for files that only include host OS headers
 HOST_INCLUDES= -I.
+# Use libepoll shim on FreeBSD
+ifeq ($(TGT_OS),FreeBSD)
+HOST_INCLUDES+= -I/usr/local/include/libepoll-shim
+endif
+
 ifndef DEBUG
 HOST_CFLAGS = -O2 -frename-registers  -funswitch-loops -fweb -Wno-format-truncation
 else
@@ -524,11 +536,15 @@ EXTRA_TCP_STACKS_SRCS+=		\
 	bbr.c
 endif
 
+ 
+ifneq ($(TGT_OS),FreeBSD)
 ifndef FF_KNI
 FF_HOST_SRCS+=         \
         ff_dpdk_kni.c
-endif
+endif		#FF_KNI
-endif
+endif		#FreeBSD OS Check
+
+endif		#INET6
 
 ifdef FF_IPFW
 NETIPFW_SRCS+=             \

lib/ff_dpdk_if.c

@@ -1179,7 +1179,8 @@ protocol_filter(const void *data, uint16_t len)
     if(ether_type == RTE_ETHER_TYPE_ARP)
         return FILTER_ARP;
 
-#ifdef INET6
+#if (!defined(__FreeBSD__) && defined(INET6) ) || \
+    ( defined(__FreeBSD__) && defined(INET6) && defined(FF_KNI))
     if (ether_type == RTE_ETHER_TYPE_IPV6) {
         return ff_kni_proto_filter(data,
             len, ether_type);