cmhi
/
f-stack
mirror of https://github.com/F-Stack/f-stack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support DPDK 20.11.0(LTS).

This commit is contained in:
fengbojiang 2021-02-05 16:42:07 +08:00
parent ba22d55c70
commit 8d76b62e73
14 changed files with 182 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
README.md
View File

@ -40,8 +40,10 @@ Currently, besides authorized DNS server of DNSPod, there are various products i
cd f-stack
# Compile DPDK
cd dpdk/usertools
./dpdk-setup.sh # compile with x86_64-native-linuxapp-gcc
cd dpdk/
meson build
ninja -C build
ninja -C build install
# Set hugepage
# single-node system
@ -60,7 +62,7 @@ Currently, besides authorized DNS server of DNSPod, there are various products i
# Install python for running DPDK python scripts
sudo apt install python # On ubuntu
# Offload NIC
modprobe uio
insmod /data/f-stack/dpdk/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko
@ -69,20 +71,27 @@ Currently, besides authorized DNS server of DNSPod, there are various products i
ifconfig eth0 down
python dpdk-devbind.py --bind=igb_uio eth0 # assuming that use 10GE NIC and eth0
# Install DPDK
cd ../x86_64-native-linuxapp-gcc
make install
# On Ubuntu, use gawk instead of the default mawk.
#sudo apt-get install gawk # or execute `sudo update-alternatives --config awk` to choose gawk.
# Install dependencies for F-Stack
sudo apt install gcc make libssl-dev # On ubuntu
# Upgrade pkg-config while version < 0.28
#cd /data
#wget https://pkg-config.freedesktop.org/releases/pkg-config-0.29.2.tar.gz
#tar xzvf pkg-config-0.29.2.tar.gz
#cd pkg-config-0.29.2
#./configure --with-internal-glib
#make
#make install
#mv /usr/bin/pkg-config /usr/bin/pkg-config.bak
#ln -s /usr/local/bin/pkg-config /usr/bin/pkg-config
# Compile F-Stack
export FF_PATH=/data/f-stack
export FF_DPDK=/data/f-stack/dpdk/x86_64-native-linuxapp-gcc
cd ../../lib/
export PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig:/usr/lib/pkgconfig
cd /data/f-stack/lib/
make
# Install F-STACK
@ -120,10 +129,6 @@ for more details, see [nginx guide](https://github.com/F-Stack/f-stack/blob/mast
echo 1 > /sys/class/net/veth0/carrier # if `carrier=on` not set while `insmod rte_kni.ko`
# route add -net ... # other route rules
## Binary Release
We provide a f-stack-binary-release package that you can use F-Stack directly without compiling. For more details, see [Binary_Release_Quick_Start](https://github.com/F-Stack/f-stack/blob/master/doc/F-Stack_Binary_Release_Quick_Start.md).
## Nginx Testing Result
Test environment

9
app/micro_thread/Makefile
View File

@ -24,15 +24,16 @@ ifeq ($(FF_PATH),)
$(warning FF_PATH environment variable not defined, default FF_PATH=/usr/local)
endif
ifeq ($(FF_DPDK),)
FF_DPDK=/usr/local
$(warning FF_DPDK environment variable not defined, default FF_DPDK=/usr/local)
ifneq ($(shell pkg-config --exists libdpdk && echo 0),0)
$(error "no installation of DPDK found, maybe you shuld export environment variable `PKG_CONFIG_PATH`")
endif
PKGCONF ?= pkg-config
DEBUG= -g
BINARY = libmt.a
FF_LIBS+= $(shell $(PKGCONF) --static --libs libdpdk)
FF_LIBS+= -L${FF_PATH}/lib -Wl,--whole-archive,-lfstack,--no-whole-archive
FF_LIBS+= -L${FF_DPDK}/lib -Wl,--whole-archive,-ldpdk,--no-whole-archive
FF_LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -lpthread -lnuma
# Comment the following line if you are not using the gnu c compiler

8
app/nginx-1.16.1/auto/make
View File

@ -23,13 +23,13 @@ if [ ! $FF_PATH ]; then
echo "FF_PATH environment variable not defined, default FF_PATH=$FF_PATH"
fi
if [ ! $FF_DPDK ]; then
FF_DPDK=/usr/local
echo "FF_DPDK environment variable not defined, default FF_DPDK=$FF_DPDK"
if [ ! $(pkg-config --exists libdpdk && echo 1) ]; then
echo "error: no installation of DPDK found, maybe you shuld export environment variable \`PKG_CONFIG_PATH\`"
exit 1
fi
CORE_LIBS="$CORE_LIBS \$(shell pkg-config --static --libs libdpdk)"
CORE_LIBS="$CORE_LIBS -L$FF_PATH/lib -Wl,--whole-archive,-lfstack,--no-whole-archive"
CORE_LIBS="$CORE_LIBS -L$FF_DPDK/lib -Wl,--whole-archive,-ldpdk,--no-whole-archive"
CORE_LIBS="$CORE_LIBS -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -lpthread -lnuma"
cat << END > $NGX_MAKEFILE

9
app/redis-5.0.5/src/Makefile
View File

@ -17,9 +17,8 @@ ifeq ($(FF_PATH),)
$(warning FF_PATH environment variable not defined, default FF_PATH=/usr/local)
endif
ifeq ($(FF_DPDK),)
FF_DPDK=/usr/local
$(warning FF_DPDK environment variable not defined, default FF_DPDK=/usr/local)
ifneq ($(shell pkg-config --exists libdpdk && echo 0),0)
$(error "no installation of DPDK found, maybe you shuld export environment variable `PKG_CONFIG_PATH`")
endif
release_hdr := $(shell sh -c './mkreleasehdr.sh')
@ -158,8 +157,10 @@ endif
FINAL_CFLAGS+= -DHAVE_FF_KQUEUE
FINAL_CFLAGS+= -I$(FF_PATH)/lib
PKGCONF ?= pkg-config
FINAL_LIBS+= $(shell $(PKGCONF) --static --libs libdpdk)
FINAL_LIBS+= -L${FF_PATH}/lib -Wl,--whole-archive,-lfstack,--no-whole-archive
FINAL_LIBS+= -L${FF_DPDK}/lib -Wl,--whole-archive,-ldpdk,--no-whole-archive
FINAL_LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -lpthread -lnuma
REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS)

134
doc/F-Stack_Build_Guide.md
View File

@ -1,61 +1,73 @@
# F-Stack Build GUide
The procedures to compile f-stack in different linux releases is almost the same, but there are still some points you need to pay attention to. This document aims to help you solve some of the problems you may meet when compiling f-stack in different linux releases.
```sh
$ sudo -i
# in centos and redhat
$ yum install -y git gcc openssl-devel kernel-devel-$(uname -r) bc numactl-devel python
# in ubuntu
$ apt-get install git gcc openssl libssl-dev linux-headers-$(uname -r) bc libnuma1 libnuma-dev libpcre3 libpcre3-dev zlib1g-dev python
$ mkdir /data/f-stack
$ git clone https://github.com/F-Stack/f-stack.git /data/f-stack
# compile dpdk
$ cd /data/f-stack/dpdk
$ make config T=x86_64-native-linuxapp-gcc
$ make
# Compile f-stack lib
$ export FF_PATH=/data/f-stack
$ export FF_DPDK=/data/f-stack/dpdk/build
$ cd /data/f-stack/lib
$ make
# Compile Nginx
$ cd ../app/nginx-1.16.1
$ ./configure --prefix=/usr/local/nginx_fstack --with-ff_module
$ make
$ make install
# Compile Redis
$ cd ../redis-5.0.5
$ make
# Compile f-stack tools
$ cd ../../tools
$ make
# Compile helloworld examples
$ cd ../examples
$ make
```
## Compile tools in Ubuntu
- remove '\\' in statement printf at f-stack/tools/netstat/Makefile line 70, now it should be:
```
- printf("\#define\tN%s\t%s\n", toupper($$2), i++);
+ printf("#define\tN%s\t%s\n", toupper($$2), i++);
```
## Compile dpdk in virtual machine
- f-stack/dpdk/lib/librte_eal/linuxapp/igb_uio/igb_uio.c line 279:
```
- if (pci_intx_mask_supported(udev->pdev)) {
+ if (true || pci_intx_mask_supported(udev->pdev)) {
```
# F-Stack Build GUide
The procedures to compile f-stack in different linux releases is almost the same, but there are still some points you need to pay attention to. This document aims to help you solve some of the problems you may meet when compiling f-stack in different linux releases.
```sh
$ sudo -i
# in centos and redhat
$ yum install -y git gcc openssl-devel kernel-devel-$(uname -r) bc numactl-devel python
# in ubuntu
$ apt-get install git gcc openssl libssl-dev linux-headers-$(uname -r) bc libnuma1 libnuma-dev libpcre3 libpcre3-dev zlib1g-dev python
$ mkdir /data/f-stack
$ git clone https://github.com/F-Stack/f-stack.git /data/f-stack
# compile dpdk
$ cd /data/f-stack/dpdk
$ meson build
$ ninja -C build
$ ninja -C build install
# Upgrade pkg-config while version < 0.28
$ cd /data
$ wget https://pkg-config.freedesktop.org/releases/pkg-config-0.29.2.tar.gz
$ tar xzvf pkg-config-0.29.2.tar.gz
$ cd pkg-config-0.29.2
$ ./configure --with-internal-glib
$ make
$ make install
$ mv /usr/bin/pkg-config /usr/bin/pkg-config.bak
$ ln -s /usr/local/bin/pkg-config /usr/bin/pkg-config
# Compile f-stack lib
$ export FF_PATH=/data/f-stack
$ export PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig:/usr/lib/pkgconfig
$ cd /data/f-stack/lib
$ make
# Compile Nginx
$ cd ../app/nginx-1.16.1
$ ./configure --prefix=/usr/local/nginx_fstack --with-ff_module
$ make
$ make install
# Compile Redis
$ cd ../redis-5.0.5
$ make
# Compile f-stack tools
$ cd ../../tools
$ make
# Compile helloworld examples
$ cd ../examples
$ make
```
## Compile tools in Ubuntu
- remove '\\' in statement printf at f-stack/tools/netstat/Makefile line 70, now it should be:
```
- printf("\#define\tN%s\t%s\n", toupper($$2), i++);
+ printf("#define\tN%s\t%s\n", toupper($$2), i++);
```
## Compile dpdk in virtual machine
- f-stack/dpdk/lib/librte_eal/linuxapp/igb_uio/igb_uio.c line 279:
```
- if (pci_intx_mask_supported(udev->pdev)) {
+ if (true || pci_intx_mask_supported(udev->pdev)) {
```

23
doc/F-Stack_Quick_Start_Guide.md
View File

@ -16,10 +16,10 @@ See Intel DPDK [linux_gsg](http://dpdk.org/doc/guides/linux_gsg/index.html)
Read DPDK Quick Started Guide or run the command below
cd /data/f-stack/dpdk/tools
./dpdk-setup.sh
Compile with x86_64-native-linuxapp-gcc
cd /data/f-stack/dpdk
meson build
ninja -C build
ninja -C build install
## Set hugepage
@ -52,9 +52,20 @@ The mount point can be made permanent across reboots, by adding the following li
## Compile lib
# Upgrade pkg-config while version < 0.28
cd /data/
wget https://pkg-config.freedesktop.org/releases/pkg-config-0.29.2.tar.gz
tar xzvf pkg-config-0.29.2.tar.gz
cd pkg-config-0.29.2
./configure --with-internal-glib
make
make install
mv /usr/bin/pkg-config /usr/bin/pkg-config.bak
ln -s /usr/local/bin/pkg-config /usr/bin/pkg-config
export FF_PATH=/data/f-stack
export FF_DPDK=/data/f-stack/dpdk/x86_64-native-linuxapp-gcc
cd ../../
export PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig:/usr/lib/pkgconfig
cd /data/f-stack
cd lib
make

13
doc/Launch_F-Stack_on_AWS_EC2_in_one_minute.md
View File

@ -53,9 +53,20 @@
sed "s/#tcp_port=80/tcp_port=80/" -i /data/f-stack/config.ini
sed "s/#vlanstrip=1/vlanstrip=1/" -i /data/f-stack/config.ini
# Upgrade pkg-config while version < 0.28
cd /data/
wget https://pkg-config.freedesktop.org/releases/pkg-config-0.29.2.tar.gz
tar xzvf pkg-config-0.29.2.tar.gz
cd pkg-config-0.29.2
./configure --with-internal-glib
make
make install
mv /usr/bin/pkg-config /usr/bin/pkg-config.bak
ln -s /usr/local/bin/pkg-config /usr/bin/pkg-config
# Compile F-Stack lib
export FF_PATH=/data/f-stack
export FF_DPDK=/data/f-stack/dpdk/build
export PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig:/usr/lib/pkgconfig
cd /data/f-stack/lib
make

15
example/Makefile
View File

@ -4,18 +4,21 @@ ifeq ($(FF_PATH),)
FF_PATH=${TOPDIR}
endif
ifeq ($(FF_DPDK),)
FF_DPDK=${TOPDIR}/dpdk/x86_64-native-linuxapp-gcc
ifneq ($(shell pkg-config --exists libdpdk && echo 0),0)
$(error "no installation of DPDK found, maybe you shuld export environment variable `PKG_CONFIG_PATH`")
endif
PKGCONF ?= pkg-config
CFLAGS += -O -gdwarf-2 $(shell $(PKGCONF) --cflags libdpdk)
LIBS+= $(shell $(PKGCONF) --static --libs libdpdk)
LIBS+= -L${FF_PATH}/lib -Wl,--whole-archive,-lfstack,--no-whole-archive
LIBS+= -L${FF_DPDK}/lib -Wl,--whole-archive,-ldpdk,--no-whole-archive
LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -pthread -lnuma
TARGET="helloworld"
all:
cc -O -gdwarf-2 -I../lib -o ${TARGET} main.c ${LIBS}
cc -O -gdwarf-2 -I../lib -o ${TARGET}_epoll main_epoll.c ${LIBS}
cc ${CFLAGS} -o ${TARGET} main.c ${LIBS}
cc ${CFLAGS} -o ${TARGET}_epoll main_epoll.c ${LIBS}
.PHONY: clean
clean:

22
lib/Makefile
View File

@ -35,27 +35,13 @@ FF_INET6=1
include ${TOPDIR}/mk/kern.pre.mk
ifeq ($(FF_DPDK),)
ifeq (${MACHINE_CPUARCH},aarch64)
FF_DPDK=${TOPDIR}/dpdk/build
else
FF_DPDK=${TOPDIR}/dpdk/x86_64-native-linuxapp-gcc
endif
ifneq ($(shell pkg-config --exists libdpdk && echo 0),0)
$(error "no installation of DPDK found, maybe you shuld export environment variable `PKG_CONFIG_PATH`")
endif
ifdef RTE_SDK
ifeq (${MACHINE_CPUARCH},aarch64)
FF_DPDK=${RTE_SDK}/build
else
FF_DPDK=${RTE_SDK}/x86_64-native-linuxapp-gcc
endif
endif
PKGCONF ?= pkg-config
DPDK_CFLAGS= -Wall -Wno-deprecated-declarations -Werror -include ${FF_DPDK}/include/rte_config.h
DPDK_CFLAGS+= -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3
DPDK_CFLAGS+= -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2
DPDK_CFLAGS+= -DRTE_COMPILE_TIME_CPUFLAGS=RTE_CPUFLAG_SSE,RTE_CPUFLAG_SSE2,RTE_CPUFLAG_SSE3,RTE_CPUFLAG_SSSE3,RTE_CPUFLAG_SSE4_1,RTE_CPUFLAG_SSE4_2
DPDK_CFLAGS+= -I${FF_DPDK}/include
DPDK_CFLAGS= -Wall -Wno-deprecated-declarations -Werror $(shell $(PKGCONF) --cflags libdpdk)
KERNPREINCLUDES:= ${INCLUDES}
INCLUDES= -I${OVERRIDE_INCLUDES_ROOT} ${KERNPREINCLUDES}

18
lib/ff_dpdk_if.c
View File

@ -791,19 +791,9 @@ init_port_start(void)
//RSS reta update will failed when enable flow isolate
#ifndef FF_FLOW_ISOLATE
if (nb_queues > 1) {
/* set HW rss hash function to Toeplitz. */
if (!rte_eth_dev_filter_supported(port_id, RTE_ETH_FILTER_HASH)) {
struct rte_eth_hash_filter_info info = {0};
info.info_type = RTE_ETH_HASH_FILTER_GLOBAL_CONFIG;
info.info.global_conf.hash_func = RTE_ETH_HASH_FUNCTION_TOEPLITZ;
if (rte_eth_dev_filter_ctrl(port_id, RTE_ETH_FILTER_HASH,
RTE_ETH_FILTER_SET, &info) < 0) {
rte_exit(EXIT_FAILURE, "port[%d] set hash func failed\n",
port_id);
}
}
/*
* FIXME: modify RSS set to FDIR
*/
set_rss_table(port_id, dev_info.reta_size, nb_queues);
}
#endif
@ -1977,7 +1967,7 @@ ff_dpdk_run(loop_func_t loop, void *arg) {
sizeof(struct loop_routine), 0);
lr->loop = loop;
lr->arg = arg;
rte_eal_mp_remote_launch(main_loop, lr, CALL_MASTER);
rte_eal_mp_remote_launch(main_loop, lr, CALL_MAIN);
rte_eal_mp_wait_lcore();
rte_free(lr);
}

12
tools/README.md
View File

@ -2,6 +2,18 @@
Compile ff tools
# Upgrade pkg-config while version < 0.28
#wget https://pkg-config.freedesktop.org/releases/pkg-config-0.29.2.tar.gz
#tar xzvf pkg-config-0.29.2.tar.gz
#cd pkg-config-0.29.2
#./configure --with-internal-glib
#make
#make install
#mv /usr/bin/pkg-config /usr/bin/pkg-config.bak
#ln -s /usr/local/bin/pkg-config /usr/bin/pkg-config
export PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig:/usr/lib/pkgconfig
make
Install ff tools, all ff tools will be installed to `/usr/local/bin/f-stack/`, and some soft link will be created in `/usr/local/bin`, such as `ff_top`,`ff_traffic`, etc.

12
tools/compat/Makefile
View File

@ -1,7 +1,7 @@
TOPDIR?=${CURDIR}/../..
ifeq ($(FF_DPDK),)
FF_DPDK=${TOPDIR}/dpdk/x86_64-native-linuxapp-gcc
ifneq ($(shell pkg-config --exists libdpdk && echo 0),0)
$(error "no installation of DPDK found, maybe you shuld export environment variable `PKG_CONFIG_PATH`")
endif
TARGET=libffcompat.a
@ -13,11 +13,9 @@ ifneq (${MK_INET6_SUPPORT},"no")
CFLAGS+= -DINET6
endif
DPDK_CFLAGS= -g -Wall -Werror -Wno-deprecated-declarations -include ${FF_DPDK}/include/rte_config.h
DPDK_CFLAGS+= -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3
DPDK_CFLAGS+= -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2
DPDK_CFLAGS+= -DRTE_COMPILE_TIME_CPUFLAGS=RTE_CPUFLAG_SSE,RTE_CPUFLAG_SSE2,RTE_CPUFLAG_SSE3,RTE_CPUFLAG_SSSE3,RTE_CPUFLAG_SSE4_1,RTE_CPUFLAG_SSE4_2
DPDK_CFLAGS+= -I${FF_DPDK}/include
PKGCONF ?= pkg-config
DPDK_CFLAGS= -g -Wall -Werror -Wno-deprecated-declarations $(shell $(PKGCONF) --cflags libdpdk)
CFLAGS+= ${DPDK_CFLAGS} -I${CURDIR}/include
CFLAGS+= -I${TOPDIR}/lib -D__BSD_VISIBLE -DFSTACK

12
tools/libnetgraph/Makefile
View File

@ -3,8 +3,8 @@
TOPDIR?=${CURDIR}/../..
ifeq ($(FF_DPDK),)
FF_DPDK=${TOPDIR}/dpdk/x86_64-native-linuxapp-gcc
ifneq ($(shell pkg-config --exists libdpdk && echo 0),0)
$(error "no installation of DPDK found, maybe you shuld export environment variable `PKG_CONFIG_PATH`")
endif
PACKAGE=lib${LIB}
@ -14,11 +14,9 @@ MAN= netgraph.3
SHLIB_MAJOR= 4
DPDK_CFLAGS= -g -Wall -Werror -include ${FF_DPDK}/include/rte_config.h
DPDK_CFLAGS+= -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3
DPDK_CFLAGS+= -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2
DPDK_CFLAGS+= -DRTE_COMPILE_TIME_CPUFLAGS=RTE_CPUFLAG_SSE,RTE_CPUFLAG_SSE2,RTE_CPUFLAG_SSE3,RTE_CPUFLAG_SSSE3,RTE_CPUFLAG_SSE4_1,RTE_CPUFLAG_SSE4_2
DPDK_CFLAGS+= -I${FF_DPDK}/include
PKGCONF ?= pkg-config
DPDK_CFLAGS= -g -Wall -Werror $(shell $(PKGCONF) --cflags libdpdk)
CFLAGS+= ${DPDK_CFLAGS}
CFLAGS+= -I${TOPDIR}/lib

18
tools/prog.mk
View File

@ -27,6 +27,10 @@ ifndef TOPDIR
$(error TOPDIR must be defined.)
endif
ifneq ($(shell pkg-config --exists libdpdk && echo 0),0)
$(error "no installation of DPDK found, maybe you shuld export environment variable `PKG_CONFIG_PATH`")
endif
ifndef SRCS
ifdef PROG_CXX
SRCS= ${PROG}.cc
@ -41,30 +45,26 @@ HEADERS+= $(filter %.h,${SRCS})
OBJS+= $(patsubst %.c,%.o, $(filter %.c,${SRCS}))
OBJS+= $(patsubst %.cc,%.o, $(filter %.cc,${SRCS}))
ifeq ($(FF_DPDK),)
FF_DPDK=${TOPDIR}/dpdk/x86_64-native-linuxapp-gcc
endif
PKGCONF ?= pkg-config
FF_PROG_CFLAGS:= -g -Wall -Werror -DFSTACK -std=gnu99
FF_PROG_CFLAGS:= -g -Wall -Werror -DFSTACK -std=gnu99 $(shell $(PKGCONF) --cflags libdpdk)
FF_PROG_CFLAGS+= -I${TOPDIR}/lib -I${TOPDIR}/tools/compat
FF_PROG_CFLAGS+= -include${TOPDIR}/tools/compat/compat.h
FF_PROG_CFLAGS+= -I${TOPDIR}/tools/compat/include -D__BSD_VISIBLE
FF_PROG_CFLAGS+= -include ${FF_DPDK}/include/rte_config.h
FF_PROG_CFLAGS+= -I${FF_DPDK}/include
FF_PROG_LIBS:= -L${TOPDIR}/tools/compat -Wl,--whole-archive,-lffcompat,--no-whole-archive
FF_PROG_LIBS+= -L${FF_DPDK}/lib -Wl,--whole-archive,-ldpdk,--no-whole-archive
FF_PROG_LIBS+= $(shell $(PKGCONF) --static --libs libdpdk)
FF_PROG_LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -lpthread -lnuma
CFLAGS+= -Wno-unused-but-set-variable -Wno-unused-variable
CFLAGS+= ${FF_PROG_CFLAGS}
CXXFLAGS+= ${FF_PROG_CFLAGS}
LIBS+= ${FF_PROG_LIBS}
CFLAGS+= $(foreach n,${LIBADD},-I${TOPDIR}/tools/lib${n})
LIBS+= $(foreach n,${LIBADD},-L${TOPDIR}/tools/lib${n} -l${n})
LIBS+= ${FF_PROG_LIBS}
CLEANFILES+= ${PROGDIR}/${PROG} ${OBJS}
${PROG}: ${HEADERS} ${OBJS}