From 865ab0ed63abdb3f50879b75af8277177c230cee Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 6 Apr 2023 15:02:39 +0800 Subject: [PATCH 01/43] Add adapter for LD_PRELOAD, TBC. --- adapter/Makefile | 75 ++ adapter/ff_adapter.h | 21 + adapter/ff_declare_syscalls.h | 31 + adapter/ff_hook_syscall.c | 1538 +++++++++++++++++++++++++++++++++ adapter/ff_hook_syscall.h | 8 + adapter/ff_linux_syscall.c | 236 +++++ adapter/ff_linux_syscall.h | 13 + adapter/ff_so_zone.c | 170 ++++ adapter/ff_socket_ops.c | 461 ++++++++++ adapter/ff_socket_ops.h | 103 +++ adapter/ff_sysproto.h | 197 +++++ adapter/fstack.c | 36 + 12 files changed, 2889 insertions(+) create mode 100644 adapter/Makefile create mode 100644 adapter/ff_adapter.h create mode 100644 adapter/ff_declare_syscalls.h create mode 100644 adapter/ff_hook_syscall.c create mode 100644 adapter/ff_hook_syscall.h create mode 100644 adapter/ff_linux_syscall.c create mode 100644 adapter/ff_linux_syscall.h create mode 100644 adapter/ff_so_zone.c create mode 100644 adapter/ff_socket_ops.c create mode 100644 adapter/ff_socket_ops.h create mode 100644 adapter/ff_sysproto.h create mode 100644 adapter/fstack.c diff --git a/adapter/Makefile b/adapter/Makefile new file mode 100644 index 000000000..ccbc5b72b --- /dev/null +++ b/adapter/Makefile @@ -0,0 +1,75 @@ +TOPDIR=.. + +ifeq ($(FF_PATH),) + FF_PATH=${TOPDIR} +endif + +ifneq ($(shell pkg-config --exists libdpdk && echo 0),0) + $(error "No installation of DPDK found, maybe you should export environment variable `PKG_CONFIG_PATH`") +endif + +DEBUG=-O0 -gdwarf-2 -g3 -DDEBUG + +PKGCONF ?= pkg-config + +ifndef DEBUG + CFLAGS+= -g -O2 +else + CFLAGS+= ${DEBUG} +endif +CFLAGS += -fPIC -Wall -Werror $(shell $(PKGCONF) --cflags libdpdk) + +INCLUDES= -I. -I${FF_PATH}/lib + +FF_LIBS= -L${FF_PATH}/lib -Wl,--whole-archive,-lfstack,--no-whole-archive + +DPDK_LIBS+= $(shell $(PKGCONF) --static --libs libdpdk) +DPDK_LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -pthread -lnuma + +#DPDK_CFLAGS= -Wall -Werror -include ${FF_DPDK}/include/rte_config.h +#DPDK_CFLAGS+= -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 +#DPDK_CFLAGS+= -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 +#DPDK_CFLAGS+= -DRTE_COMPILE_TIME_CPUFLAGS=RTE_CPUFLAG_SSE,RTE_CPUFLAG_SSE2,RTE_CPUFLAG_SSE3,RTE_CPUFLAG_SSSE3,RTE_CPUFLAG_SSE4_1,RTE_CPUFLAG_SSE4_2 +#DPDK_CFLAGS+= -I${FF_DPDK}/include + +CFLAGS+= ${INCLUDES} ${DPDK_CFLAGS} + +FSTACK_LIBS= ${FF_LIBS} ${DPDK_LIBS} +FF_SYSCALL_LIBS= + +FSTACK_SRCS= \ + fstack.c \ + ff_so_zone.c \ + ff_socket_ops.c + +FF_SYSCALL_SRCS= \ + ff_so_zone.c \ + ff_hook_syscall.c \ + ff_linux_syscall.c + +FSTACK_OBJS= $(patsubst %.c,%.o,${FSTACK_SRCS}) + +FF_SYSCALL_OBJS= $(patsubst %.c,%.o,${FF_SYSCALL_SRCS}) + +#TARGET= fstack libff_syscall.a +TARGET= fstack libff_syscall.so + +all: ${TARGET} + +fstack: ${FSTACK_OBJS} + cc -o $@ $^ ${FSTACK_LIBS} + +#libff_syscall.a: ${FF_SYSCALL_OBJS} +libff_syscall.so: ${FF_SYSCALL_OBJS} + #ar -rcs $@ $^ ${FF_SYSCALL_LIBS} + ${CC} -fPIC -shared -o $@ $^ ${FF_SYSCALL_LIBS} + +${FSTACK_OBJS}: %.o: %.c + ${CC} -c $(CFLAGS) ${PROF} $< + +${FF_SYSCALL_OBJS}: %.o: %.c + ${CC} -c $(CFLAGS) ${PROF} $< + +.PHONY: clean +clean: + rm -f *.o ${TARGET} diff --git a/adapter/ff_adapter.h b/adapter/ff_adapter.h new file mode 100644 index 000000000..39ee00cc4 --- /dev/null +++ b/adapter/ff_adapter.h @@ -0,0 +1,21 @@ +#ifndef _FF_ADAPTER_H +#define _FF_ADAPTER_H + +/* socket.h */ +//#define SOCK_CLOEXEC 0x10000000 +//#define SOCK_NONBLOCK 0x20000000 +#define SOCK_FSTACK 0x01000000 +#define SOCK_KERNEL 0x02000000 + +int ff_adapter_init(); +//int __attribute__((constructor)) ff_adapter_init(int argc, char * const argv[]); + +/*- + * Verify whether the socket is supported by fstack or not. + */ +int fstack_territory(int domain, int type, int protocol); + +/* Tell whether a 'sockfd' belongs to fstack. */ +int is_fstack_fd(int fd); + +#endif diff --git a/adapter/ff_declare_syscalls.h b/adapter/ff_declare_syscalls.h new file mode 100644 index 000000000..28d3b9808 --- /dev/null +++ b/adapter/ff_declare_syscalls.h @@ -0,0 +1,31 @@ +FF_SYSCALL_DECL(int, socket, (int, int, int)); +FF_SYSCALL_DECL(int, bind, (int, const struct sockaddr *, socklen_t)); +FF_SYSCALL_DECL(int, listen, (int, int)); +FF_SYSCALL_DECL(int, shutdown, (int, int)); +FF_SYSCALL_DECL(int, getsockname, (int, struct sockaddr *, socklen_t *)); +FF_SYSCALL_DECL(int, getpeername, (int, struct sockaddr *, socklen_t *)); +FF_SYSCALL_DECL(int, getsockopt, (int, int, int, void *, socklen_t *)); +FF_SYSCALL_DECL(int, setsockopt, (int, int, int, const void *, socklen_t)); +FF_SYSCALL_DECL(int, accept, (int, struct sockaddr *, socklen_t *)); +FF_SYSCALL_DECL(int, accept4,(int, struct sockaddr *, socklen_t *, int flags)); +FF_SYSCALL_DECL(int, connect, (int, const struct sockaddr *, socklen_t)); +FF_SYSCALL_DECL(ssize_t, recv, (int, void *, size_t, int)); +FF_SYSCALL_DECL(ssize_t, send, (int, const void *, size_t, int)); +FF_SYSCALL_DECL(ssize_t, read, (int, void *, size_t)); +FF_SYSCALL_DECL(ssize_t, write, (int, const void *, size_t)); +FF_SYSCALL_DECL(ssize_t, writev, (int, const struct iovec *, int)); +FF_SYSCALL_DECL(ssize_t, readv, (int, const struct iovec *, int)); +FF_SYSCALL_DECL(ssize_t, sendto, (int, const void *, size_t, int, + const struct sockaddr *, socklen_t)); +FF_SYSCALL_DECL(ssize_t, recvfrom, (int, void *, size_t, int, + struct sockaddr *, socklen_t *)); +FF_SYSCALL_DECL(ssize_t, sendmsg, (int, const struct msghdr *, int flags)); +FF_SYSCALL_DECL(ssize_t, recvmsg, (int, struct msghdr *, int flags)); +FF_SYSCALL_DECL(int, close, (int)); +FF_SYSCALL_DECL(int, ioctl, (int, unsigned long, unsigned long)); +FF_SYSCALL_DECL(int, fcntl, (int, int, unsigned long)); +FF_SYSCALL_DECL(int, epoll_create, (int)); +FF_SYSCALL_DECL(int, epoll_ctl, (int, int, int, struct epoll_event *)); +FF_SYSCALL_DECL(int, epoll_wait, (int, struct epoll_event *, int, int)); +FF_SYSCALL_DECL(pid_t, fork, (void)); +#undef FF_SYSCALL_DECL diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c new file mode 100644 index 000000000..26bcdcd9d --- /dev/null +++ b/adapter/ff_hook_syscall.c @@ -0,0 +1,1538 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "ff_config.h" +#include "ff_socket_ops.h" +#include "ff_sysproto.h" +#include "ff_event.h" +#include "ff_hook_syscall.h" +#include "ff_linux_syscall.h" +#include "ff_adapter.h" + +#define NS_PER_SECOND 1000000000 + +#ifndef likely +#define likely(x) __builtin_expect((x),1) +#endif + +#ifndef unlikely +#define unlikely(x) __builtin_expect((x),0) +#endif + +#define strong_alias(name, aliasname) \ + extern __typeof (name) aliasname __attribute__ ((alias (#name))); + +#undef FF_SYSCALL_DECL +#define FF_SYSCALL_DECL(ret, fn, args) strong_alias(ff_hook_##fn, fn) +#include + +#define share_mem_alloc(size) rte_malloc(NULL, (size), 0) +#define share_mem_free(addr) rte_free((addr)) + +#define CHECK_FD_OWNERSHIP(name, args) \ +{ \ + if (!is_fstack_fd(fd)) { \ + return ff_linux_##name args; \ + } \ + fd = restore_fstack_fd(fd); \ +} + +#define DEFINE_REQ_ARGS(name) \ + struct ff_##name##_args *args; \ + int ret = -1; \ + size_t size = sizeof(struct ff_##name##_args); \ + args = share_mem_alloc(size); \ + if (args == NULL) { \ + errno = ENOMEM; \ + return ret; \ + } + +#define ACQUIRE_ZONE_LOCK(exp) do { \ + while (1) { \ + rte_spinlock_lock(&sc->lock); \ + if (sc->status == exp) { \ + break; \ + } \ + rte_spinlock_unlock(&sc->lock); \ + rte_pause(); \ + } \ +} while (0) + +#define RELEASE_ZONE_LOCK(s) do { \ + sc->status = s; \ + rte_spinlock_unlock(&sc->lock); \ +} while (0) + +#define SYSCALL(op, arg) do { \ + ACQUIRE_ZONE_LOCK(FF_SC_IDLE); \ + sc->ops = (op); \ + sc->args = (arg); \ + RELEASE_ZONE_LOCK(FF_SC_REQ); \ + ACQUIRE_ZONE_LOCK(FF_SC_REP); \ + ret = sc->result; \ + if (ret < 0) { \ + errno = sc->error; \ + } \ + RELEASE_ZONE_LOCK(FF_SC_IDLE); \ +} while (0) + +#define RETURN() do { \ + share_mem_free(args); \ + return ret; \ +} while (0) + +#define RETURN_ERROR(err) do { \ + errno = err; \ + share_mem_free(args); \ + return ret; \ +} while (0) + +/* per thread separate initialization dpdk lib and attach sc */ +static __thread int inited = 0; +static __thread struct ff_so_context *sc; + +/* process-level initialization flag */ +static int proc_inited = 0; + +/* Use from lcore 2 by default, can set by environment variable FF_INITIAL_LCORE_ID */ +#define INITIAL_LCORE_ID_DEFAULT 0x4 /* lcore 2 */ +#define INITIAL_LCORE_ID_MAX 0x4000000000000 /* lcore 50 */ +#define FF_INITIAL_LCORE_ID_STR "FF_INITIAL_LCORE_ID" +static uint64_t initial_lcore_id = INITIAL_LCORE_ID_DEFAULT; +static int worker_id = 0; +rte_spinlock_t worker_id_lock; + +/* The num of F-Stack process instance, default 1 */ +#define NB_FSTACK_INSTANCE_DEFAULT 1 +#define FF_NB_FSTACK_INSTANCE_STR "FF_NB_FSTACK_INSTANCE" +static int nb_procs = NB_FSTACK_INSTANCE_DEFAULT; + +#define FF_KERNEL_MAX_FD_DEFAULT 1024 +static int ff_kernel_max_fd = FF_KERNEL_MAX_FD_DEFAULT; + +static inline int convert_fstack_fd(int sockfd) { + return sockfd + ff_kernel_max_fd; +} + +/* Restore socket fd. */ +static inline int restore_fstack_fd(int sockfd) { + if(sockfd < ff_kernel_max_fd) { + return sockfd; + } + + return sockfd - ff_kernel_max_fd; +} + +int is_fstack_fd(int sockfd) { + if (unlikely(inited == 0/* && ff_adapter_init() < 0*/)) { + return 0; + } + + /* FIXED ME: ff_linux_socket not limit fd < ff_kernel_max_fd, may be Misjudgment */ + return sockfd >= ff_kernel_max_fd; +} + +int +fstack_territory(int domain, int type, int protocol) +{ + /* Remove creation flags */ + type &= ~SOCK_CLOEXEC; + type &= ~SOCK_NONBLOCK; + type &= ~SOCK_FSTACK; + type &= ~SOCK_KERNEL; + + if ((AF_INET != domain && AF_INET6 != domain) || (SOCK_STREAM != type && + SOCK_DGRAM != type)) { + return 0; + } + + return 1; +} + +/* + * APP need set type |= SOCK_FSTACK + */ +int +ff_hook_socket(int domain, int type, int protocol) +{ + DEBUG_LOG("ff_hook_socket, domain:%d, type:%d, protocol:%d\n", domain, type, protocol); + if (unlikely(fstack_territory(domain, type, protocol) == 0)) { + return ff_linux_socket(domain, type, protocol); + } + + if (unlikely(type & SOCK_KERNEL) && !(type & SOCK_FSTACK)) { + type &= ~SOCK_KERNEL; + return ff_linux_socket(domain, type, protocol); + } + + if (unlikely(inited == 0 && ff_adapter_init() < 0)) { + return ff_linux_socket(domain, type, protocol); + } + + type &= ~SOCK_FSTACK; + + DEFINE_REQ_ARGS(socket); + + args->domain = domain; + args->type = type; + args->protocol = protocol; + + SYSCALL(FF_SO_SOCKET, args); + + if (ret >= 0) { + ret = convert_fstack_fd(ret); + } + + RETURN(); +} + +int +ff_hook_bind(int fd, const struct sockaddr *addr, + socklen_t addrlen) +{ + if (addr == NULL) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(bind, (fd, addr, addrlen)); + + DEFINE_REQ_ARGS(bind); + struct sockaddr *sh_addr; + + sh_addr = share_mem_alloc(addrlen); + if (sh_addr == NULL) { + RETURN_ERROR(ENOMEM); + } + rte_memcpy(sh_addr, addr, addrlen); + + args->fd = fd; + args->addr = sh_addr; + args->addrlen = addrlen; + + SYSCALL(FF_SO_BIND, args); + + share_mem_free(sh_addr); + RETURN(); +} + +int +ff_hook_listen(int fd, int backlog) +{ + CHECK_FD_OWNERSHIP(listen, (fd, backlog)); + + DEFINE_REQ_ARGS(listen); + + args->fd = fd; + args->backlog = backlog; + + SYSCALL(FF_SO_LISTEN, args); + + RETURN(); +} + +int +ff_hook_shutdown(int fd, int how) +{ + CHECK_FD_OWNERSHIP(shutdown, (fd, how)); + + DEFINE_REQ_ARGS(shutdown); + + args->fd = fd; + args->how = how; + + SYSCALL(FF_SO_SHUTDOWN, args); + + RETURN(); +} + +int +ff_hook_getsockname(int fd, struct sockaddr *name, + socklen_t *namelen) +{ + if (name == NULL || namelen == NULL) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(getsockname, (fd, name, namelen)); + + DEFINE_REQ_ARGS(getsockname); + struct sockaddr *sh_name; + socklen_t *sh_namelen; + + sh_name = share_mem_alloc(*namelen); + if (sh_name == NULL) { + RETURN_ERROR(ENOMEM); + } + + sh_namelen = share_mem_alloc(sizeof(socklen_t)); + if (sh_namelen == NULL) { + share_mem_free(sh_name); + RETURN_ERROR(ENOMEM); + } + *sh_namelen = *namelen; + + args->fd = fd; + args->name = sh_name; + args->namelen = sh_namelen; + + SYSCALL(FF_SO_GETSOCKNAME, args); + + if (ret == 0) { + socklen_t cplen = *namelen ? *sh_namelen > *namelen + : *sh_namelen; + rte_memcpy(name, sh_name, cplen); + *namelen = *sh_namelen; + } + + share_mem_free(sh_name); + share_mem_free(sh_namelen); + RETURN(); +} + +int +ff_hook_getpeername(int fd, struct sockaddr *name, + socklen_t *namelen) +{ + if (name == NULL || namelen == NULL) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(getpeername, (fd, name, namelen)); + + DEFINE_REQ_ARGS(getpeername); + struct sockaddr *sh_name; + socklen_t *sh_namelen; + + sh_name = share_mem_alloc(*namelen); + if (sh_name == NULL) { + RETURN_ERROR(ENOMEM); + } + + sh_namelen = share_mem_alloc(sizeof(socklen_t)); + if (sh_namelen == NULL) { + share_mem_free(sh_name); + RETURN_ERROR(ENOMEM); + } + *sh_namelen = *namelen; + + args->fd = fd; + args->name = sh_name; + args->namelen = sh_namelen; + + SYSCALL(FF_SO_GETPEERNAME, args); + + if (ret == 0) { + socklen_t cplen = *namelen ? *sh_namelen > *namelen + : *sh_namelen; + rte_memcpy(name, sh_name, cplen); + *namelen = *sh_namelen; + } + + share_mem_free(sh_name); + share_mem_free(sh_namelen); + RETURN(); +} + +int +ff_hook_getsockopt(int fd, int level, int optname, + void *optval, socklen_t *optlen) +{ + if (optlen == NULL) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(getsockopt, (fd, level, optname, + optval, optlen)); + + DEFINE_REQ_ARGS(getsockopt); + void *sh_optval = NULL; + socklen_t *sh_optlen; + + if (optval != NULL) { + sh_optval = share_mem_alloc(*optlen); + if (sh_optval == NULL) { + RETURN_ERROR(ENOMEM); + } + } + + sh_optlen = share_mem_alloc(sizeof(socklen_t)); + if (sh_optlen == NULL) { + if (sh_optval) { + share_mem_free(sh_optval); + } + + RETURN_ERROR(ENOMEM); + } + *sh_optlen = *optlen; + + args->fd = fd; + args->level = level; + args->name = optname; + args->optval = sh_optval; + args->optlen = sh_optlen; + + SYSCALL(FF_SO_GETSOCKOPT, args); + + if (ret == 0) { + if (optval) { + rte_memcpy(optval, sh_optval, *sh_optlen); + share_mem_free(sh_optval); + } + *optlen = *sh_optlen; + } + + share_mem_free(sh_optlen); + RETURN(); +} + +int +ff_hook_setsockopt(int fd, int level, int optname, + const void *optval, socklen_t optlen) +{ + if (optval == NULL && optlen != 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(setsockopt, (fd, level, optname, + optval, optlen)); + + DEFINE_REQ_ARGS(setsockopt); + void *sh_optval = NULL; + + if (optval != NULL) { + sh_optval = share_mem_alloc(optlen); + if (sh_optval == NULL) { + RETURN_ERROR(ENOMEM); + } + } + + args->fd = fd; + args->level = level; + args->name = optname; + args->optval = sh_optval; + args->optlen = optlen; + + SYSCALL(FF_SO_SETSOCKOPT, args); + + if (sh_optval) { + share_mem_free(sh_optval); + } + + RETURN(); +} + +int +ff_hook_accept(int fd, struct sockaddr *addr, socklen_t *addrlen) +{ + if ((addr == NULL && addrlen != NULL) || + (addr != NULL && addrlen == NULL)) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(accept, (fd, addr, addrlen)); + + DEFINE_REQ_ARGS(accept); + struct sockaddr *sh_addr = NULL; + socklen_t *sh_addrlen = NULL; + + if (addr != NULL) { + sh_addr = share_mem_alloc(*addrlen); + if (sh_addr == NULL) { + RETURN_ERROR(ENOMEM); + } + + sh_addrlen = share_mem_alloc(sizeof(socklen_t)); + if (sh_addrlen == NULL) { + share_mem_free(sh_addr); + RETURN_ERROR(ENOMEM); + } + *sh_addrlen = *addrlen; + } + + args->fd = fd; + args->addr = sh_addr; + args->addrlen = sh_addrlen; + + SYSCALL(FF_SO_ACCEPT, args); + + if (ret > 0) { + ret = convert_fstack_fd(ret); + } + + if (addr) { + if (ret > 0) { + socklen_t cplen = *sh_addrlen > *addrlen ? + *addrlen : *sh_addrlen; + rte_memcpy(addr, sh_addr, cplen); + *addrlen = *sh_addrlen; + } + share_mem_free(sh_addr); + share_mem_free(sh_addrlen); + } + + RETURN(); +} + +int +ff_hook_accept4(int fd, struct sockaddr *addr, + socklen_t *addrlen, int flags) +{ + CHECK_FD_OWNERSHIP(accept4, (fd, addr, addrlen, flags)); + + errno = ENOSYS; + return -1; +} + +int +ff_hook_connect(int fd, const struct sockaddr *addr, + socklen_t addrlen) +{ + if (addr == NULL) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(connect, (fd, addr, addrlen)); + + DEFINE_REQ_ARGS(connect); + struct sockaddr *sh_addr; + + sh_addr = share_mem_alloc(addrlen); + if (sh_addr == NULL) { + RETURN_ERROR(ENOMEM); + } + rte_memcpy(sh_addr, addr, addrlen); + + args->fd = fd; + args->addr = sh_addr; + args->addrlen = addrlen; + + SYSCALL(FF_SO_CONNECT, args); + + share_mem_free(sh_addr); + RETURN(); +} + +ssize_t +ff_hook_recv(int fd, void *buf, size_t len, int flags) +{ + return ff_hook_recvfrom(fd, buf, len, flags, NULL, NULL); +} + +ssize_t +ff_hook_recvfrom(int fd, void *buf, size_t len, int flags, + struct sockaddr *from, socklen_t *fromlen) +{ + if (buf == NULL || len == 0) { + errno = EINVAL; + return -1; + } + + if ((from == NULL && fromlen != NULL) || + (from != NULL && fromlen == NULL)) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(recvfrom, (fd, buf, len, flags, from, fromlen)); + + DEFINE_REQ_ARGS(recvfrom); + void *sh_buf; + struct sockaddr *sh_from = NULL; + socklen_t *sh_fromlen = NULL; + + if (from != NULL) { + sh_from = share_mem_alloc(*fromlen); + if (sh_from == NULL) { + RETURN_ERROR(ENOMEM); + } + + sh_fromlen = share_mem_alloc(sizeof(socklen_t)); + if (sh_fromlen == NULL) { + share_mem_free(sh_from); + RETURN_ERROR(ENOMEM); + } + } + + sh_buf = share_mem_alloc(len); + if (sh_buf == NULL) { + if (sh_from) { + share_mem_free(sh_from); + } + if (sh_fromlen) { + share_mem_free(sh_fromlen); + } + RETURN_ERROR(ENOMEM); + } + + args->fd = fd; + args->buf = sh_buf; + args->len = len; + args->flags = flags; + args->from = sh_from; + args->fromlen = sh_fromlen; + + SYSCALL(FF_SO_RECVFROM, args); + + if (ret >= 0) { + rte_memcpy(buf, sh_buf, ret); + if (from) { + socklen_t cplen = *fromlen ? *sh_fromlen > *fromlen + : *sh_fromlen; + rte_memcpy(from, sh_from, cplen); + *fromlen = *sh_fromlen; + } + } + + if (from) { + share_mem_free(sh_from); + share_mem_free(sh_fromlen); + } + + share_mem_free(sh_buf); + RETURN(); +} + +static void +iovec_share_free(struct iovec *iov, int iovcnt) +{ + int i; + + if (iov == NULL) { + return; + } + + for (i = 0; i < iovcnt; i++) { + if (iov[i].iov_base) { + share_mem_free(iov[i].iov_base); + } + } + + share_mem_free(iov); +} + +static struct iovec * +iovec_share_alloc(const struct iovec *iov, int iovcnt) +{ + struct iovec *sh_iov; + int i; + + if (iov == NULL || iovcnt == 0) { + return NULL; + } + + sh_iov = share_mem_alloc(sizeof(struct iovec) * iovcnt); + if (sh_iov == NULL) { + return NULL; + } + + for (i = 0; i < iovcnt; i++) { + sh_iov[i].iov_len = iov[i].iov_len; + void *iov_base = share_mem_alloc(sh_iov[i].iov_len); + sh_iov[i].iov_base = iov_base; + + if (iov_base == NULL) { + goto ERROR; + } + } + + return sh_iov; + +ERROR: + iovec_share_free(sh_iov, iovcnt); + return NULL; +} + +static void +iovec_local2share(struct iovec *share, const struct iovec *local, + int iovcnt) +{ + int i; + + if (share == NULL || local == NULL || iovcnt == 0) { + return; + } + + for (i = 0; i < iovcnt; i++) { + assert(share[i].iov_len == local[i].iov_len); + + rte_memcpy(share[i].iov_base, local[i].iov_base, + share[i].iov_len); + } +} + +static void +iovec_share2local(struct iovec *share, + const struct iovec *local, int iovcnt, + ssize_t total, int copy) +{ + int i; + for (i = 0; i < iovcnt && total > 0; i++) { + ssize_t count = local[i].iov_len; + if (total <= count) { + count = total; + } + + share[i].iov_base = + (char *)share[i].iov_base - count; + share[i].iov_len += count; + + if (copy) { + rte_memcpy(local[i].iov_base, + share[i].iov_base, count); + } + + total -= count; + } +} + +static void +msghdr_share_free(struct msghdr *msg) +{ + if (msg == NULL) { + return; + } + + if (msg->msg_name) { + share_mem_free(msg->msg_name); + } + + if (msg->msg_control) { + share_mem_free(msg->msg_control); + } + + if (msg->msg_iov) { + iovec_share_free(msg->msg_iov, msg->msg_iovlen); + } + + share_mem_free(msg); +} + +static struct msghdr * +msghdr_share_alloc(const struct msghdr *msg) +{ + struct msghdr *hdr; + + if (msg == NULL) { + return NULL; + } + + hdr = share_mem_alloc(sizeof(struct msghdr)); + if (hdr == NULL) { + return NULL; + } + memset(hdr, 0, sizeof(struct msghdr)); + + hdr->msg_namelen = msg->msg_namelen; + hdr->msg_iovlen = msg->msg_iovlen; + hdr->msg_controllen = msg->msg_controllen; + hdr->msg_flags = msg->msg_flags; + + if (msg->msg_name) { + hdr->msg_name = share_mem_alloc(hdr->msg_namelen); + if (hdr->msg_name == NULL) { + goto ERROR; + } + } + + if (msg->msg_control) { + hdr->msg_control = share_mem_alloc(hdr->msg_controllen); + if (hdr->msg_control == NULL) { + goto ERROR; + } + } + + hdr->msg_iov = iovec_share_alloc(msg->msg_iov, hdr->msg_iovlen); + if (hdr->msg_iov == NULL) { + goto ERROR; + } + + return hdr; + +ERROR: + msghdr_share_free(hdr); + return NULL; +} + +static void +msghdr_share_memcpy(const struct msghdr *dst, const struct msghdr *src) +{ + if (dst == NULL || src == NULL) { + return; + } + + assert((dst->msg_name == NULL && src->msg_name == NULL) + || (dst->msg_name != NULL && src->msg_name != NULL)); + assert(dst->msg_namelen == src->msg_namelen); + + assert((dst->msg_control == NULL && src->msg_control == NULL) + || (dst->msg_control != NULL && src->msg_control != NULL)); + assert(dst->msg_controllen == src->msg_controllen); + + if (dst->msg_name) { + rte_memcpy(dst->msg_name, src->msg_name, src->msg_namelen); + } + + if (dst->msg_control) { + rte_memcpy(dst->msg_control, src->msg_control, + src->msg_controllen); + } + + //do iovec_memcpy by caller. +} + +ssize_t +ff_hook_recvmsg(int fd, struct msghdr *msg, int flags) +{ + if (msg == NULL || msg->msg_iov == NULL || + msg->msg_iovlen == 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(recvmsg, (fd, msg, flags)); + + DEFINE_REQ_ARGS(recvmsg); + struct msghdr *sh_msg; + + sh_msg = msghdr_share_alloc(msg); + if (sh_msg == NULL) { + RETURN_ERROR(ENOMEM); + } + + args->fd = fd; + args->msg = sh_msg; + args->flags = flags; + + SYSCALL(FF_SO_RECVMSG, args); + + if (ret >= 0) { + msghdr_share_memcpy(msg, sh_msg); + if (ret > 0) { + iovec_share2local(sh_msg->msg_iov, + msg->msg_iov, msg->msg_iovlen, + ret, 1); + } + } + + msghdr_share_free(sh_msg); + RETURN(); +} + +ssize_t +ff_hook_read(int fd, void *buf, size_t len) +{ + if (buf == NULL || len == 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(read, (fd, buf, len)); + + DEFINE_REQ_ARGS(read); + void *sh_buf; + + sh_buf = share_mem_alloc(len); + if (sh_buf == NULL) { + RETURN_ERROR(ENOMEM); + } + + args->fd = fd; + args->buf = sh_buf; + args->len = len; + + SYSCALL(FF_SO_READ, args); + + if (ret > 0) { + rte_memcpy(buf, sh_buf, ret); + } + + share_mem_free(sh_buf); + RETURN(); +} + +ssize_t +ff_hook_readv(int fd, const struct iovec *iov, int iovcnt) +{ + if (iov == NULL || iovcnt == 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(readv, (fd, iov, iovcnt)); + + DEFINE_REQ_ARGS(readv); + struct iovec *sh_iov; + + sh_iov = iovec_share_alloc(iov, iovcnt); + if (sh_iov == NULL) { + RETURN_ERROR(ENOMEM); + } + + args->fd = fd; + args->iov = sh_iov; + args->iovcnt = iovcnt; + + SYSCALL(FF_SO_READV, args); + + if (ret > 0) { + iovec_share2local(sh_iov, iov, iovcnt, ret, 1); + } + + iovec_share_free(sh_iov, iovcnt); + RETURN(); +} + +ssize_t +ff_hook_sendto(int fd, const void *buf, size_t len, int flags, + const struct sockaddr *to, socklen_t tolen) +{ + if (buf == NULL || len == 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(sendto, (fd, buf, len, flags, to, tolen)); + + DEFINE_REQ_ARGS(sendto); + void *sh_buf; + void *sh_to = NULL; + + sh_buf = share_mem_alloc(len); + if (sh_buf == NULL) { + RETURN_ERROR(ENOMEM); + } + rte_memcpy(sh_buf, buf, len); + + if (to) { + sh_to = share_mem_alloc(tolen); + if (sh_to == NULL) { + share_mem_free(sh_buf); + RETURN_ERROR(ENOMEM); + } + rte_memcpy(sh_to, to, tolen); + } + + args->fd = fd; + args->buf = sh_buf; + args->len = len; + args->flags = flags; + args->to = sh_to; + args->tolen = tolen; + + SYSCALL(FF_SO_SENDTO, args); + + share_mem_free(sh_buf); + if (sh_to) { + share_mem_free(sh_to); + } + RETURN(); +} + +ssize_t +ff_hook_sendmsg(int fd, const struct msghdr *msg, int flags) +{ + if (msg == NULL || msg->msg_iov == NULL || + msg->msg_iovlen == 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(sendmsg, (fd, msg, flags)); + + DEFINE_REQ_ARGS(sendmsg); + struct msghdr *sh_msg; + + sh_msg = msghdr_share_alloc(msg); + if (sh_msg == NULL) { + RETURN_ERROR(ENOMEM); + } + msghdr_share_memcpy(sh_msg, msg); + iovec_local2share(sh_msg->msg_iov, + msg->msg_iov, msg->msg_iovlen); + + args->fd = fd; + args->msg = sh_msg; + args->flags = flags; + + SYSCALL(FF_SO_SENDMSG, args); + + if (ret > 0) { + iovec_share2local(sh_msg->msg_iov, + msg->msg_iov, msg->msg_iovlen, + ret, 0); + } + + msghdr_share_free(sh_msg); + RETURN(); +} + +ssize_t +ff_hook_send(int fd, const void *buf, size_t len, int flags) +{ + return ff_hook_sendto(fd, buf, len, flags, NULL, 0); +} + +ssize_t +ff_hook_write(int fd, const void *buf, size_t len) +{ + if (buf == NULL || len == 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(write, (fd, buf, len)); + + DEFINE_REQ_ARGS(write); + void *sh_buf; + + sh_buf = share_mem_alloc(len); + if (sh_buf == NULL) { + RETURN_ERROR(ENOMEM); + } + rte_memcpy(sh_buf, buf, len); + + args->fd = fd; + args->buf = sh_buf; + args->len = len; + + SYSCALL(FF_SO_WRITE, args); + + share_mem_free(sh_buf); + RETURN(); +} + +ssize_t +ff_hook_writev(int fd, const struct iovec *iov, int iovcnt) +{ + if (iov == NULL || iovcnt == 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(writev, (fd, iov, iovcnt)); + + DEFINE_REQ_ARGS(writev); + struct iovec *sh_iov; + + sh_iov = iovec_share_alloc(iov, iovcnt); + if (sh_iov == NULL) { + RETURN_ERROR(ENOMEM); + } + iovec_local2share(sh_iov, iov, iovcnt); + + args->fd = fd; + args->iov = sh_iov; + args->iovcnt = iovcnt; + + SYSCALL(FF_SO_WRITEV, args); + + if (ret > 0) { + iovec_share2local(sh_iov, iov, iovcnt, ret, 0); + } + + iovec_share_free(sh_iov, iovcnt); + RETURN(); +} + +int +ff_hook_close(int fd) +{ + CHECK_FD_OWNERSHIP(close, (fd)); + + DEFINE_REQ_ARGS(close); + + args->fd = fd; + + SYSCALL(FF_SO_CLOSE, args); + + RETURN(); +} + +int +ff_hook_ioctl(int fd, unsigned long req, unsigned long data) +{ + #ifndef FIOASYNC + #define FIOASYNC 0x5452 + #endif + #ifndef FIONBIO + #define FIONBIO 0x5421 + #endif + + if (req != FIOASYNC && req != FIONBIO) { + errno = ENOTSUP; + return -1; + } + + CHECK_FD_OWNERSHIP(ioctl, (fd, req, data)); + + DEFINE_REQ_ARGS(ioctl); + unsigned long *sh_data; + + sh_data = share_mem_alloc(sizeof(int)); + if (sh_data == NULL) { + RETURN_ERROR(ENOMEM); + } + *sh_data = *((int *)data); + + args->fd = fd; + args->com = req; + args->data = sh_data; + + SYSCALL(FF_SO_IOCTL, args); + + if (ret == 0) { + *((int *)data) = *sh_data; + } + + RETURN(); +} + +int +ff_hook_fcntl(int fd, int cmd, unsigned long data) +{ + CHECK_FD_OWNERSHIP(fcntl, (fd, cmd, data)); + + DEFINE_REQ_ARGS(fcntl); + + args->fd = fd; + args->cmd = cmd; + args->data = data; + + SYSCALL(FF_SO_FCNTL, args); + + RETURN(); +} + +/* + * Use F-Stack stack by default. + * + * If fdsize set SOCK_KERNEL(0x01000000) and not set SOCK_FSTACK(0x02000000), means use kernel stack. + * And the max fdsize shoud be <= (SOCK_KERNEL - 1). + * + * If fdsize set [1, 16], means use kernel stack, need to consider a better implementation. + */ +int +ff_hook_epoll_create(int fdsize) +{ + DEBUG_LOG("ff_hook_epoll_create, fdsize:%d\n", fdsize); + if (inited == 0 || ((fdsize & SOCK_KERNEL) && !(fdsize & SOCK_FSTACK)) || (fdsize >= 1 && fdsize <= 16)) { + fdsize &= ~SOCK_KERNEL; + return ff_linux_epoll_create(fdsize); + } + + DEFINE_REQ_ARGS(epoll_create); + + args->size = size; + + SYSCALL(FF_SO_EPOLL_CREATE, args); + + if (ret >= 0) { + ret = convert_fstack_fd(ret); + } + + RETURN(); +} + +int +ff_hook_epoll_ctl(int epfd, int op, int fd, + struct epoll_event *event) +{ + DEBUG_LOG("ff_hook_epoll_ctl, epfd:%d, op:%d, fd:%d\n", epfd, op, fd); + CHECK_FD_OWNERSHIP(epoll_ctl, (epfd, op, fd, event)); + + DEFINE_REQ_ARGS(epoll_ctl); + struct epoll_event *sh_event = NULL; + + if ((!event && op != EPOLL_CTL_DEL) || + (op != EPOLL_CTL_ADD && + op != EPOLL_CTL_MOD && + op != EPOLL_CTL_DEL)) { + errno = EINVAL; + return -1; + } + + if (event) { + sh_event = share_mem_alloc(sizeof(struct epoll_event)); + if (sh_event == NULL) { + RETURN_ERROR(ENOMEM); + } + rte_memcpy(sh_event, event, sizeof(struct epoll_event)); + } + + args->epfd = restore_fstack_fd(epfd); + args->op = op; + args->fd = fd; + args->event = sh_event; + + SYSCALL(FF_SO_EPOLL_CTL, args); + + if (sh_event) { + share_mem_free(sh_event); + } + + RETURN(); +} + +int +ff_hook_epoll_wait(int epfd, struct epoll_event *events, + int maxevents, int timeout) +{ + //DEBUG_LOG("ff_hook_epoll_wait, epfd:%d, maxevents:%d, timeout:%d\n", epfd, maxevents, timeout); + int fd = epfd; + CHECK_FD_OWNERSHIP(epoll_wait, (epfd, events, maxevents, timeout)); + + DEFINE_REQ_ARGS(epoll_wait); + struct epoll_event *sh_events; + + sh_events = share_mem_alloc(sizeof(struct epoll_event) * maxevents); + if (sh_events == NULL) { + RETURN_ERROR(ENOMEM); + } + + args->epfd = fd; + args->events = sh_events; + args->maxevents = maxevents; + args->timeout = timeout; + + /* for timeout, Although not really effective in FreeBSD stack */ + //SYSCALL(FF_SO_EPOLL_WAIT, args); + ACQUIRE_ZONE_LOCK(FF_SC_IDLE); + sc->ops = FF_SO_EPOLL_WAIT; + sc->args = args; + RELEASE_ZONE_LOCK(FF_SC_REQ); + + if (timeout > 0) { + struct timespec abs_timeout; + + clock_gettime(CLOCK_REALTIME, &abs_timeout); + DEBUG_LOG("before wait, sec:%ld, nsec:%ld\n", abs_timeout.tv_sec, abs_timeout.tv_nsec); + abs_timeout.tv_sec += timeout / 1000; + abs_timeout.tv_nsec += timeout * 1000; + if (abs_timeout.tv_nsec > NS_PER_SECOND) { + abs_timeout.tv_nsec -= NS_PER_SECOND; + abs_timeout.tv_sec += 1; + } + + DEBUG_LOG("ready to wait, sec:%ld, nsec:%ld\n", abs_timeout.tv_sec, abs_timeout.tv_nsec); + ret = sem_timedwait(&sc->wait_sem, &abs_timeout); + + clock_gettime(CLOCK_REALTIME, &abs_timeout); + DEBUG_LOG("after wait, sec:%ld, nsec:%ld\n", abs_timeout.tv_sec, abs_timeout.tv_nsec); + } else { + ret = sem_wait(&sc->wait_sem); + } + + rte_spinlock_lock(&sc->lock); + + if (ret == -1 && sc->status == FF_SC_REQ) { + if (errno == ETIMEDOUT) { + ret = 0; + } + } else { + ret = sc->result; + if (ret < 0) { + errno = sc->error; + } + } + + sc->status = FF_SC_IDLE; + rte_spinlock_unlock(&sc->lock); + + if (ret > 0) { + int i; + for (i = 0; i < ret; i++) { + rte_memcpy(&events[i], &sh_events[i], sizeof(struct epoll_event)); + } + } + + if (sh_events) { + share_mem_free(sh_events); + } + + RETURN(); +} + +pid_t +ff_hook_fork(void) +{ + return ff_linux_fork(); +} + +int +kqueue() +{ + int ret = -1; + + if (unlikely(inited == 0)) { + errno = ENOSYS; + return -1; + } + + SYSCALL(FF_SO_KQUEUE, NULL); + + if (ret >= 0) { + ret = convert_fstack_fd(ret); + } + + return ret; +} + +int +kevent(int kq, const struct kevent *changelist, int nchanges, + struct kevent *eventlist, int nevents, + const struct timespec *timeout) +{ + if (unlikely(inited == 0 && ff_adapter_init() < 0)) { + errno = ENOSYS; + return -1; + } + + kq = restore_fstack_fd(kq); + + DEFINE_REQ_ARGS(kevent); + struct kevent *sh_changelist = NULL; + struct kevent *sh_eventlist = NULL; + + if (changelist != NULL && nchanges > 0) { + sh_changelist = share_mem_alloc(sizeof(struct kevent) * nchanges); + if (sh_changelist == NULL) { + RETURN_ERROR(ENOMEM); + } + + rte_memcpy(sh_changelist, changelist, sizeof(struct kevent) * nchanges); + + struct kevent *kev; + int i = 0; + for(i = 0; i < nchanges; i++) { + kev = (struct kevent *)&sh_changelist[i]; + switch (kev->filter) { + case EVFILT_READ: + case EVFILT_WRITE: + case EVFILT_VNODE: + kev->ident = restore_fstack_fd(kev->ident); + break; + default: + break; + } + } + } + + if (eventlist != NULL && nevents > 0) { + sh_eventlist = share_mem_alloc(sizeof(struct kevent) * nevents); + if (sh_eventlist == NULL) { + share_mem_free(sh_changelist); + RETURN_ERROR(ENOMEM); + } + } + + args->kq = kq; + args->changelist = sh_changelist; + args->nchanges = nchanges; + args->eventlist = sh_eventlist; + args->nevents = nevents; + args->timeout = NULL; + + rte_spinlock_lock(&sc->lock); + + sc->ops = FF_SO_KEVENT; + sc->args = args; + sc->status = FF_SC_REQ; + + rte_spinlock_unlock(&sc->lock); + + if (timeout != NULL) { + struct timespec abs_timeout; + + clock_gettime(CLOCK_REALTIME, &abs_timeout); + abs_timeout.tv_sec += timeout->tv_sec; + abs_timeout.tv_nsec += timeout->tv_nsec; + if (abs_timeout.tv_nsec > NS_PER_SECOND) { + abs_timeout.tv_nsec -= NS_PER_SECOND; + abs_timeout.tv_sec += 1; + } + + ret = sem_timedwait(&sc->wait_sem, &abs_timeout); + } else { + ret = sem_wait(&sc->wait_sem); + } + + rte_spinlock_lock(&sc->lock); + + if (ret == -1 && sc->status == FF_SC_REQ) { + if (errno == ETIMEDOUT) { + ret = 0; + } + } else { + ret = sc->result; + if (ret < 0) { + errno = sc->error; + } + } + + sc->status = FF_SC_IDLE; + + rte_spinlock_unlock(&sc->lock); + + if (ret > 0) { + if (eventlist && nevents) { + rte_memcpy(eventlist, sh_eventlist, + sizeof(struct kevent) * ret); + } + } + + if (sh_changelist) { + share_mem_free(sh_changelist); + } + + if (sh_eventlist) { + share_mem_free(sh_eventlist); + } + + RETURN(); +} + +int +ff_adapter_init() +//int __attribute__((constructor)) +//ff_adapter_init(int argc, char * const argv[]) +{ + int ret; + + if (inited) { + return 0; + } + + if (proc_inited == 0) { + rte_spinlock_init(&worker_id_lock); + rte_spinlock_lock(&worker_id_lock); + + /* + * get ulimit -n to distinguish fd between kernel and F-Stack + */ + struct rlimit rlmt; + ret = getrlimit(RLIMIT_NOFILE, &rlmt); + if (ret < 0) { + ERR_LOG("getrlimit(RLIMIT_NOFILE) failed, use default ff_kernel_max_fd:%d\n", ff_kernel_max_fd); + return -1; + } else { + ff_kernel_max_fd = (int)rlmt.rlim_cur; + } + ERR_LOG("getrlimit(RLIMIT_NOFILE) successed, sed ff_kernel_max_fd:%d, and rlim_max is %ld\n", + ff_kernel_max_fd, rlmt.rlim_max); + + /* + * Get environment variable FF_INITIAL_LCORE_ID to set initial_lcore_id + * + * If need later, modify to get config from config file, + * it can consider multiplex F-Stack config.ini + */ + char *ff_init_lcore_id = getenv(FF_INITIAL_LCORE_ID_STR); + if (ff_init_lcore_id != NULL) { + char *pos = strchr(ff_init_lcore_id, '='); + if (pos != NULL) { + initial_lcore_id = (uint64_t)strtoull(pos, NULL, 16); + if (initial_lcore_id > ((uint64_t)INITIAL_LCORE_ID_MAX) /*== UINT64_MAX*/) { + initial_lcore_id = INITIAL_LCORE_ID_DEFAULT; + ERR_LOG("get invalid FF_INITIAL_LCORE_ID=%s, to use default value 0x%0lx\n", + ff_init_lcore_id, initial_lcore_id); + } + } + ERR_LOG("get FF_INITIAL_LCORE_ID=%s, use 0x%0lx\n", + ff_init_lcore_id, initial_lcore_id); + } + else { + ERR_LOG("environment variable FF_INITIAL_LCORE_ID not found, to use default value 0x%0lx\n", + initial_lcore_id); + } + + + /* + * Get environment variable FF_NB_FSTACK_INSTANCE to set nb_procs. + */ + char *ff_nb_procs = getenv(FF_NB_FSTACK_INSTANCE_STR); + if (ff_nb_procs != NULL) { + char *pos = strchr(ff_nb_procs, '='); + if (pos != NULL) { + nb_procs = (uint32_t)strtoul(pos, NULL, 10); + if (nb_procs == -1 /*UINT32_MAX*/) { + nb_procs = NB_FSTACK_INSTANCE_DEFAULT; + ERR_LOG("get invalid FF_NB_FSTACK_INSTANCE=%s, to use default value %d\n", + ff_nb_procs, nb_procs); + } + } + ERR_LOG("get FF_NB_FSTACK_INSTANCE=%s, use %d\n", + ff_nb_procs, nb_procs); + } + else { + ERR_LOG("environment variable FF_NB_FSTACK_INSTANCE not found, to use default value %d\n", + nb_procs); + } + + if (proc_inited == 0) { + proc_inited = 1; + } + } else { + rte_spinlock_lock(&worker_id_lock); + } + + char buf[RTE_MAX_LCORE] = {0}; + sprintf(buf, "-c%lx", initial_lcore_id << worker_id); + + char *dpdk_argv[] = { + "ff-adapter", buf, "-n4", + "--proc-type=secondary", + /* RTE_LOG_WARNING */ + "--log-level=5", + }; + + printf("\n"); + DEBUG_LOG("rte_eal_init, argc:%ld/%ld=%ld\n", sizeof(dpdk_argv), sizeof(dpdk_argv[0]), sizeof(dpdk_argv)/sizeof(dpdk_argv[0])); + for (int i=0; i < sizeof(dpdk_argv)/sizeof(dpdk_argv[0]); i++) { + printf("%s ", dpdk_argv[i]); + } + printf("\n"); + ret = rte_eal_init(sizeof(dpdk_argv)/sizeof(dpdk_argv[0]), + dpdk_argv); + DEBUG_LOG("rte_eal_init ret:%d\n", ret); + if (ret < 0) { + ERR_LOG("ff_adapter_init failed with EAL initialization\n"); + return ret; + } + + sc = ff_attach_so_context(worker_id % nb_procs); + if (sc == NULL) { + ERR_LOG("ff_attach_so_context failed\n"); + return -1; + } + + worker_id++; + inited = 1; + + rte_spinlock_unlock(&worker_id_lock); + + ERR_LOG("ff_adapter_init success, sc:%p, status:%d, ops:%d\n", sc, sc->status, sc->ops); + + return 0; +} + +void __attribute__((destructor)) +ff_adapter_exit() +{ + ff_detach_so_context(sc); + ERR_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc); +} diff --git a/adapter/ff_hook_syscall.h b/adapter/ff_hook_syscall.h new file mode 100644 index 000000000..a88a28544 --- /dev/null +++ b/adapter/ff_hook_syscall.h @@ -0,0 +1,8 @@ +#ifndef _FF_HOOK_SYSCALL_H +#define _FF_HOOK_SYSCALL_H + +#undef FF_SYSCALL_DECL +#define FF_SYSCALL_DECL(ret, fn, args) extern ret ff_hook_##fn args +#include + +#endif diff --git a/adapter/ff_linux_syscall.c b/adapter/ff_linux_syscall.c new file mode 100644 index 000000000..88614abd4 --- /dev/null +++ b/adapter/ff_linux_syscall.c @@ -0,0 +1,236 @@ +#include +#include +#include +#include + +#include "ff_socket_ops.h" +#include "ff_linux_syscall.h" + +#define SYSCALL(symbol, para) { \ + if (linux_syscall_inited == 0) { \ + if (linux_syscall_init() != 0) { \ + return -1; \ + } \ + } \ + if (syscalls.pf_##symbol) { \ + return syscalls.pf_##symbol para; \ + } \ + errno = ENOSYS; \ + return -1; \ +} + +struct ff_linux_syscall { + #define FF_SYSCALL_DECL(ret, fn, args) ret (*pf_##fn) args; + #include "ff_declare_syscalls.h" +}; + +static int linux_syscall_inited; + +static struct ff_linux_syscall syscalls = { 0 }; + +static void *linux_lib_handle = NULL; + +pthread_mutex_t syscall_init_mutex = PTHREAD_MUTEX_INITIALIZER; + +static inline int +linux_syscall_load_symbol() +{ + linux_lib_handle = dlopen ("libc.so.6", RTLD_NOW | RTLD_GLOBAL); + if (linux_lib_handle == NULL) { + ERR_LOG("cannot dlopen libc.so.6] err_string=%s", dlerror()); + return -1; + } + + #define FF_SYSCALL_DECL(ret, fn, args) \ + syscalls.pf_##fn = (typeof(syscalls.pf_##fn))dlsym(linux_lib_handle, #fn); + #include + + return 0; +} + +static inline int +linux_syscall_init() +{ + if (linux_syscall_inited) { + return 0; + } + + pthread_mutex_lock(&syscall_init_mutex); + if (linux_syscall_inited) { + pthread_mutex_unlock(&syscall_init_mutex); + return 0; + } + + if (linux_syscall_load_symbol() != 0) { + pthread_mutex_unlock(&syscall_init_mutex); + return -1; + } + + linux_syscall_inited = 1; + + pthread_mutex_unlock(&syscall_init_mutex); + + return 0; +} + +int +ff_linux_socket(int domain, int type, int protocol) +{ + DEBUG_LOG("ff_linux_socket, domain:%d, type:%d, protocol:%d\n", domain, type, protocol); + SYSCALL(socket, (domain, type, protocol)); +} + +int +ff_linux_bind(int s, const struct sockaddr *addr, + socklen_t addrlen) +{ + SYSCALL(bind, (s, addr, addrlen)); +} + +int +ff_linux_listen(int s, int backlog) +{ + SYSCALL(listen, (s, backlog)); +} + +int ff_linux_shutdown(int s, int how) +{ + SYSCALL(shutdown, (s, how)); +} + +int ff_linux_getsockname(int s, struct sockaddr *name, + socklen_t *namelen) +{ + SYSCALL(getsockname, (s, name, namelen)); +} + +int ff_linux_getpeername(int s, struct sockaddr *name, + socklen_t *namelen) +{ + SYSCALL(getpeername, (s, name, namelen)); +} + +int ff_linux_getsockopt(int s, int level, int optname, + void *optval, socklen_t *optlen) +{ + SYSCALL(getsockopt, (s, level, optname, optval, optlen)); +} + +int ff_linux_setsockopt(int s, int level, int optname, + const void *optval, socklen_t optlen) +{ + SYSCALL(setsockopt, (s, level, optname, optval, optlen)); +} + +int ff_linux_accept(int s, struct sockaddr *addr, socklen_t *addrlen) +{ + SYSCALL(accept, (s, addr, addrlen)); +} + +int ff_linux_accept4(int s, struct sockaddr *addr, + socklen_t *addrlen, int flags) +{ + SYSCALL(accept4, (s, addr, addrlen, flags)); +} + +int ff_linux_connect(int s, const struct sockaddr *addr, + socklen_t addrlen) +{ + SYSCALL(connect, (s, addr, addrlen)); +} + +ssize_t ff_linux_recv(int s, void *buf, size_t len, int flags) +{ + SYSCALL(recv, (s, buf, len, flags)); +} + +ssize_t ff_linux_send(int s, const void *buf, size_t len, int flags) +{ + SYSCALL(send, (s, buf, len, flags)); +} + +ssize_t ff_linux_read(int s, void *buf, size_t len) +{ + SYSCALL(read, (s, buf, len)); +} + +ssize_t ff_linux_write(int s, const void *buf, size_t len) +{ + SYSCALL(write, (s, buf, len)); +} + +ssize_t ff_linux_writev(int s, const struct iovec *iov, int iovcnt) +{ + SYSCALL(writev, (s, iov, iovcnt)); +} + +ssize_t ff_linux_readv(int s, const struct iovec *iov, int iovcnt) +{ + SYSCALL(readv, (s, iov, iovcnt)); +} + +ssize_t ff_linux_sendto(int s, const void *buf, size_t len, int flags, + const struct sockaddr *to, socklen_t tolen) +{ + SYSCALL(sendto, (s, buf, len, flags, to, tolen)); +} + +ssize_t ff_linux_recvfrom(int s, void *buf, size_t len, int flags, + struct sockaddr *from, socklen_t *fromlen) + +{ + SYSCALL(recvfrom, (s, buf, len, flags, from, fromlen)); +} + +ssize_t ff_linux_sendmsg(int s, const struct msghdr *msg, int flags) +{ + SYSCALL(sendmsg, (s, msg, flags)); +} + +ssize_t ff_linux_recvmsg(int s, struct msghdr *msg, int flags) +{ + SYSCALL(recvmsg, (s, msg, flags)) +} + +int ff_linux_close(int s) +{ + SYSCALL(close, (s)); +} + +int ff_linux_ioctl(int s, unsigned long req, unsigned long data) +{ + SYSCALL(ioctl, (s, req, data)); +} + +int ff_linux_fcntl(int s, int cmd, unsigned long data) +{ + SYSCALL(fcntl, (s, cmd, data)); +} + +int ff_linux_epoll_create(int size) +{ + DEBUG_LOG("ff_linux_epoll_create, fdsize:%d\n", size); + SYSCALL(epoll_create, (size)); +} + +int ff_linux_epoll_ctl(int epfd, int op, int fd, + struct epoll_event *event) +{ + DEBUG_LOG("ff_linux_epoll_ctl, epfd:%d, op:%d, fd:%d\n", epfd, op, fd); + SYSCALL(epoll_ctl, (epfd, op, fd, event)); +} + +int +ff_linux_epoll_wait(int epfd, struct epoll_event *events, + int maxevents, int timeout) +{ + DEBUG_LOG("ff_linux_epoll_wait, epfd:%d, maxevents:%d, timeout:%d\n", epfd, maxevents, timeout); + SYSCALL(epoll_wait, (epfd, events, maxevents, timeout)); +} + +pid_t +ff_linux_fork(void) +{ + SYSCALL(fork, ()); +} + diff --git a/adapter/ff_linux_syscall.h b/adapter/ff_linux_syscall.h new file mode 100644 index 000000000..c6fc96582 --- /dev/null +++ b/adapter/ff_linux_syscall.h @@ -0,0 +1,13 @@ +#ifndef _FF_LINUX_SYSCALL_H_ +#define _FF_LINUX_SYSCALL_H_ + +#include +#include +#include +#include + +#undef FF_SYSCALL_DECL +#define FF_SYSCALL_DECL(ret, fn, args) ret ff_linux_##fn args +#include "ff_declare_syscalls.h" + +#endif diff --git a/adapter/ff_so_zone.c b/adapter/ff_so_zone.c new file mode 100644 index 000000000..2ecfe3a9b --- /dev/null +++ b/adapter/ff_so_zone.c @@ -0,0 +1,170 @@ +#include +#include + +#include +#include + +#include "ff_config.h" +#include "ff_socket_ops.h" + +#define SOCKET_OPS_ZONE_NAME "ff_socket_ops_zone_%d" + +#define SOCKET_OPS_CONTEXT_MAX_NUM 32 + +#define SOCKET_OPS_CONTEXT_NAME_SIZE 32 +#define SOCKET_OPS_CONTEXT_NAME "ff_so_context_" + +static uint16_t ff_max_so_context = SOCKET_OPS_CONTEXT_MAX_NUM; +struct ff_socket_ops_zone *ff_so_zone; + +int +ff_set_max_so_context(uint16_t count) +{ + if (ff_so_zone) { + ERR_LOG("Can not set: memzone has inited\n"); + return -1; + } + + /*if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + ERR_LOG("Can not set: process is not primary\n"); + return 1; + }*/ + + ff_max_so_context = count; + + return 0; +} + +int +ff_create_so_memzone() +{ + if (ff_so_zone) { + ERR_LOG("Can not create memzone: memzone has inited\n"); + return -1; + } + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + uint16_t i, proc_id; + for (proc_id = 0; proc_id < ff_global_cfg.dpdk.nb_procs; proc_id++) { + struct ff_socket_ops_zone *so_zone_tmp; + const struct rte_memzone *mz; + char zn[64]; + + size_t zone_size = sizeof(struct ff_socket_ops_zone) + + sizeof(struct ff_so_context) * ff_max_so_context; + snprintf(zn, sizeof(zn), SOCKET_OPS_ZONE_NAME, proc_id); + ERR_LOG("To create memzone:%s\n", zn); + + mz = rte_memzone_reserve(zn, zone_size, rte_socket_id(), 0); + if (mz == NULL) { + ERR_LOG("Cannot reserve memory zone:%s\n", zn); + return -1; + } + + memset(mz->addr, 0, zone_size); + so_zone_tmp = mz->addr; + + rte_spinlock_init(&so_zone_tmp->lock); + so_zone_tmp->count = ff_max_so_context; + so_zone_tmp->free = so_zone_tmp->count; + so_zone_tmp->sc = (struct ff_so_context *)(so_zone_tmp + 1); + + for (i = 0; i < ff_max_so_context; i++) { + struct ff_so_context *sc = &so_zone_tmp->sc[i]; + rte_spinlock_init(&sc->lock); + sc->status = FF_SC_IDLE; + sc->inuse = 0; + + if (sem_init(&sc->wait_sem, 1, 0) == -1) { + ERR_LOG("Initialize semaphore failed:%d\n", errno); + return -1; + } + } + + if (proc_id == 0) { + ff_so_zone = so_zone_tmp; + } + } + }else { + const struct rte_memzone *mz; + char zn[64]; + + snprintf(zn, sizeof(zn), SOCKET_OPS_ZONE_NAME, ff_global_cfg.dpdk.proc_id); + ERR_LOG("To lookup memzone:%s\n", zn); + + mz = rte_memzone_lookup(zn); + if (mz == NULL) { + ERR_LOG("Lookup memory zone:%s failed\n", zn); + return -1; + } + + ff_so_zone = mz->addr; + } + + return 0; +} + +struct ff_so_context * +ff_attach_so_context(int proc_id) +{ + struct ff_so_context *sc = NULL; + uint16_t i; + + if (ff_so_zone == NULL) { + const struct rte_memzone *mz; + char zn[64]; + + snprintf(zn, sizeof(zn), SOCKET_OPS_ZONE_NAME, proc_id); + ERR_LOG("To lookup memzone:%s\n", zn); + + mz = rte_memzone_lookup(zn); + if (mz == NULL) { + ERR_LOG("Lookup memory zone:%s failed\n", zn); + return NULL; + } + + ff_so_zone = mz->addr; + } + + rte_spinlock_lock(&ff_so_zone->lock); + + if (ff_so_zone->free == 0) { + ERR_LOG("Attach memzone failed: no free context\n"); + rte_spinlock_unlock(&ff_so_zone->lock); + return NULL; + } + + /* i从count-free开始尝试，直到一轮结束 */ + for (i = 0; i < ff_so_zone->count; i++) { + sc = &ff_so_zone->sc[i]; + if (sc->inuse == 0) { + sc->inuse = 1; + rte_spinlock_init(&sc->lock); + sc->status = FF_SC_IDLE; + ff_so_zone->free--; + break; + } + } + DEBUG_LOG("attach sc:%p, i:%d\n", sc, i); + + rte_spinlock_unlock(&ff_so_zone->lock); + + return sc; +} + +void +ff_detach_so_context(struct ff_so_context *sc) +{ + if (ff_so_zone == NULL || sc == NULL) { + return; + } + + rte_spinlock_lock(&ff_so_zone->lock); + + sc->inuse = 0; + + ff_so_zone->free++; + DEBUG_LOG("detach sc:%p, status:%d, ops:%d\n", sc, sc->status, sc->ops); + + rte_spinlock_unlock(&ff_so_zone->lock); +} diff --git a/adapter/ff_socket_ops.c b/adapter/ff_socket_ops.c new file mode 100644 index 000000000..d7a5201a3 --- /dev/null +++ b/adapter/ff_socket_ops.c @@ -0,0 +1,461 @@ +#include +#include + +#include "ff_socket_ops.h" +#include "ff_sysproto.h" +#include "ff_api.h" +#include "ff_epoll.h" + +#undef FF_SYSCALL_DECL +#define FF_SYSCALL_DECL(ret, fn, none) \ + static ret ff_sys_##fn(struct ff_##fn##_args *args); +#include +static int ff_sys_kqueue(struct ff_kqueue_args *args); +static int ff_sys_kevent(struct ff_kevent_args *args); + +extern struct ff_socket_ops_zone *ff_so_zone; + +#define FF_MAX_BOUND_NUM 8 + + +struct ff_bound_info { + int fd; + struct sockaddr addr; +}; + +static struct ff_bound_info ff_bound_fds[FF_MAX_BOUND_NUM]; + +static int +sockaddr_cmp(struct sockaddr *a, struct sockaddr *b) +{ + struct sockaddr_in *sina, *sinb; + sina = (struct sockaddr_in *)a; + sinb = (struct sockaddr_in *)b; + + if (sina->sin_family != sinb->sin_family) { + return 1; + } + + if (sina->sin_port != sinb->sin_port) { + return 1; + } + + if (sina->sin_addr.s_addr != sinb->sin_addr.s_addr) { + return 1; + } + + return 0; +} + +static int +sockaddr_is_bound(struct sockaddr *addr) +{ + int i; + + for (i = 0; i < FF_MAX_BOUND_NUM; i++) { + struct ff_bound_info *info = &ff_bound_fds[i]; + if (info->fd == 0) { + continue; + } + + if (sockaddr_cmp(&info->addr, addr) == 0) { + return info->fd; + } + } + + return 0; +} + +static int +sockaddr_bind(int fd, struct sockaddr *addr) +{ + int i; + + for (i = 0; i < FF_MAX_BOUND_NUM; i++) { + struct ff_bound_info *info = &ff_bound_fds[i]; + if (info->fd != 0) { + continue; + } + + info->fd = fd; + rte_memcpy(&info->addr, addr, sizeof(struct sockaddr)); + + return 0; + } + + return -1; +} + +static int +sockaddr_unbind(int fd) +{ + int i; + + for (i = 0; i < FF_MAX_BOUND_NUM; i++) { + struct ff_bound_info *info = &ff_bound_fds[i]; + if (info->fd != fd) { + continue; + } + + info->fd = 0; + + return 0; + } + + return -1; +} + +static int +ff_sys_socket(struct ff_socket_args *args) +{ + return ff_socket(args->domain, args->type, args->protocol); +} + +static int +ff_sys_bind(struct ff_bind_args *args) +{ + int bound_fd; + int ret; + + bound_fd = sockaddr_is_bound(args->addr); + if (bound_fd != 0 && bound_fd != args->fd) { + return ff_dup2(bound_fd, args->fd); + } + + ret = ff_bind(args->fd, args->addr, args->addrlen); + if (ret == 0) { + sockaddr_bind(args->fd, args->addr); + } + + return ret; +} + +static int +ff_sys_listen(struct ff_listen_args *args) +{ + return ff_listen(args->fd, args->backlog); +} + +static int +ff_sys_shutdown(struct ff_shutdown_args *args) +{ + return ff_shutdown(args->fd, args->how); +} + +static int +ff_sys_getsockname(struct ff_getsockname_args *args) +{ + return ff_getsockname(args->fd, args->name, args->namelen); +} + +static int +ff_sys_getpeername(struct ff_getpeername_args *args) +{ + return ff_getpeername(args->fd, args->name, args->namelen); +} + +static int +ff_sys_getsockopt(struct ff_getsockopt_args *args) +{ + return ff_getsockopt(args->fd, args->level, args->name, + args->optval, args->optlen); +} + +static int +ff_sys_setsockopt(struct ff_setsockopt_args *args) +{ + return ff_setsockopt(args->fd, args->level, args->name, + args->optval, args->optlen); +} + +static int +ff_sys_accept(struct ff_accept_args *args) +{ + return ff_accept(args->fd, args->addr, args->addrlen); +} + +static int +ff_sys_accept4(struct ff_accept4_args *args) +{ + errno = ENOSYS; + return -1; +} + +static int +ff_sys_connect(struct ff_connect_args *args) +{ + return ff_connect(args->fd, args->addr, args->addrlen); +} + +static ssize_t +ff_sys_recv(struct ff_recv_args *args) +{ + return ff_recv(args->fd, args->buf, args->len, args->flags); +} + +static ssize_t +ff_sys_recvfrom(struct ff_recvfrom_args *args) +{ + return ff_recvfrom(args->fd, args->buf, args->len, args->flags, + args->from, args->fromlen); +} + +static ssize_t +ff_sys_recvmsg(struct ff_recvmsg_args *args) +{ + return ff_recvmsg(args->fd, args->msg, args->flags); +} + +static ssize_t +ff_sys_read(struct ff_read_args *args) +{ + return ff_read(args->fd, args->buf, args->len); +} + +static ssize_t +ff_sys_readv(struct ff_readv_args *args) +{ + return ff_readv(args->fd, args->iov, args->iovcnt); +} + +static ssize_t +ff_sys_send(struct ff_send_args *args) +{ + return ff_send(args->fd, args->buf, args->len, args->flags); +} + +static ssize_t +ff_sys_sendto(struct ff_sendto_args *args) +{ + return ff_sendto(args->fd, args->buf, args->len, args->flags, + args->to, args->tolen); +} + +static ssize_t +ff_sys_sendmsg(struct ff_sendmsg_args *args) +{ + return ff_sendmsg(args->fd, args->msg, args->flags); +} + +static ssize_t +ff_sys_write(struct ff_write_args *args) +{ + return ff_write(args->fd, args->buf, args->len); +} + +static ssize_t +ff_sys_writev(struct ff_writev_args *args) +{ + return ff_writev(args->fd, args->iov, args->iovcnt); +} + +static int +ff_sys_close(struct ff_close_args *args) +{ + sockaddr_unbind(args->fd); + return ff_close(args->fd); +} + +static int +ff_sys_ioctl(struct ff_ioctl_args *args) +{ + return ff_ioctl(args->fd, args->com, args->data); +} + +static int +ff_sys_fcntl(struct ff_fcntl_args *args) +{ + return ff_fcntl(args->fd, args->cmd, args->data); +} + +static int +ff_sys_epoll_create(struct ff_epoll_create_args *args) +{ + DEBUG_LOG("to run ff_epoll_create, size:%d\n", args->size); + return ff_epoll_create(args->size); +} + +static int +ff_sys_epoll_ctl(struct ff_epoll_ctl_args *args) +{ + DEBUG_LOG("to run ff_epoll_ctl, epfd:%d, op:%d, fd:%d\n", + args->epfd, args->op, args->fd); + return ff_epoll_ctl(args->epfd, args->op, args->fd, + args->event); +} + +static int +ff_sys_epoll_wait(struct ff_epoll_wait_args *args) +{ + DEBUG_LOG("to run ff_epoll_wait, epfd:%d, maxevents:%d, timeout:%d\n", + args->epfd, args->maxevents, args->timeout); + return ff_epoll_wait(args->epfd, args->events, + args->maxevents, args->timeout); +} + +static int +ff_sys_kqueue(struct ff_kqueue_args *args) +{ + return ff_kqueue(); +} + +static int +ff_sys_kevent(struct ff_kevent_args *args) +{ + return ff_kevent(args->kq, args->changelist, args->nchanges, + args->eventlist, args->nevents, args->timeout); +} + +static pid_t +ff_sys_fork(struct ff_fork_args *args) +{ + errno = ENOSYS; + return -1; +} + +static int +ff_so_handler(int ops, void *args) +{ + DEBUG_LOG("ff_so_handler ops:%d, epoll create ops:%d\n", ops, FF_SO_EPOLL_CREATE); + switch(ops) { + case FF_SO_SOCKET: + return ff_sys_socket((struct ff_socket_args *)args); + case FF_SO_BIND: + return ff_sys_bind((struct ff_bind_args *)args); + case FF_SO_LISTEN: + return ff_sys_listen((struct ff_listen_args *)args); + case FF_SO_CONNECT: + return ff_sys_connect((struct ff_connect_args *)args); + case FF_SO_SHUTDOWN: + return ff_sys_shutdown((struct ff_shutdown_args *)args); + case FF_SO_GETSOCKNAME: + return ff_sys_getsockname((struct ff_getsockname_args *)args); + case FF_SO_GETPEERNAME: + return ff_sys_getpeername((struct ff_getpeername_args *)args); + case FF_SO_GETSOCKOPT: + return ff_sys_getsockopt((struct ff_getsockopt_args *)args); + case FF_SO_SETSOCKOPT: + return ff_sys_setsockopt((struct ff_setsockopt_args *)args); + case FF_SO_ACCEPT: + return ff_sys_accept((struct ff_accept_args *)args); + case FF_SO_ACCEPT4: + return ff_sys_accept4((struct ff_accept4_args *)args); + case FF_SO_RECV: + return ff_sys_recv((struct ff_recv_args *)args); + case FF_SO_RECVFROM: + return ff_sys_recvfrom((struct ff_recvfrom_args *)args); + case FF_SO_RECVMSG: + return ff_sys_recvmsg((struct ff_recvmsg_args *)args); + case FF_SO_READ: + return ff_sys_read((struct ff_read_args *)args); + case FF_SO_READV: + return ff_sys_readv((struct ff_readv_args *)args); + case FF_SO_SEND: + return ff_sys_send((struct ff_send_args *)args); + case FF_SO_SENDTO: + return ff_sys_sendto((struct ff_sendto_args *)args); + case FF_SO_SENDMSG: + return ff_sys_sendmsg((struct ff_sendmsg_args *)args); + case FF_SO_WRITE: + return ff_sys_write((struct ff_write_args *)args); + case FF_SO_WRITEV: + return ff_sys_writev((struct ff_writev_args *)args); + case FF_SO_CLOSE: + return ff_sys_close((struct ff_close_args *)args); + case FF_SO_IOCTL: + return ff_sys_ioctl((struct ff_ioctl_args *)args); + case FF_SO_FCNTL: + return ff_sys_fcntl((struct ff_fcntl_args *)args); + case FF_SO_EPOLL_CREATE: + return ff_sys_epoll_create((struct ff_epoll_create_args *)args); + case FF_SO_EPOLL_CTL: + return ff_sys_epoll_ctl((struct ff_epoll_ctl_args *)args); + case FF_SO_EPOLL_WAIT: + return ff_sys_epoll_wait((struct ff_epoll_wait_args *)args); + case FF_SO_KQUEUE: + return ff_sys_kqueue((struct ff_kqueue_args *)args); + case FF_SO_KEVENT: + return ff_sys_kevent((struct ff_kevent_args *)args); + case FF_SO_FORK: + return ff_sys_fork((struct ff_fork_args *)args); + default: + break; + } + + errno = EINVAL; + return (-1); +} + +static void +ff_handle_kevent(struct ff_so_context *sc) +{ + struct ff_kevent_args *ka = sc->args; + + errno = 0; + sc->result = ff_sys_kevent(ka); + sc->error = errno; + + if (ka->nchanges) { + ka->nchanges = 0; + } + + if (sc->result == 0 && ka->nevents != 0) { + return; + } + + sc->status = FF_SC_REP; + sem_post(&sc->wait_sem); +} + +static inline void +ff_handle_socket_ops(struct ff_so_context *sc) +{ + if (sc->inuse == 0) { + return; + } + + if (!rte_spinlock_trylock(&sc->lock)) { + return; + } + + if (sc->status != FF_SC_REQ) { + rte_spinlock_unlock(&sc->lock); + return; + } + + DEBUG_LOG("ff_handle_socket_ops sc:%p, status:%d, ops:%d\n", sc, sc->status, sc->ops); + + if (sc->ops == FF_SO_KEVENT) { + ff_handle_kevent(sc); + rte_spinlock_unlock(&sc->lock); + return; + } + + errno = 0; + sc->result = ff_so_handler(sc->ops, sc->args); + sc->error = errno; + sc->status = FF_SC_REP; + + if (sc->ops == FF_SO_EPOLL_WAIT) { + sem_post(&sc->wait_sem); + } + + rte_spinlock_unlock(&sc->lock); +} + +void +ff_handle_each_context() +{ + uint16_t i; + + rte_spinlock_lock(&ff_so_zone->lock); + + for (i = 0; i < ff_so_zone->count; i++) { + struct ff_so_context *sc = &ff_so_zone->sc[i]; + + ff_handle_socket_ops(sc); + } + + rte_spinlock_unlock(&ff_so_zone->lock); +} + diff --git a/adapter/ff_socket_ops.h b/adapter/ff_socket_ops.h new file mode 100644 index 000000000..df0405364 --- /dev/null +++ b/adapter/ff_socket_ops.h @@ -0,0 +1,103 @@ +#ifndef _FF_SOCKET_OPS_H_ +#define _FF_SOCKET_OPS_H_ + +#include + +#include +#include + +#define ERR_LOG(fmt, ...) do { \ + printf("file:%s, line:%u, fun:%s, "fmt, __FILE__, __LINE__, __func__, ##__VA_ARGS__); \ + } while (0) + +#ifdef DEBUG +#define DEBUG_LOG ERR_LOG +#else +#define DEBUG_LOG(...) +#endif + +enum FF_SOCKET_OPS { + FF_SO_SOCKET, + FF_SO_LISTEN, + FF_SO_BIND, + FF_SO_CONNECT, + FF_SO_SHUTDOWN, + FF_SO_GETSOCKNAME, + FF_SO_GETPEERNAME, + FF_SO_GETSOCKOPT, + FF_SO_SETSOCKOPT, + FF_SO_ACCEPT, + FF_SO_ACCEPT4, + FF_SO_RECV, + FF_SO_RECVFROM, + FF_SO_RECVMSG, + FF_SO_READ, + FF_SO_READV, + FF_SO_SEND, + FF_SO_SENDTO, + FF_SO_SENDMSG, + FF_SO_WRITE, + FF_SO_WRITEV, + FF_SO_CLOSE, + FF_SO_IOCTL, + FF_SO_FCNTL, + FF_SO_EPOLL_CREATE, + FF_SO_EPOLL_CTL, + FF_SO_EPOLL_WAIT, + FF_SO_KQUEUE, + FF_SO_KEVENT, + FF_SO_FORK, +}; + +enum FF_SO_CONTEXT_STATUS { + FF_SC_IDLE, + FF_SC_REQ, + FF_SC_REP, +}; + +struct ff_socket_ops_zone { + rte_spinlock_t lock; + + /* total number of so_contex */ + uint16_t count; + + /* free number of so_context */ + uint16_t free; + + struct ff_so_context *sc; +} __attribute__((packed)); + +struct ff_so_context { + rte_spinlock_t lock; + + int status; + + sem_t wait_sem; + + enum FF_SOCKET_OPS ops; + + void *args; + + /* result of ops processing */ + ssize_t result; + /* errno if failed */ + int error; + + /* 1 if used, else 0 */ + int inuse; + + // listen fd, refcount.. +} __attribute__((packed)); + +extern struct ff_socket_ops_zone *ff_so_zone; + +/* For primary process */ +int ff_set_max_so_context(uint16_t count); +int ff_create_so_memzone(); +void ff_handle_each_context(); + +/* For secondary process */ +struct ff_so_context *ff_attach_so_context(int proc_id); +void ff_detach_so_context(struct ff_so_context *context); + +#endif diff --git a/adapter/ff_sysproto.h b/adapter/ff_sysproto.h new file mode 100644 index 000000000..57e63eca8 --- /dev/null +++ b/adapter/ff_sysproto.h @@ -0,0 +1,197 @@ +#ifndef _FF_SYSPROTO_H_ +#define _FF_SYSPROTO_H_ + +#include +#include +#include +#include + +struct ff_socket_args { + int domain; + int type; + int protocol; +}; + +struct ff_bind_args { + int fd; + void *addr; + int addrlen; +}; + +struct ff_listen_args { + int fd; + int backlog; +}; + +struct ff_shutdown_args { + int fd; + int how; +}; + +struct ff_getpeername_args { + int fd; + void *name; + socklen_t *namelen; +}; + +struct ff_getsockname_args { + int fd; + void *name; + socklen_t *namelen; +}; + +struct ff_setsockopt_args { + int fd; + int level; + int name; + void *optval; + socklen_t optlen; +}; + +struct ff_getsockopt_args { + int fd; + int level; + int name; + void *optval; + socklen_t *optlen; +}; + +struct ff_accept_args { + int fd; + void *addr; + socklen_t *addrlen; +}; + +struct ff_accept4_args { + int fd; + void *addr; + socklen_t *addrlen; + int flags; +}; + +struct ff_connect_args { + int fd; + void *addr; + int addrlen; +}; + +struct ff_recv_args { + int fd; + void *buf; + size_t len; + int flags; +}; + +struct ff_recvfrom_args { + int fd; + void *buf; + size_t len; + int flags; + void *from; + socklen_t *fromlen; +}; + +struct ff_recvmsg_args { + int fd; + struct msghdr *msg; + int flags; +}; + +struct ff_read_args { + int fd; + void *buf; + size_t len; +}; + +struct ff_readv_args { + int fd; + struct iovec *iov; + int iovcnt; +}; + +struct ff_send_args { + int fd; + void *buf; + size_t len; + int flags; +}; + +struct ff_sendto_args { + int fd; + void *buf; + size_t len; + int flags; + void *to; + int tolen; +}; + +struct ff_sendmsg_args { + int fd; + struct msghdr * msg; + int flags; +}; + +struct ff_write_args { + int fd; + void *buf; + size_t len; +}; + +struct ff_writev_args { + int fd; + struct iovec *iov; + int iovcnt; +}; + +struct ff_close_args { + int fd; +}; + +struct ff_ioctl_args { + int fd; + unsigned long com; + void *data; +}; + +struct ff_fcntl_args { + int fd; + int cmd; + long data; +}; + +struct ff_epoll_create_args { + int size; +}; + +struct ff_epoll_ctl_args { + int epfd; + int op; + int fd; + struct epoll_event *event; +}; + +struct ff_epoll_wait_args { + int epfd; + struct epoll_event *events; + int maxevents; + int timeout; +}; + +struct ff_kqueue_args { + +}; + +struct ff_kevent_args { + int kq; + struct kevent *changelist; + int nchanges; + struct kevent *eventlist; + int nevents; + struct timespec *timeout; +}; + +struct ff_fork_args { + +}; + +#endif diff --git a/adapter/fstack.c b/adapter/fstack.c new file mode 100644 index 000000000..9d1031a5a --- /dev/null +++ b/adapter/fstack.c @@ -0,0 +1,36 @@ +#include "ff_api.h" +#include "ff_socket_ops.h" + +#define WORKERS 32 + +int +loop(void *arg) +{ + ff_handle_each_context(); + + return 0; +} + +int +main(int argc, char * argv[]) +{ + int ret; + + ff_init(argc, argv); + + ret = ff_set_max_so_context(WORKERS); + if (ret < 0) { + return -1; + } + + ret = ff_create_so_memzone(); + if (ret < 0) { + return -1; + } + + ERR_LOG("ff_create_so_memzone successful\n"); + + ff_run(loop, NULL); + + return 0; +} From fe7e360eca5c71931e0ca356997cc282853eca03 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 6 Apr 2023 18:02:47 +0800 Subject: [PATCH 02/43] Add FF_THREAD_SOCKET in Makefile, Whether to use thread-level socket, default enable. --- adapter/Makefile | 16 ++++++++++++++-- adapter/ff_hook_syscall.c | 16 +++++++++++++--- adapter/ff_socket_ops.h | 6 +++--- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/adapter/Makefile b/adapter/Makefile index ccbc5b72b..e759f4a55 100644 --- a/adapter/Makefile +++ b/adapter/Makefile @@ -8,15 +8,27 @@ ifneq ($(shell pkg-config --exists libdpdk && echo 0),0) $(error "No installation of DPDK found, maybe you should export environment variable `PKG_CONFIG_PATH`") endif -DEBUG=-O0 -gdwarf-2 -g3 -DDEBUG +DEBUG=-O0 -gdwarf-2 -g3 + +# Per thread separate initialization dpdk lib and attach sc when needed, +# such as listen same port in different threads, and socket can use in own thread. +# Default enable. +# +# If disable it, one socket can use in all threads. +FF_THREAD_SOCKET=1 PKGCONF ?= pkg-config ifndef DEBUG - CFLAGS+= -g -O2 + CFLAGS+= -g -O2 -DNDEBUG else CFLAGS+= ${DEBUG} endif + +ifdef FF_THREAD_SOCKET + CFLAGS+= -DFF_THREAD_SOCKET +endif + CFLAGS += -fPIC -Wall -Werror $(shell $(PKGCONF) --cflags libdpdk) INCLUDES= -I. -I${FF_PATH}/lib diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index 26bcdcd9d..3a6ff5f65 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -95,9 +95,19 @@ return ret; \ } while (0) -/* per thread separate initialization dpdk lib and attach sc */ -static __thread int inited = 0; -static __thread struct ff_so_context *sc; +/* + * Per thread separate initialization dpdk lib and attach sc when needed, + * such as listen same port in different threads, and socket can use in own thread. + * + * Otherwise, one socket can use in all threads. + */ +#ifdef FF_THREAD_SOCKET +#define __FF_THREAD __thread +#else +#define __FF_THREAD +#endif +static __FF_THREAD int inited = 0; +static __FF_THREAD struct ff_so_context *sc; /* process-level initialization flag */ static int proc_inited = 0; diff --git a/adapter/ff_socket_ops.h b/adapter/ff_socket_ops.h index df0405364..63b12857d 100644 --- a/adapter/ff_socket_ops.h +++ b/adapter/ff_socket_ops.h @@ -10,10 +10,10 @@ printf("file:%s, line:%u, fun:%s, "fmt, __FILE__, __LINE__, __func__, ##__VA_ARGS__); \ } while (0) -#ifdef DEBUG -#define DEBUG_LOG ERR_LOG -#else +#ifdef NDEBUG #define DEBUG_LOG(...) +#else +#define DEBUG_LOG ERR_LOG #endif enum FF_SOCKET_OPS { From edcac626c521f47077f421f1c747e40a66e636fb Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 6 Apr 2023 18:23:35 +0800 Subject: [PATCH 03/43] Optimize some code. --- adapter/ff_so_zone.c | 33 +++++++++++++++++++++++++++++---- adapter/ff_socket_ops.c | 25 +++++++++++++++++-------- adapter/ff_socket_ops.h | 5 ++++- 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/adapter/ff_so_zone.c b/adapter/ff_so_zone.c index 2ecfe3a9b..f324232ef 100644 --- a/adapter/ff_so_zone.c +++ b/adapter/ff_so_zone.c @@ -9,7 +9,8 @@ #define SOCKET_OPS_ZONE_NAME "ff_socket_ops_zone_%d" -#define SOCKET_OPS_CONTEXT_MAX_NUM 32 +/* Must be power of 2 */ +#define SOCKET_OPS_CONTEXT_MAX_NUM (2 << 5) #define SOCKET_OPS_CONTEXT_NAME_SIZE 32 #define SOCKET_OPS_CONTEXT_NAME "ff_so_context_" @@ -17,6 +18,12 @@ static uint16_t ff_max_so_context = SOCKET_OPS_CONTEXT_MAX_NUM; struct ff_socket_ops_zone *ff_so_zone; +static inline int +is_power_of_2(uint64_t n) +{ + return (n != 0 && ((n & (n - 1)) == 0)); +} + int ff_set_max_so_context(uint16_t count) { @@ -30,6 +37,12 @@ ff_set_max_so_context(uint16_t count) return 1; }*/ + if (!is_power_of_2(count)) { + ERR_LOG("Can not set: count:%d is not power of 2, use default:%d\n", + count, ff_max_so_context); + return -1; + } + ff_max_so_context = count; return 0; @@ -66,7 +79,9 @@ ff_create_so_memzone() rte_spinlock_init(&so_zone_tmp->lock); so_zone_tmp->count = ff_max_so_context; + so_zone_tmp->mask = so_zone_tmp->count - 1; so_zone_tmp->free = so_zone_tmp->count; + so_zone_tmp->idx = 0; so_zone_tmp->sc = (struct ff_so_context *)(so_zone_tmp + 1); for (i = 0; i < ff_max_so_context; i++) { @@ -134,18 +149,28 @@ ff_attach_so_context(int proc_id) return NULL; } - /* i从count-free开始尝试，直到一轮结束 */ for (i = 0; i < ff_so_zone->count; i++) { - sc = &ff_so_zone->sc[i]; + uint16_t idx = (ff_so_zone->idx + i) & ff_so_zone->mask; + sc = &ff_so_zone->sc[idx]; if (sc->inuse == 0) { sc->inuse = 1; rte_spinlock_init(&sc->lock); sc->status = FF_SC_IDLE; ff_so_zone->free--; + ff_so_zone->idx = idx; break; } } - DEBUG_LOG("attach sc:%p, i:%d\n", sc, i); + + if (unlikely(i == ff_so_zone->count)) { + ERR_LOG("Attach memzone failed: instance %d no free context," + " fetel error of so status, all sc inuse, count:%d, free:%d\n", + proc_id, ff_so_zone->count, ff_so_zone->free); + sc = NULL; + } + + DEBUG_LOG("attach sc:%p, so count:%d, free:%d, idx:%d, i:%d\n", + sc, ff_so_zone->count, ff_so_zone->free, ff_so_zone->idx, i); rte_spinlock_unlock(&ff_so_zone->lock); diff --git a/adapter/ff_socket_ops.c b/adapter/ff_socket_ops.c index d7a5201a3..1b13cca32 100644 --- a/adapter/ff_socket_ops.c +++ b/adapter/ff_socket_ops.c @@ -410,10 +410,6 @@ ff_handle_kevent(struct ff_so_context *sc) static inline void ff_handle_socket_ops(struct ff_so_context *sc) { - if (sc->inuse == 0) { - return; - } - if (!rte_spinlock_trylock(&sc->lock)) { return; } @@ -446,14 +442,27 @@ ff_handle_socket_ops(struct ff_so_context *sc) void ff_handle_each_context() { - uint16_t i; + uint16_t i, nb_handled; rte_spinlock_lock(&ff_so_zone->lock); - for (i = 0; i < ff_so_zone->count; i++) { - struct ff_so_context *sc = &ff_so_zone->sc[i]; + assert(ff_so_zone->count >= ff_so_zone->free); + nb_handled = ff_so_zone->count - ff_so_zone->free; + if (nb_handled) { + for (i = 0; i < ff_so_zone->count; i++) { + struct ff_so_context *sc = &ff_so_zone->sc[i]; - ff_handle_socket_ops(sc); + if (sc->inuse == 0) { + continue; + } + + ff_handle_socket_ops(sc); + + nb_handled--; + if (!nb_handled) { + break; + } + } } rte_spinlock_unlock(&ff_so_zone->lock); diff --git a/adapter/ff_socket_ops.h b/adapter/ff_socket_ops.h index 63b12857d..eaee49086 100644 --- a/adapter/ff_socket_ops.h +++ b/adapter/ff_socket_ops.h @@ -58,12 +58,15 @@ enum FF_SO_CONTEXT_STATUS { struct ff_socket_ops_zone { rte_spinlock_t lock; - /* total number of so_contex */ + /* total number of so_contex, must be power of 2 */ uint16_t count; + uint16_t mask; /* free number of so_context */ uint16_t free; + uint16_t idx; + struct ff_so_context *sc; } __attribute__((packed)); From 0efc8b3374d51555aa4f8ca91a41d1ddc0d4e0d5 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 6 Apr 2023 19:09:55 +0800 Subject: [PATCH 04/43] Fixed some issue in multi instances. --- adapter/ff_hook_syscall.c | 88 ++++++++++++++++----------------------- adapter/ff_so_zone.c | 6 ++- adapter/ff_socket_ops.c | 2 - adapter/ff_socket_ops.h | 14 ++++++- 4 files changed, 53 insertions(+), 57 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index 3a6ff5f65..a968c98a9 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -95,17 +95,6 @@ return ret; \ } while (0) -/* - * Per thread separate initialization dpdk lib and attach sc when needed, - * such as listen same port in different threads, and socket can use in own thread. - * - * Otherwise, one socket can use in all threads. - */ -#ifdef FF_THREAD_SOCKET -#define __FF_THREAD __thread -#else -#define __FF_THREAD -#endif static __FF_THREAD int inited = 0; static __FF_THREAD struct ff_so_context *sc; @@ -1428,6 +1417,7 @@ ff_adapter_init() } if (proc_inited == 0) { + /* May conflict */ rte_spinlock_init(&worker_id_lock); rte_spinlock_lock(&worker_id_lock); @@ -1453,14 +1443,11 @@ ff_adapter_init() */ char *ff_init_lcore_id = getenv(FF_INITIAL_LCORE_ID_STR); if (ff_init_lcore_id != NULL) { - char *pos = strchr(ff_init_lcore_id, '='); - if (pos != NULL) { - initial_lcore_id = (uint64_t)strtoull(pos, NULL, 16); - if (initial_lcore_id > ((uint64_t)INITIAL_LCORE_ID_MAX) /*== UINT64_MAX*/) { - initial_lcore_id = INITIAL_LCORE_ID_DEFAULT; - ERR_LOG("get invalid FF_INITIAL_LCORE_ID=%s, to use default value 0x%0lx\n", - ff_init_lcore_id, initial_lcore_id); - } + initial_lcore_id = (uint64_t)strtoull(ff_init_lcore_id, NULL, 16); + if (initial_lcore_id > ((uint64_t)INITIAL_LCORE_ID_MAX) /*== UINT64_MAX*/) { + initial_lcore_id = INITIAL_LCORE_ID_DEFAULT; + ERR_LOG("get invalid FF_INITIAL_LCORE_ID=%s, to use default value 0x%0lx\n", + ff_init_lcore_id, initial_lcore_id); } ERR_LOG("get FF_INITIAL_LCORE_ID=%s, use 0x%0lx\n", ff_init_lcore_id, initial_lcore_id); @@ -1470,20 +1457,16 @@ ff_adapter_init() initial_lcore_id); } - /* * Get environment variable FF_NB_FSTACK_INSTANCE to set nb_procs. */ char *ff_nb_procs = getenv(FF_NB_FSTACK_INSTANCE_STR); if (ff_nb_procs != NULL) { - char *pos = strchr(ff_nb_procs, '='); - if (pos != NULL) { - nb_procs = (uint32_t)strtoul(pos, NULL, 10); - if (nb_procs == -1 /*UINT32_MAX*/) { - nb_procs = NB_FSTACK_INSTANCE_DEFAULT; - ERR_LOG("get invalid FF_NB_FSTACK_INSTANCE=%s, to use default value %d\n", - ff_nb_procs, nb_procs); - } + nb_procs = (uint32_t)strtoul(ff_nb_procs, NULL, 10); + if (nb_procs == -1 /*UINT32_MAX*/) { + nb_procs = NB_FSTACK_INSTANCE_DEFAULT; + ERR_LOG("get invalid FF_NB_FSTACK_INSTANCE=%s, to use default value %d\n", + ff_nb_procs, nb_procs); } ERR_LOG("get FF_NB_FSTACK_INSTANCE=%s, use %d\n", ff_nb_procs, nb_procs); @@ -1493,6 +1476,30 @@ ff_adapter_init() nb_procs); } + char buf[RTE_MAX_LCORE] = {0}; + sprintf(buf, "-c%lx", initial_lcore_id/* << worker_id*/); + + char *dpdk_argv[] = { + "ff-adapter", buf, "-n4", + "--proc-type=secondary", + /* RTE_LOG_WARNING */ + "--log-level=5", + }; + + printf("\n"); + DEBUG_LOG("rte_eal_init, argc:%ld/%ld=%ld\n", sizeof(dpdk_argv), sizeof(dpdk_argv[0]), sizeof(dpdk_argv)/sizeof(dpdk_argv[0])); + for (int i=0; i < sizeof(dpdk_argv)/sizeof(dpdk_argv[0]); i++) { + printf("%s ", dpdk_argv[i]); + } + printf("\n"); + ret = rte_eal_init(sizeof(dpdk_argv)/sizeof(dpdk_argv[0]), + dpdk_argv); + DEBUG_LOG("rte_eal_init ret:%d\n", ret); + if (ret < 0) { + ERR_LOG("ff_adapter_init failed with EAL initialization\n"); + return ret; + } + if (proc_inited == 0) { proc_inited = 1; } @@ -1500,30 +1507,7 @@ ff_adapter_init() rte_spinlock_lock(&worker_id_lock); } - char buf[RTE_MAX_LCORE] = {0}; - sprintf(buf, "-c%lx", initial_lcore_id << worker_id); - - char *dpdk_argv[] = { - "ff-adapter", buf, "-n4", - "--proc-type=secondary", - /* RTE_LOG_WARNING */ - "--log-level=5", - }; - - printf("\n"); - DEBUG_LOG("rte_eal_init, argc:%ld/%ld=%ld\n", sizeof(dpdk_argv), sizeof(dpdk_argv[0]), sizeof(dpdk_argv)/sizeof(dpdk_argv[0])); - for (int i=0; i < sizeof(dpdk_argv)/sizeof(dpdk_argv[0]); i++) { - printf("%s ", dpdk_argv[i]); - } - printf("\n"); - ret = rte_eal_init(sizeof(dpdk_argv)/sizeof(dpdk_argv[0]), - dpdk_argv); - DEBUG_LOG("rte_eal_init ret:%d\n", ret); - if (ret < 0) { - ERR_LOG("ff_adapter_init failed with EAL initialization\n"); - return ret; - } - + DEBUG_LOG("worker_id:%d, nb_procs:%d\n", worker_id, nb_procs); sc = ff_attach_so_context(worker_id % nb_procs); if (sc == NULL) { ERR_LOG("ff_attach_so_context failed\n"); diff --git a/adapter/ff_so_zone.c b/adapter/ff_so_zone.c index f324232ef..09f9f84f2 100644 --- a/adapter/ff_so_zone.c +++ b/adapter/ff_so_zone.c @@ -16,7 +16,7 @@ #define SOCKET_OPS_CONTEXT_NAME "ff_so_context_" static uint16_t ff_max_so_context = SOCKET_OPS_CONTEXT_MAX_NUM; -struct ff_socket_ops_zone *ff_so_zone; +__FF_THREAD struct ff_socket_ops_zone *ff_so_zone; static inline int is_power_of_2(uint64_t n) @@ -125,6 +125,8 @@ ff_attach_so_context(int proc_id) struct ff_so_context *sc = NULL; uint16_t i; + DEBUG_LOG("proc_id:%d, ff_so_zone:%p\n", proc_id, ff_so_zone); + if (ff_so_zone == NULL) { const struct rte_memzone *mz; char zn[64]; @@ -157,7 +159,7 @@ ff_attach_so_context(int proc_id) rte_spinlock_init(&sc->lock); sc->status = FF_SC_IDLE; ff_so_zone->free--; - ff_so_zone->idx = idx; + ff_so_zone->idx = idx + 1; break; } } diff --git a/adapter/ff_socket_ops.c b/adapter/ff_socket_ops.c index 1b13cca32..baef7015f 100644 --- a/adapter/ff_socket_ops.c +++ b/adapter/ff_socket_ops.c @@ -13,8 +13,6 @@ static int ff_sys_kqueue(struct ff_kqueue_args *args); static int ff_sys_kevent(struct ff_kevent_args *args); -extern struct ff_socket_ops_zone *ff_so_zone; - #define FF_MAX_BOUND_NUM 8 diff --git a/adapter/ff_socket_ops.h b/adapter/ff_socket_ops.h index eaee49086..8c97d8275 100644 --- a/adapter/ff_socket_ops.h +++ b/adapter/ff_socket_ops.h @@ -6,6 +6,18 @@ #include #include +/* + * Per thread separate initialization dpdk lib and attach sc when needed, + * such as listen same port in different threads, and socket can use in own thread. + * + * Otherwise, one socket can use in all threads. + */ +#ifdef FF_THREAD_SOCKET +#define __FF_THREAD __thread +#else +#define __FF_THREAD +#endif + #define ERR_LOG(fmt, ...) do { \ printf("file:%s, line:%u, fun:%s, "fmt, __FILE__, __LINE__, __func__, ##__VA_ARGS__); \ } while (0) @@ -92,7 +104,7 @@ struct ff_so_context { // listen fd, refcount.. } __attribute__((packed)); -extern struct ff_socket_ops_zone *ff_so_zone; +extern __FF_THREAD struct ff_socket_ops_zone *ff_so_zone; /* For primary process */ int ff_set_max_so_context(uint16_t count); From 799c39d5f0ffc40b6129f45e6978e828794a3528 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 6 Apr 2023 21:48:57 +0800 Subject: [PATCH 05/43] Fix the issue that not detach sc while destructor in exit. --- adapter/ff_hook_syscall.c | 32 +++++++++++++++++++++++++++++++- adapter/ff_so_zone.c | 17 ++++++++++++++--- adapter/ff_socket_ops.h | 8 +++----- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index a968c98a9..1b9f4e9ce 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -98,6 +98,14 @@ static __FF_THREAD int inited = 0; static __FF_THREAD struct ff_so_context *sc; +/* + * Use pthread_key_create/pthread_setspecific/pthread_key_delete in FF_THREAD_SOCKET mode, + * because ff_so_zone is thread level. + */ +#ifdef FF_THREAD_SOCKET +static pthread_key_t key; +#endif + /* process-level initialization flag */ static int proc_inited = 0; @@ -1405,6 +1413,15 @@ kevent(int kq, const struct kevent *changelist, int nchanges, RETURN(); } +#ifdef FF_THREAD_SOCKET +static void +thread_destructor(void *sc) +{ + DEBUG_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc); + ff_detach_so_context(sc); +} +#endif + int ff_adapter_init() //int __attribute__((constructor)) @@ -1421,6 +1438,11 @@ ff_adapter_init() rte_spinlock_init(&worker_id_lock); rte_spinlock_lock(&worker_id_lock); +#ifdef FF_THREAD_SOCKET + pthread_key_create(&key, thread_destructor); + DEBUG_LOG("pthread key:%d\n", key); +#endif + /* * get ulimit -n to distinguish fd between kernel and F-Stack */ @@ -1514,6 +1536,10 @@ ff_adapter_init() return -1; } +#ifdef FF_THREAD_SOCKET + pthread_setspecific(key, sc); +#endif + worker_id++; inited = 1; @@ -1527,6 +1553,10 @@ ff_adapter_init() void __attribute__((destructor)) ff_adapter_exit() { +#ifdef FF_THREAD_SOCKET + pthread_key_delete(key); +#else + DEBUG_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc); ff_detach_so_context(sc); - ERR_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc); +#endif } diff --git a/adapter/ff_so_zone.c b/adapter/ff_so_zone.c index 09f9f84f2..2d7634b76 100644 --- a/adapter/ff_so_zone.c +++ b/adapter/ff_so_zone.c @@ -88,6 +88,7 @@ ff_create_so_memzone() struct ff_so_context *sc = &so_zone_tmp->sc[i]; rte_spinlock_init(&sc->lock); sc->status = FF_SC_IDLE; + sc->idx = i; sc->inuse = 0; if (sem_init(&sc->wait_sem, 1, 0) == -1) { @@ -182,16 +183,26 @@ ff_attach_so_context(int proc_id) void ff_detach_so_context(struct ff_so_context *sc) { + DEBUG_LOG("ff_so_zone:%p, sc:%p\n", ff_so_zone, sc); + if (ff_so_zone == NULL || sc == NULL) { return; } + DEBUG_LOG("detach sc:%p, ops:%d, status:%d, idx:%d, inuse:%d, so free:%u, idx:%u\n", + sc, sc->ops, sc->status, sc->idx, sc->inuse, ff_so_zone->free, ff_so_zone->idx); + rte_spinlock_lock(&ff_so_zone->lock); - sc->inuse = 0; + if (sc->inuse == 1) { + sc->inuse = 0; - ff_so_zone->free++; - DEBUG_LOG("detach sc:%p, status:%d, ops:%d\n", sc, sc->status, sc->ops); + ff_so_zone->free++; + ff_so_zone->idx = sc->idx; + } + + DEBUG_LOG("detach sc:%p, ops:%d, status:%d, idx:%d, inuse:%d, so free:%u, idx:%u\n", + sc, sc->ops, sc->status, sc->idx, sc->inuse, ff_so_zone->free, ff_so_zone->idx); rte_spinlock_unlock(&ff_so_zone->lock); } diff --git a/adapter/ff_socket_ops.h b/adapter/ff_socket_ops.h index 8c97d8275..761549c26 100644 --- a/adapter/ff_socket_ops.h +++ b/adapter/ff_socket_ops.h @@ -84,14 +84,12 @@ struct ff_socket_ops_zone { struct ff_so_context { rte_spinlock_t lock; - - int status; - sem_t wait_sem; + void *args; enum FF_SOCKET_OPS ops; - - void *args; + int status; + int idx; /* result of ops processing */ ssize_t result; From 7492bfbcb6d5e4733ab20d8b41ff02f38dfbd143 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 6 Apr 2023 21:53:02 +0800 Subject: [PATCH 06/43] Add helloworld_stack_epoll for demo. --- example/Makefile | 3 +- example/main_stack_epoll.c | 202 +++++++++++++++++++++++++++++++++++++ 2 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 example/main_stack_epoll.c diff --git a/example/Makefile b/example/Makefile index 5c52df9a6..2ca45b0fc 100644 --- a/example/Makefile +++ b/example/Makefile @@ -10,7 +10,7 @@ endif PKGCONF ?= pkg-config -CFLAGS += -O -gdwarf-2 $(shell $(PKGCONF) --cflags libdpdk) +CFLAGS += -O0 -g -gdwarf-2 $(shell $(PKGCONF) --cflags libdpdk) LIBS+= $(shell $(PKGCONF) --static --libs libdpdk) LIBS+= -L${FF_PATH}/lib -Wl,--whole-archive,-lfstack,--no-whole-archive @@ -20,6 +20,7 @@ TARGET="helloworld" all: cc ${CFLAGS} -DINET6 -o ${TARGET} main.c ${LIBS} cc ${CFLAGS} -o ${TARGET}_epoll main_epoll.c ${LIBS} + cc ${CFLAGS} -I ../adapter -o ${TARGET}_stack_epoll main_stack_epoll.c ${LIBS} .PHONY: clean clean: diff --git a/example/main_stack_epoll.c b/example/main_stack_epoll.c new file mode 100644 index 000000000..e3278f1dc --- /dev/null +++ b/example/main_stack_epoll.c @@ -0,0 +1,202 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_WORKERS 128 +pthread_t hworker[MAX_WORKERS]; +pthread_spinlock_t worker_lock; +#define MAX_EVENTS 512 + +static int exit_flag = 0; + +char html[] = +"HTTP/1.1 200 OK\r\n" +"Server: F-Stack\r\n" +"Date: Sat, 25 Feb 2017 09:26:33 GMT\r\n" +"Content-Type: text/html\r\n" +"Content-Length: 438\r\n" +"Last-Modified: Tue, 21 Feb 2017 09:44:03 GMT\r\n" +"Connection: keep-alive\r\n" +"Accept-Ranges: bytes\r\n" +"\r\n" +"\r\n" +"\r\n" +"\r\n" +"Welcome to F-Stack!\r\n" +"\r\n" +"\r\n" +"\r\n" +"

Welcome to F-Stack!

\r\n" +"\r\n" +"

For online documentation and support please refer to\r\n" +"F-Stack.org.
\r\n" +"\r\n" +"

Thank you for using F-Stack.

\r\n" +"\r\n" +""; + +void sig_term(int sig) +{ + printf("we caught signal %d, to exit helloworld\n", sig); + exit_flag = 1; + return; +} + +void *loop(void *arg) +{ + struct epoll_event ev; + struct epoll_event events[MAX_EVENTS]; + int epfd; + int sockfd; + int thread_id; + + thread_id = *(int *)arg; + pthread_spin_unlock(&worker_lock); + + printf("start thread %d\n", thread_id); + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + printf("thread %d, sockfd:%d\n", thread_id, sockfd); + if (sockfd < 0) { + printf("thread %d, ff_socket failed\n", thread_id); + exit(1); + } + + int on = 1; + ioctl(sockfd, FIONBIO, &on); + + struct sockaddr_in my_addr; + bzero(&my_addr, sizeof(my_addr)); + my_addr.sin_family = AF_INET; + my_addr.sin_port = htons(80); + my_addr.sin_addr.s_addr = htonl(INADDR_ANY); + + int ret = bind(sockfd, (const struct sockaddr *)&my_addr, sizeof(my_addr)); + if (ret < 0) { + printf("thread %d, ff_bind failed\n", thread_id); + exit(1); + } + + ret = listen(sockfd, MAX_EVENTS); + if (ret < 0) { + printf("thread %d, ff_listen failed\n", thread_id); + exit(1); + } + + epfd = epoll_create(0); + if (epfd <= 0) { + printf("thread %d, ff_epoll_create failed, errno:%d, %s\n", + thread_id, errno, strerror(errno)); + exit(1); + } + ev.data.fd = sockfd; + ev.events = EPOLLIN; + epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev); + + /* Wait for events to happen */ + while (!exit_flag) { + int nevents = epoll_wait(epfd, events, MAX_EVENTS, 100); + if (nevents <= 0) { + if (nevents) { + printf("thread %d, hello world epoll wait ret %d, errno:%d, %s\n", + thread_id, nevents, errno, strerror(errno)); + } + //usleep(100); + sleep(1); + } + printf("thread %d, get nevents:%d\n", thread_id, nevents); + int i; + + for (i = 0; i < nevents; ++i) { + /* Handle new connect */ + if (events[i].data.fd == sockfd) { + while (1) { + int nclientfd = accept(sockfd, NULL, NULL); + if (nclientfd < 0) { + break; + } + + /* Add to event list */ + ev.data.fd = nclientfd; + ev.events = EPOLLIN; + if (epoll_ctl(epfd, EPOLL_CTL_ADD, nclientfd, &ev) != 0) { + printf("thread %d, ff_epoll_ctl failed:%d, %s\n", + thread_id, errno, strerror(errno)); + break; + } + } + } else { + if (events[i].events & EPOLLERR ) { + /* Simply close socket */ + epoll_ctl(epfd, EPOLL_CTL_DEL, events[i].data.fd, NULL); + close(events[i].data.fd); + } else if (events[i].events & EPOLLIN) { + char buf[256]; + size_t readlen = read( events[i].data.fd, buf, sizeof(buf)); + if(readlen > 0) { + write( events[i].data.fd, html, sizeof(html) - 1); + } else { + epoll_ctl(epfd, EPOLL_CTL_DEL, events[i].data.fd, NULL); + close(events[i].data.fd); + } + } else { + printf("thread %d, unknown event: %8.8X\n", events[i].events); + } + } + } + } +} + +int main(int argc, char * argv[]) +{ + int i, worker_num; + + signal(SIGTERM, sig_term); + + if (argc == 1) { + worker_num = 1; + } else { + worker_num = atoi(argv[1]); + } + printf("to init %d workers.\n", worker_num); + + pthread_spin_init(&worker_lock, PTHREAD_PROCESS_SHARED); + + for (i = 0; i < worker_num; i++) { + pthread_spin_lock(&worker_lock); + if(pthread_create(&hworker[i], NULL, loop, (void *)&i) < 0) { + printf("create loop thread failed., errno:%d/%s\n", + errno, strerror(errno)); + pthread_spin_unlock(&worker_lock); + pthread_spin_destroy(&worker_lock); + return -1; + } + sleep(5); + } + + for (i = 0; i < worker_num; i++) { + pthread_join(hworker[i], NULL); + } + + pthread_spin_destroy(&worker_lock); + + return 0; +} From baceb8fd64ad28fb2ba354ba89848b6878ab2220 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Fri, 7 Apr 2023 10:43:37 +0800 Subject: [PATCH 07/43] 1. Add FF_PROC_ID environment variable to fix the issue that secondary APP's process get sc from the first ff_so_zone. 2. SIGINT can destructor sc. --- adapter/ff_hook_syscall.c | 26 ++++++++++++++++++++++++-- example/main_stack_epoll.c | 1 + 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index 1b9f4e9ce..a6bb0895c 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -114,7 +114,10 @@ static int proc_inited = 0; #define INITIAL_LCORE_ID_MAX 0x4000000000000 /* lcore 50 */ #define FF_INITIAL_LCORE_ID_STR "FF_INITIAL_LCORE_ID" static uint64_t initial_lcore_id = INITIAL_LCORE_ID_DEFAULT; -static int worker_id = 0; + +#define WORKER_ID_DEFAULT 0 +#define FF_PROC_ID_STR "FF_PROC_ID" +static int worker_id = WORKER_ID_DEFAULT; rte_spinlock_t worker_id_lock; /* The num of F-Stack process instance, default 1 */ @@ -1199,7 +1202,7 @@ int ff_hook_epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) { - //DEBUG_LOG("ff_hook_epoll_wait, epfd:%d, maxevents:%d, timeout:%d\n", epfd, maxevents, timeout); + DEBUG_LOG("ff_hook_epoll_wait, epfd:%d, maxevents:%d, timeout:%d\n", epfd, maxevents, timeout); int fd = epfd; CHECK_FD_OWNERSHIP(epoll_wait, (epfd, events, maxevents, timeout)); @@ -1498,6 +1501,25 @@ ff_adapter_init() nb_procs); } + /* + * Get environment variable FF_PROC_ID to set nb_procs. + */ + char *ff_worker_id = getenv(FF_PROC_ID_STR); + if (ff_worker_id != NULL) { + worker_id = (uint32_t)strtoul(ff_worker_id, NULL, 10); + if (worker_id == -1 /*UINT32_MAX*/) { + worker_id = WORKER_ID_DEFAULT; + ERR_LOG("get invalid FF_PROC_ID=%s, to use default value %d\n", + ff_worker_id, worker_id); + } + ERR_LOG("get FF_PROC_ID=%s, use %d\n", + ff_worker_id, worker_id); + } + else { + ERR_LOG("environment variable FF_PROC_ID not found, to use default value %d\n", + worker_id); + } + char buf[RTE_MAX_LCORE] = {0}; sprintf(buf, "-c%lx", initial_lcore_id/* << worker_id*/); diff --git a/example/main_stack_epoll.c b/example/main_stack_epoll.c index e3278f1dc..bc5a321a4 100644 --- a/example/main_stack_epoll.c +++ b/example/main_stack_epoll.c @@ -169,6 +169,7 @@ int main(int argc, char * argv[]) { int i, worker_num; + signal(SIGINT, sig_term); signal(SIGTERM, sig_term); if (argc == 1) { From 5a01e6f99ba097719de153f995f26fedbb26a1f8 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Fri, 7 Apr 2023 11:26:30 +0800 Subject: [PATCH 08/43] Optimize some code of main_stack_epoll. --- example/main_stack_epoll.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/example/main_stack_epoll.c b/example/main_stack_epoll.c index bc5a321a4..d7acc034b 100644 --- a/example/main_stack_epoll.c +++ b/example/main_stack_epoll.c @@ -69,17 +69,19 @@ void *loop(void *arg) int thread_id; thread_id = *(int *)arg; - pthread_spin_unlock(&worker_lock); - printf("start thread %d\n", thread_id); sockfd = socket(AF_INET, SOCK_STREAM, 0); printf("thread %d, sockfd:%d\n", thread_id, sockfd); if (sockfd < 0) { printf("thread %d, ff_socket failed\n", thread_id); + pthread_spin_unlock(&worker_lock); exit(1); } + /* socket will init adapter,so unlock after socket */ + pthread_spin_unlock(&worker_lock); + int on = 1; ioctl(sockfd, FIONBIO, &on); @@ -158,7 +160,7 @@ void *loop(void *arg) close(events[i].data.fd); } } else { - printf("thread %d, unknown event: %8.8X\n", events[i].events); + printf("thread %d, unknown event: %8.8X\n", thread_id, events[i].events); } } } @@ -190,7 +192,7 @@ int main(int argc, char * argv[]) pthread_spin_destroy(&worker_lock); return -1; } - sleep(5); + //sleep(1); } for (i = 0; i < worker_num; i++) { From c0677726be5ffcd2df7b93eb141660d65082313f Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Fri, 7 Apr 2023 11:27:34 +0800 Subject: [PATCH 09/43] Add main_stack_epoll_pipeline, and must disable FF_THREAD_SOCKET before build adapter. --- example/Makefile | 1 + example/main_stack_epoll_pipeline.c | 181 ++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 example/main_stack_epoll_pipeline.c diff --git a/example/Makefile b/example/Makefile index 2ca45b0fc..c0db5350c 100644 --- a/example/Makefile +++ b/example/Makefile @@ -21,6 +21,7 @@ all: cc ${CFLAGS} -DINET6 -o ${TARGET} main.c ${LIBS} cc ${CFLAGS} -o ${TARGET}_epoll main_epoll.c ${LIBS} cc ${CFLAGS} -I ../adapter -o ${TARGET}_stack_epoll main_stack_epoll.c ${LIBS} + cc ${CFLAGS} -I ../adapter -o ${TARGET}_stack_epoll_pipeline main_stack_epoll_pipeline.c ${LIBS} .PHONY: clean clean: diff --git a/example/main_stack_epoll_pipeline.c b/example/main_stack_epoll_pipeline.c new file mode 100644 index 000000000..675dee24a --- /dev/null +++ b/example/main_stack_epoll_pipeline.c @@ -0,0 +1,181 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_WORKERS 128 +pthread_t hworker[MAX_WORKERS]; + +#define MAX_EVENTS 512 +struct epoll_event ev; +struct epoll_event events[MAX_EVENTS]; +int epfd; +int sockfd; + +static int exit_flag = 0; + +char html[] = +"HTTP/1.1 200 OK\r\n" +"Server: F-Stack\r\n" +"Date: Sat, 25 Feb 2017 09:26:33 GMT\r\n" +"Content-Type: text/html\r\n" +"Content-Length: 438\r\n" +"Last-Modified: Tue, 21 Feb 2017 09:44:03 GMT\r\n" +"Connection: keep-alive\r\n" +"Accept-Ranges: bytes\r\n" +"\r\n" +"\r\n" +"\r\n" +"\r\n" +"Welcome to F-Stack!\r\n" +"\r\n" +"\r\n" +"\r\n" +"

Welcome to F-Stack!

\r\n" +"\r\n" +"

For online documentation and support please refer to\r\n" +"F-Stack.org.
\r\n" +"\r\n" +"

Thank you for using F-Stack.

\r\n" +"\r\n" +""; + +void sig_term(int sig) +{ + printf("we caught signal %d, to exit helloworld\n", sig); + exit_flag = 1; + return; +} + +void *loop(void *arg) +{ + /* Wait for events to happen */ + while (!exit_flag) { + int nevents = epoll_wait(epfd, events, MAX_EVENTS, 100); + if (nevents <= 0) { + if (nevents) { + printf("hello world epoll wait ret %d, errno:%d, %s\n", + nevents, errno, strerror(errno)); + } + //usleep(100); + sleep(1); + } + printf("get nevents:%d\n", nevents); + int i; + + for (i = 0; i < nevents; ++i) { + /* Handle new connect */ + if (events[i].data.fd == sockfd) { + while (1) { + int nclientfd = accept(sockfd, NULL, NULL); + if (nclientfd < 0) { + break; + } + + /* Add to event list */ + ev.data.fd = nclientfd; + ev.events = EPOLLIN; + if (epoll_ctl(epfd, EPOLL_CTL_ADD, nclientfd, &ev) != 0) { + printf("ff_epoll_ctl failed:%d, %s\n", + errno, strerror(errno)); + break; + } + } + } else { + if (events[i].events & EPOLLERR ) { + /* Simply close socket */ + epoll_ctl(epfd, EPOLL_CTL_DEL, events[i].data.fd, NULL); + close(events[i].data.fd); + } else if (events[i].events & EPOLLIN) { + char buf[256]; + size_t readlen = read( events[i].data.fd, buf, sizeof(buf)); + if(readlen > 0) { + write( events[i].data.fd, html, sizeof(html) - 1); + } else { + epoll_ctl(epfd, EPOLL_CTL_DEL, events[i].data.fd, NULL); + close(events[i].data.fd); + } + } else { + printf("unknown event: %8.8X\n", events[i].events); + } + } + } + } +} + +int main(int argc, char * argv[]) +{ + int i, worker_num = 1; + + signal(SIGINT, sig_term); + signal(SIGTERM, sig_term); + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + printf("sockfd:%d\n", sockfd); + if (sockfd < 0) { + printf("ff_socket failed\n"); + exit(1); + } + + int on = 1; + ioctl(sockfd, FIONBIO, &on); + + struct sockaddr_in my_addr; + bzero(&my_addr, sizeof(my_addr)); + my_addr.sin_family = AF_INET; + my_addr.sin_port = htons(80); + my_addr.sin_addr.s_addr = htonl(INADDR_ANY); + + int ret = bind(sockfd, (const struct sockaddr *)&my_addr, sizeof(my_addr)); + if (ret < 0) { + printf("ff_bind failed\n"); + exit(1); + } + + ret = listen(sockfd, MAX_EVENTS); + if (ret < 0) { + printf("ff_listen failed\n"); + exit(1); + } + + epfd = epoll_create(0); + if (epfd <= 0) { + printf("ff_epoll_create failed, errno:%d, %s\n", + errno, strerror(errno)); + exit(1); + } + ev.data.fd = sockfd; + ev.events = EPOLLIN; + epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev); + + for (i = 0; i < worker_num; i++) { + if(pthread_create(&hworker[i], NULL, loop, (void *)&i) < 0) { + printf("create loop thread failed., errno:%d/%s\n", + errno, strerror(errno)); + return -1; + } + } + + for (i = 0; i < worker_num; i++) { + pthread_join(hworker[i], NULL); + } + + return 0; +} From 88318c48c4149ac63178e9b823081bff180d7bd2 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Fri, 7 Apr 2023 12:00:02 +0800 Subject: [PATCH 10/43] Fix lock issue. --- example/main_stack_epoll.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/example/main_stack_epoll.c b/example/main_stack_epoll.c index d7acc034b..dbc4809b3 100644 --- a/example/main_stack_epoll.c +++ b/example/main_stack_epoll.c @@ -181,10 +181,10 @@ int main(int argc, char * argv[]) } printf("to init %d workers.\n", worker_num); - pthread_spin_init(&worker_lock, PTHREAD_PROCESS_SHARED); + pthread_spin_init(&worker_lock, PTHREAD_PROCESS_PRIVATE); + pthread_spin_lock(&worker_lock); for (i = 0; i < worker_num; i++) { - pthread_spin_lock(&worker_lock); if(pthread_create(&hworker[i], NULL, loop, (void *)&i) < 0) { printf("create loop thread failed., errno:%d/%s\n", errno, strerror(errno)); @@ -192,6 +192,7 @@ int main(int argc, char * argv[]) pthread_spin_destroy(&worker_lock); return -1; } + pthread_spin_lock(&worker_lock); //sleep(1); } From e8ed03b52191b087d59bb2b6ea2db49ef48a880a Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Fri, 7 Apr 2023 13:44:41 +0800 Subject: [PATCH 11/43] Modify FF_SOCKET_THREAD disable by default. --- adapter/Makefile | 4 +- example/Makefile | 2 +- example/main_stack_epoll.c | 119 +++++++----------- ...ine.c => main_stack_epoll_thread_socket.c} | 119 +++++++++++------- 4 files changed, 122 insertions(+), 122 deletions(-) rename example/{main_stack_epoll_pipeline.c => main_stack_epoll_thread_socket.c} (71%) diff --git a/adapter/Makefile b/adapter/Makefile index e759f4a55..e6b8e6d4d 100644 --- a/adapter/Makefile +++ b/adapter/Makefile @@ -12,10 +12,10 @@ DEBUG=-O0 -gdwarf-2 -g3 # Per thread separate initialization dpdk lib and attach sc when needed, # such as listen same port in different threads, and socket can use in own thread. -# Default enable. +# Default disable. # # If disable it, one socket can use in all threads. -FF_THREAD_SOCKET=1 +#FF_THREAD_SOCKET=1 PKGCONF ?= pkg-config diff --git a/example/Makefile b/example/Makefile index c0db5350c..20ac50be2 100644 --- a/example/Makefile +++ b/example/Makefile @@ -21,7 +21,7 @@ all: cc ${CFLAGS} -DINET6 -o ${TARGET} main.c ${LIBS} cc ${CFLAGS} -o ${TARGET}_epoll main_epoll.c ${LIBS} cc ${CFLAGS} -I ../adapter -o ${TARGET}_stack_epoll main_stack_epoll.c ${LIBS} - cc ${CFLAGS} -I ../adapter -o ${TARGET}_stack_epoll_pipeline main_stack_epoll_pipeline.c ${LIBS} + cc ${CFLAGS} -I ../adapter -o ${TARGET}_stack_epoll_thread_socket main_stack_epoll_thread_socket.c ${LIBS} .PHONY: clean clean: diff --git a/example/main_stack_epoll.c b/example/main_stack_epoll.c index dbc4809b3..675dee24a 100644 --- a/example/main_stack_epoll.c +++ b/example/main_stack_epoll.c @@ -16,8 +16,12 @@ #define MAX_WORKERS 128 pthread_t hworker[MAX_WORKERS]; -pthread_spinlock_t worker_lock; + #define MAX_EVENTS 512 +struct epoll_event ev; +struct epoll_event events[MAX_EVENTS]; +int epfd; +int sockfd; static int exit_flag = 0; @@ -62,69 +66,18 @@ void sig_term(int sig) void *loop(void *arg) { - struct epoll_event ev; - struct epoll_event events[MAX_EVENTS]; - int epfd; - int sockfd; - int thread_id; - - thread_id = *(int *)arg; - printf("start thread %d\n", thread_id); - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - printf("thread %d, sockfd:%d\n", thread_id, sockfd); - if (sockfd < 0) { - printf("thread %d, ff_socket failed\n", thread_id); - pthread_spin_unlock(&worker_lock); - exit(1); - } - - /* socket will init adapter,so unlock after socket */ - pthread_spin_unlock(&worker_lock); - - int on = 1; - ioctl(sockfd, FIONBIO, &on); - - struct sockaddr_in my_addr; - bzero(&my_addr, sizeof(my_addr)); - my_addr.sin_family = AF_INET; - my_addr.sin_port = htons(80); - my_addr.sin_addr.s_addr = htonl(INADDR_ANY); - - int ret = bind(sockfd, (const struct sockaddr *)&my_addr, sizeof(my_addr)); - if (ret < 0) { - printf("thread %d, ff_bind failed\n", thread_id); - exit(1); - } - - ret = listen(sockfd, MAX_EVENTS); - if (ret < 0) { - printf("thread %d, ff_listen failed\n", thread_id); - exit(1); - } - - epfd = epoll_create(0); - if (epfd <= 0) { - printf("thread %d, ff_epoll_create failed, errno:%d, %s\n", - thread_id, errno, strerror(errno)); - exit(1); - } - ev.data.fd = sockfd; - ev.events = EPOLLIN; - epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev); - /* Wait for events to happen */ while (!exit_flag) { int nevents = epoll_wait(epfd, events, MAX_EVENTS, 100); if (nevents <= 0) { if (nevents) { - printf("thread %d, hello world epoll wait ret %d, errno:%d, %s\n", - thread_id, nevents, errno, strerror(errno)); + printf("hello world epoll wait ret %d, errno:%d, %s\n", + nevents, errno, strerror(errno)); } //usleep(100); sleep(1); } - printf("thread %d, get nevents:%d\n", thread_id, nevents); + printf("get nevents:%d\n", nevents); int i; for (i = 0; i < nevents; ++i) { @@ -140,8 +93,8 @@ void *loop(void *arg) ev.data.fd = nclientfd; ev.events = EPOLLIN; if (epoll_ctl(epfd, EPOLL_CTL_ADD, nclientfd, &ev) != 0) { - printf("thread %d, ff_epoll_ctl failed:%d, %s\n", - thread_id, errno, strerror(errno)); + printf("ff_epoll_ctl failed:%d, %s\n", + errno, strerror(errno)); break; } } @@ -160,7 +113,7 @@ void *loop(void *arg) close(events[i].data.fd); } } else { - printf("thread %d, unknown event: %8.8X\n", thread_id, events[i].events); + printf("unknown event: %8.8X\n", events[i].events); } } } @@ -169,38 +122,60 @@ void *loop(void *arg) int main(int argc, char * argv[]) { - int i, worker_num; + int i, worker_num = 1; signal(SIGINT, sig_term); signal(SIGTERM, sig_term); - if (argc == 1) { - worker_num = 1; - } else { - worker_num = atoi(argv[1]); + sockfd = socket(AF_INET, SOCK_STREAM, 0); + printf("sockfd:%d\n", sockfd); + if (sockfd < 0) { + printf("ff_socket failed\n"); + exit(1); } - printf("to init %d workers.\n", worker_num); - pthread_spin_init(&worker_lock, PTHREAD_PROCESS_PRIVATE); - pthread_spin_lock(&worker_lock); + int on = 1; + ioctl(sockfd, FIONBIO, &on); + + struct sockaddr_in my_addr; + bzero(&my_addr, sizeof(my_addr)); + my_addr.sin_family = AF_INET; + my_addr.sin_port = htons(80); + my_addr.sin_addr.s_addr = htonl(INADDR_ANY); + + int ret = bind(sockfd, (const struct sockaddr *)&my_addr, sizeof(my_addr)); + if (ret < 0) { + printf("ff_bind failed\n"); + exit(1); + } + + ret = listen(sockfd, MAX_EVENTS); + if (ret < 0) { + printf("ff_listen failed\n"); + exit(1); + } + + epfd = epoll_create(0); + if (epfd <= 0) { + printf("ff_epoll_create failed, errno:%d, %s\n", + errno, strerror(errno)); + exit(1); + } + ev.data.fd = sockfd; + ev.events = EPOLLIN; + epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev); for (i = 0; i < worker_num; i++) { if(pthread_create(&hworker[i], NULL, loop, (void *)&i) < 0) { printf("create loop thread failed., errno:%d/%s\n", errno, strerror(errno)); - pthread_spin_unlock(&worker_lock); - pthread_spin_destroy(&worker_lock); return -1; } - pthread_spin_lock(&worker_lock); - //sleep(1); } for (i = 0; i < worker_num; i++) { pthread_join(hworker[i], NULL); } - pthread_spin_destroy(&worker_lock); - return 0; } diff --git a/example/main_stack_epoll_pipeline.c b/example/main_stack_epoll_thread_socket.c similarity index 71% rename from example/main_stack_epoll_pipeline.c rename to example/main_stack_epoll_thread_socket.c index 675dee24a..dbc4809b3 100644 --- a/example/main_stack_epoll_pipeline.c +++ b/example/main_stack_epoll_thread_socket.c @@ -16,12 +16,8 @@ #define MAX_WORKERS 128 pthread_t hworker[MAX_WORKERS]; - +pthread_spinlock_t worker_lock; #define MAX_EVENTS 512 -struct epoll_event ev; -struct epoll_event events[MAX_EVENTS]; -int epfd; -int sockfd; static int exit_flag = 0; @@ -66,18 +62,69 @@ void sig_term(int sig) void *loop(void *arg) { + struct epoll_event ev; + struct epoll_event events[MAX_EVENTS]; + int epfd; + int sockfd; + int thread_id; + + thread_id = *(int *)arg; + printf("start thread %d\n", thread_id); + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + printf("thread %d, sockfd:%d\n", thread_id, sockfd); + if (sockfd < 0) { + printf("thread %d, ff_socket failed\n", thread_id); + pthread_spin_unlock(&worker_lock); + exit(1); + } + + /* socket will init adapter,so unlock after socket */ + pthread_spin_unlock(&worker_lock); + + int on = 1; + ioctl(sockfd, FIONBIO, &on); + + struct sockaddr_in my_addr; + bzero(&my_addr, sizeof(my_addr)); + my_addr.sin_family = AF_INET; + my_addr.sin_port = htons(80); + my_addr.sin_addr.s_addr = htonl(INADDR_ANY); + + int ret = bind(sockfd, (const struct sockaddr *)&my_addr, sizeof(my_addr)); + if (ret < 0) { + printf("thread %d, ff_bind failed\n", thread_id); + exit(1); + } + + ret = listen(sockfd, MAX_EVENTS); + if (ret < 0) { + printf("thread %d, ff_listen failed\n", thread_id); + exit(1); + } + + epfd = epoll_create(0); + if (epfd <= 0) { + printf("thread %d, ff_epoll_create failed, errno:%d, %s\n", + thread_id, errno, strerror(errno)); + exit(1); + } + ev.data.fd = sockfd; + ev.events = EPOLLIN; + epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev); + /* Wait for events to happen */ while (!exit_flag) { int nevents = epoll_wait(epfd, events, MAX_EVENTS, 100); if (nevents <= 0) { if (nevents) { - printf("hello world epoll wait ret %d, errno:%d, %s\n", - nevents, errno, strerror(errno)); + printf("thread %d, hello world epoll wait ret %d, errno:%d, %s\n", + thread_id, nevents, errno, strerror(errno)); } //usleep(100); sleep(1); } - printf("get nevents:%d\n", nevents); + printf("thread %d, get nevents:%d\n", thread_id, nevents); int i; for (i = 0; i < nevents; ++i) { @@ -93,8 +140,8 @@ void *loop(void *arg) ev.data.fd = nclientfd; ev.events = EPOLLIN; if (epoll_ctl(epfd, EPOLL_CTL_ADD, nclientfd, &ev) != 0) { - printf("ff_epoll_ctl failed:%d, %s\n", - errno, strerror(errno)); + printf("thread %d, ff_epoll_ctl failed:%d, %s\n", + thread_id, errno, strerror(errno)); break; } } @@ -113,7 +160,7 @@ void *loop(void *arg) close(events[i].data.fd); } } else { - printf("unknown event: %8.8X\n", events[i].events); + printf("thread %d, unknown event: %8.8X\n", thread_id, events[i].events); } } } @@ -122,60 +169,38 @@ void *loop(void *arg) int main(int argc, char * argv[]) { - int i, worker_num = 1; + int i, worker_num; signal(SIGINT, sig_term); signal(SIGTERM, sig_term); - sockfd = socket(AF_INET, SOCK_STREAM, 0); - printf("sockfd:%d\n", sockfd); - if (sockfd < 0) { - printf("ff_socket failed\n"); - exit(1); + if (argc == 1) { + worker_num = 1; + } else { + worker_num = atoi(argv[1]); } + printf("to init %d workers.\n", worker_num); - int on = 1; - ioctl(sockfd, FIONBIO, &on); - - struct sockaddr_in my_addr; - bzero(&my_addr, sizeof(my_addr)); - my_addr.sin_family = AF_INET; - my_addr.sin_port = htons(80); - my_addr.sin_addr.s_addr = htonl(INADDR_ANY); - - int ret = bind(sockfd, (const struct sockaddr *)&my_addr, sizeof(my_addr)); - if (ret < 0) { - printf("ff_bind failed\n"); - exit(1); - } - - ret = listen(sockfd, MAX_EVENTS); - if (ret < 0) { - printf("ff_listen failed\n"); - exit(1); - } - - epfd = epoll_create(0); - if (epfd <= 0) { - printf("ff_epoll_create failed, errno:%d, %s\n", - errno, strerror(errno)); - exit(1); - } - ev.data.fd = sockfd; - ev.events = EPOLLIN; - epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev); + pthread_spin_init(&worker_lock, PTHREAD_PROCESS_PRIVATE); + pthread_spin_lock(&worker_lock); for (i = 0; i < worker_num; i++) { if(pthread_create(&hworker[i], NULL, loop, (void *)&i) < 0) { printf("create loop thread failed., errno:%d/%s\n", errno, strerror(errno)); + pthread_spin_unlock(&worker_lock); + pthread_spin_destroy(&worker_lock); return -1; } + pthread_spin_lock(&worker_lock); + //sleep(1); } for (i = 0; i < worker_num; i++) { pthread_join(hworker[i], NULL); } + pthread_spin_destroy(&worker_lock); + return 0; } From f27d862f85c095fd56663d8a309b5c427c34d178 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Fri, 7 Apr 2023 20:39:31 +0800 Subject: [PATCH 12/43] 1. Add kevent demo, need link libff_syscall.so, not LD_PRELOAD. 2. Fix some issues. --- adapter/ff_hook_syscall.c | 22 ++- adapter/ff_hook_syscall.h | 4 + adapter/ff_socket_ops.c | 10 +- example/Makefile | 6 +- example/main_stack.c | 210 ++++++++++++++++++++ example/main_stack_epoll.c | 32 ++- example/main_stack_epoll_thread_socket.c | 29 ++- example/main_stack_thread_socket.c | 239 +++++++++++++++++++++++ 8 files changed, 534 insertions(+), 18 deletions(-) create mode 100644 example/main_stack.c create mode 100644 example/main_stack_thread_socket.c diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index a6bb0895c..eb775c738 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -846,6 +846,8 @@ ff_hook_recvmsg(int fd, struct msghdr *msg, int flags) ssize_t ff_hook_read(int fd, void *buf, size_t len) { + DEBUG_LOG("ff_hook_read, fd:%d, len:%lu\n", fd, len); + if (buf == NULL || len == 0) { errno = EINVAL; return -1; @@ -1000,6 +1002,8 @@ ff_hook_send(int fd, const void *buf, size_t len, int flags) ssize_t ff_hook_write(int fd, const void *buf, size_t len) { + DEBUG_LOG("ff_hook_write, fd:%d, len:%lu\n", fd, len); + if (buf == NULL || len == 0) { errno = EINVAL; return -1; @@ -1062,6 +1066,8 @@ ff_hook_writev(int fd, const struct iovec *iov, int iovcnt) int ff_hook_close(int fd) { + DEBUG_LOG("ff_hook_close, fd:%d\n", fd); + CHECK_FD_OWNERSHIP(close, (fd)); DEFINE_REQ_ARGS(close); @@ -1288,6 +1294,8 @@ kqueue() { int ret = -1; + DEBUG_LOG("run kqueue\n"); + if (unlikely(inited == 0)) { errno = ENOSYS; return -1; @@ -1299,6 +1307,8 @@ kqueue() ret = convert_fstack_fd(ret); } + DEBUG_LOG("get fd:%d\n", ret); + return ret; } @@ -1307,6 +1317,11 @@ kevent(int kq, const struct kevent *changelist, int nchanges, struct kevent *eventlist, int nevents, const struct timespec *timeout) { + int i; + struct kevent *kev; + + DEBUG_LOG("kq:%d, nchanges:%d, nevents:%d\n", kq, nchanges, nevents); + if (unlikely(inited == 0 && ff_adapter_init() < 0)) { errno = ENOSYS; return -1; @@ -1326,8 +1341,6 @@ kevent(int kq, const struct kevent *changelist, int nchanges, rte_memcpy(sh_changelist, changelist, sizeof(struct kevent) * nchanges); - struct kevent *kev; - int i = 0; for(i = 0; i < nchanges; i++) { kev = (struct kevent *)&sh_changelist[i]; switch (kev->filter) { @@ -1402,6 +1415,11 @@ kevent(int kq, const struct kevent *changelist, int nchanges, if (eventlist && nevents) { rte_memcpy(eventlist, sh_eventlist, sizeof(struct kevent) * ret); + + for (i = 0; i < nevents; i++) { + kev = &eventlist[i]; + kev->ident = convert_fstack_fd(kev->ident); + } } } diff --git a/adapter/ff_hook_syscall.h b/adapter/ff_hook_syscall.h index a88a28544..66ad9828e 100644 --- a/adapter/ff_hook_syscall.h +++ b/adapter/ff_hook_syscall.h @@ -5,4 +5,8 @@ #define FF_SYSCALL_DECL(ret, fn, args) extern ret ff_hook_##fn args #include +extern int kqueue(void); +extern int kevent(int kq, const struct kevent *changelist, int nchanges, + struct kevent *eventlist, int nevents, const struct timespec *timeout); + #endif diff --git a/adapter/ff_socket_ops.c b/adapter/ff_socket_ops.c index baef7015f..c9038d08e 100644 --- a/adapter/ff_socket_ops.c +++ b/adapter/ff_socket_ops.c @@ -207,6 +207,7 @@ ff_sys_recvmsg(struct ff_recvmsg_args *args) static ssize_t ff_sys_read(struct ff_read_args *args) { + DEBUG_LOG("ff_sys_read, fd:%d, len:%lu\n", args->fd, args->len); return ff_read(args->fd, args->buf, args->len); } @@ -238,6 +239,7 @@ ff_sys_sendmsg(struct ff_sendmsg_args *args) static ssize_t ff_sys_write(struct ff_write_args *args) { + DEBUG_LOG("ff_sys_write, fd:%d, len:%lu\n", args->fd, args->len); return ff_write(args->fd, args->buf, args->len); } @@ -250,6 +252,7 @@ ff_sys_writev(struct ff_writev_args *args) static int ff_sys_close(struct ff_close_args *args) { + DEBUG_LOG("ff_sys_close, fd:%d\n", args->fd); sockaddr_unbind(args->fd); return ff_close(args->fd); } @@ -389,6 +392,9 @@ ff_handle_kevent(struct ff_so_context *sc) { struct ff_kevent_args *ka = sc->args; + DEBUG_LOG("ff_handle_kevent sc:%p, status:%d, ops:%d, kq:%d, nchanges:%d, nevents:%d\n", + sc, sc->status, sc->ops, ka->kq, ka->nchanges, ka->nevents); + errno = 0; sc->result = ff_sys_kevent(ka); sc->error = errno; @@ -397,9 +403,9 @@ ff_handle_kevent(struct ff_so_context *sc) ka->nchanges = 0; } - if (sc->result == 0 && ka->nevents != 0) { + /*if (sc->result == 0 && ka->nevents != 0) { return; - } + }*/ sc->status = FF_SC_REP; sem_post(&sc->wait_sem); diff --git a/example/Makefile b/example/Makefile index 20ac50be2..11fa396e2 100644 --- a/example/Makefile +++ b/example/Makefile @@ -20,8 +20,10 @@ TARGET="helloworld" all: cc ${CFLAGS} -DINET6 -o ${TARGET} main.c ${LIBS} cc ${CFLAGS} -o ${TARGET}_epoll main_epoll.c ${LIBS} - cc ${CFLAGS} -I ../adapter -o ${TARGET}_stack_epoll main_stack_epoll.c ${LIBS} - cc ${CFLAGS} -I ../adapter -o ${TARGET}_stack_epoll_thread_socket main_stack_epoll_thread_socket.c ${LIBS} + cc ${CFLAGS} -I ${FF_PATH}/adapter -L ${FF_PATH}/adapter -lff_syscall -o ${TARGET}_stack main_stack.c ${LIBS} + cc ${CFLAGS} -I ${FF_PATH}/adapter -L ${FF_PATH}/adapter -lff_syscall -o ${TARGET}_stack_thread_socket main_stack_thread_socket.c ${LIBS} + cc ${CFLAGS} -I ${FF_PATH}/adapter -o ${TARGET}_stack_epoll main_stack_epoll.c ${LIBS} + cc ${CFLAGS} -I ${FF_PATH}/adapter -o ${TARGET}_stack_epoll_thread_socket main_stack_epoll_thread_socket.c ${LIBS} .PHONY: clean clean: diff --git a/example/main_stack.c b/example/main_stack.c new file mode 100644 index 000000000..9de5bde7a --- /dev/null +++ b/example/main_stack.c @@ -0,0 +1,210 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +//#include "ff_config.h" +//#include "ff_api.h" +#include "ff_event.h" +#include "ff_adapter.h" +#include "ff_hook_syscall.h" + +pthread_t hworker; + +#define MAX_EVENTS 512 + +/* kevent set */ +struct kevent kevSet; +/* events */ +struct kevent events[MAX_EVENTS]; +/* kq */ +int kq; +int sockfd; + +static int exit_flag = 0; + +char html[] = +"HTTP/1.1 200 OK\r\n" +"Server: F-Stack\r\n" +"Date: Sat, 25 Feb 2017 09:26:33 GMT\r\n" +"Content-Type: text/html\r\n" +"Content-Length: 438\r\n" +"Last-Modified: Tue, 21 Feb 2017 09:44:03 GMT\r\n" +"Connection: keep-alive\r\n" +"Accept-Ranges: bytes\r\n" +"\r\n" +"\r\n" +"\r\n" +"\r\n" +"Welcome to F-Stack!\r\n" +"\r\n" +"\r\n" +"\r\n" +"

Welcome to F-Stack!

\r\n" +"\r\n" +"

For online documentation and support please refer to\r\n" +"F-Stack.org.
\r\n" +"\r\n" +"

Thank you for using F-Stack.

\r\n" +"\r\n" +""; + +void sig_term(int sig) +{ + printf("we caught signal %d, to exit helloworld\n", sig); + exit_flag = 1; + return; +} + +void *loop(void *arg) +{ + /* Wait for events to happen */ + while (!exit_flag) { + int nevents = kevent(kq, NULL, 0, events, MAX_EVENTS, NULL); + int i; + + if (nevents <= 0) { + if (nevents) { + printf("ff_kevent failed:%d, %s\n", errno, + strerror(errno)); + return NULL; + } + //usleep(100); + sleep(1); + } + printf("get nevents:%d\n", nevents); + + for (i = 0; i < nevents; ++i) { + struct kevent event = events[i]; + int clientfd = (int)event.ident; + + /* Handle disconnect */ + if (event.flags & EV_EOF) { + /* Simply close socket */ + close(clientfd); + } else if (clientfd == sockfd) { + int available = (int)event.data; + do { + int nclientfd = accept(clientfd, NULL, NULL); + if (nclientfd < 0) { + printf("ff_accept failed:%d, %s\n", errno, + strerror(errno)); + break; + } + + /* Add to event list */ + EV_SET(&kevSet, nclientfd, EVFILT_READ, EV_ADD, 0, 0, NULL); + + if(kevent(kq, &kevSet, 1, NULL, 0, NULL) < 0) { + printf("ff_kevent error:%d, %s\n", errno, + strerror(errno)); + close(nclientfd); + break; + } + + available--; + } while (available); + } else if (event.filter == EVFILT_READ) { + char buf[256]; + ssize_t readlen = read(clientfd, buf, sizeof(buf)); + ssize_t writelen = write(clientfd, html, sizeof(html) - 1); + if (writelen < 0){ + printf("ff_write failed:%d, %s\n", errno, + strerror(errno)); + close(clientfd); + } + } else { + printf("unknown event: %8.8X\n", event.flags); + } + } + } + + return NULL; +} + +int main(int argc, char * argv[]) +{ + signal(SIGINT, sig_term); + signal(SIGTERM, sig_term); + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + printf("sockfd:%d\n", sockfd); + if (sockfd < 0) { + printf("ff_socket failed, sockfd:%d, errno:%d, %s\n", sockfd, errno, strerror(errno)); + return -1;; + } + + /* Set non blocking */ + int on = 1; + ioctl(sockfd, FIONBIO, &on); + + struct sockaddr_in my_addr; + bzero(&my_addr, sizeof(my_addr)); + my_addr.sin_family = AF_INET; + my_addr.sin_port = htons(80); + my_addr.sin_addr.s_addr = htonl(INADDR_ANY); + + int ret = bind(sockfd, (const struct sockaddr *)&my_addr, sizeof(my_addr)); + if (ret < 0) { + printf("ff_bind failed, sockfd:%d, errno:%d, %s\n", sockfd, errno, strerror(errno)); + close(sockfd); + return -1; + } + + ret = listen(sockfd, MAX_EVENTS); + if (ret < 0) { + printf("ff_listen failed, sockfd:%d, errno:%d, %s\n", sockfd, errno, strerror(errno)); + close(sockfd); + return -1; + } + + kq = kqueue(); + printf("kq:%d\n", kq); + if (kq < 0) { + printf("ff_kqueue failed, errno:%d, %s\n", errno, strerror(errno)); + close(sockfd); + return -1; + } + + EV_SET(&kevSet, sockfd, EVFILT_READ, EV_ADD, 0, MAX_EVENTS, NULL); + /* Update kqueue */ + ret = kevent(kq, &kevSet, 1, NULL, 0, NULL); + if (ret < 0) { + printf("kevent failed\n"); + close(kq); + close(sockfd); + return -1; + } + + if(pthread_create(&hworker, NULL, loop, NULL) < 0) { + printf("create loop thread failed., errno:%d/%s\n", + errno, strerror(errno)); + close(kq); + close(sockfd); + return -1; + } + + pthread_join(hworker, NULL); + + close(kq); + close(sockfd); + + return 0; +} diff --git a/example/main_stack_epoll.c b/example/main_stack_epoll.c index 675dee24a..d012dedc1 100644 --- a/example/main_stack_epoll.c +++ b/example/main_stack_epoll.c @@ -69,16 +69,18 @@ void *loop(void *arg) /* Wait for events to happen */ while (!exit_flag) { int nevents = epoll_wait(epfd, events, MAX_EVENTS, 100); + int i; + if (nevents <= 0) { if (nevents) { printf("hello world epoll wait ret %d, errno:%d, %s\n", nevents, errno, strerror(errno)); + break; } //usleep(100); sleep(1); } printf("get nevents:%d\n", nevents); - int i; for (i = 0; i < nevents; ++i) { /* Handle new connect */ @@ -95,6 +97,7 @@ void *loop(void *arg) if (epoll_ctl(epfd, EPOLL_CTL_ADD, nclientfd, &ev) != 0) { printf("ff_epoll_ctl failed:%d, %s\n", errno, strerror(errno)); + close(nclientfd); break; } } @@ -118,6 +121,8 @@ void *loop(void *arg) } } } + + return NULL; } int main(int argc, char * argv[]) @@ -131,7 +136,7 @@ int main(int argc, char * argv[]) printf("sockfd:%d\n", sockfd); if (sockfd < 0) { printf("ff_socket failed\n"); - exit(1); + return -1; } int on = 1; @@ -146,29 +151,41 @@ int main(int argc, char * argv[]) int ret = bind(sockfd, (const struct sockaddr *)&my_addr, sizeof(my_addr)); if (ret < 0) { printf("ff_bind failed\n"); - exit(1); + close(sockfd); + return -1; } ret = listen(sockfd, MAX_EVENTS); if (ret < 0) { printf("ff_listen failed\n"); - exit(1); + close(sockfd); + return -1; } epfd = epoll_create(0); if (epfd <= 0) { printf("ff_epoll_create failed, errno:%d, %s\n", errno, strerror(errno)); - exit(1); + close(sockfd); + return -1; } + ev.data.fd = sockfd; ev.events = EPOLLIN; - epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev); + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev); + if (ret < 0) { + printf("ff_listen failed\n"); + close(epfd); + close(sockfd); + return -1; + } for (i = 0; i < worker_num; i++) { if(pthread_create(&hworker[i], NULL, loop, (void *)&i) < 0) { printf("create loop thread failed., errno:%d/%s\n", errno, strerror(errno)); + close(epfd); + close(sockfd); return -1; } } @@ -177,5 +194,8 @@ int main(int argc, char * argv[]) pthread_join(hworker[i], NULL); } + close(epfd); + close(sockfd); + return 0; } diff --git a/example/main_stack_epoll_thread_socket.c b/example/main_stack_epoll_thread_socket.c index dbc4809b3..4a2e0faf3 100644 --- a/example/main_stack_epoll_thread_socket.c +++ b/example/main_stack_epoll_thread_socket.c @@ -76,7 +76,7 @@ void *loop(void *arg) if (sockfd < 0) { printf("thread %d, ff_socket failed\n", thread_id); pthread_spin_unlock(&worker_lock); - exit(1); + return NULL; } /* socket will init adapter,so unlock after socket */ @@ -94,38 +94,49 @@ void *loop(void *arg) int ret = bind(sockfd, (const struct sockaddr *)&my_addr, sizeof(my_addr)); if (ret < 0) { printf("thread %d, ff_bind failed\n", thread_id); - exit(1); + close(sockfd); + return NULL; } ret = listen(sockfd, MAX_EVENTS); if (ret < 0) { printf("thread %d, ff_listen failed\n", thread_id); - exit(1); + close(sockfd); + return NULL; } epfd = epoll_create(0); if (epfd <= 0) { printf("thread %d, ff_epoll_create failed, errno:%d, %s\n", thread_id, errno, strerror(errno)); - exit(1); + close(sockfd); + return NULL; } ev.data.fd = sockfd; ev.events = EPOLLIN; - epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev); + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev); + if (ret < 0) { + printf("ff_listen failed\n"); + close(epfd); + close(sockfd); + return NULL; + } /* Wait for events to happen */ while (!exit_flag) { int nevents = epoll_wait(epfd, events, MAX_EVENTS, 100); + int i; + if (nevents <= 0) { if (nevents) { printf("thread %d, hello world epoll wait ret %d, errno:%d, %s\n", thread_id, nevents, errno, strerror(errno)); + break; } //usleep(100); sleep(1); } printf("thread %d, get nevents:%d\n", thread_id, nevents); - int i; for (i = 0; i < nevents; ++i) { /* Handle new connect */ @@ -142,6 +153,7 @@ void *loop(void *arg) if (epoll_ctl(epfd, EPOLL_CTL_ADD, nclientfd, &ev) != 0) { printf("thread %d, ff_epoll_ctl failed:%d, %s\n", thread_id, errno, strerror(errno)); + close(nclientfd); break; } } @@ -165,6 +177,11 @@ void *loop(void *arg) } } } + + close(epfd); + close(sockfd); + + return NULL; } int main(int argc, char * argv[]) diff --git a/example/main_stack_thread_socket.c b/example/main_stack_thread_socket.c new file mode 100644 index 000000000..1ca97e29f --- /dev/null +++ b/example/main_stack_thread_socket.c @@ -0,0 +1,239 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +//#include "ff_config.h" +//#include "ff_api.h" +#include "ff_event.h" +#include "ff_adapter.h" +#include "ff_hook_syscall.h" + +#define MAX_WORKERS 128 +pthread_t hworker[MAX_WORKERS]; +pthread_spinlock_t worker_lock; +#define MAX_EVENTS 512 + +static int exit_flag = 0; + +char html[] = +"HTTP/1.1 200 OK\r\n" +"Server: F-Stack\r\n" +"Date: Sat, 25 Feb 2017 09:26:33 GMT\r\n" +"Content-Type: text/html\r\n" +"Content-Length: 438\r\n" +"Last-Modified: Tue, 21 Feb 2017 09:44:03 GMT\r\n" +"Connection: keep-alive\r\n" +"Accept-Ranges: bytes\r\n" +"\r\n" +"\r\n" +"\r\n" +"\r\n" +"Welcome to F-Stack!\r\n" +"\r\n" +"\r\n" +"\r\n" +"

Welcome to F-Stack!

\r\n" +"\r\n" +"

For online documentation and support please refer to\r\n" +"F-Stack.org.
\r\n" +"\r\n" +"

Thank you for using F-Stack.

\r\n" +"\r\n" +""; + +void sig_term(int sig) +{ + printf("we caught signal %d, to exit helloworld\n", sig); + exit_flag = 1; + return; +} + +void *loop(void *arg) +{ + /* kevent set */ + struct kevent kevSet; + /* events */ + struct kevent events[MAX_EVENTS]; + /* kq */ + int kq; + int sockfd; + + int thread_id; + + thread_id = *(int *)arg; + printf("start thread %d\n", thread_id); + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + printf("thread %d, sockfd:%d\n", thread_id, sockfd); + if (sockfd < 0) { + printf("thread %d, ff_socket failed\n", thread_id); + pthread_spin_unlock(&worker_lock); + return NULL; + } + + /* socket will init adapter,so unlock after socket */ + pthread_spin_unlock(&worker_lock); + + int on = 1; + ioctl(sockfd, FIONBIO, &on); + + struct sockaddr_in my_addr; + bzero(&my_addr, sizeof(my_addr)); + my_addr.sin_family = AF_INET; + my_addr.sin_port = htons(80); + my_addr.sin_addr.s_addr = htonl(INADDR_ANY); + + int ret = bind(sockfd, (const struct sockaddr *)&my_addr, sizeof(my_addr)); + if (ret < 0) { + printf("thread %d, ff_bind failed\n", thread_id); + close(sockfd); + return NULL; + } + + ret = listen(sockfd, MAX_EVENTS); + if (ret < 0) { + printf("thread %d, ff_listen failed\n", thread_id); + close(sockfd); + return NULL; + } + + + kq = kqueue(); + printf("thread %d, kq:%d\n", thread_id, kq); + if (kq < 0) { + printf("thread %d, ff_kqueue failed, errno:%d, %s\n", thread_id, errno, strerror(errno)); + close(sockfd); + return NULL; + } + + EV_SET(&kevSet, sockfd, EVFILT_READ, EV_ADD, 0, MAX_EVENTS, NULL); + /* Update kqueue */ + ret = kevent(kq, &kevSet, 1, NULL, 0, NULL); + if (ret < 0) { + printf("thread %d, kevent failed\n", thread_id); + close(kq); + close(sockfd); + return NULL; + } + + /* Wait for events to happen */ + while (!exit_flag) { + int nevents = kevent(kq, NULL, 0, events, MAX_EVENTS, NULL); + int i; + + if (nevents <= 0) { + if (nevents) { + printf("thread %d, ff_kevent failed:%d, %s\n", thread_id, errno, + strerror(errno)); + return NULL; + } + //usleep(100); + sleep(1); + } + printf("thread %d, get nevents:%d\n", thread_id, nevents); + + for (i = 0; i < nevents; ++i) { + struct kevent event = events[i]; + int clientfd = (int)event.ident; + + /* Handle disconnect */ + if (event.flags & EV_EOF) { + /* Simply close socket */ + close(clientfd); + } else if (clientfd == sockfd) { + int available = (int)event.data; + do { + int nclientfd = accept(clientfd, NULL, NULL); + if (nclientfd < 0) { + printf("thread %d, ff_accept failed:%d, %s\n", thread_id, errno, + strerror(errno)); + break; + } + + /* Add to event list */ + EV_SET(&kevSet, nclientfd, EVFILT_READ, EV_ADD, 0, 0, NULL); + + if(kevent(kq, &kevSet, 1, NULL, 0, NULL) < 0) { + printf("thread %d, ff_kevent error:%d, %s\n", thread_id, errno, + strerror(errno)); + close(nclientfd); + break; + } + + available--; + } while (available); + } else if (event.filter == EVFILT_READ) { + char buf[256]; + ssize_t readlen = read(clientfd, buf, sizeof(buf)); + ssize_t writelen = write(clientfd, html, sizeof(html) - 1); + if (writelen < 0){ + printf("thread %d, ff_write failed:%d, %s\n", thread_id, errno, + strerror(errno)); + close(clientfd); + } + } else { + printf("thread %d, unknown event: %8.8X\n", thread_id, event.flags); + } + } + } + + close(kq); + close(sockfd); + + return NULL; +} + +int main(int argc, char * argv[]) +{ + int i, worker_num; + + signal(SIGINT, sig_term); + signal(SIGTERM, sig_term); + + if (argc == 1) { + worker_num = 1; + } else { + worker_num = atoi(argv[1]); + } + printf("to init %d workers.\n", worker_num); + + pthread_spin_init(&worker_lock, PTHREAD_PROCESS_PRIVATE); + pthread_spin_lock(&worker_lock); + + for (i = 0; i < worker_num; i++) { + if(pthread_create(&hworker[i], NULL, loop, (void *)&i) < 0) { + printf("create loop thread failed., errno:%d/%s\n", + errno, strerror(errno)); + pthread_spin_unlock(&worker_lock); + pthread_spin_destroy(&worker_lock); + return -1; + } + pthread_spin_lock(&worker_lock); + } + + for (i = 0; i < worker_num; i++) { + pthread_join(hworker[i], NULL); + } + + pthread_spin_destroy(&worker_lock); + + return 0; +} From 8766cedde13765cd098cf02bf8338f158891acc2 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Sat, 8 Apr 2023 09:29:50 +0800 Subject: [PATCH 13/43] Add timeout in kevent. --- adapter/ff_hook_syscall.c | 1 + example/main_stack.c | 6 ++++-- example/main_stack_thread_socket.c | 6 ++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index eb775c738..f3369953a 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -71,6 +71,7 @@ rte_spinlock_unlock(&sc->lock); \ } while (0) +/* NOTE: deadlock prone while fstack adapter run error */ #define SYSCALL(op, arg) do { \ ACQUIRE_ZONE_LOCK(FF_SC_IDLE); \ sc->ops = (op); \ diff --git a/example/main_stack.c b/example/main_stack.c index 9de5bde7a..56201b5f1 100644 --- a/example/main_stack.c +++ b/example/main_stack.c @@ -32,6 +32,8 @@ struct kevent events[MAX_EVENTS]; int kq; int sockfd; +struct timespec timeout = {0, 100000}; + static int exit_flag = 0; char html[] = @@ -77,7 +79,7 @@ void *loop(void *arg) { /* Wait for events to happen */ while (!exit_flag) { - int nevents = kevent(kq, NULL, 0, events, MAX_EVENTS, NULL); + int nevents = kevent(kq, NULL, 0, events, MAX_EVENTS, &timeout); int i; if (nevents <= 0) { @@ -185,7 +187,7 @@ int main(int argc, char * argv[]) EV_SET(&kevSet, sockfd, EVFILT_READ, EV_ADD, 0, MAX_EVENTS, NULL); /* Update kqueue */ - ret = kevent(kq, &kevSet, 1, NULL, 0, NULL); + ret = kevent(kq, &kevSet, 1, NULL, 0, &timeout); if (ret < 0) { printf("kevent failed\n"); close(kq); diff --git a/example/main_stack_thread_socket.c b/example/main_stack_thread_socket.c index 1ca97e29f..4d58c13ab 100644 --- a/example/main_stack_thread_socket.c +++ b/example/main_stack_thread_socket.c @@ -25,6 +25,8 @@ pthread_t hworker[MAX_WORKERS]; pthread_spinlock_t worker_lock; #define MAX_EVENTS 512 +struct timespec timeout = {0, 100000}; + static int exit_flag = 0; char html[] = @@ -126,7 +128,7 @@ void *loop(void *arg) EV_SET(&kevSet, sockfd, EVFILT_READ, EV_ADD, 0, MAX_EVENTS, NULL); /* Update kqueue */ - ret = kevent(kq, &kevSet, 1, NULL, 0, NULL); + ret = kevent(kq, &kevSet, 1, NULL, 0, &timeout); if (ret < 0) { printf("thread %d, kevent failed\n", thread_id); close(kq); @@ -136,7 +138,7 @@ void *loop(void *arg) /* Wait for events to happen */ while (!exit_flag) { - int nevents = kevent(kq, NULL, 0, events, MAX_EVENTS, NULL); + int nevents = kevent(kq, NULL, 0, events, MAX_EVENTS, &timeout); int i; if (nevents <= 0) { From 1d09ad5003e885edb785c7fa9118fb0791f3295d Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Tue, 11 Apr 2023 17:51:34 +0800 Subject: [PATCH 14/43] Fixed some issue that check the return value of sem_timedwait. --- adapter/ff_hook_syscall.c | 42 +++++++++++++++++++++++++++++---------- adapter/ff_socket_ops.h | 6 +++--- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index f3369953a..73ea9c41c 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -1233,6 +1233,7 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, sc->args = args; RELEASE_ZONE_LOCK(FF_SC_REQ); + errno = 0; if (timeout > 0) { struct timespec abs_timeout; @@ -1256,11 +1257,16 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, rte_spinlock_lock(&sc->lock); - if (ret == -1 && sc->status == FF_SC_REQ) { - if (errno == ETIMEDOUT) { - ret = 0; - } - } else { + /* + * After sem_timedwait, and before lock sc, sc->status may be modify from FF_SC_REQ to FF_SC_RSP, + * so it can't use to check. + * + * And only ret == 0, means sem_timedwait return normal, + * can set ret = sc->result, otherwise may use last sc->result. + */ + if (ret == -1 && errno == ETIMEDOUT /* sc->status == FF_SC_REQ */) { + ret = 0; + } else if (ret == 0) { ret = sc->result; if (ret < 0) { errno = sc->error; @@ -1272,6 +1278,11 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, if (ret > 0) { int i; + if (unlikely(ret > maxevents)) { + ERR_LOG("return events:%d, maxevents:%d, set return events to maxevents, may be some error occur\n", + ret, maxevents); + ret = maxevents; + } for (i = 0; i < ret; i++) { rte_memcpy(&events[i], &sh_events[i], sizeof(struct epoll_event)); } @@ -1319,6 +1330,7 @@ kevent(int kq, const struct kevent *changelist, int nchanges, const struct timespec *timeout) { int i; + int maxevents = nevents; struct kevent *kev; DEBUG_LOG("kq:%d, nchanges:%d, nevents:%d\n", kq, nchanges, nevents); @@ -1397,11 +1409,16 @@ kevent(int kq, const struct kevent *changelist, int nchanges, rte_spinlock_lock(&sc->lock); - if (ret == -1 && sc->status == FF_SC_REQ) { - if (errno == ETIMEDOUT) { - ret = 0; - } - } else { + /* + * After sem_timedwait, and before lock sc, sc->status may be modify from FF_SC_REQ to FF_SC_RSP, + * so it can't use to check. + * + * And only ret == 0, means sem_timedwait return normal, + * can set ret = sc->result, otherwise may use last sc->result. + */ + if (ret == -1 && errno == ETIMEDOUT /* sc->status == FF_SC_REQ */) { + ret = 0; + } else if (ret == 0) { ret = sc->result; if (ret < 0) { errno = sc->error; @@ -1414,6 +1431,11 @@ kevent(int kq, const struct kevent *changelist, int nchanges, if (ret > 0) { if (eventlist && nevents) { + if (unlikely(nevents > maxevents)) { + ERR_LOG("return events:%d, maxevents:%d, set return events to maxevents, may be some error occur\n", + nevents, maxevents); + nevents = maxevents; + } rte_memcpy(eventlist, sh_eventlist, sizeof(struct kevent) * ret); diff --git a/adapter/ff_socket_ops.h b/adapter/ff_socket_ops.h index 761549c26..7378b8415 100644 --- a/adapter/ff_socket_ops.h +++ b/adapter/ff_socket_ops.h @@ -39,7 +39,7 @@ enum FF_SOCKET_OPS { FF_SO_GETSOCKOPT, FF_SO_SETSOCKOPT, FF_SO_ACCEPT, - FF_SO_ACCEPT4, + FF_SO_ACCEPT4, // 10 FF_SO_RECV, FF_SO_RECVFROM, FF_SO_RECVMSG, @@ -49,7 +49,7 @@ enum FF_SOCKET_OPS { FF_SO_SENDTO, FF_SO_SENDMSG, FF_SO_WRITE, - FF_SO_WRITEV, + FF_SO_WRITEV, // 20 FF_SO_CLOSE, FF_SO_IOCTL, FF_SO_FCNTL, @@ -58,7 +58,7 @@ enum FF_SOCKET_OPS { FF_SO_EPOLL_WAIT, FF_SO_KQUEUE, FF_SO_KEVENT, - FF_SO_FORK, + FF_SO_FORK, // 29 }; enum FF_SO_CONTEXT_STATUS { From e58d683b24ea2af89efb06b7d9c8a8c1fa38c89c Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 12 Apr 2023 10:15:22 +0800 Subject: [PATCH 15/43] burst copy in ff_hook_epoll_wait. --- adapter/ff_hook_syscall.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index 73ea9c41c..ff242f5da 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -1277,15 +1277,12 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, rte_spinlock_unlock(&sc->lock); if (ret > 0) { - int i; if (unlikely(ret > maxevents)) { ERR_LOG("return events:%d, maxevents:%d, set return events to maxevents, may be some error occur\n", ret, maxevents); ret = maxevents; } - for (i = 0; i < ret; i++) { - rte_memcpy(&events[i], &sh_events[i], sizeof(struct epoll_event)); - } + rte_memcpy(events, sh_events, sizeof(struct epoll_event) * ret); } if (sh_events) { From 03b606ff9baf140745921fbbb4acfa82638b1a6e Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 12 Apr 2023 12:00:53 +0800 Subject: [PATCH 16/43] Support timeout is NULL or 0 in libffsyscall.so. And support it in main_stack.c. --- adapter/ff_adapter.h | 2 + adapter/ff_hook_syscall.c | 41 +++++++++++- adapter/ff_socket_ops.c | 82 ++++++++++++++---------- example/main_stack.c | 6 +- example/main_stack_epoll.c | 7 +- example/main_stack_epoll_thread_socket.c | 7 +- example/main_stack_thread_socket.c | 7 +- 7 files changed, 112 insertions(+), 40 deletions(-) diff --git a/adapter/ff_adapter.h b/adapter/ff_adapter.h index 39ee00cc4..a985993a5 100644 --- a/adapter/ff_adapter.h +++ b/adapter/ff_adapter.h @@ -10,6 +10,8 @@ int ff_adapter_init(); //int __attribute__((constructor)) ff_adapter_init(int argc, char * const argv[]); +void alarm_event_sem(); + /*- * Verify whether the socket is supported by fstack or not. */ diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index ff242f5da..601c7925c 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -129,6 +129,8 @@ static int nb_procs = NB_FSTACK_INSTANCE_DEFAULT; #define FF_KERNEL_MAX_FD_DEFAULT 1024 static int ff_kernel_max_fd = FF_KERNEL_MAX_FD_DEFAULT; +static int need_alarm_sem = 0; + static inline int convert_fstack_fd(int sockfd) { return sockfd + ff_kernel_max_fd; } @@ -1231,6 +1233,11 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, ACQUIRE_ZONE_LOCK(FF_SC_IDLE); sc->ops = FF_SO_EPOLL_WAIT; sc->args = args; + + if (timeout == 0) { + need_alarm_sem = 1; + } + RELEASE_ZONE_LOCK(FF_SC_REQ); errno = 0; @@ -1257,6 +1264,10 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, rte_spinlock_lock(&sc->lock); + if (timeout == 0) { + need_alarm_sem = 1; + } + /* * After sem_timedwait, and before lock sc, sc->status may be modify from FF_SC_REQ to FF_SC_RSP, * so it can't use to check. @@ -1380,6 +1391,10 @@ kevent(int kq, const struct kevent *changelist, int nchanges, args->nevents = nevents; args->timeout = NULL; + if (timeout == NULL) { + need_alarm_sem = 1; + } + rte_spinlock_lock(&sc->lock); sc->ops = FF_SO_KEVENT; @@ -1406,6 +1421,10 @@ kevent(int kq, const struct kevent *changelist, int nchanges, rte_spinlock_lock(&sc->lock); + if (timeout == NULL) { + need_alarm_sem = 0; + } + /* * After sem_timedwait, and before lock sc, sc->status may be modify from FF_SC_REQ to FF_SC_RSP, * so it can't use to check. @@ -1616,7 +1635,27 @@ ff_adapter_exit() #ifdef FF_THREAD_SOCKET pthread_key_delete(key); #else - DEBUG_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc); + ERR_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc); ff_detach_so_context(sc); #endif } + +void +alarm_event_sem() +{ +#ifndef FF_THREAD_SOCKET + DEBUG_LOG("check whether need to alarm sem sc:%p, status:%d, ops:%d\n", + sc, sc->status, sc->ops, need_alarm_sem); + rte_spinlock_lock(&sc->lock); + if (need_alarm_sem == 1) { + ERR_LOG("alarm sc:%p, status:%d, ops:%d\n", sc, sc->status, sc->ops); + sem_post(&sc->wait_sem); + need_alarm_sem = 0; + } + rte_spinlock_unlock(&sc->lock); + + DEBUG_LOG("finish alarm sem sc:%p, status:%d, ops:%d\n", + sc, sc->status, sc->ops, need_alarm_sem); +#endif +} + diff --git a/adapter/ff_socket_ops.c b/adapter/ff_socket_ops.c index c9038d08e..abd3c7038 100644 --- a/adapter/ff_socket_ops.c +++ b/adapter/ff_socket_ops.c @@ -15,6 +15,8 @@ static int ff_sys_kevent(struct ff_kevent_args *args); #define FF_MAX_BOUND_NUM 8 +/* Where to call sem_post in kevent or epoll_wait */ +static int sem_flag = 0; struct ff_bound_info { int fd; @@ -288,10 +290,25 @@ ff_sys_epoll_ctl(struct ff_epoll_ctl_args *args) static int ff_sys_epoll_wait(struct ff_epoll_wait_args *args) { + int ret; + DEBUG_LOG("to run ff_epoll_wait, epfd:%d, maxevents:%d, timeout:%d\n", args->epfd, args->maxevents, args->timeout); - return ff_epoll_wait(args->epfd, args->events, + ret = ff_epoll_wait(args->epfd, args->events, args->maxevents, args->timeout); + + /* + * If timeout is 0, and no event triggered, + * no post sem, and next loop will continue to call ff_sys_epoll_wait, + * until some event triggered + */ + if (args->timeout == 0 && ret == 0 && args->maxevents != 0) { + sem_flag = 0; + } else { + sem_flag = 1; + } + + return ret; } static int @@ -303,8 +320,27 @@ ff_sys_kqueue(struct ff_kqueue_args *args) static int ff_sys_kevent(struct ff_kevent_args *args) { - return ff_kevent(args->kq, args->changelist, args->nchanges, + int ret; + + ret = ff_kevent(args->kq, args->changelist, args->nchanges, args->eventlist, args->nevents, args->timeout); + + if (args->nchanges) { + args->nchanges = 0; + } + + /* + * If timeout is NULL, and no event triggered, + * no post sem, and next loop will continue to call ff_sys_kevent, + * until some event triggered + */ + if (args->timeout == NULL && ret == 0 && args->nevents != 0) { + sem_flag = 0; + } else { + sem_flag = 1; + } + + return ret; } static pid_t @@ -387,30 +423,6 @@ ff_so_handler(int ops, void *args) return (-1); } -static void -ff_handle_kevent(struct ff_so_context *sc) -{ - struct ff_kevent_args *ka = sc->args; - - DEBUG_LOG("ff_handle_kevent sc:%p, status:%d, ops:%d, kq:%d, nchanges:%d, nevents:%d\n", - sc, sc->status, sc->ops, ka->kq, ka->nchanges, ka->nevents); - - errno = 0; - sc->result = ff_sys_kevent(ka); - sc->error = errno; - - if (ka->nchanges) { - ka->nchanges = 0; - } - - /*if (sc->result == 0 && ka->nevents != 0) { - return; - }*/ - - sc->status = FF_SC_REP; - sem_post(&sc->wait_sem); -} - static inline void ff_handle_socket_ops(struct ff_so_context *sc) { @@ -425,19 +437,19 @@ ff_handle_socket_ops(struct ff_so_context *sc) DEBUG_LOG("ff_handle_socket_ops sc:%p, status:%d, ops:%d\n", sc, sc->status, sc->ops); - if (sc->ops == FF_SO_KEVENT) { - ff_handle_kevent(sc); - rte_spinlock_unlock(&sc->lock); - return; - } - errno = 0; sc->result = ff_so_handler(sc->ops, sc->args); sc->error = errno; - sc->status = FF_SC_REP; - if (sc->ops == FF_SO_EPOLL_WAIT) { - sem_post(&sc->wait_sem); + if (sc->ops == FF_SO_EPOLL_WAIT || sc->ops == FF_SO_KEVENT) { + if (sem_flag == 1) { + sc->status = FF_SC_REP; + sem_post(&sc->wait_sem); + } else { + // do nothing with this sc + } + } else { + sc->status = FF_SC_REP; } rte_spinlock_unlock(&sc->lock); diff --git a/example/main_stack.c b/example/main_stack.c index 56201b5f1..c5dc89065 100644 --- a/example/main_stack.c +++ b/example/main_stack.c @@ -72,6 +72,7 @@ void sig_term(int sig) { printf("we caught signal %d, to exit helloworld\n", sig); exit_flag = 1; + alarm_event_sem(); return; } @@ -79,6 +80,9 @@ void *loop(void *arg) { /* Wait for events to happen */ while (!exit_flag) { + /* + * If timeout is NULL, must call alarm_event_sem(); + */ int nevents = kevent(kq, NULL, 0, events, MAX_EVENTS, &timeout); int i; @@ -91,7 +95,7 @@ void *loop(void *arg) //usleep(100); sleep(1); } - printf("get nevents:%d\n", nevents); + //printf("get nevents:%d\n", nevents); for (i = 0; i < nevents; ++i) { struct kevent event = events[i]; diff --git a/example/main_stack_epoll.c b/example/main_stack_epoll.c index d012dedc1..654726857 100644 --- a/example/main_stack_epoll.c +++ b/example/main_stack_epoll.c @@ -61,6 +61,7 @@ void sig_term(int sig) { printf("we caught signal %d, to exit helloworld\n", sig); exit_flag = 1; + //alarm_event_sem(); return; } @@ -68,6 +69,10 @@ void *loop(void *arg) { /* Wait for events to happen */ while (!exit_flag) { + /* + * If not call alarm_event_sem, and epoll_wait timeout is 0, + * it can't exit normal, so timeout can't set to 0. + */ int nevents = epoll_wait(epfd, events, MAX_EVENTS, 100); int i; @@ -80,7 +85,7 @@ void *loop(void *arg) //usleep(100); sleep(1); } - printf("get nevents:%d\n", nevents); + //printf("get nevents:%d\n", nevents); for (i = 0; i < nevents; ++i) { /* Handle new connect */ diff --git a/example/main_stack_epoll_thread_socket.c b/example/main_stack_epoll_thread_socket.c index 4a2e0faf3..8f31136cd 100644 --- a/example/main_stack_epoll_thread_socket.c +++ b/example/main_stack_epoll_thread_socket.c @@ -57,6 +57,7 @@ void sig_term(int sig) { printf("we caught signal %d, to exit helloworld\n", sig); exit_flag = 1; + //alarm_event_sem(); return; } @@ -124,6 +125,10 @@ void *loop(void *arg) /* Wait for events to happen */ while (!exit_flag) { + /* + * If not call alarm_event_sem, and epoll_wait timeout is 0, + * it can't exit normal, so timeout can't set to 0. + */ int nevents = epoll_wait(epfd, events, MAX_EVENTS, 100); int i; @@ -136,7 +141,7 @@ void *loop(void *arg) //usleep(100); sleep(1); } - printf("thread %d, get nevents:%d\n", thread_id, nevents); + //printf("thread %d, get nevents:%d\n", thread_id, nevents); for (i = 0; i < nevents; ++i) { /* Handle new connect */ diff --git a/example/main_stack_thread_socket.c b/example/main_stack_thread_socket.c index 4d58c13ab..86afbd5fa 100644 --- a/example/main_stack_thread_socket.c +++ b/example/main_stack_thread_socket.c @@ -65,6 +65,7 @@ void sig_term(int sig) { printf("we caught signal %d, to exit helloworld\n", sig); exit_flag = 1; + //alarm_event_sem(); return; } @@ -138,6 +139,10 @@ void *loop(void *arg) /* Wait for events to happen */ while (!exit_flag) { + /* + * If not call alarm_event_sem, and epoll_wait timeout is NULL, + * it can't exit normal, so timeout can't set to NULL. + */ int nevents = kevent(kq, NULL, 0, events, MAX_EVENTS, &timeout); int i; @@ -150,7 +155,7 @@ void *loop(void *arg) //usleep(100); sleep(1); } - printf("thread %d, get nevents:%d\n", thread_id, nevents); + //printf("thread %d, get nevents:%d\n", thread_id, nevents); for (i = 0; i < nevents; ++i) { struct kevent event = events[i]; From 739ea21764cab4cfc98953aec477de9cfb24411f Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 12 Apr 2023 12:03:34 +0800 Subject: [PATCH 17/43] Add some log. --- example/main_stack_epoll.c | 1 + example/main_stack_epoll_thread_socket.c | 1 + 2 files changed, 2 insertions(+) diff --git a/example/main_stack_epoll.c b/example/main_stack_epoll.c index 654726857..e56006acb 100644 --- a/example/main_stack_epoll.c +++ b/example/main_stack_epoll.c @@ -168,6 +168,7 @@ int main(int argc, char * argv[]) } epfd = epoll_create(0); + printf("epfd:%d\n", epfd); if (epfd <= 0) { printf("ff_epoll_create failed, errno:%d, %s\n", errno, strerror(errno)); diff --git a/example/main_stack_epoll_thread_socket.c b/example/main_stack_epoll_thread_socket.c index 8f31136cd..863ce21fb 100644 --- a/example/main_stack_epoll_thread_socket.c +++ b/example/main_stack_epoll_thread_socket.c @@ -107,6 +107,7 @@ void *loop(void *arg) } epfd = epoll_create(0); + printf("thread %d, epfd:%d\n", thread_id, epfd); if (epfd <= 0) { printf("thread %d, ff_epoll_create failed, errno:%d, %s\n", thread_id, errno, strerror(errno)); From 2471d2560cca3b027a9deea9dd9670cf1bc72ff4 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 12 Apr 2023 17:48:39 +0800 Subject: [PATCH 18/43] Modify some log. --- adapter/ff_so_zone.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/adapter/ff_so_zone.c b/adapter/ff_so_zone.c index 2d7634b76..eb2f6525c 100644 --- a/adapter/ff_so_zone.c +++ b/adapter/ff_so_zone.c @@ -172,7 +172,7 @@ ff_attach_so_context(int proc_id) sc = NULL; } - DEBUG_LOG("attach sc:%p, so count:%d, free:%d, idx:%d, i:%d\n", + ERR_LOG("attach sc:%p, so count:%d, free:%d, idx:%d, i:%d\n", sc, ff_so_zone->count, ff_so_zone->free, ff_so_zone->idx, i); rte_spinlock_unlock(&ff_so_zone->lock); @@ -183,13 +183,13 @@ ff_attach_so_context(int proc_id) void ff_detach_so_context(struct ff_so_context *sc) { - DEBUG_LOG("ff_so_zone:%p, sc:%p\n", ff_so_zone, sc); + ERR_LOG("ff_so_zone:%p, sc:%p\n", ff_so_zone, sc); if (ff_so_zone == NULL || sc == NULL) { return; } - DEBUG_LOG("detach sc:%p, ops:%d, status:%d, idx:%d, inuse:%d, so free:%u, idx:%u\n", + ERR_LOG("detach sc:%p, ops:%d, status:%d, idx:%d, inuse:%d, so free:%u, idx:%u\n", sc, sc->ops, sc->status, sc->idx, sc->inuse, ff_so_zone->free, ff_so_zone->idx); rte_spinlock_lock(&ff_so_zone->lock); @@ -201,7 +201,7 @@ ff_detach_so_context(struct ff_so_context *sc) ff_so_zone->idx = sc->idx; } - DEBUG_LOG("detach sc:%p, ops:%d, status:%d, idx:%d, inuse:%d, so free:%u, idx:%u\n", + ERR_LOG("detach sc:%p, ops:%d, status:%d, idx:%d, inuse:%d, so free:%u, idx:%u\n", sc, sc->ops, sc->status, sc->idx, sc->inuse, ff_so_zone->free, ff_so_zone->idx); rte_spinlock_unlock(&ff_so_zone->lock); From f85c52e3413b14211711bbe5068ba7127baa4149 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 12 Apr 2023 17:49:16 +0800 Subject: [PATCH 19/43] Don't sleep while no event triggerd in demo. --- example/main_stack.c | 2 +- example/main_stack_epoll.c | 2 +- example/main_stack_epoll_thread_socket.c | 2 +- example/main_stack_thread_socket.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/example/main_stack.c b/example/main_stack.c index c5dc89065..a521dfa83 100644 --- a/example/main_stack.c +++ b/example/main_stack.c @@ -93,7 +93,7 @@ void *loop(void *arg) return NULL; } //usleep(100); - sleep(1); + //sleep(1); } //printf("get nevents:%d\n", nevents); diff --git a/example/main_stack_epoll.c b/example/main_stack_epoll.c index e56006acb..25f7a36de 100644 --- a/example/main_stack_epoll.c +++ b/example/main_stack_epoll.c @@ -83,7 +83,7 @@ void *loop(void *arg) break; } //usleep(100); - sleep(1); + //sleep(1); } //printf("get nevents:%d\n", nevents); diff --git a/example/main_stack_epoll_thread_socket.c b/example/main_stack_epoll_thread_socket.c index 863ce21fb..7cbe69e95 100644 --- a/example/main_stack_epoll_thread_socket.c +++ b/example/main_stack_epoll_thread_socket.c @@ -140,7 +140,7 @@ void *loop(void *arg) break; } //usleep(100); - sleep(1); + //sleep(1); } //printf("thread %d, get nevents:%d\n", thread_id, nevents); diff --git a/example/main_stack_thread_socket.c b/example/main_stack_thread_socket.c index 86afbd5fa..0bc7704e2 100644 --- a/example/main_stack_thread_socket.c +++ b/example/main_stack_thread_socket.c @@ -153,7 +153,7 @@ void *loop(void *arg) return NULL; } //usleep(100); - sleep(1); + //sleep(1); } //printf("thread %d, get nevents:%d\n", thread_id, nevents); From dfb462caa7b3aca04470d1d65a54edbae1757332 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 12 Apr 2023 18:55:27 +0800 Subject: [PATCH 20/43] Save the event num kevent or epoll_wait returned, Use for burst process event in one F-Stack loop to improve performance. The performance of single instance improve from 4500 QPS to 200, 000 QPS. --- adapter/ff_socket_ops.c | 58 ++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/adapter/ff_socket_ops.c b/adapter/ff_socket_ops.c index abd3c7038..0423b0f77 100644 --- a/adapter/ff_socket_ops.c +++ b/adapter/ff_socket_ops.c @@ -18,6 +18,14 @@ static int ff_sys_kevent(struct ff_kevent_args *args); /* Where to call sem_post in kevent or epoll_wait */ static int sem_flag = 0; +/* + * The event num kevent or epoll_wait returned. + * Use for burst process event in one F-Stack loop to improve performance. + */ +#define EVENT_LOOP_TIMES 32 +static int ff_event_loop_nb = 0; +static int ff_next_event_flag = 0; + struct ff_bound_info { int fd; struct sockaddr addr; @@ -442,6 +450,18 @@ ff_handle_socket_ops(struct ff_so_context *sc) sc->error = errno; if (sc->ops == FF_SO_EPOLL_WAIT || sc->ops == FF_SO_KEVENT) { + if (ff_event_loop_nb > 0) { + ff_next_event_flag = 1; + } else { + ff_next_event_flag = 0; + } + + if (sc->result > 0) { + ff_event_loop_nb = (sc->result * EVENT_LOOP_TIMES); + } else { + ff_event_loop_nb = 0; + } + if (sem_flag == 1) { sc->status = FF_SC_REP; sem_post(&sc->wait_sem); @@ -460,27 +480,35 @@ ff_handle_each_context() { uint16_t i, nb_handled; - rte_spinlock_lock(&ff_so_zone->lock); + while(1) { + rte_spinlock_lock(&ff_so_zone->lock); - assert(ff_so_zone->count >= ff_so_zone->free); - nb_handled = ff_so_zone->count - ff_so_zone->free; - if (nb_handled) { - for (i = 0; i < ff_so_zone->count; i++) { - struct ff_so_context *sc = &ff_so_zone->sc[i]; + assert(ff_so_zone->count >= ff_so_zone->free); + nb_handled = ff_so_zone->count - ff_so_zone->free; + if (nb_handled) { + for (i = 0; i < ff_so_zone->count; i++) { + struct ff_so_context *sc = &ff_so_zone->sc[i]; - if (sc->inuse == 0) { - continue; - } + if (sc->inuse == 0) { + continue; + } - ff_handle_socket_ops(sc); + ff_handle_socket_ops(sc); - nb_handled--; - if (!nb_handled) { - break; + nb_handled--; + if (!nb_handled) { + break; + } } } - } - rte_spinlock_unlock(&ff_so_zone->lock); + rte_spinlock_unlock(&ff_so_zone->lock); + + if (--ff_event_loop_nb <= 0 || ff_next_event_flag == 1) { + break; + } + + rte_pause(); + } } From e45925dd799e4b3d30acb69e6de88349c34d3fd0 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 12 Apr 2023 18:57:59 +0800 Subject: [PATCH 21/43] Don't free the memory malloc with rte_malloc in most commonly used socket API, to improve proformance. The performance of single instance from 200,000 QPS improve to 260,000QPS. NOTE:May cause memory leak. --- adapter/ff_hook_syscall.c | 427 +++++++++++++++++++++++++++----------- 1 file changed, 303 insertions(+), 124 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index 601c7925c..42c9ddb16 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -55,6 +55,32 @@ return ret; \ } +/* Always use __thread, but no __FF_THREAD */ +static __thread struct ff_shutdown_args *shutdown_args = NULL; +static __thread struct ff_accept_args *accept_args = NULL; +static __thread struct ff_connect_args *connect_args = NULL; +static __thread struct ff_recvfrom_args *recvfrom_args = NULL; +static __thread struct ff_read_args *read_args = NULL; +static __thread struct ff_sendto_args *sendto_args = NULL; +static __thread struct ff_write_args *write_args = NULL; +static __thread struct ff_close_args *close_args = NULL; +static __thread struct ff_epoll_ctl_args *epoll_ctl_args = NULL; +static __thread struct ff_epoll_wait_args *epoll_wait_args = NULL; +static __thread struct ff_kevent_args *kevent_args = NULL; + +#define DEFINE_REQ_ARGS_STATIC(name) \ + int ret = -1; \ + struct ff_##name##_args *args = NULL; \ + if (name##_args == NULL) { \ + size_t size = sizeof(struct ff_##name##_args); \ + name##_args = share_mem_alloc(size); \ + if (name##_args == NULL) { \ + errno = ENOMEM; \ + return ret; \ + } \ + } \ + args = name##_args; + #define ACQUIRE_ZONE_LOCK(exp) do { \ while (1) { \ rte_spinlock_lock(&sc->lock); \ @@ -85,6 +111,15 @@ RELEASE_ZONE_LOCK(FF_SC_IDLE); \ } while (0) +#define RETURN_NOFREE() do { \ + return ret; \ +} while (0) + +#define RETURN_ERROR_NOFREE(err) do { \ + errno = err; \ + return ret; \ +} while (0) + #define RETURN() do { \ share_mem_free(args); \ return ret; \ @@ -99,13 +134,7 @@ static __FF_THREAD int inited = 0; static __FF_THREAD struct ff_so_context *sc; -/* - * Use pthread_key_create/pthread_setspecific/pthread_key_delete in FF_THREAD_SOCKET mode, - * because ff_so_zone is thread level. - */ -#ifdef FF_THREAD_SOCKET static pthread_key_t key; -#endif /* process-level initialization flag */ static int proc_inited = 0; @@ -257,14 +286,14 @@ ff_hook_shutdown(int fd, int how) { CHECK_FD_OWNERSHIP(shutdown, (fd, how)); - DEFINE_REQ_ARGS(shutdown); + DEFINE_REQ_ARGS_STATIC(shutdown); args->fd = fd; args->how = how; SYSCALL(FF_SO_SHUTDOWN, args); - RETURN(); + RETURN_NOFREE(); } int @@ -458,27 +487,41 @@ ff_hook_accept(int fd, struct sockaddr *addr, socklen_t *addrlen) CHECK_FD_OWNERSHIP(accept, (fd, addr, addrlen)); - DEFINE_REQ_ARGS(accept); - struct sockaddr *sh_addr = NULL; - socklen_t *sh_addrlen = NULL; + DEFINE_REQ_ARGS_STATIC(accept); + static __thread struct sockaddr *sh_addr = NULL; + static __thread socklen_t sh_addr_len = 0; + static __thread socklen_t *sh_addrlen = NULL; if (addr != NULL) { - sh_addr = share_mem_alloc(*addrlen); - if (sh_addr == NULL) { - RETURN_ERROR(ENOMEM); + if (sh_addr == NULL || sh_addr_len < *addrlen) { + if(sh_addr) { + share_mem_free(sh_addr); + } + + sh_addr_len = *addrlen; + sh_addr = share_mem_alloc(sh_addr_len); + if (sh_addr == NULL) { + RETURN_ERROR_NOFREE(ENOMEM); + } } - sh_addrlen = share_mem_alloc(sizeof(socklen_t)); if (sh_addrlen == NULL) { - share_mem_free(sh_addr); - RETURN_ERROR(ENOMEM); + sh_addrlen = share_mem_alloc(sizeof(socklen_t)); + if (sh_addrlen == NULL) { + //share_mem_free(sh_addr); // Don't free + RETURN_ERROR_NOFREE(ENOMEM); + } } *sh_addrlen = *addrlen; + + args->addr = sh_addr; + args->addrlen = sh_addrlen; + }else { + args->addr = NULL; + args->addrlen = NULL; } args->fd = fd; - args->addr = sh_addr; - args->addrlen = sh_addrlen; SYSCALL(FF_SO_ACCEPT, args); @@ -493,11 +536,11 @@ ff_hook_accept(int fd, struct sockaddr *addr, socklen_t *addrlen) rte_memcpy(addr, sh_addr, cplen); *addrlen = *sh_addrlen; } - share_mem_free(sh_addr); - share_mem_free(sh_addrlen); + //share_mem_free(sh_addr); // Don't free + //share_mem_free(sh_addrlen); } - RETURN(); + RETURN_NOFREE(); } int @@ -521,12 +564,20 @@ ff_hook_connect(int fd, const struct sockaddr *addr, CHECK_FD_OWNERSHIP(connect, (fd, addr, addrlen)); - DEFINE_REQ_ARGS(connect); - struct sockaddr *sh_addr; + DEFINE_REQ_ARGS_STATIC(connect); + static __thread struct sockaddr *sh_addr = NULL; + static __thread socklen_t sh_addr_len = 0; - sh_addr = share_mem_alloc(addrlen); - if (sh_addr == NULL) { - RETURN_ERROR(ENOMEM); + if (sh_addr == NULL || sh_addr_len < addrlen) { + if(sh_addr) { + share_mem_free(sh_addr); + } + + sh_addr_len = addrlen; + sh_addr = share_mem_alloc(sh_addr_len); + if (sh_addr == NULL) { + RETURN_ERROR_NOFREE(ENOMEM); + } } rte_memcpy(sh_addr, addr, addrlen); @@ -536,8 +587,8 @@ ff_hook_connect(int fd, const struct sockaddr *addr, SYSCALL(FF_SO_CONNECT, args); - share_mem_free(sh_addr); - RETURN(); + //share_mem_free(sh_addr); + RETURN_NOFREE(); } ssize_t @@ -563,41 +614,57 @@ ff_hook_recvfrom(int fd, void *buf, size_t len, int flags, CHECK_FD_OWNERSHIP(recvfrom, (fd, buf, len, flags, from, fromlen)); - DEFINE_REQ_ARGS(recvfrom); - void *sh_buf; - struct sockaddr *sh_from = NULL; - socklen_t *sh_fromlen = NULL; + DEFINE_REQ_ARGS_STATIC(recvfrom); + static __thread void *sh_buf; + static __thread size_t sh_buf_len = 0; + static __thread struct sockaddr *sh_from = NULL; + static __thread socklen_t sh_from_len = 0; + static __thread socklen_t *sh_fromlen = NULL; if (from != NULL) { - sh_from = share_mem_alloc(*fromlen); - if (sh_from == NULL) { - RETURN_ERROR(ENOMEM); + if (sh_from == NULL || sh_from_len < *fromlen) { + if (sh_from) { + share_mem_free(sh_from); + } + + sh_from_len = *fromlen; + sh_from = share_mem_alloc(sh_from_len); + if (sh_from == NULL) { + RETURN_ERROR_NOFREE(ENOMEM); + } } - sh_fromlen = share_mem_alloc(sizeof(socklen_t)); if (sh_fromlen == NULL) { - share_mem_free(sh_from); - RETURN_ERROR(ENOMEM); + sh_fromlen = share_mem_alloc(sizeof(socklen_t)); + if (sh_fromlen == NULL) { + //share_mem_free(sh_from); + RETURN_ERROR_NOFREE(ENOMEM); + } } + + args->from = sh_from; + args->fromlen = sh_fromlen; + } else { + args->from = NULL; + args->fromlen = NULL; } - sh_buf = share_mem_alloc(len); - if (sh_buf == NULL) { - if (sh_from) { - share_mem_free(sh_from); + if (sh_buf == NULL || sh_buf_len < (len * 4)) { + if (sh_buf) { + share_mem_free(sh_buf); } - if (sh_fromlen) { - share_mem_free(sh_fromlen); + + sh_buf_len = len * 4; + sh_buf = share_mem_alloc(sh_buf_len); + if (sh_buf == NULL) { + RETURN_ERROR_NOFREE(ENOMEM); } - RETURN_ERROR(ENOMEM); } args->fd = fd; args->buf = sh_buf; args->len = len; args->flags = flags; - args->from = sh_from; - args->fromlen = sh_fromlen; SYSCALL(FF_SO_RECVFROM, args); @@ -611,13 +678,14 @@ ff_hook_recvfrom(int fd, void *buf, size_t len, int flags, } } - if (from) { + /*if (from) { share_mem_free(sh_from); share_mem_free(sh_fromlen); } - share_mem_free(sh_buf); - RETURN(); + share_mem_free(sh_buf);*/ + + RETURN_NOFREE(); } static void @@ -858,12 +926,22 @@ ff_hook_read(int fd, void *buf, size_t len) CHECK_FD_OWNERSHIP(read, (fd, buf, len)); - DEFINE_REQ_ARGS(read); - void *sh_buf; + DEFINE_REQ_ARGS_STATIC(read); + static __thread void *sh_buf = NULL; + static __thread size_t sh_buf_len = 0; - sh_buf = share_mem_alloc(len); - if (sh_buf == NULL) { - RETURN_ERROR(ENOMEM); + /* alloc or realloc sh_buf */ + if (sh_buf == NULL || sh_buf_len < (len * 4)) { + if (sh_buf) { + share_mem_free(sh_buf);; + } + + /* alloc 4 times buf space */ + sh_buf_len = len * 4; + sh_buf = share_mem_alloc(sh_buf_len); + if (sh_buf == NULL) { + RETURN_ERROR_NOFREE(ENOMEM); + } } args->fd = fd; @@ -876,8 +954,9 @@ ff_hook_read(int fd, void *buf, size_t len) rte_memcpy(buf, sh_buf, ret); } - share_mem_free(sh_buf); - RETURN(); + //share_mem_free(sh_buf); + + RETURN_NOFREE(); } ssize_t @@ -923,39 +1002,60 @@ ff_hook_sendto(int fd, const void *buf, size_t len, int flags, CHECK_FD_OWNERSHIP(sendto, (fd, buf, len, flags, to, tolen)); - DEFINE_REQ_ARGS(sendto); - void *sh_buf; - void *sh_to = NULL; + DEFINE_REQ_ARGS_STATIC(sendto); + static __thread void *sh_buf = NULL; + static __thread size_t sh_buf_len = 0; + static __thread void *sh_to = NULL; + static __thread socklen_t sh_to_len = 0; - sh_buf = share_mem_alloc(len); - if (sh_buf == NULL) { - RETURN_ERROR(ENOMEM); + if (sh_buf == NULL || sh_buf_len < (len * 4)) { + if (sh_buf) { + share_mem_free(sh_buf);; + } + + /* alloc 4 times buf space */ + sh_buf_len = len * 4; + sh_buf = share_mem_alloc(sh_buf_len); + if (sh_buf == NULL) { + RETURN_ERROR_NOFREE(ENOMEM); + } } rte_memcpy(sh_buf, buf, len); if (to) { - sh_to = share_mem_alloc(tolen); - if (sh_to == NULL) { - share_mem_free(sh_buf); - RETURN_ERROR(ENOMEM); + if (sh_to == NULL || sh_to_len < tolen) { + if (sh_to) { + share_mem_free(sh_to); + } + + sh_to_len = tolen; + sh_to = share_mem_alloc(sh_to_len); + if (sh_to == NULL) { + //share_mem_free(sh_buf); + RETURN_ERROR_NOFREE(ENOMEM); + } } rte_memcpy(sh_to, to, tolen); + args->to = sh_to; + args->tolen = tolen; + } else { + args->to = NULL; + args->tolen = 0; } args->fd = fd; args->buf = sh_buf; args->len = len; args->flags = flags; - args->to = sh_to; - args->tolen = tolen; SYSCALL(FF_SO_SENDTO, args); - share_mem_free(sh_buf); + /*share_mem_free(sh_buf); if (sh_to) { share_mem_free(sh_to); - } - RETURN(); + }*/ + + RETURN_NOFREE(); } ssize_t @@ -1014,12 +1114,20 @@ ff_hook_write(int fd, const void *buf, size_t len) CHECK_FD_OWNERSHIP(write, (fd, buf, len)); - DEFINE_REQ_ARGS(write); - void *sh_buf; + DEFINE_REQ_ARGS_STATIC(write); + static __thread void *sh_buf = NULL; + static __thread size_t sh_buf_len = 0; - sh_buf = share_mem_alloc(len); - if (sh_buf == NULL) { - RETURN_ERROR(ENOMEM); + if (sh_buf == NULL || sh_buf_len < (len * 4)) { + if (sh_buf) { + share_mem_free(sh_buf); + } + + sh_buf_len = len * 4; + sh_buf = share_mem_alloc(sh_buf_len); + if (sh_buf == NULL) { + RETURN_ERROR_NOFREE(ENOMEM); + } } rte_memcpy(sh_buf, buf, len); @@ -1029,8 +1137,9 @@ ff_hook_write(int fd, const void *buf, size_t len) SYSCALL(FF_SO_WRITE, args); - share_mem_free(sh_buf); - RETURN(); + //share_mem_free(sh_buf); + + RETURN_NOFREE(); } ssize_t @@ -1073,13 +1182,13 @@ ff_hook_close(int fd) CHECK_FD_OWNERSHIP(close, (fd)); - DEFINE_REQ_ARGS(close); + DEFINE_REQ_ARGS_STATIC(close); args->fd = fd; SYSCALL(FF_SO_CLOSE, args); - RETURN(); + RETURN_NOFREE(); } int @@ -1174,8 +1283,8 @@ ff_hook_epoll_ctl(int epfd, int op, int fd, DEBUG_LOG("ff_hook_epoll_ctl, epfd:%d, op:%d, fd:%d\n", epfd, op, fd); CHECK_FD_OWNERSHIP(epoll_ctl, (epfd, op, fd, event)); - DEFINE_REQ_ARGS(epoll_ctl); - struct epoll_event *sh_event = NULL; + DEFINE_REQ_ARGS_STATIC(epoll_ctl); + static __thread struct epoll_event *sh_event = NULL; if ((!event && op != EPOLL_CTL_DEL) || (op != EPOLL_CTL_ADD && @@ -1186,25 +1295,29 @@ ff_hook_epoll_ctl(int epfd, int op, int fd, } if (event) { - sh_event = share_mem_alloc(sizeof(struct epoll_event)); if (sh_event == NULL) { - RETURN_ERROR(ENOMEM); + sh_event = share_mem_alloc(sizeof(struct epoll_event)); + if (sh_event == NULL) { + RETURN_ERROR_NOFREE(ENOMEM); + } } rte_memcpy(sh_event, event, sizeof(struct epoll_event)); + args->event = sh_event; + } else { + args->event = NULL; } args->epfd = restore_fstack_fd(epfd); args->op = op; args->fd = fd; - args->event = sh_event; SYSCALL(FF_SO_EPOLL_CTL, args); - if (sh_event) { + /*if (sh_event) { share_mem_free(sh_event); - } + }*/ - RETURN(); + RETURN_NOFREE(); } int @@ -1215,12 +1328,14 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, int fd = epfd; CHECK_FD_OWNERSHIP(epoll_wait, (epfd, events, maxevents, timeout)); - DEFINE_REQ_ARGS(epoll_wait); - struct epoll_event *sh_events; + DEFINE_REQ_ARGS_STATIC(epoll_wait); + static __thread struct epoll_event *sh_events = NULL; - sh_events = share_mem_alloc(sizeof(struct epoll_event) * maxevents); if (sh_events == NULL) { - RETURN_ERROR(ENOMEM); + sh_events = share_mem_alloc(sizeof(struct epoll_event) * maxevents); + if (sh_events == NULL) { + RETURN_ERROR_NOFREE(ENOMEM); + } } args->epfd = fd; @@ -1296,11 +1411,17 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, rte_memcpy(events, sh_events, sizeof(struct epoll_event) * ret); } - if (sh_events) { + /* + * Don't free, to improve proformance. + * Will cause memory leak while APP exit , but fstack adapter not exit. + * May set them as gloabl variable and free in thread_destructor. + */ + /*if (sh_events) { share_mem_free(sh_events); - } + sh_events = NULL; + }*/ - RETURN(); + RETURN_NOFREE(); } pid_t @@ -1350,16 +1471,24 @@ kevent(int kq, const struct kevent *changelist, int nchanges, kq = restore_fstack_fd(kq); - DEFINE_REQ_ARGS(kevent); - struct kevent *sh_changelist = NULL; - struct kevent *sh_eventlist = NULL; + DEFINE_REQ_ARGS_STATIC(kevent); + static __thread struct kevent *sh_changelist = NULL; + static __thread int sh_changelist_len = 0; + static __thread struct kevent *sh_eventlist = NULL; + static __thread int sh_eventlist_len = 0; if (changelist != NULL && nchanges > 0) { - sh_changelist = share_mem_alloc(sizeof(struct kevent) * nchanges); - if (sh_changelist == NULL) { - RETURN_ERROR(ENOMEM); - } + if (sh_changelist == NULL || sh_changelist_len < nchanges) { + if (sh_changelist) { + share_mem_free(sh_changelist); + } + sh_changelist_len = nchanges; + sh_changelist = share_mem_alloc(sizeof(struct kevent) * sh_changelist_len); + if (sh_changelist == NULL) { + RETURN_ERROR_NOFREE(ENOMEM); + } + } rte_memcpy(sh_changelist, changelist, sizeof(struct kevent) * nchanges); for(i = 0; i < nchanges; i++) { @@ -1374,22 +1503,35 @@ kevent(int kq, const struct kevent *changelist, int nchanges, break; } } + args->changelist = sh_changelist; + args->nchanges = nchanges; + } else { + args->changelist = NULL; + args->nchanges = 0; } if (eventlist != NULL && nevents > 0) { - sh_eventlist = share_mem_alloc(sizeof(struct kevent) * nevents); - if (sh_eventlist == NULL) { - share_mem_free(sh_changelist); - RETURN_ERROR(ENOMEM); + if (sh_eventlist == NULL || sh_eventlist_len < nevents) { + if (sh_eventlist) { + share_mem_free(sh_eventlist); + } + + sh_eventlist_len = nevents; + sh_eventlist = share_mem_alloc(sizeof(struct kevent) * sh_eventlist_len); + if (sh_eventlist == NULL) { + //share_mem_free(sh_changelist); // don't free + RETURN_ERROR_NOFREE(ENOMEM); + } } + args->eventlist = sh_eventlist; + args->nevents = nevents; + } else { + args->eventlist = NULL; + args->nevents = 0; } args->kq = kq; - args->changelist = sh_changelist; - args->nchanges = nchanges; - args->eventlist = sh_eventlist; - args->nevents = nevents; - args->timeout = NULL; + args->timeout = (struct timespec *)timeout; if (timeout == NULL) { need_alarm_sem = 1; @@ -1462,26 +1604,67 @@ kevent(int kq, const struct kevent *changelist, int nchanges, } } - if (sh_changelist) { + /* + * Don't free, to improve performance. + * Will cause memory leak while APP exit , but fstack adapter not exit. + * May set them as gloabl variable and free in thread_destructor. + */ + /*if (sh_changelist) { share_mem_free(sh_changelist); + sh_changelist = NULL; } if (sh_eventlist) { share_mem_free(sh_eventlist); - } + sh_eventlist = NULL; + }*/ - RETURN(); + RETURN_NOFREE(); } -#ifdef FF_THREAD_SOCKET static void thread_destructor(void *sc) { +#ifdef FF_THREAD_SOCKET DEBUG_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc); ff_detach_so_context(sc); -} #endif + if (shutdown_args) { + share_mem_free(shutdown_args); + } + if (accept_args) { + share_mem_free(accept_args); + } + if (connect_args) { + share_mem_free(connect_args); + } + if (recvfrom_args) { + share_mem_free(recvfrom_args); + } + if (read_args) { + share_mem_free(read_args); + } + if (sendto_args) { + share_mem_free(sendto_args); + } + if (write_args) { + share_mem_free(write_args); + } + if (close_args) { + share_mem_free(close_args); + } + if (epoll_ctl_args) { + share_mem_free(epoll_ctl_args); + } + if (epoll_wait_args) { + share_mem_free(epoll_wait_args); + } + if (kevent_args) { + share_mem_free(kevent_args); + } +} + int ff_adapter_init() //int __attribute__((constructor)) @@ -1498,10 +1681,8 @@ ff_adapter_init() rte_spinlock_init(&worker_id_lock); rte_spinlock_lock(&worker_id_lock); -#ifdef FF_THREAD_SOCKET pthread_key_create(&key, thread_destructor); DEBUG_LOG("pthread key:%d\n", key); -#endif /* * get ulimit -n to distinguish fd between kernel and F-Stack @@ -1615,9 +1796,7 @@ ff_adapter_init() return -1; } -#ifdef FF_THREAD_SOCKET pthread_setspecific(key, sc); -#endif worker_id++; inited = 1; @@ -1632,9 +1811,9 @@ ff_adapter_init() void __attribute__((destructor)) ff_adapter_exit() { -#ifdef FF_THREAD_SOCKET pthread_key_delete(key); -#else + +#ifndef FF_THREAD_SOCKET ERR_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc); ff_detach_so_context(sc); #endif From d3fabc9cefa9fd082bb9d782ee99d9af788bbd24 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 13 Apr 2023 19:05:46 +0800 Subject: [PATCH 22/43] some opt. --- adapter/ff_hook_syscall.c | 3 +++ example/main_stack.c | 2 +- example/main_stack_epoll.c | 2 +- example/main_stack_epoll_thread_socket.c | 17 ++++++++++++++++- example/main_stack_thread_socket.c | 2 +- 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index 42c9ddb16..3a4a5c40f 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -148,6 +148,7 @@ static uint64_t initial_lcore_id = INITIAL_LCORE_ID_DEFAULT; #define WORKER_ID_DEFAULT 0 #define FF_PROC_ID_STR "FF_PROC_ID" static int worker_id = WORKER_ID_DEFAULT; +static __thread int thread_id = 0; rte_spinlock_t worker_id_lock; /* The num of F-Stack process instance, default 1 */ @@ -158,6 +159,7 @@ static int nb_procs = NB_FSTACK_INSTANCE_DEFAULT; #define FF_KERNEL_MAX_FD_DEFAULT 1024 static int ff_kernel_max_fd = FF_KERNEL_MAX_FD_DEFAULT; +/* not support thread socket now */ static int need_alarm_sem = 0; static inline int convert_fstack_fd(int sockfd) { @@ -1798,6 +1800,7 @@ ff_adapter_init() pthread_setspecific(key, sc); + thread_id = worker_id; worker_id++; inited = 1; diff --git a/example/main_stack.c b/example/main_stack.c index a521dfa83..46439f5a5 100644 --- a/example/main_stack.c +++ b/example/main_stack.c @@ -137,7 +137,7 @@ void *loop(void *arg) close(clientfd); } } else { - printf("unknown event: %8.8X\n", event.flags); + printf("unknown event: %d:%8.8X\n", i, event.flags); } } } diff --git a/example/main_stack_epoll.c b/example/main_stack_epoll.c index 25f7a36de..a51bbeac6 100644 --- a/example/main_stack_epoll.c +++ b/example/main_stack_epoll.c @@ -121,7 +121,7 @@ void *loop(void *arg) close(events[i].data.fd); } } else { - printf("unknown event: %8.8X\n", events[i].events); + printf("unknown event: %d:%8.8X\n", i, events[i].events); } } } diff --git a/example/main_stack_epoll_thread_socket.c b/example/main_stack_epoll_thread_socket.c index 7cbe69e95..e2f9bf030 100644 --- a/example/main_stack_epoll_thread_socket.c +++ b/example/main_stack_epoll_thread_socket.c @@ -1,3 +1,5 @@ +#define _GNU_SOURCE +#include #include #include #include @@ -178,7 +180,7 @@ void *loop(void *arg) close(events[i].data.fd); } } else { - printf("thread %d, unknown event: %8.8X\n", thread_id, events[i].events); + printf("thread %d, unknown event: %d:%8.8X\n", thread_id, i, events[i].events); } } } @@ -215,6 +217,19 @@ int main(int argc, char * argv[]) pthread_spin_destroy(&worker_lock); return -1; } + if (i > 0) { + cpu_set_t cpuinfo; + int lcore_id = 2 + i; + + CPU_ZERO(&cpuinfo); + CPU_SET_S(lcore_id, sizeof(cpuinfo), &cpuinfo); + if(0 != pthread_setaffinity_np(hworker[i], sizeof(cpu_set_t), &cpuinfo)) + { + printf("set affinity recver faild\n"); + exit(0); + } + printf("set affinity recver sucssed, thread:%d, lcore_id:%d\n", i, lcore_id); + } pthread_spin_lock(&worker_lock); //sleep(1); } diff --git a/example/main_stack_thread_socket.c b/example/main_stack_thread_socket.c index 0bc7704e2..fa55c74b3 100644 --- a/example/main_stack_thread_socket.c +++ b/example/main_stack_thread_socket.c @@ -197,7 +197,7 @@ void *loop(void *arg) close(clientfd); } } else { - printf("thread %d, unknown event: %8.8X\n", thread_id, event.flags); + printf("thread %d, unknown event: %d:%8.8X\n", thread_id, i, event.flags); } } } From 4af5a643fb371e1248abd6b7f9e0aeaa7981a6d9 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 13 Apr 2023 19:27:57 +0800 Subject: [PATCH 23/43] 1. Fixed some issue that sem_timewait and sem_post. 2. Modify timeout of epoll_wait/kevent from 100us to 100ms by default. 3. set cpu affinity in main_stack_epoll_thread socket. --- adapter/ff_hook_syscall.c | 33 ++++++++++++++++++++++++------ example/main_stack.c | 3 ++- example/main_stack_thread_socket.c | 3 ++- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index 3a4a5c40f..86f7d05b4 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -1351,20 +1351,30 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, sc->ops = FF_SO_EPOLL_WAIT; sc->args = args; + /* + * sc->result, sc->error must reset in epoll_wait and kevent. + * Otherwise can access last sc call's result. + * + * Because if sem_timedwait timeouted, but fstack instance still + * call sem_post later, and next or next's next sem_timedwait will + * return 0 directly, then get invalid result and error. + */ + sc->result = 0; + sc->error = 0; + errno = 0; if (timeout == 0) { need_alarm_sem = 1; } RELEASE_ZONE_LOCK(FF_SC_REQ); - errno = 0; if (timeout > 0) { struct timespec abs_timeout; clock_gettime(CLOCK_REALTIME, &abs_timeout); DEBUG_LOG("before wait, sec:%ld, nsec:%ld\n", abs_timeout.tv_sec, abs_timeout.tv_nsec); abs_timeout.tv_sec += timeout / 1000; - abs_timeout.tv_nsec += timeout * 1000; + abs_timeout.tv_nsec += timeout * 1000 * 1000; if (abs_timeout.tv_nsec > NS_PER_SECOND) { abs_timeout.tv_nsec -= NS_PER_SECOND; abs_timeout.tv_sec += 1; @@ -1535,16 +1545,27 @@ kevent(int kq, const struct kevent *changelist, int nchanges, args->kq = kq; args->timeout = (struct timespec *)timeout; - if (timeout == NULL) { - need_alarm_sem = 1; - } - rte_spinlock_lock(&sc->lock); sc->ops = FF_SO_KEVENT; sc->args = args; sc->status = FF_SC_REQ; + /* + * sc->result, sc->error must reset in epoll_wait and kevent. + * Otherwise can access last sc call's result. + * + * Because if sem_timedwait timeouted, but fstack instance still + * call sem_post later, and next or next's next sem_timedwait will + * return 0 directly, then get invalid result and error. + */ + sc->result = 0; + sc->error = 0; + errno = 0; + if (timeout == NULL) { + need_alarm_sem = 1; + } + rte_spinlock_unlock(&sc->lock); if (timeout != NULL) { diff --git a/example/main_stack.c b/example/main_stack.c index 46439f5a5..936f488ae 100644 --- a/example/main_stack.c +++ b/example/main_stack.c @@ -32,7 +32,8 @@ struct kevent events[MAX_EVENTS]; int kq; int sockfd; -struct timespec timeout = {0, 100000}; +/* 100 ms */ +struct timespec timeout = {0, 100000000}; static int exit_flag = 0; diff --git a/example/main_stack_thread_socket.c b/example/main_stack_thread_socket.c index fa55c74b3..4f7155163 100644 --- a/example/main_stack_thread_socket.c +++ b/example/main_stack_thread_socket.c @@ -25,7 +25,8 @@ pthread_t hworker[MAX_WORKERS]; pthread_spinlock_t worker_lock; #define MAX_EVENTS 512 -struct timespec timeout = {0, 100000}; +/* 100 ms */ +struct timespec timeout = {0, 100000000}; static int exit_flag = 0; From 3cd0ab86b6204dd1473fb5f3eea362934014747f Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Fri, 14 Apr 2023 14:37:15 +0800 Subject: [PATCH 24/43] 1. Reduce the number of locks with dirty read first, and then try to lock sc and real read. QPS:360k -> 380k. 2. memory alignment. QPS:380k -> 400k. --- adapter/ff_hook_syscall.c | 8 ++++++-- adapter/ff_so_zone.c | 22 ++++++++++++---------- adapter/ff_socket_ops.c | 31 ++++++++++++++++++++++--------- adapter/ff_socket_ops.h | 35 ++++++++++++++++++++--------------- 4 files changed, 60 insertions(+), 36 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index 86f7d05b4..1d012d64e 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -81,14 +81,17 @@ static __thread struct ff_kevent_args *kevent_args = NULL; } \ args = name##_args; +/* Dirty read first, and then try to lock sc and real read. */ #define ACQUIRE_ZONE_LOCK(exp) do { \ while (1) { \ + while (sc->status != exp) { \ + rte_pause(); \ + } \ rte_spinlock_lock(&sc->lock); \ if (sc->status == exp) { \ break; \ } \ rte_spinlock_unlock(&sc->lock); \ - rte_pause(); \ } \ } while (0) @@ -1545,7 +1548,8 @@ kevent(int kq, const struct kevent *changelist, int nchanges, args->kq = kq; args->timeout = (struct timespec *)timeout; - rte_spinlock_lock(&sc->lock); + ACQUIRE_ZONE_LOCK(FF_SC_IDLE); + //rte_spinlock_lock(&sc->lock); sc->ops = FF_SO_KEVENT; sc->args = args; diff --git a/adapter/ff_so_zone.c b/adapter/ff_so_zone.c index eb2f6525c..9f5555950 100644 --- a/adapter/ff_so_zone.c +++ b/adapter/ff_so_zone.c @@ -9,9 +9,6 @@ #define SOCKET_OPS_ZONE_NAME "ff_socket_ops_zone_%d" -/* Must be power of 2 */ -#define SOCKET_OPS_CONTEXT_MAX_NUM (2 << 5) - #define SOCKET_OPS_CONTEXT_NAME_SIZE 32 #define SOCKET_OPS_CONTEXT_NAME "ff_so_context_" @@ -43,6 +40,10 @@ ff_set_max_so_context(uint16_t count) return -1; } + if (count > SOCKET_OPS_CONTEXT_MAX_NUM) { + count = SOCKET_OPS_CONTEXT_MAX_NUM; + } + ff_max_so_context = count; return 0; @@ -82,6 +83,7 @@ ff_create_so_memzone() so_zone_tmp->mask = so_zone_tmp->count - 1; so_zone_tmp->free = so_zone_tmp->count; so_zone_tmp->idx = 0; + memset(so_zone_tmp->inuse, 0, SOCKET_OPS_CONTEXT_MAX_NUM); so_zone_tmp->sc = (struct ff_so_context *)(so_zone_tmp + 1); for (i = 0; i < ff_max_so_context; i++) { @@ -89,7 +91,7 @@ ff_create_so_memzone() rte_spinlock_init(&sc->lock); sc->status = FF_SC_IDLE; sc->idx = i; - sc->inuse = 0; + //so_zone_tmp->inuse[i] = 0; if (sem_init(&sc->wait_sem, 1, 0) == -1) { ERR_LOG("Initialize semaphore failed:%d\n", errno); @@ -155,8 +157,8 @@ ff_attach_so_context(int proc_id) for (i = 0; i < ff_so_zone->count; i++) { uint16_t idx = (ff_so_zone->idx + i) & ff_so_zone->mask; sc = &ff_so_zone->sc[idx]; - if (sc->inuse == 0) { - sc->inuse = 1; + if (ff_so_zone->inuse[idx] == 0) { + ff_so_zone->inuse[idx] = 1; rte_spinlock_init(&sc->lock); sc->status = FF_SC_IDLE; ff_so_zone->free--; @@ -190,19 +192,19 @@ ff_detach_so_context(struct ff_so_context *sc) } ERR_LOG("detach sc:%p, ops:%d, status:%d, idx:%d, inuse:%d, so free:%u, idx:%u\n", - sc, sc->ops, sc->status, sc->idx, sc->inuse, ff_so_zone->free, ff_so_zone->idx); + sc, sc->ops, sc->status, sc->idx, ff_so_zone->inuse[sc->idx], ff_so_zone->free, ff_so_zone->idx); rte_spinlock_lock(&ff_so_zone->lock); - if (sc->inuse == 1) { - sc->inuse = 0; + if (ff_so_zone->inuse[sc->idx] == 1) { + ff_so_zone->inuse[sc->idx] = 0; ff_so_zone->free++; ff_so_zone->idx = sc->idx; } ERR_LOG("detach sc:%p, ops:%d, status:%d, idx:%d, inuse:%d, so free:%u, idx:%u\n", - sc, sc->ops, sc->status, sc->idx, sc->inuse, ff_so_zone->free, ff_so_zone->idx); + sc, sc->ops, sc->status, sc->idx, ff_so_zone->inuse[sc->idx], ff_so_zone->free, ff_so_zone->idx); rte_spinlock_unlock(&ff_so_zone->lock); } diff --git a/adapter/ff_socket_ops.c b/adapter/ff_socket_ops.c index 0423b0f77..0b31e79be 100644 --- a/adapter/ff_socket_ops.c +++ b/adapter/ff_socket_ops.c @@ -450,6 +450,8 @@ ff_handle_socket_ops(struct ff_so_context *sc) sc->error = errno; if (sc->ops == FF_SO_EPOLL_WAIT || sc->ops == FF_SO_KEVENT) { + DEBUG_LOG("ff_event_loop_nb:%d/%d, ff_next_event_flag:%d\n", + ff_event_loop_nb, ff_next_event_flag); if (ff_event_loop_nb > 0) { ff_next_event_flag = 1; } else { @@ -478,22 +480,33 @@ ff_handle_socket_ops(struct ff_so_context *sc) void ff_handle_each_context() { - uint16_t i, nb_handled; + uint16_t i, nb_handled, tmp; + + ff_event_loop_nb = 0; + + rte_spinlock_lock(&ff_so_zone->lock); + + assert(ff_so_zone->count >= ff_so_zone->free); + tmp = nb_handled = ff_so_zone->count - ff_so_zone->free; while(1) { - rte_spinlock_lock(&ff_so_zone->lock); - - assert(ff_so_zone->count >= ff_so_zone->free); - nb_handled = ff_so_zone->count - ff_so_zone->free; + nb_handled = tmp; if (nb_handled) { for (i = 0; i < ff_so_zone->count; i++) { struct ff_so_context *sc = &ff_so_zone->sc[i]; - if (sc->inuse == 0) { + DEBUG_LOG("so:%p, so->count:%d,%p, sc:%p, sc->inuse:%d,%p, i:%d, nb:%d, all_nb:%d\n", + ff_so_zone, ff_so_zone->count, &ff_so_zone->count, + sc, ff_so_zone->inuse[i], &ff_so_zone->inuse[i], i, nb_handled, tmp); + + if (ff_so_zone->inuse[i] == 0) { continue; } - ff_handle_socket_ops(sc); + /* Dirty read first, and then try to lock sc and real read. */ + if (sc->status == FF_SC_REQ) { + ff_handle_socket_ops(sc); + } nb_handled--; if (!nb_handled) { @@ -502,13 +515,13 @@ ff_handle_each_context() } } - rte_spinlock_unlock(&ff_so_zone->lock); - if (--ff_event_loop_nb <= 0 || ff_next_event_flag == 1) { break; } rte_pause(); } + + rte_spinlock_unlock(&ff_so_zone->lock); } diff --git a/adapter/ff_socket_ops.h b/adapter/ff_socket_ops.h index 7378b8415..d6e57490b 100644 --- a/adapter/ff_socket_ops.h +++ b/adapter/ff_socket_ops.h @@ -28,6 +28,9 @@ #define DEBUG_LOG ERR_LOG #endif +/* Must be power of 2 */ +#define SOCKET_OPS_CONTEXT_MAX_NUM (1 << 5) + enum FF_SOCKET_OPS { FF_SO_SOCKET, FF_SO_LISTEN, @@ -71,36 +74,38 @@ struct ff_socket_ops_zone { rte_spinlock_t lock; /* total number of so_contex, must be power of 2 */ - uint16_t count; - uint16_t mask; + uint8_t count; + uint8_t mask; /* free number of so_context */ - uint16_t free; + uint8_t free; - uint16_t idx; + uint8_t idx; + /* 1 if used, else 0, most access */ + uint8_t inuse[SOCKET_OPS_CONTEXT_MAX_NUM]; struct ff_so_context *sc; -} __attribute__((packed)); + + uint8_t padding[16]; +} __attribute__((aligned(RTE_CACHE_LINE_SIZE))); struct ff_so_context { - rte_spinlock_t lock; - sem_t wait_sem; + enum FF_SOCKET_OPS ops; + enum FF_SO_CONTEXT_STATUS status; void *args; - enum FF_SOCKET_OPS ops; - int status; - int idx; + rte_spinlock_t lock; - /* result of ops processing */ - ssize_t result; /* errno if failed */ int error; + /* result of ops processing */ + int result; + int idx; - /* 1 if used, else 0 */ - int inuse; + sem_t wait_sem; /* 32 bytes */ // listen fd, refcount.. -} __attribute__((packed)); +} __attribute__((aligned(RTE_CACHE_LINE_SIZE))); extern __FF_THREAD struct ff_socket_ops_zone *ff_so_zone; From cf587718c2fbdd2fd7109fb5634ef953324e0e42 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 19 Apr 2023 13:54:10 +0800 Subject: [PATCH 25/43] set cpu affinity in main_stack_thread_socket.c. --- example/main_stack_thread_socket.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/example/main_stack_thread_socket.c b/example/main_stack_thread_socket.c index 4f7155163..4966e2cdf 100644 --- a/example/main_stack_thread_socket.c +++ b/example/main_stack_thread_socket.c @@ -1,3 +1,5 @@ +#define _GNU_SOURCE +#include #include #include #include @@ -234,6 +236,19 @@ int main(int argc, char * argv[]) pthread_spin_destroy(&worker_lock); return -1; } + if (i > 0) { + cpu_set_t cpuinfo; + int lcore_id = 2 + i; + + CPU_ZERO(&cpuinfo); + CPU_SET_S(lcore_id, sizeof(cpuinfo), &cpuinfo); + if(0 != pthread_setaffinity_np(hworker[i], sizeof(cpu_set_t), &cpuinfo)) + { + printf("set affinity recver faild\n"); + exit(0); + } + printf("set affinity recver sucssed, thread:%d, lcore_id:%d\n", i, lcore_id); + } pthread_spin_lock(&worker_lock); } From dd7dc378efd44c314bbffcbf06e9f4ebb8024de4 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Sun, 23 Apr 2023 10:51:56 +0800 Subject: [PATCH 26/43] opt code. --- adapter/Makefile | 2 +- adapter/ff_hook_syscall.c | 44 ++++++++++++++++++++++++++------------ adapter/ff_hook_syscall.h | 2 +- adapter/ff_linux_syscall.c | 5 +++++ adapter/ff_socket_ops.c | 13 +++++++---- adapter/ff_socket_ops.h | 3 ++- 6 files changed, 48 insertions(+), 21 deletions(-) diff --git a/adapter/Makefile b/adapter/Makefile index e6b8e6d4d..e02451764 100644 --- a/adapter/Makefile +++ b/adapter/Makefile @@ -74,7 +74,7 @@ fstack: ${FSTACK_OBJS} #libff_syscall.a: ${FF_SYSCALL_OBJS} libff_syscall.so: ${FF_SYSCALL_OBJS} #ar -rcs $@ $^ ${FF_SYSCALL_LIBS} - ${CC} -fPIC -shared -o $@ $^ ${FF_SYSCALL_LIBS} + ${CC} -fPIC -shared -o $@ $^ ${FF_SYSCALL_LIBS} ${DPDK_LIBS} ${FSTACK_OBJS}: %.o: %.c ${CC} -c $(CFLAGS) ${PROF} $< diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index 1d012d64e..110b23cba 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -17,6 +18,9 @@ #include "ff_linux_syscall.h" #include "ff_adapter.h" +/* Just for so, no used */ +struct ff_config ff_global_cfg; + #define NS_PER_SECOND 1000000000 #ifndef likely @@ -238,6 +242,8 @@ ff_hook_socket(int domain, int type, int protocol) ret = convert_fstack_fd(ret); } + DEBUG_LOG("ff_hook_socket return fd:%d\n", ret); + RETURN(); } @@ -484,6 +490,8 @@ ff_hook_setsockopt(int fd, int level, int optname, int ff_hook_accept(int fd, struct sockaddr *addr, socklen_t *addrlen) { + DEBUG_LOG("ff_hook_accept, fd:%d, addr:%p, len:%p\n", fd, addr, addrlen); + if ((addr == NULL && addrlen != NULL) || (addr != NULL && addrlen == NULL)) { errno = EINVAL; @@ -1263,7 +1271,7 @@ int ff_hook_epoll_create(int fdsize) { DEBUG_LOG("ff_hook_epoll_create, fdsize:%d\n", fdsize); - if (inited == 0 || ((fdsize & SOCK_KERNEL) && !(fdsize & SOCK_FSTACK)) || (fdsize >= 1 && fdsize <= 16)) { + if (inited == 0 || ((fdsize & SOCK_KERNEL) && !(fdsize & SOCK_FSTACK))/* || (fdsize >= 1 && fdsize <= 16)*/) { fdsize &= ~SOCK_KERNEL; return ff_linux_epoll_create(fdsize); } @@ -1278,6 +1286,8 @@ ff_hook_epoll_create(int fdsize) ret = convert_fstack_fd(ret); } + DEBUG_LOG("ff_hook_epoll_create return fd:%d\n", ret); + RETURN(); } @@ -1442,6 +1452,7 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, pid_t ff_hook_fork(void) { + DEBUG_LOG("ff_hook_fork\n"); return ff_linux_fork(); } @@ -1655,6 +1666,7 @@ thread_destructor(void *sc) #ifdef FF_THREAD_SOCKET DEBUG_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc); ff_detach_so_context(sc); + sc = NULL; #endif if (shutdown_args) { @@ -1692,6 +1704,18 @@ thread_destructor(void *sc) } } +void __attribute__((destructor)) +ff_adapter_exit() +{ + pthread_key_delete(key); + +#ifndef FF_THREAD_SOCKET + ERR_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc); + ff_detach_so_context(sc); + sc = NULL; +#endif +} + int ff_adapter_init() //int __attribute__((constructor)) @@ -1711,6 +1735,9 @@ ff_adapter_init() pthread_key_create(&key, thread_destructor); DEBUG_LOG("pthread key:%d\n", key); + //atexit(ff_adapter_exit); + //on_exit(ff_adapter_exit, NULL); + /* * get ulimit -n to distinguish fd between kernel and F-Stack */ @@ -1836,22 +1863,11 @@ ff_adapter_init() return 0; } -void __attribute__((destructor)) -ff_adapter_exit() -{ - pthread_key_delete(key); - -#ifndef FF_THREAD_SOCKET - ERR_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc); - ff_detach_so_context(sc); -#endif -} - void alarm_event_sem() { #ifndef FF_THREAD_SOCKET - DEBUG_LOG("check whether need to alarm sem sc:%p, status:%d, ops:%d\n", + DEBUG_LOG("check whether need to alarm sem sc:%p, status:%d, ops:%d, need_alarm_sem:%d\n", sc, sc->status, sc->ops, need_alarm_sem); rte_spinlock_lock(&sc->lock); if (need_alarm_sem == 1) { @@ -1861,7 +1877,7 @@ alarm_event_sem() } rte_spinlock_unlock(&sc->lock); - DEBUG_LOG("finish alarm sem sc:%p, status:%d, ops:%d\n", + DEBUG_LOG("finish alarm sem sc:%p, status:%d, ops:%d, need_alarm_sem:%d\n", sc, sc->status, sc->ops, need_alarm_sem); #endif } diff --git a/adapter/ff_hook_syscall.h b/adapter/ff_hook_syscall.h index 66ad9828e..2f943b40a 100644 --- a/adapter/ff_hook_syscall.h +++ b/adapter/ff_hook_syscall.h @@ -7,6 +7,6 @@ extern int kqueue(void); extern int kevent(int kq, const struct kevent *changelist, int nchanges, - struct kevent *eventlist, int nevents, const struct timespec *timeout); + struct kevent *eventlist, int nevents, const struct timespec *timeout); #endif diff --git a/adapter/ff_linux_syscall.c b/adapter/ff_linux_syscall.c index 88614abd4..7c950e23a 100644 --- a/adapter/ff_linux_syscall.c +++ b/adapter/ff_linux_syscall.c @@ -124,6 +124,7 @@ int ff_linux_setsockopt(int s, int level, int optname, int ff_linux_accept(int s, struct sockaddr *addr, socklen_t *addrlen) { + DEBUG_LOG("ff_linux_accept, fd:%d, addr:%p, len:%p\n", s, addr, addrlen); SYSCALL(accept, (s, addr, addrlen)); } @@ -151,11 +152,13 @@ ssize_t ff_linux_send(int s, const void *buf, size_t len, int flags) ssize_t ff_linux_read(int s, void *buf, size_t len) { + DEBUG_LOG("ff_linux_read, fd:%d, buf:%p, len:%lu\n", s, buf, len); SYSCALL(read, (s, buf, len)); } ssize_t ff_linux_write(int s, const void *buf, size_t len) { + DEBUG_LOG("ff_linux_write, fd:%d, buf:%p, len:%lu\n", s, buf, len); SYSCALL(write, (s, buf, len)); } @@ -194,6 +197,7 @@ ssize_t ff_linux_recvmsg(int s, struct msghdr *msg, int flags) int ff_linux_close(int s) { + DEBUG_LOG("ff_linux_close, fd:%d\n", s); SYSCALL(close, (s)); } @@ -231,6 +235,7 @@ ff_linux_epoll_wait(int epfd, struct epoll_event *events, pid_t ff_linux_fork(void) { + DEBUG_LOG("ff_linux_fork\n"); SYSCALL(fork, ()); } diff --git a/adapter/ff_socket_ops.c b/adapter/ff_socket_ops.c index 0b31e79be..b747d538f 100644 --- a/adapter/ff_socket_ops.c +++ b/adapter/ff_socket_ops.c @@ -450,7 +450,7 @@ ff_handle_socket_ops(struct ff_so_context *sc) sc->error = errno; if (sc->ops == FF_SO_EPOLL_WAIT || sc->ops == FF_SO_KEVENT) { - DEBUG_LOG("ff_event_loop_nb:%d/%d, ff_next_event_flag:%d\n", + DEBUG_LOG("ff_event_loop_nb:%d, ff_next_event_flag:%d\n", ff_event_loop_nb, ff_next_event_flag); if (ff_event_loop_nb > 0) { ff_next_event_flag = 1; @@ -481,6 +481,7 @@ void ff_handle_each_context() { uint16_t i, nb_handled, tmp; + static uint64_t loop_count = 0; ff_event_loop_nb = 0; @@ -495,9 +496,11 @@ ff_handle_each_context() for (i = 0; i < ff_so_zone->count; i++) { struct ff_so_context *sc = &ff_so_zone->sc[i]; - DEBUG_LOG("so:%p, so->count:%d,%p, sc:%p, sc->inuse:%d,%p, i:%d, nb:%d, all_nb:%d\n", - ff_so_zone, ff_so_zone->count, &ff_so_zone->count, - sc, ff_so_zone->inuse[i], &ff_so_zone->inuse[i], i, nb_handled, tmp); + if ((loop_count & 1048575) == 0) { + DEBUG_LOG("so:%p, so->count:%d,%p, sc:%p, sc->inuse:%d,%p, i:%d, nb:%d, all_nb:%d\n", + ff_so_zone, ff_so_zone->count, &ff_so_zone->count, + sc, ff_so_zone->inuse[i], &ff_so_zone->inuse[i], i, nb_handled, tmp); + } if (ff_so_zone->inuse[i] == 0) { continue; @@ -523,5 +526,7 @@ ff_handle_each_context() } rte_spinlock_unlock(&ff_so_zone->lock); + + loop_count++; } diff --git a/adapter/ff_socket_ops.h b/adapter/ff_socket_ops.h index d6e57490b..5e74e22e6 100644 --- a/adapter/ff_socket_ops.h +++ b/adapter/ff_socket_ops.h @@ -19,7 +19,8 @@ #endif #define ERR_LOG(fmt, ...) do { \ - printf("file:%s, line:%u, fun:%s, "fmt, __FILE__, __LINE__, __func__, ##__VA_ARGS__); \ + printf("file:%s, line:%u, fun:%s, thread self tid:%ld, "fmt, \ + __FILE__, __LINE__, __func__, pthread_self(), ##__VA_ARGS__); \ } while (0) #ifdef NDEBUG From ac0321e70e28282a1bbfc78d5a4cdeab562f38ae Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Sun, 23 Apr 2023 11:32:33 +0800 Subject: [PATCH 27/43] Add refcoount in sc for fork and detach. --- adapter/ff_hook_syscall.c | 22 +++++++++++++++++++++- adapter/ff_so_zone.c | 28 ++++++++++++++++++++-------- adapter/ff_socket_ops.h | 5 ++++- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index 110b23cba..737bf3794 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -1452,8 +1452,28 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, pid_t ff_hook_fork(void) { + pid_t pid; + DEBUG_LOG("ff_hook_fork\n"); - return ff_linux_fork(); + if (sc) { + rte_spinlock_lock(&sc->lock); + } + + pid = ff_linux_fork(); + + if (sc) { + /* Parent process set refcount. */ + if (pid > 0) { + sc->refcount++; + } + + /* Parent process unlock sc, fork success of failed. */ + if (pid != 0) { + rte_spinlock_unlock(&sc->lock); + } + } + + return pid; } int diff --git a/adapter/ff_so_zone.c b/adapter/ff_so_zone.c index 9f5555950..8e694ef74 100644 --- a/adapter/ff_so_zone.c +++ b/adapter/ff_so_zone.c @@ -91,6 +91,7 @@ ff_create_so_memzone() rte_spinlock_init(&sc->lock); sc->status = FF_SC_IDLE; sc->idx = i; + sc->refcount = 0; //so_zone_tmp->inuse[i] = 0; if (sem_init(&sc->wait_sem, 1, 0) == -1) { @@ -161,6 +162,7 @@ ff_attach_so_context(int proc_id) ff_so_zone->inuse[idx] = 1; rte_spinlock_init(&sc->lock); sc->status = FF_SC_IDLE; + sc->refcount = 1; ff_so_zone->free--; ff_so_zone->idx = idx + 1; break; @@ -191,20 +193,30 @@ ff_detach_so_context(struct ff_so_context *sc) return; } - ERR_LOG("detach sc:%p, ops:%d, status:%d, idx:%d, inuse:%d, so free:%u, idx:%u\n", - sc, sc->ops, sc->status, sc->idx, ff_so_zone->inuse[sc->idx], ff_so_zone->free, ff_so_zone->idx); + ERR_LOG("detach sc:%p, ops:%d, status:%d, idx:%d, sc->refcount:%d, inuse:%d, so free:%u, idx:%u\n", + sc, sc->ops, sc->status, sc->idx, sc->refcount, ff_so_zone->inuse[sc->idx], ff_so_zone->free, ff_so_zone->idx); rte_spinlock_lock(&ff_so_zone->lock); + rte_spinlock_lock(&sc->lock); - if (ff_so_zone->inuse[sc->idx] == 1) { - ff_so_zone->inuse[sc->idx] = 0; + if (sc->refcount > 1) { + ERR_LOG("sc refcount > 1, to sub it, sc:%p, ops:%d, status:%d, idx:%d, sc->refcount:%d, inuse:%d, so free:%u, idx:%u\n", + sc, sc->ops, sc->status, sc->idx, sc->refcount, ff_so_zone->inuse[sc->idx], ff_so_zone->free, ff_so_zone->idx); + sc->refcount--; + } else { + ERR_LOG("sc refcount is 1, to detach it, sc:%p, ops:%d, status:%d, idx:%d, sc->refcount:%d, inuse:%d, so free:%u, idx:%u\n", + sc, sc->ops, sc->status, sc->idx, sc->refcount, ff_so_zone->inuse[sc->idx], ff_so_zone->free, ff_so_zone->idx); + if (ff_so_zone->inuse[sc->idx] == 1) { + ff_so_zone->inuse[sc->idx] = 0; - ff_so_zone->free++; - ff_so_zone->idx = sc->idx; + ff_so_zone->free++; + ff_so_zone->idx = sc->idx; + } } - ERR_LOG("detach sc:%p, ops:%d, status:%d, idx:%d, inuse:%d, so free:%u, idx:%u\n", - sc, sc->ops, sc->status, sc->idx, ff_so_zone->inuse[sc->idx], ff_so_zone->free, ff_so_zone->idx); + ERR_LOG("detach sc:%p, ops:%d, status:%d, idx:%d, sc->refcount:%d, inuse:%d, so free:%u, idx:%u\n", + sc, sc->ops, sc->status, sc->idx, sc->refcount, ff_so_zone->inuse[sc->idx], ff_so_zone->free, ff_so_zone->idx); + rte_spinlock_unlock(&sc->lock); rte_spinlock_unlock(&ff_so_zone->lock); } diff --git a/adapter/ff_socket_ops.h b/adapter/ff_socket_ops.h index 5e74e22e6..436a12d42 100644 --- a/adapter/ff_socket_ops.h +++ b/adapter/ff_socket_ops.h @@ -91,6 +91,7 @@ struct ff_socket_ops_zone { } __attribute__((aligned(RTE_CACHE_LINE_SIZE))); struct ff_so_context { + /* CACHE LINE 0 */ enum FF_SOCKET_OPS ops; enum FF_SO_CONTEXT_STATUS status; void *args; @@ -105,7 +106,9 @@ struct ff_so_context { sem_t wait_sem; /* 32 bytes */ - // listen fd, refcount.. + /* CACHE LINE 1 */ + /* listen fd, refcount.. */ + int refcount; } __attribute__((aligned(RTE_CACHE_LINE_SIZE))); extern __FF_THREAD struct ff_socket_ops_zone *ff_so_zone; From 77d7051991e278fbdb12d846816434bf2a4c629e Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Tue, 25 Apr 2023 11:59:10 +0800 Subject: [PATCH 28/43] Nginx can basic work with libff_syscall.so without any code modify now. 1. Add FF_KERNEL_EVENT argument, default disable. If enable FF_KERNEL_EVENT, epoll_create/epoll_clt/epoll_wait always call f-stack and system API at the same time. Use for some scenarios similar to Nginx. And F-Stack's kern.maxfiles value shoudn't > 65536. 2. Fix the issue that abs_timeout possibly is negative numbers. 3. If timeout < 0, such as -1, ff_hook_epoll_wait will run loop always until return some events. 3. Add some log. --- adapter/Makefile | 8 ++ adapter/ff_hook_syscall.c | 178 ++++++++++++++++++++++++++++++++----- adapter/ff_linux_syscall.c | 11 +++ adapter/ff_socket_ops.c | 2 + 4 files changed, 177 insertions(+), 22 deletions(-) diff --git a/adapter/Makefile b/adapter/Makefile index e02451764..743b1de4e 100644 --- a/adapter/Makefile +++ b/adapter/Makefile @@ -17,6 +17,10 @@ DEBUG=-O0 -gdwarf-2 -g3 # If disable it, one socket can use in all threads. #FF_THREAD_SOCKET=1 +# If enable FF_KERNEL_EVENT, epoll_create/epoll_clt/epoll_wait always call f-stack and system API at the same time. +# Use for some scenarios similar to Nginx. +#FF_KERNEL_EVENT=1 + PKGCONF ?= pkg-config ifndef DEBUG @@ -29,6 +33,10 @@ ifdef FF_THREAD_SOCKET CFLAGS+= -DFF_THREAD_SOCKET endif +ifdef FF_KERNEL_EVENT + CFLAGS+= -DFF_KERNEL_EVENT +endif + CFLAGS += -fPIC -Wall -Werror $(shell $(PKGCONF) --cflags libdpdk) INCLUDES= -I. -I${FF_PATH}/lib diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index 737bf3794..6faa6eaf2 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -119,11 +119,13 @@ static __thread struct ff_kevent_args *kevent_args = NULL; } while (0) #define RETURN_NOFREE() do { \ + DEBUG_LOG("RETURN_NOFREE ret:%d, errno:%d\n", ret, errno); \ return ret; \ } while (0) #define RETURN_ERROR_NOFREE(err) do { \ errno = err; \ + DEBUG_LOG("RETURN_ERROR_NOFREE ret:%d, errno:%d\n", ret, errno); \ return ret; \ } while (0) @@ -143,6 +145,12 @@ static __FF_THREAD struct ff_so_context *sc; static pthread_key_t key; +#ifdef FF_KERNEL_EVENT +/* kern.maxfiles: 33554432 */ +#define FF_MAX_FREEBSD_FILES 65536 +int fstack_kernel_fd_map[FF_MAX_FREEBSD_FILES]; +#endif + /* process-level initialization flag */ static int proc_inited = 0; @@ -607,6 +615,8 @@ ff_hook_connect(int fd, const struct sockaddr *addr, ssize_t ff_hook_recv(int fd, void *buf, size_t len, int flags) { + DEBUG_LOG("ff_hook_recv, fd:%d, buf:%p, len:%lu, flags:%d\n", + fd, buf, len, flags); return ff_hook_recvfrom(fd, buf, len, flags, NULL, NULL); } @@ -614,6 +624,9 @@ ssize_t ff_hook_recvfrom(int fd, void *buf, size_t len, int flags, struct sockaddr *from, socklen_t *fromlen) { + DEBUG_LOG("ff_hook_recvfrom, fd:%d, buf:%p, len:%lu, flags:%d, from:%p, fromlen:%p\n", + fd, buf, len, flags, from, fromlen); + if (buf == NULL || len == 0) { errno = EINVAL; return -1; @@ -892,6 +905,8 @@ msghdr_share_memcpy(const struct msghdr *dst, const struct msghdr *src) ssize_t ff_hook_recvmsg(int fd, struct msghdr *msg, int flags) { + DEBUG_LOG("ff_hook_recvmsg, fd:%d, msg:%p, flags:%d\n", fd, msg, flags); + if (msg == NULL || msg->msg_iov == NULL || msg->msg_iovlen == 0) { errno = EINVAL; @@ -930,7 +945,7 @@ ff_hook_recvmsg(int fd, struct msghdr *msg, int flags) ssize_t ff_hook_read(int fd, void *buf, size_t len) { - DEBUG_LOG("ff_hook_read, fd:%d, len:%lu\n", fd, len); + DEBUG_LOG("ff_hook_read, fd:%d, buf:%p, len:%lu\n", fd, buf, len); if (buf == NULL || len == 0) { errno = EINVAL; @@ -975,6 +990,8 @@ ff_hook_read(int fd, void *buf, size_t len) ssize_t ff_hook_readv(int fd, const struct iovec *iov, int iovcnt) { + DEBUG_LOG("ff_hook_readv, fd:%d, iov:%p, iovcnt:%d\n", fd, iov, iovcnt); + if (iov == NULL || iovcnt == 0) { errno = EINVAL; return -1; @@ -1008,6 +1025,9 @@ ssize_t ff_hook_sendto(int fd, const void *buf, size_t len, int flags, const struct sockaddr *to, socklen_t tolen) { + DEBUG_LOG("ff_hook_sendto, fd:%d, buf:%p, len:%lu, flags:%d, to:%p, tolen:%d\n", + fd, buf, len, flags, to, tolen); + if (buf == NULL || len == 0) { errno = EINVAL; return -1; @@ -1074,6 +1094,9 @@ ff_hook_sendto(int fd, const void *buf, size_t len, int flags, ssize_t ff_hook_sendmsg(int fd, const struct msghdr *msg, int flags) { + DEBUG_LOG("ff_hook_sendmsg, fd:%d, msg:%p, flags:%d\n", + fd, msg, flags); + if (msg == NULL || msg->msg_iov == NULL || msg->msg_iovlen == 0) { errno = EINVAL; @@ -1112,6 +1135,7 @@ ff_hook_sendmsg(int fd, const struct msghdr *msg, int flags) ssize_t ff_hook_send(int fd, const void *buf, size_t len, int flags) { + DEBUG_LOG("ff_hook_send, fd:%d, buf:%p, len:%lu, flags:%d\n", fd, buf, len, flags); return ff_hook_sendto(fd, buf, len, flags, NULL, 0); } @@ -1158,6 +1182,8 @@ ff_hook_write(int fd, const void *buf, size_t len) ssize_t ff_hook_writev(int fd, const struct iovec *iov, int iovcnt) { + DEBUG_LOG("ff_hook_writev, fd:%d, iov:%p, iovcnt:%d\n", fd, iov, iovcnt); + if (iov == NULL || iovcnt == 0) { errno = EINVAL; return -1; @@ -1270,6 +1296,7 @@ ff_hook_fcntl(int fd, int cmd, unsigned long data) int ff_hook_epoll_create(int fdsize) { + DEBUG_LOG("ff_hook_epoll_create, fdsize:%d\n", fdsize); if (inited == 0 || ((fdsize & SOCK_KERNEL) && !(fdsize & SOCK_FSTACK))/* || (fdsize >= 1 && fdsize <= 16)*/) { fdsize &= ~SOCK_KERNEL; @@ -1283,6 +1310,13 @@ ff_hook_epoll_create(int fdsize) SYSCALL(FF_SO_EPOLL_CREATE, args); if (ret >= 0) { +#ifdef FF_KERNEL_EVENT + int kernel_fd; + + kernel_fd = ff_linux_epoll_create(fdsize); + fstack_kernel_fd_map[ret] = kernel_fd; + ERR_LOG("ff_hook_epoll_create fstack fd:%d, FF_KERNEL_EVENT kernel_fd:%d:\n", ret, kernel_fd); +#endif ret = convert_fstack_fd(ret); } @@ -1295,8 +1329,30 @@ int ff_hook_epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { + int ff_epfd; + DEBUG_LOG("ff_hook_epoll_ctl, epfd:%d, op:%d, fd:%d\n", epfd, op, fd); + +#ifdef FF_KERNEL_EVENT + if (unlikely(!is_fstack_fd(fd))) { + if (is_fstack_fd(epfd)) { + ff_epfd = restore_fstack_fd(epfd); + if (likely(fstack_kernel_fd_map[ff_epfd] > 0)) { + epfd = fstack_kernel_fd_map[ff_epfd]; + DEBUG_LOG("ff_epfd:%d, kernel epfd:%d\n", ff_epfd, epfd); + } else { + ERR_LOG("invalid fd and ff_epfd:%d, epfd:%d, op:%d, fd:%d\n", ff_epfd, epfd, op, fd); + errno = EBADF; + return -1; + } + } + return ff_linux_epoll_ctl(epfd, op, fd, event); + } + fd = restore_fstack_fd(fd); +#else CHECK_FD_OWNERSHIP(epoll_ctl, (epfd, op, fd, event)); +#endif + ff_epfd = restore_fstack_fd(epfd); DEFINE_REQ_ARGS_STATIC(epoll_ctl); static __thread struct epoll_event *sh_event = NULL; @@ -1322,7 +1378,7 @@ ff_hook_epoll_ctl(int epfd, int op, int fd, args->event = NULL; } - args->epfd = restore_fstack_fd(epfd); + args->epfd = ff_epfd; args->op = op; args->fd = fd; @@ -1341,23 +1397,67 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, { DEBUG_LOG("ff_hook_epoll_wait, epfd:%d, maxevents:%d, timeout:%d\n", epfd, maxevents, timeout); int fd = epfd; + struct timespec abs_timeout; + CHECK_FD_OWNERSHIP(epoll_wait, (epfd, events, maxevents, timeout)); DEFINE_REQ_ARGS_STATIC(epoll_wait); static __thread struct epoll_event *sh_events = NULL; + static __thread int sh_events_len = 0; - if (sh_events == NULL) { - sh_events = share_mem_alloc(sizeof(struct epoll_event) * maxevents); +#ifdef FF_KERNEL_EVENT + /* maxevents must >= 2, if use FF_KERNEL_EVENT */ + if (unlikely(maxevents < 2)) { + ERR_LOG("maxevents must >= 2, if use FF_KERNEL_EVENT, now is %d\n", maxevents); + RETURN_ERROR_NOFREE(EINVAL); + } + + int kernel_ret = 0; + int kernel_maxevents = kernel_maxevents = maxevents / 16; + + if (kernel_maxevents > SOCKET_OPS_CONTEXT_MAX_NUM) { + kernel_maxevents = SOCKET_OPS_CONTEXT_MAX_NUM; + } else if (kernel_maxevents <= 0) { + kernel_maxevents = 1; + } + maxevents -= kernel_maxevents; +#endif + + if (sh_events == NULL || sh_events_len < maxevents) { + if (sh_events) { + share_mem_free(sh_events); + } + + sh_events_len = maxevents; + sh_events = share_mem_alloc(sizeof(struct epoll_event) * sh_events_len); if (sh_events == NULL) { RETURN_ERROR_NOFREE(ENOMEM); } } + if (timeout > 0) { + clock_gettime(CLOCK_REALTIME, &abs_timeout); + DEBUG_LOG("before wait, sec:%ld, nsec:%ld\n", abs_timeout.tv_sec, abs_timeout.tv_nsec); + abs_timeout.tv_sec += timeout / 1000; + /* must % 1000 first, otherwise type(int) maybe overflow, and sem_timedwait failed */ + abs_timeout.tv_nsec += (timeout % 1000) * 1000 * 1000; + if (abs_timeout.tv_nsec > NS_PER_SECOND) { + abs_timeout.tv_nsec -= NS_PER_SECOND; + abs_timeout.tv_sec += 1; + } + if (unlikely(abs_timeout.tv_sec < 0 || abs_timeout.tv_nsec < 0)) { + ERR_LOG("invalid timeout argument, the sec:%ld, nsec:%ld\n", + abs_timeout.tv_sec, abs_timeout.tv_nsec); + RETURN_ERROR_NOFREE(EINVAL); + } + } + args->epfd = fd; args->events = sh_events; args->maxevents = maxevents; args->timeout = timeout; +RETRY: /* for timeout, Although not really effective in FreeBSD stack */ //SYSCALL(FF_SO_EPOLL_WAIT, args); ACQUIRE_ZONE_LOCK(FF_SC_IDLE); @@ -1375,24 +1475,33 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, sc->result = 0; sc->error = 0; errno = 0; - if (timeout == 0) { + if (timeout <= 0) { need_alarm_sem = 1; } RELEASE_ZONE_LOCK(FF_SC_REQ); - if (timeout > 0) { - struct timespec abs_timeout; - - clock_gettime(CLOCK_REALTIME, &abs_timeout); - DEBUG_LOG("before wait, sec:%ld, nsec:%ld\n", abs_timeout.tv_sec, abs_timeout.tv_nsec); - abs_timeout.tv_sec += timeout / 1000; - abs_timeout.tv_nsec += timeout * 1000 * 1000; - if (abs_timeout.tv_nsec > NS_PER_SECOND) { - abs_timeout.tv_nsec -= NS_PER_SECOND; - abs_timeout.tv_sec += 1; +#ifdef FF_KERNEL_EVENT + /* + * Call ff_linux_epoll_wait before sem_timedwait/sem_wait. + * And set timeout is 0. + * + * If there are events return, and move event offset to unused event for copy F-Stack events. + */ + DEBUG_LOG("call ff_linux_epoll_wait at the same time, epfd:%d, fstack_kernel_fd_map[epfd]:%d, kernel_maxevents:%d\n", + fd, fstack_kernel_fd_map[fd], kernel_maxevents); + if (likely(fstack_kernel_fd_map[fd] > 0)) { + kernel_ret = ff_linux_epoll_wait(fstack_kernel_fd_map[fd], events, kernel_maxevents, 0); + DEBUG_LOG("ff_linux_epoll_wait kernel_ret:%d, errno:%d\n", ret, errno); + if (kernel_ret < 0) { + kernel_ret = 0; + } else if (kernel_ret > 0) { + events += kernel_ret; } + } +#endif + if (timeout > 0) { DEBUG_LOG("ready to wait, sec:%ld, nsec:%ld\n", abs_timeout.tv_sec, abs_timeout.tv_nsec); ret = sem_timedwait(&sc->wait_sem, &abs_timeout); @@ -1404,8 +1513,8 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, rte_spinlock_lock(&sc->lock); - if (timeout == 0) { - need_alarm_sem = 1; + if (timeout <= 0) { + need_alarm_sem = 0; } /* @@ -1415,9 +1524,11 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, * And only ret == 0, means sem_timedwait return normal, * can set ret = sc->result, otherwise may use last sc->result. */ - if (ret == -1 && errno == ETIMEDOUT /* sc->status == FF_SC_REQ */) { + DEBUG_LOG("sem wait, ret:%d, sc->result:%d, sc->errno:%d\n", + ret, sc->result, sc->error); + if (unlikely(ret == -1 && errno == ETIMEDOUT /* sc->status == FF_SC_REQ */)) { ret = 0; - } else if (ret == 0) { + } else if (likely(ret == 0)) { ret = sc->result; if (ret < 0) { errno = sc->error; @@ -1427,7 +1538,7 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, sc->status = FF_SC_IDLE; rte_spinlock_unlock(&sc->lock); - if (ret > 0) { + if (likely(ret > 0)) { if (unlikely(ret > maxevents)) { ERR_LOG("return events:%d, maxevents:%d, set return events to maxevents, may be some error occur\n", ret, maxevents); @@ -1436,6 +1547,23 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events, rte_memcpy(events, sh_events, sizeof(struct epoll_event) * ret); } +#ifdef FF_KERNEL_EVENT + if (unlikely(kernel_ret > 0)) { + if (likely(ret > 0)) { + ret += kernel_ret; + } else { + ret = kernel_ret; + } + } +#endif + + /* If timeout is -1, always retry epoll_wait until ret not 0 */ + if (timeout <= 0 && ret == 0) { + //usleep(100); + rte_pause(); + goto RETRY; + } + /* * Don't free, to improve proformance. * Will cause memory leak while APP exit , but fstack adapter not exit. @@ -1613,8 +1741,14 @@ kevent(int kq, const struct kevent *changelist, int nchanges, abs_timeout.tv_nsec -= NS_PER_SECOND; abs_timeout.tv_sec += 1; } - - ret = sem_timedwait(&sc->wait_sem, &abs_timeout); + if (unlikely(abs_timeout.tv_sec < 0 || abs_timeout.tv_nsec < 0)) { + ERR_LOG("invalid timeout argument, the sec:%ld, nsec:%ld\n", + abs_timeout.tv_sec, abs_timeout.tv_nsec); + errno = EINVAL; + ret = -1; + } else { + ret = sem_timedwait(&sc->wait_sem, &abs_timeout); + } } else { ret = sem_wait(&sc->wait_sem); } diff --git a/adapter/ff_linux_syscall.c b/adapter/ff_linux_syscall.c index 7c950e23a..3f6f950a9 100644 --- a/adapter/ff_linux_syscall.c +++ b/adapter/ff_linux_syscall.c @@ -142,11 +142,13 @@ int ff_linux_connect(int s, const struct sockaddr *addr, ssize_t ff_linux_recv(int s, void *buf, size_t len, int flags) { + DEBUG_LOG("ff_linux_recv, fd:%d, buf:%p, len:%lu, flags:%d\n", s, buf, len, flags); SYSCALL(recv, (s, buf, len, flags)); } ssize_t ff_linux_send(int s, const void *buf, size_t len, int flags) { + DEBUG_LOG("ff_linux_send, fd:%d, buf:%p, len:%lu, flags:%d\n", s, buf, len, flags); SYSCALL(send, (s, buf, len, flags)); } @@ -164,17 +166,21 @@ ssize_t ff_linux_write(int s, const void *buf, size_t len) ssize_t ff_linux_writev(int s, const struct iovec *iov, int iovcnt) { + DEBUG_LOG("ff_linux_writev, fd:%d, iov:%p, iovcnt:%d\n", s, iov, iovcnt); SYSCALL(writev, (s, iov, iovcnt)); } ssize_t ff_linux_readv(int s, const struct iovec *iov, int iovcnt) { + DEBUG_LOG("ff_linux_readv, fd:%d, iov:%p, iovcnt:%d\n", s, iov, iovcnt); SYSCALL(readv, (s, iov, iovcnt)); } ssize_t ff_linux_sendto(int s, const void *buf, size_t len, int flags, const struct sockaddr *to, socklen_t tolen) { + DEBUG_LOG("ff_linux_sendto, fd:%d, buf:%p, len:%lu, flags:%d, to:%p, tolen:%d\n", + s, buf, len, flags, to, tolen); SYSCALL(sendto, (s, buf, len, flags, to, tolen)); } @@ -182,16 +188,21 @@ ssize_t ff_linux_recvfrom(int s, void *buf, size_t len, int flags, struct sockaddr *from, socklen_t *fromlen) { + DEBUG_LOG("ff_linux_recvfrom, fd:%d, buf:%p, len:%lu, flags:%d, from:%p, fromlen:%p\n", + s, buf, len, flags, from, fromlen); SYSCALL(recvfrom, (s, buf, len, flags, from, fromlen)); } ssize_t ff_linux_sendmsg(int s, const struct msghdr *msg, int flags) { + DEBUG_LOG("ff_linux_sendmsg, fd:%d, msg:%p, flags:%d\n", + s, msg, flags); SYSCALL(sendmsg, (s, msg, flags)); } ssize_t ff_linux_recvmsg(int s, struct msghdr *msg, int flags) { + DEBUG_LOG("ff_linux_recvmsg, fd:%d, msg:%p, flags:%d\n", s, msg, flags); SYSCALL(recvmsg, (s, msg, flags)) } diff --git a/adapter/ff_socket_ops.c b/adapter/ff_socket_ops.c index b747d538f..7c8fa3fb5 100644 --- a/adapter/ff_socket_ops.c +++ b/adapter/ff_socket_ops.c @@ -428,6 +428,7 @@ ff_so_handler(int ops, void *args) } errno = EINVAL; + DEBUG_LOG("ff_so_handler error:%d, ops:%d\n", errno, ops); return (-1); } @@ -448,6 +449,7 @@ ff_handle_socket_ops(struct ff_so_context *sc) errno = 0; sc->result = ff_so_handler(sc->ops, sc->args); sc->error = errno; + DEBUG_LOG("ff_handle_socket_ops error:%d, ops:%d, result:%d\n", errno, sc->ops, sc->result); if (sc->ops == FF_SO_EPOLL_WAIT || sc->ops == FF_SO_KEVENT) { DEBUG_LOG("ff_event_loop_nb:%d, ff_next_event_flag:%d\n", From a791578c8a72a8f13816bf522164d49fe65adb24 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Tue, 25 Apr 2023 15:01:22 +0800 Subject: [PATCH 29/43] Fix the issue that memory leak in ff_hook_ioctl and ff_hook_getsockopt. --- adapter/ff_hook_syscall.c | 36 +++++++++++++++++++++++------------- adapter/ff_linux_syscall.c | 2 ++ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index 6faa6eaf2..a3b0eb603 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -131,12 +131,14 @@ static __thread struct ff_kevent_args *kevent_args = NULL; #define RETURN() do { \ share_mem_free(args); \ + DEBUG_LOG("RETURN ret:%d, errno:%d\n", ret, errno); \ return ret; \ } while (0) #define RETURN_ERROR(err) do { \ errno = err; \ share_mem_free(args); \ + DEBUG_LOG("RETURN_ERROR ret:%d, errno:%d\n", ret, errno); \ return ret; \ } while (0) @@ -267,7 +269,7 @@ ff_hook_bind(int fd, const struct sockaddr *addr, CHECK_FD_OWNERSHIP(bind, (fd, addr, addrlen)); DEFINE_REQ_ARGS(bind); - struct sockaddr *sh_addr; + struct sockaddr *sh_addr = NULL; sh_addr = share_mem_alloc(addrlen); if (sh_addr == NULL) { @@ -327,8 +329,8 @@ ff_hook_getsockname(int fd, struct sockaddr *name, CHECK_FD_OWNERSHIP(getsockname, (fd, name, namelen)); DEFINE_REQ_ARGS(getsockname); - struct sockaddr *sh_name; - socklen_t *sh_namelen; + struct sockaddr *sh_name = NULL; + socklen_t *sh_namelen = NULL; sh_name = share_mem_alloc(*namelen); if (sh_name == NULL) { @@ -372,8 +374,8 @@ ff_hook_getpeername(int fd, struct sockaddr *name, CHECK_FD_OWNERSHIP(getpeername, (fd, name, namelen)); DEFINE_REQ_ARGS(getpeername); - struct sockaddr *sh_name; - socklen_t *sh_namelen; + struct sockaddr *sh_name = NULL; + socklen_t *sh_namelen = NULL; sh_name = share_mem_alloc(*namelen); if (sh_name == NULL) { @@ -419,7 +421,7 @@ ff_hook_getsockopt(int fd, int level, int optname, DEFINE_REQ_ARGS(getsockopt); void *sh_optval = NULL; - socklen_t *sh_optlen; + socklen_t *sh_optlen = NULL; if (optval != NULL) { sh_optval = share_mem_alloc(*optlen); @@ -449,11 +451,13 @@ ff_hook_getsockopt(int fd, int level, int optname, if (ret == 0) { if (optval) { rte_memcpy(optval, sh_optval, *sh_optlen); - share_mem_free(sh_optval); } *optlen = *sh_optlen; } + if (sh_optval) { + share_mem_free(sh_optval); + } share_mem_free(sh_optlen); RETURN(); } @@ -568,6 +572,8 @@ int ff_hook_accept4(int fd, struct sockaddr *addr, socklen_t *addrlen, int flags) { + DEBUG_LOG("ff_hook_accept4, fd:%d, addr:%p, addrlen:%p, flags:%d\n", fd, addr, addrlen, flags); + CHECK_FD_OWNERSHIP(accept4, (fd, addr, addrlen, flags)); errno = ENOSYS; @@ -578,6 +584,8 @@ int ff_hook_connect(int fd, const struct sockaddr *addr, socklen_t addrlen) { + DEBUG_LOG("ff_hook_connect, fd:%d, addr:%p, addrlen:%u\n", fd, addr, addrlen); + if (addr == NULL) { errno = EINVAL; return -1; @@ -641,7 +649,7 @@ ff_hook_recvfrom(int fd, void *buf, size_t len, int flags, CHECK_FD_OWNERSHIP(recvfrom, (fd, buf, len, flags, from, fromlen)); DEFINE_REQ_ARGS_STATIC(recvfrom); - static __thread void *sh_buf; + static __thread void *sh_buf = NULL; static __thread size_t sh_buf_len = 0; static __thread struct sockaddr *sh_from = NULL; static __thread socklen_t sh_from_len = 0; @@ -916,7 +924,7 @@ ff_hook_recvmsg(int fd, struct msghdr *msg, int flags) CHECK_FD_OWNERSHIP(recvmsg, (fd, msg, flags)); DEFINE_REQ_ARGS(recvmsg); - struct msghdr *sh_msg; + struct msghdr *sh_msg = NULL; sh_msg = msghdr_share_alloc(msg); if (sh_msg == NULL) { @@ -1000,7 +1008,7 @@ ff_hook_readv(int fd, const struct iovec *iov, int iovcnt) CHECK_FD_OWNERSHIP(readv, (fd, iov, iovcnt)); DEFINE_REQ_ARGS(readv); - struct iovec *sh_iov; + struct iovec *sh_iov = NULL; sh_iov = iovec_share_alloc(iov, iovcnt); if (sh_iov == NULL) { @@ -1106,7 +1114,7 @@ ff_hook_sendmsg(int fd, const struct msghdr *msg, int flags) CHECK_FD_OWNERSHIP(sendmsg, (fd, msg, flags)); DEFINE_REQ_ARGS(sendmsg); - struct msghdr *sh_msg; + struct msghdr *sh_msg = NULL; sh_msg = msghdr_share_alloc(msg); if (sh_msg == NULL) { @@ -1192,7 +1200,7 @@ ff_hook_writev(int fd, const struct iovec *iov, int iovcnt) CHECK_FD_OWNERSHIP(writev, (fd, iov, iovcnt)); DEFINE_REQ_ARGS(writev); - struct iovec *sh_iov; + struct iovec *sh_iov = NULL; sh_iov = iovec_share_alloc(iov, iovcnt); if (sh_iov == NULL) { @@ -1248,7 +1256,7 @@ ff_hook_ioctl(int fd, unsigned long req, unsigned long data) CHECK_FD_OWNERSHIP(ioctl, (fd, req, data)); DEFINE_REQ_ARGS(ioctl); - unsigned long *sh_data; + unsigned long *sh_data = NULL; sh_data = share_mem_alloc(sizeof(int)); if (sh_data == NULL) { @@ -1266,6 +1274,8 @@ ff_hook_ioctl(int fd, unsigned long req, unsigned long data) *((int *)data) = *sh_data; } + share_mem_free(sh_data); + RETURN(); } diff --git a/adapter/ff_linux_syscall.c b/adapter/ff_linux_syscall.c index 3f6f950a9..371332346 100644 --- a/adapter/ff_linux_syscall.c +++ b/adapter/ff_linux_syscall.c @@ -131,12 +131,14 @@ int ff_linux_accept(int s, struct sockaddr *addr, socklen_t *addrlen) int ff_linux_accept4(int s, struct sockaddr *addr, socklen_t *addrlen, int flags) { + DEBUG_LOG("ff_linux_accept4, fd:%d, addr:%p, addrlen:%p, flags:%d\n", s, addr, addrlen, flags); SYSCALL(accept4, (s, addr, addrlen, flags)); } int ff_linux_connect(int s, const struct sockaddr *addr, socklen_t addrlen) { + DEBUG_LOG("ff_linux_connect, fd:%d, addr:%p, addrlen:%u\n", s, addr, addrlen); SYSCALL(connect, (s, addr, addrlen)); } From f98fd1a6159e57844abdd2c90589dd8939d155e6 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 26 Apr 2023 15:29:15 +0800 Subject: [PATCH 30/43] Don't free the memory malloc with rte_malloc in more socket API to improve proformance, such as ff_hook_writev(), ff_hook_ioctl(), ff_hook_setsockopt(), etc. But there are some others socket API not support this now, such as ff_hook_recvmsg(), ff_hook_readv(), ff_hook_sendmsg, because it not be used now, and do it is relatively troublesome, So If you need call them very frequently, maybe you should support it, and can see ff_hook_writev(). Now it will cause memory lead, need to fix it later. --- adapter/ff_hook_syscall.c | 414 ++++++++++++++++++++++++++++++++------ 1 file changed, 348 insertions(+), 66 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index a3b0eb603..d8828d45d 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -61,17 +61,34 @@ struct ff_config ff_global_cfg; /* Always use __thread, but no __FF_THREAD */ static __thread struct ff_shutdown_args *shutdown_args = NULL; +static __thread struct ff_getsockname_args *getsockname_args = NULL; +static __thread struct ff_getpeername_args *getpeername_args = NULL; +static __thread struct ff_setsockopt_args *setsockopt_args = NULL; static __thread struct ff_accept_args *accept_args = NULL; static __thread struct ff_connect_args *connect_args = NULL; static __thread struct ff_recvfrom_args *recvfrom_args = NULL; +static __thread struct ff_recvmsg_args *recvmsg_args = NULL; static __thread struct ff_read_args *read_args = NULL; +static __thread struct ff_readv_args *readv_args = NULL; static __thread struct ff_sendto_args *sendto_args = NULL; +static __thread struct ff_sendmsg_args *sendmsg_args = NULL; static __thread struct ff_write_args *write_args = NULL; +static __thread struct ff_writev_args *writev_args = NULL; static __thread struct ff_close_args *close_args = NULL; +static __thread struct ff_ioctl_args *ioctl_args = NULL; +static __thread struct ff_fcntl_args *fcntl_args = NULL; static __thread struct ff_epoll_ctl_args *epoll_ctl_args = NULL; static __thread struct ff_epoll_wait_args *epoll_wait_args = NULL; static __thread struct ff_kevent_args *kevent_args = NULL; +#define IOV_MAX 64 +#define IOV_LEN_MAX 2048 + +static __thread struct iovec *sh_iov_static = NULL; +static __thread void *sh_iov_static_base[IOV_MAX]; +static __thread int sh_iov_static_fill_idx_local = 0; +static __thread int sh_iov_static_fill_idx_share = 0; + #define DEFINE_REQ_ARGS_STATIC(name) \ int ret = -1; \ struct ff_##name##_args *args = NULL; \ @@ -328,19 +345,29 @@ ff_hook_getsockname(int fd, struct sockaddr *name, CHECK_FD_OWNERSHIP(getsockname, (fd, name, namelen)); - DEFINE_REQ_ARGS(getsockname); - struct sockaddr *sh_name = NULL; - socklen_t *sh_namelen = NULL; + DEFINE_REQ_ARGS_STATIC(getsockname); + static __thread struct sockaddr *sh_name = NULL; + static __thread socklen_t sh_name_len = 0; + static __thread socklen_t *sh_namelen = NULL; - sh_name = share_mem_alloc(*namelen); - if (sh_name == NULL) { - RETURN_ERROR(ENOMEM); + if (sh_name == NULL || sh_name_len < *namelen) { + if (sh_name) { + share_mem_free(sh_name); + } + + sh_name_len = *namelen; + sh_name = share_mem_alloc(sh_name_len); + if (sh_name == NULL) { + RETURN_ERROR_NOFREE(ENOMEM); + } } - sh_namelen = share_mem_alloc(sizeof(socklen_t)); if (sh_namelen == NULL) { - share_mem_free(sh_name); - RETURN_ERROR(ENOMEM); + sh_namelen = share_mem_alloc(sizeof(socklen_t)); + if (sh_namelen == NULL) { + //share_mem_free(sh_name); + RETURN_ERROR_NOFREE(ENOMEM); + } } *sh_namelen = *namelen; @@ -357,9 +384,10 @@ ff_hook_getsockname(int fd, struct sockaddr *name, *namelen = *sh_namelen; } - share_mem_free(sh_name); - share_mem_free(sh_namelen); - RETURN(); + //share_mem_free(sh_name); + //share_mem_free(sh_namelen); + + RETURN_NOFREE(); } int @@ -373,19 +401,29 @@ ff_hook_getpeername(int fd, struct sockaddr *name, CHECK_FD_OWNERSHIP(getpeername, (fd, name, namelen)); - DEFINE_REQ_ARGS(getpeername); - struct sockaddr *sh_name = NULL; - socklen_t *sh_namelen = NULL; + DEFINE_REQ_ARGS_STATIC(getpeername); + static __thread struct sockaddr *sh_name = NULL; + static __thread socklen_t sh_name_len = 0; + static __thread socklen_t *sh_namelen = NULL; - sh_name = share_mem_alloc(*namelen); - if (sh_name == NULL) { - RETURN_ERROR(ENOMEM); + if (sh_name == NULL || sh_name_len < *namelen) { + if (sh_name) { + share_mem_free(sh_name); + } + + sh_name_len = *namelen; + sh_name = share_mem_alloc(sh_name_len); + if (sh_name == NULL) { + RETURN_ERROR_NOFREE(ENOMEM); + } } - sh_namelen = share_mem_alloc(sizeof(socklen_t)); if (sh_namelen == NULL) { - share_mem_free(sh_name); - RETURN_ERROR(ENOMEM); + sh_namelen = share_mem_alloc(sizeof(socklen_t)); + if (sh_namelen == NULL) { + //share_mem_free(sh_name); + RETURN_ERROR_NOFREE(ENOMEM); + } } *sh_namelen = *namelen; @@ -402,9 +440,10 @@ ff_hook_getpeername(int fd, struct sockaddr *name, *namelen = *sh_namelen; } - share_mem_free(sh_name); - share_mem_free(sh_namelen); - RETURN(); + //share_mem_free(sh_name); + //share_mem_free(sh_namelen); + + RETURN_NOFREE(); } int @@ -474,13 +513,21 @@ ff_hook_setsockopt(int fd, int level, int optname, CHECK_FD_OWNERSHIP(setsockopt, (fd, level, optname, optval, optlen)); - DEFINE_REQ_ARGS(setsockopt); - void *sh_optval = NULL; + DEFINE_REQ_ARGS_STATIC(setsockopt); + static __thread void *sh_optval = NULL; + static __thread socklen_t sh_optval_len = 0; if (optval != NULL) { - sh_optval = share_mem_alloc(optlen); - if (sh_optval == NULL) { - RETURN_ERROR(ENOMEM); + if (sh_optval == NULL || sh_optval_len < optlen) { + if (sh_optval) { + share_mem_free(sh_optval); + } + + sh_optval_len = optlen; + sh_optval = share_mem_alloc(sh_optval_len); + if (sh_optval == NULL) { + RETURN_ERROR_NOFREE(ENOMEM); + } } } @@ -492,11 +539,11 @@ ff_hook_setsockopt(int fd, int level, int optname, SYSCALL(FF_SO_SETSOCKOPT, args); - if (sh_optval) { + /*if (sh_optval) { share_mem_free(sh_optval); - } + }*/ - RETURN(); + RETURN_NOFREE(); } int @@ -617,6 +664,7 @@ ff_hook_connect(int fd, const struct sockaddr *addr, SYSCALL(FF_SO_CONNECT, args); //share_mem_free(sh_addr); + RETURN_NOFREE(); } @@ -815,6 +863,145 @@ iovec_share2local(struct iovec *share, } } +static struct iovec * +iovec_share_alloc_s() +{ + int i, iovcnt = IOV_MAX; + + sh_iov_static = share_mem_alloc(sizeof(struct iovec) * iovcnt); + if (sh_iov_static == NULL) { + ERR_LOG("share_mem_alloc shiov failed, oom\n"); + errno = ENOMEM; + return NULL; + } + + for (i = 0; i < iovcnt; i++) { + sh_iov_static[i].iov_len = IOV_LEN_MAX; + void *iov_base = share_mem_alloc(sh_iov_static[i].iov_len); + sh_iov_static[i].iov_base = iov_base; + sh_iov_static_base[i] = iov_base; + + if (iov_base == NULL) { + ERR_LOG("share_mem_alloc iov_base:%d failed, oom\n", i); + errno = ENOMEM; + goto ERROR; + } + } + + ERR_LOG("iovec_share_alloc_s alloc sh_iov_static:%p success, iovcnt:%d, per iov_len:%d\n", + sh_iov_static, IOV_MAX, IOV_LEN_MAX); + + return sh_iov_static; + +ERROR: + iovec_share_free(sh_iov_static, i); + return NULL; +} + +static int +_iovec_local2share_s(const struct iovec *local, int iovcnt, size_t skip) +{ + int i, j; + size_t len, total = 0; + + DEBUG_LOG("_iovec_local2share_s local iov:%p, iovcnt:%d, skip:%lu, sh_iov_static:%p, " + "first iov_base:%p, iov_len:%lu\n", + local, iovcnt, skip, sh_iov_static, + sh_iov_static[0].iov_base, sh_iov_static[0].iov_len); + + if (local == NULL || iovcnt == 0) { + errno = EINVAL; + return -1; + } + + for (i = sh_iov_static_fill_idx_local, j = 0; i < iovcnt && j < IOV_MAX; i++, j++) { + DEBUG_LOG("local[%d].iov_len:%lu, skip:%lu, total:%lu\n", + i, local[i].iov_len, skip, total); + + if (local[i].iov_len <= skip) { + skip -= local[i].iov_len; + continue; + } + + if ((local[i].iov_len - skip) <= IOV_LEN_MAX) { + sh_iov_static[j].iov_len = local[i].iov_len - skip; + rte_memcpy(sh_iov_static[j].iov_base, local[i].iov_base, + sh_iov_static[j].iov_len); + total += sh_iov_static[j].iov_len; + DEBUG_LOG("sh_iov_static[%d].iov_base:%p, len:%lu, skip:%lu, total:%lu\n", + j, sh_iov_static[j].iov_base, sh_iov_static[j].iov_len, skip, total); + } else { + len = local[i].iov_len - skip; + DEBUG_LOG("local[%d].iov_len:%lu, skip:%lu, total:%lu, len(iov_len - skip):%lu\n", + i, local[i].iov_len, skip, total, len); + for (; j < IOV_MAX ; j++) { + sh_iov_static[j].iov_len = RTE_MIN(IOV_LEN_MAX, len); + rte_memcpy(sh_iov_static[j].iov_base, local[i].iov_base + (local[i].iov_len - len), + sh_iov_static[j].iov_len); + + len -= sh_iov_static[j].iov_len; + total += sh_iov_static[j].iov_len; + + DEBUG_LOG("sh_iov_static[%d].iov_base:%p, len:%lu, skip:%lu, total:%lu, len:%lu\n", + j, sh_iov_static[j].iov_base, sh_iov_static[j].iov_len, skip, total, len); + + if (len == 0) { + break; + } + } + + if (j == IOV_MAX) { + ERR_LOG("Too large buf to send/write, you best to reduce it.\n"); + break; + } + } + } + + sh_iov_static_fill_idx_local = i; + sh_iov_static_fill_idx_share = j; + + DEBUG_LOG("sh_iov_static_fill_idx_local(i):%d, sh_iov_static_fill_idx_share(j):%d, skip:%lu, total:%lu\n", + sh_iov_static_fill_idx_local, sh_iov_static_fill_idx_share, skip, total); + + return total; +} + +static int +iovec_local2share_s(const struct iovec *iov, int iovcnt, size_t skip) +{ + int sent = 0; + + DEBUG_LOG("iovec_local2share_s iov:%p, iovcnt:%d, skip:%lu, sh_iov_static:%p\n", + iov, iovcnt, skip, sh_iov_static); + + if (sh_iov_static == NULL) { + sh_iov_static = iovec_share_alloc_s(); + if (sh_iov_static == NULL) { + ERR_LOG("iovec_share_alloc_s failed, oom\n"); + errno = ENOMEM; + return -1; + } + } + + sent = _iovec_local2share_s(iov, iovcnt, skip); + + return sent; +} + +static void +iovec_share2local_s() +{ + int i; + + DEBUG_LOG("iovec_share2local_s sh_iov_static:%p, sh_iov_static_fill_idx_share:%d\n", + sh_iov_static, sh_iov_static_fill_idx_share); + + for (i = 0; i < sh_iov_static_fill_idx_share; i++) { + sh_iov_static[i].iov_base = sh_iov_static_base[i]; + sh_iov_static[i].iov_len = IOV_LEN_MAX; + } +} + static void msghdr_share_free(struct msghdr *msg) { @@ -923,12 +1110,21 @@ ff_hook_recvmsg(int fd, struct msghdr *msg, int flags) CHECK_FD_OWNERSHIP(recvmsg, (fd, msg, flags)); - DEFINE_REQ_ARGS(recvmsg); + DEFINE_REQ_ARGS_STATIC(recvmsg); + + /* + * If calling very frequently, + * may need to not free the memory malloc with rte_malloc, + * to improve proformance. + * + * Because this API support it relatively troublesome, + * so no support right now. + */ struct msghdr *sh_msg = NULL; sh_msg = msghdr_share_alloc(msg); if (sh_msg == NULL) { - RETURN_ERROR(ENOMEM); + RETURN_ERROR_NOFREE(ENOMEM); } args->fd = fd; @@ -947,7 +1143,8 @@ ff_hook_recvmsg(int fd, struct msghdr *msg, int flags) } msghdr_share_free(sh_msg); - RETURN(); + + RETURN_NOFREE(); } ssize_t @@ -1007,12 +1204,18 @@ ff_hook_readv(int fd, const struct iovec *iov, int iovcnt) CHECK_FD_OWNERSHIP(readv, (fd, iov, iovcnt)); - DEFINE_REQ_ARGS(readv); + DEFINE_REQ_ARGS_STATIC(readv); + + /* + * If calling very frequently, + * may need to not free the memory malloc with rte_malloc, + * to improve proformance, see ff_hook_writev(). + */ struct iovec *sh_iov = NULL; sh_iov = iovec_share_alloc(iov, iovcnt); if (sh_iov == NULL) { - RETURN_ERROR(ENOMEM); + RETURN_ERROR_NOFREE(ENOMEM); } args->fd = fd; @@ -1026,7 +1229,8 @@ ff_hook_readv(int fd, const struct iovec *iov, int iovcnt) } iovec_share_free(sh_iov, iovcnt); - RETURN(); + + RETURN_NOFREE(); } ssize_t @@ -1113,12 +1317,21 @@ ff_hook_sendmsg(int fd, const struct msghdr *msg, int flags) CHECK_FD_OWNERSHIP(sendmsg, (fd, msg, flags)); - DEFINE_REQ_ARGS(sendmsg); + DEFINE_REQ_ARGS_STATIC(sendmsg); + + /* + * If calling very frequently, + * may need to not free the memory malloc with rte_malloc, + * to improve proformance. + * + * Because this API support it relatively troublesome, + * so no support right now. + */ struct msghdr *sh_msg = NULL; sh_msg = msghdr_share_alloc(msg); if (sh_msg == NULL) { - RETURN_ERROR(ENOMEM); + RETURN_ERROR_NOFREE(ENOMEM); } msghdr_share_memcpy(sh_msg, msg); iovec_local2share(sh_msg->msg_iov, @@ -1137,7 +1350,8 @@ ff_hook_sendmsg(int fd, const struct msghdr *msg, int flags) } msghdr_share_free(sh_msg); - RETURN(); + + RETURN_NOFREE(); } ssize_t @@ -1190,6 +1404,9 @@ ff_hook_write(int fd, const void *buf, size_t len) ssize_t ff_hook_writev(int fd, const struct iovec *iov, int iovcnt) { + size_t sent = 0; + int ret_s = -1; + DEBUG_LOG("ff_hook_writev, fd:%d, iov:%p, iovcnt:%d\n", fd, iov, iovcnt); if (iov == NULL || iovcnt == 0) { @@ -1199,27 +1416,57 @@ ff_hook_writev(int fd, const struct iovec *iov, int iovcnt) CHECK_FD_OWNERSHIP(writev, (fd, iov, iovcnt)); - DEFINE_REQ_ARGS(writev); - struct iovec *sh_iov = NULL; - - sh_iov = iovec_share_alloc(iov, iovcnt); - if (sh_iov == NULL) { - RETURN_ERROR(ENOMEM); - } - iovec_local2share(sh_iov, iov, iovcnt); + DEFINE_REQ_ARGS_STATIC(writev); + errno = 0; args->fd = fd; - args->iov = sh_iov; - args->iovcnt = iovcnt; - SYSCALL(FF_SO_WRITEV, args); + do { + sh_iov_static_fill_idx_local = 0; + sh_iov_static_fill_idx_share = 0; + ret_s = iovec_local2share_s(iov, iovcnt, sent); + DEBUG_LOG("iovec_local2share_s ret_s:%d, iov:%p, ipvcnt:%d, send:%lu, " + "sh_iov_static:%p, sh_iov_static_fill_idx_local:%d, sh_iov_static_fill_idx_share:%d\n", + ret_s, iov, iovcnt, sent, + sh_iov_static, sh_iov_static_fill_idx_local, sh_iov_static_fill_idx_share); + if (ret_s < 0) { + ERR_LOG("get_iovec_share failed, iov:%p, iovcnt:%d, sh_iov_static_fill_idx_local:%d," + " sh_iov_static_fill_idx_share:%d", + iov, iovcnt, sh_iov_static_fill_idx_local, + sh_iov_static_fill_idx_share); + return -1; + } - if (ret > 0) { - iovec_share2local(sh_iov, iov, iovcnt, ret, 0); + args->iov = sh_iov_static; + args->iovcnt = sh_iov_static_fill_idx_share; + + SYSCALL(FF_SO_WRITEV, args); + + /* + * This API can be igroned while use sh_iov_static_base[i] directly + * in _iovec_local2share_s. But don't do like that now + */ + iovec_share2local_s(); + + if (ret > 0) { + sent += ret; + } + + /* + * Don't try to send again in this case. + */ + DEBUG_LOG("iovec_local2share_s ret_s:%d, f-stack writev ret:%d, total sent:%lu\n", ret_s, ret, sent); + if (ret != ret_s) { + break; + } + } while (sh_iov_static_fill_idx_local < iovcnt); + sh_iov_static_fill_idx_share = 0; + + if (sent > 0) { + ret = sent; } - iovec_share_free(sh_iov, iovcnt); - RETURN(); + RETURN_NOFREE(); } int @@ -1255,14 +1502,17 @@ ff_hook_ioctl(int fd, unsigned long req, unsigned long data) CHECK_FD_OWNERSHIP(ioctl, (fd, req, data)); - DEFINE_REQ_ARGS(ioctl); - unsigned long *sh_data = NULL; + DEFINE_REQ_ARGS_STATIC(ioctl); + + static __thread unsigned long *sh_data = NULL; - sh_data = share_mem_alloc(sizeof(int)); if (sh_data == NULL) { - RETURN_ERROR(ENOMEM); + sh_data = share_mem_alloc(sizeof(int)); + if (sh_data == NULL) { + RETURN_ERROR_NOFREE(ENOMEM); + } } - *sh_data = *((int *)data); + *((int *)sh_data) = *((int *)data); args->fd = fd; args->com = req; @@ -1271,12 +1521,12 @@ ff_hook_ioctl(int fd, unsigned long req, unsigned long data) SYSCALL(FF_SO_IOCTL, args); if (ret == 0) { - *((int *)data) = *sh_data; + *((int *)data) = *((int *)sh_data); } - share_mem_free(sh_data); + //share_mem_free(sh_data); - RETURN(); + RETURN_NOFREE(); } int @@ -1284,7 +1534,7 @@ ff_hook_fcntl(int fd, int cmd, unsigned long data) { CHECK_FD_OWNERSHIP(fcntl, (fd, cmd, data)); - DEFINE_REQ_ARGS(fcntl); + DEFINE_REQ_ARGS_STATIC(fcntl); args->fd = fd; args->cmd = cmd; @@ -1292,7 +1542,7 @@ ff_hook_fcntl(int fd, int cmd, unsigned long data) SYSCALL(FF_SO_FCNTL, args); - RETURN(); + RETURN_NOFREE(); } /* @@ -1836,6 +2086,15 @@ thread_destructor(void *sc) if (shutdown_args) { share_mem_free(shutdown_args); } + if (getsockname_args) { + share_mem_free(getsockname_args); + } + if (getpeername_args) { + share_mem_free(getpeername_args); + } + if (setsockopt_args) { + share_mem_free(setsockopt_args); + } if (accept_args) { share_mem_free(accept_args); } @@ -1845,18 +2104,36 @@ thread_destructor(void *sc) if (recvfrom_args) { share_mem_free(recvfrom_args); } + if (recvmsg_args) { + share_mem_free(recvmsg_args); + } if (read_args) { share_mem_free(read_args); } + if (readv_args) { + share_mem_free(readv_args); + } if (sendto_args) { share_mem_free(sendto_args); } + if (sendmsg_args) { + share_mem_free(sendmsg_args); + } if (write_args) { share_mem_free(write_args); } + if (writev_args) { + share_mem_free(writev_args); + } if (close_args) { share_mem_free(close_args); } + if (ioctl_args) { + share_mem_free(ioctl_args); + } + if (fcntl_args) { + share_mem_free(fcntl_args); + } if (epoll_ctl_args) { share_mem_free(epoll_ctl_args); } @@ -1866,6 +2143,11 @@ thread_destructor(void *sc) if (kevent_args) { share_mem_free(kevent_args); } + + if (sh_iov_static) { + iovec_share2local_s(); + iovec_share_free(sh_iov_static, IOV_MAX); + } } void __attribute__((destructor)) From 3240dd0dad15edac84f71f16d115e73a068cc9bf Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 27 Apr 2023 17:24:32 +0800 Subject: [PATCH 31/43] Add FF_MULTI_SC macro definition to support some special application scenarios, let the child process worker inherit the specified sc. Like Nginx with reuseport. Parent process socket/bind/listen multi sockets, and use them in different child process worker. All child process workers must be forked by the same process, scilicet: support master fork child1, [child1 fork child2], chilid2 fork worker1/worker2/worker3... But not support master fork worker1, worker fork worker2, worker2 fork worker3... 2. modify some log. --- adapter/Makefile | 6 ++- adapter/ff_hook_syscall.c | 100 ++++++++++++++++++++++++++++++++----- adapter/ff_linux_syscall.c | 6 +-- adapter/ff_socket_ops.h | 5 +- 4 files changed, 98 insertions(+), 19 deletions(-) diff --git a/adapter/Makefile b/adapter/Makefile index 743b1de4e..01ec7d820 100644 --- a/adapter/Makefile +++ b/adapter/Makefile @@ -8,7 +8,7 @@ ifneq ($(shell pkg-config --exists libdpdk && echo 0),0) $(error "No installation of DPDK found, maybe you should export environment variable `PKG_CONFIG_PATH`") endif -DEBUG=-O0 -gdwarf-2 -g3 +#DEBUG=-O0 -gdwarf-2 -g3 # Per thread separate initialization dpdk lib and attach sc when needed, # such as listen same port in different threads, and socket can use in own thread. @@ -37,6 +37,10 @@ ifdef FF_KERNEL_EVENT CFLAGS+= -DFF_KERNEL_EVENT endif +ifdef FF_MULTI_SC + CFLAGS+= -DFF_MULTI_SC +endif + CFLAGS += -fPIC -Wall -Werror $(shell $(PKGCONF) --cflags libdpdk) INCLUDES= -I. -I${FF_PATH}/lib diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index d8828d45d..e035df397 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -162,6 +162,30 @@ static __thread int sh_iov_static_fill_idx_share = 0; static __FF_THREAD int inited = 0; static __FF_THREAD struct ff_so_context *sc; +/* + * For parent process socket/bind/listen multi sockets + * and use them in different child process, + * like Nginx with reuseport. + */ +#ifdef FF_MULTI_SC +typedef struct ff_multi_sc { + int worker_id; + //int fd; + struct ff_so_context *sc; +} ff_multi_sc_type; + +static ff_multi_sc_type scs[SOCKET_OPS_CONTEXT_MAX_NUM]; + +/* + * For child worker process, + * All workers must be forked by the same process, scilicet + * support master fork child1, [child1 fork child2], chilid2 fork worker1/worker2/worker3... + * But not support master fork worker1, worker fork worker2, worker2 fork worker3... + */ +#define CURRENT_WORKER_ID_DEFAULT 0 +static int current_worker_id = CURRENT_WORKER_ID_DEFAULT; +#endif + static pthread_key_t key; #ifdef FF_KERNEL_EVENT @@ -182,7 +206,6 @@ static uint64_t initial_lcore_id = INITIAL_LCORE_ID_DEFAULT; #define WORKER_ID_DEFAULT 0 #define FF_PROC_ID_STR "FF_PROC_ID" static int worker_id = WORKER_ID_DEFAULT; -static __thread int thread_id = 0; rte_spinlock_t worker_id_lock; /* The num of F-Stack process instance, default 1 */ @@ -241,7 +264,7 @@ fstack_territory(int domain, int type, int protocol) int ff_hook_socket(int domain, int type, int protocol) { - DEBUG_LOG("ff_hook_socket, domain:%d, type:%d, protocol:%d\n", domain, type, protocol); + ERR_LOG("ff_hook_socket, domain:%d, type:%d, protocol:%d\n", domain, type, protocol); if (unlikely(fstack_territory(domain, type, protocol) == 0)) { return ff_linux_socket(domain, type, protocol); } @@ -251,9 +274,19 @@ ff_hook_socket(int domain, int type, int protocol) return ff_linux_socket(domain, type, protocol); } - if (unlikely(inited == 0 && ff_adapter_init() < 0)) { - return ff_linux_socket(domain, type, protocol); + if (unlikely(inited == 0)) { + if (ff_adapter_init() < 0) { + return ff_linux_socket(domain, type, protocol); + } } +#ifdef FF_MULTI_SC + else { + if (ff_adapter_init() < 0) { + ERR_LOG("FF_MUTLI_SC ff_adapter_init failed\n"); + return -1; + } + } +#endif type &= ~SOCK_FSTACK; @@ -269,7 +302,7 @@ ff_hook_socket(int domain, int type, int protocol) ret = convert_fstack_fd(ret); } - DEBUG_LOG("ff_hook_socket return fd:%d\n", ret); + ERR_LOG("ff_hook_socket return fd:%d\n", ret); RETURN(); } @@ -278,6 +311,8 @@ int ff_hook_bind(int fd, const struct sockaddr *addr, socklen_t addrlen) { + ERR_LOG("ff_hook_bind, fd:%d, addr:%p, addrlen:%d\n", fd, addr, addrlen); + if (addr == NULL) { errno = EINVAL; return -1; @@ -307,6 +342,8 @@ ff_hook_bind(int fd, const struct sockaddr *addr, int ff_hook_listen(int fd, int backlog) { + ERR_LOG("ff_hook_listen, fd:%d, backlog:%d\n", fd, backlog); + CHECK_FD_OWNERSHIP(listen, (fd, backlog)); DEFINE_REQ_ARGS(listen); @@ -1557,7 +1594,7 @@ int ff_hook_epoll_create(int fdsize) { - DEBUG_LOG("ff_hook_epoll_create, fdsize:%d\n", fdsize); + ERR_LOG("ff_hook_epoll_create, fdsize:%d\n", fdsize); if (inited == 0 || ((fdsize & SOCK_KERNEL) && !(fdsize & SOCK_FSTACK))/* || (fdsize >= 1 && fdsize <= 16)*/) { fdsize &= ~SOCK_KERNEL; return ff_linux_epoll_create(fdsize); @@ -1580,7 +1617,7 @@ ff_hook_epoll_create(int fdsize) ret = convert_fstack_fd(ret); } - DEBUG_LOG("ff_hook_epoll_create return fd:%d\n", ret); + ERR_LOG("ff_hook_epoll_create return fd:%d\n", ret); RETURN(); } @@ -1842,7 +1879,12 @@ ff_hook_fork(void) { pid_t pid; - DEBUG_LOG("ff_hook_fork\n"); + ERR_LOG("ff_hook_fork\n"); +#ifdef FF_MULTI_SC + /* Let the child process inherit the specified sc */ + sc = scs[current_worker_id].sc; +#endif + if (sc) { rte_spinlock_lock(&sc->lock); } @@ -1853,6 +1895,19 @@ ff_hook_fork(void) /* Parent process set refcount. */ if (pid > 0) { sc->refcount++; + ERR_LOG("parent process, chilid pid:%d, sc:%p, sc->refcount:%d\n", + pid, sc, sc->refcount); +#ifdef FF_MULTI_SC + current_worker_id++; + ERR_LOG("parent process, current_worker_id++:%d\n", current_worker_id); +#endif + } + else if (pid == 0) { + ERR_LOG("chilid process, sc:%p, sc->refcount:%d\n", + sc, sc->refcount); +#ifdef FF_MULTI_SC + ERR_LOG("chilid process, current_worker_id:%d\n", current_worker_id); +#endif } /* Parent process unlock sc, fork success of failed. */ @@ -2156,9 +2211,21 @@ ff_adapter_exit() pthread_key_delete(key); #ifndef FF_THREAD_SOCKET - ERR_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc); - ff_detach_so_context(sc); - sc = NULL; + +#ifdef FF_MULTI_SC + if (current_worker_id == worker_id) { + int i; + for (i = 0; i < worker_id; i ++) { + ERR_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), scs[i].sc); + ff_detach_so_context(scs[i].sc); + } + } else +#endif + { + ERR_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc); + ff_detach_so_context(sc); + sc = NULL; + } #endif } @@ -2169,9 +2236,13 @@ ff_adapter_init() { int ret; + ERR_LOG("inited:%d, proc_inited:%d\n", inited, proc_inited); + +#ifndef FF_MULTI_SC if (inited) { return 0; } +#endif if (proc_inited == 0) { /* May conflict */ @@ -2240,7 +2311,7 @@ ff_adapter_init() } /* - * Get environment variable FF_PROC_ID to set nb_procs. + * Get environment variable FF_PROC_ID to set worker_id. */ char *ff_worker_id = getenv(FF_PROC_ID_STR); if (ff_worker_id != NULL) { @@ -2298,7 +2369,10 @@ ff_adapter_init() pthread_setspecific(key, sc); - thread_id = worker_id; +#ifdef FF_MULTI_SC + scs[worker_id].worker_id = worker_id; + scs[worker_id].sc = sc; +#endif worker_id++; inited = 1; diff --git a/adapter/ff_linux_syscall.c b/adapter/ff_linux_syscall.c index 371332346..5c0550a3c 100644 --- a/adapter/ff_linux_syscall.c +++ b/adapter/ff_linux_syscall.c @@ -76,7 +76,7 @@ linux_syscall_init() int ff_linux_socket(int domain, int type, int protocol) { - DEBUG_LOG("ff_linux_socket, domain:%d, type:%d, protocol:%d\n", domain, type, protocol); + ERR_LOG("ff_linux_socket, domain:%d, type:%d, protocol:%d\n", domain, type, protocol); SYSCALL(socket, (domain, type, protocol)); } @@ -226,14 +226,14 @@ int ff_linux_fcntl(int s, int cmd, unsigned long data) int ff_linux_epoll_create(int size) { - DEBUG_LOG("ff_linux_epoll_create, fdsize:%d\n", size); + ERR_LOG("ff_linux_epoll_create, fdsize:%d\n", size); SYSCALL(epoll_create, (size)); } int ff_linux_epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { - DEBUG_LOG("ff_linux_epoll_ctl, epfd:%d, op:%d, fd:%d\n", epfd, op, fd); + ERR_LOG("ff_linux_epoll_ctl, epfd:%d, op:%d, fd:%d\n", epfd, op, fd); SYSCALL(epoll_ctl, (epfd, op, fd, event)); } diff --git a/adapter/ff_socket_ops.h b/adapter/ff_socket_ops.h index 436a12d42..b09186b4f 100644 --- a/adapter/ff_socket_ops.h +++ b/adapter/ff_socket_ops.h @@ -1,6 +1,7 @@ #ifndef _FF_SOCKET_OPS_H_ #define _FF_SOCKET_OPS_H_ +#include #include #include @@ -19,8 +20,8 @@ #endif #define ERR_LOG(fmt, ...) do { \ - printf("file:%s, line:%u, fun:%s, thread self tid:%ld, "fmt, \ - __FILE__, __LINE__, __func__, pthread_self(), ##__VA_ARGS__); \ + printf("file:%s, line:%u, fun:%s, pid:%d, "fmt, \ + __FILE__, __LINE__, __func__, getpid(), ##__VA_ARGS__); \ } while (0) #ifdef NDEBUG From 8ae79cd43890850a6042c72d7e039b020d5586a3 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 27 Apr 2023 18:34:44 +0800 Subject: [PATCH 32/43] support mutli ff_so_zones. --- adapter/ff_hook_syscall.c | 12 +++++++----- adapter/ff_so_zone.c | 20 ++++++++++++++++---- adapter/ff_socket_ops.h | 3 +++ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index e035df397..02c2a8552 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -1881,8 +1881,9 @@ ff_hook_fork(void) ERR_LOG("ff_hook_fork\n"); #ifdef FF_MULTI_SC - /* Let the child process inherit the specified sc */ + /* Let the child process inherit the specified sc and ff_so_zone*/ sc = scs[current_worker_id].sc; + ff_so_zone = ff_so_zones[current_worker_id]; #endif if (sc) { @@ -1895,16 +1896,16 @@ ff_hook_fork(void) /* Parent process set refcount. */ if (pid > 0) { sc->refcount++; - ERR_LOG("parent process, chilid pid:%d, sc:%p, sc->refcount:%d\n", - pid, sc, sc->refcount); + ERR_LOG("parent process, chilid pid:%d, sc:%p, sc->refcount:%d, ff_so_zone:%p\n", + pid, sc, sc->refcount, ff_so_zone); #ifdef FF_MULTI_SC current_worker_id++; ERR_LOG("parent process, current_worker_id++:%d\n", current_worker_id); #endif } else if (pid == 0) { - ERR_LOG("chilid process, sc:%p, sc->refcount:%d\n", - sc, sc->refcount); + ERR_LOG("chilid process, sc:%p, sc->refcount:%d, ff_so_zone:%p\n", + sc, sc->refcount, ff_so_zone); #ifdef FF_MULTI_SC ERR_LOG("chilid process, current_worker_id:%d\n", current_worker_id); #endif @@ -2217,6 +2218,7 @@ ff_adapter_exit() int i; for (i = 0; i < worker_id; i ++) { ERR_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), scs[i].sc); + ff_so_zone = ff_so_zones[i]; ff_detach_so_context(scs[i].sc); } } else diff --git a/adapter/ff_so_zone.c b/adapter/ff_so_zone.c index 8e694ef74..90ddabfc7 100644 --- a/adapter/ff_so_zone.c +++ b/adapter/ff_so_zone.c @@ -14,6 +14,9 @@ static uint16_t ff_max_so_context = SOCKET_OPS_CONTEXT_MAX_NUM; __FF_THREAD struct ff_socket_ops_zone *ff_so_zone; +#ifdef FF_MULTI_SC +struct ff_socket_ops_zone *ff_so_zones[SOCKET_OPS_CONTEXT_MAX_NUM] = {NULL}; +#endif static inline int is_power_of_2(uint64_t n) @@ -124,18 +127,22 @@ ff_create_so_memzone() } struct ff_so_context * -ff_attach_so_context(int proc_id) +ff_attach_so_context(int idx) { struct ff_so_context *sc = NULL; uint16_t i; - DEBUG_LOG("proc_id:%d, ff_so_zone:%p\n", proc_id, ff_so_zone); +#ifdef FF_MULTI_SC + ff_so_zone = ff_so_zones[idx]; +#endif + + DEBUG_LOG("proc_id:%d, ff_so_zone:%p\n", idx, ff_so_zone); if (ff_so_zone == NULL) { const struct rte_memzone *mz; char zn[64]; - snprintf(zn, sizeof(zn), SOCKET_OPS_ZONE_NAME, proc_id); + snprintf(zn, sizeof(zn), SOCKET_OPS_ZONE_NAME, idx); ERR_LOG("To lookup memzone:%s\n", zn); mz = rte_memzone_lookup(zn); @@ -145,6 +152,11 @@ ff_attach_so_context(int proc_id) } ff_so_zone = mz->addr; + +#ifdef FF_MULTI_SC + ff_so_zones[idx] = ff_so_zone; + ERR_LOG("FF_MULTI_SC f_so_zones[%d]:%p\n", idx, ff_so_zones[idx]); +#endif } rte_spinlock_lock(&ff_so_zone->lock); @@ -172,7 +184,7 @@ ff_attach_so_context(int proc_id) if (unlikely(i == ff_so_zone->count)) { ERR_LOG("Attach memzone failed: instance %d no free context," " fetel error of so status, all sc inuse, count:%d, free:%d\n", - proc_id, ff_so_zone->count, ff_so_zone->free); + idx, ff_so_zone->count, ff_so_zone->free); sc = NULL; } diff --git a/adapter/ff_socket_ops.h b/adapter/ff_socket_ops.h index b09186b4f..10b20014f 100644 --- a/adapter/ff_socket_ops.h +++ b/adapter/ff_socket_ops.h @@ -113,6 +113,9 @@ struct ff_so_context { } __attribute__((aligned(RTE_CACHE_LINE_SIZE))); extern __FF_THREAD struct ff_socket_ops_zone *ff_so_zone; +#ifdef FF_MULTI_SC +extern struct ff_socket_ops_zone *ff_so_zones[SOCKET_OPS_CONTEXT_MAX_NUM]; +#endif /* For primary process */ int ff_set_max_so_context(uint16_t count); From 85173ced832404fa214869abdf4022d314ef4896 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 27 Apr 2023 20:41:54 +0800 Subject: [PATCH 33/43] try reduce the number of call ff_linux_epoll_wait(). --- adapter/ff_hook_syscall.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index 02c2a8552..ef52d569d 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -1788,13 +1788,17 @@ RETRY: DEBUG_LOG("call ff_linux_epoll_wait at the same time, epfd:%d, fstack_kernel_fd_map[epfd]:%d, kernel_maxevents:%d\n", fd, fstack_kernel_fd_map[fd], kernel_maxevents); if (likely(fstack_kernel_fd_map[fd] > 0)) { - kernel_ret = ff_linux_epoll_wait(fstack_kernel_fd_map[fd], events, kernel_maxevents, 0); - DEBUG_LOG("ff_linux_epoll_wait kernel_ret:%d, errno:%d\n", ret, errno); - if (kernel_ret < 0) { - kernel_ret = 0; - } else if (kernel_ret > 0) { - events += kernel_ret; + static uint64_t count = 0; + if (unlikely(count & 0xff == 0)) { + kernel_ret = ff_linux_epoll_wait(fstack_kernel_fd_map[fd], events, kernel_maxevents, 0); + DEBUG_LOG("ff_linux_epoll_wait kernel_ret:%d, errno:%d\n", ret, errno); + if (kernel_ret < 0) { + kernel_ret = 0; + } else if (kernel_ret > 0) { + events += kernel_ret; + } } + count++; } #endif From 1b25a84cc6d055b1cdab3aba121a8c63d26a4b5b Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 27 Apr 2023 20:47:43 +0800 Subject: [PATCH 34/43] Fix waring. --- adapter/ff_hook_syscall.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index ef52d569d..af7d23226 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -1789,7 +1789,7 @@ RETRY: fd, fstack_kernel_fd_map[fd], kernel_maxevents); if (likely(fstack_kernel_fd_map[fd] > 0)) { static uint64_t count = 0; - if (unlikely(count & 0xff == 0)) { + if (unlikely((count & 0xff) == 0)) { kernel_ret = ff_linux_epoll_wait(fstack_kernel_fd_map[fd], events, kernel_maxevents, 0); DEBUG_LOG("ff_linux_epoll_wait kernel_ret:%d, errno:%d\n", ret, errno); if (kernel_ret < 0) { From 3d3266071cbf71c14d7ebb4f4425608b1ae29604 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Fri, 28 Apr 2023 12:26:11 +0800 Subject: [PATCH 35/43] Use ff_global_cfg.dpdk.pkt_tx_delay to delay ff_handle_each_context to improve the performance of nginx. --- adapter/ff_socket_ops.c | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/adapter/ff_socket_ops.c b/adapter/ff_socket_ops.c index 7c8fa3fb5..d02ec510f 100644 --- a/adapter/ff_socket_ops.c +++ b/adapter/ff_socket_ops.c @@ -5,6 +5,7 @@ #include "ff_sysproto.h" #include "ff_api.h" #include "ff_epoll.h" +#include "ff_config.h" #undef FF_SYSCALL_DECL #define FF_SYSCALL_DECL(ret, fn, none) \ @@ -24,7 +25,7 @@ static int sem_flag = 0; */ #define EVENT_LOOP_TIMES 32 static int ff_event_loop_nb = 0; -static int ff_next_event_flag = 0; +//static int ff_next_event_flag = 0; struct ff_bound_info { int fd; @@ -452,7 +453,7 @@ ff_handle_socket_ops(struct ff_so_context *sc) DEBUG_LOG("ff_handle_socket_ops error:%d, ops:%d, result:%d\n", errno, sc->ops, sc->result); if (sc->ops == FF_SO_EPOLL_WAIT || sc->ops == FF_SO_KEVENT) { - DEBUG_LOG("ff_event_loop_nb:%d, ff_next_event_flag:%d\n", + /*DEBUG_LOG("ff_event_loop_nb:%d, ff_next_event_flag:%d\n", ff_event_loop_nb, ff_next_event_flag); if (ff_event_loop_nb > 0) { ff_next_event_flag = 1; @@ -464,7 +465,7 @@ ff_handle_socket_ops(struct ff_so_context *sc) ff_event_loop_nb = (sc->result * EVENT_LOOP_TIMES); } else { ff_event_loop_nb = 0; - } + }*/ if (sem_flag == 1) { sc->status = FF_SC_REP; @@ -484,9 +485,18 @@ ff_handle_each_context() { uint16_t i, nb_handled, tmp; static uint64_t loop_count = 0; + static uint64_t cur_tsc, diff_tsc, drain_tsc = 0; + + if (unlikely(drain_tsc == 0 && ff_global_cfg.dpdk.pkt_tx_delay)) { + drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * ff_global_cfg.dpdk.pkt_tx_delay; + ERR_LOG("ff_global_cfg.dpdk.handle_sc_delay%d, drain_tsc:%lu\n", + ff_global_cfg.dpdk.pkt_tx_delay, drain_tsc); + } ff_event_loop_nb = 0; + cur_tsc = rte_rdtsc(); + rte_spinlock_lock(&ff_so_zone->lock); assert(ff_so_zone->count >= ff_so_zone->free); @@ -520,7 +530,12 @@ ff_handle_each_context() } } - if (--ff_event_loop_nb <= 0 || ff_next_event_flag == 1) { + /*if (--ff_event_loop_nb <= 0 || ff_next_event_flag == 1) { + break; + }*/ + diff_tsc = rte_rdtsc() - cur_tsc; + DEBUG_LOG("cur_tsc:%lu, diff_tsc:%lu, drain_tsc:%lu\n", cur_tsc, diff_tsc, drain_tsc); + if (diff_tsc >= drain_tsc) { break; } @@ -530,5 +545,9 @@ ff_handle_each_context() rte_spinlock_unlock(&ff_so_zone->lock); loop_count++; + + DEBUG_LOG("loop_count:%lu, nb:%d, all_nb:%d\n", + loop_count, nb_handled, tmp/*, ff_event_loop_nb, ff_next_event_flag*/); + //, ff_event_loop_nb:%d, ff_next_event_flag:%d } From edeb90e0b568f63105d21b1eda88f3e57ce7938d Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Fri, 28 Apr 2023 14:48:18 +0800 Subject: [PATCH 36/43] fix an issue that close sockfd while Nginx exiting. --- adapter/ff_hook_syscall.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index af7d23226..d32945b8d 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -170,7 +170,7 @@ static __FF_THREAD struct ff_so_context *sc; #ifdef FF_MULTI_SC typedef struct ff_multi_sc { int worker_id; - //int fd; + int fd; struct ff_so_context *sc; } ff_multi_sc_type; @@ -298,6 +298,10 @@ ff_hook_socket(int domain, int type, int protocol) SYSCALL(FF_SO_SOCKET, args); +#ifdef FF_MULTI_SC + scs[worker_id - 1].fd = ret; +#endif + if (ret >= 0) { ret = convert_fstack_fd(ret); } @@ -1515,6 +1519,25 @@ ff_hook_close(int fd) DEFINE_REQ_ARGS_STATIC(close); +#ifdef FF_MULTI_SC + /* + * Hear don't care if the fd belong to this worker sc, + * just scs[i].fd == fd, to close it + * until the loop close all fd. + */ + if (unlikely(current_worker_id == worker_id)) { + int i; + for (i = 0; i < worker_id; i++) { + if (scs[i].fd == fd) { + ERR_LOG("worker_id:%d, fd:%d, sc:%p, sc->fd:%d, sc->worker_id:%d\n", + i, fd, scs[i].sc, scs[i].fd, scs[i].worker_id); + sc = scs[i].sc; + scs[i].fd = -1; + break; + } + } + } +#endif args->fd = fd; SYSCALL(FF_SO_CLOSE, args); @@ -2220,7 +2243,7 @@ ff_adapter_exit() #ifdef FF_MULTI_SC if (current_worker_id == worker_id) { int i; - for (i = 0; i < worker_id; i ++) { + for (i = 0; i < worker_id; i++) { ERR_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), scs[i].sc); ff_so_zone = ff_so_zones[i]; ff_detach_so_context(scs[i].sc); @@ -2377,6 +2400,7 @@ ff_adapter_init() #ifdef FF_MULTI_SC scs[worker_id].worker_id = worker_id; + scs[worker_id].fd = -1; scs[worker_id].sc = sc; #endif worker_id++; From 94d26c8a13544987afbf2ef9431fb10b47be3223 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 3 May 2023 12:02:19 +0800 Subject: [PATCH 37/43] modify IOV_MAX to 16. --- adapter/ff_hook_syscall.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index d32945b8d..fbed0d17a 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -81,7 +81,7 @@ static __thread struct ff_epoll_ctl_args *epoll_ctl_args = NULL; static __thread struct ff_epoll_wait_args *epoll_wait_args = NULL; static __thread struct ff_kevent_args *kevent_args = NULL; -#define IOV_MAX 64 +#define IOV_MAX 16 #define IOV_LEN_MAX 2048 static __thread struct iovec *sh_iov_static = NULL; From b64f360ebcda945d819dbb85705ac2b60b402267 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 3 May 2023 16:59:40 +0800 Subject: [PATCH 38/43] 1. Add main_stack_epoll_kernel.c for helloworld_stack_epoll_kernel that support f-stack and kernel stack at the same time. 2. move all main_stack*.c to adapter directory. --- adapter/Makefile | 12 +- adapter/ff_hook_syscall.c | 2 +- adapter/ff_linux_syscall.c | 2 + {example => adapter}/main_stack.c | 9 +- {example => adapter}/main_stack_epoll.c | 5 +- adapter/main_stack_epoll_kernel.c | 255 ++++++++++++++++++ .../main_stack_epoll_thread_socket.c | 24 +- .../main_stack_thread_socket.c | 4 +- example/Makefile | 4 - 9 files changed, 294 insertions(+), 23 deletions(-) rename {example => adapter}/main_stack.c (95%) rename {example => adapter}/main_stack_epoll.c (97%) create mode 100644 adapter/main_stack_epoll_kernel.c rename {example => adapter}/main_stack_epoll_thread_socket.c (91%) rename {example => adapter}/main_stack_thread_socket.c (97%) diff --git a/adapter/Makefile b/adapter/Makefile index 01ec7d820..551462ca9 100644 --- a/adapter/Makefile +++ b/adapter/Makefile @@ -45,10 +45,11 @@ CFLAGS += -fPIC -Wall -Werror $(shell $(PKGCONF) --cflags libdpdk) INCLUDES= -I. -I${FF_PATH}/lib +LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -pthread -lnuma FF_LIBS= -L${FF_PATH}/lib -Wl,--whole-archive,-lfstack,--no-whole-archive DPDK_LIBS+= $(shell $(PKGCONF) --static --libs libdpdk) -DPDK_LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -pthread -lnuma +DPDK_LIBS+= ${LIBS} #DPDK_CFLAGS= -Wall -Werror -include ${FF_DPDK}/include/rte_config.h #DPDK_CFLAGS+= -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 @@ -76,7 +77,7 @@ FSTACK_OBJS= $(patsubst %.c,%.o,${FSTACK_SRCS}) FF_SYSCALL_OBJS= $(patsubst %.c,%.o,${FF_SYSCALL_SRCS}) #TARGET= fstack libff_syscall.a -TARGET= fstack libff_syscall.so +TARGET= fstack libff_syscall.so example all: ${TARGET} @@ -88,6 +89,13 @@ libff_syscall.so: ${FF_SYSCALL_OBJS} #ar -rcs $@ $^ ${FF_SYSCALL_LIBS} ${CC} -fPIC -shared -o $@ $^ ${FF_SYSCALL_LIBS} ${DPDK_LIBS} +example: + cc ${CFLAGS} -I ${FF_PATH}/adapter/syscall -L ${FF_PATH}/adapter/syscall -lff_syscall -o helloworld_stack main_stack.c ${LIBS} + cc ${CFLAGS} -I ${FF_PATH}/adapter/syscall -L ${FF_PATH}/adapter/syscall -lff_syscall -o helloworld_stack_thread_socket main_stack_thread_socket.c ${LIBS} + cc ${CFLAGS} -I ${FF_PATH}/adapter/syscall -o helloworld_stack_epoll main_stack_epoll.c ${LIBS} + cc ${CFLAGS} -I ${FF_PATH}/adapter/syscall -o helloworld_stack_epoll_thread_socket main_stack_epoll_thread_socket.c ${LIBS} + cc ${CFLAGS} -I ${FF_PATH}/adapter/syscall -o helloworld_stack_epoll_kernel main_stack_epoll_kernel.c ${LIBS} + ${FSTACK_OBJS}: %.o: %.c ${CC} -c $(CFLAGS) ${PROF} $< diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c index fbed0d17a..811a7a74c 100644 --- a/adapter/ff_hook_syscall.c +++ b/adapter/ff_hook_syscall.c @@ -1814,7 +1814,7 @@ RETRY: static uint64_t count = 0; if (unlikely((count & 0xff) == 0)) { kernel_ret = ff_linux_epoll_wait(fstack_kernel_fd_map[fd], events, kernel_maxevents, 0); - DEBUG_LOG("ff_linux_epoll_wait kernel_ret:%d, errno:%d\n", ret, errno); + DEBUG_LOG("ff_linux_epoll_wait count:%lu, kernel_ret:%d, errno:%d\n", count, ret, errno); if (kernel_ret < 0) { kernel_ret = 0; } else if (kernel_ret > 0) { diff --git a/adapter/ff_linux_syscall.c b/adapter/ff_linux_syscall.c index 5c0550a3c..adfa2e60c 100644 --- a/adapter/ff_linux_syscall.c +++ b/adapter/ff_linux_syscall.c @@ -84,12 +84,14 @@ int ff_linux_bind(int s, const struct sockaddr *addr, socklen_t addrlen) { + ERR_LOG("ff_linux_bind, fd:%d, addr:%p, addrlen:%u\n", s, addr, addrlen); SYSCALL(bind, (s, addr, addrlen)); } int ff_linux_listen(int s, int backlog) { + ERR_LOG("ff_linux_listen, fd:%d, backlog:%d\n", s, backlog); SYSCALL(listen, (s, backlog)); } diff --git a/example/main_stack.c b/adapter/main_stack.c similarity index 95% rename from example/main_stack.c rename to adapter/main_stack.c index 936f488ae..86da3b898 100644 --- a/example/main_stack.c +++ b/adapter/main_stack.c @@ -32,8 +32,7 @@ struct kevent events[MAX_EVENTS]; int kq; int sockfd; -/* 100 ms */ -struct timespec timeout = {0, 100000000}; +struct timespec timeout = {0, 100000}; static int exit_flag = 0; @@ -133,12 +132,12 @@ void *loop(void *arg) ssize_t readlen = read(clientfd, buf, sizeof(buf)); ssize_t writelen = write(clientfd, html, sizeof(html) - 1); if (writelen < 0){ - printf("ff_write failed:%d, %s\n", errno, - strerror(errno)); + printf("ff_write failed, readlen:%lu, writelen:%lu, :%d, %s\n", + readlen, writelen, errno, strerror(errno)); close(clientfd); } } else { - printf("unknown event: %d:%8.8X\n", i, event.flags); + printf("unknown event: %8.8X\n", event.flags); } } } diff --git a/example/main_stack_epoll.c b/adapter/main_stack_epoll.c similarity index 97% rename from example/main_stack_epoll.c rename to adapter/main_stack_epoll.c index a51bbeac6..a1ae19557 100644 --- a/example/main_stack_epoll.c +++ b/adapter/main_stack_epoll.c @@ -83,7 +83,7 @@ void *loop(void *arg) break; } //usleep(100); - //sleep(1); + sleep(1); } //printf("get nevents:%d\n", nevents); @@ -92,6 +92,7 @@ void *loop(void *arg) if (events[i].data.fd == sockfd) { while (1) { int nclientfd = accept(sockfd, NULL, NULL); + printf("accept sockfd:%d, nclientfd:%d, errono:%d/%s\n", sockfd, nclientfd, errno, strerror(errno)); if (nclientfd < 0) { break; } @@ -167,7 +168,7 @@ int main(int argc, char * argv[]) return -1; } - epfd = epoll_create(0); + epfd = epoll_create(512); printf("epfd:%d\n", epfd); if (epfd <= 0) { printf("ff_epoll_create failed, errno:%d, %s\n", diff --git a/adapter/main_stack_epoll_kernel.c b/adapter/main_stack_epoll_kernel.c new file mode 100644 index 000000000..0557ffe79 --- /dev/null +++ b/adapter/main_stack_epoll_kernel.c @@ -0,0 +1,255 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SOCK_FSTACK 0x01000000 +#define SOCK_KERNEL 0x02000000 + +#define MAX_WORKERS 128 +pthread_t hworker[MAX_WORKERS]; + +#define MAX_EVENTS 512 +struct epoll_event ev; +struct epoll_event events[MAX_EVENTS]; +int epfd; +int sockfd, sockfd_kernel; + +static int exit_flag = 0; + +char html[] = +"HTTP/1.1 200 OK\r\n" +"Server: F-Stack\r\n" +"Date: Sat, 25 Feb 2017 09:26:33 GMT\r\n" +"Content-Type: text/html\r\n" +"Content-Length: 438\r\n" +"Last-Modified: Tue, 21 Feb 2017 09:44:03 GMT\r\n" +"Connection: keep-alive\r\n" +"Accept-Ranges: bytes\r\n" +"\r\n" +"\r\n" +"\r\n" +"\r\n" +"Welcome to F-Stack!\r\n" +"\r\n" +"\r\n" +"\r\n" +"

Welcome to F-Stack!

\r\n" +"\r\n" +"

For online documentation and support please refer to\r\n" +"F-Stack.org.
\r\n" +"\r\n" +"

Thank you for using F-Stack.

\r\n" +"\r\n" +""; + +void sig_term(int sig) +{ + printf("we caught signal %d, to exit helloworld\n", sig); + exit_flag = 1; + //alarm_event_sem(); + return; +} + +void *loop(void *arg) +{ + /* Wait for events to happen */ + while (!exit_flag) { + /* + * If not call alarm_event_sem, and epoll_wait timeout is 0, + * it can't exit normal, so timeout can't set to 0. + */ + int nevents = epoll_wait(epfd, events, MAX_EVENTS, -1); + int i; + + if (nevents <= 0) { + if (nevents) { + printf("hello world epoll wait ret %d, errno:%d, %s\n", + nevents, errno, strerror(errno)); + break; + } + usleep(100); + //sleep(1); + } + //printf("get nevents:%d\n", nevents); + + for (i = 0; i < nevents; ++i) { + /* Handle new connect */ + if (events[i].data.fd == sockfd || events[i].data.fd == sockfd_kernel) { + while (1) { + int nclientfd = accept(events[i].data.fd, NULL, NULL); + printf("accept sockfd(_kernel):%d, nclientfd:%d, errono:%d/%s\n", events[i].data.fd, nclientfd, errno, strerror(errno)); + if (nclientfd < 0) { + break; + } + + /* Add to event list */ + ev.data.fd = nclientfd; + ev.events = EPOLLIN; + if (epoll_ctl(epfd, EPOLL_CTL_ADD, nclientfd, &ev) != 0) { + printf("ff_epoll_ctl failed:%d, %s\n", + errno, strerror(errno)); + close(nclientfd); + break; + } + if (events[i].data.fd == sockfd_kernel) { + break; + } + } + } else { + if (events[i].events & EPOLLERR ) { + /* Simply close socket */ + epoll_ctl(epfd, EPOLL_CTL_DEL, events[i].data.fd, NULL); + close(events[i].data.fd); + } else if (events[i].events & EPOLLIN) { + char buf[256]; + size_t readlen = read( events[i].data.fd, buf, sizeof(buf)); + if(readlen > 0) { + write( events[i].data.fd, html, sizeof(html) - 1); + } else { + epoll_ctl(epfd, EPOLL_CTL_DEL, events[i].data.fd, NULL); + close(events[i].data.fd); + } + } else { + printf("unknown event: %d:%8.8X\n", i, events[i].events); + } + } + } + } + + return NULL; +} + +int main(int argc, char * argv[]) +{ + int i, worker_num = 1; + + signal(SIGINT, sig_term); + signal(SIGTERM, sig_term); + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + printf("sockfd:%d\n", sockfd); + if (sockfd < 0) { + printf("ff_socket failed\n"); + return -1; + } + + int on = 1; + ioctl(sockfd, FIONBIO, &on); + + struct sockaddr_in my_addr; + bzero(&my_addr, sizeof(my_addr)); + my_addr.sin_family = AF_INET; + my_addr.sin_port = htons(80); + my_addr.sin_addr.s_addr = htonl(INADDR_ANY); + + int ret = bind(sockfd, (const struct sockaddr *)&my_addr, sizeof(my_addr)); + if (ret < 0) { + printf("ff_bind failed\n"); + close(sockfd); + return -1; + } + + ret = listen(sockfd, MAX_EVENTS); + if (ret < 0) { + printf("ff_listen failed\n"); + close(sockfd); + return -1; + } + + sockfd_kernel = socket(AF_INET, SOCK_STREAM | SOCK_KERNEL, 0); + printf("sockfd_kernel:%d\n", sockfd_kernel); + if (sockfd_kernel < 0) { + printf("ff_socket failed\n"); + return -1; + } + + bzero(&my_addr, sizeof(my_addr)); + my_addr.sin_family = AF_INET; + my_addr.sin_port = htons(80); + my_addr.sin_addr.s_addr = htonl(INADDR_ANY); + + ret = bind(sockfd_kernel, (const struct sockaddr *)&my_addr, sizeof(my_addr)); + if (ret < 0) { + printf("ff_bind failed\n"); + close(sockfd); + close(sockfd_kernel); + return -1; + } + + ret = listen(sockfd_kernel, MAX_EVENTS); + if (ret < 0) { + printf("ff_listen failed\n"); + close(sockfd); + close(sockfd_kernel); + return -1; + } + + epfd = epoll_create(512); + printf("epfd:%d\n", epfd); + if (epfd <= 0) { + printf("ff_epoll_create failed, errno:%d, %s\n", + errno, strerror(errno)); + close(sockfd); + return -1; + } + + ev.data.fd = sockfd; + ev.events = EPOLLIN; + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev); + if (ret < 0) { + printf("ff_listen failed\n"); + close(epfd); + close(sockfd); + close(sockfd_kernel); + return -1; + } + ev.data.fd = sockfd_kernel; + ev.events = EPOLLIN; + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd_kernel, &ev); + if (ret < 0) { + printf("ff_listen failed\n"); + close(epfd); + close(sockfd); + close(sockfd_kernel); + return -1; + } + + for (i = 0; i < worker_num; i++) { + if(pthread_create(&hworker[i], NULL, loop, (void *)&i) < 0) { + printf("create loop thread failed., errno:%d/%s\n", + errno, strerror(errno)); + close(epfd); + close(sockfd); + close(sockfd_kernel); + return -1; + } + } + + for (i = 0; i < worker_num; i++) { + pthread_join(hworker[i], NULL); + } + + close(epfd); + close(sockfd); + close(sockfd_kernel); + + return 0; +} diff --git a/example/main_stack_epoll_thread_socket.c b/adapter/main_stack_epoll_thread_socket.c similarity index 91% rename from example/main_stack_epoll_thread_socket.c rename to adapter/main_stack_epoll_thread_socket.c index e2f9bf030..f4d6317c0 100644 --- a/example/main_stack_epoll_thread_socket.c +++ b/adapter/main_stack_epoll_thread_socket.c @@ -63,6 +63,8 @@ void sig_term(int sig) return; } +#define SOCK_FSTACK 0x01000000 +#define SOCK_KERNEL 0x02000000 void *loop(void *arg) { struct epoll_event ev; @@ -73,8 +75,8 @@ void *loop(void *arg) thread_id = *(int *)arg; printf("start thread %d\n", thread_id); - - sockfd = socket(AF_INET, SOCK_STREAM, 0); + + sockfd = socket(AF_INET, SOCK_STREAM | SOCK_FSTACK, 0); printf("thread %d, sockfd:%d\n", thread_id, sockfd); if (sockfd < 0) { printf("thread %d, ff_socket failed\n", thread_id); @@ -86,7 +88,11 @@ void *loop(void *arg) pthread_spin_unlock(&worker_lock); int on = 1; - ioctl(sockfd, FIONBIO, &on); + //ioctl(sockfd, FIONBIO, &on); + if (setsockopt(sockfd, SOL_SOCKET, SO_REUSEPORT, &on, sizeof(on)) < 0) { + printf("thread %d, setsockopt SO_REUSEPORT failed\n", thread_id); + return NULL; + } struct sockaddr_in my_addr; bzero(&my_addr, sizeof(my_addr)); @@ -108,7 +114,7 @@ void *loop(void *arg) return NULL; } - epfd = epoll_create(0); + epfd = epoll_create(512|SOCK_FSTACK); printf("thread %d, epfd:%d\n", thread_id, epfd); if (epfd <= 0) { printf("thread %d, ff_epoll_create failed, errno:%d, %s\n", @@ -144,12 +150,16 @@ void *loop(void *arg) //usleep(100); //sleep(1); } + /*if (nevents > 100) { + printf("error event nb:%d, to exit\n", nevents); + break; + }*/ //printf("thread %d, get nevents:%d\n", thread_id, nevents); for (i = 0; i < nevents; ++i) { /* Handle new connect */ if (events[i].data.fd == sockfd) { - while (1) { + //while (1) { int nclientfd = accept(sockfd, NULL, NULL); if (nclientfd < 0) { break; @@ -164,7 +174,7 @@ void *loop(void *arg) close(nclientfd); break; } - } + //} } else { if (events[i].events & EPOLLERR ) { /* Simply close socket */ @@ -180,7 +190,7 @@ void *loop(void *arg) close(events[i].data.fd); } } else { - printf("thread %d, unknown event: %d:%8.8X\n", thread_id, i, events[i].events); + printf("thread %d, unknown event: %8.8X\n", thread_id, events[i].events); } } } diff --git a/example/main_stack_thread_socket.c b/adapter/main_stack_thread_socket.c similarity index 97% rename from example/main_stack_thread_socket.c rename to adapter/main_stack_thread_socket.c index 4966e2cdf..2f3b2f410 100644 --- a/example/main_stack_thread_socket.c +++ b/adapter/main_stack_thread_socket.c @@ -195,8 +195,8 @@ void *loop(void *arg) ssize_t readlen = read(clientfd, buf, sizeof(buf)); ssize_t writelen = write(clientfd, html, sizeof(html) - 1); if (writelen < 0){ - printf("thread %d, ff_write failed:%d, %s\n", thread_id, errno, - strerror(errno)); + printf("thread %d, ff_write failed, readlen:%lu, writelen:%lu, :%d, %s\n", + thread_id, readlen, writelen, errno, strerror(errno)); close(clientfd); } } else { diff --git a/example/Makefile b/example/Makefile index 11fa396e2..77d413d92 100644 --- a/example/Makefile +++ b/example/Makefile @@ -20,10 +20,6 @@ TARGET="helloworld" all: cc ${CFLAGS} -DINET6 -o ${TARGET} main.c ${LIBS} cc ${CFLAGS} -o ${TARGET}_epoll main_epoll.c ${LIBS} - cc ${CFLAGS} -I ${FF_PATH}/adapter -L ${FF_PATH}/adapter -lff_syscall -o ${TARGET}_stack main_stack.c ${LIBS} - cc ${CFLAGS} -I ${FF_PATH}/adapter -L ${FF_PATH}/adapter -lff_syscall -o ${TARGET}_stack_thread_socket main_stack_thread_socket.c ${LIBS} - cc ${CFLAGS} -I ${FF_PATH}/adapter -o ${TARGET}_stack_epoll main_stack_epoll.c ${LIBS} - cc ${CFLAGS} -I ${FF_PATH}/adapter -o ${TARGET}_stack_epoll_thread_socket main_stack_epoll_thread_socket.c ${LIBS} .PHONY: clean clean: From 8f5f1dfbb048665e3ed77b6600ae44fbb8dcaf57 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 3 May 2023 17:04:14 +0800 Subject: [PATCH 39/43] Move adapter diretcory's all file to adapter/syscall. --- adapter/{ => syscall}/Makefile | 0 adapter/{ => syscall}/ff_adapter.h | 0 adapter/{ => syscall}/ff_declare_syscalls.h | 0 adapter/{ => syscall}/ff_hook_syscall.c | 0 adapter/{ => syscall}/ff_hook_syscall.h | 0 adapter/{ => syscall}/ff_linux_syscall.c | 0 adapter/{ => syscall}/ff_linux_syscall.h | 0 adapter/{ => syscall}/ff_so_zone.c | 0 adapter/{ => syscall}/ff_socket_ops.c | 0 adapter/{ => syscall}/ff_socket_ops.h | 0 adapter/{ => syscall}/ff_sysproto.h | 0 adapter/{ => syscall}/fstack.c | 0 adapter/{ => syscall}/main_stack.c | 0 adapter/{ => syscall}/main_stack_epoll.c | 0 adapter/{ => syscall}/main_stack_epoll_kernel.c | 0 adapter/{ => syscall}/main_stack_epoll_thread_socket.c | 0 adapter/{ => syscall}/main_stack_thread_socket.c | 0 17 files changed, 0 insertions(+), 0 deletions(-) rename adapter/{ => syscall}/Makefile (100%) rename adapter/{ => syscall}/ff_adapter.h (100%) rename adapter/{ => syscall}/ff_declare_syscalls.h (100%) rename adapter/{ => syscall}/ff_hook_syscall.c (100%) rename adapter/{ => syscall}/ff_hook_syscall.h (100%) rename adapter/{ => syscall}/ff_linux_syscall.c (100%) rename adapter/{ => syscall}/ff_linux_syscall.h (100%) rename adapter/{ => syscall}/ff_so_zone.c (100%) rename adapter/{ => syscall}/ff_socket_ops.c (100%) rename adapter/{ => syscall}/ff_socket_ops.h (100%) rename adapter/{ => syscall}/ff_sysproto.h (100%) rename adapter/{ => syscall}/fstack.c (100%) rename adapter/{ => syscall}/main_stack.c (100%) rename adapter/{ => syscall}/main_stack_epoll.c (100%) rename adapter/{ => syscall}/main_stack_epoll_kernel.c (100%) rename adapter/{ => syscall}/main_stack_epoll_thread_socket.c (100%) rename adapter/{ => syscall}/main_stack_thread_socket.c (100%) diff --git a/adapter/Makefile b/adapter/syscall/Makefile similarity index 100% rename from adapter/Makefile rename to adapter/syscall/Makefile diff --git a/adapter/ff_adapter.h b/adapter/syscall/ff_adapter.h similarity index 100% rename from adapter/ff_adapter.h rename to adapter/syscall/ff_adapter.h diff --git a/adapter/ff_declare_syscalls.h b/adapter/syscall/ff_declare_syscalls.h similarity index 100% rename from adapter/ff_declare_syscalls.h rename to adapter/syscall/ff_declare_syscalls.h diff --git a/adapter/ff_hook_syscall.c b/adapter/syscall/ff_hook_syscall.c similarity index 100% rename from adapter/ff_hook_syscall.c rename to adapter/syscall/ff_hook_syscall.c diff --git a/adapter/ff_hook_syscall.h b/adapter/syscall/ff_hook_syscall.h similarity index 100% rename from adapter/ff_hook_syscall.h rename to adapter/syscall/ff_hook_syscall.h diff --git a/adapter/ff_linux_syscall.c b/adapter/syscall/ff_linux_syscall.c similarity index 100% rename from adapter/ff_linux_syscall.c rename to adapter/syscall/ff_linux_syscall.c diff --git a/adapter/ff_linux_syscall.h b/adapter/syscall/ff_linux_syscall.h similarity index 100% rename from adapter/ff_linux_syscall.h rename to adapter/syscall/ff_linux_syscall.h diff --git a/adapter/ff_so_zone.c b/adapter/syscall/ff_so_zone.c similarity index 100% rename from adapter/ff_so_zone.c rename to adapter/syscall/ff_so_zone.c diff --git a/adapter/ff_socket_ops.c b/adapter/syscall/ff_socket_ops.c similarity index 100% rename from adapter/ff_socket_ops.c rename to adapter/syscall/ff_socket_ops.c diff --git a/adapter/ff_socket_ops.h b/adapter/syscall/ff_socket_ops.h similarity index 100% rename from adapter/ff_socket_ops.h rename to adapter/syscall/ff_socket_ops.h diff --git a/adapter/ff_sysproto.h b/adapter/syscall/ff_sysproto.h similarity index 100% rename from adapter/ff_sysproto.h rename to adapter/syscall/ff_sysproto.h diff --git a/adapter/fstack.c b/adapter/syscall/fstack.c similarity index 100% rename from adapter/fstack.c rename to adapter/syscall/fstack.c diff --git a/adapter/main_stack.c b/adapter/syscall/main_stack.c similarity index 100% rename from adapter/main_stack.c rename to adapter/syscall/main_stack.c diff --git a/adapter/main_stack_epoll.c b/adapter/syscall/main_stack_epoll.c similarity index 100% rename from adapter/main_stack_epoll.c rename to adapter/syscall/main_stack_epoll.c diff --git a/adapter/main_stack_epoll_kernel.c b/adapter/syscall/main_stack_epoll_kernel.c similarity index 100% rename from adapter/main_stack_epoll_kernel.c rename to adapter/syscall/main_stack_epoll_kernel.c diff --git a/adapter/main_stack_epoll_thread_socket.c b/adapter/syscall/main_stack_epoll_thread_socket.c similarity index 100% rename from adapter/main_stack_epoll_thread_socket.c rename to adapter/syscall/main_stack_epoll_thread_socket.c diff --git a/adapter/main_stack_thread_socket.c b/adapter/syscall/main_stack_thread_socket.c similarity index 100% rename from adapter/main_stack_thread_socket.c rename to adapter/syscall/main_stack_thread_socket.c From f52469fde1abb71542e862b7cf3ed6335ffb8d15 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 3 May 2023 17:06:40 +0800 Subject: [PATCH 40/43] move /app/micro_thread to adapter/micro_thread. --- {app => adapter}/micro_thread/Makefile | 0 {app => adapter}/micro_thread/arch_ctx.S | 0 {app => adapter}/micro_thread/echo.cpp | 0 {app => adapter}/micro_thread/ff_hook.cpp | 0 {app => adapter}/micro_thread/ff_hook.h | 0 {app => adapter}/micro_thread/hash_list.h | 0 {app => adapter}/micro_thread/heap.h | 0 {app => adapter}/micro_thread/heap_timer.cpp | 0 {app => adapter}/micro_thread/heap_timer.h | 0 {app => adapter}/micro_thread/kqueue_proxy.cpp | 0 {app => adapter}/micro_thread/kqueue_proxy.h | 0 {app => adapter}/micro_thread/micro_thread.cpp | 0 {app => adapter}/micro_thread/micro_thread.h | 0 {app => adapter}/micro_thread/mt_action.cpp | 0 {app => adapter}/micro_thread/mt_action.h | 0 {app => adapter}/micro_thread/mt_api.cpp | 0 {app => adapter}/micro_thread/mt_api.h | 0 {app => adapter}/micro_thread/mt_cache.cpp | 0 {app => adapter}/micro_thread/mt_cache.h | 0 {app => adapter}/micro_thread/mt_concurrent.cpp | 0 {app => adapter}/micro_thread/mt_concurrent.h | 0 {app => adapter}/micro_thread/mt_connection.cpp | 0 {app => adapter}/micro_thread/mt_connection.h | 0 {app => adapter}/micro_thread/mt_incl.h | 0 {app => adapter}/micro_thread/mt_mbuf_pool.cpp | 0 {app => adapter}/micro_thread/mt_mbuf_pool.h | 0 {app => adapter}/micro_thread/mt_msg.h | 0 {app => adapter}/micro_thread/mt_net.cpp | 0 {app => adapter}/micro_thread/mt_net.h | 0 {app => adapter}/micro_thread/mt_net_api.h | 0 {app => adapter}/micro_thread/mt_notify.cpp | 0 {app => adapter}/micro_thread/mt_notify.h | 0 {app => adapter}/micro_thread/mt_session.cpp | 0 {app => adapter}/micro_thread/mt_session.h | 0 {app => adapter}/micro_thread/mt_sys_hook.cpp | 0 {app => adapter}/micro_thread/mt_sys_hook.h | 0 {app => adapter}/micro_thread/mt_version.h | 0 {app => adapter}/micro_thread/valgrind/valgrind.h | 0 38 files changed, 0 insertions(+), 0 deletions(-) rename {app => adapter}/micro_thread/Makefile (100%) rename {app => adapter}/micro_thread/arch_ctx.S (100%) rename {app => adapter}/micro_thread/echo.cpp (100%) rename {app => adapter}/micro_thread/ff_hook.cpp (100%) rename {app => adapter}/micro_thread/ff_hook.h (100%) rename {app => adapter}/micro_thread/hash_list.h (100%) rename {app => adapter}/micro_thread/heap.h (100%) rename {app => adapter}/micro_thread/heap_timer.cpp (100%) rename {app => adapter}/micro_thread/heap_timer.h (100%) rename {app => adapter}/micro_thread/kqueue_proxy.cpp (100%) rename {app => adapter}/micro_thread/kqueue_proxy.h (100%) rename {app => adapter}/micro_thread/micro_thread.cpp (100%) rename {app => adapter}/micro_thread/micro_thread.h (100%) rename {app => adapter}/micro_thread/mt_action.cpp (100%) rename {app => adapter}/micro_thread/mt_action.h (100%) rename {app => adapter}/micro_thread/mt_api.cpp (100%) rename {app => adapter}/micro_thread/mt_api.h (100%) rename {app => adapter}/micro_thread/mt_cache.cpp (100%) rename {app => adapter}/micro_thread/mt_cache.h (100%) rename {app => adapter}/micro_thread/mt_concurrent.cpp (100%) rename {app => adapter}/micro_thread/mt_concurrent.h (100%) rename {app => adapter}/micro_thread/mt_connection.cpp (100%) rename {app => adapter}/micro_thread/mt_connection.h (100%) rename {app => adapter}/micro_thread/mt_incl.h (100%) rename {app => adapter}/micro_thread/mt_mbuf_pool.cpp (100%) rename {app => adapter}/micro_thread/mt_mbuf_pool.h (100%) rename {app => adapter}/micro_thread/mt_msg.h (100%) rename {app => adapter}/micro_thread/mt_net.cpp (100%) rename {app => adapter}/micro_thread/mt_net.h (100%) rename {app => adapter}/micro_thread/mt_net_api.h (100%) rename {app => adapter}/micro_thread/mt_notify.cpp (100%) rename {app => adapter}/micro_thread/mt_notify.h (100%) rename {app => adapter}/micro_thread/mt_session.cpp (100%) rename {app => adapter}/micro_thread/mt_session.h (100%) rename {app => adapter}/micro_thread/mt_sys_hook.cpp (100%) rename {app => adapter}/micro_thread/mt_sys_hook.h (100%) rename {app => adapter}/micro_thread/mt_version.h (100%) rename {app => adapter}/micro_thread/valgrind/valgrind.h (100%) diff --git a/app/micro_thread/Makefile b/adapter/micro_thread/Makefile similarity index 100% rename from app/micro_thread/Makefile rename to adapter/micro_thread/Makefile diff --git a/app/micro_thread/arch_ctx.S b/adapter/micro_thread/arch_ctx.S similarity index 100% rename from app/micro_thread/arch_ctx.S rename to adapter/micro_thread/arch_ctx.S diff --git a/app/micro_thread/echo.cpp b/adapter/micro_thread/echo.cpp similarity index 100% rename from app/micro_thread/echo.cpp rename to adapter/micro_thread/echo.cpp diff --git a/app/micro_thread/ff_hook.cpp b/adapter/micro_thread/ff_hook.cpp similarity index 100% rename from app/micro_thread/ff_hook.cpp rename to adapter/micro_thread/ff_hook.cpp diff --git a/app/micro_thread/ff_hook.h b/adapter/micro_thread/ff_hook.h similarity index 100% rename from app/micro_thread/ff_hook.h rename to adapter/micro_thread/ff_hook.h diff --git a/app/micro_thread/hash_list.h b/adapter/micro_thread/hash_list.h similarity index 100% rename from app/micro_thread/hash_list.h rename to adapter/micro_thread/hash_list.h diff --git a/app/micro_thread/heap.h b/adapter/micro_thread/heap.h similarity index 100% rename from app/micro_thread/heap.h rename to adapter/micro_thread/heap.h diff --git a/app/micro_thread/heap_timer.cpp b/adapter/micro_thread/heap_timer.cpp similarity index 100% rename from app/micro_thread/heap_timer.cpp rename to adapter/micro_thread/heap_timer.cpp diff --git a/app/micro_thread/heap_timer.h b/adapter/micro_thread/heap_timer.h similarity index 100% rename from app/micro_thread/heap_timer.h rename to adapter/micro_thread/heap_timer.h diff --git a/app/micro_thread/kqueue_proxy.cpp b/adapter/micro_thread/kqueue_proxy.cpp similarity index 100% rename from app/micro_thread/kqueue_proxy.cpp rename to adapter/micro_thread/kqueue_proxy.cpp diff --git a/app/micro_thread/kqueue_proxy.h b/adapter/micro_thread/kqueue_proxy.h similarity index 100% rename from app/micro_thread/kqueue_proxy.h rename to adapter/micro_thread/kqueue_proxy.h diff --git a/app/micro_thread/micro_thread.cpp b/adapter/micro_thread/micro_thread.cpp similarity index 100% rename from app/micro_thread/micro_thread.cpp rename to adapter/micro_thread/micro_thread.cpp diff --git a/app/micro_thread/micro_thread.h b/adapter/micro_thread/micro_thread.h similarity index 100% rename from app/micro_thread/micro_thread.h rename to adapter/micro_thread/micro_thread.h diff --git a/app/micro_thread/mt_action.cpp b/adapter/micro_thread/mt_action.cpp similarity index 100% rename from app/micro_thread/mt_action.cpp rename to adapter/micro_thread/mt_action.cpp diff --git a/app/micro_thread/mt_action.h b/adapter/micro_thread/mt_action.h similarity index 100% rename from app/micro_thread/mt_action.h rename to adapter/micro_thread/mt_action.h diff --git a/app/micro_thread/mt_api.cpp b/adapter/micro_thread/mt_api.cpp similarity index 100% rename from app/micro_thread/mt_api.cpp rename to adapter/micro_thread/mt_api.cpp diff --git a/app/micro_thread/mt_api.h b/adapter/micro_thread/mt_api.h similarity index 100% rename from app/micro_thread/mt_api.h rename to adapter/micro_thread/mt_api.h diff --git a/app/micro_thread/mt_cache.cpp b/adapter/micro_thread/mt_cache.cpp similarity index 100% rename from app/micro_thread/mt_cache.cpp rename to adapter/micro_thread/mt_cache.cpp diff --git a/app/micro_thread/mt_cache.h b/adapter/micro_thread/mt_cache.h similarity index 100% rename from app/micro_thread/mt_cache.h rename to adapter/micro_thread/mt_cache.h diff --git a/app/micro_thread/mt_concurrent.cpp b/adapter/micro_thread/mt_concurrent.cpp similarity index 100% rename from app/micro_thread/mt_concurrent.cpp rename to adapter/micro_thread/mt_concurrent.cpp diff --git a/app/micro_thread/mt_concurrent.h b/adapter/micro_thread/mt_concurrent.h similarity index 100% rename from app/micro_thread/mt_concurrent.h rename to adapter/micro_thread/mt_concurrent.h diff --git a/app/micro_thread/mt_connection.cpp b/adapter/micro_thread/mt_connection.cpp similarity index 100% rename from app/micro_thread/mt_connection.cpp rename to adapter/micro_thread/mt_connection.cpp diff --git a/app/micro_thread/mt_connection.h b/adapter/micro_thread/mt_connection.h similarity index 100% rename from app/micro_thread/mt_connection.h rename to adapter/micro_thread/mt_connection.h diff --git a/app/micro_thread/mt_incl.h b/adapter/micro_thread/mt_incl.h similarity index 100% rename from app/micro_thread/mt_incl.h rename to adapter/micro_thread/mt_incl.h diff --git a/app/micro_thread/mt_mbuf_pool.cpp b/adapter/micro_thread/mt_mbuf_pool.cpp similarity index 100% rename from app/micro_thread/mt_mbuf_pool.cpp rename to adapter/micro_thread/mt_mbuf_pool.cpp diff --git a/app/micro_thread/mt_mbuf_pool.h b/adapter/micro_thread/mt_mbuf_pool.h similarity index 100% rename from app/micro_thread/mt_mbuf_pool.h rename to adapter/micro_thread/mt_mbuf_pool.h diff --git a/app/micro_thread/mt_msg.h b/adapter/micro_thread/mt_msg.h similarity index 100% rename from app/micro_thread/mt_msg.h rename to adapter/micro_thread/mt_msg.h diff --git a/app/micro_thread/mt_net.cpp b/adapter/micro_thread/mt_net.cpp similarity index 100% rename from app/micro_thread/mt_net.cpp rename to adapter/micro_thread/mt_net.cpp diff --git a/app/micro_thread/mt_net.h b/adapter/micro_thread/mt_net.h similarity index 100% rename from app/micro_thread/mt_net.h rename to adapter/micro_thread/mt_net.h diff --git a/app/micro_thread/mt_net_api.h b/adapter/micro_thread/mt_net_api.h similarity index 100% rename from app/micro_thread/mt_net_api.h rename to adapter/micro_thread/mt_net_api.h diff --git a/app/micro_thread/mt_notify.cpp b/adapter/micro_thread/mt_notify.cpp similarity index 100% rename from app/micro_thread/mt_notify.cpp rename to adapter/micro_thread/mt_notify.cpp diff --git a/app/micro_thread/mt_notify.h b/adapter/micro_thread/mt_notify.h similarity index 100% rename from app/micro_thread/mt_notify.h rename to adapter/micro_thread/mt_notify.h diff --git a/app/micro_thread/mt_session.cpp b/adapter/micro_thread/mt_session.cpp similarity index 100% rename from app/micro_thread/mt_session.cpp rename to adapter/micro_thread/mt_session.cpp diff --git a/app/micro_thread/mt_session.h b/adapter/micro_thread/mt_session.h similarity index 100% rename from app/micro_thread/mt_session.h rename to adapter/micro_thread/mt_session.h diff --git a/app/micro_thread/mt_sys_hook.cpp b/adapter/micro_thread/mt_sys_hook.cpp similarity index 100% rename from app/micro_thread/mt_sys_hook.cpp rename to adapter/micro_thread/mt_sys_hook.cpp diff --git a/app/micro_thread/mt_sys_hook.h b/adapter/micro_thread/mt_sys_hook.h similarity index 100% rename from app/micro_thread/mt_sys_hook.h rename to adapter/micro_thread/mt_sys_hook.h diff --git a/app/micro_thread/mt_version.h b/adapter/micro_thread/mt_version.h similarity index 100% rename from app/micro_thread/mt_version.h rename to adapter/micro_thread/mt_version.h diff --git a/app/micro_thread/valgrind/valgrind.h b/adapter/micro_thread/valgrind/valgrind.h similarity index 100% rename from app/micro_thread/valgrind/valgrind.h rename to adapter/micro_thread/valgrind/valgrind.h From 8d38554a6fec4cab2005e921f578e308d8a1cc3e Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 3 May 2023 17:09:29 +0800 Subject: [PATCH 41/43] Add LICENSE. --- adapter/syscall/LICENSE | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 adapter/syscall/LICENSE diff --git a/adapter/syscall/LICENSE b/adapter/syscall/LICENSE new file mode 100644 index 000000000..1f72a9380 --- /dev/null +++ b/adapter/syscall/LICENSE @@ -0,0 +1,25 @@ +This is BSD 2-Clause licensed. +Copyright (C) 2023 THL A29 Limited, a Tencent company. +All rights reserved. + +Terms of the BSD 2-Clause License: +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 6c13a60c87fa5f1fb9bc6727ac67d5c8c34327d2 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 3 May 2023 17:16:01 +0800 Subject: [PATCH 42/43] Add adapter/README.md. --- adapter/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 adapter/README.md diff --git a/adapter/README.md b/adapter/README.md new file mode 100644 index 000000000..66ebcc854 --- /dev/null +++ b/adapter/README.md @@ -0,0 +1,13 @@ +This directory stores some adapters based on the F-Stack lib library, currently includes `micro_thread` and `syscall`. + +## micro_thread + +Provides micro thread interface. Various applications with stateful applications can easily use F-Stack to get high performance without processing complex asynchronous logic. + +## syscall + +Hijack Linux kernel syscall with f-stack api, can use `LD_PRELOAD` to support existing applications, such as Nginx. + +It also can support f-stack and kernel stack at the same time. + + \ No newline at end of file From 3e1a3d808235258fd9ce134c8d25f277001c458c Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Wed, 3 May 2023 23:23:09 +0800 Subject: [PATCH 43/43] =?UTF-8?q?Add=20README.md=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adapter/syscall/README.md | 4 ++++ adapter/syscall/nginx-close.png | Bin 0 -> 14664 bytes adapter/syscall/nginx-keep-alive.png | Bin 0 -> 14828 bytes 3 files changed, 4 insertions(+) create mode 100644 adapter/syscall/README.md create mode 100644 adapter/syscall/nginx-close.png create mode 100644 adapter/syscall/nginx-keep-alive.png diff --git a/adapter/syscall/README.md b/adapter/syscall/README.md new file mode 100644 index 000000000..8f19fd357 --- /dev/null +++ b/adapter/syscall/README.md @@ -0,0 +1,4 @@ +# F-Stack LD_PRELOAD Beta Introduction + +[To be translated](https://mp.weixin.qq.com/s?__biz=MzI5ODY3Nzc5OA==&tempkey=MTIxNl9Kd0pabEJ4em95c3J6OFJJdUYzZ2F2aGthLURKQmplYnJIVmE0cTFsSzNqdkhFM3JjV1F1SjlrMWFxeVo0Nks0b0lwOG5UbEZueVRTb0RFcnQ5LW42UE1aNVFUYW9mMHVaRjFiWXBKcG9VY0FYTTJIdUgxcU5UdTltb0hId0EtODBHR3lZM1FuX1FxdHNMQkJsWExTb3IyQ0MycDkxWXlBTllNOVlBfn4%3D&chksm=eca3623edbd4eb28e8d7dc68001f46381d39f4ffca91037a4ad8f8ed5b59bad2354ca273601b&token=39899456&lang=zh_CN#rd). + diff --git a/adapter/syscall/nginx-close.png b/adapter/syscall/nginx-close.png new file mode 100644 index 0000000000000000000000000000000000000000..855688050ebeb602fa5bdf98c283ba519bd5e8b7 GIT binary patch literal 14664 zcmb_@bySsK+vcIWB&1tHx{;KYP66o<=}zhH5)lxjyFux0l#mn<>5y*l(9Aym-tU`t z=9^hFYt8(@I>33Jop)UKwXYqerXq)hPKpkJK(ORrNNYeKaO>cQ4iy>v2QJ$>76PG$ z$V*FTd3`!q3~7E)C$tUz#&}-&_sH-XdN$r1l&-Io zUM!buU2nf&!-w{`P#vcj(|lJez&KLh`;J-AZXi;mDRM4@GJm?3WV_mIjquikoKbEg zLLQ%ycunx0h9w1hv#u+?uiNnYn^4l-T}5G7Wuwwot}ZD{Y@9;s7;Fd3|Op!ME6Sy(usX{?OqF}aH9}zTH3rihEUv_ zKL_}ld4i#ExWsRD7tCX@a=E)`7f4h3ZHBUu@bJ_>1%G@$Cu*LFrQE|W;f$Y0Slj}e zqM2xaEbZ%=+v{`iL0gb~jtbWgHwPWg<<4&=#D0=z*NAV#HUtc2a<0x@(+6j!*Ss`X zx!Aw@%ggSy(m@Hw-C@|5uRJ8>_Ux=&$_;xVkQlPO8G@($1bg|E>ARn;c6WEjCnmUd zo72aXzPzebM^>jj*b{=7vLtH{II0bXG>Lnccd-WJ_-wj$j?u+lIr;}=F zYujj%(bA^JfyaKL^DN*h+gzg;63ey|el7{43@lcL%ZU-pK`fW+Pd5Aak0vU*x*Ml-l&l!pFz0UHXjpR9>KF4RwtK)nR@CW5lPfq$7L}t zzGx?6>cHdPwRk3XV%j0r9=1A`m><*P7c>wW^WkDjjOXlo7abVGhMY~O69a?-ZjZxX)B%V47@El zjYkzubhPWX-i{Do84ql9QB^sd;1T5a2E1oS<$Y@88nA)9(i6|{BmCu)qaxIR)o;IxVYmsvH#Z{K8cntEOOqr37e3i>K>RLR?BD*I zkJ{&5!v#$`9O##zHTUosU7sZjj>vwhK}?zl5p-?l>4LK&TpoUWFlJ*&mPZl-Uax(2 zq*lyr!2v(hUg~Zi`Gx?*1LdktL&F6GwIlqkg))1;$T!5Wf7;EVkO!#U9k~dnw$i5~ z18YrBJ8hch&gvx6y=t4Or%8VY>v(ZfDCf+b-BX@-6#U66w)}JA7p+UPVxDy`m?#<6 z&GaUFL`#tV)9FgQD64sTqJhtZz(n8b(j@3Rgieti%n>Bxsqcd+ImD#-2sg^mD+B?9 z)xw~WzFcj9q;RJ}y8b=V{6ecHvH|1PO&y(x`1pABu^-dZeSVaAA9KtM*`m>fCMLe1yI-8a70h;q<>k?8X=zy* zV&xSmY7i&XMVBMGeS2m)6nht8sHqt;G^7w`BUO-lhQ0EB>iPFdrj}~756f#$^`%Rb zJ~`OD72l`C7*fj5{!_zZytj6W*d4Yts4S0fcRKut{Zyr0`6n$M30<&6(N7ZxIoj*iL;Q%%^t zYm*U=Wa4q-D{+ku88&3t5p(wO5!TbwOH55oO+LdLWw0Z`A7$wH()CSrca|kvb>1b_ z(9lrOW1kpeGF7B3|BP|Mu7{H$U0LN&eT$G|zUVu%N^npR94{Xq4KAw5-SvfThwodB z?~1$`oUu`FV!n)(33G2QlzFH!Mh&h@7#mZiu<3_i?9DPBv`K$K>rJ!b-*(V~G9gb> zt4zA2$|X?t7b>PEW@jg`={L}Cev6A++uN44uS58l7`C9sVV&Qo;vF(23ErsUgoVQ>6O6vd49f)5cH;Z$qeEsrMl?`q8KZSiFyA(DmbnNYAzc_TzVmVDYwhcDg4E z6`#dLG0DSPnc!vf{0Ro(y7irgkr72UAMe$xSMtv|ChP{m;zT76{)nUgU9AjOFO}U8 zoo(NglNih?JNfJpY@olR7Kpgyp;0Kv$SWsak{CV&L_`d^dC~-Z;AMGV%OF+%0{`~Y zw3-N0lSGm%rS>xedi~xpMOnf2@v2y>BH=TF@u`Zcst*M>eeW=Q>=s)-VEEuF zzYp!zZP-;eu6mA|D`ul?6Euqk%RMrFM$V8wiz;6AOmyixEo48Q^*^6XY2Dx1k|WM*?K+QVTyy7S- zEsX-OLa$U;Cnhgg#bBGQNBqYaO5)|%QPLAdV7e%s%LxMmvgAT;OefI{66)%BPoF-m zt3p!is{aFH~1Zp>`ogAgtM-?+qe1p?kRD}HD!w1qI;Y=zN91u$tgr6~L zP3s9I3m-0@>K_~&2zsA0xwyC-A22Ib+Bq>T(7klxHCExz8u# zjS7m6uG-_xwxuL*vU3==gyzd7z(X4Be)do^D7@?I(QPDsf(Nx||8CQS**}gj{vQCQ z!h&QX79ZF`)57ND=TKD=gE<5UzujD*#c;|Ku&&YXa3%MDHL5Co{6!0i6=Ch6P%jX~ zwALgl4;916)3Pqd&BK$t-P5CiM?lMVu(Ep523_)5ixXJF4Y*yWdaouy8!f}>`l!>= zt}G0_n-fo(D;uZ{b8b!ygh_Xk=*hDtu1JX zISw&`0row`BA$l^R;CFDzZzXlacHrCXLI-0;0WI5TrdAkhxlR^s;W5Y85wXTz?caL z=wXAW>P=(P{Pb4Ao@8sODn3gkartO4S1JM*6|u(SK#zT2qJSoxOPyic29?9J8I~#4 zdy4HG9AsfJg55A0i;SP%SBy4Y2t3qsmWc!cPA4ox3?@(wf)Ma7H8lc+2x=AeJOpe3z<4Qwh&~u2gvNAC-ao6eQaIJ^^+EiX7(=*`w zJ4`P6eVcy@SUtgh<>%xC0v9SI0izVspwvNld#`7*d4`sKc)l~08z$P|us8%3brKUS z#-dwCm5PM|n1fE0LH9d4IS8b%u+U&Ggb52a`lus2jnvljo4d=E5FFso6}q*wXdmeF zUIUy_`2M{QNYan(6^C3>yTe355FLOu3t%dfp<7!{e2&T_Z&6B03YZRZNDyp9po&lz z+IGZIHE&rj3@zp|Y;i+lV`KX|G_(dLO9FyOLA`0VP@MG~%7O;m!nLks@!sBE?`m%Z z9E6PDZUy!bbvNgq9_wUk`PGP*@k}X|cora%kfz`jmd8I1TuS&cb4))9T zI<-iouZzWUVuF_}Nzj}6I%UKd`+G+1;~HeYHS^d;T{gMOJN_Z70KHaX zq=xByOKCgG-fFE@cadi2?9Vcjlj-#wfeQ<1YwNkkOL03hgMk@Gs@rbGs@VfOQ^oJz zzYnR@Z*=LJsaOk}vI-Nq!eC~8tsD2ve&2%8A021k8_fd}h~%IiW3oO6FoU;N*>GNI zdAXFF++fL;jg8Hk7>;_IN0l#`=i+31c}UEOv48pC_(R(Dn4yXacE@=kheQ&+L7Nv| z=Vd=flJ3;Gx|UXGbToQp`8#TkPqL!>S;ss?o;5nodECz%6gXR_tFf?=l)Gml+Nk~{ zaYw3eCEWT|h|OvfPWZf)s{h2fyQf~o8%D}ELYcjfMVX6>~S|O!o}E z3E9JYLV#T+W|6f=Z=WoL^j+9?aLz zH94DqwvnG2fXH#4zh_!lJ%|} zO0^8--@_T+jVv$X8=C*-`S8lbAyM5{;XC1z`V76smeeB@uUl;XdAftlnUu1p|=_r6TK zG*~z6jT@9nDII)O*}c_U>6`j&P4v8|>$wRqkYS4v1m&aOC6=qqxF611_-kCZ|quLx0E>j+l>m z&;leimwduP1944V=jg1>#Y5Yn)0ye%8$Gi-Esogg0VdPR+?U5f=C)dg*_!29`1$RtL-y_8dB ztU>Dw!*~ol%kJmrFbp51?u#!03X49OQNA3i+nyh^3^6UHkIEvsU0KK#nrvsaSg1pT zPrdz`!hVZ}%`hF6!YazLtTkR$j51kxV}NW|#V_1yD4cSBRNQ}GCgYRJH^`qvP8h`Y z;}jlY&$*c12+4)*%m`lPsqS~e4&-f*+y%}_A)R?ytjm|f23CoK&m1Jxy?KkMYg1}V zi6p0G*=ByG;~o%6mkmeqZXIDd!k@sfzUH!KQ*|{CWrC2@ts_LyyB~s~y<|TZ&X6ZOi(Xq|J;X>KLP;?eR9-P^wTmw@aI3uaR-Z-5UK6 zDGB?wN}@unEO}$zZa%$*_l=>0=u1;6m5KCUgsAPJw@$NckPB z>*jmuGB%Oxgp$YSah08MGE+<`cGlce=W2Z>-$2uBryuunn%au`vYqsc7;fe#yodH4*5*hB?!X8wnfgd%uUX8ktgfT;IG$1z9m2%MCgbn# z|Jmln$M0Imim_96(=&F`!*iHT?`vGbK=svR`CVi!3WOo$=}L;u0u_7OJg)D;Jw620 zAFl~W!G0DMF(AUjExg^zgw-gISmZyBh3es%bJP1dC#+a!;Q9$x6Rlk;O|C3}B1k#) z-eYA%b9dIW4RK>hY*Uj$%vZ3kW-uI!EJ;g(R^c%y6LxSH5Y6ULSlJY<64uNI<5xk6 z@O5kqhHH5U0L~Wznb{#B&6Oo5z!AQ)#xA6jS4(u8ikO{i|X*zrJild6G8* zvy>tp)l!5;7e^fwdu4T3R~~O~?;5WYGrRdZ<|sn?Jh#45W$AB|izO-+@YB{xKV#O! zWPqWAWbeG%g9M5wlf9Wr01XseWZpYA-N4^Qqev~`&}_1k2YAxbMn?l=gekRX1LcJf z_y;KX5Zv7P^YZiOm6u~&U0uz!xZCP+>~p!D&{s7ItaAH$;u zR4=Q-5H9;@B549j?7jy-$%oe+Qb`cPx1a@pi1s#f^>Bx9tgl_}SzI}nU zf#lf_S^)tUR>R#qN=-?z;n%g@o+7%`4~W=Pz=t=ql>0FHgN#$Bk>uH4gZcM7N#$KR z-l-?nHt$;!EUdBzseh8zS{6B6U&e4Ztxj#9h0fu;yp~<%MafLjz&hOinCND;4iaMyIAd zA%YF`{I;Hps*Fj&3Z)1X>iKoN9aRUVq=y}l<#)#U&{0O`%?Bo#WqnoC<9ttQ-&sye zbs84|@nBfT{i+XmS2&^xZ}*Nul3O{SxMvs!$E6 zONUApI@TF=zkH8*j~@YnfL)bXY8UTQepnUIJzsx7Ia5|OtI8r%F9@fZcEWYQF!=pd z+di55C%S7nLbTd&+5De~co>-Kt7~@XYYu~#D<{zysqAB(z6ysD;J)9_WYvF?wu;?p z)TC}UyNvr4aGrW>M8pG&gKyXNct5;av3XI=t8$rD+qNA4s4)e9uPH!~?`aahqH=(U<&h4y;R;qjS$L6>{*6 zDO)1L@;J_YA+Anht+`0so)j^4u;q?v(ocB-Qvo*fAFJ|Df~d&e#WO`QaL6 zVC^!u3+mz|Q_W%2?M_h7_7K$HuNxG1=whAdO<`r1Yr|OMvU{ZW>^eM1<`1T`)6+Sh zKWpUAtIh8@qTJtNa$>%&zLC>yRPEq1qsDy>0@wFU;RrzO00e^+7WS?6fpqLVpTlBn z<*=aL^|vtKp+E@DN`mUu+x)m}q~Abp!rW9Fz+nHMAYAW?+W6}j5m2A6V~Ifm(JY8^ zgqAj9D`vec@~Z|BT8)A!0_V-_gW2ZwneC5>qn)L`C>Udg6l0<1gx5l;ch%z&gecrz z61~9Uw#hH%a%fUnv`Nlk`#r227TTlh-~`A1m|o}dS?V-;#AK);M8rNf!9n^Gh2e*V zv^?Gd;@a;U^LZFd?_EC$O=(=HHAf-hF#6=gCwRy7jNP!MV8TZRF}WMA@Eg%d#@@g& z`ao={_Va1)Ro-2ipKMMsdTDLNE`_tm)4rps1H7cqCBo^y1LLH%{V%Jc(X(y* zJHvPW;!6P!cjIMxzZxCoBqX|%m$Rl4k{q|bXOgzNa^_5%EFLTnPE2O9wz<{tp;a^} zlz1G5eM@=4!jZ*(7`4kcanWaLhaqUC74vGqzZHdYo8~8-jng=W{xp>QNNB3Omfx?H zf5%R61+wq$&YQ7h1n+rVT?mUaFmjEu-?NQ(hqB+#&o9w}1Xd5P^~X4mi9-7*`ab#d zl5Lv^gBNcYGgCI)N#)`+#D0+eoYd5Rdy-W725kA6X0Ku~t1=`&U*>3>*b~{nXCyf0LYCsUA^YG@Y)oPi)<*_5_q! zm+44aOW3d>}@$4Uo*PQ*vrjK{=)AdNY8!EQV{-G zz8LGBLW6=86aGmYbv8ry$j1%`8^Dmas$-cwH?*(~rWOM@gp`a-s@$fZtWWB(YTjTy z8Tc>*xsu82R+&KLp5e4%-xPPMPsyStR(P3Tn-0S&_KY~7J5GrqzuQ7WY3j3Pq^%~t z49|Yd+k~~LV%7ZSW)cXD)v;S>;;L#q%@z#pXv_{fHk1OxtXY}RY2x7Q?+%?0{hgMs z#)4?W7_f?&xNXTN3ZTIRQ$o29!Pb)4DrG;Y*sQG8iyv1ScZwaptoNnr2=c#@fjkE} z_-nxSTL-xpyu*Ne8o+OsJ3-8NPA=+A0EjwB;N+x6gYp%gRvFiQ(MUM!{2h8 z%jX-@drHzIF@P|gQS)r_Y1uv+T+9~)DC+AwIzhi{-p`*uoA8p>`d*!;ElXZjnWaC!9tyf35X+7#kV4DawpN24oujH@PfXVPF61~0Rd(5^Roq4 zwm-m3Rd=UR2i|9Uant(hIXm_F;UofU8J()Q`FKDOU{uq|vG?N@sO z9?<0!@Bu5#)VEyP5Dix^)3#Ej4)Y5^w{2Vi<6vC4&tWqP09v4*noZI3&O6M$L@HMu ze?kmO|93Y?_;?TiZRP-Df43GqN%n`S10)8q%M}C|LvXUzKi6PSc6xehCU)_+3O02Y z-2*cVyju27rz?tL3N=+iggo}ojnBz?Uml+o4|qUAy%BRTuer|_7%nbm7#ds~au2`8 zOhD(5=OA55FQtgeSBv3U@P~7~J7NXcZo2x&OFmOaWUVI*1FQ`ve9-fSVSV&vK7*Sd z_{7(BG89~QE3Rpc_Z>S;@U%lt06gR4L_nF?Z~eCcqocYG_5&3WenKET!EAd?{=DOg zKD?E~1em1m@ixnXep+On8S&Kq%t>&I+&q0%3Isxe0Mi0E|H)JLA!uxDT;HC^hXe{A zwc!KqD)K6MmJr8#+52f*-DWR`i>(xGI&N>;o9d;6AlI={<2_@n19jj1IWBT>e{x8T zX)iLYDB#MT^xT-;1{MT~WwpOr`9jX6Ed741N|{#}Ng#;$4_u>G2@V7t2INw+v$4go zMBZ#ZmXi3?dWENAm`8BhV_nBkV>_ip*ZVXM`u1xLY37STzmiAT=Adj`aFkW$JYVHFKwoe3Lr)o(6@~N5PxoNZf^B<*#1{Tf7^FAr<|0dUI7T?tn-3*HmwNF zYn3;Csc0NvsG`GO;lr757?Ch!Lh2s>l~W`8i%RkgAW)|M_?!*wq0a=|+D7MgXqx>4 zosO$h5PNk8?i-$>(ol!OqwLR7+;X<75RX>ENIF!e|C9v~lomhxYo#kxSqaeP0P(uE z&3Fo7HC^IgqD!lX7)^iDWem-QJZXr;uF+it?hbghv)&*}0an3De}!8P+46sZkz(g_ zW+5TnVY^;QFD<3sTLmkph7ObI^tq!2F=ROO*ru}s@qq!>PU-+!cr99eeK!>Z$(|Bn z$g}i;@gF`g?0#_f-%b${0bl7&3PJBo=y(-{dI?EZDcz(>p*M(Z}HeoW&B(QT-yMRNE^#(UO z{rpv<(XUm87q*ttW0o}^cHK#7!qO`yT0)9t+Zp~6f0SJol-8?0nVhmS0(sv=$~^h? zX5C)ngme%i>qyPJ?=Xr44#Su{C+0c+eNJvOlR0o`#K;JU) z>>FJ1AGU0=R$;Rba4eQPO|3n|@9*1TVqvY0eB@Chcv5s$LAu*D>o@wfhm+lst-Ezr zarL44V#>e=9BoL-$jF?kc>OMQ;`rv)dQq!R_ebhqP{{V(5SCLeq5}oQkZl3$(RmUD zaztJOtCT{G_x_1D8MfeV?lMQOHrYw*1faA4sI8Ge0ic1jA7#2)&NY`3J#@UZbE`aa z&i1o}r7$Ey9jBaCZN>z70iL0+2L_z{HiTpJb7pv#_;yTeH6Zzv_AlYP|76L}1wV!q zI}3a=!nboWYivh!m>?*`=B`N68&qpmhtXLvr30nOPQ-h5%4*{I?|k5~93toE?q>@^ zY9DHBZEVzDy((3=x6<=FVIM!mhje{%M{&|0qrUYQ2hb^&LfE6Pt!xHRc$~*KLXfVu z3#jR6$e1ZxYQu=qVjO9^jx;9;xmvp2c$^M00=a^ql~O%?p0@on1rLmcJR*i8<~n#L zvMo0EhNrp(HiA&fG|4cw5>hl@m^wi+z;+kljX-sdZVRX9rEwR&yY;aL$W+kufoFO! zf|u5&81kG!R#d`Myi?tZ*tqY zq+uxg#h{O}zg<8)2zp;k+7}iU#z~CHg;Yy?n*Y!!-n$}q1c973=)Y@R6sr7DWbFZs zAZ>T%Oq<4?{|me*#Or$g9eMGqD~|mawu{dBNKK9G=tv%rU4l3t=jLolHOX5V?Lr(1 z$Xc-ip;~p~QjjkAMl#O5N?kvxu3}?;dXF*PhIrBmMuDclq#xs^8yw6!d_C*UWvko- zQq>Y9cw$$0D@T17&7LcZ_{JgN&<31;m>XH@T~@8m+b+Y#4f3PJz0?>_Bs1EyHpBjV z9D`G#xO`c|Y#Ss6$*4!0Sv)x&ERFM~I7P(^f!Yz=-qbG*UhB#s54R}NA|I$ooZC*C zx_?W1obVc>4Mkntg>Cof4(;$E-6mI1&orJqMRf9gq5~)$ri_wj_oM{IMk}$Iy4h_q zu~5hZ4Xaaj4xTunw3@+I_}5=$1FALU{c-98(tYkT*hwxpb0M7)nhL7=(sUglUQxxB z*@}w+y`=-ObYtSDn#D)ojW6G|kc!L^*@MpS-Vwu2c4?IxFzlu-Bj+4g=am+?SjMa9f|Cp`87wY+dYVpz!Kzjba@6 zb&*)>`s<`vqmX>n)vxfm5kdkChmy~_EKSCpH5W#xwWzYi&?k(|VrKO&M6zrXPTpT+eySr&6zf!9ncXaaN|D zEr*>f9WXI5rJWyxgtP5;CXZ<-g2oI|j>x%)&NOXavI+6j$oL6--%m@LD2dDP$EzV@ z0ihPxj_X-TNolf5IEEroOU9x>t4m{CfRi8+xuyN=sQ*4V8Xu1lz1zj)Q z6Sbk9@7#aU%{4olk0|P2i}@nloyI_>3)!~BFUjfI?d7GJ2RMsHc&4$K5)e}NsNtG; zo3Q1)17a@%+&Q!>bn&K`9dzeihAB4|Z+fjOc=6QTFwKWp@ibH5&Diz4J3_|JIwL3; z5MW462nL;xGE~=qkx2Y^Fv>Yf$VeLsdsbPnZ{5B9bf+@rv(TuEN{2RVYu#V88T zC6XKK7LJv9j_l_-dlfBpoB7q$H!QVm@RHKJhQtHO_uk^#DAqDcuY_1U)|Z!jRFyPS zf@whv)$e}NC7f9k4;<6#L1}xsU zI=uWpX)5j{h1vHWk@sHx$%J@#&e#}R1{F283r^Sr1%Lb>P1S$0bUi{pERy|&j3D>E0tFrB z$5Ox&7ZkmIB?qN=`rGr?=?De^%F4kZ)IahjqXF438rd zP!aN+3s}STHu59(a}tlZeE_`o(=93!^s|9Sa?=bE6=#C-+BFSYQ+qar@R(BsgP-A#ehRU_-m|X z$sKyl6aWcBt0#~E%?yEnnBpSvKtbgAGlau(t4rkI10!v^KTvg5PoV;dW5a~}zO^Pq zT1@A!?&}4e=qRo|VIuMv97L?+`pD@Sv~{j6->t{JFrt;CWGb2VH~QGGb6k+in_`~; zF$h{hQA*{bbNiRL^+oDqdmbNpSIT6)M0EZ`{TK%p_tJ`|pxbqu%Es@gHCn{$SAR?IYlmzw%kId*#TTKXi8D8jC>r zV5A07_QSb@^Fek1E49z z6$Af{Be{_0-AFd)aYQQDxitm^R!x3gbdWhL>Qr_HVQUhjH?63s_|-wmp(6^U?m{f^ z%=JNl_^91^XmY&&J+T*5#z6dx5wdHr+IB!R0(Q7ye-6sXK$~7lJV4M1Q?{{5ELxOa zMH3F6FAsdjB9m>m13bX>=Hp%rkJnq&o*>pc+zYFygA(Ns`PZaBQfbR}Yye_Id6NYR^SHgzQwI*ZNhM0me? zWz}IOqhk83!Eqt-6M234tIMix&rA19kd4f~;y^%wEe6ty)W|E|`$!vFT2M2;?(lki z8H0bhX81y_nDwq1ineeu3pKww-Gubqpx!|#TB1#giWt&Sm$TTx6s=}*RZI_H88gtAO?YS7!<4M z!;e3{e3ZQ8xU#U~QSs`*-Oiwu2*Npcn$=<=2StMH22(WuV*SLTB+ZM2x&G5(t~mUt ze}24+_4z@(ws&Vx)+a}k(H1_=O5LDOpU4e#M8N9DFalYT$A*2Q$*&lL848EIVB$ zbi{$Pq%ghiz!sCiHf0=Vu*4I*Cd6Z}K}gx6NEEwqgakfQ(-{%^=_8Kp=#6X4Y?N@L zU7-dJu#TMUl^_Va9^tNv9<-K=9^tT8O^L^N>86t4Da7Swa^S5uc3crMPEvDy5+6Qv zN;sEdzn!h_fraRW04ma0!eo%NeplTI0g}*C2LKNA5%~f6*X4-v+&w$(xe%BNeIU8G z#ja@2aqw84C0yJ9jwyl8RxQZU3lD}!=zbnimlsePv)r-IoHx=Q4?hqtia#&_0|&eZ zX#14PmnbIto(_hzelOlZLee6cWFBFflW5CJN_`|tsg;2VfV2e(;AjvDP)wE&LUm$3 zf82MWxP3UQ4y4=TES?H`)+^d1*aR1_>+>jk;gmzS`Q&Z#C5OqX<2^#veXpk^9IM5a z${1taHz9n-i84&XHeN zFBW-7xGFrtQ`bwI@fiqTBNIrQE&GpK4zb=<;1v1rte|HoLuIWR{UP)v(u47Kzyb{R zdR!QN>S#hYZk`YSaGy6*D%M@E?VY2xjbJ6ufJCe*|j1OxO=apu!b<_paNC&oEL< zs+NdJL)}>dtF5*CfKY#-F1!|~@^OF48T`#&+@ns7D0kUyp;zrk)F`_ip?*Oes~odO zA=AdePnM*Zn*M}e>y^O4w)1z&{ssdU@Gb`C{L;bvUoWvk%7u9Ie(qI-536CeBU_M| z&qW0BROdY$la4%l(-*vCY>s_M~LH56@`rvEjc7z z($YL*6;xdxY-Vr7`s?4{yjwm~)nzh-gBeUkF>wfmaZlcukTcKsa=5k*_-A`ZQh;>l ztK!4MQ)UczMt7&UqD=YoV7Y(^dE|6(U3rZdLfGp9GJC- zQzf^eT)E32=gZA^ZngNFT|NYj==qGT6D3RWJ!!&fN z+&v7|E)NelH2xnBi!)6q@$u9d((p|8|M5Kj%ym9#R9WBxYXVrkPJG*WpJjD&F+ai3&>6U3_3{63IqQFWYwLgdqBzp-1GW6XIDf)NP(=K5 znTs~047&@XSUs5j6^*Uo2f^6u5uTN;C-Y3n_cO=D*b_JR%)hsIfBq(De1+ z>x5z-Zs}to&-{*scdxRWXQLMXJZcIH#D$MWs5w%d6@S8T-TZw&n}U{YB=#~clkN+n-e$x4L^*^qYSz4;L}{5u27<1y@avH<8HkYKSR3=w9| z5JpA)=b?37m6C`XP!IuYk-uA73oqeQ_Enl6t^oa1a>iO~eEt!u9T7aQbm{k0-!zZJt9^nfJ_}xn&Y%y)<>TWksi>H4VFxDgMS(NmC-?scNBh6J zX{O5$8Oad90RzyR1-@vd{ZYV?67u|!Bx>A0w+!zdwBt}~DLbOL!3{}>yo`!;m841V Fe*^EI3ts>L literal 0 HcmV?d00001 diff --git a/adapter/syscall/nginx-keep-alive.png b/adapter/syscall/nginx-keep-alive.png new file mode 100644 index 0000000000000000000000000000000000000000..7847275236810bcfa2e8ae76605f7a63e0ef0992 GIT binary patch literal 14828 zcmb_@WmHvN+wKOWTcjJLq`O1hfTW;=(kV!HcWz2V5s(xRknZkOKuWs1k(918x6kvv z@AI8+yyKiP&iTP`xYu5LtvTl%*L_|09j>Y(hmApo0f9iU<)1>ILm&t%;Ex^+1$=^# z{w4|np@GOlAFI2k{F?JqO`ZtgyxuqAG7O?5sHk31ELDCe%2=xWd_#}Urg1>g4|-(7 z(Nwec-nIOp)4swll{1C8Mx$gLQTcr?dG=?vUu@Q^zw#gEzsEx;dj~W#nUGX~Z_vW`V-YkTzO%G=(CUZfLdp5`GNdRS#Op=w}I;*tu3!FVD%mLL$eky6yhbV51Ah36yqCIb>}@MMK~ z%*#hH$nA^jA$ZbXRf7Dx2(%xuaiQfV>6(Ez(Z*P{bG4?Mi`1+^;LqnckH}D>KAG!L z7f4=TfBpOP6eZo<%*@Ex*jR0YE^}4&`SZZ(Y3-td#}618jdrV&hS$`Gza(EZ<(aN? z7IrKx?rB1y51bUKlz%yq8ik_TR3ALF-Pbg4tf0_w>Cut}+fc<2QZ|ti$p z4txg@u`Uyh_TGNT0Ir`3c&1{2?h=OX|Han+$36d#7p!~0N*}2{px13anDF7Ce<1V8 z^9S>p_J`ak80@KuBAN+jVMO4#B`zCpq&LUtRNoRlqBPo9mDc77ayH(ihDG%!z1zA6 zZ`8o@;Zaa&nbNt@+Ue1d{mH8p~}{9zqsU!CyP9GO{I9^2RyjgQe_5TtXVEZVpr zp)iex+HUs0epx^~A2B}t!{B9DN5JK!XSC%LD0J(Q1XxZx7A`bd$si-X?Q2atYtm=b zq4^@jp%4nyw?!|W5$OFns9GhPqtt!G`hjP-@1HjeOPmzhx?yAObuYqRrx)PcJ|X>0 z76*S@;su)@AxgiOG+Op3r)zC5hqIShNKdk}=mI<9gVGsA> zGUm}0dx?b(apuD|F2>Bx?JH8QYwpq`U&9r*%3|!l?>#S3sfpP4!hRW_YvTiZf3G7E zc7G#gAz^M*W~_DYnS~yC=9fRWlWf03s*YbnxJg$9H0NJ`UPaZO2xZ>liVA0LzSatOcKUr^*dmgd4!Cuf)?kb&&B9GEMXe0f z4ZehHEI+mlLe9PqMaFayFyQ8g1SFLj)(Mf`cxq6vpoo|PyWTA&VRChWE~@v%S8s0rp0$dE_7$0HCm z7$RGPj4d#iDub#sWcxT<;nG5$!f3A;R_Q0PvMh%LfJYd2sdn;KmQH*g};9z zN@G`1?Lw=-3oJKD$8bn<0OfCY_yyd-!Zm}{X3MYlu@2Os^j~Id@a6d*Gd2zpyry9W z)|}{Io!JMoa82Kh4e&lAa9kw9@0z{eK8ab7`MGKuNiojtT-U`S&mlD7N|y^BHIaky zR8#$%29VQ<%%FS?OEGJ&@hB=~9%zAxOOO;S^?^m9Y%oN`OPI`Br^7&EGSF2K$y($> z_n($5d=JIO=K8JfY7lIhPE?=ibgwFFmfkRsxQMqG%k1*&H->s^f&Tvf&0oK;z518z z*|v3|c{z=nPZIZcTEG9&u<*Qk#wGEI>h4rgK|xhVr@)0D*51h!)ufrY6aX8yRKe%< zGor*n-QL;Z&&O6nRDO$aSC4#vbm6z%z2M7Iy~NMW9e^uw4uPn?dPO?oM39|X8&SVu(^!b}Wtab~g+_rW!^`#u@rm9MCpLg7sfv`KdQ4z6{rOS&A0c_dF%; z(S-%vl8MNG0E9yQCNd2T4Q_EswR~-9!75yiEUMGfyj)LY*FTvgWU*D0Gynix8SnS(>t%#6X&(b3q|H&Z?5U&%P` z6z;?=99q+t%W~XP1@6Yih?JC+n_F855OsC+&zYIVJsS)Pr~$g+GZvO~3fvz|MqYdp zXd?MQLq$s3R%Y^zi;K%YC51v=Q!}%+)*b$3L11|AJ^k2FZ3xAfL0nby^OJl_Es4{z z5HnVwWwJ4l`m)K>1?!PQoG-B2dr^At`}7f8s|Dk|Mdqm)wMjsXM)13C-p!dm`0*aa z=J1wlco|qO&)kXT#dm!qYY(&G+@%|%%r1b-%*uPXg+HRd{Q0?PXQE`~{BVVYl5!1P zlmR|wR^F$wxIV>zJ^fsRkm*@kXJ_X<0oz%$ptjbl4k#vY?_TxW0;$26)w8oE#u|BD z|L}116fuwc`HvEko9=F}4i7qTztzV>Ndt_O;_^Uvj>r$-Zy<8s+}vaaD|!naNzclP z`SIh&$;vk`QoxFC!HPav95CA#y{Zl+uDx~Myji-w_THVXBM1ozamZ&@dkufFq#o|6 zM<_&M7OcN-4=zu34kt}wz&h2i;OkUg7+f7tV%<+je*6PzAD5tuxFeDILom+t^igl) zpZ!p~262O@r;FJC@UgH(!B@|LnqzRcF`LdddZQ+CgmG{m@ooK}S#T!vk&$y7vG0)c z)DIJY6yjJyB~gl-S#&zingljxygWVa&ku~E-o7=yxjfN6y+xK=|UKpir@?K&pZ*Tc8_ROMk$#;E`dms(6eF88WiEa1P%$#GGcwF71)t zw8k#2tXcfbd1wYDrRTC>H4mbAOc}DU|81!w%;9wRm8qFon+2caXCgJkhl^=t#6`A9 zmg;&3gik(6Y&Q_-{TKWVZCbY%B68uND_VUWz1-rvq_N70I=^>`F}iYxcj(iX!1-;u z$|^QBHTv+-;rH3uO8VVJI)U!YIqN@Q*%?Uf{hN;Y7C-xM=1qBziN}}hKQ}k)XHDa= z|D>$ZR7yX+WZ<#sRVu7Z!)8gguM!Fw|wvJBM?aftJiQkRY;Z|~QLcB0>Xyf=1^uxH??p=Q;}R1RE|qWCm@Wb0ctSZ!{F?k^op!5ELBj_2`na z_kw~b@b)c^ZnHP(t5>hw&zHh7CY)C9sg(n_$WE5@HE@0L411c4jI13VTln~b30XB4 z!DFAOsl|?0JM%E;r#^fg^m2 z&3SUM?wRe-?CA%tzCmr#-60PP20{gfQFI!KG?N^BUsNIQjgLMv3FTH3A!)ehT@IY) zc59^8`+`Ta@MS=L{v({x-Um6BFNZ>oV;!C1d0xC2QCEDqN6+QNa+id{WLOe8`4ij5 z=$T!@P6+>GQv5=LboD59Wj-j0;tUBye;1uGGKBu)Zt^6;ijIus-jmghJ#G2XoW>>3cOS2XM7_`m@ zbx}it^#_eTD5*BoDYD{)`T^UTyo>m?CInLOeHX{G!LLD$bJkINySKT<63A|nseX3h z)q0|-1X~jlK2V;g9&i1nNX@wJ#4&K4;yQWqg8f@J&)2phEAzNd0FfjI8kt+Z_v)Xh z+|1oazZ%G5?cLZ0ktUdpwULX6PYJmhULlQ`huEYO=)Fc>=U_^UiL1HfA(>r&N94?> z=6UZq8i0i?*2|}8wl3Ob_gnKawAH;2`*!2UTeq1TjJnbhRI0EAQsIf&rFf2(eNO6C zrHM;v6}olRBMWD7+z&R@L^c(7h%smZdfeqED? zo_9y10V_1Fw#6A^sb6&Y-lE+@Exl)DM2<2WarANs-@MQ4=>ZbP?9xM zAn|LDr!IMm3S_t1)v^1;a?yLnt)KQyV!6{T4C!s39hzao1_reG#;JI;>i+vjBM z0p7^c>4*ukpFZ4KVxM|p&s)ELN5i2M2~JC+T3uaTC`Btva;{xtQpqA&j9TMN#j+B`F#fK!Aw;#6@FJMz|+Bu_#TOMe`Ufp+7l5SB(`7dLYs=kJej6$ zAoEd3c$4_HP$vu6v}?KEus9Ys(qaBxFC&LDGcF4ZPI5UZS3P7;R(5jY_4M-U(03A| zh!i0nmS&C!Y;Tt?)UCS@;Sv&pfj~*$vVYeWw1bj?Bj#V@=RgIy6X#I>Q}bC>6{Dz; za9nanerhSj2}3$LIUySh<>>-j^OjOH6rfS~*1)4b04o7YF#(d-l%;IAmnGL2Cu3(; zdK#d(tBuaFZ80gI{*A%Xqb?h#RBX`l_DL8H@;+{|5+i9R#QrasP-u!rko}=HxD4)n zv^$BHy3zLJy`P_zgB53jutqHrH9V{avK_0zV_)B9w#LgGYtjwtcl2MwnVW)DMD$7i z#asliEas2kJtvX~5dG9k6d38N+IW3?){p#vr(J|dWAy!RvPwfVLT_0 zTfpNgp5*o<()fG=6!V01%E*pp=x0%DbSu>s+3?CR1iby1Y+!2?lD-r^K0ZrlVNXu3 z(GCodL2(|DUzKNprHtbjy5x6B)m@U(Q65-@x8ln@C9JI#`Sb-vtIQXCkm5SRa7_#i zAr}`HlXXr^7?_wH3w+ny_ERjmTytkh0)){ShpZc1-iyLYIHyLh+O!{t-G~Gwq5lMyXpkTWqg!R1XY|SxHT5#$2&lP7pgMu}v2$u#ulN zMVMI$Sn9E7erW~CwiKKfg}CQ~tgI~NM~~Wz^%~xNAjoiWFCrxoOCk!Y?v^+wv~9lP zfzX&=t$e>*XSK#B2m$$*LG|BAeR!|K55Cc*&s^=$ONQP50^pEb3>5%wa&O+SfHx-`TJhGBB#bXi&S$C(=)rO9t;pInpInQOXUE0T> zYKY+sd3c^-lrqlgQa3r#`5;BW`x=abG+#=O81E-Lmf56#GPs<`Um3jNR}aYG4v7;c0W(f|b+>k8f2fEtuo539bp3Bvn1zZ}TKKYXGoB zRJ_PlSqj3HfI#4t4Jd<7kvN)Z8uDH^4~?IHbRuz_=*d&G5-=XCuw|PduPoL~;<@Mc zg@6ikc6~_GCXT0s5G0h*ydqrk`ZJW6!|HmYPZR5{v>QZ^QuJOkAKNl;WDdA_=lCa@ zq|20O1dyh^W2X{luG!f~GX-Hwj|dYB|I8unMnrjxs8+}1LpMM$Lff00ty?3xu^&H% zq>8#_R#rNnA{E#4%0=KzAEgTil1r9!tR9$uZ#E~_)H$WSO0&_{V&zTDH+oDwAg9pz zs#i%l;Kskn!iBu_z{kb8w93QqjRBO^A=V}B!yk$%t+Z7jqsQd2S6|yZNiuJtNyJgH zJ26Wv+aGBK_{A>_h!{GF-y+q*mJQd9sTy2=0mL=DiyDG6U4OQ3N|XBzScSvL)guT^ z&2quZixXHvZlkakxwQ`Sa&yjZouX=t0|UVFp96cWD@5h`((lU#eshgno;P z5USEimn)EJAkiB>{%;VN(2BIal$Y+j3DYZ)+FP{oe_<2EJ}s9wXAx7mTh@#fdTRR> z;tW|7g%OP!rmLXp1tBXG5G&_QY0vT&zo;Qx+e|xK8wds1f#mq=M#y;+@AC5UU3z*< zetv#k5DCQ74-1r^dLvz@=URNBz$YBJU|M3Q*4@f_+xz? z&LK1=g-OE2-(x$GL?r&eET47KjY?0k(BPu+y_jdy>z1@3j+Iw|?g-wfI{ry>N4!d7 zrNO%`-9c;@i3nPqgcuDyeeg`Q|M(hBiH^7-ybqd%Vz_9pwBn zQ1MZwJ`Fi>3K0j*ircGfV*ah`W=8P!FDs$VAogu@+4_OWUBvJ4+-gx+SXhhy2F+kE z)`>emn6!S%U7c{LYTbS%tIZVK(ZnnZcQ%x4_wCep<#9{e_pp80=r(Mp$xU<4!q{w?*w*ssK z9-Azk#dXX%q;hX6qk)LiCFSSmXMfcJqK&$ShK#<3RZf{=t6WU)$LjKlFF!6vR??l9 z7j8FRB-Z+(_l;Y3TJ}k>EtCU)Hxo$3#wHRZ+t@T8n<9fa8P>n0V&{LfD#S#P&;AQ- z;a2?1jF5G{+WF3FcbfM4xD^NSrF3EOe&*+t0HG;I`{Jl5^B_?Lv?g-c6_AHY5xe;3 zk;R`9QDEJoZPcm&8a3YXhxzq|?gQFjdm)OiB_%O!TR+MDR36Abdxk3_BErSZ&3y$) z{)7?(_!FEcTttuL0MvPv@I0j&5x>Y}yB0L=J-O2+!j~t%cT|rw-{`SNAeRs-Se>Md zgg!meCalzahI(y$&yk^+-6Xw_L?DM{bwN2G1}M}@8^++LGJw=i;>jX zP`nr_KO}m)CJ6Xd+b*rL?WuHL*hn~dX9nf5Puf!S?es^12m7mo^!dZ!DNW5G;maB$mfPE*PW-*sxQbND>7^bjjUYk5`%acyu2y)zDIQW^?>na zdc=4(Cbq_)ee+5Z0aAE=v|u_5+xlTB{lP+8x~2eKIpfUm2~{K+?Bi(sWd%lVkGRP~ z8&RbWSzy+db9Jbpqz@ntiA>reJs{p>9HA z5JkOM@J42(^3vR~#LxbCu59yjYG^%HuLA;!WZIPoHi|dXzpqF1Ph5j3ft!shqo%5^ z9y~j%i$)7*Hp1(pgqD=!S{K%xsZL9qJnA*1_p+&MMY!vWKg4Jk-^ECZ4@k(uh?@#$^E4qhq7*& z{nsIDkm4WfXIBn@C;@&$BPmHO>b3_pGFqBN)we2#-xJoA=zb}-py0oVQ>!r ziljC^Z*{_3Mk$XFFqt!GerrB%JzB2~eZ%4lLO?MfB-;n7BbCqvt zX+m%^rH;*$J4TVS=NBG=aG3iHx%>hr9@pA>UiLL{EPQ2=o0yv2E@~>TD-KcAo*;06Rp7@i#Gz@|6 zzi-kFSAQj}!nVTFpa{7f_+9CE(z{a?a-C_9Qc%zgFO3d^SU z3@pO7Qhvo)0YDSg7EK4EQ`O)X3abTK*w+chDVkd&rCXth;_!Zu8pS_+C&7M&x+M(6 zD>l}bk%74e8&sW8^lmQ3BwA;?>E7Pnx(z(kQ`)_&Pu|UkqAV51KW(ZgJK&z=iZpd* z{iN+hi7~015&;7RpxK~ua;O}-wf=>LYykc|ASNF4^}oyCji=gv7gVw5r5x&jL2|XI zReQ152k~F?M8askz(BRce-mt^bk7iQr{7|z>CGy zFu7@sbA+{@J4lh;D!ZtVRq-S2!s}^VejuB?HX(V@tuty&HIWqHG>Afc)QV)+hoB>` z5_QFWG>&Vx%?e?}8)GW;Cuz%1adPR8br zVliQN=Zqhwp>GG48ZNCzX-@yh@H@*`!f!jM-y1-4s z%#nH?rt^whkOl%F&4u>@kb9oB>+6eG5vVOvWEed(s2GSJmmrREn8`UE*ufZ;JmT-q zYK}CPcB!NEGI7@rNT{aMgg6addn=g>$g}YB5VLV5i8j{wIhk@m(6M-*l=?UUthOSfkpD)sWT@EKOb#^(x<}QFVJl9eG@+1S z+46BbeB~lg87|Oi_pS5Y4u*BVP#=F+WZpEQo7JvFh>;R$?*EQr!wU-wH&L;uK0fw7 zW&;Z56uX{!YApt5+hq8SV?|gQ`S+kf$2?AL z{|k2SbjPZjPt z7{2L~!Yct@L)v*L;I5uE7~A%R86yr5KQ`6xnzp2Sf9TogE4O{-=0``%$jp(EoWuB? zDyJ0}15kn?`joY~5t~IKhWFeanNp1UP+Rq2I7bDdQv$)@pqJO&f$g^8ciH+Jv>csd z&&ipY;UZ~UQ`5Y+_d9J9Kg!1C#a2V!{v;?vvnG8Z`*O&bikJ9k=uW-*>GM`#i=nt; zVW6n{yb1sl2q_owlmLF;h&xMX;R>|oRk!7Q&@2BpO|*84pi_iFrZ?DOnp~7^-hMnF zyLmp=C-wwx~W_Ids)Y+i=m<{lh9rjiRx}I>NnmW^5 zEN+qTFU(v`zf$7<@((WRncGC5-AiIrTR5)aW`vW|w)*Sfx8zD(Oj)6PB_G zPSn;St-rm0KQ=zbze&oG6-Py}|CAP1(Z{xsyDwYE;EuTKSq*WpAQ-#JeGaC^Gw{`Up8LtH5lLB-n6eA!j_hnrf?6Y z)^0oGXrXRJcm!GDfDo7!JB?hV>g-^b#QRWV_ePIFL4!*1{4Uqrhc%e?x8D~`ApuHv zHoUUhS*haADzn|H-WJtPb-_@y>%3*a9F2+7{;jAKe0o?P^>Y9KKoaVEBFc`Vd^Ccd zPh&xHtfi*6YY84c^8<^t_`6lCRvJ^VqKqfH6kG-{5O${aWd+ZTxmR|)LRYDB=gf8@&JT)3m z#&zH;U3P1^N+qGDj=5cr<(0m^^_?QSP2iXIURTzYNw9Eqeu{OoT=FNipI^)QF?Orm z%I-YEAEpCv);B9}%(k)vyJ8uq*%bX#mRX{3g%~umFY}*02yGh~oiFPg82~Yo!Hi)$ z-xv-w0cU4tsP}$+!@!dK4FX)jsTh`gT%a8`IDJd(fA?`xfwTP)cVNl9SOotv%C!Q{ zyus~8?a}S4#hVIkEp~>uxVX>N)c{Q5g@o^qG6p$$&L>Y`bXm((S+uvCsFD9u`|W@I zIMPGc718fH?SKZf>&9ebTozmN$xp-;fqpt@!o$!cUa9mS+*lq9_ zkRC_uqNUU2xP74U;SH^?QQ=I)5hyiM>NocOYNetI5Y`(hsIzz_n^%;Tg_Qd&T~|=bf9ej8Tn%|EZ;}!xaw)m?!LgjRJttYK6h` zQg7=@x1dWwb?UjaZ+h1wk!s(6J3p@#a$JQ?w)nNU?-bQ^=BE3S-SpqNO2baE9Y1RV ze$3W8t~P>Wc=eZ#aqSvvTb`m|%#AcSQ#GtEKL{}-F^LBHqLVJQ2-|r5xf$6uES2QCiA%lw8t`{&nG_-e>{qnk=p``F))w8H_@1@mqPOxLu0E>&uvT zI)@rX1u{^F)~Y;~?EgV_k^0~qQnyHo`?+`UI!13jNR-5a5KM3`bxZog)U3womM+B+>+IhJZh z)cPED02lom@xiskB!IaTHuv|x9j<&^I=fQS)y2vhhJj=+y5N3XsH-y#nN`)yet;{yqH{K4}Jl~kmSYBovp3M05t;1Q&n3#qPLehSy0#RqZeKb zE+Wuy0o8Tesp*O?yKsjt8`BrPa!hrBqUrl9jBEl1yASY&Jlu4C-;%%^KAOLhO`gj3 z{-CD1`$CW`uZ$&ZlZIg$B+`^Nn5U}~Q`@?p$XR&sMfU}IY zdJ5Hs?*M!c>zp?1FOJM0QYtDquD_;30A(h+AT~-HH0ek9eTEIlhhm9~WP=NUpsICQ zPJn}GJQG0(1Qhkqd6PjJKzP*sqE!nuA^ETlI2yQ802ErJceJ`-Q-96Wm~=%D;^Ulu zN_moki0PrE+Y(&1V%NTKK?QsOQtI7@v#Kf4wP!(xVNCMjwWp`&(A-=!pg|$6K(72e zvQ=s`Q~muo452Pb6R7KFpyOu)k7-Go3Q#UlF4sIl%$N{I*5-)WUOR_F9J+d>j>P+W zppj#9cefL8;1JUGnHoEw7RHFVk<>cT;@&(nBVqak%Ge+sN>nc2H42D_q!3s_?AhiE z-P#}+Pz7H|D=8^SX=tPfJD~tn3-JeS0D#hjF1%qeJU1cjl&^mmM$!;ahU*)m!{i&@ zvor`c17n=-3~cazteyQZ4qXyKN8)nt{@KcGozuaEcN-oD8yggKrU=+f^PQoCNW<)| zisCT8E;Qlk%ZPf#<+=>9^*PWy@U=e`pLwTq@xYPV^ZCj&zS<)rk{7x_^?}*8QE*?b5oy;CuuGuRB;0M@0~JKCP@vs+N0X~* zXoLWE=>qw1W54Gl8|iW5mdq60xDf2d{H@{Uh6~B12PJqEk%B7@c-8xEH#G((z`J>2 z;TFToRNWFfg`Fuuew@==vz116Hit7{8b$q`{u1kYi1w zt`*JMlN!|6yOV@>8gUeJ>(IFR< zMZ86m35~EK@j9DU+r#H-iyS943+3?{UHKn;$1{=M{{F7*v4Ys-WK0O2v?1s*dh+B6 zTn7fi4AIFX;5!aNTqq}-Pao{{esqfZ##G^OdW?GLaF?E8S9`A&58{Sc{BV}c0m#qaG8A&Uy*BF^j!5ow3Ui{FtKVZ2KxnUgGknjjoGlNCNO!D$N6bsEfKn~vx9 zkHo3$2!C>YLxaQ3r3=Kdst~x<@yLTf|HKtWbJrz%Y*Xq$}*G}H!!N_fY_6|Aq!W&2HLpM@`#~ zILkfUq|Lch2}tFji7Ck4B|9bU_afye~{ zx@Qv=i0Lg(8E>cJbJ_A$A~dp7TOlfr$9s=eN#xuT^q5SFHPlE;HMRBsaX( z^3~kj90@5Y0uAvtGb`&8psSUam#>nW>!ck~v8oydz>JWH2)?~b1!n!_^5CR;ZMPnB0T^a$aEZ5)aEs?EnHcGerzQf{`4tGR=ibew zW91PM5vrP+0!`YQh912;Me=p>NRZiP<{-E0pVanO%V9=lx?h4|JjfkFUH3Hmo@?;e zs(u4!qo%L_79^Bsc0ciKJbE)Nm37)GLO5v}GAac3rB2tw%0Pae$7RfnYUNVF2lgxt z76Vuw8JQ5>uPf`XqZYSZ>=0tm zZSo`pFsBV~Abs;*@gyDRv$|pdqas6YSCjmii-+BuJ|rnurYb|Y#O#zRUpbumy zIwm8V*!fjM{P7;8lxA_f^9Rh?pX!_To;v93NygZLCS(VKjw7Z$gwg$8uMQwQu$Mq$ z^bxA5VKXAbA3D;PG&k%V09rk?yE~xf-aL1FdH!fhIJV==fi;%~AD>-{q9IS&!jf0n zr=(;2XzTu81pZJ?@}nyL)_>-;eI`6!q?V5cf@1Xh$R$&R2{E52c01tw_OfHbwdc+A zmv!$`gLc`EeD{y~MQc&`@YSLoZvNe&H@daT3nn3&MTgCX|2YYnFeIhcLAj$nlpdt;;=ARcsop@N31|Cwp& z#Ib#tq$u4!X~mz_mzT*xsGULk`nFw!0%vq$!b&|kJGs`ErEpDDc#>GNc(v#~Qn7#} zTQ2!r1$IxF909(XXQUU&m98x43QioRo(01cXg)=M8hWz}49`{AOYtX7Y2gSLNKHtu zy4Knb80a!PBX=w;fl}MtfRNeq;I#fvc9B(z*{}?rQ3c$h*_P>buW(fABpg#14IP?Y z&q!X&&3d1i&2C?_Glofi5T(@cCS+|neKLEp;2X_xaH7w!b=kl)Q#Q-h@$0jOL7=Y)-G|qq*8Q^4mI_cPW*cGc z^VZEnK`z%XEVa3IQkuiBJqF#5iOa6%F+w@<)gp{Za{V0^&|waK?&r2s@JL=&Oa>-F zARR);g`VT$XUSkp#H`DOj~`5y^hf`v`KbRmtMlJ3@X&)kZFuWBSNqyXJs8)5j{KC2 z^5G!q2ZY{VBRcvUh{uUw;z(vXtnBaR(XCY{(53{}?M%tQ!M^_vV}F2cb}2(SF*_SI zKR>_dB%`)yFJ!Dy6!)1KBN$Ixi{8k+X{+S(XNcd;gkKT7L);Wf!l`KkxMHXOn!L%1 zR%&>yZ+F>ODfyL!|BH_xKLh|2UVRqO(aESmOGj7#1iZrE_yW!HuK@<42?u2`+vK&i zpj=d8lwN`C*@P$xX+?}FHR8rzY1pf0t#(mF_~hE(IR85#d_t+`8NM`3snHIs^h=f0 zZ4P(}KE7J{|6$ndzwH_5;`&eHY4QVB8lRcF7m40&4Q%j8t39~js((kDfZd9kpcat% za%rsokpt1`f>6+{4MBgXQA%ZF=_3s70`XD{y^q0%#&(L>>a;Y$7)-lg``+EXxcbxV zpL6GD^_Qj?{MYc>zVex*By7E#OGW$7nAC^ylcx^}lBN;8ly^$2bXT??_Nq(qPgYy% z5zzevlMV0V{cw?6M=4jb>&2{99V>PA69zkpP4 zz^63Zi^!{ptK);3IL*AKprO6=$xBELspjc(0$QCN47~o;Kf`NuzdpD>N}RQ2-rJg( zRUl5W)bYHfIB;RG#Us$a8;seF^htyOrfp2|z zgNJnR*Zfq>d##4yVzUtVIFisC{gqBp)KXDXXZ1gOuIv< zw^vDXrKY_%j+w7YlSSTa8CnsPzR`<@0xgq~XFI~2D?|Q^7n|8)i&dtFVIQ#GrzW7u zhmI~9ba2C9d_bn6qx(q$20Z@%UYLTs0|i4^M}~_Ye8N$|)|LYzg)fa3{bvwz_!gHl X