From 865ab0ed63abdb3f50879b75af8277177c230cee Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Thu, 6 Apr 2023 15:02:39 +0800 Subject: [PATCH] Add adapter for LD_PRELOAD, TBC. --- adapter/Makefile | 75 ++ adapter/ff_adapter.h | 21 + adapter/ff_declare_syscalls.h | 31 + adapter/ff_hook_syscall.c | 1538 +++++++++++++++++++++++++++++++++ adapter/ff_hook_syscall.h | 8 + adapter/ff_linux_syscall.c | 236 +++++ adapter/ff_linux_syscall.h | 13 + adapter/ff_so_zone.c | 170 ++++ adapter/ff_socket_ops.c | 461 ++++++++++ adapter/ff_socket_ops.h | 103 +++ adapter/ff_sysproto.h | 197 +++++ adapter/fstack.c | 36 + 12 files changed, 2889 insertions(+) create mode 100644 adapter/Makefile create mode 100644 adapter/ff_adapter.h create mode 100644 adapter/ff_declare_syscalls.h create mode 100644 adapter/ff_hook_syscall.c create mode 100644 adapter/ff_hook_syscall.h create mode 100644 adapter/ff_linux_syscall.c create mode 100644 adapter/ff_linux_syscall.h create mode 100644 adapter/ff_so_zone.c create mode 100644 adapter/ff_socket_ops.c create mode 100644 adapter/ff_socket_ops.h create mode 100644 adapter/ff_sysproto.h create mode 100644 adapter/fstack.c diff --git a/adapter/Makefile b/adapter/Makefile new file mode 100644 index 000000000..ccbc5b72b --- /dev/null +++ b/adapter/Makefile @@ -0,0 +1,75 @@ +TOPDIR=.. + +ifeq ($(FF_PATH),) + FF_PATH=${TOPDIR} +endif + +ifneq ($(shell pkg-config --exists libdpdk && echo 0),0) + $(error "No installation of DPDK found, maybe you should export environment variable `PKG_CONFIG_PATH`") +endif + +DEBUG=-O0 -gdwarf-2 -g3 -DDEBUG + +PKGCONF ?= pkg-config + +ifndef DEBUG + CFLAGS+= -g -O2 +else + CFLAGS+= ${DEBUG} +endif +CFLAGS += -fPIC -Wall -Werror $(shell $(PKGCONF) --cflags libdpdk) + +INCLUDES= -I. -I${FF_PATH}/lib + +FF_LIBS= -L${FF_PATH}/lib -Wl,--whole-archive,-lfstack,--no-whole-archive + +DPDK_LIBS+= $(shell $(PKGCONF) --static --libs libdpdk) +DPDK_LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -pthread -lnuma + +#DPDK_CFLAGS= -Wall -Werror -include ${FF_DPDK}/include/rte_config.h +#DPDK_CFLAGS+= -march=native -DRTE_MACHINE_CPUFLAG_SSE -DRTE_MACHINE_CPUFLAG_SSE2 -DRTE_MACHINE_CPUFLAG_SSE3 +#DPDK_CFLAGS+= -DRTE_MACHINE_CPUFLAG_SSSE3 -DRTE_MACHINE_CPUFLAG_SSE4_1 -DRTE_MACHINE_CPUFLAG_SSE4_2 +#DPDK_CFLAGS+= -DRTE_COMPILE_TIME_CPUFLAGS=RTE_CPUFLAG_SSE,RTE_CPUFLAG_SSE2,RTE_CPUFLAG_SSE3,RTE_CPUFLAG_SSSE3,RTE_CPUFLAG_SSE4_1,RTE_CPUFLAG_SSE4_2 +#DPDK_CFLAGS+= -I${FF_DPDK}/include + +CFLAGS+= ${INCLUDES} ${DPDK_CFLAGS} + +FSTACK_LIBS= ${FF_LIBS} ${DPDK_LIBS} +FF_SYSCALL_LIBS= + +FSTACK_SRCS= \ + fstack.c \ + ff_so_zone.c \ + ff_socket_ops.c + +FF_SYSCALL_SRCS= \ + ff_so_zone.c \ + ff_hook_syscall.c \ + ff_linux_syscall.c + +FSTACK_OBJS= $(patsubst %.c,%.o,${FSTACK_SRCS}) + +FF_SYSCALL_OBJS= $(patsubst %.c,%.o,${FF_SYSCALL_SRCS}) + +#TARGET= fstack libff_syscall.a +TARGET= fstack libff_syscall.so + +all: ${TARGET} + +fstack: ${FSTACK_OBJS} + cc -o $@ $^ ${FSTACK_LIBS} + +#libff_syscall.a: ${FF_SYSCALL_OBJS} +libff_syscall.so: ${FF_SYSCALL_OBJS} + #ar -rcs $@ $^ ${FF_SYSCALL_LIBS} + ${CC} -fPIC -shared -o $@ $^ ${FF_SYSCALL_LIBS} + +${FSTACK_OBJS}: %.o: %.c + ${CC} -c $(CFLAGS) ${PROF} $< + +${FF_SYSCALL_OBJS}: %.o: %.c + ${CC} -c $(CFLAGS) ${PROF} $< + +.PHONY: clean +clean: + rm -f *.o ${TARGET} diff --git a/adapter/ff_adapter.h b/adapter/ff_adapter.h new file mode 100644 index 000000000..39ee00cc4 --- /dev/null +++ b/adapter/ff_adapter.h @@ -0,0 +1,21 @@ +#ifndef _FF_ADAPTER_H +#define _FF_ADAPTER_H + +/* socket.h */ +//#define SOCK_CLOEXEC 0x10000000 +//#define SOCK_NONBLOCK 0x20000000 +#define SOCK_FSTACK 0x01000000 +#define SOCK_KERNEL 0x02000000 + +int ff_adapter_init(); +//int __attribute__((constructor)) ff_adapter_init(int argc, char * const argv[]); + +/*- + * Verify whether the socket is supported by fstack or not. + */ +int fstack_territory(int domain, int type, int protocol); + +/* Tell whether a 'sockfd' belongs to fstack. */ +int is_fstack_fd(int fd); + +#endif diff --git a/adapter/ff_declare_syscalls.h b/adapter/ff_declare_syscalls.h new file mode 100644 index 000000000..28d3b9808 --- /dev/null +++ b/adapter/ff_declare_syscalls.h @@ -0,0 +1,31 @@ +FF_SYSCALL_DECL(int, socket, (int, int, int)); +FF_SYSCALL_DECL(int, bind, (int, const struct sockaddr *, socklen_t)); +FF_SYSCALL_DECL(int, listen, (int, int)); +FF_SYSCALL_DECL(int, shutdown, (int, int)); +FF_SYSCALL_DECL(int, getsockname, (int, struct sockaddr *, socklen_t *)); +FF_SYSCALL_DECL(int, getpeername, (int, struct sockaddr *, socklen_t *)); +FF_SYSCALL_DECL(int, getsockopt, (int, int, int, void *, socklen_t *)); +FF_SYSCALL_DECL(int, setsockopt, (int, int, int, const void *, socklen_t)); +FF_SYSCALL_DECL(int, accept, (int, struct sockaddr *, socklen_t *)); +FF_SYSCALL_DECL(int, accept4,(int, struct sockaddr *, socklen_t *, int flags)); +FF_SYSCALL_DECL(int, connect, (int, const struct sockaddr *, socklen_t)); +FF_SYSCALL_DECL(ssize_t, recv, (int, void *, size_t, int)); +FF_SYSCALL_DECL(ssize_t, send, (int, const void *, size_t, int)); +FF_SYSCALL_DECL(ssize_t, read, (int, void *, size_t)); +FF_SYSCALL_DECL(ssize_t, write, (int, const void *, size_t)); +FF_SYSCALL_DECL(ssize_t, writev, (int, const struct iovec *, int)); +FF_SYSCALL_DECL(ssize_t, readv, (int, const struct iovec *, int)); +FF_SYSCALL_DECL(ssize_t, sendto, (int, const void *, size_t, int, + const struct sockaddr *, socklen_t)); +FF_SYSCALL_DECL(ssize_t, recvfrom, (int, void *, size_t, int, + struct sockaddr *, socklen_t *)); +FF_SYSCALL_DECL(ssize_t, sendmsg, (int, const struct msghdr *, int flags)); +FF_SYSCALL_DECL(ssize_t, recvmsg, (int, struct msghdr *, int flags)); +FF_SYSCALL_DECL(int, close, (int)); +FF_SYSCALL_DECL(int, ioctl, (int, unsigned long, unsigned long)); +FF_SYSCALL_DECL(int, fcntl, (int, int, unsigned long)); +FF_SYSCALL_DECL(int, epoll_create, (int)); +FF_SYSCALL_DECL(int, epoll_ctl, (int, int, int, struct epoll_event *)); +FF_SYSCALL_DECL(int, epoll_wait, (int, struct epoll_event *, int, int)); +FF_SYSCALL_DECL(pid_t, fork, (void)); +#undef FF_SYSCALL_DECL diff --git a/adapter/ff_hook_syscall.c b/adapter/ff_hook_syscall.c new file mode 100644 index 000000000..26bcdcd9d --- /dev/null +++ b/adapter/ff_hook_syscall.c @@ -0,0 +1,1538 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "ff_config.h" +#include "ff_socket_ops.h" +#include "ff_sysproto.h" +#include "ff_event.h" +#include "ff_hook_syscall.h" +#include "ff_linux_syscall.h" +#include "ff_adapter.h" + +#define NS_PER_SECOND 1000000000 + +#ifndef likely +#define likely(x) __builtin_expect((x),1) +#endif + +#ifndef unlikely +#define unlikely(x) __builtin_expect((x),0) +#endif + +#define strong_alias(name, aliasname) \ + extern __typeof (name) aliasname __attribute__ ((alias (#name))); + +#undef FF_SYSCALL_DECL +#define FF_SYSCALL_DECL(ret, fn, args) strong_alias(ff_hook_##fn, fn) +#include + +#define share_mem_alloc(size) rte_malloc(NULL, (size), 0) +#define share_mem_free(addr) rte_free((addr)) + +#define CHECK_FD_OWNERSHIP(name, args) \ +{ \ + if (!is_fstack_fd(fd)) { \ + return ff_linux_##name args; \ + } \ + fd = restore_fstack_fd(fd); \ +} + +#define DEFINE_REQ_ARGS(name) \ + struct ff_##name##_args *args; \ + int ret = -1; \ + size_t size = sizeof(struct ff_##name##_args); \ + args = share_mem_alloc(size); \ + if (args == NULL) { \ + errno = ENOMEM; \ + return ret; \ + } + +#define ACQUIRE_ZONE_LOCK(exp) do { \ + while (1) { \ + rte_spinlock_lock(&sc->lock); \ + if (sc->status == exp) { \ + break; \ + } \ + rte_spinlock_unlock(&sc->lock); \ + rte_pause(); \ + } \ +} while (0) + +#define RELEASE_ZONE_LOCK(s) do { \ + sc->status = s; \ + rte_spinlock_unlock(&sc->lock); \ +} while (0) + +#define SYSCALL(op, arg) do { \ + ACQUIRE_ZONE_LOCK(FF_SC_IDLE); \ + sc->ops = (op); \ + sc->args = (arg); \ + RELEASE_ZONE_LOCK(FF_SC_REQ); \ + ACQUIRE_ZONE_LOCK(FF_SC_REP); \ + ret = sc->result; \ + if (ret < 0) { \ + errno = sc->error; \ + } \ + RELEASE_ZONE_LOCK(FF_SC_IDLE); \ +} while (0) + +#define RETURN() do { \ + share_mem_free(args); \ + return ret; \ +} while (0) + +#define RETURN_ERROR(err) do { \ + errno = err; \ + share_mem_free(args); \ + return ret; \ +} while (0) + +/* per thread separate initialization dpdk lib and attach sc */ +static __thread int inited = 0; +static __thread struct ff_so_context *sc; + +/* process-level initialization flag */ +static int proc_inited = 0; + +/* Use from lcore 2 by default, can set by environment variable FF_INITIAL_LCORE_ID */ +#define INITIAL_LCORE_ID_DEFAULT 0x4 /* lcore 2 */ +#define INITIAL_LCORE_ID_MAX 0x4000000000000 /* lcore 50 */ +#define FF_INITIAL_LCORE_ID_STR "FF_INITIAL_LCORE_ID" +static uint64_t initial_lcore_id = INITIAL_LCORE_ID_DEFAULT; +static int worker_id = 0; +rte_spinlock_t worker_id_lock; + +/* The num of F-Stack process instance, default 1 */ +#define NB_FSTACK_INSTANCE_DEFAULT 1 +#define FF_NB_FSTACK_INSTANCE_STR "FF_NB_FSTACK_INSTANCE" +static int nb_procs = NB_FSTACK_INSTANCE_DEFAULT; + +#define FF_KERNEL_MAX_FD_DEFAULT 1024 +static int ff_kernel_max_fd = FF_KERNEL_MAX_FD_DEFAULT; + +static inline int convert_fstack_fd(int sockfd) { + return sockfd + ff_kernel_max_fd; +} + +/* Restore socket fd. */ +static inline int restore_fstack_fd(int sockfd) { + if(sockfd < ff_kernel_max_fd) { + return sockfd; + } + + return sockfd - ff_kernel_max_fd; +} + +int is_fstack_fd(int sockfd) { + if (unlikely(inited == 0/* && ff_adapter_init() < 0*/)) { + return 0; + } + + /* FIXED ME: ff_linux_socket not limit fd < ff_kernel_max_fd, may be Misjudgment */ + return sockfd >= ff_kernel_max_fd; +} + +int +fstack_territory(int domain, int type, int protocol) +{ + /* Remove creation flags */ + type &= ~SOCK_CLOEXEC; + type &= ~SOCK_NONBLOCK; + type &= ~SOCK_FSTACK; + type &= ~SOCK_KERNEL; + + if ((AF_INET != domain && AF_INET6 != domain) || (SOCK_STREAM != type && + SOCK_DGRAM != type)) { + return 0; + } + + return 1; +} + +/* + * APP need set type |= SOCK_FSTACK + */ +int +ff_hook_socket(int domain, int type, int protocol) +{ + DEBUG_LOG("ff_hook_socket, domain:%d, type:%d, protocol:%d\n", domain, type, protocol); + if (unlikely(fstack_territory(domain, type, protocol) == 0)) { + return ff_linux_socket(domain, type, protocol); + } + + if (unlikely(type & SOCK_KERNEL) && !(type & SOCK_FSTACK)) { + type &= ~SOCK_KERNEL; + return ff_linux_socket(domain, type, protocol); + } + + if (unlikely(inited == 0 && ff_adapter_init() < 0)) { + return ff_linux_socket(domain, type, protocol); + } + + type &= ~SOCK_FSTACK; + + DEFINE_REQ_ARGS(socket); + + args->domain = domain; + args->type = type; + args->protocol = protocol; + + SYSCALL(FF_SO_SOCKET, args); + + if (ret >= 0) { + ret = convert_fstack_fd(ret); + } + + RETURN(); +} + +int +ff_hook_bind(int fd, const struct sockaddr *addr, + socklen_t addrlen) +{ + if (addr == NULL) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(bind, (fd, addr, addrlen)); + + DEFINE_REQ_ARGS(bind); + struct sockaddr *sh_addr; + + sh_addr = share_mem_alloc(addrlen); + if (sh_addr == NULL) { + RETURN_ERROR(ENOMEM); + } + rte_memcpy(sh_addr, addr, addrlen); + + args->fd = fd; + args->addr = sh_addr; + args->addrlen = addrlen; + + SYSCALL(FF_SO_BIND, args); + + share_mem_free(sh_addr); + RETURN(); +} + +int +ff_hook_listen(int fd, int backlog) +{ + CHECK_FD_OWNERSHIP(listen, (fd, backlog)); + + DEFINE_REQ_ARGS(listen); + + args->fd = fd; + args->backlog = backlog; + + SYSCALL(FF_SO_LISTEN, args); + + RETURN(); +} + +int +ff_hook_shutdown(int fd, int how) +{ + CHECK_FD_OWNERSHIP(shutdown, (fd, how)); + + DEFINE_REQ_ARGS(shutdown); + + args->fd = fd; + args->how = how; + + SYSCALL(FF_SO_SHUTDOWN, args); + + RETURN(); +} + +int +ff_hook_getsockname(int fd, struct sockaddr *name, + socklen_t *namelen) +{ + if (name == NULL || namelen == NULL) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(getsockname, (fd, name, namelen)); + + DEFINE_REQ_ARGS(getsockname); + struct sockaddr *sh_name; + socklen_t *sh_namelen; + + sh_name = share_mem_alloc(*namelen); + if (sh_name == NULL) { + RETURN_ERROR(ENOMEM); + } + + sh_namelen = share_mem_alloc(sizeof(socklen_t)); + if (sh_namelen == NULL) { + share_mem_free(sh_name); + RETURN_ERROR(ENOMEM); + } + *sh_namelen = *namelen; + + args->fd = fd; + args->name = sh_name; + args->namelen = sh_namelen; + + SYSCALL(FF_SO_GETSOCKNAME, args); + + if (ret == 0) { + socklen_t cplen = *namelen ? *sh_namelen > *namelen + : *sh_namelen; + rte_memcpy(name, sh_name, cplen); + *namelen = *sh_namelen; + } + + share_mem_free(sh_name); + share_mem_free(sh_namelen); + RETURN(); +} + +int +ff_hook_getpeername(int fd, struct sockaddr *name, + socklen_t *namelen) +{ + if (name == NULL || namelen == NULL) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(getpeername, (fd, name, namelen)); + + DEFINE_REQ_ARGS(getpeername); + struct sockaddr *sh_name; + socklen_t *sh_namelen; + + sh_name = share_mem_alloc(*namelen); + if (sh_name == NULL) { + RETURN_ERROR(ENOMEM); + } + + sh_namelen = share_mem_alloc(sizeof(socklen_t)); + if (sh_namelen == NULL) { + share_mem_free(sh_name); + RETURN_ERROR(ENOMEM); + } + *sh_namelen = *namelen; + + args->fd = fd; + args->name = sh_name; + args->namelen = sh_namelen; + + SYSCALL(FF_SO_GETPEERNAME, args); + + if (ret == 0) { + socklen_t cplen = *namelen ? *sh_namelen > *namelen + : *sh_namelen; + rte_memcpy(name, sh_name, cplen); + *namelen = *sh_namelen; + } + + share_mem_free(sh_name); + share_mem_free(sh_namelen); + RETURN(); +} + +int +ff_hook_getsockopt(int fd, int level, int optname, + void *optval, socklen_t *optlen) +{ + if (optlen == NULL) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(getsockopt, (fd, level, optname, + optval, optlen)); + + DEFINE_REQ_ARGS(getsockopt); + void *sh_optval = NULL; + socklen_t *sh_optlen; + + if (optval != NULL) { + sh_optval = share_mem_alloc(*optlen); + if (sh_optval == NULL) { + RETURN_ERROR(ENOMEM); + } + } + + sh_optlen = share_mem_alloc(sizeof(socklen_t)); + if (sh_optlen == NULL) { + if (sh_optval) { + share_mem_free(sh_optval); + } + + RETURN_ERROR(ENOMEM); + } + *sh_optlen = *optlen; + + args->fd = fd; + args->level = level; + args->name = optname; + args->optval = sh_optval; + args->optlen = sh_optlen; + + SYSCALL(FF_SO_GETSOCKOPT, args); + + if (ret == 0) { + if (optval) { + rte_memcpy(optval, sh_optval, *sh_optlen); + share_mem_free(sh_optval); + } + *optlen = *sh_optlen; + } + + share_mem_free(sh_optlen); + RETURN(); +} + +int +ff_hook_setsockopt(int fd, int level, int optname, + const void *optval, socklen_t optlen) +{ + if (optval == NULL && optlen != 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(setsockopt, (fd, level, optname, + optval, optlen)); + + DEFINE_REQ_ARGS(setsockopt); + void *sh_optval = NULL; + + if (optval != NULL) { + sh_optval = share_mem_alloc(optlen); + if (sh_optval == NULL) { + RETURN_ERROR(ENOMEM); + } + } + + args->fd = fd; + args->level = level; + args->name = optname; + args->optval = sh_optval; + args->optlen = optlen; + + SYSCALL(FF_SO_SETSOCKOPT, args); + + if (sh_optval) { + share_mem_free(sh_optval); + } + + RETURN(); +} + +int +ff_hook_accept(int fd, struct sockaddr *addr, socklen_t *addrlen) +{ + if ((addr == NULL && addrlen != NULL) || + (addr != NULL && addrlen == NULL)) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(accept, (fd, addr, addrlen)); + + DEFINE_REQ_ARGS(accept); + struct sockaddr *sh_addr = NULL; + socklen_t *sh_addrlen = NULL; + + if (addr != NULL) { + sh_addr = share_mem_alloc(*addrlen); + if (sh_addr == NULL) { + RETURN_ERROR(ENOMEM); + } + + sh_addrlen = share_mem_alloc(sizeof(socklen_t)); + if (sh_addrlen == NULL) { + share_mem_free(sh_addr); + RETURN_ERROR(ENOMEM); + } + *sh_addrlen = *addrlen; + } + + args->fd = fd; + args->addr = sh_addr; + args->addrlen = sh_addrlen; + + SYSCALL(FF_SO_ACCEPT, args); + + if (ret > 0) { + ret = convert_fstack_fd(ret); + } + + if (addr) { + if (ret > 0) { + socklen_t cplen = *sh_addrlen > *addrlen ? + *addrlen : *sh_addrlen; + rte_memcpy(addr, sh_addr, cplen); + *addrlen = *sh_addrlen; + } + share_mem_free(sh_addr); + share_mem_free(sh_addrlen); + } + + RETURN(); +} + +int +ff_hook_accept4(int fd, struct sockaddr *addr, + socklen_t *addrlen, int flags) +{ + CHECK_FD_OWNERSHIP(accept4, (fd, addr, addrlen, flags)); + + errno = ENOSYS; + return -1; +} + +int +ff_hook_connect(int fd, const struct sockaddr *addr, + socklen_t addrlen) +{ + if (addr == NULL) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(connect, (fd, addr, addrlen)); + + DEFINE_REQ_ARGS(connect); + struct sockaddr *sh_addr; + + sh_addr = share_mem_alloc(addrlen); + if (sh_addr == NULL) { + RETURN_ERROR(ENOMEM); + } + rte_memcpy(sh_addr, addr, addrlen); + + args->fd = fd; + args->addr = sh_addr; + args->addrlen = addrlen; + + SYSCALL(FF_SO_CONNECT, args); + + share_mem_free(sh_addr); + RETURN(); +} + +ssize_t +ff_hook_recv(int fd, void *buf, size_t len, int flags) +{ + return ff_hook_recvfrom(fd, buf, len, flags, NULL, NULL); +} + +ssize_t +ff_hook_recvfrom(int fd, void *buf, size_t len, int flags, + struct sockaddr *from, socklen_t *fromlen) +{ + if (buf == NULL || len == 0) { + errno = EINVAL; + return -1; + } + + if ((from == NULL && fromlen != NULL) || + (from != NULL && fromlen == NULL)) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(recvfrom, (fd, buf, len, flags, from, fromlen)); + + DEFINE_REQ_ARGS(recvfrom); + void *sh_buf; + struct sockaddr *sh_from = NULL; + socklen_t *sh_fromlen = NULL; + + if (from != NULL) { + sh_from = share_mem_alloc(*fromlen); + if (sh_from == NULL) { + RETURN_ERROR(ENOMEM); + } + + sh_fromlen = share_mem_alloc(sizeof(socklen_t)); + if (sh_fromlen == NULL) { + share_mem_free(sh_from); + RETURN_ERROR(ENOMEM); + } + } + + sh_buf = share_mem_alloc(len); + if (sh_buf == NULL) { + if (sh_from) { + share_mem_free(sh_from); + } + if (sh_fromlen) { + share_mem_free(sh_fromlen); + } + RETURN_ERROR(ENOMEM); + } + + args->fd = fd; + args->buf = sh_buf; + args->len = len; + args->flags = flags; + args->from = sh_from; + args->fromlen = sh_fromlen; + + SYSCALL(FF_SO_RECVFROM, args); + + if (ret >= 0) { + rte_memcpy(buf, sh_buf, ret); + if (from) { + socklen_t cplen = *fromlen ? *sh_fromlen > *fromlen + : *sh_fromlen; + rte_memcpy(from, sh_from, cplen); + *fromlen = *sh_fromlen; + } + } + + if (from) { + share_mem_free(sh_from); + share_mem_free(sh_fromlen); + } + + share_mem_free(sh_buf); + RETURN(); +} + +static void +iovec_share_free(struct iovec *iov, int iovcnt) +{ + int i; + + if (iov == NULL) { + return; + } + + for (i = 0; i < iovcnt; i++) { + if (iov[i].iov_base) { + share_mem_free(iov[i].iov_base); + } + } + + share_mem_free(iov); +} + +static struct iovec * +iovec_share_alloc(const struct iovec *iov, int iovcnt) +{ + struct iovec *sh_iov; + int i; + + if (iov == NULL || iovcnt == 0) { + return NULL; + } + + sh_iov = share_mem_alloc(sizeof(struct iovec) * iovcnt); + if (sh_iov == NULL) { + return NULL; + } + + for (i = 0; i < iovcnt; i++) { + sh_iov[i].iov_len = iov[i].iov_len; + void *iov_base = share_mem_alloc(sh_iov[i].iov_len); + sh_iov[i].iov_base = iov_base; + + if (iov_base == NULL) { + goto ERROR; + } + } + + return sh_iov; + +ERROR: + iovec_share_free(sh_iov, iovcnt); + return NULL; +} + +static void +iovec_local2share(struct iovec *share, const struct iovec *local, + int iovcnt) +{ + int i; + + if (share == NULL || local == NULL || iovcnt == 0) { + return; + } + + for (i = 0; i < iovcnt; i++) { + assert(share[i].iov_len == local[i].iov_len); + + rte_memcpy(share[i].iov_base, local[i].iov_base, + share[i].iov_len); + } +} + +static void +iovec_share2local(struct iovec *share, + const struct iovec *local, int iovcnt, + ssize_t total, int copy) +{ + int i; + for (i = 0; i < iovcnt && total > 0; i++) { + ssize_t count = local[i].iov_len; + if (total <= count) { + count = total; + } + + share[i].iov_base = + (char *)share[i].iov_base - count; + share[i].iov_len += count; + + if (copy) { + rte_memcpy(local[i].iov_base, + share[i].iov_base, count); + } + + total -= count; + } +} + +static void +msghdr_share_free(struct msghdr *msg) +{ + if (msg == NULL) { + return; + } + + if (msg->msg_name) { + share_mem_free(msg->msg_name); + } + + if (msg->msg_control) { + share_mem_free(msg->msg_control); + } + + if (msg->msg_iov) { + iovec_share_free(msg->msg_iov, msg->msg_iovlen); + } + + share_mem_free(msg); +} + +static struct msghdr * +msghdr_share_alloc(const struct msghdr *msg) +{ + struct msghdr *hdr; + + if (msg == NULL) { + return NULL; + } + + hdr = share_mem_alloc(sizeof(struct msghdr)); + if (hdr == NULL) { + return NULL; + } + memset(hdr, 0, sizeof(struct msghdr)); + + hdr->msg_namelen = msg->msg_namelen; + hdr->msg_iovlen = msg->msg_iovlen; + hdr->msg_controllen = msg->msg_controllen; + hdr->msg_flags = msg->msg_flags; + + if (msg->msg_name) { + hdr->msg_name = share_mem_alloc(hdr->msg_namelen); + if (hdr->msg_name == NULL) { + goto ERROR; + } + } + + if (msg->msg_control) { + hdr->msg_control = share_mem_alloc(hdr->msg_controllen); + if (hdr->msg_control == NULL) { + goto ERROR; + } + } + + hdr->msg_iov = iovec_share_alloc(msg->msg_iov, hdr->msg_iovlen); + if (hdr->msg_iov == NULL) { + goto ERROR; + } + + return hdr; + +ERROR: + msghdr_share_free(hdr); + return NULL; +} + +static void +msghdr_share_memcpy(const struct msghdr *dst, const struct msghdr *src) +{ + if (dst == NULL || src == NULL) { + return; + } + + assert((dst->msg_name == NULL && src->msg_name == NULL) + || (dst->msg_name != NULL && src->msg_name != NULL)); + assert(dst->msg_namelen == src->msg_namelen); + + assert((dst->msg_control == NULL && src->msg_control == NULL) + || (dst->msg_control != NULL && src->msg_control != NULL)); + assert(dst->msg_controllen == src->msg_controllen); + + if (dst->msg_name) { + rte_memcpy(dst->msg_name, src->msg_name, src->msg_namelen); + } + + if (dst->msg_control) { + rte_memcpy(dst->msg_control, src->msg_control, + src->msg_controllen); + } + + //do iovec_memcpy by caller. +} + +ssize_t +ff_hook_recvmsg(int fd, struct msghdr *msg, int flags) +{ + if (msg == NULL || msg->msg_iov == NULL || + msg->msg_iovlen == 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(recvmsg, (fd, msg, flags)); + + DEFINE_REQ_ARGS(recvmsg); + struct msghdr *sh_msg; + + sh_msg = msghdr_share_alloc(msg); + if (sh_msg == NULL) { + RETURN_ERROR(ENOMEM); + } + + args->fd = fd; + args->msg = sh_msg; + args->flags = flags; + + SYSCALL(FF_SO_RECVMSG, args); + + if (ret >= 0) { + msghdr_share_memcpy(msg, sh_msg); + if (ret > 0) { + iovec_share2local(sh_msg->msg_iov, + msg->msg_iov, msg->msg_iovlen, + ret, 1); + } + } + + msghdr_share_free(sh_msg); + RETURN(); +} + +ssize_t +ff_hook_read(int fd, void *buf, size_t len) +{ + if (buf == NULL || len == 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(read, (fd, buf, len)); + + DEFINE_REQ_ARGS(read); + void *sh_buf; + + sh_buf = share_mem_alloc(len); + if (sh_buf == NULL) { + RETURN_ERROR(ENOMEM); + } + + args->fd = fd; + args->buf = sh_buf; + args->len = len; + + SYSCALL(FF_SO_READ, args); + + if (ret > 0) { + rte_memcpy(buf, sh_buf, ret); + } + + share_mem_free(sh_buf); + RETURN(); +} + +ssize_t +ff_hook_readv(int fd, const struct iovec *iov, int iovcnt) +{ + if (iov == NULL || iovcnt == 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(readv, (fd, iov, iovcnt)); + + DEFINE_REQ_ARGS(readv); + struct iovec *sh_iov; + + sh_iov = iovec_share_alloc(iov, iovcnt); + if (sh_iov == NULL) { + RETURN_ERROR(ENOMEM); + } + + args->fd = fd; + args->iov = sh_iov; + args->iovcnt = iovcnt; + + SYSCALL(FF_SO_READV, args); + + if (ret > 0) { + iovec_share2local(sh_iov, iov, iovcnt, ret, 1); + } + + iovec_share_free(sh_iov, iovcnt); + RETURN(); +} + +ssize_t +ff_hook_sendto(int fd, const void *buf, size_t len, int flags, + const struct sockaddr *to, socklen_t tolen) +{ + if (buf == NULL || len == 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(sendto, (fd, buf, len, flags, to, tolen)); + + DEFINE_REQ_ARGS(sendto); + void *sh_buf; + void *sh_to = NULL; + + sh_buf = share_mem_alloc(len); + if (sh_buf == NULL) { + RETURN_ERROR(ENOMEM); + } + rte_memcpy(sh_buf, buf, len); + + if (to) { + sh_to = share_mem_alloc(tolen); + if (sh_to == NULL) { + share_mem_free(sh_buf); + RETURN_ERROR(ENOMEM); + } + rte_memcpy(sh_to, to, tolen); + } + + args->fd = fd; + args->buf = sh_buf; + args->len = len; + args->flags = flags; + args->to = sh_to; + args->tolen = tolen; + + SYSCALL(FF_SO_SENDTO, args); + + share_mem_free(sh_buf); + if (sh_to) { + share_mem_free(sh_to); + } + RETURN(); +} + +ssize_t +ff_hook_sendmsg(int fd, const struct msghdr *msg, int flags) +{ + if (msg == NULL || msg->msg_iov == NULL || + msg->msg_iovlen == 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(sendmsg, (fd, msg, flags)); + + DEFINE_REQ_ARGS(sendmsg); + struct msghdr *sh_msg; + + sh_msg = msghdr_share_alloc(msg); + if (sh_msg == NULL) { + RETURN_ERROR(ENOMEM); + } + msghdr_share_memcpy(sh_msg, msg); + iovec_local2share(sh_msg->msg_iov, + msg->msg_iov, msg->msg_iovlen); + + args->fd = fd; + args->msg = sh_msg; + args->flags = flags; + + SYSCALL(FF_SO_SENDMSG, args); + + if (ret > 0) { + iovec_share2local(sh_msg->msg_iov, + msg->msg_iov, msg->msg_iovlen, + ret, 0); + } + + msghdr_share_free(sh_msg); + RETURN(); +} + +ssize_t +ff_hook_send(int fd, const void *buf, size_t len, int flags) +{ + return ff_hook_sendto(fd, buf, len, flags, NULL, 0); +} + +ssize_t +ff_hook_write(int fd, const void *buf, size_t len) +{ + if (buf == NULL || len == 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(write, (fd, buf, len)); + + DEFINE_REQ_ARGS(write); + void *sh_buf; + + sh_buf = share_mem_alloc(len); + if (sh_buf == NULL) { + RETURN_ERROR(ENOMEM); + } + rte_memcpy(sh_buf, buf, len); + + args->fd = fd; + args->buf = sh_buf; + args->len = len; + + SYSCALL(FF_SO_WRITE, args); + + share_mem_free(sh_buf); + RETURN(); +} + +ssize_t +ff_hook_writev(int fd, const struct iovec *iov, int iovcnt) +{ + if (iov == NULL || iovcnt == 0) { + errno = EINVAL; + return -1; + } + + CHECK_FD_OWNERSHIP(writev, (fd, iov, iovcnt)); + + DEFINE_REQ_ARGS(writev); + struct iovec *sh_iov; + + sh_iov = iovec_share_alloc(iov, iovcnt); + if (sh_iov == NULL) { + RETURN_ERROR(ENOMEM); + } + iovec_local2share(sh_iov, iov, iovcnt); + + args->fd = fd; + args->iov = sh_iov; + args->iovcnt = iovcnt; + + SYSCALL(FF_SO_WRITEV, args); + + if (ret > 0) { + iovec_share2local(sh_iov, iov, iovcnt, ret, 0); + } + + iovec_share_free(sh_iov, iovcnt); + RETURN(); +} + +int +ff_hook_close(int fd) +{ + CHECK_FD_OWNERSHIP(close, (fd)); + + DEFINE_REQ_ARGS(close); + + args->fd = fd; + + SYSCALL(FF_SO_CLOSE, args); + + RETURN(); +} + +int +ff_hook_ioctl(int fd, unsigned long req, unsigned long data) +{ + #ifndef FIOASYNC + #define FIOASYNC 0x5452 + #endif + #ifndef FIONBIO + #define FIONBIO 0x5421 + #endif + + if (req != FIOASYNC && req != FIONBIO) { + errno = ENOTSUP; + return -1; + } + + CHECK_FD_OWNERSHIP(ioctl, (fd, req, data)); + + DEFINE_REQ_ARGS(ioctl); + unsigned long *sh_data; + + sh_data = share_mem_alloc(sizeof(int)); + if (sh_data == NULL) { + RETURN_ERROR(ENOMEM); + } + *sh_data = *((int *)data); + + args->fd = fd; + args->com = req; + args->data = sh_data; + + SYSCALL(FF_SO_IOCTL, args); + + if (ret == 0) { + *((int *)data) = *sh_data; + } + + RETURN(); +} + +int +ff_hook_fcntl(int fd, int cmd, unsigned long data) +{ + CHECK_FD_OWNERSHIP(fcntl, (fd, cmd, data)); + + DEFINE_REQ_ARGS(fcntl); + + args->fd = fd; + args->cmd = cmd; + args->data = data; + + SYSCALL(FF_SO_FCNTL, args); + + RETURN(); +} + +/* + * Use F-Stack stack by default. + * + * If fdsize set SOCK_KERNEL(0x01000000) and not set SOCK_FSTACK(0x02000000), means use kernel stack. + * And the max fdsize shoud be <= (SOCK_KERNEL - 1). + * + * If fdsize set [1, 16], means use kernel stack, need to consider a better implementation. + */ +int +ff_hook_epoll_create(int fdsize) +{ + DEBUG_LOG("ff_hook_epoll_create, fdsize:%d\n", fdsize); + if (inited == 0 || ((fdsize & SOCK_KERNEL) && !(fdsize & SOCK_FSTACK)) || (fdsize >= 1 && fdsize <= 16)) { + fdsize &= ~SOCK_KERNEL; + return ff_linux_epoll_create(fdsize); + } + + DEFINE_REQ_ARGS(epoll_create); + + args->size = size; + + SYSCALL(FF_SO_EPOLL_CREATE, args); + + if (ret >= 0) { + ret = convert_fstack_fd(ret); + } + + RETURN(); +} + +int +ff_hook_epoll_ctl(int epfd, int op, int fd, + struct epoll_event *event) +{ + DEBUG_LOG("ff_hook_epoll_ctl, epfd:%d, op:%d, fd:%d\n", epfd, op, fd); + CHECK_FD_OWNERSHIP(epoll_ctl, (epfd, op, fd, event)); + + DEFINE_REQ_ARGS(epoll_ctl); + struct epoll_event *sh_event = NULL; + + if ((!event && op != EPOLL_CTL_DEL) || + (op != EPOLL_CTL_ADD && + op != EPOLL_CTL_MOD && + op != EPOLL_CTL_DEL)) { + errno = EINVAL; + return -1; + } + + if (event) { + sh_event = share_mem_alloc(sizeof(struct epoll_event)); + if (sh_event == NULL) { + RETURN_ERROR(ENOMEM); + } + rte_memcpy(sh_event, event, sizeof(struct epoll_event)); + } + + args->epfd = restore_fstack_fd(epfd); + args->op = op; + args->fd = fd; + args->event = sh_event; + + SYSCALL(FF_SO_EPOLL_CTL, args); + + if (sh_event) { + share_mem_free(sh_event); + } + + RETURN(); +} + +int +ff_hook_epoll_wait(int epfd, struct epoll_event *events, + int maxevents, int timeout) +{ + //DEBUG_LOG("ff_hook_epoll_wait, epfd:%d, maxevents:%d, timeout:%d\n", epfd, maxevents, timeout); + int fd = epfd; + CHECK_FD_OWNERSHIP(epoll_wait, (epfd, events, maxevents, timeout)); + + DEFINE_REQ_ARGS(epoll_wait); + struct epoll_event *sh_events; + + sh_events = share_mem_alloc(sizeof(struct epoll_event) * maxevents); + if (sh_events == NULL) { + RETURN_ERROR(ENOMEM); + } + + args->epfd = fd; + args->events = sh_events; + args->maxevents = maxevents; + args->timeout = timeout; + + /* for timeout, Although not really effective in FreeBSD stack */ + //SYSCALL(FF_SO_EPOLL_WAIT, args); + ACQUIRE_ZONE_LOCK(FF_SC_IDLE); + sc->ops = FF_SO_EPOLL_WAIT; + sc->args = args; + RELEASE_ZONE_LOCK(FF_SC_REQ); + + if (timeout > 0) { + struct timespec abs_timeout; + + clock_gettime(CLOCK_REALTIME, &abs_timeout); + DEBUG_LOG("before wait, sec:%ld, nsec:%ld\n", abs_timeout.tv_sec, abs_timeout.tv_nsec); + abs_timeout.tv_sec += timeout / 1000; + abs_timeout.tv_nsec += timeout * 1000; + if (abs_timeout.tv_nsec > NS_PER_SECOND) { + abs_timeout.tv_nsec -= NS_PER_SECOND; + abs_timeout.tv_sec += 1; + } + + DEBUG_LOG("ready to wait, sec:%ld, nsec:%ld\n", abs_timeout.tv_sec, abs_timeout.tv_nsec); + ret = sem_timedwait(&sc->wait_sem, &abs_timeout); + + clock_gettime(CLOCK_REALTIME, &abs_timeout); + DEBUG_LOG("after wait, sec:%ld, nsec:%ld\n", abs_timeout.tv_sec, abs_timeout.tv_nsec); + } else { + ret = sem_wait(&sc->wait_sem); + } + + rte_spinlock_lock(&sc->lock); + + if (ret == -1 && sc->status == FF_SC_REQ) { + if (errno == ETIMEDOUT) { + ret = 0; + } + } else { + ret = sc->result; + if (ret < 0) { + errno = sc->error; + } + } + + sc->status = FF_SC_IDLE; + rte_spinlock_unlock(&sc->lock); + + if (ret > 0) { + int i; + for (i = 0; i < ret; i++) { + rte_memcpy(&events[i], &sh_events[i], sizeof(struct epoll_event)); + } + } + + if (sh_events) { + share_mem_free(sh_events); + } + + RETURN(); +} + +pid_t +ff_hook_fork(void) +{ + return ff_linux_fork(); +} + +int +kqueue() +{ + int ret = -1; + + if (unlikely(inited == 0)) { + errno = ENOSYS; + return -1; + } + + SYSCALL(FF_SO_KQUEUE, NULL); + + if (ret >= 0) { + ret = convert_fstack_fd(ret); + } + + return ret; +} + +int +kevent(int kq, const struct kevent *changelist, int nchanges, + struct kevent *eventlist, int nevents, + const struct timespec *timeout) +{ + if (unlikely(inited == 0 && ff_adapter_init() < 0)) { + errno = ENOSYS; + return -1; + } + + kq = restore_fstack_fd(kq); + + DEFINE_REQ_ARGS(kevent); + struct kevent *sh_changelist = NULL; + struct kevent *sh_eventlist = NULL; + + if (changelist != NULL && nchanges > 0) { + sh_changelist = share_mem_alloc(sizeof(struct kevent) * nchanges); + if (sh_changelist == NULL) { + RETURN_ERROR(ENOMEM); + } + + rte_memcpy(sh_changelist, changelist, sizeof(struct kevent) * nchanges); + + struct kevent *kev; + int i = 0; + for(i = 0; i < nchanges; i++) { + kev = (struct kevent *)&sh_changelist[i]; + switch (kev->filter) { + case EVFILT_READ: + case EVFILT_WRITE: + case EVFILT_VNODE: + kev->ident = restore_fstack_fd(kev->ident); + break; + default: + break; + } + } + } + + if (eventlist != NULL && nevents > 0) { + sh_eventlist = share_mem_alloc(sizeof(struct kevent) * nevents); + if (sh_eventlist == NULL) { + share_mem_free(sh_changelist); + RETURN_ERROR(ENOMEM); + } + } + + args->kq = kq; + args->changelist = sh_changelist; + args->nchanges = nchanges; + args->eventlist = sh_eventlist; + args->nevents = nevents; + args->timeout = NULL; + + rte_spinlock_lock(&sc->lock); + + sc->ops = FF_SO_KEVENT; + sc->args = args; + sc->status = FF_SC_REQ; + + rte_spinlock_unlock(&sc->lock); + + if (timeout != NULL) { + struct timespec abs_timeout; + + clock_gettime(CLOCK_REALTIME, &abs_timeout); + abs_timeout.tv_sec += timeout->tv_sec; + abs_timeout.tv_nsec += timeout->tv_nsec; + if (abs_timeout.tv_nsec > NS_PER_SECOND) { + abs_timeout.tv_nsec -= NS_PER_SECOND; + abs_timeout.tv_sec += 1; + } + + ret = sem_timedwait(&sc->wait_sem, &abs_timeout); + } else { + ret = sem_wait(&sc->wait_sem); + } + + rte_spinlock_lock(&sc->lock); + + if (ret == -1 && sc->status == FF_SC_REQ) { + if (errno == ETIMEDOUT) { + ret = 0; + } + } else { + ret = sc->result; + if (ret < 0) { + errno = sc->error; + } + } + + sc->status = FF_SC_IDLE; + + rte_spinlock_unlock(&sc->lock); + + if (ret > 0) { + if (eventlist && nevents) { + rte_memcpy(eventlist, sh_eventlist, + sizeof(struct kevent) * ret); + } + } + + if (sh_changelist) { + share_mem_free(sh_changelist); + } + + if (sh_eventlist) { + share_mem_free(sh_eventlist); + } + + RETURN(); +} + +int +ff_adapter_init() +//int __attribute__((constructor)) +//ff_adapter_init(int argc, char * const argv[]) +{ + int ret; + + if (inited) { + return 0; + } + + if (proc_inited == 0) { + rte_spinlock_init(&worker_id_lock); + rte_spinlock_lock(&worker_id_lock); + + /* + * get ulimit -n to distinguish fd between kernel and F-Stack + */ + struct rlimit rlmt; + ret = getrlimit(RLIMIT_NOFILE, &rlmt); + if (ret < 0) { + ERR_LOG("getrlimit(RLIMIT_NOFILE) failed, use default ff_kernel_max_fd:%d\n", ff_kernel_max_fd); + return -1; + } else { + ff_kernel_max_fd = (int)rlmt.rlim_cur; + } + ERR_LOG("getrlimit(RLIMIT_NOFILE) successed, sed ff_kernel_max_fd:%d, and rlim_max is %ld\n", + ff_kernel_max_fd, rlmt.rlim_max); + + /* + * Get environment variable FF_INITIAL_LCORE_ID to set initial_lcore_id + * + * If need later, modify to get config from config file, + * it can consider multiplex F-Stack config.ini + */ + char *ff_init_lcore_id = getenv(FF_INITIAL_LCORE_ID_STR); + if (ff_init_lcore_id != NULL) { + char *pos = strchr(ff_init_lcore_id, '='); + if (pos != NULL) { + initial_lcore_id = (uint64_t)strtoull(pos, NULL, 16); + if (initial_lcore_id > ((uint64_t)INITIAL_LCORE_ID_MAX) /*== UINT64_MAX*/) { + initial_lcore_id = INITIAL_LCORE_ID_DEFAULT; + ERR_LOG("get invalid FF_INITIAL_LCORE_ID=%s, to use default value 0x%0lx\n", + ff_init_lcore_id, initial_lcore_id); + } + } + ERR_LOG("get FF_INITIAL_LCORE_ID=%s, use 0x%0lx\n", + ff_init_lcore_id, initial_lcore_id); + } + else { + ERR_LOG("environment variable FF_INITIAL_LCORE_ID not found, to use default value 0x%0lx\n", + initial_lcore_id); + } + + + /* + * Get environment variable FF_NB_FSTACK_INSTANCE to set nb_procs. + */ + char *ff_nb_procs = getenv(FF_NB_FSTACK_INSTANCE_STR); + if (ff_nb_procs != NULL) { + char *pos = strchr(ff_nb_procs, '='); + if (pos != NULL) { + nb_procs = (uint32_t)strtoul(pos, NULL, 10); + if (nb_procs == -1 /*UINT32_MAX*/) { + nb_procs = NB_FSTACK_INSTANCE_DEFAULT; + ERR_LOG("get invalid FF_NB_FSTACK_INSTANCE=%s, to use default value %d\n", + ff_nb_procs, nb_procs); + } + } + ERR_LOG("get FF_NB_FSTACK_INSTANCE=%s, use %d\n", + ff_nb_procs, nb_procs); + } + else { + ERR_LOG("environment variable FF_NB_FSTACK_INSTANCE not found, to use default value %d\n", + nb_procs); + } + + if (proc_inited == 0) { + proc_inited = 1; + } + } else { + rte_spinlock_lock(&worker_id_lock); + } + + char buf[RTE_MAX_LCORE] = {0}; + sprintf(buf, "-c%lx", initial_lcore_id << worker_id); + + char *dpdk_argv[] = { + "ff-adapter", buf, "-n4", + "--proc-type=secondary", + /* RTE_LOG_WARNING */ + "--log-level=5", + }; + + printf("\n"); + DEBUG_LOG("rte_eal_init, argc:%ld/%ld=%ld\n", sizeof(dpdk_argv), sizeof(dpdk_argv[0]), sizeof(dpdk_argv)/sizeof(dpdk_argv[0])); + for (int i=0; i < sizeof(dpdk_argv)/sizeof(dpdk_argv[0]); i++) { + printf("%s ", dpdk_argv[i]); + } + printf("\n"); + ret = rte_eal_init(sizeof(dpdk_argv)/sizeof(dpdk_argv[0]), + dpdk_argv); + DEBUG_LOG("rte_eal_init ret:%d\n", ret); + if (ret < 0) { + ERR_LOG("ff_adapter_init failed with EAL initialization\n"); + return ret; + } + + sc = ff_attach_so_context(worker_id % nb_procs); + if (sc == NULL) { + ERR_LOG("ff_attach_so_context failed\n"); + return -1; + } + + worker_id++; + inited = 1; + + rte_spinlock_unlock(&worker_id_lock); + + ERR_LOG("ff_adapter_init success, sc:%p, status:%d, ops:%d\n", sc, sc->status, sc->ops); + + return 0; +} + +void __attribute__((destructor)) +ff_adapter_exit() +{ + ff_detach_so_context(sc); + ERR_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc); +} diff --git a/adapter/ff_hook_syscall.h b/adapter/ff_hook_syscall.h new file mode 100644 index 000000000..a88a28544 --- /dev/null +++ b/adapter/ff_hook_syscall.h @@ -0,0 +1,8 @@ +#ifndef _FF_HOOK_SYSCALL_H +#define _FF_HOOK_SYSCALL_H + +#undef FF_SYSCALL_DECL +#define FF_SYSCALL_DECL(ret, fn, args) extern ret ff_hook_##fn args +#include + +#endif diff --git a/adapter/ff_linux_syscall.c b/adapter/ff_linux_syscall.c new file mode 100644 index 000000000..88614abd4 --- /dev/null +++ b/adapter/ff_linux_syscall.c @@ -0,0 +1,236 @@ +#include +#include +#include +#include + +#include "ff_socket_ops.h" +#include "ff_linux_syscall.h" + +#define SYSCALL(symbol, para) { \ + if (linux_syscall_inited == 0) { \ + if (linux_syscall_init() != 0) { \ + return -1; \ + } \ + } \ + if (syscalls.pf_##symbol) { \ + return syscalls.pf_##symbol para; \ + } \ + errno = ENOSYS; \ + return -1; \ +} + +struct ff_linux_syscall { + #define FF_SYSCALL_DECL(ret, fn, args) ret (*pf_##fn) args; + #include "ff_declare_syscalls.h" +}; + +static int linux_syscall_inited; + +static struct ff_linux_syscall syscalls = { 0 }; + +static void *linux_lib_handle = NULL; + +pthread_mutex_t syscall_init_mutex = PTHREAD_MUTEX_INITIALIZER; + +static inline int +linux_syscall_load_symbol() +{ + linux_lib_handle = dlopen ("libc.so.6", RTLD_NOW | RTLD_GLOBAL); + if (linux_lib_handle == NULL) { + ERR_LOG("cannot dlopen libc.so.6] err_string=%s", dlerror()); + return -1; + } + + #define FF_SYSCALL_DECL(ret, fn, args) \ + syscalls.pf_##fn = (typeof(syscalls.pf_##fn))dlsym(linux_lib_handle, #fn); + #include + + return 0; +} + +static inline int +linux_syscall_init() +{ + if (linux_syscall_inited) { + return 0; + } + + pthread_mutex_lock(&syscall_init_mutex); + if (linux_syscall_inited) { + pthread_mutex_unlock(&syscall_init_mutex); + return 0; + } + + if (linux_syscall_load_symbol() != 0) { + pthread_mutex_unlock(&syscall_init_mutex); + return -1; + } + + linux_syscall_inited = 1; + + pthread_mutex_unlock(&syscall_init_mutex); + + return 0; +} + +int +ff_linux_socket(int domain, int type, int protocol) +{ + DEBUG_LOG("ff_linux_socket, domain:%d, type:%d, protocol:%d\n", domain, type, protocol); + SYSCALL(socket, (domain, type, protocol)); +} + +int +ff_linux_bind(int s, const struct sockaddr *addr, + socklen_t addrlen) +{ + SYSCALL(bind, (s, addr, addrlen)); +} + +int +ff_linux_listen(int s, int backlog) +{ + SYSCALL(listen, (s, backlog)); +} + +int ff_linux_shutdown(int s, int how) +{ + SYSCALL(shutdown, (s, how)); +} + +int ff_linux_getsockname(int s, struct sockaddr *name, + socklen_t *namelen) +{ + SYSCALL(getsockname, (s, name, namelen)); +} + +int ff_linux_getpeername(int s, struct sockaddr *name, + socklen_t *namelen) +{ + SYSCALL(getpeername, (s, name, namelen)); +} + +int ff_linux_getsockopt(int s, int level, int optname, + void *optval, socklen_t *optlen) +{ + SYSCALL(getsockopt, (s, level, optname, optval, optlen)); +} + +int ff_linux_setsockopt(int s, int level, int optname, + const void *optval, socklen_t optlen) +{ + SYSCALL(setsockopt, (s, level, optname, optval, optlen)); +} + +int ff_linux_accept(int s, struct sockaddr *addr, socklen_t *addrlen) +{ + SYSCALL(accept, (s, addr, addrlen)); +} + +int ff_linux_accept4(int s, struct sockaddr *addr, + socklen_t *addrlen, int flags) +{ + SYSCALL(accept4, (s, addr, addrlen, flags)); +} + +int ff_linux_connect(int s, const struct sockaddr *addr, + socklen_t addrlen) +{ + SYSCALL(connect, (s, addr, addrlen)); +} + +ssize_t ff_linux_recv(int s, void *buf, size_t len, int flags) +{ + SYSCALL(recv, (s, buf, len, flags)); +} + +ssize_t ff_linux_send(int s, const void *buf, size_t len, int flags) +{ + SYSCALL(send, (s, buf, len, flags)); +} + +ssize_t ff_linux_read(int s, void *buf, size_t len) +{ + SYSCALL(read, (s, buf, len)); +} + +ssize_t ff_linux_write(int s, const void *buf, size_t len) +{ + SYSCALL(write, (s, buf, len)); +} + +ssize_t ff_linux_writev(int s, const struct iovec *iov, int iovcnt) +{ + SYSCALL(writev, (s, iov, iovcnt)); +} + +ssize_t ff_linux_readv(int s, const struct iovec *iov, int iovcnt) +{ + SYSCALL(readv, (s, iov, iovcnt)); +} + +ssize_t ff_linux_sendto(int s, const void *buf, size_t len, int flags, + const struct sockaddr *to, socklen_t tolen) +{ + SYSCALL(sendto, (s, buf, len, flags, to, tolen)); +} + +ssize_t ff_linux_recvfrom(int s, void *buf, size_t len, int flags, + struct sockaddr *from, socklen_t *fromlen) + +{ + SYSCALL(recvfrom, (s, buf, len, flags, from, fromlen)); +} + +ssize_t ff_linux_sendmsg(int s, const struct msghdr *msg, int flags) +{ + SYSCALL(sendmsg, (s, msg, flags)); +} + +ssize_t ff_linux_recvmsg(int s, struct msghdr *msg, int flags) +{ + SYSCALL(recvmsg, (s, msg, flags)) +} + +int ff_linux_close(int s) +{ + SYSCALL(close, (s)); +} + +int ff_linux_ioctl(int s, unsigned long req, unsigned long data) +{ + SYSCALL(ioctl, (s, req, data)); +} + +int ff_linux_fcntl(int s, int cmd, unsigned long data) +{ + SYSCALL(fcntl, (s, cmd, data)); +} + +int ff_linux_epoll_create(int size) +{ + DEBUG_LOG("ff_linux_epoll_create, fdsize:%d\n", size); + SYSCALL(epoll_create, (size)); +} + +int ff_linux_epoll_ctl(int epfd, int op, int fd, + struct epoll_event *event) +{ + DEBUG_LOG("ff_linux_epoll_ctl, epfd:%d, op:%d, fd:%d\n", epfd, op, fd); + SYSCALL(epoll_ctl, (epfd, op, fd, event)); +} + +int +ff_linux_epoll_wait(int epfd, struct epoll_event *events, + int maxevents, int timeout) +{ + DEBUG_LOG("ff_linux_epoll_wait, epfd:%d, maxevents:%d, timeout:%d\n", epfd, maxevents, timeout); + SYSCALL(epoll_wait, (epfd, events, maxevents, timeout)); +} + +pid_t +ff_linux_fork(void) +{ + SYSCALL(fork, ()); +} + diff --git a/adapter/ff_linux_syscall.h b/adapter/ff_linux_syscall.h new file mode 100644 index 000000000..c6fc96582 --- /dev/null +++ b/adapter/ff_linux_syscall.h @@ -0,0 +1,13 @@ +#ifndef _FF_LINUX_SYSCALL_H_ +#define _FF_LINUX_SYSCALL_H_ + +#include +#include +#include +#include + +#undef FF_SYSCALL_DECL +#define FF_SYSCALL_DECL(ret, fn, args) ret ff_linux_##fn args +#include "ff_declare_syscalls.h" + +#endif diff --git a/adapter/ff_so_zone.c b/adapter/ff_so_zone.c new file mode 100644 index 000000000..2ecfe3a9b --- /dev/null +++ b/adapter/ff_so_zone.c @@ -0,0 +1,170 @@ +#include +#include + +#include +#include + +#include "ff_config.h" +#include "ff_socket_ops.h" + +#define SOCKET_OPS_ZONE_NAME "ff_socket_ops_zone_%d" + +#define SOCKET_OPS_CONTEXT_MAX_NUM 32 + +#define SOCKET_OPS_CONTEXT_NAME_SIZE 32 +#define SOCKET_OPS_CONTEXT_NAME "ff_so_context_" + +static uint16_t ff_max_so_context = SOCKET_OPS_CONTEXT_MAX_NUM; +struct ff_socket_ops_zone *ff_so_zone; + +int +ff_set_max_so_context(uint16_t count) +{ + if (ff_so_zone) { + ERR_LOG("Can not set: memzone has inited\n"); + return -1; + } + + /*if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + ERR_LOG("Can not set: process is not primary\n"); + return 1; + }*/ + + ff_max_so_context = count; + + return 0; +} + +int +ff_create_so_memzone() +{ + if (ff_so_zone) { + ERR_LOG("Can not create memzone: memzone has inited\n"); + return -1; + } + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + uint16_t i, proc_id; + for (proc_id = 0; proc_id < ff_global_cfg.dpdk.nb_procs; proc_id++) { + struct ff_socket_ops_zone *so_zone_tmp; + const struct rte_memzone *mz; + char zn[64]; + + size_t zone_size = sizeof(struct ff_socket_ops_zone) + + sizeof(struct ff_so_context) * ff_max_so_context; + snprintf(zn, sizeof(zn), SOCKET_OPS_ZONE_NAME, proc_id); + ERR_LOG("To create memzone:%s\n", zn); + + mz = rte_memzone_reserve(zn, zone_size, rte_socket_id(), 0); + if (mz == NULL) { + ERR_LOG("Cannot reserve memory zone:%s\n", zn); + return -1; + } + + memset(mz->addr, 0, zone_size); + so_zone_tmp = mz->addr; + + rte_spinlock_init(&so_zone_tmp->lock); + so_zone_tmp->count = ff_max_so_context; + so_zone_tmp->free = so_zone_tmp->count; + so_zone_tmp->sc = (struct ff_so_context *)(so_zone_tmp + 1); + + for (i = 0; i < ff_max_so_context; i++) { + struct ff_so_context *sc = &so_zone_tmp->sc[i]; + rte_spinlock_init(&sc->lock); + sc->status = FF_SC_IDLE; + sc->inuse = 0; + + if (sem_init(&sc->wait_sem, 1, 0) == -1) { + ERR_LOG("Initialize semaphore failed:%d\n", errno); + return -1; + } + } + + if (proc_id == 0) { + ff_so_zone = so_zone_tmp; + } + } + }else { + const struct rte_memzone *mz; + char zn[64]; + + snprintf(zn, sizeof(zn), SOCKET_OPS_ZONE_NAME, ff_global_cfg.dpdk.proc_id); + ERR_LOG("To lookup memzone:%s\n", zn); + + mz = rte_memzone_lookup(zn); + if (mz == NULL) { + ERR_LOG("Lookup memory zone:%s failed\n", zn); + return -1; + } + + ff_so_zone = mz->addr; + } + + return 0; +} + +struct ff_so_context * +ff_attach_so_context(int proc_id) +{ + struct ff_so_context *sc = NULL; + uint16_t i; + + if (ff_so_zone == NULL) { + const struct rte_memzone *mz; + char zn[64]; + + snprintf(zn, sizeof(zn), SOCKET_OPS_ZONE_NAME, proc_id); + ERR_LOG("To lookup memzone:%s\n", zn); + + mz = rte_memzone_lookup(zn); + if (mz == NULL) { + ERR_LOG("Lookup memory zone:%s failed\n", zn); + return NULL; + } + + ff_so_zone = mz->addr; + } + + rte_spinlock_lock(&ff_so_zone->lock); + + if (ff_so_zone->free == 0) { + ERR_LOG("Attach memzone failed: no free context\n"); + rte_spinlock_unlock(&ff_so_zone->lock); + return NULL; + } + + /* i从count-free开始尝试，直到一轮结束 */ + for (i = 0; i < ff_so_zone->count; i++) { + sc = &ff_so_zone->sc[i]; + if (sc->inuse == 0) { + sc->inuse = 1; + rte_spinlock_init(&sc->lock); + sc->status = FF_SC_IDLE; + ff_so_zone->free--; + break; + } + } + DEBUG_LOG("attach sc:%p, i:%d\n", sc, i); + + rte_spinlock_unlock(&ff_so_zone->lock); + + return sc; +} + +void +ff_detach_so_context(struct ff_so_context *sc) +{ + if (ff_so_zone == NULL || sc == NULL) { + return; + } + + rte_spinlock_lock(&ff_so_zone->lock); + + sc->inuse = 0; + + ff_so_zone->free++; + DEBUG_LOG("detach sc:%p, status:%d, ops:%d\n", sc, sc->status, sc->ops); + + rte_spinlock_unlock(&ff_so_zone->lock); +} diff --git a/adapter/ff_socket_ops.c b/adapter/ff_socket_ops.c new file mode 100644 index 000000000..d7a5201a3 --- /dev/null +++ b/adapter/ff_socket_ops.c @@ -0,0 +1,461 @@ +#include +#include + +#include "ff_socket_ops.h" +#include "ff_sysproto.h" +#include "ff_api.h" +#include "ff_epoll.h" + +#undef FF_SYSCALL_DECL +#define FF_SYSCALL_DECL(ret, fn, none) \ + static ret ff_sys_##fn(struct ff_##fn##_args *args); +#include +static int ff_sys_kqueue(struct ff_kqueue_args *args); +static int ff_sys_kevent(struct ff_kevent_args *args); + +extern struct ff_socket_ops_zone *ff_so_zone; + +#define FF_MAX_BOUND_NUM 8 + + +struct ff_bound_info { + int fd; + struct sockaddr addr; +}; + +static struct ff_bound_info ff_bound_fds[FF_MAX_BOUND_NUM]; + +static int +sockaddr_cmp(struct sockaddr *a, struct sockaddr *b) +{ + struct sockaddr_in *sina, *sinb; + sina = (struct sockaddr_in *)a; + sinb = (struct sockaddr_in *)b; + + if (sina->sin_family != sinb->sin_family) { + return 1; + } + + if (sina->sin_port != sinb->sin_port) { + return 1; + } + + if (sina->sin_addr.s_addr != sinb->sin_addr.s_addr) { + return 1; + } + + return 0; +} + +static int +sockaddr_is_bound(struct sockaddr *addr) +{ + int i; + + for (i = 0; i < FF_MAX_BOUND_NUM; i++) { + struct ff_bound_info *info = &ff_bound_fds[i]; + if (info->fd == 0) { + continue; + } + + if (sockaddr_cmp(&info->addr, addr) == 0) { + return info->fd; + } + } + + return 0; +} + +static int +sockaddr_bind(int fd, struct sockaddr *addr) +{ + int i; + + for (i = 0; i < FF_MAX_BOUND_NUM; i++) { + struct ff_bound_info *info = &ff_bound_fds[i]; + if (info->fd != 0) { + continue; + } + + info->fd = fd; + rte_memcpy(&info->addr, addr, sizeof(struct sockaddr)); + + return 0; + } + + return -1; +} + +static int +sockaddr_unbind(int fd) +{ + int i; + + for (i = 0; i < FF_MAX_BOUND_NUM; i++) { + struct ff_bound_info *info = &ff_bound_fds[i]; + if (info->fd != fd) { + continue; + } + + info->fd = 0; + + return 0; + } + + return -1; +} + +static int +ff_sys_socket(struct ff_socket_args *args) +{ + return ff_socket(args->domain, args->type, args->protocol); +} + +static int +ff_sys_bind(struct ff_bind_args *args) +{ + int bound_fd; + int ret; + + bound_fd = sockaddr_is_bound(args->addr); + if (bound_fd != 0 && bound_fd != args->fd) { + return ff_dup2(bound_fd, args->fd); + } + + ret = ff_bind(args->fd, args->addr, args->addrlen); + if (ret == 0) { + sockaddr_bind(args->fd, args->addr); + } + + return ret; +} + +static int +ff_sys_listen(struct ff_listen_args *args) +{ + return ff_listen(args->fd, args->backlog); +} + +static int +ff_sys_shutdown(struct ff_shutdown_args *args) +{ + return ff_shutdown(args->fd, args->how); +} + +static int +ff_sys_getsockname(struct ff_getsockname_args *args) +{ + return ff_getsockname(args->fd, args->name, args->namelen); +} + +static int +ff_sys_getpeername(struct ff_getpeername_args *args) +{ + return ff_getpeername(args->fd, args->name, args->namelen); +} + +static int +ff_sys_getsockopt(struct ff_getsockopt_args *args) +{ + return ff_getsockopt(args->fd, args->level, args->name, + args->optval, args->optlen); +} + +static int +ff_sys_setsockopt(struct ff_setsockopt_args *args) +{ + return ff_setsockopt(args->fd, args->level, args->name, + args->optval, args->optlen); +} + +static int +ff_sys_accept(struct ff_accept_args *args) +{ + return ff_accept(args->fd, args->addr, args->addrlen); +} + +static int +ff_sys_accept4(struct ff_accept4_args *args) +{ + errno = ENOSYS; + return -1; +} + +static int +ff_sys_connect(struct ff_connect_args *args) +{ + return ff_connect(args->fd, args->addr, args->addrlen); +} + +static ssize_t +ff_sys_recv(struct ff_recv_args *args) +{ + return ff_recv(args->fd, args->buf, args->len, args->flags); +} + +static ssize_t +ff_sys_recvfrom(struct ff_recvfrom_args *args) +{ + return ff_recvfrom(args->fd, args->buf, args->len, args->flags, + args->from, args->fromlen); +} + +static ssize_t +ff_sys_recvmsg(struct ff_recvmsg_args *args) +{ + return ff_recvmsg(args->fd, args->msg, args->flags); +} + +static ssize_t +ff_sys_read(struct ff_read_args *args) +{ + return ff_read(args->fd, args->buf, args->len); +} + +static ssize_t +ff_sys_readv(struct ff_readv_args *args) +{ + return ff_readv(args->fd, args->iov, args->iovcnt); +} + +static ssize_t +ff_sys_send(struct ff_send_args *args) +{ + return ff_send(args->fd, args->buf, args->len, args->flags); +} + +static ssize_t +ff_sys_sendto(struct ff_sendto_args *args) +{ + return ff_sendto(args->fd, args->buf, args->len, args->flags, + args->to, args->tolen); +} + +static ssize_t +ff_sys_sendmsg(struct ff_sendmsg_args *args) +{ + return ff_sendmsg(args->fd, args->msg, args->flags); +} + +static ssize_t +ff_sys_write(struct ff_write_args *args) +{ + return ff_write(args->fd, args->buf, args->len); +} + +static ssize_t +ff_sys_writev(struct ff_writev_args *args) +{ + return ff_writev(args->fd, args->iov, args->iovcnt); +} + +static int +ff_sys_close(struct ff_close_args *args) +{ + sockaddr_unbind(args->fd); + return ff_close(args->fd); +} + +static int +ff_sys_ioctl(struct ff_ioctl_args *args) +{ + return ff_ioctl(args->fd, args->com, args->data); +} + +static int +ff_sys_fcntl(struct ff_fcntl_args *args) +{ + return ff_fcntl(args->fd, args->cmd, args->data); +} + +static int +ff_sys_epoll_create(struct ff_epoll_create_args *args) +{ + DEBUG_LOG("to run ff_epoll_create, size:%d\n", args->size); + return ff_epoll_create(args->size); +} + +static int +ff_sys_epoll_ctl(struct ff_epoll_ctl_args *args) +{ + DEBUG_LOG("to run ff_epoll_ctl, epfd:%d, op:%d, fd:%d\n", + args->epfd, args->op, args->fd); + return ff_epoll_ctl(args->epfd, args->op, args->fd, + args->event); +} + +static int +ff_sys_epoll_wait(struct ff_epoll_wait_args *args) +{ + DEBUG_LOG("to run ff_epoll_wait, epfd:%d, maxevents:%d, timeout:%d\n", + args->epfd, args->maxevents, args->timeout); + return ff_epoll_wait(args->epfd, args->events, + args->maxevents, args->timeout); +} + +static int +ff_sys_kqueue(struct ff_kqueue_args *args) +{ + return ff_kqueue(); +} + +static int +ff_sys_kevent(struct ff_kevent_args *args) +{ + return ff_kevent(args->kq, args->changelist, args->nchanges, + args->eventlist, args->nevents, args->timeout); +} + +static pid_t +ff_sys_fork(struct ff_fork_args *args) +{ + errno = ENOSYS; + return -1; +} + +static int +ff_so_handler(int ops, void *args) +{ + DEBUG_LOG("ff_so_handler ops:%d, epoll create ops:%d\n", ops, FF_SO_EPOLL_CREATE); + switch(ops) { + case FF_SO_SOCKET: + return ff_sys_socket((struct ff_socket_args *)args); + case FF_SO_BIND: + return ff_sys_bind((struct ff_bind_args *)args); + case FF_SO_LISTEN: + return ff_sys_listen((struct ff_listen_args *)args); + case FF_SO_CONNECT: + return ff_sys_connect((struct ff_connect_args *)args); + case FF_SO_SHUTDOWN: + return ff_sys_shutdown((struct ff_shutdown_args *)args); + case FF_SO_GETSOCKNAME: + return ff_sys_getsockname((struct ff_getsockname_args *)args); + case FF_SO_GETPEERNAME: + return ff_sys_getpeername((struct ff_getpeername_args *)args); + case FF_SO_GETSOCKOPT: + return ff_sys_getsockopt((struct ff_getsockopt_args *)args); + case FF_SO_SETSOCKOPT: + return ff_sys_setsockopt((struct ff_setsockopt_args *)args); + case FF_SO_ACCEPT: + return ff_sys_accept((struct ff_accept_args *)args); + case FF_SO_ACCEPT4: + return ff_sys_accept4((struct ff_accept4_args *)args); + case FF_SO_RECV: + return ff_sys_recv((struct ff_recv_args *)args); + case FF_SO_RECVFROM: + return ff_sys_recvfrom((struct ff_recvfrom_args *)args); + case FF_SO_RECVMSG: + return ff_sys_recvmsg((struct ff_recvmsg_args *)args); + case FF_SO_READ: + return ff_sys_read((struct ff_read_args *)args); + case FF_SO_READV: + return ff_sys_readv((struct ff_readv_args *)args); + case FF_SO_SEND: + return ff_sys_send((struct ff_send_args *)args); + case FF_SO_SENDTO: + return ff_sys_sendto((struct ff_sendto_args *)args); + case FF_SO_SENDMSG: + return ff_sys_sendmsg((struct ff_sendmsg_args *)args); + case FF_SO_WRITE: + return ff_sys_write((struct ff_write_args *)args); + case FF_SO_WRITEV: + return ff_sys_writev((struct ff_writev_args *)args); + case FF_SO_CLOSE: + return ff_sys_close((struct ff_close_args *)args); + case FF_SO_IOCTL: + return ff_sys_ioctl((struct ff_ioctl_args *)args); + case FF_SO_FCNTL: + return ff_sys_fcntl((struct ff_fcntl_args *)args); + case FF_SO_EPOLL_CREATE: + return ff_sys_epoll_create((struct ff_epoll_create_args *)args); + case FF_SO_EPOLL_CTL: + return ff_sys_epoll_ctl((struct ff_epoll_ctl_args *)args); + case FF_SO_EPOLL_WAIT: + return ff_sys_epoll_wait((struct ff_epoll_wait_args *)args); + case FF_SO_KQUEUE: + return ff_sys_kqueue((struct ff_kqueue_args *)args); + case FF_SO_KEVENT: + return ff_sys_kevent((struct ff_kevent_args *)args); + case FF_SO_FORK: + return ff_sys_fork((struct ff_fork_args *)args); + default: + break; + } + + errno = EINVAL; + return (-1); +} + +static void +ff_handle_kevent(struct ff_so_context *sc) +{ + struct ff_kevent_args *ka = sc->args; + + errno = 0; + sc->result = ff_sys_kevent(ka); + sc->error = errno; + + if (ka->nchanges) { + ka->nchanges = 0; + } + + if (sc->result == 0 && ka->nevents != 0) { + return; + } + + sc->status = FF_SC_REP; + sem_post(&sc->wait_sem); +} + +static inline void +ff_handle_socket_ops(struct ff_so_context *sc) +{ + if (sc->inuse == 0) { + return; + } + + if (!rte_spinlock_trylock(&sc->lock)) { + return; + } + + if (sc->status != FF_SC_REQ) { + rte_spinlock_unlock(&sc->lock); + return; + } + + DEBUG_LOG("ff_handle_socket_ops sc:%p, status:%d, ops:%d\n", sc, sc->status, sc->ops); + + if (sc->ops == FF_SO_KEVENT) { + ff_handle_kevent(sc); + rte_spinlock_unlock(&sc->lock); + return; + } + + errno = 0; + sc->result = ff_so_handler(sc->ops, sc->args); + sc->error = errno; + sc->status = FF_SC_REP; + + if (sc->ops == FF_SO_EPOLL_WAIT) { + sem_post(&sc->wait_sem); + } + + rte_spinlock_unlock(&sc->lock); +} + +void +ff_handle_each_context() +{ + uint16_t i; + + rte_spinlock_lock(&ff_so_zone->lock); + + for (i = 0; i < ff_so_zone->count; i++) { + struct ff_so_context *sc = &ff_so_zone->sc[i]; + + ff_handle_socket_ops(sc); + } + + rte_spinlock_unlock(&ff_so_zone->lock); +} + diff --git a/adapter/ff_socket_ops.h b/adapter/ff_socket_ops.h new file mode 100644 index 000000000..df0405364 --- /dev/null +++ b/adapter/ff_socket_ops.h @@ -0,0 +1,103 @@ +#ifndef _FF_SOCKET_OPS_H_ +#define _FF_SOCKET_OPS_H_ + +#include + +#include +#include + +#define ERR_LOG(fmt, ...) do { \ + printf("file:%s, line:%u, fun:%s, "fmt, __FILE__, __LINE__, __func__, ##__VA_ARGS__); \ + } while (0) + +#ifdef DEBUG +#define DEBUG_LOG ERR_LOG +#else +#define DEBUG_LOG(...) +#endif + +enum FF_SOCKET_OPS { + FF_SO_SOCKET, + FF_SO_LISTEN, + FF_SO_BIND, + FF_SO_CONNECT, + FF_SO_SHUTDOWN, + FF_SO_GETSOCKNAME, + FF_SO_GETPEERNAME, + FF_SO_GETSOCKOPT, + FF_SO_SETSOCKOPT, + FF_SO_ACCEPT, + FF_SO_ACCEPT4, + FF_SO_RECV, + FF_SO_RECVFROM, + FF_SO_RECVMSG, + FF_SO_READ, + FF_SO_READV, + FF_SO_SEND, + FF_SO_SENDTO, + FF_SO_SENDMSG, + FF_SO_WRITE, + FF_SO_WRITEV, + FF_SO_CLOSE, + FF_SO_IOCTL, + FF_SO_FCNTL, + FF_SO_EPOLL_CREATE, + FF_SO_EPOLL_CTL, + FF_SO_EPOLL_WAIT, + FF_SO_KQUEUE, + FF_SO_KEVENT, + FF_SO_FORK, +}; + +enum FF_SO_CONTEXT_STATUS { + FF_SC_IDLE, + FF_SC_REQ, + FF_SC_REP, +}; + +struct ff_socket_ops_zone { + rte_spinlock_t lock; + + /* total number of so_contex */ + uint16_t count; + + /* free number of so_context */ + uint16_t free; + + struct ff_so_context *sc; +} __attribute__((packed)); + +struct ff_so_context { + rte_spinlock_t lock; + + int status; + + sem_t wait_sem; + + enum FF_SOCKET_OPS ops; + + void *args; + + /* result of ops processing */ + ssize_t result; + /* errno if failed */ + int error; + + /* 1 if used, else 0 */ + int inuse; + + // listen fd, refcount.. +} __attribute__((packed)); + +extern struct ff_socket_ops_zone *ff_so_zone; + +/* For primary process */ +int ff_set_max_so_context(uint16_t count); +int ff_create_so_memzone(); +void ff_handle_each_context(); + +/* For secondary process */ +struct ff_so_context *ff_attach_so_context(int proc_id); +void ff_detach_so_context(struct ff_so_context *context); + +#endif diff --git a/adapter/ff_sysproto.h b/adapter/ff_sysproto.h new file mode 100644 index 000000000..57e63eca8 --- /dev/null +++ b/adapter/ff_sysproto.h @@ -0,0 +1,197 @@ +#ifndef _FF_SYSPROTO_H_ +#define _FF_SYSPROTO_H_ + +#include +#include +#include +#include + +struct ff_socket_args { + int domain; + int type; + int protocol; +}; + +struct ff_bind_args { + int fd; + void *addr; + int addrlen; +}; + +struct ff_listen_args { + int fd; + int backlog; +}; + +struct ff_shutdown_args { + int fd; + int how; +}; + +struct ff_getpeername_args { + int fd; + void *name; + socklen_t *namelen; +}; + +struct ff_getsockname_args { + int fd; + void *name; + socklen_t *namelen; +}; + +struct ff_setsockopt_args { + int fd; + int level; + int name; + void *optval; + socklen_t optlen; +}; + +struct ff_getsockopt_args { + int fd; + int level; + int name; + void *optval; + socklen_t *optlen; +}; + +struct ff_accept_args { + int fd; + void *addr; + socklen_t *addrlen; +}; + +struct ff_accept4_args { + int fd; + void *addr; + socklen_t *addrlen; + int flags; +}; + +struct ff_connect_args { + int fd; + void *addr; + int addrlen; +}; + +struct ff_recv_args { + int fd; + void *buf; + size_t len; + int flags; +}; + +struct ff_recvfrom_args { + int fd; + void *buf; + size_t len; + int flags; + void *from; + socklen_t *fromlen; +}; + +struct ff_recvmsg_args { + int fd; + struct msghdr *msg; + int flags; +}; + +struct ff_read_args { + int fd; + void *buf; + size_t len; +}; + +struct ff_readv_args { + int fd; + struct iovec *iov; + int iovcnt; +}; + +struct ff_send_args { + int fd; + void *buf; + size_t len; + int flags; +}; + +struct ff_sendto_args { + int fd; + void *buf; + size_t len; + int flags; + void *to; + int tolen; +}; + +struct ff_sendmsg_args { + int fd; + struct msghdr * msg; + int flags; +}; + +struct ff_write_args { + int fd; + void *buf; + size_t len; +}; + +struct ff_writev_args { + int fd; + struct iovec *iov; + int iovcnt; +}; + +struct ff_close_args { + int fd; +}; + +struct ff_ioctl_args { + int fd; + unsigned long com; + void *data; +}; + +struct ff_fcntl_args { + int fd; + int cmd; + long data; +}; + +struct ff_epoll_create_args { + int size; +}; + +struct ff_epoll_ctl_args { + int epfd; + int op; + int fd; + struct epoll_event *event; +}; + +struct ff_epoll_wait_args { + int epfd; + struct epoll_event *events; + int maxevents; + int timeout; +}; + +struct ff_kqueue_args { + +}; + +struct ff_kevent_args { + int kq; + struct kevent *changelist; + int nchanges; + struct kevent *eventlist; + int nevents; + struct timespec *timeout; +}; + +struct ff_fork_args { + +}; + +#endif diff --git a/adapter/fstack.c b/adapter/fstack.c new file mode 100644 index 000000000..9d1031a5a --- /dev/null +++ b/adapter/fstack.c @@ -0,0 +1,36 @@ +#include "ff_api.h" +#include "ff_socket_ops.h" + +#define WORKERS 32 + +int +loop(void *arg) +{ + ff_handle_each_context(); + + return 0; +} + +int +main(int argc, char * argv[]) +{ + int ret; + + ff_init(argc, argv); + + ret = ff_set_max_so_context(WORKERS); + if (ret < 0) { + return -1; + } + + ret = ff_create_so_memzone(); + if (ret < 0) { + return -1; + } + + ERR_LOG("ff_create_so_memzone successful\n"); + + ff_run(loop, NULL); + + return 0; +}