From c41d837fae6321df80f951abdae046561ce399f2 Mon Sep 17 00:00:00 2001 From: chenwei Date: Wed, 31 Jan 2018 13:37:58 +0800 Subject: [PATCH 1/5] Nginx : ngx_configure_listening_sockets 1. Nginx based on fstack delays setting up server on fstack until ngx_worker_process_init. ngx_configure_listening_sockets should as well. 2. FStack does not support IP_PKTINFO --- app/nginx-1.11.10/src/core/ngx_connection.c | 64 +++++++++++-------- .../src/os/unix/ngx_process_cycle.c | 4 ++ 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/app/nginx-1.11.10/src/core/ngx_connection.c b/app/nginx-1.11.10/src/core/ngx_connection.c index 9739fed66..fd4d2ada3 100644 --- a/app/nginx-1.11.10/src/core/ngx_connection.c +++ b/app/nginx-1.11.10/src/core/ngx_connection.c @@ -13,6 +13,33 @@ #if (NGX_HAVE_FSTACK) extern int fstack_territory(int domain, int type, int protocol); extern int is_fstack_fd(int sockfd); + +#define ngx_ff_skip_listening_socket(ls) \ + if (ngx_process <= NGX_PROCESS_MASTER) { \ + \ + /* process master, kernel network stack*/ \ + if (!ls[i].belong_to_host) { \ + /* We should continue to process the listening socket, */ \ + /* if it is not supported by fstack. */ \ + if (fstack_territory(ls[i].sockaddr->sa_family, ls[i].type, 0)) { \ + continue; \ + } \ + } \ + } else if (NGX_PROCESS_WORKER == ngx_process) { \ + /* process worker, fstack */ \ + if (ls[i].belong_to_host) { \ + continue; \ + } \ + \ + if (!fstack_territory(ls[i].sockaddr->sa_family, ls[i].type, 0)) { \ + continue; \ + } \ + } else { \ + ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, \ + "unexpected process type: %d, ignored", \ + ngx_process); \ + exit(1); \ + } #endif @@ -411,31 +438,7 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle) #if (NGX_HAVE_FSTACK) - if (ngx_process <= NGX_PROCESS_MASTER) { - - /* process master, kernel network stack*/ - if (!ls[i].belong_to_host) { - /* We should continue to process the listening socket, - if it is not supported by fstack.*/ - if (fstack_territory(ls[i].sockaddr->sa_family, ls[i].type, 0)) { - continue; - } - } - } else if (NGX_PROCESS_WORKER == ngx_process) { - /* process worker, fstack */ - if (ls[i].belong_to_host) { - continue; - } - - if (!fstack_territory(ls[i].sockaddr->sa_family, ls[i].type, 0)) { - continue; - } - } else { - ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, - "unexpected process type: %d, ignored", - ngx_process); - exit(1); - } + ngx_ff_skip_listening_socket(ls); #endif @@ -695,6 +698,12 @@ ngx_configure_listening_sockets(ngx_cycle_t *cycle) ls = cycle->listening.elts; for (i = 0; i < cycle->listening.nelts; i++) { +#if (NGX_HAVE_FSTACK) + + ngx_ff_skip_listening_socket(ls); + +#endif + ls[i].log = *ls[i].logp; if (ls[i].rcvbuf != -1) { @@ -941,7 +950,8 @@ ngx_configure_listening_sockets(ngx_cycle_t *cycle) &ls[i].addr_text); } } - +/* TODO: configure , auto/feature */ +#if !(NGX_HAVE_FSTACK) #elif (NGX_HAVE_IP_PKTINFO) if (ls[i].wildcard @@ -960,7 +970,7 @@ ngx_configure_listening_sockets(ngx_cycle_t *cycle) &ls[i].addr_text); } } - +#endif /* NGX_HAVE_FSTACK */ #endif #if (NGX_HAVE_INET6 && NGX_HAVE_IPV6_RECVPKTINFO) diff --git a/app/nginx-1.11.10/src/os/unix/ngx_process_cycle.c b/app/nginx-1.11.10/src/os/unix/ngx_process_cycle.c index 9ccfe6fdd..abfb814ef 100644 --- a/app/nginx-1.11.10/src/os/unix/ngx_process_cycle.c +++ b/app/nginx-1.11.10/src/os/unix/ngx_process_cycle.c @@ -1002,6 +1002,10 @@ ngx_worker_process_init(ngx_cycle_t *cycle, ngx_int_t worker) "ngx_open_listening_sockets failed"); exit(2); } + + if (!ngx_test_config) { + ngx_configure_listening_sockets(cycle); + } } #endif From 19acab5d5f254bd107300828251500ec439be36e Mon Sep 17 00:00:00 2001 From: chenwei Date: Wed, 31 Jan 2018 13:47:44 +0800 Subject: [PATCH 2/5] Nginx: hijack `recvmsg`. Since Nginx calls `recvmsg` in stream with udp, we must hijack this function, so that the network IO can pass through f-stack. --- app/nginx-1.11.10/src/event/modules/ngx_ff_module.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c b/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c index 4bf5537d8..75bd3de44 100644 --- a/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c +++ b/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c @@ -109,6 +109,7 @@ static ssize_t (*real_send)(int, const void *, size_t, int); static ssize_t (*real_sendto)(int, const void *, size_t, int, const struct sockaddr*, socklen_t); static ssize_t (*real_sendmsg)(int, const struct msghdr*, int); +static ssize_t (*real_recvmsg)(int, struct msghdr *, int); static ssize_t (*real_writev)(int, const struct iovec *, int); static ssize_t (*real_readv)(int, const struct iovec *, int); @@ -292,6 +293,16 @@ sendmsg(int sockfd, const struct msghdr *msg, int flags) return SYSCALL(sendmsg)(sockfd, msg, flags); } +ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags) +{ + if(is_fstack_fd(sockfd)){ + sockfd = restore_fstack_fd(sockfd); + return ff_recvmsg(sockfd, msg, flags); + } + + return SYSCALL(recvmsg)(sockfd, msg, flags); +} + ssize_t recv(int sockfd, void *buf, size_t len, int flags) { From d65c1f94b40da2137d6723635147e267462e98a7 Mon Sep 17 00:00:00 2001 From: chenwei Date: Thu, 1 Feb 2018 16:30:46 +0800 Subject: [PATCH 3/5] Nginx : replace macro with inline function to implement ```ngx_ff_skip_listening_socket``` --- app/nginx-1.11.10/src/core/ngx_connection.c | 65 ++++++++++++--------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/app/nginx-1.11.10/src/core/ngx_connection.c b/app/nginx-1.11.10/src/core/ngx_connection.c index fd4d2ada3..47e29ba43 100644 --- a/app/nginx-1.11.10/src/core/ngx_connection.c +++ b/app/nginx-1.11.10/src/core/ngx_connection.c @@ -14,32 +14,37 @@ extern int fstack_territory(int domain, int type, int protocol); extern int is_fstack_fd(int sockfd); -#define ngx_ff_skip_listening_socket(ls) \ - if (ngx_process <= NGX_PROCESS_MASTER) { \ - \ - /* process master, kernel network stack*/ \ - if (!ls[i].belong_to_host) { \ - /* We should continue to process the listening socket, */ \ - /* if it is not supported by fstack. */ \ - if (fstack_territory(ls[i].sockaddr->sa_family, ls[i].type, 0)) { \ - continue; \ - } \ - } \ - } else if (NGX_PROCESS_WORKER == ngx_process) { \ - /* process worker, fstack */ \ - if (ls[i].belong_to_host) { \ - continue; \ - } \ - \ - if (!fstack_territory(ls[i].sockaddr->sa_family, ls[i].type, 0)) { \ - continue; \ - } \ - } else { \ - ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, \ - "unexpected process type: %d, ignored", \ - ngx_process); \ - exit(1); \ - } +static ngx_inline int +ngx_ff_skip_listening_socket(ngx_cycle_t *cycle, const ngx_listening_t *ls) +{ + if (ngx_process <= NGX_PROCESS_MASTER) { + + /* process master, kernel network stack*/ + if (!ls->belong_to_host) { + /* We should continue to process the listening socket, + if it is not supported by fstack. */ + if (fstack_territory(ls->sockaddr->sa_family, ls->type, 0)) { + return 1; + } + } + } else if (NGX_PROCESS_WORKER == ngx_process) { + /* process worker, fstack */ + if (ls->belong_to_host) { + return 1; + } + + if (!fstack_territory(ls->sockaddr->sa_family, ls->type, 0)) { + return 1; + } + } else { + ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, + "unexpected process type: %d, ignored", + ngx_process); + exit(1); + } + + return 0; +} #endif @@ -438,7 +443,9 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle) #if (NGX_HAVE_FSTACK) - ngx_ff_skip_listening_socket(ls); + if(ngx_ff_skip_listening_socket(cycle, &ls[i])){ + continue; + } #endif @@ -700,7 +707,9 @@ ngx_configure_listening_sockets(ngx_cycle_t *cycle) #if (NGX_HAVE_FSTACK) - ngx_ff_skip_listening_socket(ls); + if(ngx_ff_skip_listening_socket(cycle, &ls[i])){ + continue; + } #endif From d8829f7e5a4b7c6773d63d437b4a7a20f6ab8328 Mon Sep 17 00:00:00 2001 From: chenwei Date: Fri, 2 Feb 2018 15:28:58 +0800 Subject: [PATCH 4/5] API : fix bug of ff_sendmsg and ff_recvmsg freebsd and linux have a different "struct sockaddr". In ff_recvmsg and ff_recvmsg, ```msg->msg_name```,which can refer to address, can be expected to be converted . --- lib/ff_syscall_wrapper.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/ff_syscall_wrapper.c b/lib/ff_syscall_wrapper.c index a710a9cad..b071d7992 100644 --- a/lib/ff_syscall_wrapper.c +++ b/lib/ff_syscall_wrapper.c @@ -334,9 +334,9 @@ linux2freebsd_sockaddr(const struct linux_sockaddr *linux, return; } - bzero(freebsd, sizeof(struct sockaddr)); - freebsd->sa_len = addrlen; + /* #linux and #freebsd may point to the same address */ freebsd->sa_family = linux->sa_family; + freebsd->sa_len = addrlen; bcopy(linux->sa_data, freebsd->sa_data, sizeof(linux->sa_data)); } @@ -610,6 +610,11 @@ ff_sendmsg(int s, const struct msghdr *msg, int flags) { int rc; + if (msg->msg_name != NULL) { + linux2freebsd_sockaddr(msg->msg_name, + sizeof(struct linux_sockaddr), msg->msg_name); + } + if ((rc = sendit(curthread, s, __DECONST(struct msghdr *, msg), flags))) goto kern_fail; @@ -676,6 +681,8 @@ ff_recvmsg(int s, struct msghdr *msg, int flags) } rc = curthread->td_retval[0]; + freebsd2linux_sockaddr(msg->msg_name, msg->msg_name); + return (rc); kern_fail: ff_os_errno(rc); From 6c629acab4f9a6a90a62bb048dba4129070429af Mon Sep 17 00:00:00 2001 From: chenwei Date: Fri, 2 Feb 2018 17:58:17 +0800 Subject: [PATCH 5/5] API : ff_sendmsg, use transient variables to avoid polluting user's data. --- lib/ff_syscall_wrapper.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/ff_syscall_wrapper.c b/lib/ff_syscall_wrapper.c index b071d7992..3e1e484df 100644 --- a/lib/ff_syscall_wrapper.c +++ b/lib/ff_syscall_wrapper.c @@ -609,13 +609,20 @@ ssize_t ff_sendmsg(int s, const struct msghdr *msg, int flags) { int rc; + struct sockaddr freebsd_sa; + void *linux_sa = msg->msg_name; - if (msg->msg_name != NULL) { - linux2freebsd_sockaddr(msg->msg_name, - sizeof(struct linux_sockaddr), msg->msg_name); + if (linux_sa != NULL) { + linux2freebsd_sockaddr(linux_sa, + sizeof(struct linux_sockaddr), &freebsd_sa); + __DECONST(struct msghdr *, msg)->msg_name = &freebsd_sa; } - if ((rc = sendit(curthread, s, __DECONST(struct msghdr *, msg), flags))) + rc = sendit(curthread, s, __DECONST(struct msghdr *, msg), flags); + + __DECONST(struct msghdr *, msg)->msg_name = linux_sa; + + if (rc) goto kern_fail; return (rc);