From d1d95d5185f61b22ed9fc92e92e765989b34c548 Mon Sep 17 00:00:00 2001 From: chenwei Date: Mon, 5 Feb 2018 15:55:20 +0800 Subject: [PATCH 1/4] Nginx : directive proxy_kernel_network_stack 1. Add a new directive proxy_kernel_network_stack : Syntax: proxy_kernel_network_stack on | off; Default: proxy_kernel_network_stack off; Context: http, stream, mail, server This directive is available only when NGX_HAVE_FF_STACK is defined. Determines whether proxy should go throught kernel network stack or fstack. 2.Update F-Stack_Nginx_APP_Guide.md --- .../src/event/ngx_event_connect.c | 16 ++++++++++ .../src/event/ngx_event_connect.h | 4 +++ .../src/http/modules/ngx_http_proxy_module.c | 29 +++++++++++++++++++ .../src/http/ngx_http_core_module.c | 2 ++ .../src/mail/ngx_mail_core_module.c | 6 ++-- .../src/mail/ngx_mail_proxy_module.c | 26 +++++++++++++++++ .../src/stream/ngx_stream_proxy_module.c | 27 +++++++++++++++++ doc/F-Stack_Nginx_APP_Guide.md | 26 +++++++++++++---- 8 files changed, 128 insertions(+), 8 deletions(-) diff --git a/app/nginx-1.11.10/src/event/ngx_event_connect.c b/app/nginx-1.11.10/src/event/ngx_event_connect.c index 5662094c..09cd2340 100644 --- a/app/nginx-1.11.10/src/event/ngx_event_connect.c +++ b/app/nginx-1.11.10/src/event/ngx_event_connect.c @@ -38,7 +38,23 @@ ngx_event_connect_peer(ngx_peer_connection_t *pc) type = (pc->type ? pc->type : SOCK_STREAM); +#if (NGX_HAVE_FSTACK) + /* + We need to explicitly call the needed socket() function + to create socket! + */ + static int (*real_socket)(int, int, int); + if (pc->belong_to_host) { + if (!real_socket) { + real_socket = dlsym(RTLD_NEXT, "socket"); + } + s = real_socket(pc->sockaddr->sa_family, type, 0); + } else { + s = ngx_socket(pc->sockaddr->sa_family, type, 0); + } +#else s = ngx_socket(pc->sockaddr->sa_family, type, 0); +#endif ngx_log_debug2(NGX_LOG_DEBUG_EVENT, pc->log, 0, "%s socket %d", (type == SOCK_STREAM) ? "stream" : "dgram", s); diff --git a/app/nginx-1.11.10/src/event/ngx_event_connect.h b/app/nginx-1.11.10/src/event/ngx_event_connect.h index 72d21d7f..5d87eeeb 100644 --- a/app/nginx-1.11.10/src/event/ngx_event_connect.h +++ b/app/nginx-1.11.10/src/event/ngx_event_connect.h @@ -66,6 +66,10 @@ struct ngx_peer_connection_s { /* ngx_connection_log_error_e */ unsigned log_error:2; +#if (NGX_HAVE_FSTACK) + unsigned belong_to_host:1; +#endif + NGX_COMPAT_BEGIN(2) NGX_COMPAT_END }; diff --git a/app/nginx-1.11.10/src/http/modules/ngx_http_proxy_module.c b/app/nginx-1.11.10/src/http/modules/ngx_http_proxy_module.c index 1a84d78b..efad4162 100644 --- a/app/nginx-1.11.10/src/http/modules/ngx_http_proxy_module.c +++ b/app/nginx-1.11.10/src/http/modules/ngx_http_proxy_module.c @@ -101,6 +101,10 @@ typedef struct { ngx_str_t ssl_certificate_key; ngx_array_t *ssl_passwords; #endif + +#if (NGX_HAVE_FSTACK) + ngx_flag_t kernel_network_stack; +#endif } ngx_http_proxy_loc_conf_t; @@ -713,6 +717,17 @@ static ngx_command_t ngx_http_proxy_commands[] = { 0, NULL }, +#endif + +#if (NGX_HAVE_FSTACK) + + { ngx_string("proxy_kernel_network_stack"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_proxy_loc_conf_t, kernel_network_stack), + NULL }, + #endif ngx_null_command @@ -917,6 +932,10 @@ ngx_http_proxy_handler(ngx_http_request_t *r) u->accel = 1; +#if (NGX_HAVE_FSTACK) + u->peer.belong_to_host = plcf->kernel_network_stack; +#endif + if (!plcf->upstream.request_buffering && plcf->body_values == NULL && plcf->upstream.pass_request_body && (!r->headers_in.chunked @@ -2902,6 +2921,10 @@ ngx_http_proxy_create_loc_conf(ngx_conf_t *cf) ngx_str_set(&conf->upstream.module, "proxy"); +#if (NGX_HAVE_FSTACK) + conf->kernel_network_stack = NGX_CONF_UNSET; +#endif + return conf; } @@ -3396,6 +3419,12 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) #endif } +#if (NGX_HAVE_FSTACK) + /* By default, we set up a proxy on fstack */ + ngx_conf_merge_value(conf->kernel_network_stack, + prev->kernel_network_stack, 0); +#endif + return NGX_CONF_OK; } diff --git a/app/nginx-1.11.10/src/http/ngx_http_core_module.c b/app/nginx-1.11.10/src/http/ngx_http_core_module.c index 5bb6591e..734db7fc 100644 --- a/app/nginx-1.11.10/src/http/ngx_http_core_module.c +++ b/app/nginx-1.11.10/src/http/ngx_http_core_module.c @@ -3455,7 +3455,9 @@ ngx_http_core_create_srv_conf(ngx_conf_t *cf) cscf->ignore_invalid_headers = NGX_CONF_UNSET; cscf->merge_slashes = NGX_CONF_UNSET; cscf->underscores_in_headers = NGX_CONF_UNSET; +#if (NGX_HAVE_FSTACK) cscf->kernel_network_stack = NGX_CONF_UNSET; +#endif return cscf; } diff --git a/app/nginx-1.11.10/src/mail/ngx_mail_core_module.c b/app/nginx-1.11.10/src/mail/ngx_mail_core_module.c index fc8876d3..561e3c0d 100644 --- a/app/nginx-1.11.10/src/mail/ngx_mail_core_module.c +++ b/app/nginx-1.11.10/src/mail/ngx_mail_core_module.c @@ -67,10 +67,10 @@ static ngx_command_t ngx_mail_core_commands[] = { #if (NGX_HAVE_FSTACK) { ngx_string("kernel_network_stack"), - NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, + NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_FLAG, ngx_conf_set_flag_slot, - NGX_HTTP_SRV_CONF_OFFSET, - offsetof(ngx_http_core_srv_conf_t, kernel_network_stack), + NGX_MAIL_SRV_CONF_OFFSET, + offsetof(ngx_mail_core_srv_conf_t, kernel_network_stack), NULL }, #endif diff --git a/app/nginx-1.11.10/src/mail/ngx_mail_proxy_module.c b/app/nginx-1.11.10/src/mail/ngx_mail_proxy_module.c index 007284b6..ff2cbc45 100644 --- a/app/nginx-1.11.10/src/mail/ngx_mail_proxy_module.c +++ b/app/nginx-1.11.10/src/mail/ngx_mail_proxy_module.c @@ -18,6 +18,10 @@ typedef struct { ngx_flag_t xclient; size_t buffer_size; ngx_msec_t timeout; + +#if (NGX_HAVE_FSTACK) + ngx_flag_t kernel_network_stack; +#endif } ngx_mail_proxy_conf_t; @@ -74,6 +78,15 @@ static ngx_command_t ngx_mail_proxy_commands[] = { offsetof(ngx_mail_proxy_conf_t, xclient), NULL }, +#if (NGX_HAVE_FSTACK) + { ngx_string("proxy_kernel_network_stack"), + NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_MAIL_SRV_CONF_OFFSET, + offsetof(ngx_mail_proxy_conf_t, kernel_network_stack), + NULL }, +#endif + ngx_null_command }; @@ -135,6 +148,10 @@ ngx_mail_proxy_init(ngx_mail_session_t *s, ngx_addr_t *peer) p->upstream.log = s->connection->log; p->upstream.log_error = NGX_ERROR_ERR; +#if (NGX_HAVE_FSTACK) + p->upstream.belong_to_host = pcf->kernel_network_stack; +#endif + rc = ngx_event_connect_peer(&p->upstream); if (rc == NGX_ERROR || rc == NGX_BUSY || rc == NGX_DECLINED) { @@ -1102,6 +1119,10 @@ ngx_mail_proxy_create_conf(ngx_conf_t *cf) pcf->buffer_size = NGX_CONF_UNSET_SIZE; pcf->timeout = NGX_CONF_UNSET_MSEC; +#if (NGX_HAVE_FSTACK) + pcf->kernel_network_stack = NGX_CONF_UNSET; +#endif + return pcf; } @@ -1119,5 +1140,10 @@ ngx_mail_proxy_merge_conf(ngx_conf_t *cf, void *parent, void *child) (size_t) ngx_pagesize); ngx_conf_merge_msec_value(conf->timeout, prev->timeout, 24 * 60 * 60000); +#if (NGX_HAVE_FSTACK) + ngx_conf_merge_value(conf->kernel_network_stack, + prev->kernel_network_stack, 0); +#endif + return NGX_CONF_OK; } diff --git a/app/nginx-1.11.10/src/stream/ngx_stream_proxy_module.c b/app/nginx-1.11.10/src/stream/ngx_stream_proxy_module.c index 87117b02..3b9200e4 100644 --- a/app/nginx-1.11.10/src/stream/ngx_stream_proxy_module.c +++ b/app/nginx-1.11.10/src/stream/ngx_stream_proxy_module.c @@ -51,6 +51,10 @@ typedef struct { ngx_ssl_t *ssl; #endif +#if (NGX_HAVE_FSTACK) + ngx_flag_t kernel_network_stack; +#endif + ngx_stream_upstream_srv_conf_t *upstream; ngx_stream_complex_value_t *upstream_value; } ngx_stream_proxy_srv_conf_t; @@ -313,6 +317,15 @@ static ngx_command_t ngx_stream_proxy_commands[] = { #endif +#if (NGX_HAVE_FSTACK) + { ngx_string("proxy_kernel_network_stack"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_flag_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_proxy_srv_conf_t, kernel_network_stack), + NULL }, +#endif + ngx_null_command }; @@ -387,6 +400,10 @@ ngx_stream_proxy_handler(ngx_stream_session_t *s) u->peer.type = c->type; u->start_sec = ngx_time(); +#if (NGX_HAVE_FSTACK) + u->peer.belong_to_host = pscf->kernel_network_stack; +#endif + c->write->handler = ngx_stream_proxy_downstream_handler; c->read->handler = ngx_stream_proxy_downstream_handler; @@ -1860,6 +1877,10 @@ ngx_stream_proxy_create_srv_conf(ngx_conf_t *cf) conf->ssl_passwords = NGX_CONF_UNSET_PTR; #endif +#if (NGX_HAVE_FSTACK) + conf->kernel_network_stack = NGX_CONF_UNSET; +#endif + return conf; } @@ -1943,6 +1964,12 @@ ngx_stream_proxy_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) #endif +#if (NGX_HAVE_FSTACK) + /* By default, we set up a proxy on fstack */ + ngx_conf_merge_value(conf->kernel_network_stack, + prev->kernel_network_stack, 0); +#endif + return NGX_CONF_OK; } diff --git a/doc/F-Stack_Nginx_APP_Guide.md b/doc/F-Stack_Nginx_APP_Guide.md index 39066351..80686cea 100644 --- a/doc/F-Stack_Nginx_APP_Guide.md +++ b/doc/F-Stack_Nginx_APP_Guide.md @@ -42,21 +42,37 @@ first one to start | | | | | - a major addition to the worker process is fstack-handling：ff_init();ff_run(worker_process_cycle); worker_process_cycle(handle channel/host/fstack event). -- a new directive `kernel_network_stack` : +## What's Different? +### New directives: +All the directives below are available only when ```NGX_HAVE_FSTACK``` is defined. ``` Syntax: kernel_network_stack on | off; Default: kernel_network_stack off; Context: http, server - This directive is available only when ```NGX_HAVE_FSTACK``` is defined. + Determines whether server should run on kernel network stack or fstack. ``` -Note that: +``` + Syntax: proxy_kernel_network_stack on | off; + Default: kernel_network_stack off; + Context: http, stream, mail, server -- the `reload` is not graceful, service will still be unavailable during the process of reloading. + Determines whether proxy should go through kernel network stack or fstack. +``` -- necessary modifies in nginx.conf: +``` + Syntax: schedule_timeout time; + Default: schedule_timeout 30m; + Context: http, server + Sets a time interval for polling kernel_network_stack. The default value is 30 msec. +``` + +### Command-line `reload` +the `reload` is not graceful, service will still be unavailable during the process of reloading. + +### Necessary modifies in nginx.conf: ``` user root; # root account is necessary. fstack_conf f-stack.conf; # path of f-stack configuration file, default: $NGX_PREFIX/conf/f-stack.conf. From edbcaf6eef2afeccf34fb0d2d05f7b518fbbca5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=A8=81?= Date: Fri, 9 Feb 2018 20:09:05 +0800 Subject: [PATCH 2/4] Update F-Stack_Nginx_APP_Guide.md --- doc/F-Stack_Nginx_APP_Guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/F-Stack_Nginx_APP_Guide.md b/doc/F-Stack_Nginx_APP_Guide.md index 80686cea..e94130f0 100644 --- a/doc/F-Stack_Nginx_APP_Guide.md +++ b/doc/F-Stack_Nginx_APP_Guide.md @@ -63,7 +63,7 @@ All the directives below are available only when ```NGX_HAVE_FSTACK``` is define ``` Syntax: schedule_timeout time; - Default: schedule_timeout 30m; + Default: schedule_timeout 30ms; Context: http, server Sets a time interval for polling kernel_network_stack. The default value is 30 msec. From 76e16b226f8deadbe975a77fae948053f49fe646 Mon Sep 17 00:00:00 2001 From: chenwei Date: Sat, 24 Feb 2018 16:45:10 +0800 Subject: [PATCH 3/4] Nginx : add a creation flag SOCK_FSTACK(create-fstack-socket) for socket() 1. `#define SOCK_FSTACK 0x1000` 2. when we want to create socket by fstack, we code like this : `s = ngx_socket(domain, type | SOCK_FSTACK , protocol);` --- app/nginx-1.11.10/auto/modules | 1 + app/nginx-1.11.10/src/core/ngx_connection.c | 4 +++- .../src/event/modules/ngx_ff_module.c | 21 ++++++++++++++----- .../src/event/ngx_event_connect.c | 21 +++++++------------ 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/app/nginx-1.11.10/auto/modules b/app/nginx-1.11.10/auto/modules index 61051ee4..116ac7e8 100644 --- a/app/nginx-1.11.10/auto/modules +++ b/app/nginx-1.11.10/auto/modules @@ -58,6 +58,7 @@ fi if [ $USE_FSTACK = YES ]; then have=NGX_HAVE_FSTACK . auto/have have=NGX_HAVE_FSTACK . auto/have_headers + have=SOCK_FSTACK value=0x1000 . auto/define CORE_SRCS="$CORE_SRCS $KQUEUE_SRCS" EVENT_MODULES="$EVENT_MODULES $KQUEUE_MODULE" fi diff --git a/app/nginx-1.11.10/src/core/ngx_connection.c b/app/nginx-1.11.10/src/core/ngx_connection.c index 47e29ba4..46c0ebc0 100644 --- a/app/nginx-1.11.10/src/core/ngx_connection.c +++ b/app/nginx-1.11.10/src/core/ngx_connection.c @@ -15,7 +15,7 @@ extern int fstack_territory(int domain, int type, int protocol); extern int is_fstack_fd(int sockfd); static ngx_inline int -ngx_ff_skip_listening_socket(ngx_cycle_t *cycle, const ngx_listening_t *ls) +ngx_ff_skip_listening_socket(ngx_cycle_t *cycle, ngx_listening_t *ls) { if (ngx_process <= NGX_PROCESS_MASTER) { @@ -36,6 +36,8 @@ ngx_ff_skip_listening_socket(ngx_cycle_t *cycle, const ngx_listening_t *ls) if (!fstack_territory(ls->sockaddr->sa_family, ls->type, 0)) { return 1; } + + ls->type |= SOCK_FSTACK; } else { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "unexpected process type: %d, ignored", diff --git a/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c b/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c index 75bd3de4..0bef5177 100644 --- a/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c +++ b/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c @@ -72,6 +72,7 @@ #include #include +#include #include "ff_api.h" #define _GNU_SOURCE @@ -208,11 +209,16 @@ ff_mod_init(const char *conf, int proc_id, int proc_type) { int fstack_territory(int domain, int type, int protocol) { - if ((AF_INET != domain) || (SOCK_STREAM != type && SOCK_DGRAM != type)) { - return 0; - } + /* Remove creation flags */ + type &= ~SOCK_CLOEXEC; + type &= ~SOCK_NONBLOCK; + type &= ~SOCK_FSTACK; - return 1; + if ((AF_INET != domain) || (SOCK_STREAM != type && SOCK_DGRAM != type)) { + return 0; + } + + return 1; } int @@ -223,10 +229,15 @@ socket(int domain, int type, int protocol) return SYSCALL(socket)(domain, type, protocol); } - if ((AF_INET != domain) || (SOCK_STREAM != type && SOCK_DGRAM != type)) { + if (unlikely(fstack_territory(domain, type, protocol) == 0)) { return SYSCALL(socket)(domain, type, protocol); } + if (unlikely((type & SOCK_FSTACK) == 0)) { + return SYSCALL(socket)(domain, type, protocol); + } + + type &= ~SOCK_FSTACK; sock = ff_socket(domain, type, protocol); if (sock != -1) { diff --git a/app/nginx-1.11.10/src/event/ngx_event_connect.c b/app/nginx-1.11.10/src/event/ngx_event_connect.c index 09cd2340..4343340c 100644 --- a/app/nginx-1.11.10/src/event/ngx_event_connect.c +++ b/app/nginx-1.11.10/src/event/ngx_event_connect.c @@ -39,23 +39,18 @@ ngx_event_connect_peer(ngx_peer_connection_t *pc) type = (pc->type ? pc->type : SOCK_STREAM); #if (NGX_HAVE_FSTACK) - /* - We need to explicitly call the needed socket() function - to create socket! + /* + We use a creation flags created by fstack's adaptable layer to + to explicitly call the needed socket() function. */ - static int (*real_socket)(int, int, int); - if (pc->belong_to_host) { - if (!real_socket) { - real_socket = dlsym(RTLD_NEXT, "socket"); - } - s = real_socket(pc->sockaddr->sa_family, type, 0); - } else { - s = ngx_socket(pc->sockaddr->sa_family, type, 0); + if (!pc->belong_to_host) { + type |= SOCK_FSTACK; } -#else - s = ngx_socket(pc->sockaddr->sa_family, type, 0); #endif + s = ngx_socket(pc->sockaddr->sa_family, type, 0); + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, pc->log, 0, "%s socket %d", (type == SOCK_STREAM) ? "stream" : "dgram", s); From 42f547056e360fe704c2b9e14643d6eabad472ff Mon Sep 17 00:00:00 2001 From: chenwei Date: Sat, 24 Feb 2018 17:08:59 +0800 Subject: [PATCH 4/4] Nginx: revise indent. --- app/nginx-1.11.10/src/event/modules/ngx_ff_module.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c b/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c index 0bef5177..48045097 100644 --- a/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c +++ b/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c @@ -210,8 +210,8 @@ int fstack_territory(int domain, int type, int protocol) { /* Remove creation flags */ - type &= ~SOCK_CLOEXEC; - type &= ~SOCK_NONBLOCK; + type &= ~SOCK_CLOEXEC; + type &= ~SOCK_NONBLOCK; type &= ~SOCK_FSTACK; if ((AF_INET != domain) || (SOCK_STREAM != type && SOCK_DGRAM != type)) { @@ -233,9 +233,9 @@ socket(int domain, int type, int protocol) return SYSCALL(socket)(domain, type, protocol); } - if (unlikely((type & SOCK_FSTACK) == 0)) { - return SYSCALL(socket)(domain, type, protocol); - } + if (unlikely((type & SOCK_FSTACK) == 0)) { + return SYSCALL(socket)(domain, type, protocol); + } type &= ~SOCK_FSTACK; sock = ff_socket(domain, type, protocol); @@ -287,7 +287,7 @@ sendto(int sockfd, const void *buf, size_t len, int flags, if(is_fstack_fd(sockfd)){ sockfd = restore_fstack_fd(sockfd); return ff_sendto(sockfd, buf, len, flags, - (struct linux_sockaddr *)dest_addr, addrlen); + (struct linux_sockaddr *)dest_addr, addrlen); } return SYSCALL(sendto)(sockfd, buf, len, flags, dest_addr, addrlen);