From d1d95d5185f61b22ed9fc92e92e765989b34c548 Mon Sep 17 00:00:00 2001 From: chenwei Date: Mon, 5 Feb 2018 15:55:20 +0800 Subject: [PATCH] Nginx : directive proxy_kernel_network_stack 1. Add a new directive proxy_kernel_network_stack : Syntax: proxy_kernel_network_stack on | off; Default: proxy_kernel_network_stack off; Context: http, stream, mail, server This directive is available only when NGX_HAVE_FF_STACK is defined. Determines whether proxy should go throught kernel network stack or fstack. 2.Update F-Stack_Nginx_APP_Guide.md --- .../src/event/ngx_event_connect.c | 16 ++++++++++ .../src/event/ngx_event_connect.h | 4 +++ .../src/http/modules/ngx_http_proxy_module.c | 29 +++++++++++++++++++ .../src/http/ngx_http_core_module.c | 2 ++ .../src/mail/ngx_mail_core_module.c | 6 ++-- .../src/mail/ngx_mail_proxy_module.c | 26 +++++++++++++++++ .../src/stream/ngx_stream_proxy_module.c | 27 +++++++++++++++++ doc/F-Stack_Nginx_APP_Guide.md | 26 +++++++++++++---- 8 files changed, 128 insertions(+), 8 deletions(-) diff --git a/app/nginx-1.11.10/src/event/ngx_event_connect.c b/app/nginx-1.11.10/src/event/ngx_event_connect.c index 5662094c..09cd2340 100644 --- a/app/nginx-1.11.10/src/event/ngx_event_connect.c +++ b/app/nginx-1.11.10/src/event/ngx_event_connect.c @@ -38,7 +38,23 @@ ngx_event_connect_peer(ngx_peer_connection_t *pc) type = (pc->type ? pc->type : SOCK_STREAM); +#if (NGX_HAVE_FSTACK) + /* + We need to explicitly call the needed socket() function + to create socket! + */ + static int (*real_socket)(int, int, int); + if (pc->belong_to_host) { + if (!real_socket) { + real_socket = dlsym(RTLD_NEXT, "socket"); + } + s = real_socket(pc->sockaddr->sa_family, type, 0); + } else { + s = ngx_socket(pc->sockaddr->sa_family, type, 0); + } +#else s = ngx_socket(pc->sockaddr->sa_family, type, 0); +#endif ngx_log_debug2(NGX_LOG_DEBUG_EVENT, pc->log, 0, "%s socket %d", (type == SOCK_STREAM) ? "stream" : "dgram", s); diff --git a/app/nginx-1.11.10/src/event/ngx_event_connect.h b/app/nginx-1.11.10/src/event/ngx_event_connect.h index 72d21d7f..5d87eeeb 100644 --- a/app/nginx-1.11.10/src/event/ngx_event_connect.h +++ b/app/nginx-1.11.10/src/event/ngx_event_connect.h @@ -66,6 +66,10 @@ struct ngx_peer_connection_s { /* ngx_connection_log_error_e */ unsigned log_error:2; +#if (NGX_HAVE_FSTACK) + unsigned belong_to_host:1; +#endif + NGX_COMPAT_BEGIN(2) NGX_COMPAT_END }; diff --git a/app/nginx-1.11.10/src/http/modules/ngx_http_proxy_module.c b/app/nginx-1.11.10/src/http/modules/ngx_http_proxy_module.c index 1a84d78b..efad4162 100644 --- a/app/nginx-1.11.10/src/http/modules/ngx_http_proxy_module.c +++ b/app/nginx-1.11.10/src/http/modules/ngx_http_proxy_module.c @@ -101,6 +101,10 @@ typedef struct { ngx_str_t ssl_certificate_key; ngx_array_t *ssl_passwords; #endif + +#if (NGX_HAVE_FSTACK) + ngx_flag_t kernel_network_stack; +#endif } ngx_http_proxy_loc_conf_t; @@ -713,6 +717,17 @@ static ngx_command_t ngx_http_proxy_commands[] = { 0, NULL }, +#endif + +#if (NGX_HAVE_FSTACK) + + { ngx_string("proxy_kernel_network_stack"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_proxy_loc_conf_t, kernel_network_stack), + NULL }, + #endif ngx_null_command @@ -917,6 +932,10 @@ ngx_http_proxy_handler(ngx_http_request_t *r) u->accel = 1; +#if (NGX_HAVE_FSTACK) + u->peer.belong_to_host = plcf->kernel_network_stack; +#endif + if (!plcf->upstream.request_buffering && plcf->body_values == NULL && plcf->upstream.pass_request_body && (!r->headers_in.chunked @@ -2902,6 +2921,10 @@ ngx_http_proxy_create_loc_conf(ngx_conf_t *cf) ngx_str_set(&conf->upstream.module, "proxy"); +#if (NGX_HAVE_FSTACK) + conf->kernel_network_stack = NGX_CONF_UNSET; +#endif + return conf; } @@ -3396,6 +3419,12 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) #endif } +#if (NGX_HAVE_FSTACK) + /* By default, we set up a proxy on fstack */ + ngx_conf_merge_value(conf->kernel_network_stack, + prev->kernel_network_stack, 0); +#endif + return NGX_CONF_OK; } diff --git a/app/nginx-1.11.10/src/http/ngx_http_core_module.c b/app/nginx-1.11.10/src/http/ngx_http_core_module.c index 5bb6591e..734db7fc 100644 --- a/app/nginx-1.11.10/src/http/ngx_http_core_module.c +++ b/app/nginx-1.11.10/src/http/ngx_http_core_module.c @@ -3455,7 +3455,9 @@ ngx_http_core_create_srv_conf(ngx_conf_t *cf) cscf->ignore_invalid_headers = NGX_CONF_UNSET; cscf->merge_slashes = NGX_CONF_UNSET; cscf->underscores_in_headers = NGX_CONF_UNSET; +#if (NGX_HAVE_FSTACK) cscf->kernel_network_stack = NGX_CONF_UNSET; +#endif return cscf; } diff --git a/app/nginx-1.11.10/src/mail/ngx_mail_core_module.c b/app/nginx-1.11.10/src/mail/ngx_mail_core_module.c index fc8876d3..561e3c0d 100644 --- a/app/nginx-1.11.10/src/mail/ngx_mail_core_module.c +++ b/app/nginx-1.11.10/src/mail/ngx_mail_core_module.c @@ -67,10 +67,10 @@ static ngx_command_t ngx_mail_core_commands[] = { #if (NGX_HAVE_FSTACK) { ngx_string("kernel_network_stack"), - NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, + NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_FLAG, ngx_conf_set_flag_slot, - NGX_HTTP_SRV_CONF_OFFSET, - offsetof(ngx_http_core_srv_conf_t, kernel_network_stack), + NGX_MAIL_SRV_CONF_OFFSET, + offsetof(ngx_mail_core_srv_conf_t, kernel_network_stack), NULL }, #endif diff --git a/app/nginx-1.11.10/src/mail/ngx_mail_proxy_module.c b/app/nginx-1.11.10/src/mail/ngx_mail_proxy_module.c index 007284b6..ff2cbc45 100644 --- a/app/nginx-1.11.10/src/mail/ngx_mail_proxy_module.c +++ b/app/nginx-1.11.10/src/mail/ngx_mail_proxy_module.c @@ -18,6 +18,10 @@ typedef struct { ngx_flag_t xclient; size_t buffer_size; ngx_msec_t timeout; + +#if (NGX_HAVE_FSTACK) + ngx_flag_t kernel_network_stack; +#endif } ngx_mail_proxy_conf_t; @@ -74,6 +78,15 @@ static ngx_command_t ngx_mail_proxy_commands[] = { offsetof(ngx_mail_proxy_conf_t, xclient), NULL }, +#if (NGX_HAVE_FSTACK) + { ngx_string("proxy_kernel_network_stack"), + NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_MAIL_SRV_CONF_OFFSET, + offsetof(ngx_mail_proxy_conf_t, kernel_network_stack), + NULL }, +#endif + ngx_null_command }; @@ -135,6 +148,10 @@ ngx_mail_proxy_init(ngx_mail_session_t *s, ngx_addr_t *peer) p->upstream.log = s->connection->log; p->upstream.log_error = NGX_ERROR_ERR; +#if (NGX_HAVE_FSTACK) + p->upstream.belong_to_host = pcf->kernel_network_stack; +#endif + rc = ngx_event_connect_peer(&p->upstream); if (rc == NGX_ERROR || rc == NGX_BUSY || rc == NGX_DECLINED) { @@ -1102,6 +1119,10 @@ ngx_mail_proxy_create_conf(ngx_conf_t *cf) pcf->buffer_size = NGX_CONF_UNSET_SIZE; pcf->timeout = NGX_CONF_UNSET_MSEC; +#if (NGX_HAVE_FSTACK) + pcf->kernel_network_stack = NGX_CONF_UNSET; +#endif + return pcf; } @@ -1119,5 +1140,10 @@ ngx_mail_proxy_merge_conf(ngx_conf_t *cf, void *parent, void *child) (size_t) ngx_pagesize); ngx_conf_merge_msec_value(conf->timeout, prev->timeout, 24 * 60 * 60000); +#if (NGX_HAVE_FSTACK) + ngx_conf_merge_value(conf->kernel_network_stack, + prev->kernel_network_stack, 0); +#endif + return NGX_CONF_OK; } diff --git a/app/nginx-1.11.10/src/stream/ngx_stream_proxy_module.c b/app/nginx-1.11.10/src/stream/ngx_stream_proxy_module.c index 87117b02..3b9200e4 100644 --- a/app/nginx-1.11.10/src/stream/ngx_stream_proxy_module.c +++ b/app/nginx-1.11.10/src/stream/ngx_stream_proxy_module.c @@ -51,6 +51,10 @@ typedef struct { ngx_ssl_t *ssl; #endif +#if (NGX_HAVE_FSTACK) + ngx_flag_t kernel_network_stack; +#endif + ngx_stream_upstream_srv_conf_t *upstream; ngx_stream_complex_value_t *upstream_value; } ngx_stream_proxy_srv_conf_t; @@ -313,6 +317,15 @@ static ngx_command_t ngx_stream_proxy_commands[] = { #endif +#if (NGX_HAVE_FSTACK) + { ngx_string("proxy_kernel_network_stack"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_flag_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_proxy_srv_conf_t, kernel_network_stack), + NULL }, +#endif + ngx_null_command }; @@ -387,6 +400,10 @@ ngx_stream_proxy_handler(ngx_stream_session_t *s) u->peer.type = c->type; u->start_sec = ngx_time(); +#if (NGX_HAVE_FSTACK) + u->peer.belong_to_host = pscf->kernel_network_stack; +#endif + c->write->handler = ngx_stream_proxy_downstream_handler; c->read->handler = ngx_stream_proxy_downstream_handler; @@ -1860,6 +1877,10 @@ ngx_stream_proxy_create_srv_conf(ngx_conf_t *cf) conf->ssl_passwords = NGX_CONF_UNSET_PTR; #endif +#if (NGX_HAVE_FSTACK) + conf->kernel_network_stack = NGX_CONF_UNSET; +#endif + return conf; } @@ -1943,6 +1964,12 @@ ngx_stream_proxy_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) #endif +#if (NGX_HAVE_FSTACK) + /* By default, we set up a proxy on fstack */ + ngx_conf_merge_value(conf->kernel_network_stack, + prev->kernel_network_stack, 0); +#endif + return NGX_CONF_OK; } diff --git a/doc/F-Stack_Nginx_APP_Guide.md b/doc/F-Stack_Nginx_APP_Guide.md index 39066351..80686cea 100644 --- a/doc/F-Stack_Nginx_APP_Guide.md +++ b/doc/F-Stack_Nginx_APP_Guide.md @@ -42,21 +42,37 @@ first one to start | | | | | - a major addition to the worker process is fstack-handling：ff_init();ff_run(worker_process_cycle); worker_process_cycle(handle channel/host/fstack event). -- a new directive `kernel_network_stack` : +## What's Different? +### New directives: +All the directives below are available only when ```NGX_HAVE_FSTACK``` is defined. ``` Syntax: kernel_network_stack on | off; Default: kernel_network_stack off; Context: http, server - This directive is available only when ```NGX_HAVE_FSTACK``` is defined. + Determines whether server should run on kernel network stack or fstack. ``` -Note that: +``` + Syntax: proxy_kernel_network_stack on | off; + Default: kernel_network_stack off; + Context: http, stream, mail, server -- the `reload` is not graceful, service will still be unavailable during the process of reloading. + Determines whether proxy should go through kernel network stack or fstack. +``` -- necessary modifies in nginx.conf: +``` + Syntax: schedule_timeout time; + Default: schedule_timeout 30m; + Context: http, server + Sets a time interval for polling kernel_network_stack. The default value is 30 msec. +``` + +### Command-line `reload` +the `reload` is not graceful, service will still be unavailable during the process of reloading. + +### Necessary modifies in nginx.conf: ``` user root; # root account is necessary. fstack_conf f-stack.conf; # path of f-stack configuration file, default: $NGX_PREFIX/conf/f-stack.conf.