diff --git a/app/nginx-1.11.10/src/event/modules/ngx_ff_channel.c b/app/nginx-1.11.10/src/event/modules/ngx_ff_channel.c index fbaf8c37..c52a2ad1 100644 --- a/app/nginx-1.11.10/src/event/modules/ngx_ff_channel.c +++ b/app/nginx-1.11.10/src/event/modules/ngx_ff_channel.c @@ -46,18 +46,14 @@ static ngx_int_t ngx_ff_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags); static ngx_int_t ngx_ff_create_connection(ngx_cycle_t *cycle); static void ngx_ff_delete_connection(); -static void ngx_ff_primary_channel_handler(ngx_event_t *ev); static void ngx_ff_worker_channel_handler(ngx_event_t *ev); static void *ngx_ff_channel_thread_main(void *args); static ngx_int_t ngx_ff_add_channel_event(ngx_cycle_t *cycle, ngx_fd_t fd, ngx_int_t event, ngx_event_handler_pt handler); +static ngx_int_t ngx_ff_process_channel_events(ngx_cycle_t *cycle); ngx_int_t ngx_ff_start_worker_channel(ngx_cycle_t *cycle, ngx_fd_t fd, ngx_int_t event); -ngx_int_t ngx_ff_start_primary_channel(ngx_cycle_t *cycle, - ngx_fd_t fd, ngx_int_t event); -ngx_int_t ngx_ff_process_channel_events(ngx_cycle_t *cycle); - struct channel_thread_args { ngx_cycle_t *cycle; @@ -650,79 +646,6 @@ ngx_ff_worker_channel_handler(ngx_event_t *ev) } } -static void -ngx_ff_primary_channel_handler(ngx_event_t *ev) -{ - ngx_int_t n; - ngx_channel_t ch; - ngx_connection_t *c; - - if (ev->timedout) { - ev->timedout = 0; - return; - } - - c = ev->data; - - ngx_log_debug0(NGX_LOG_DEBUG_CORE, ev->log, 0, "primary channel handler"); - - for ( ;; ) { - - n = ngx_read_channel(c->fd, &ch, sizeof(ngx_channel_t), ev->log); - - ngx_log_debug1(NGX_LOG_DEBUG_CORE, ev->log, 0, "channel: %i", n); - - if (n == NGX_ERROR) { - ngx_ff_epoll_del_event(c->read, NGX_READ_EVENT, NGX_CLOSE_EVENT); - close(c->fd); - ngx_ff_delete_connection(); - return; - } - - if (n == NGX_AGAIN) { - return; - } - - ngx_log_debug1(NGX_LOG_DEBUG_CORE, ev->log, 0, - "channel command: %ui", ch.command); - - switch (ch.command) { - case NGX_CMD_TERMINATE: - ngx_terminate = 1; - break; - - case NGX_CMD_REOPEN: - ngx_reopen = 1; - break; - - case NGX_CMD_OPEN_CHANNEL: - - ngx_log_debug3(NGX_LOG_DEBUG_CORE, ev->log, 0, - "get channel s:%i pid:%P fd:%d", - ch.slot, ch.pid, ch.fd); - - ngx_processes[ch.slot].pid = ch.pid; - ngx_processes[ch.slot].channel[0] = ch.fd; - break; - - case NGX_CMD_CLOSE_CHANNEL: - - ngx_log_debug4(NGX_LOG_DEBUG_CORE, ev->log, 0, - "close channel s:%i pid:%P our:%P fd:%d", - ch.slot, ch.pid, ngx_processes[ch.slot].pid, - ngx_processes[ch.slot].channel[0]); - - if (close(ngx_processes[ch.slot].channel[0]) == -1) { - ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno, - "close() channel failed"); - } - - ngx_processes[ch.slot].channel[0] = -1; - break; - } - } -} - static void * ngx_ff_channel_thread_main(void *args) { @@ -746,7 +669,7 @@ ngx_ff_channel_thread_main(void *args) return NULL; } -ngx_int_t +static ngx_int_t ngx_ff_process_channel_events(ngx_cycle_t *cycle) { return ngx_ff_epoll_process_events(cycle, 500, NGX_UPDATE_TIME); @@ -769,7 +692,7 @@ ngx_ff_start_worker_channel(ngx_cycle_t *cycle, ngx_fd_t fd, cta->event = event; cta->handler = ngx_ff_worker_channel_handler; - ret = pthread_create(&channel_thread, NULL, + ret = pthread_create(&channel_thread, NULL, ngx_ff_channel_thread_main, (void *)cta); if (ret != 0) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, @@ -782,12 +705,4 @@ ngx_ff_start_worker_channel(ngx_cycle_t *cycle, ngx_fd_t fd, return NGX_OK; } -ngx_int_t -ngx_ff_start_primary_channel(ngx_cycle_t *cycle, - ngx_fd_t fd, ngx_int_t event) -{ - return ngx_ff_add_channel_event(cycle, fd, event, - ngx_ff_primary_channel_handler); -} - #endif diff --git a/app/nginx-1.11.10/src/os/unix/ngx_process_cycle.c b/app/nginx-1.11.10/src/os/unix/ngx_process_cycle.c index 896ff70d..ea4f2083 100644 --- a/app/nginx-1.11.10/src/os/unix/ngx_process_cycle.c +++ b/app/nginx-1.11.10/src/os/unix/ngx_process_cycle.c @@ -25,9 +25,6 @@ static void ngx_worker_process_exit(ngx_cycle_t *cycle); #if (NGX_HAVE_FSTACK) extern ngx_int_t ngx_ff_start_worker_channel(ngx_cycle_t *cycle, ngx_fd_t fd, ngx_int_t event); -extern ngx_int_t ngx_ff_start_primary_channel(ngx_cycle_t *cycle, - ngx_fd_t fd, ngx_int_t event); -extern ngx_int_t ngx_ff_process_channel_events(ngx_cycle_t *cycle); #else static void ngx_channel_handler(ngx_event_t *ev); #endif @@ -80,141 +77,7 @@ static ngx_log_t ngx_exit_log; static ngx_open_file_t ngx_exit_log_file; #if (NGX_HAVE_FSTACK) - -static void -ngx_ff_primary_process_exit(ngx_cycle_t *cycle) -{ - ngx_uint_t i; - - for (i = 0; cycle->modules[i]; i++) { - if (cycle->modules[i]->exit_process) { - cycle->modules[i]->exit_process(cycle); - } - } - - /* - * Copy ngx_cycle->log related data to the special static exit cycle, - * log, and log file structures enough to allow a signal handler to log. - * The handler may be called when standard ngx_cycle->log allocated from - * ngx_cycle->pool is already destroyed. - */ - - ngx_exit_log = *ngx_log_get_file_log(ngx_cycle->log); - - ngx_exit_log_file.fd = ngx_exit_log.file->fd; - ngx_exit_log.file = &ngx_exit_log_file; - ngx_exit_log.next = NULL; - ngx_exit_log.writer = NULL; - - ngx_exit_cycle.log = &ngx_exit_log; - ngx_exit_cycle.files = ngx_cycle->files; - ngx_exit_cycle.files_n = ngx_cycle->files_n; - ngx_cycle = &ngx_exit_cycle; - - ngx_destroy_pool(cycle->pool); - - // wait worker process exited. - ngx_msleep(500); - - ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, "exit"); - - exit(0); -} - -static void -ngx_ff_primary_process_cycle(ngx_cycle_t *cycle, void *data) -{ - ngx_core_conf_t *ccf; - ngx_uint_t i; - ngx_int_t n; - - ngx_process = NGX_PROCESS_WORKER; - - ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, - ngx_core_module); - - ngx_setproctitle("ff primary process"); - - if (ff_mod_init((const char *)ccf->fstack_conf.data, 0, 1)) { - exit(2); - } - - for (i = 0; cycle->modules[i]; i++) { - if (cycle->modules[i]->init_process) { - if (cycle->modules[i]->init_process(cycle) == NGX_ERROR) { - /* fatal */ - exit(2); - } - } - } - - for (n = 0; n < ngx_last_process; n++) { - - if (ngx_processes[n].pid == -1) { - continue; - } - - if (n == ngx_process_slot) { - continue; - } - - if (ngx_processes[n].channel[1] == -1) { - continue; - } - - if (close(ngx_processes[n].channel[1]) == -1) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, - "close() channel failed"); - } - } - - if (close(ngx_processes[ngx_process_slot].channel[0]) == -1) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, - "close() channel failed"); - } - - if (ngx_ff_start_primary_channel(cycle, ngx_channel, NGX_READ_EVENT)) { - exit(2); - } - - for ( ;; ) { - - if (ngx_terminate) { - ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "exiting"); - ngx_ff_primary_process_exit(cycle); - } - - if (ngx_reopen) { - ngx_reopen = 0; - ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "reopening logs"); - ngx_reopen_files(cycle, -1); - } - - (void) ngx_ff_process_channel_events(cycle); - } -} - -static void -ngx_start_ff_primary_process(ngx_cycle_t *cycle) -{ - ngx_channel_t ch; - ngx_memzero(&ch, sizeof(ngx_channel_t)); - - ch.command = NGX_CMD_OPEN_CHANNEL; - - ngx_spawn_process(cycle, ngx_ff_primary_process_cycle, - NULL, "ff primary process", - NGX_PROCESS_RESPAWN); - - ch.pid = ngx_processes[ngx_process_slot].pid; - ch.slot = ngx_process_slot; - ch.fd = ngx_processes[ngx_process_slot].channel[0]; - - ngx_pass_open_channel(cycle, &ch); - - // wait for primary process startup. - ngx_sleep(1); -} +static ngx_int_t ngx_ff_primary; #endif void @@ -226,6 +89,7 @@ ngx_master_process_cycle(ngx_cycle_t *cycle) ngx_int_t i; #if (NGX_HAVE_FSTACK) ngx_uint_t sigio; + ngx_uint_t sig_worker_quit = 0; #else ngx_uint_t n, sigio; #endif @@ -281,16 +145,6 @@ ngx_master_process_cycle(ngx_cycle_t *cycle) ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); -#if (NGX_HAVE_FSTACK) - if (ccf->fstack_conf.len == 0) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, - "fstack_conf null"); - exit(2); - } - - ngx_start_ff_primary_process(cycle); -#endif - ngx_start_worker_processes(cycle, ccf->worker_processes, NGX_PROCESS_RESPAWN); ngx_start_cache_manager_processes(cycle, 0); @@ -383,6 +237,20 @@ ngx_master_process_cycle(ngx_cycle_t *cycle) } if (ngx_reconfigure) { +#if (NGX_HAVE_FSTACK) + if (!sig_worker_quit) { + sig_worker_quit = 1; + ngx_signal_worker_processes(cycle, + ngx_signal_value(NGX_SHUTDOWN_SIGNAL)); + continue; + } + + if (live) { + continue; + } + + sig_worker_quit = 0; +#endif ngx_reconfigure = 0; if (ngx_new_binary) { @@ -527,6 +395,12 @@ ngx_start_worker_processes(ngx_cycle_t *cycle, ngx_int_t n, ngx_int_t type) ch.fd = ngx_processes[ngx_process_slot].channel[0]; ngx_pass_open_channel(cycle, &ch); +#if (NGX_HAVE_FSTACK) + if (i == 0) { + // wait for ff_primary worker process startup. + ngx_sleep(1); + } +#endif } } @@ -780,7 +654,11 @@ ngx_reap_children(ngx_cycle_t *cycle) if (ngx_processes[i].respawn && !ngx_processes[i].exiting && !ngx_terminate - && !ngx_quit) + && !ngx_quit +#if (NGX_HAVE_FSTACK) + && !ngx_reconfigure +#endif + ) { if (ngx_spawn_process(cycle, ngx_processes[i].proc, ngx_processes[i].data, @@ -1115,7 +993,12 @@ ngx_worker_process_init(ngx_cycle_t *cycle, ngx_int_t worker) exit(2); } - if (ff_mod_init((const char *)ccf->fstack_conf.data, worker, 0)) { + if (worker == 0) { + ngx_ff_primary = 1; + } + + if (ff_mod_init((const char *)ccf->fstack_conf.data, worker, + ngx_ff_primary)) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "ff_mod_init failed"); exit(2); @@ -1247,6 +1130,12 @@ ngx_worker_process_exit(ngx_cycle_t *cycle) ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, "exit"); +#if (NGX_HAVE_FSTACK) + if (ngx_ff_primary) { + // wait for secondary worker processes to exit. + ngx_msleep(500); + } +#endif exit(0); } diff --git a/doc/F-Stack_Nginx_APP_Guide.md b/doc/F-Stack_Nginx_APP_Guide.md index 12a50086..a77c88e3 100644 --- a/doc/F-Stack_Nginx_APP_Guide.md +++ b/doc/F-Stack_Nginx_APP_Guide.md @@ -8,52 +8,47 @@ F-Stack is an open source network framework based on DPDK. F-Stack supports stan ``` - +--------+ - | -+------------------------+ | - channel: socketpair | signal(reload, quit..) -+------------------------+ | - | - +---------v--------+ - | | - +----------------+ master process +---------------+ - | | | | - | channel +----------+-------+ | - | | channel | - | channel | | - | | | - +---------+----------+ +----------+--------+ +---------+--------+ - | | | | | | - | ff primary process | | worker process | | worker process | - | | | | | | - +--------------------+ +-------------------+ +------------------+ - ff_init(primary) +--------+ +-------+ - loop: | | | | - handle channel event | fstack | |channel| - | main | | event | - | loop | |thread | - | thread | | | - | | | | - +--------+ +-------+ - woker loop: - process handle - cycle channel - event + +--------+ + +------------------------+ | + channel: socketpair | + +------------------------+ | signal(reload, quit..) + | + | + +---------v--------+ + | | + +----------------+ master process +---------------+ + | | | | + | channel +----------+-------+ | + | | channel | + | channel | | + | | | + +---------+----------+ +----------+--------+ +---------+--------+ +first one to start | | | | | | + last one to exit<-+ primary worker | | secondary worker | | secondary worker | + | | | | | | + +--------------------+ +-------------------+ +------------------+ + +--------+ +-------+ +--------+ +-------+ + | | | | | | | | + | fstack | |channel| | fstack | |channel| + | main | | event | | main | | event | + | loop | |thread | | loop | |thread | + | thread | | | | thread | | | + | | | | | | | | + +--------+ +-------+ +--------+ +-------+ + woker loop: worker loop: + process handle process handle + cycle channel cycle channel + event event ``` -- spawn an extra process ff primary: ff_init(primary);loop(handle channel event).This process doesn't handle SIGQUIT. +- spawn primary worker firstly, and then wait for primary startup, continue to spawn secondary workers. -- worker process has 2 threads. main thread: ff_init(secondary);ff_run(worker_process_cycle), channel thread: loop(handle channel event). +- worker process has 2 threads. main thread: ff_init();ff_run(worker_process_cycle), channel thread: loop(handle channel event). Note that: -- kni couldn't be enabled in this version, because kni is only processed by primary and worker `ff_primary` won't execute `ff_run()`. -- supported nginx signals: reload(HUP)/reopen(USR1)/stop(TERM). - -- unsupported nginx signals: NGX_CHANGEBIN_SIGNAL(USR2). - -- when use `nginx -s reload`, you should make sure that `woker_processes` in nginx.conf and f-stack.conf haven't be modified. +- the `reload` is not graceful, service will still be unavailable during the process of reloading. - necessary modifies in nginx.conf: