Fix nginx coredump on virtio vm and support kni.

1.close #67.
2.replace ff_primary process to primary worker process(the first one to
start and the last one to exit).
3.it's not graceful to reload.

app/nginx-1.11.10/src/event/modules/ngx_ff_channel.c

@@ -46,18 +46,14 @@ static ngx_int_t ngx_ff_epoll_process_events(ngx_cycle_t *cycle,
     ngx_msec_t timer, ngx_uint_t flags);
 static ngx_int_t ngx_ff_create_connection(ngx_cycle_t *cycle);
 static void ngx_ff_delete_connection();
-static void ngx_ff_primary_channel_handler(ngx_event_t *ev);
 static void ngx_ff_worker_channel_handler(ngx_event_t *ev);
 static void *ngx_ff_channel_thread_main(void *args);
 static ngx_int_t ngx_ff_add_channel_event(ngx_cycle_t *cycle,
     ngx_fd_t fd, ngx_int_t event, ngx_event_handler_pt handler);
+static ngx_int_t ngx_ff_process_channel_events(ngx_cycle_t *cycle);
 
 ngx_int_t ngx_ff_start_worker_channel(ngx_cycle_t *cycle,
     ngx_fd_t fd, ngx_int_t event);
-ngx_int_t ngx_ff_start_primary_channel(ngx_cycle_t *cycle,
-    ngx_fd_t fd, ngx_int_t event);
-ngx_int_t ngx_ff_process_channel_events(ngx_cycle_t *cycle);
-
 
 struct channel_thread_args {
     ngx_cycle_t *cycle;
@@ -650,79 +646,6 @@ ngx_ff_worker_channel_handler(ngx_event_t *ev)
     }
 }
 
-static void
-ngx_ff_primary_channel_handler(ngx_event_t *ev)
-{
-    ngx_int_t          n;
-    ngx_channel_t      ch;
-    ngx_connection_t  *c;
-
-    if (ev->timedout) {
-        ev->timedout = 0;
-        return;
-    }
-
-    c = ev->data;
-
-    ngx_log_debug0(NGX_LOG_DEBUG_CORE, ev->log, 0, "primary channel handler");
-
-    for ( ;; ) {
-
-        n = ngx_read_channel(c->fd, &ch, sizeof(ngx_channel_t), ev->log);
-
-        ngx_log_debug1(NGX_LOG_DEBUG_CORE, ev->log, 0, "channel: %i", n);
-
-        if (n == NGX_ERROR) {
-            ngx_ff_epoll_del_event(c->read, NGX_READ_EVENT, NGX_CLOSE_EVENT);
-            close(c->fd);
-            ngx_ff_delete_connection();
-            return;
-        }
-
-        if (n == NGX_AGAIN) {
-            return;
-        }
-
-        ngx_log_debug1(NGX_LOG_DEBUG_CORE, ev->log, 0,
-                       "channel command: %ui", ch.command);
-
-        switch (ch.command) {
-        case NGX_CMD_TERMINATE:
-            ngx_terminate = 1;
-            break;
-
-        case NGX_CMD_REOPEN:
-            ngx_reopen = 1;
-            break;
-
-        case NGX_CMD_OPEN_CHANNEL:
-
-            ngx_log_debug3(NGX_LOG_DEBUG_CORE, ev->log, 0,
-                           "get channel s:%i pid:%P fd:%d",
-                           ch.slot, ch.pid, ch.fd);
-
-            ngx_processes[ch.slot].pid = ch.pid;
-            ngx_processes[ch.slot].channel[0] = ch.fd;
-            break;
-
-        case NGX_CMD_CLOSE_CHANNEL:
-
-            ngx_log_debug4(NGX_LOG_DEBUG_CORE, ev->log, 0,
-                           "close channel s:%i pid:%P our:%P fd:%d",
-                           ch.slot, ch.pid, ngx_processes[ch.slot].pid,
-                           ngx_processes[ch.slot].channel[0]);
-
-            if (close(ngx_processes[ch.slot].channel[0]) == -1) {
-                ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno,
-                              "close() channel failed");
-            }
-
-            ngx_processes[ch.slot].channel[0] = -1;
-            break;
-        }
-    }
-}
-
 static void *
 ngx_ff_channel_thread_main(void *args)
 {
@@ -746,7 +669,7 @@ ngx_ff_channel_thread_main(void *args)
     return NULL;
 }
 
-ngx_int_t
+static ngx_int_t
 ngx_ff_process_channel_events(ngx_cycle_t *cycle)
 {
     return ngx_ff_epoll_process_events(cycle, 500, NGX_UPDATE_TIME);
@@ -782,12 +705,4 @@ ngx_ff_start_worker_channel(ngx_cycle_t *cycle, ngx_fd_t fd,
     return NGX_OK;
 }
 
-ngx_int_t
-ngx_ff_start_primary_channel(ngx_cycle_t *cycle,
-    ngx_fd_t fd, ngx_int_t event)
-{
-    return ngx_ff_add_channel_event(cycle, fd, event,
-        ngx_ff_primary_channel_handler);
-}
-
 #endif

app/nginx-1.11.10/src/os/unix/ngx_process_cycle.c

@@ -25,9 +25,6 @@ static void ngx_worker_process_exit(ngx_cycle_t *cycle);
 #if (NGX_HAVE_FSTACK)
 extern ngx_int_t ngx_ff_start_worker_channel(ngx_cycle_t *cycle,
     ngx_fd_t fd, ngx_int_t event);
-extern ngx_int_t ngx_ff_start_primary_channel(ngx_cycle_t *cycle,
-    ngx_fd_t fd, ngx_int_t event);
-extern ngx_int_t ngx_ff_process_channel_events(ngx_cycle_t *cycle);
 #else
 static void ngx_channel_handler(ngx_event_t *ev);
 #endif
@@ -80,141 +77,7 @@ static ngx_log_t        ngx_exit_log;
 static ngx_open_file_t  ngx_exit_log_file;
 
 #if (NGX_HAVE_FSTACK)
-
+static ngx_int_t ngx_ff_primary;
-static void
-ngx_ff_primary_process_exit(ngx_cycle_t *cycle)
-{
-    ngx_uint_t         i;
-
-    for (i = 0; cycle->modules[i]; i++) {
-        if (cycle->modules[i]->exit_process) {
-            cycle->modules[i]->exit_process(cycle);
-        }
-    }
-
-    /*
-     * Copy ngx_cycle->log related data to the special static exit cycle,
-     * log, and log file structures enough to allow a signal handler to log.
-     * The handler may be called when standard ngx_cycle->log allocated from
-     * ngx_cycle->pool is already destroyed.
-     */
-
-    ngx_exit_log = *ngx_log_get_file_log(ngx_cycle->log);
-
-    ngx_exit_log_file.fd = ngx_exit_log.file->fd;
-    ngx_exit_log.file = &ngx_exit_log_file;
-    ngx_exit_log.next = NULL;
-    ngx_exit_log.writer = NULL;
-
-    ngx_exit_cycle.log = &ngx_exit_log;
-    ngx_exit_cycle.files = ngx_cycle->files;
-    ngx_exit_cycle.files_n = ngx_cycle->files_n;
-    ngx_cycle = &ngx_exit_cycle;
-
-    ngx_destroy_pool(cycle->pool);
-
-    // wait worker process exited.
-    ngx_msleep(500);
-
-    ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, "exit");
-
-    exit(0);
-}
-
-static void
-ngx_ff_primary_process_cycle(ngx_cycle_t *cycle, void *data)
-{
-    ngx_core_conf_t   *ccf;
-    ngx_uint_t        i;
-    ngx_int_t         n;
-
-    ngx_process = NGX_PROCESS_WORKER;
-
-    ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx,
-        ngx_core_module);
-
-    ngx_setproctitle("ff primary process");
-
-    if (ff_mod_init((const char *)ccf->fstack_conf.data, 0, 1)) {
-        exit(2);
-    }
-
-    for (i = 0; cycle->modules[i]; i++) {
-        if (cycle->modules[i]->init_process) {
-            if (cycle->modules[i]->init_process(cycle) == NGX_ERROR) {
-                /* fatal */
-                exit(2);
-            }
-        }
-    }
-
-    for (n = 0; n < ngx_last_process; n++) {
-
-        if (ngx_processes[n].pid == -1) {
-            continue;
-        }
-
-        if (n == ngx_process_slot) {
-            continue;
-        }
-
-        if (ngx_processes[n].channel[1] == -1) {
-            continue;
-        }
-
-        if (close(ngx_processes[n].channel[1]) == -1) {
-            ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
-                          "close() channel failed");
-        }
-    }
-
-    if (close(ngx_processes[ngx_process_slot].channel[0]) == -1) {
-        ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
-                      "close() channel failed");
-    }
-
-    if (ngx_ff_start_primary_channel(cycle, ngx_channel, NGX_READ_EVENT)) {
-        exit(2);
-    }
-
-    for ( ;; ) {
-
-        if (ngx_terminate) {
-            ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "exiting");
-            ngx_ff_primary_process_exit(cycle);
-        }
-
-        if (ngx_reopen) {
-            ngx_reopen = 0;
-            ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "reopening logs");
-            ngx_reopen_files(cycle, -1);
-        }
-
-        (void) ngx_ff_process_channel_events(cycle);
-    }
-}
-
-static void
-ngx_start_ff_primary_process(ngx_cycle_t *cycle)
-{
-    ngx_channel_t  ch;
-    ngx_memzero(&ch, sizeof(ngx_channel_t));
-
-    ch.command = NGX_CMD_OPEN_CHANNEL;
-
-    ngx_spawn_process(cycle, ngx_ff_primary_process_cycle,
-                      NULL, "ff primary process",
-                      NGX_PROCESS_RESPAWN);
-
-    ch.pid = ngx_processes[ngx_process_slot].pid;
-    ch.slot = ngx_process_slot;
-    ch.fd = ngx_processes[ngx_process_slot].channel[0];
-
-    ngx_pass_open_channel(cycle, &ch);
-
-    // wait for primary process startup.
-    ngx_sleep(1);
-}
 #endif
 
 void
@@ -226,6 +89,7 @@ ngx_master_process_cycle(ngx_cycle_t *cycle)
     ngx_int_t          i;
 #if (NGX_HAVE_FSTACK)
     ngx_uint_t         sigio;
+    ngx_uint_t         sig_worker_quit = 0;
 #else
     ngx_uint_t         n, sigio;
 #endif
@@ -281,16 +145,6 @@ ngx_master_process_cycle(ngx_cycle_t *cycle)
 
     ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
 
-#if (NGX_HAVE_FSTACK)
-    if (ccf->fstack_conf.len == 0) {
-        ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
-                      "fstack_conf null");
-        exit(2);
-    }
-
-    ngx_start_ff_primary_process(cycle);
-#endif
-
     ngx_start_worker_processes(cycle, ccf->worker_processes,
                                NGX_PROCESS_RESPAWN);
     ngx_start_cache_manager_processes(cycle, 0);
@@ -383,6 +237,20 @@ ngx_master_process_cycle(ngx_cycle_t *cycle)
         }
 
         if (ngx_reconfigure) {
+#if (NGX_HAVE_FSTACK)
+            if (!sig_worker_quit) {
+                sig_worker_quit = 1;
+                ngx_signal_worker_processes(cycle,
+                    ngx_signal_value(NGX_SHUTDOWN_SIGNAL));
+                continue;
+            }
+
+            if (live) {
+                continue;
+            }
+
+            sig_worker_quit = 0;
+#endif
             ngx_reconfigure = 0;
 
             if (ngx_new_binary) {
@@ -527,6 +395,12 @@ ngx_start_worker_processes(ngx_cycle_t *cycle, ngx_int_t n, ngx_int_t type)
         ch.fd = ngx_processes[ngx_process_slot].channel[0];
 
         ngx_pass_open_channel(cycle, &ch);
+#if (NGX_HAVE_FSTACK)
+        if (i == 0) {
+            // wait for ff_primary worker process startup.
+            ngx_sleep(1);
+        }
+#endif
     }
 }
 
@@ -780,7 +654,11 @@ ngx_reap_children(ngx_cycle_t *cycle)
             if (ngx_processes[i].respawn
                 && !ngx_processes[i].exiting
                 && !ngx_terminate
-                && !ngx_quit)
+                && !ngx_quit
+#if (NGX_HAVE_FSTACK)
+                && !ngx_reconfigure
+#endif
+            )
             {
                 if (ngx_spawn_process(cycle, ngx_processes[i].proc,
                                       ngx_processes[i].data,
@@ -1115,7 +993,12 @@ ngx_worker_process_init(ngx_cycle_t *cycle, ngx_int_t worker)
             exit(2);
         }
 
-        if (ff_mod_init((const char *)ccf->fstack_conf.data, worker, 0)) {
+        if (worker == 0) {
+            ngx_ff_primary = 1;
+        }
+
+        if (ff_mod_init((const char *)ccf->fstack_conf.data, worker,
+            ngx_ff_primary)) {
             ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
                           "ff_mod_init failed");
             exit(2);
@@ -1247,6 +1130,12 @@ ngx_worker_process_exit(ngx_cycle_t *cycle)
 
     ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, "exit");
 
+#if (NGX_HAVE_FSTACK)
+    if (ngx_ff_primary) {
+        // wait for secondary worker processes to exit.
+        ngx_msleep(500);
+    }
+#endif
     exit(0);
 }
 

doc/F-Stack_Nginx_APP_Guide.md

@@ -9,10 +9,10 @@ F-Stack is an open source network framework based on DPDK. F-Stack supports stan
 ```
 
                                                         +--------+
+                         +------------------------+     |
+                            channel: socketpair         |
+                         +------------------------+     |  signal(reload, quit..)
                                                         |
-+------------------------+                          |
-   channel: socketpair                              |  signal(reload, quit..)
-+------------------------+                          |
                                                         |
                                               +---------v--------+
                                               |                  |
@@ -23,37 +23,32 @@ F-Stack is an open source network framework based on DPDK. F-Stack supports stan
                              |                  channel  |                       |
                              |                           |                       |
                    +---------+----------+     +----------+--------+    +---------+--------+
-               |                    |     |                   |    |                  |
+first one to start |                    |     |                   |    |                  |
-               | ff primary process |     |  worker process   |    |  worker process  |
+ last one to exit<-+   primary worker   |     |  secondary worker |    | secondary worker |
                    |                    |     |                   |    |                  |
                    +--------------------+     +-------------------+    +------------------+
-                ff_init(primary)          +--------+  +-------+
+                   +--------+  +-------+      +--------+  +-------+
-                loop:                     |        |  |       |
+                   |        |  |       |      |        |  |       |
-                 handle channel event     | fstack |  |channel|
+                   | fstack |  |channel|      | fstack |  |channel|
-                                          |  main  |  | event |
+                   |  main  |  | event |      |  main  |  | event |
-                                          |  loop  |  |thread |
+                   |  loop  |  |thread |      |  loop  |  |thread |
-                                          | thread |  |       |
+                   | thread |  |       |      | thread |  |       |
-                                          |        |  |       |
+                   |        |  |       |      |        |  |       |
-                                          +--------+  +-------+
+                   +--------+  +-------+      +--------+  +-------+
-                                           woker       loop:
+                    woker       loop:          worker      loop:
-                                          process      handle
+                   process      handle        process      handle
-                                           cycle      channel
+                    cycle      channel         cycle      channel
-                                                       event
+                                event                      event
 
 ```
 
-- spawn an extra process ff primary: ff_init(primary);loop(handle channel event).This process doesn't handle SIGQUIT.
+- spawn primary worker firstly, and then wait for primary startup, continue to spawn secondary workers.
 
-- worker process has 2 threads. main thread: ff_init(secondary);ff_run(worker_process_cycle), channel thread: loop(handle channel event).
+- worker process has 2 threads. main thread: ff_init();ff_run(worker_process_cycle), channel thread: loop(handle channel event).
 
 Note that:
-- kni couldn't be enabled in this version, because kni is only processed by primary and worker `ff_primary` won't execute `ff_run()`.
 
-- supported nginx signals: reload(HUP)/reopen(USR1)/stop(TERM).
+- the `reload` is not graceful, service will still be unavailable during the process of reloading.
-
-- unsupported nginx signals: NGX_CHANGEBIN_SIGNAL(USR2).
-
-- when use `nginx -s reload`, you should make sure that `woker_processes` in nginx.conf and f-stack.conf haven't be modified.
 
 - necessary modifies in nginx.conf: