Merge pull request #159 from chadwill/master

Nginx :  ngx_configure_listening_sockets and hijack `recvmsg`

app/nginx-1.11.10/src/core/ngx_connection.c

@@ -13,6 +13,38 @@
 #if (NGX_HAVE_FSTACK)
 extern int fstack_territory(int domain, int type, int protocol);
 extern int is_fstack_fd(int sockfd);
+
+static ngx_inline int
+ngx_ff_skip_listening_socket(ngx_cycle_t *cycle, const ngx_listening_t  *ls)
+{
+    if (ngx_process <= NGX_PROCESS_MASTER) {
+
+        /* process master,  kernel network stack*/
+        if (!ls->belong_to_host) {
+            /* We should continue to process the listening socket,
+                  if it is not supported by fstack. */
+            if (fstack_territory(ls->sockaddr->sa_family, ls->type, 0)) {
+                return 1;
+            }
+        }
+    } else if (NGX_PROCESS_WORKER == ngx_process) {
+        /* process worker, fstack */
+        if (ls->belong_to_host) {
+            return 1;
+        }
+
+        if (!fstack_territory(ls->sockaddr->sa_family, ls->type, 0)) {
+            return 1;
+        }
+    } else {
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                            "unexpected process type: %d, ignored",
+                            ngx_process);
+        exit(1);
+    }
+
+    return 0;
+}
 #endif
 
 
@@ -411,30 +443,8 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle)
 
 #if (NGX_HAVE_FSTACK)
 
-            if (ngx_process <= NGX_PROCESS_MASTER) {
-
-                /* process master,  kernel network stack*/
-                if (!ls[i].belong_to_host) {
-                    /* We should continue to process the listening socket, 
-                        if it is not supported by fstack.*/
-                    if (fstack_territory(ls[i].sockaddr->sa_family, ls[i].type, 0)) {
-                        continue;
-                    }
-                }
-            } else if (NGX_PROCESS_WORKER == ngx_process) {
-                /* process worker, fstack */
-                if (ls[i].belong_to_host) {
-                    continue;
-                }
-
-                if (!fstack_territory(ls[i].sockaddr->sa_family, ls[i].type, 0)) {
-                    continue;
-                }
-            } else {
-                ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
-                                  "unexpected process type: %d, ignored",
-                                  ngx_process);
-                exit(1);
+            if(ngx_ff_skip_listening_socket(cycle, &ls[i])){
+                continue;
             }
 
 #endif
@@ -695,6 +705,14 @@ ngx_configure_listening_sockets(ngx_cycle_t *cycle)
     ls = cycle->listening.elts;
     for (i = 0; i < cycle->listening.nelts; i++) {
 
+#if (NGX_HAVE_FSTACK)
+
+        if(ngx_ff_skip_listening_socket(cycle, &ls[i])){
+            continue;
+        }
+
+#endif
+
         ls[i].log = *ls[i].logp;
 
         if (ls[i].rcvbuf != -1) {
@@ -941,7 +959,8 @@ ngx_configure_listening_sockets(ngx_cycle_t *cycle)
                               &ls[i].addr_text);
             }
         }
-
+/* TODO: configure ,  auto/feature  */
+#if !(NGX_HAVE_FSTACK)
 #elif (NGX_HAVE_IP_PKTINFO)
 
         if (ls[i].wildcard
@@ -960,7 +979,7 @@ ngx_configure_listening_sockets(ngx_cycle_t *cycle)
                               &ls[i].addr_text);
             }
         }
-
+#endif /* NGX_HAVE_FSTACK */
 #endif
 
 #if (NGX_HAVE_INET6 && NGX_HAVE_IPV6_RECVPKTINFO)

app/nginx-1.11.10/src/event/modules/ngx_ff_module.c

@@ -109,6 +109,7 @@ static ssize_t (*real_send)(int, const void *, size_t, int);
 static ssize_t (*real_sendto)(int, const void *, size_t, int,
     const struct sockaddr*, socklen_t);
 static ssize_t (*real_sendmsg)(int, const struct msghdr*, int);
+static ssize_t (*real_recvmsg)(int, struct msghdr *, int);
 static ssize_t (*real_writev)(int, const struct iovec *, int);
 static ssize_t (*real_readv)(int, const struct iovec *, int);
 
@@ -292,6 +293,16 @@ sendmsg(int sockfd, const struct msghdr *msg, int flags)
     return SYSCALL(sendmsg)(sockfd, msg, flags);
 }
 
+ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags)
+{
+    if(is_fstack_fd(sockfd)){
+        sockfd = restore_fstack_fd(sockfd);
+        return ff_recvmsg(sockfd, msg, flags);
+    }
+
+    return SYSCALL(recvmsg)(sockfd, msg, flags);
+}
+
 ssize_t
 recv(int sockfd, void *buf, size_t len, int flags)
 {

app/nginx-1.11.10/src/os/unix/ngx_process_cycle.c

@@ -1002,6 +1002,10 @@ ngx_worker_process_init(ngx_cycle_t *cycle, ngx_int_t worker)
                           "ngx_open_listening_sockets failed");
             exit(2);
         }
+
+        if (!ngx_test_config) {
+            ngx_configure_listening_sockets(cycle);
+        }
     }
 #endif
 

lib/ff_syscall_wrapper.c

@@ -334,9 +334,9 @@ linux2freebsd_sockaddr(const struct linux_sockaddr *linux,
         return;
     }
 
-    bzero(freebsd, sizeof(struct sockaddr));
-    freebsd->sa_len = addrlen;
+    /* #linux and #freebsd may point to the same address */
     freebsd->sa_family = linux->sa_family;
+    freebsd->sa_len = addrlen;
     bcopy(linux->sa_data, freebsd->sa_data, sizeof(linux->sa_data));
 }
 
@@ -609,8 +609,20 @@ ssize_t
 ff_sendmsg(int s, const struct msghdr *msg, int flags)
 {
     int rc;
+    struct sockaddr freebsd_sa;
+    void *linux_sa = msg->msg_name;
 
-    if ((rc = sendit(curthread, s, __DECONST(struct msghdr *, msg), flags)))
+    if (linux_sa != NULL) {
+        linux2freebsd_sockaddr(linux_sa,
+            sizeof(struct linux_sockaddr), &freebsd_sa);
+        __DECONST(struct msghdr *, msg)->msg_name = &freebsd_sa;
+    }
+
+    rc = sendit(curthread, s, __DECONST(struct msghdr *, msg), flags);
+
+    __DECONST(struct msghdr *, msg)->msg_name = linux_sa;
+
+    if (rc)
         goto kern_fail;
 
     return (rc);
@@ -676,6 +688,8 @@ ff_recvmsg(int s, struct msghdr *msg, int flags)
     }
     rc = curthread->td_retval[0];
 
+    freebsd2linux_sockaddr(msg->msg_name, msg->msg_name);
+
     return (rc);
 kern_fail:
     ff_os_errno(rc);