From e340d433ea27c25cb1e2bdb467618483c2fcf5a7 Mon Sep 17 00:00:00 2001 From: logwang Date: Thu, 11 Jan 2018 18:03:24 +0800 Subject: [PATCH] Nginx: fix crash when server configuration item [`kernel_network_stack`] is on and uses `proxy_pass`. When nginx is configured like this: ``` server { listen 8000; kernel_network_stack on; location / { proxy_pass http://127.0.0.1:8080/; } } ``` nginx will crash, becasue kernel network stack is handled in a single thread, but we have hijacked all the socket apis, it causes that all apis enter to f-stack's path which is in main thread. --- app/nginx-1.11.10/src/core/ngx_connection.c | 15 +++++--- .../src/event/modules/ngx_ff_module.c | 36 +++++++++---------- app/nginx-1.11.10/src/event/ngx_event.c | 2 +- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/app/nginx-1.11.10/src/core/ngx_connection.c b/app/nginx-1.11.10/src/core/ngx_connection.c index 45d90e66b..9fe1dbf6a 100644 --- a/app/nginx-1.11.10/src/core/ngx_connection.c +++ b/app/nginx-1.11.10/src/core/ngx_connection.c @@ -10,6 +10,12 @@ #include +#if (NGX_HAVE_FSTACK) +extern int fstack_territory(int domain, int type, int protocol); +extern int is_fstack_fd(int sockfd); +#endif + + ngx_os_io_t ngx_io; @@ -376,11 +382,6 @@ ngx_set_inherited_sockets(ngx_cycle_t *cycle) return NGX_OK; } -#if (NGX_HAVE_FSTACK) -extern int - fstack_territory(int domain, int type, int protocol); -#endif - ngx_int_t ngx_open_listening_sockets(ngx_cycle_t *cycle) { @@ -1166,6 +1167,10 @@ ngx_get_connection(ngx_socket_t s, ngx_log_t *log) wev->write = 1; +#if (NGX_HAVE_FSTACK) + rev->belong_to_host = wev->belong_to_host = is_fstack_fd(s) ? 0 : 1; +#endif + return c; } diff --git a/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c b/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c index 0d37baff5..4bf5537d8 100644 --- a/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c +++ b/app/nginx-1.11.10/src/event/modules/ngx_ff_module.c @@ -119,7 +119,7 @@ static int (*real_ioctl)(int, int, void *); static int (*real_gettimeofday)(struct timeval *tv, struct timezone *tz); -static int inited; +static __thread int inited; #define SYSCALL(func) \ ({ \ @@ -151,7 +151,7 @@ static inline int restore_fstack_fd(int sockfd) { } /* Tell whether a 'sockfd' belongs to fstack. */ -static inline int is_fstack_fd(int sockfd) { +int is_fstack_fd(int sockfd) { if (unlikely(inited == 0)) { return 0; } @@ -242,7 +242,7 @@ bind(int sockfd, const struct sockaddr *addr, socklen_t addrlen) sockfd = restore_fstack_fd(sockfd); return ff_bind(sockfd, (struct linux_sockaddr *)addr, addrlen); } - + return SYSCALL(bind)(sockfd, addr, addrlen); } @@ -253,7 +253,7 @@ connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen) sockfd = restore_fstack_fd(sockfd); return ff_connect(sockfd, (struct linux_sockaddr *)addr, addrlen); } - + return SYSCALL(connect)(sockfd, addr, addrlen); } @@ -264,7 +264,7 @@ send(int sockfd, const void *buf, size_t len, int flags) sockfd = restore_fstack_fd(sockfd); return ff_send(sockfd, buf, len, flags); } - + return SYSCALL(send)(sockfd, buf, len, flags); } @@ -288,7 +288,7 @@ sendmsg(int sockfd, const struct msghdr *msg, int flags) sockfd = restore_fstack_fd(sockfd); return ff_sendmsg(sockfd, msg, flags); } - + return SYSCALL(sendmsg)(sockfd, msg, flags); } @@ -299,7 +299,7 @@ recv(int sockfd, void *buf, size_t len, int flags) sockfd = restore_fstack_fd(sockfd); return ff_recv(sockfd, buf, len, flags); } - + return SYSCALL(recv)(sockfd, buf, len, flags); } @@ -310,7 +310,7 @@ listen(int sockfd, int backlog) sockfd = restore_fstack_fd(sockfd); return ff_listen(sockfd, backlog); } - + return SYSCALL(listen)(sockfd, backlog); } @@ -322,7 +322,7 @@ getsockopt(int sockfd, int level, int optname, sockfd = restore_fstack_fd(sockfd); return ff_getsockopt(sockfd, level, optname, optval, optlen); } - + return SYSCALL(getsockopt)(sockfd, level, optname, optval, optlen); } @@ -334,7 +334,7 @@ setsockopt (int sockfd, int level, int optname, sockfd = restore_fstack_fd(sockfd); return ff_setsockopt(sockfd, level, optname, optval, optlen); } - + return SYSCALL(setsockopt)(sockfd, level, optname, optval, optlen); } @@ -351,7 +351,7 @@ accept(int sockfd, struct sockaddr *addr, socklen_t *addrlen) return rc; } - + return SYSCALL(accept)(sockfd, addr, addrlen); } @@ -368,7 +368,7 @@ accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags) return rc; } - + return SYSCALL(accept4)(sockfd, addr, addrlen, flags); } @@ -379,7 +379,7 @@ close(int sockfd) sockfd = restore_fstack_fd(sockfd); return ff_close(sockfd); } - + return SYSCALL(close)(sockfd); } @@ -390,7 +390,7 @@ writev(int sockfd, const struct iovec *iov, int iovcnt) sockfd = restore_fstack_fd(sockfd); return ff_writev(sockfd, iov, iovcnt); } - + return SYSCALL(writev)(sockfd, iov, iovcnt); } @@ -401,7 +401,7 @@ readv(int sockfd, const struct iovec *iov, int iovcnt) sockfd = restore_fstack_fd(sockfd); return ff_readv(sockfd, iov, iovcnt); } - + return SYSCALL(readv)(sockfd, iov, iovcnt); } @@ -412,7 +412,7 @@ read(int sockfd, void *buf, size_t count) sockfd = restore_fstack_fd(sockfd); return ff_read(sockfd, buf, count); } - + return SYSCALL(read)(sockfd, buf, count); } @@ -423,7 +423,7 @@ write(int sockfd, const void *buf, size_t count) sockfd = restore_fstack_fd(sockfd); return ff_write(sockfd, buf, count); } - + return SYSCALL(write)(sockfd, buf, count); } @@ -434,7 +434,7 @@ ioctl(int sockfd, int request, void *p) sockfd = restore_fstack_fd(sockfd); return ff_ioctl(sockfd, request, p); } - + return SYSCALL(ioctl)(sockfd, request, p); } diff --git a/app/nginx-1.11.10/src/event/ngx_event.c b/app/nginx-1.11.10/src/event/ngx_event.c index 71016c95b..2b7b50a87 100644 --- a/app/nginx-1.11.10/src/event/ngx_event.c +++ b/app/nginx-1.11.10/src/event/ngx_event.c @@ -783,7 +783,7 @@ ngx_event_process_init(ngx_cycle_t *cycle) #if (NGX_HAVE_FSTACK) /* Note when nginx running on fstack, make sure that add the right fd to kqueue !! */ - c->read->belong_to_host = c->write->belong_to_host = ls[i].belong_to_host; + c->read->belong_to_host = c->write->belong_to_host = ls[i].belong_to_host; #endif #if (NGX_HAVE_DEFERRED_ACCEPT)