Support timeout is NULL or 0 in libffsyscall.so.

And support it in main_stack.c.

adapter/ff_adapter.h

@@ -10,6 +10,8 @@
 int ff_adapter_init();
 //int __attribute__((constructor)) ff_adapter_init(int argc, char * const argv[]);
 
+void alarm_event_sem();
+
 /*-
  * Verify whether the socket is supported by fstack or not.
  */

adapter/ff_hook_syscall.c

@@ -129,6 +129,8 @@ static int nb_procs = NB_FSTACK_INSTANCE_DEFAULT;
 #define FF_KERNEL_MAX_FD_DEFAULT    1024
 static int ff_kernel_max_fd = FF_KERNEL_MAX_FD_DEFAULT;
 
+static int need_alarm_sem = 0;
+
 static inline int convert_fstack_fd(int sockfd) {
     return sockfd + ff_kernel_max_fd;
 }
@@ -1231,6 +1233,11 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events,
     ACQUIRE_ZONE_LOCK(FF_SC_IDLE);
     sc->ops = FF_SO_EPOLL_WAIT;
     sc->args = args;
+
+    if (timeout == 0) {
+        need_alarm_sem = 1;
+    }
+
     RELEASE_ZONE_LOCK(FF_SC_REQ);
 
     errno = 0;
@@ -1257,6 +1264,10 @@ ff_hook_epoll_wait(int epfd, struct epoll_event *events,
 
     rte_spinlock_lock(&sc->lock);
 
+    if (timeout == 0) {
+        need_alarm_sem = 1;
+    }
+
     /*
      * After sem_timedwait, and before lock sc, sc->status may be modify from FF_SC_REQ to FF_SC_RSP,
      * so it can't use to check.
@@ -1380,6 +1391,10 @@ kevent(int kq, const struct kevent *changelist, int nchanges,
     args->nevents = nevents;
     args->timeout = NULL;
 
+    if (timeout == NULL) {
+        need_alarm_sem = 1;
+    }
+
     rte_spinlock_lock(&sc->lock);
 
     sc->ops = FF_SO_KEVENT;
@@ -1406,6 +1421,10 @@ kevent(int kq, const struct kevent *changelist, int nchanges,
 
     rte_spinlock_lock(&sc->lock);
 
+    if (timeout == NULL) {
+        need_alarm_sem = 0;
+    }
+
     /*
      * After sem_timedwait, and before lock sc, sc->status may be modify from FF_SC_REQ to FF_SC_RSP,
      * so it can't use to check.
@@ -1616,7 +1635,27 @@ ff_adapter_exit()
 #ifdef FF_THREAD_SOCKET
     pthread_key_delete(key);
 #else
-    DEBUG_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc);
+    ERR_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc);
     ff_detach_so_context(sc);
 #endif
 }
+
+void
+alarm_event_sem()
+{
+#ifndef FF_THREAD_SOCKET
+    DEBUG_LOG("check whether need to alarm sem sc:%p, status:%d, ops:%d\n",
+        sc, sc->status, sc->ops, need_alarm_sem);
+    rte_spinlock_lock(&sc->lock);
+    if (need_alarm_sem == 1) {
+        ERR_LOG("alarm sc:%p, status:%d, ops:%d\n", sc, sc->status, sc->ops);
+        sem_post(&sc->wait_sem);
+        need_alarm_sem = 0;
+    }
+    rte_spinlock_unlock(&sc->lock);
+
+    DEBUG_LOG("finish alarm sem sc:%p, status:%d, ops:%d\n",
+        sc, sc->status, sc->ops, need_alarm_sem);
+#endif
+}
+

adapter/ff_socket_ops.c

@@ -15,6 +15,8 @@ static int ff_sys_kevent(struct ff_kevent_args *args);
 
 #define FF_MAX_BOUND_NUM 8
 
+/* Where to call sem_post in kevent or epoll_wait */
+static int sem_flag = 0;
 
 struct ff_bound_info {
     int fd;
@@ -288,10 +290,25 @@ ff_sys_epoll_ctl(struct ff_epoll_ctl_args *args)
 static int
 ff_sys_epoll_wait(struct ff_epoll_wait_args *args)
 {
+    int ret;
+
     DEBUG_LOG("to run ff_epoll_wait, epfd:%d, maxevents:%d, timeout:%d\n",
         args->epfd, args->maxevents, args->timeout);
-    return ff_epoll_wait(args->epfd, args->events,
+    ret = ff_epoll_wait(args->epfd, args->events,
         args->maxevents, args->timeout);
+
+    /*
+     * If timeout is 0, and no event triggered,
+     * no post sem, and next loop will continue to call ff_sys_epoll_wait,
+     * until some event triggered
+     */
+    if (args->timeout == 0 && ret == 0 && args->maxevents != 0) {
+        sem_flag = 0;
+    } else {
+        sem_flag = 1;
+    }
+
+    return ret;
 }
 
 static int
@@ -303,8 +320,27 @@ ff_sys_kqueue(struct ff_kqueue_args *args)
 static int
 ff_sys_kevent(struct ff_kevent_args *args)
 {
-    return ff_kevent(args->kq, args->changelist, args->nchanges,
+    int ret;
+
+    ret = ff_kevent(args->kq, args->changelist, args->nchanges,
         args->eventlist, args->nevents, args->timeout);
+
+    if (args->nchanges) {
+        args->nchanges = 0;
+    }
+
+    /*
+     * If timeout is NULL, and no event triggered,
+     * no post sem, and next loop will continue to call ff_sys_kevent,
+     * until some event triggered
+     */
+    if (args->timeout == NULL && ret == 0 && args->nevents != 0) {
+        sem_flag = 0;
+    } else {
+        sem_flag = 1;
+    }
+
+    return ret;
 }
 
 static pid_t
@@ -387,30 +423,6 @@ ff_so_handler(int ops, void *args)
     return (-1);
 }
 
-static void
-ff_handle_kevent(struct ff_so_context *sc)
-{
-    struct ff_kevent_args *ka = sc->args;
-
-    DEBUG_LOG("ff_handle_kevent sc:%p, status:%d, ops:%d, kq:%d, nchanges:%d, nevents:%d\n",
-        sc, sc->status, sc->ops, ka->kq, ka->nchanges, ka->nevents);
-
-    errno = 0;
-    sc->result = ff_sys_kevent(ka);
-    sc->error = errno;
-
-    if (ka->nchanges) {
-        ka->nchanges = 0;
-    }
-
-    /*if (sc->result == 0 && ka->nevents != 0) {
-        return;
-    }*/
-
-    sc->status = FF_SC_REP;
-    sem_post(&sc->wait_sem);
-}
-
 static inline void
 ff_handle_socket_ops(struct ff_so_context *sc)
 {
@@ -425,19 +437,19 @@ ff_handle_socket_ops(struct ff_so_context *sc)
 
     DEBUG_LOG("ff_handle_socket_ops sc:%p, status:%d, ops:%d\n", sc, sc->status, sc->ops);
 
-    if (sc->ops == FF_SO_KEVENT) {
-        ff_handle_kevent(sc);
-        rte_spinlock_unlock(&sc->lock);
-        return;
-    }
-
     errno = 0;
     sc->result = ff_so_handler(sc->ops, sc->args);
     sc->error = errno;
-    sc->status = FF_SC_REP;
 
-    if (sc->ops == FF_SO_EPOLL_WAIT) {
-        sem_post(&sc->wait_sem);
+    if (sc->ops == FF_SO_EPOLL_WAIT || sc->ops == FF_SO_KEVENT) {
+        if (sem_flag == 1) {
+            sc->status = FF_SC_REP;
+            sem_post(&sc->wait_sem);
+        } else {
+            // do nothing with this sc
+        }
+    } else {
+        sc->status = FF_SC_REP;
     }
 
     rte_spinlock_unlock(&sc->lock);

example/main_stack.c

@@ -72,6 +72,7 @@ void sig_term(int sig)
 {
     printf("we caught signal %d, to exit helloworld\n", sig);
     exit_flag = 1;
+    alarm_event_sem();
     return;
 }
 
@@ -79,6 +80,9 @@ void *loop(void *arg)
 {
     /* Wait for events to happen */
     while (!exit_flag) {
+        /*
+         * If timeout is NULL, must call alarm_event_sem();
+         */
         int nevents = kevent(kq, NULL, 0, events, MAX_EVENTS, &timeout);
         int i;
 
@@ -91,7 +95,7 @@ void *loop(void *arg)
             //usleep(100);
             sleep(1);
         }
-        printf("get nevents:%d\n", nevents);
+        //printf("get nevents:%d\n", nevents);
 
         for (i = 0; i < nevents; ++i) {
             struct kevent event = events[i];

example/main_stack_epoll.c

@@ -61,6 +61,7 @@ void sig_term(int sig)
 {
     printf("we caught signal %d, to exit helloworld\n", sig);
     exit_flag = 1;
+    //alarm_event_sem();
     return;
 }
 
@@ -68,6 +69,10 @@ void *loop(void *arg)
 {
     /* Wait for events to happen */
     while (!exit_flag) {
+        /*
+         * If not call alarm_event_sem, and epoll_wait timeout is 0,
+         * it can't exit normal, so timeout can't set to 0.
+         */
         int nevents = epoll_wait(epfd, events, MAX_EVENTS, 100);
         int i;
 
@@ -80,7 +85,7 @@ void *loop(void *arg)
             //usleep(100);
             sleep(1);
         }
-        printf("get nevents:%d\n", nevents);
+        //printf("get nevents:%d\n", nevents);
 
         for (i = 0; i < nevents; ++i) {
             /* Handle new connect */

example/main_stack_epoll_thread_socket.c

@@ -57,6 +57,7 @@ void sig_term(int sig)
 {
     printf("we caught signal %d, to exit helloworld\n", sig);
     exit_flag = 1;
+    //alarm_event_sem();
     return;
 }
 
@@ -124,6 +125,10 @@ void *loop(void *arg)
 
     /* Wait for events to happen */
     while (!exit_flag) {
+        /*
+         * If not call alarm_event_sem, and epoll_wait timeout is 0,
+         * it can't exit normal, so timeout can't set to 0.
+         */
         int nevents = epoll_wait(epfd, events, MAX_EVENTS, 100);
         int i;
 
@@ -136,7 +141,7 @@ void *loop(void *arg)
             //usleep(100);
             sleep(1);
         }
-        printf("thread %d, get nevents:%d\n", thread_id, nevents);
+        //printf("thread %d, get nevents:%d\n", thread_id, nevents);
 
         for (i = 0; i < nevents; ++i) {
             /* Handle new connect */

example/main_stack_thread_socket.c

@@ -65,6 +65,7 @@ void sig_term(int sig)
 {
     printf("we caught signal %d, to exit helloworld\n", sig);
     exit_flag = 1;
+    //alarm_event_sem();
     return;
 }
 
@@ -138,6 +139,10 @@ void *loop(void *arg)
 
     /* Wait for events to happen */
     while (!exit_flag) {
+        /*
+         * If not call alarm_event_sem, and epoll_wait timeout is NULL,
+         * it can't exit normal, so timeout can't set to NULL.
+         */
         int nevents = kevent(kq, NULL, 0, events, MAX_EVENTS, &timeout);
         int i;
 
@@ -150,7 +155,7 @@ void *loop(void *arg)
             //usleep(100);
             sleep(1);
         }
-        printf("thread %d, get nevents:%d\n", thread_id, nevents);
+        //printf("thread %d, get nevents:%d\n", thread_id, nevents);
 
         for (i = 0; i < nevents; ++i) {
             struct kevent event = events[i];