Fix the issue that not detach sc while destructor in exit.

adapter/ff_hook_syscall.c

@@ -98,6 +98,14 @@
 static __FF_THREAD int inited = 0;
 static __FF_THREAD struct ff_so_context *sc;
 
+/*
+ * Use pthread_key_create/pthread_setspecific/pthread_key_delete in FF_THREAD_SOCKET mode,
+ * because ff_so_zone is thread level.
+ */
+#ifdef FF_THREAD_SOCKET
+static pthread_key_t key;
+#endif
+
 /* process-level initialization flag */
 static int proc_inited = 0;
 
@@ -1405,6 +1413,15 @@ kevent(int kq, const struct kevent *changelist, int nchanges,
     RETURN();
 }
 
+#ifdef FF_THREAD_SOCKET
+static void
+thread_destructor(void *sc)
+{
+    DEBUG_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc);
+    ff_detach_so_context(sc);
+}
+#endif
+
 int
 ff_adapter_init()
 //int __attribute__((constructor))
@@ -1421,6 +1438,11 @@ ff_adapter_init()
         rte_spinlock_init(&worker_id_lock);
         rte_spinlock_lock(&worker_id_lock);
 
+#ifdef FF_THREAD_SOCKET
+        pthread_key_create(&key, thread_destructor);
+        DEBUG_LOG("pthread key:%d\n", key);
+#endif
+
         /*
          * get ulimit -n to distinguish fd between kernel and F-Stack
          */
@@ -1514,6 +1536,10 @@ ff_adapter_init()
         return -1;
     }
 
+#ifdef FF_THREAD_SOCKET
+    pthread_setspecific(key, sc);
+#endif
+
     worker_id++;
     inited = 1;
 
@@ -1527,6 +1553,10 @@ ff_adapter_init()
 void __attribute__((destructor))
 ff_adapter_exit()
 {
+#ifdef FF_THREAD_SOCKET
+    pthread_key_delete(key);
+#else
+    DEBUG_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc);
     ff_detach_so_context(sc);
-    ERR_LOG("pthread self tid:%lu, detach sc:%p\n", pthread_self(), sc);
+#endif
 }

adapter/ff_so_zone.c

@@ -88,6 +88,7 @@ ff_create_so_memzone()
                 struct ff_so_context *sc = &so_zone_tmp->sc[i];
                 rte_spinlock_init(&sc->lock);
                 sc->status = FF_SC_IDLE;
+                sc->idx = i;
                 sc->inuse = 0;
 
                 if (sem_init(&sc->wait_sem, 1, 0) == -1) {
@@ -182,16 +183,26 @@ ff_attach_so_context(int proc_id)
 void
 ff_detach_so_context(struct ff_so_context *sc)
 {
+    DEBUG_LOG("ff_so_zone:%p, sc:%p\n", ff_so_zone, sc);
+
     if (ff_so_zone == NULL || sc == NULL) {
         return;
     }
 
+    DEBUG_LOG("detach sc:%p, ops:%d, status:%d, idx:%d, inuse:%d, so free:%u, idx:%u\n",
+        sc, sc->ops, sc->status, sc->idx, sc->inuse, ff_so_zone->free, ff_so_zone->idx);
+
     rte_spinlock_lock(&ff_so_zone->lock);
 
-    sc->inuse = 0;
+    if (sc->inuse == 1) {
+        sc->inuse = 0;
 
-    ff_so_zone->free++;
-    DEBUG_LOG("detach sc:%p, status:%d, ops:%d\n", sc, sc->status, sc->ops);
+        ff_so_zone->free++;
+        ff_so_zone->idx = sc->idx;
+    }
+
+    DEBUG_LOG("detach sc:%p, ops:%d, status:%d, idx:%d, inuse:%d, so free:%u, idx:%u\n",
+        sc, sc->ops, sc->status, sc->idx, sc->inuse, ff_so_zone->free, ff_so_zone->idx);
 
     rte_spinlock_unlock(&ff_so_zone->lock);
 }

adapter/ff_socket_ops.h

@@ -84,14 +84,12 @@ struct ff_socket_ops_zone {
 
 struct ff_so_context {
     rte_spinlock_t lock;
-
-    int status;
-
     sem_t wait_sem;
+    void *args;
 
     enum FF_SOCKET_OPS ops;
-
-    void *args;
+    int status;
+    int idx;
 
     /* result of ops processing */
     ssize_t result;