bug/SKALE-3662 Adding libzmq

ServerInit.cpp

@@ -42,7 +42,6 @@
 #include <unistd.h>
 
 
-
 #include "BLSPrivateKeyShareSGX.h"
 #include "sgxwallet_common.h"
 #include "third_party/intel/create_enclave.h"
@@ -68,16 +67,16 @@ using namespace std;
 void systemHealthCheck() {
     string ulimit;
     try {
-        ulimit = exec( "/bin/bash -c \"ulimit -n\"" );
-    } catch ( ... ) {
+        ulimit = exec("/bin/bash -c \"ulimit -n\"");
+    } catch (...) {
         spdlog::error("Execution of '/bin/bash -c ulimit -n' failed");
         exit(-15);
     }
-    int noFiles = strtol( ulimit.c_str(), NULL, 10 );
+    int noFiles = strtol(ulimit.c_str(), NULL, 10);
 
-    auto noUlimitCheck = getenv( "NO_ULIMIT_CHECK" ) != nullptr;
+    auto noUlimitCheck = getenv("NO_ULIMIT_CHECK") != nullptr;
 
-    if ( noFiles < 65535 && !noUlimitCheck) {
+    if (noFiles < 65535 && !noUlimitCheck) {
         string errStr =
                 "sgxwallet requires setting Linux file descriptor limit to at least 65535 "
                 "You current limit (ulimit -n) is less than 65535. \n Please set it to 65535:"
@@ -89,8 +88,8 @@ void systemHealthCheck() {
     }
 }
 
-static ZMQServer* zmqServer = nullptr;
-atomic<bool> exiting(false);
+static ZMQServer *zmqServer = nullptr;
+
 
 void initUserSpace() {
 
@@ -109,15 +108,10 @@ void initUserSpace() {
 }
 
 void exitZMQServer() {
-    auto doExit = !exiting.exchange(true);
-
-    if (doExit) {
-        spdlog::info("Exiting zmq server ...");
-        zmqServer->exitWorkers();
-        spdlog::info("Exited zmq server ...");
-        zmqServer = nullptr;
-    }
-
+    spdlog::info("Exiting zmq server ...");
+    zmqServer->exitWorkers();
+    spdlog::info("Exited zmq server ...");
+    zmqServer = nullptr;
 }
 
 uint64_t initEnclave() {
@@ -163,7 +157,7 @@ uint64_t initEnclave() {
         }
 
         spdlog::info("Enclave created and started successfully");
-        
+
         status = trustedEnclaveInit(eid, enclaveLogLevel);
     }
 
@@ -178,7 +172,6 @@ uint64_t initEnclave() {
 }
 
 
-
 void initAll(uint32_t _logLevel, bool _checkCert, bool _autoSign, bool _generateTestKeys) {
 
 
@@ -201,9 +194,9 @@ void initAll(uint32_t _logLevel, bool _checkCert, bool _autoSign, bool _generate
         uint64_t counter = 0;
 
         uint64_t initResult = 0;
-        while ((initResult = initEnclave()) != 0 && counter < 10){
+        while ((initResult = initEnclave()) != 0 && counter < 10) {
             sleep(1);
-            counter ++;
+            counter++;
         }
 
         if (initResult != 0) {

ServerWorker.cpp

@@ -15,7 +15,8 @@
 
 
 ServerWorker::ServerWorker(zmq::context_t &ctx, int sock_type) : ctx_(ctx),
-                                                                 worker_(ctx_, sock_type) {};
+                                                                 worker_(ctx_, sock_type),
+                                                                 isExitRequested(false) {};
 
 void ServerWorker::work() {
     worker_.connect("inproc://backend");
@@ -23,7 +24,7 @@ void ServerWorker::work() {
     std::string replyStr;
 
 
-    while (true) {
+    while (!isExitRequested) {
 
         Json::Value result;
         int errStatus = -1 * (10000 + __LINE__);
@@ -76,9 +77,15 @@ void ServerWorker::work() {
             spdlog::error("Exception in zmq server worker:{}", e.what());
         }
         catch (std::exception &e) {
+            if (isExitRequested) {
+                return;
+            }
             result["errorMessage"] = string(e.what());
             spdlog::error("Exception in zmq server worker:{}", e.what());
         } catch (...) {
+            if (isExitRequested) {
+                return;
+            }
             spdlog::error("Error in zmq server worker");
             result["errorMessage"] = "Error in zmq server worker";
         }
@@ -99,14 +106,22 @@ void ServerWorker::work() {
             worker_.send(replyMsg);
 
         } catch (std::exception &e) {
+            if (isExitRequested) {
+                return;
+            }
             spdlog::error("Exception in zmq server worker send :{}", e.what());
         } catch (...) {
+            if (isExitRequested) {
+                return;
+            }
             spdlog::error("Unklnown exception in zmq server worker send");
         }
-    }
 
+    }
+}
 
 
+void ServerWorker::requestExit() {
+    isExitRequested.exchange(true);
 }
 
-

ServerWorker.h

@@ -25,10 +25,13 @@ public:
 
     void work();
 
+    void requestExit();
 
 private:
     zmq::context_t &ctx_;
     zmq::socket_t worker_;
+
+    std::atomic<bool> isExitRequested;
 };
 
 

ZMQServer.cpp

@@ -63,18 +63,18 @@ void ZMQServer::run() {
 
     try {
         for (int i = 0; i < kMaxThread; ++i) {
-            worker.push_back(make_shared<ServerWorker>(ctx_, ZMQ_DEALER));
-            worker_thread.push_back(make_shared<std::thread>(std::bind(&ServerWorker::work, worker[i])));
+            workers.push_back(make_shared<ServerWorker>(ctx_, ZMQ_DEALER));
+            worker_threads.push_back(make_shared<std::thread>(std::bind(&ServerWorker::work, workers[i])));
         }
     } catch (std::exception &e) {
         spdlog::error("Could not create zmq server workers:{} ", e.what());
         exit(-102);
-    }
+    };
 
 
     try {
         zmq::proxy(static_cast<void *>(frontend_), static_cast<void *>(backend_), nullptr);
-    } catch (exception& _e) {
+    } catch (exception &_e) {
         spdlog::info("Error, exiting zmq server ... {}", _e.what());
         return;
     } catch (...) {
@@ -84,12 +84,25 @@ void ZMQServer::run() {
 }
 
 
-
 void ZMQServer::exitWorkers() {
-    spdlog::info("Emptying threads ...");
-    worker_thread.empty();
-    spdlog::info("Emptying workers ...");
-    worker.empty();
-    spdlog::info("Emptied workers ...");
+    auto doExit = !exiting.exchange(true);
+    if (doExit) {
+
+        spdlog::info("Telling workers to exit");
+
+        for (auto &&worker : workers) {
+            worker->requestExit();
+        }
+
+        spdlog::info("joining threads ...");
+        for (auto &&thread : worker_threads)
+            thread->join();
+    }
+    spdlog::info("Deleting threads ...");
+    worker_threads.empty();
+    spdlog::info("Deleting workers ...");
+    workers.empty();
+    spdlog::info("Deleted workers ...");
 }
 
+

ZMQServer.h

@@ -43,9 +43,12 @@ using namespace std;
 
 class ZMQServer {
 public:
+
+    atomic<bool> exiting;
+
     ZMQServer();
 
-    atomic<bool> isExitRequested;
+
 
     enum {
         kMaxThread = 1
@@ -60,8 +63,11 @@ private:
     zmq::socket_t frontend_;
     zmq::socket_t backend_;
 
-    std::vector<shared_ptr<ServerWorker> > worker;
-    std::vector<shared_ptr<std::thread>> worker_thread;
+    std::vector<shared_ptr<ServerWorker> > workers;
+    std::vector<shared_ptr<std::thread>> worker_threads;
+
+
+    std::atomic<bool> isExitRequested;
 
 };