SKALE-4005 check for exit

CSRManagerServer.cpp

@@ -27,6 +27,7 @@
 #include <jsonrpccpp/server/connectors/httpserver.h>
 
 #include "CSRManagerServer.h"
+#include "ExitHandler.h"
 #include "SGXException.h"
 #include "sgxwallet_common.h"
 
@@ -119,9 +120,23 @@ int CSRManagerServer::initCSRManagerServer() {
 
     if (!cs->StartListening()) {
         spdlog::info("CSR manager server could not start listening");
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-1);
     } else {
         spdlog::info("CSR manager server started on port {}", BASE_PORT + 2);
     }
     return 0;
 };
+
+int CSRManagerServer::exitServer() {
+  spdlog::info("Stoping CSRManager server");
+
+  if (!cs->StopListening()) {
+      spdlog::error("CSRManager server could not be stopped");
+      exit(-104);
+  } else {
+      spdlog::info("CSRManager server stopped");
+  }
+
+  return 0;
+}

CSRManagerServer.h

@@ -50,6 +50,8 @@ class CSRManagerServer : public abstractCSRManagerServer {
   virtual Json::Value signByHash(const string& hash, int status);
 
   static int initCSRManagerServer();
+
+  static int exitServer();
 };
 
 

ExitHandler.cpp

+#include "ExitHandler.h"
+
+void ExitHandler::exitHandler( int s ) {
+    exitHandler( s, ec_success );
+}
+
+void ExitHandler::exitHandler( int s, ExitHandler::exit_code_t ec ) {
+    m_signal = s;
+    if ( ec != ec_success ) {
+        g_ec = ec;
+    }
+    s_shouldExit = true;
+//    // HACK wait for loop in main to send exit call to consensus et al.
+//    std::this_thread::sleep_for( std::chrono::milliseconds( 2000 ) );
+}
+
+volatile bool ExitHandler::s_shouldExit = false;
+volatile int ExitHandler::m_signal = -1;
+volatile ExitHandler::exit_code_t ExitHandler::g_ec = ExitHandler::ec_success;

ExitHandler.h

+#ifndef EXITHANDLER_H
+#define EXITHANDLER_H
+
+#include <signal.h>
+
+class ExitHandler {
+public:
+    enum { KILL_TIMEOUT = 57 };
+
+    enum exit_code_t {
+        ec_success = 0,
+        ec_failure = 1,  // same as EXIT_FAILURE in stdlib.h, generic failure in main()
+        ec_termninated_by_signal = 196,
+        ec_compute_snapshot_error = 197,  // snapshot computation error
+        ec_rotation_complete = 0,         // must be zero, exit requested after rotation complete
+        ec_consensus_terminate_request = 198,  // exit requested by consensus
+        ec_web3_request = 199,                 // programmatic shutdown via Web3 call, when enabled
+        ec_state_root_mismatch = 200,  // current state root is not equal to arrived from consensus
+    };
+
+private:
+    static volatile bool s_shouldExit;
+    static volatile int m_signal;
+    static volatile exit_code_t g_ec;
+
+    ExitHandler() = delete;
+
+public:
+    static void exitHandler( int s );
+    static void exitHandler( int s, ExitHandler::exit_code_t ec );
+    static bool shouldExit() { return s_shouldExit; }
+    static int getSignal() { return m_signal; }
+    static exit_code_t requestedExitCode() { return g_ec; }
+
+};
+
+#endif // EXITHANDLER_H

LevelDB.cpp

@@ -30,6 +30,7 @@
 #include <jsonrpccpp/client.h>
 
 #include "sgxwallet_common.h"
+#include "ExitHandler.h"
 #include "SGXException.h"
 #include "LevelDB.h"
 
@@ -275,6 +276,7 @@ void LevelDB::initDataFolderAndDBs() {
 
     if (getcwd(cwd, sizeof(cwd)) == NULL) {
         spdlog::error("could not get current workin directory");
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-2);
     }
 
@@ -289,6 +291,7 @@ void LevelDB::initDataFolderAndDBs() {
         }
         else{
             spdlog::error("Couldnt create creating sgx_data folder");
+            ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
             exit(-3);
         }
     }

Makefile.am

@@ -70,7 +70,7 @@ bin_PROGRAMS = sgxwallet testw sgx_util
 ## have to be explicitly listed
 
 
-COMMON_SRC = SGXException.cpp ZMQClient.cpp BLSSignRspMessage.cpp ECDSASignRspMessage.cpp ECDSASignReqMessage.cpp BLSSignReqMessage.cpp ZMQMessage.cpp ZMQServer.cpp ServerWorker.cpp InvalidStateException.cpp  Exception.cpp InvalidArgumentException.cpp Log.cpp  \
+COMMON_SRC = SGXException.cpp ExitHandler.cpp ZMQClient.cpp BLSSignRspMessage.cpp ECDSASignRspMessage.cpp ECDSASignReqMessage.cpp BLSSignReqMessage.cpp ZMQMessage.cpp ZMQServer.cpp ServerWorker.cpp InvalidStateException.cpp Exception.cpp InvalidArgumentException.cpp Log.cpp  \
              SGXWalletServer.cpp SGXRegistrationServer.cpp CSRManagerServer.cpp BLSCrypto.cpp \
              DKGCrypto.cpp ServerInit.cpp BLSPrivateKeyShareSGX.cpp LevelDB.cpp ServerDataChecker.cpp SEKManager.cpp \
              third_party/intel/sgx_stub.c third_party/intel/sgx_detect_linux.c third_party/intel/create_enclave.c third_party/intel/oc_alloc.c \
@@ -116,12 +116,12 @@ nodist_testw_SOURCES=${nodist_sgxwallet_SOURCES}
 EXTRA_testw_DEPENDENCIES=${EXTRA_sgxwallet_DEPENDENCIES}
 testw_LDADD= ${sgxwallet_LDADD}
 
-sgx_util_SOURCES= SGXException.cpp InvalidStateException.cpp  Exception.cpp InvalidArgumentException.cpp Log.cpp sgx_util.cpp stubclient.cpp LevelDB.cpp SGXRegistrationServer.cpp CSRManagerServer.cpp
+sgx_util_SOURCES= SGXException.cpp ExitHandler.cpp InvalidStateException.cpp Exception.cpp InvalidArgumentException.cpp Log.cpp sgx_util.cpp stubclient.cpp LevelDB.cpp SGXRegistrationServer.cpp CSRManagerServer.cpp
 
 sgx_util_LDADD=-LlibBLS/deps/deps_inst/x86_or_x64/lib -Lleveldb/build -LlibBLS/build \
                    -LlibBLS/build/libff/libff \
                    -Llibzmq/build/lib/ \
                    -l:libzmq.a \
                    -l:libbls.a -l:libleveldb.a \
-                   -l:libff.a -lgmp  -ljsonrpccpp-stub -ljsonrpccpp-server -ljsonrpccpp-client -ljsonrpccpp-common -ljsoncpp -lmicrohttpd -lgnutls -lgcrypt -lidn2 -lcurl -lssl -lcrypto -lz -lpthread -ldl
+                   -l:libff.a -lgmp -ljsonrpccpp-stub -ljsonrpccpp-server -ljsonrpccpp-client -ljsonrpccpp-common -ljsoncpp -lmicrohttpd -lgnutls -lgcrypt -lidn2 -lcurl -lssl -lcrypto -lz -lpthread -ldl
 

SEKManager.cpp

@@ -33,6 +33,7 @@
 #include "common.h"
 #include "sgxwallet.h"
 
+#include "ExitHandler.h"
 #include "SGXException.h"
 #include "BLSCrypto.h"
 #include "LevelDB.h"
@@ -90,6 +91,7 @@ void validate_SEK() {
     if (!hex2carray(test_key_ptr->c_str(), &len, encr_test_key.data(),
                     BUF_LEN)) {
         spdlog::error("Corrupt test key is LevelDB");
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-4);
     }
 
@@ -108,6 +110,7 @@ void validate_SEK() {
         spdlog::error("Invalid storage key. You need to recover using backup key");
         spdlog::error("Set the correct backup key into sgx_datasgxwallet_backup_key.txt");
         spdlog::error("Then run sgxwallet using backup flag");
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-5);
     }
 }
@@ -206,17 +209,17 @@ void gen_SEK() {
 }
 
 
-static std::atomic<int> isSgxWalletExiting(0);
+//static std::atomic<int> isSgxWalletExiting(0);
 
-void  safeExit() {
+//void safeExit() {
 
-    // this is to make sure exit is only called once if called from multiple threads
+//    // this is to make sure exit is only called once if called from multiple threads
 
-    auto previousValue = isSgxWalletExiting.exchange(1);
+//    auto previousValue = isSgxWalletExiting.exchange(1);
 
-    if (previousValue != 1)
-        exit(-6);
-}
+//    if (previousValue != 1)
+//        exit(-6);
+//}
 
 void setSEK(shared_ptr <string> hex_encrypted_SEK) {
 
@@ -256,12 +259,14 @@ void enter_SEK() {
     shared_ptr <string> test_key_ptr = LevelDB::getLevelDb()->readString("TEST_KEY");
     if (test_key_ptr == nullptr) {
         spdlog::error("Error: corrupt or empty LevelDB database");
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-7);
     }
 
 
     if (!experimental::filesystem::is_regular_file(BACKUP_PATH)) {
         spdlog::error("File does not exist: "  BACKUP_PATH);
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-8);
     }
 
@@ -278,6 +283,7 @@ void enter_SEK() {
 
     while (!checkHex(sek, 16)) {
         spdlog::error("Invalid hex in key");
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-9);
     }
 

SEKManager.h

@@ -47,7 +47,7 @@ EXTERNC void initSEK();
 
 EXTERNC void setSEK();
 
-EXTERNC void safeExit();
+//EXTERNC void safeExit();
 
 
 

SGXInfoServer.cpp

@@ -31,6 +31,7 @@
 
 #include "sgxwallet_common.h"
 
+#include "ExitHandler.h"
 #include "SGXException.h"
 #include "LevelDB.h"
 
@@ -115,6 +116,7 @@ int SGXInfoServer::initInfoServer(uint32_t _logLevel, bool _autoSign, bool _chec
 
     if (!server->StartListening()) {
         spdlog::error("Info server could not start listening on port {}", BASE_PORT + 4);
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-10);
     } else {
         spdlog::info("Info server started on port {}", BASE_PORT + 4);
@@ -123,6 +125,19 @@ int SGXInfoServer::initInfoServer(uint32_t _logLevel, bool _autoSign, bool _chec
     return 0;
 }
 
+int SGXInfoServer::exitServer() {
+  spdlog::info("Stoping SGXInfo server");
+
+  if (!server->StopListening()) {
+      spdlog::error("SGXInfo server could not be stopped");
+      exit(-105);
+  } else {
+      spdlog::info("SGXInfo server stopped");
+  }
+
+  return 0;
+}
+
 shared_ptr<SGXInfoServer> SGXInfoServer::getServer() {
     CHECK_STATE(server);
     return server;

SGXInfoServer.h

@@ -61,6 +61,8 @@ public:
 
     static int initInfoServer(uint32_t _logLevel, bool _autoSign, bool _checkCerts, bool _generateTestKeys);
 
+    static int exitServer();
+
 };
 
 #endif // SGXINFOSERVER_H

SGXWalletServer.cpp

@@ -21,7 +21,9 @@
     @date 2019
 */
 
+#include <chrono>
 #include <iostream>
+#include <thread>
 
 #include "abstractstubserver.h"
 #include <jsonrpccpp/server/connectors/httpserver.h>
@@ -35,6 +37,7 @@
 #include "sgxwallet.h"
 
 
+#include "ExitHandler.h"
 #include "SGXException.h"
 #include "LevelDB.h"
 #include "BLSCrypto.h"
@@ -139,6 +142,7 @@ void SGXWalletServer::createCertsIfNeeded() {
             spdlog::info("ROOT CA CERTIFICATE IS SUCCESSFULLY GENERATED");
         } else {
             spdlog::error("ROOT CA CERTIFICATE GENERATION FAILED");
+            ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
             exit(-11);
         }
     }
@@ -156,6 +160,7 @@ void SGXWalletServer::createCertsIfNeeded() {
             spdlog::info("SERVER CERTIFICATE IS SUCCESSFULLY GENERATED");
         } else {
             spdlog::info("SERVER CERTIFICATE GENERATION FAILED");
+            ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
             exit(-12);
         }
     }
@@ -166,6 +171,7 @@ void SGXWalletServer::createCertsIfNeeded() {
         spdlog::info("SERVER CERTIFICATE IS SUCCESSFULLY VERIFIED");
     } else {
         spdlog::info("SERVER CERTIFICATE VERIFICATION FAILED");
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-12);
     }
 }
@@ -195,6 +201,7 @@ int SGXWalletServer::initHttpsServer(bool _checkCerts) {
 
     if (!server->StartListening()) {
         spdlog::error("SGX Server could not start listening");
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-13);
     } else {
         spdlog::info("SGX Server started on port {}", BASE_PORT);
@@ -214,11 +221,25 @@ int SGXWalletServer::initHttpServer() { //without ssl
                                           JSONRPC_SERVER_V2); // hybrid server (json-rpc 1.0 & 2.0)
     if (!server->StartListening()) {
         spdlog::error("Server could not start listening");
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-14);
     }
     return 0;
 }
 
+int SGXWalletServer::exitServer() {
+  spdlog::info("Stoping sgx server");
+
+  if (!server->StopListening()) {
+      spdlog::error("Sgx server could not be stopped");
+      exit(-103);
+  } else {
+      spdlog::info("Sgx server stopped");
+  }
+
+  return 0;
+}
+
 Json::Value
 SGXWalletServer::importBLSKeyShareImpl(const string &_keyShare, const string &_keyShareName) {
     COUNT_STATISTICS

SGXWalletServer.hpp

@@ -39,8 +39,6 @@ using namespace std;
 #define TOSTRING(x) STRINGIFY(x)
 
 
-
-
 class SGXWalletServer : public AbstractStubServer {
     static shared_ptr<SGXWalletServer> server;
     static shared_ptr<HttpServer> httpServer;
@@ -182,6 +180,8 @@ public:
 
     static int initHttpsServer(bool _checkCerts);
 
+    static int exitServer();
+
     static void createCertsIfNeeded();
 };
 

ServerInit.cpp

@@ -42,6 +42,7 @@
 #include <unistd.h>
 
 
+#include "ExitHandler.h"
 #include "BLSPrivateKeyShareSGX.h"
 #include "sgxwallet_common.h"
 #include "third_party/intel/create_enclave.h"
@@ -70,6 +71,7 @@ void systemHealthCheck() {
         ulimit = exec("/bin/bash -c \"ulimit -n\"");
     } catch (...) {
         spdlog::error("Execution of '/bin/bash -c ulimit -n' failed");
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-15);
     }
     int noFiles = strtol(ulimit.c_str(), NULL, 10);
@@ -84,6 +86,7 @@ void systemHealthCheck() {
                 "and setting 'DefaultLimitNOFILE=65535'\n"
                 "After that, restart sgxwallet";
         spdlog::error(errStr);
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-16);
     }
 }
@@ -116,6 +119,7 @@ uint64_t initEnclave() {
     support = get_sgx_support();
     if (!SGX_OK(support)) {
         sgx_support_perror(support);
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-17);
     }
 #endif
@@ -147,6 +151,7 @@ uint64_t initEnclave() {
             } else {
                 spdlog::error("sgx_create_enclave_search failed {} {}", ENCLAVE_NAME, status);
             }
+            ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
             exit(-21);
         }
 
@@ -222,15 +227,27 @@ void initAll(uint32_t _logLevel, bool _checkCert,
         sgxServerInited = true;
     } catch (SGXException &_e) {
         spdlog::error(_e.getMessage());
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-18);
     } catch (exception &_e) {
         spdlog::error(_e.what());
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-19);
     }
     catch (...) {
         exception_ptr p = current_exception();
         printf("Exception %s \n", p.__cxa_exception_type()->name());
         spdlog::error("Unknown exception");
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-22);
     }
 };
+
+void exitAll() {
+    SGXWalletServer::exitServer();
+    SGXRegistrationServer::exitServer();
+    CSRManagerServer::exitServer();
+    SGXInfoServer::exitServer();
+    ZMQServer::exitZMQServer();
+
+}

ServerInit.h

@@ -34,6 +34,8 @@
 
 EXTERNC void initAll(uint32_t  _logLevel, bool _checkCert, bool _checkZMQSig, bool _autoSign, bool _generateTestKeys);
 
+void exitAll();
+
 EXTERNC void initUserSpace();
 
 EXTERNC uint64_t initEnclave();

ZMQServer.cpp

@@ -29,6 +29,7 @@
 
 #include "common.h"
 
+#include "ExitHandler.h"
 #include "ZMQServer.h"
 #include "sgxwallet_common.h"
 
@@ -78,6 +79,7 @@ void ZMQServer::run() {
         frontend->bind("tcp://*:" + to_string(port));
     } catch (...) {
         spdlog::error("Server task could not bind to port:{}", port);
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(ZMQ_COULD_NOT_BIND_FRONT_END);
     }
 
@@ -88,6 +90,7 @@ void ZMQServer::run() {
         backend->bind("inproc://backend");
     } catch (exception &e) {
         spdlog::error("Could not bind to zmq backend: {}", e.what());
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(ZMQ_COULD_NOT_BIND_BACK_END);
     }
 
@@ -103,6 +106,7 @@ void ZMQServer::run() {
         }
     } catch (std::exception &e) {
         spdlog::error("Could not create zmq server workers:{} ", e.what());
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(ZMQ_COULD_NOT_CREATE_WORKERS);
     };
 
@@ -123,6 +127,7 @@ void ZMQServer::run() {
             return;
         }
         spdlog::info("Error, exiting zmq server ...");
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(ZMQ_COULD_NOT_CREATE_PROXY);
     }
 }
@@ -206,4 +211,4 @@ ZMQServer::~ZMQServer() {
     spdlog::info("Deleting ZMQ context");
     ctx_ = nullptr;
     spdlog::info("Deleted ZMQ context");
-}
\ No newline at end of file
+}

sgxwall.cpp

@@ -23,13 +23,14 @@
 
 #include <stdbool.h>
 
+#include "ExitHandler.h"
+
 #include "BLSCrypto.h"
 #include "ServerInit.h"
 
 #include "SEKManager.h"
 #include "SGXWalletServer.h"
 
-
 #include <fstream>
 
 #include "TestUtils.h"
@@ -43,6 +44,7 @@
 
 void SGXWallet::usage() {
     cerr << "usage: sgxwallet\n";
+    ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
     exit(-21);
 }
 
@@ -103,6 +105,7 @@ int main(int argc, char *argv[]) {
 
     if (argc > 1 && strlen(argv[1]) == 1) {
         SGXWallet::printUsage();
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
         exit(-22);
     }
 
@@ -110,6 +113,7 @@ int main(int argc, char *argv[]) {
         switch (opt) {
             case 'h':
                 SGXWallet::printUsage();
+                ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
                 exit(-24);
             case 'c':
                 checkClientCertOption = false;
@@ -147,6 +151,7 @@ int main(int argc, char *argv[]) {
                 break;
             default:
                 SGXWallet::printUsage();
+                ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_failure);
                 exit(-23);
                 break;
         }
@@ -208,6 +213,13 @@ int main(int argc, char *argv[]) {
 
     while (true) {
         sleep(10);
+        if ( ExitHandler::shouldExit() ) {
+            ExitHandler::exit_code_t exitCode = ExitHandler::requestedExitCode();
+            spdlog::info("Will exit with exit code {}", exitCode);
+            exitAll();
+            spdlog::info("Exiting with exit code {}", exitCode);
+            return exitCode;
+        }
     }
 
     return 0;