Merge branch 'develop' into stable

.github/workflows/dockerimageintelsubmission.yml

@@ -21,7 +21,6 @@ jobs:
           contains(github.ref, 'develop') || contains(github.ref, 'beta') ||
           contains(github.ref, 'master') || contains(github.ref, 'stable') ||
           contains(github.ref, 'SECURE_ENCLAVE_CHANGES')
-          
         run : |
           export BRANCH=${GITHUB_REF##*/}
           echo "Branch $BRANCH"

.github/workflows/dockerimagesim.yml

@@ -28,6 +28,7 @@ jobs:
         contains(github.ref, 'master') || contains(github.ref, 'stable') ||
         contains(github.ref, 'SECURE_ENCLAVE_CHANGES')
       run : |
+            sudo rm -rf /home/runner/work/sgxwallet/sgxwallet/sgx_data
             export BRANCH=${GITHUB_REF##*/}
             echo "Branch $BRANCH"
             export VERSION=$(cat VERSION)

BLSCrypto.cpp

@@ -78,6 +78,7 @@ int char2int(char _input) {
 vector<char> carray2Hex(const unsigned char *d, uint64_t _len) {
 
     CHECK_STATE(d);
+
     vector<char> _hexArray( 2 * _len + 1);
 
     char hexval[16] = {'0', '1', '2', '3', '4', '5', '6', '7',

BLSSignReqMessage.cpp

@@ -7,9 +7,11 @@
 
 
 Json::Value BLSSignReqMessage::process() {
-    auto keyName = getStringRapid("kn");
-    auto hash = getStringRapid("mh");
+    auto keyName = getStringRapid("keyShareName");
+    auto hash = getStringRapid("messageHash");
     auto t = getUint64Rapid("t");
     auto n = getUint64Rapid("n");
-    return SGXWalletServer::blsSignMessageHashImpl(keyName, hash, t, n);
+    auto result =  SGXWalletServer::blsSignMessageHashImpl(keyName, hash, t, n);
+    result["type"] = ZMQMessage::BLS_SIGN_RSP;
+    return result;
 }
\ No newline at end of file

BLSSignRspMessage.cpp

+/*
+  Copyright (C) 2018-2019 SKALE Labs
+
+  This file is part of libBLS.
+
+  libBLS is free software: you can redistribute it and/or modify
+  it under the terms of the GNU Affero General Public License as published
+  by the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  libBLS is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU Affero General Public License for more details.
+
+  You should have received a copy of the GNU Affero General Public License
+  along with libBLS.  If not, see <https://www.gnu.org/licenses/>.
+
+  @file BLSRspSignMessage.h
+  @author Stan Kladko
+  @date 2020
+*/
+
+#include "BLSSignRspMessage.h"
+#include "SGXWalletServer.hpp"
+
+
+Json::Value BLSSignRspMessage::process() {
+    assert(false);
+}
\ No newline at end of file

BLSSignRspMessage.h

+/*
+  Copyright (C) 2018-2019 SKALE Labs
+
+  This file is part of libBLS.
+
+  libBLS is free software: you can redistribute it and/or modify
+  it under the terms of the GNU Affero General Public License as published
+  by the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  libBLS is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU Affero General Public License for more details.
+
+  You should have received a copy of the GNU Affero General Public License
+  along with libBLS.  If not, see <https://www.gnu.org/licenses/>.
+
+  @file BLSRspSignMessage.h
+  @author Stan Kladko
+  @date 2020
+*/
+
+#ifndef SGXWALLET_BLSSIGNRSPMSG_H
+#define SGXWALLET_BLSSIGNRSPMSG_H
+
+#include "ZMQMessage.h"
+
+class BLSSignRspMessage : public ZMQMessage {
+public:
+
+    BLSSignRspMessage(shared_ptr<rapidjson::Document>& _d) : ZMQMessage(_d) {};
+
+    virtual Json::Value process();
+
+    string getSigShare() {
+        return getStringRapid("signatureShare");
+    }
+
+};
+
+
+#endif //SGXWALLET_BLSSIGNRSPMSG_H

CMakeLists.txt

@@ -8,6 +8,7 @@ include_directories(secure_enclave)
 include_directories(sgx-software-enable)
 include_directories(tgmp-build)
 include_directories(tgmp-build/include)
+include_directories(sgxwallet)
 
 add_executable(sgxwallet
         secure_enclave/AESUtils.c
@@ -82,4 +83,4 @@ add_executable(sgxwallet
         SGXWalletServer.hpp
         stubclient.cpp
         stubclient.h
-        testw.cpp)
+        testw.cpp ZMQClient.cpp ZMQClient.h)

CSRManagerServer.cpp

@@ -110,16 +110,29 @@ Json::Value CSRManagerServer::signByHash(const string &hash, int status) {
     return signByHashImpl(hash, status);
 }
 
-int CSRManagerServer::initCSRManagerServer() {
+void CSRManagerServer::initCSRManagerServer() {
     hs3 = make_shared<jsonrpc::HttpServer>(BASE_PORT + 2);
     hs3->BindLocalhost();
     cs = make_shared<CSRManagerServer>(*hs3, JSONRPC_SERVER_V2); // server (json-rpc 2.0)
 
+    spdlog::info("Starting csr manager server on port {} ...", BASE_PORT + 2);
+
     if (!cs->StartListening()) {
         spdlog::info("CSR manager server could not start listening");
-        exit(-1);
+        throw SGXException(CSR_MANAGER_SERVER_FAILED_TO_START, "CSRManager server could not start listening.");
     } else {
         spdlog::info("CSR manager server started on port {}", BASE_PORT + 2);
     }
-    return 0;
 };
+
+int CSRManagerServer::exitServer() {
+  spdlog::info("Stoping CSRManager server");
+
+  if (cs && !cs->StopListening()) {
+      spdlog::error("CSRManager server could not be stopped. Will forcefully terminate the app");
+  } else {
+      spdlog::info("CSRManager server stopped");
+  }
+
+  return 0;
+}

CSRManagerServer.h

@@ -49,7 +49,9 @@ class CSRManagerServer : public abstractCSRManagerServer {
   virtual Json::Value getUnsignedCSRs();
   virtual Json::Value signByHash(const string& hash, int status);
 
-  static int initCSRManagerServer();
+  static void initCSRManagerServer();
+
+  static int exitServer();
 };
 
 

ECDSASignReqMessage.cpp

-//
-// Created by kladko on 15.12.20.
-//
+/*
+  Copyright (C) 2018-2019 SKALE Labs
+
+  This file is part of libBLS.
+
+  libBLS is free software: you can redistribute it and/or modify
+  it under the terms of the GNU Affero General Public License as published
+  by the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  libBLS is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU Affero General Public License for more details.
+
+  You should have received a copy of the GNU Affero General Public License
+  along with libBLS.  If not, see <https://www.gnu.org/licenses/>.
+
+  @file ECDSASignReqMessage.cpp
+  @author Stan Kladko
+  @date 2020
+*/
 
 
 #include "SGXWalletServer.hpp"
@@ -10,8 +29,10 @@
 
 
 Json::Value ECDSASignReqMessage::process() {
-    auto base = getUint64Rapid("bs");
-    auto keyName = getStringRapid("kn");
-    auto hash = getStringRapid("mh");
-    return SGXWalletServer::ecdsaSignMessageHashImpl(base, keyName, hash);
+    auto base = getUint64Rapid("base");
+    auto keyName = getStringRapid("keyName");
+    auto hash = getStringRapid("messageHash");
+    auto result =  SGXWalletServer::ecdsaSignMessageHashImpl(base, keyName, hash);
+    result["type"] = ZMQMessage::ECDSA_SIGN_RSP;
+    return result;
 }
\ No newline at end of file

ECDSASignReqMessage.h

 /*
-  Copyright (C) 2018-2019 SKALE Labs
+  Copyright (C) 2018- SKALE Labs
 
   This file is part of libBLS.
 
@@ -33,6 +33,7 @@ public:
 
     virtual Json::Value process();
 
+
 };
 
 

ECDSASignRspMessage.cpp

+/*
+  Copyright (C) 2018- SKALE Labs
+
+  This file is part of libBLS.
+
+  libBLS is free software: you can redistribute it and/or modify
+  it under the terms of the GNU Affero General Public License as published
+  by the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  libBLS is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU Affero General Public License for more details.
+
+  You should have received a copy of the GNU Affero General Public License
+  along with libBLS.  If not, see <https://www.gnu.org/licenses/>.
+
+  @file ECDSARspSignMessage.cpp
+  @author Stan Kladko
+  @date 2020
+*/
+
+#include "SGXWalletServer.hpp"
+
+#include "ECDSASignRspMessage.h"
+
+
+
+Json::Value ECDSASignRspMessage::process() {
+    // never called
+    assert(false);
+}
+
+string ECDSASignRspMessage::getSignature() {
+
+
+
+    string r = getStringRapid( "signature_r" );
+    string v = getStringRapid( "signature_v" );
+    string s = getStringRapid("signature_s" );
+
+    auto ret = v + ":" + r.substr( 2 ) + ":" + s.substr( 2 );
+
+    return ret;
+}

ECDSASignRspMessage.h

+/*
+  Copyright (C) 2018- SKALE Labs
+
+  This file is part of libBLS.
+
+  libBLS is free software: you can redistribute it and/or modify
+  it under the terms of the GNU Affero General Public License as published
+  by the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  libBLS is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU Affero General Public License for more details.
+
+  You should have received a copy of the GNU Affero General Public License
+  along with libBLS.  If not, see <https://www.gnu.org/licenses/>.
+
+  @file ECDSARspSignMessage.h
+  @author Stan Kladko
+  @date 2020
+*/
+
+#ifndef SGXWALLET_ECDSASIGNRSPMESSAGE_H
+#define SGXWALLET_ECDSASIGNRSPMESSAGE_H
+
+#include "ZMQMessage.h"
+
+class ECDSASignRspMessage : public ZMQMessage {
+public:
+
+    ECDSASignRspMessage(shared_ptr <rapidjson::Document> &_d) : ZMQMessage(_d) {};
+
+    virtual Json::Value process();
+
+    string getSignature();
+
+
+};
+
+
+#endif //SGXWALLET_ECDSASIGNRSPMESSAGE_H

ExitHandler.cpp

+#include <chrono>
+#include <thread>
+
+#include "ExitHandler.h"
+
+void ExitHandler::exitHandler( int s ) {
+    exitHandler( s, ec_success );
+}
+
+void ExitHandler::exitHandler( int s, ExitHandler::exit_code_t ec ) {
+    m_signal = s;
+    if ( ec != ec_success ) {
+        g_ec = ec;
+    }
+    s_shouldExit = true;
+}
+
+volatile bool ExitHandler::s_shouldExit = false;
+volatile int ExitHandler::m_signal = -1;
+volatile ExitHandler::exit_code_t ExitHandler::g_ec = ExitHandler::ec_success;

ExitHandler.h

+#ifndef EXITHANDLER_H
+#define EXITHANDLER_H
+
+#include <signal.h>
+
+class ExitHandler {
+public:
+    enum exit_code_t {
+        ec_success = 0,
+        ec_initing_user_space = 202,  // error or exception while initializing user space
+    };
+
+private:
+    static volatile bool s_shouldExit;
+    static volatile int m_signal;
+    static volatile exit_code_t g_ec;
+
+    ExitHandler() = delete;
+
+public:
+    static void exitHandler( int s );
+    static void exitHandler( int s, ExitHandler::exit_code_t ec );
+    static bool shouldExit() { return s_shouldExit; }
+    static int getSignal() { return m_signal; }
+    static exit_code_t requestedExitCode() { return g_ec; }
+
+};
+
+#endif // EXITHANDLER_H

LevelDB.cpp

@@ -274,8 +274,8 @@ void LevelDB::initDataFolderAndDBs() {
     char cwd[PATH_MAX];
 
     if (getcwd(cwd, sizeof(cwd)) == NULL) {
-        spdlog::error("could not get current workin directory");
-        exit(-2);
+        spdlog::error("Could not get current working directory.");
+        throw SGXException(COULD_NOT_GET_WORKING_DIRECTORY, "Could not get current working directory.");
     }
 
     sgx_data_folder = string(cwd) + "/" + SGXDATA_FOLDER;
@@ -288,8 +288,8 @@ void LevelDB::initDataFolderAndDBs() {
             spdlog::info("Successfully created sgx_data folder");
         }
         else{
-            spdlog::error("Couldnt create creating sgx_data folder");
-            exit(-3);
+            spdlog::error("Could not create sgx_data folder.");
+            throw SGXException(ERROR_CREATING_SGX_DATA_FOLDER, "Could not create sgx_data folder.");
         }
     }
 

Makefile.am

@@ -70,11 +70,11 @@ bin_PROGRAMS = sgxwallet testw sgx_util
 ## have to be explicitly listed
 
 
-COMMON_SRC = ECDSASignReqMessage.cpp BLSSignReqMessage.cpp ZMQMessage.cpp ZMQServer.cpp ServerWorker.cpp InvalidStateException.cpp  Exception.cpp InvalidArgumentException.cpp Log.cpp  \
+COMMON_SRC = SGXException.cpp ExitHandler.cpp ZMQClient.cpp BLSSignRspMessage.cpp ECDSASignRspMessage.cpp ECDSASignReqMessage.cpp BLSSignReqMessage.cpp ZMQMessage.cpp ZMQServer.cpp ServerWorker.cpp InvalidStateException.cpp Exception.cpp InvalidArgumentException.cpp Log.cpp  \
              SGXWalletServer.cpp SGXRegistrationServer.cpp CSRManagerServer.cpp BLSCrypto.cpp \
              DKGCrypto.cpp ServerInit.cpp BLSPrivateKeyShareSGX.cpp LevelDB.cpp ServerDataChecker.cpp SEKManager.cpp \
              third_party/intel/sgx_stub.c third_party/intel/sgx_detect_linux.c third_party/intel/create_enclave.c third_party/intel/oc_alloc.c \
-             ECDSAImpl.c TestUtils.cpp sgxwallet.c SGXInfoServer.cpp ECDSACrypto.cpp 
+             ECDSAImpl.c TestUtils.cpp sgxwallet.c SGXInfoServer.cpp ECDSACrypto.cpp
 COMMON_ENCLAVE_SRC = secure_enclave_u.c secure_enclave_u.h
 
 sgxwallet_SOURCES = sgxwall.cpp $(COMMON_SRC)
@@ -116,12 +116,12 @@ nodist_testw_SOURCES=${nodist_sgxwallet_SOURCES}
 EXTRA_testw_DEPENDENCIES=${EXTRA_sgxwallet_DEPENDENCIES}
 testw_LDADD= ${sgxwallet_LDADD}
 
-sgx_util_SOURCES= InvalidStateException.cpp  Exception.cpp InvalidArgumentException.cpp Log.cpp sgx_util.cpp stubclient.cpp LevelDB.cpp SGXRegistrationServer.cpp CSRManagerServer.cpp
+sgx_util_SOURCES= SGXException.cpp ExitHandler.cpp InvalidStateException.cpp Exception.cpp InvalidArgumentException.cpp Log.cpp sgx_util.cpp stubclient.cpp LevelDB.cpp SGXRegistrationServer.cpp CSRManagerServer.cpp
 
 sgx_util_LDADD=-LlibBLS/deps/deps_inst/x86_or_x64/lib -Lleveldb/build -LlibBLS/build \
                    -LlibBLS/build/libff/libff \
                    -Llibzmq/build/lib/ \
                    -l:libzmq.a \
                    -l:libbls.a -l:libleveldb.a \
-                   -l:libff.a -lgmp  -ljsonrpccpp-stub -ljsonrpccpp-server -ljsonrpccpp-client -ljsonrpccpp-common -ljsoncpp -lmicrohttpd -lgnutls -lgcrypt -lidn2 -lcurl -lssl -lcrypto -lz -lpthread -ldl
+                   -l:libff.a -lgmp -ljsonrpccpp-stub -ljsonrpccpp-server -ljsonrpccpp-client -ljsonrpccpp-common -ljsoncpp -lmicrohttpd -lgnutls -lgcrypt -lidn2 -lcurl -lssl -lcrypto -lz -lpthread -ldl
 

SEKManager.cpp

@@ -90,7 +90,7 @@ void validate_SEK() {
     if (!hex2carray(test_key_ptr->c_str(), &len, encr_test_key.data(),
                     BUF_LEN)) {
         spdlog::error("Corrupt test key is LevelDB");
-        exit(-4);
+        throw SGXException(CORRUPT_DATABASE, "Corrupt test key is LevelDB");
     }
 
     sgx_status_t status = SGX_SUCCESS;
@@ -108,7 +108,7 @@ void validate_SEK() {
         spdlog::error("Invalid storage key. You need to recover using backup key");
         spdlog::error("Set the correct backup key into sgx_datasgxwallet_backup_key.txt");
         spdlog::error("Then run sgxwallet using backup flag");
-        exit(-5);
+        throw SGXException(INVALID_SEK, "Invalid storage key. Recover using backup key");
     }
 }
 
@@ -181,6 +181,7 @@ void gen_SEK() {
 
 
     if (!autoconfirm) {
+        sleep(10);
         string confirm_str = "I confirm";
         string buffer;
         do {
@@ -201,21 +202,6 @@ void gen_SEK() {
 
     setSEK(encrypted_SEK_ptr);
 
-    validate_SEK();
-
-}
-
-
-static std::atomic<int> isSgxWalletExiting(0);
-
-void  safeExit() {
-
-    // this is to make sure exit is only called once if called from multiple threads
-
-    auto previousValue = isSgxWalletExiting.exchange(1);
-
-    if (previousValue != 1)
-        exit(-6);
 }
 
 void setSEK(shared_ptr <string> hex_encrypted_SEK) {
@@ -241,10 +227,7 @@ void setSEK(shared_ptr <string> hex_encrypted_SEK) {
 
     HANDLE_TRUSTED_FUNCTION_ERROR(status, err_status, errMsg.data());
 
-
     validate_SEK();
-
-
 }
 
 #include "experimental/filesystem"
@@ -256,13 +239,13 @@ void enter_SEK() {
     shared_ptr <string> test_key_ptr = LevelDB::getLevelDb()->readString("TEST_KEY");
     if (test_key_ptr == nullptr) {
         spdlog::error("Error: corrupt or empty LevelDB database");
-        exit(-7);
+        throw SGXException(CORRUPT_DATABASE, "Could not find TEST_KEY in database.");
     }
 
 
     if (!experimental::filesystem::is_regular_file(BACKUP_PATH)) {
-        spdlog::error("File does not exist: "  BACKUP_PATH);
-        exit(-8);
+        spdlog::error("File does not exist: " BACKUP_PATH);
+        throw SGXException(FILE_NOT_FOUND, "File does not exist: " BACKUP_PATH);
     }
 
     ifstream sek_file(BACKUP_PATH);
@@ -278,7 +261,7 @@ void enter_SEK() {
 
     while (!checkHex(sek, 16)) {
         spdlog::error("Invalid hex in key");
-        exit(-9);
+        throw SGXException(SET_SEK_INVALID_SEK_HEX, "Invalid hex in key");
     }
 
     auto encrypted_SEK = check_and_set_SEK(sek);
@@ -298,10 +281,10 @@ void enter_SEK() {
 }
 
 void initSEK() {
-    shared_ptr <string> encrypted_SEK_ptr = LevelDB::getLevelDb()->readString("SEK");
     if (enterBackupKey) {
         enter_SEK();
     } else {
+        shared_ptr <string> encrypted_SEK_ptr = LevelDB::getLevelDb()->readString("SEK");
         if (encrypted_SEK_ptr == nullptr) {
             spdlog::warn("SEK was not created yet. Going to create SEK");
             gen_SEK();

SEKManager.h

@@ -47,8 +47,6 @@ EXTERNC void initSEK();
 
 EXTERNC void setSEK();
 
-EXTERNC void safeExit();
-
 
 
 

SGXException.cpp

+/*
+    Copyright (C) 2021-Present SKALE Labs
+
+    This file is part of sgxwallet.
+
+    sgxwallet is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    sgxwallet is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with sgxwallet.  If not, see <https://www.gnu.org/licenses/>.
+
+    @file SGXException.cpp
+    @author Stan Kladko
+    @date 2021
+*/
+
+#include "SGXException.h"
+
+const char* SGXException::what()  const noexcept {
+    return errString.c_str();
+}
+
+

SGXException.h

@@ -16,13 +16,13 @@
     You should have received a copy of the GNU Affero General Public License
     along with sgxwallet.  If not, see <https://www.gnu.org/licenses/>.
 
-    @file RPCException.h
+    @file SGXException.h
     @author Stan Kladko
     @date 2019
 */
 
-#ifndef SGXD_RPCEXCEPTION_H
-#define SGXD_RPCEXCEPTION_H
+#ifndef SGXD_SGXEXCEPTION_H
+#define SGXD_SGXEXCEPTION_H
 
 #include <string>
 #include <exception>
@@ -46,11 +46,11 @@ public:
         return errString;
     }
 
+    const char* what() const noexcept override;
+
     const int32_t getStatus() const {
         return status;
     }
-
-
 };
 
-#endif //SGXD_RPCEXCEPTION_H
+#endif //SGXD_SGXEXCEPTION_H

SGXInfoServer.cpp

@@ -107,18 +107,30 @@ Json::Value SGXInfoServer::isKeyExist(const string& key) {
     RETURN_SUCCESS(result)
 }
 
-int SGXInfoServer::initInfoServer(uint32_t _logLevel, bool _autoSign, bool _checkCerts, bool _generateTestKeys) {
+void SGXInfoServer::initInfoServer(uint32_t _logLevel, bool _autoSign, bool _checkCerts, bool _generateTestKeys) {
     httpServer = make_shared<HttpServer>(BASE_PORT + 4);
     server = make_shared<SGXInfoServer>(*httpServer, JSONRPC_SERVER_V2, _logLevel, _autoSign, _checkCerts, _generateTestKeys); // hybrid server (json-rpc 1.0 & 2.0)
 
+    spdlog::info("Starting info server on port {} ...", BASE_PORT + 4);
+
     if (!server->StartListening()) {
         spdlog::error("Info server could not start listening on port {}", BASE_PORT + 4);
-        exit(-10);
+        throw SGXException(SGX_INFO_SERVER_FAILED_TO_START, "Info server could not start listening.");
     } else {
         spdlog::info("Info server started on port {}", BASE_PORT + 4);
     }
+}
+
+int SGXInfoServer::exitServer() {
+  spdlog::info("Stoping SGXInfo server");
+
+  if (server && !server->StopListening()) {
+      spdlog::error("SGXInfo server could not be stopped. Will forcefully terminate the app");
+  } else {
+      spdlog::info("SGXInfo server stopped");
+  }
 
-    return 0;
+  return 0;
 }
 
 shared_ptr<SGXInfoServer> SGXInfoServer::getServer() {

SGXInfoServer.h

@@ -59,7 +59,9 @@ public:
 
     virtual Json::Value isKeyExist(const string& key);
 
-    static int initInfoServer(uint32_t _logLevel, bool _autoSign, bool _checkCerts, bool _generateTestKeys);
+    static void initInfoServer(uint32_t _logLevel, bool _autoSign, bool _checkCerts, bool _generateTestKeys);
+
+    static int exitServer();
 
 };
 

SGXRegistrationServer.cpp

@@ -162,20 +162,32 @@ Json::Value SGXRegistrationServer::GetCertificate(const string &hash) {
 }
 
 
-int SGXRegistrationServer::initRegistrationServer(bool _autoSign) {
+void SGXRegistrationServer::initRegistrationServer(bool _autoSign) {
     httpServer = make_shared<HttpServer>(BASE_PORT + 1);
     server = make_shared<SGXRegistrationServer>(*httpServer,
                                                 JSONRPC_SERVER_V2,
                                                 _autoSign); // hybrid server (json-rpc 1.0 & 2.0)
 
+    spdlog::info("Starting registration server on port {} ...", BASE_PORT + 1);
+
     if (!server->StartListening()) {
         spdlog::error("Registration server could not start listening on port {}", BASE_PORT + 1);
-        exit(-10);
+        throw SGXException(REGISTRATION_SERVER_FAILED_TO_START, "Registration server could not start listening.");
     } else {
         spdlog::info("Registration server started on port {}", BASE_PORT + 1);
     }
+}
+
+int SGXRegistrationServer::exitServer() {
+  spdlog::info("Stoping registration server");
+
+  if (server && !server->StopListening()) {
+      spdlog::error("Registration server could not be stopped. Will forcefully terminate the app");
+  } else {
+      spdlog::info("Registration server stopped");
+  }
 
-    return 0;
+  return 0;
 }
 
 

SGXRegistrationServer.h

@@ -60,9 +60,10 @@ public:
 
     virtual Json::Value GetCertificate(const string &hash);
 
-    static int initRegistrationServer(bool _autoSign = false);
+    static void initRegistrationServer(bool _autoSign = false);
 
+    static int exitServer();
 };
 
 
-#endif // SGXD_SGXREGISTRATIONSERVER_H
\ No newline at end of file
+#endif // SGXD_SGXREGISTRATIONSERVER_H

SGXWalletServer.cpp

@@ -21,7 +21,9 @@
     @date 2019
 */
 
+#include <chrono>
 #include <iostream>
+#include <thread>
 
 #include "abstractstubserver.h"
 #include <jsonrpccpp/server/connectors/httpserver.h>
@@ -30,11 +32,9 @@
 #include <stdlib.h>
 #include <unistd.h>
 
-
 #include "sgxwallet_common.h"
 #include "sgxwallet.h"
 
-
 #include "SGXException.h"
 #include "LevelDB.h"
 #include "BLSCrypto.h"
@@ -118,12 +118,14 @@ void SGXWalletServer::printDB() {
 #define NUM_THREADS 200
 #endif
 
+bool SGXWalletServer::verifyCert(string &_certFileName) {
+    string rootCAPath = string(SGXDATA_FOLDER) + "cert_data/rootCA.pem";
+    string verifyCert = "cert/verify_client_cert " + rootCAPath + " " + _certFileName;
+    return system(verifyCert.c_str()) == 0;
+}
 
-int SGXWalletServer::initHttpsServer(bool _checkCerts) {
-    COUNT_STATISTICS
-    spdlog::info("Entering {}", __FUNCTION__);
 
-    spdlog::info("Initing server, number of threads: {}", NUM_THREADS);
+void SGXWalletServer::createCertsIfNeeded() {
 
     string rootCAPath = string(SGXDATA_FOLDER) + "cert_data/rootCA.pem";
     string keyCAPath = string(SGXDATA_FOLDER) + "cert_data/rootCA.key";
@@ -137,7 +139,7 @@ int SGXWalletServer::initHttpsServer(bool _checkCerts) {
             spdlog::info("ROOT CA CERTIFICATE IS SUCCESSFULLY GENERATED");
         } else {
             spdlog::error("ROOT CA CERTIFICATE GENERATION FAILED");
-            exit(-11);
+            throw SGXException(FAIL_TO_CREATE_CERTIFICATE, "ROOT CA CERTIFICATE GENERATION FAILED");
         }
     }
 
@@ -154,37 +156,73 @@ int SGXWalletServer::initHttpsServer(bool _checkCerts) {
             spdlog::info("SERVER CERTIFICATE IS SUCCESSFULLY GENERATED");
         } else {
             spdlog::info("SERVER CERTIFICATE GENERATION FAILED");
-            exit(-12);
+            throw SGXException(FAIL_TO_CREATE_CERTIFICATE, "SERVER CERTIFICATE GENERATION FAILED");
         }
     }
 
+    spdlog::info("Verifying  server cert");
+
+    if (verifyCert(certPath)) {
+        spdlog::info("SERVER CERTIFICATE IS SUCCESSFULLY VERIFIED");
+    } else {
+        spdlog::info("SERVER CERTIFICATE VERIFICATION FAILED");
+        throw SGXException(FAIL_TO_VERIFY_CERTIFICATE, "SERVER CERTIFICATE VERIFICATION FAILED");
+    }
+}
+
+
+void SGXWalletServer::initHttpsServer(bool _checkCerts) {
+    COUNT_STATISTICS
+    spdlog::info("Entering {}", __FUNCTION__);
+    spdlog::info("Initing server, number of threads: {}", NUM_THREADS);
+
+    string certPath = string(SGXDATA_FOLDER) + "cert_data/SGXServerCert.crt";
+    string keyPath = string(SGXDATA_FOLDER) + "cert_data/SGXServerCert.key";
+    string rootCAPath = string(SGXDATA_FOLDER) + "cert_data/rootCA.pem";
+    string keyCAPath = string(SGXDATA_FOLDER) + "cert_data/rootCA.key";
+
     httpServer = make_shared<HttpServer>(BASE_PORT, certPath, keyPath, rootCAPath, _checkCerts,
                                          NUM_THREADS);
 
     server = make_shared<SGXWalletServer>(*httpServer,
                                           JSONRPC_SERVER_V2); // hybrid server (json-rpc 1.0 & 2.0)
 
+    spdlog::info("Starting sgx server on port {} ...", BASE_PORT);
+
     if (!server->StartListening()) {
         spdlog::error("SGX Server could not start listening");
-        exit(-13);
+        throw SGXException(SGX_SERVER_FAILED_TO_START, "Https server could not start listening.");
     } else {
         spdlog::info("SGX Server started on port {}", BASE_PORT);
     }
-    return 0;
 }
 
-int SGXWalletServer::initHttpServer() { //without ssl
+void SGXWalletServer::initHttpServer() { //without ssl
     COUNT_STATISTICS
     spdlog::info("Entering {}", __FUNCTION__);
+
+    spdlog::info("Starting sgx http server on port {} ...", BASE_PORT + 3);
+
     httpServer = make_shared<HttpServer>(BASE_PORT + 3, "", "", "", false,
                                          NUM_THREADS);
     server = make_shared<SGXWalletServer>(*httpServer,
                                           JSONRPC_SERVER_V2); // hybrid server (json-rpc 1.0 & 2.0)
     if (!server->StartListening()) {
         spdlog::error("Server could not start listening");
-        exit(-14);
+        throw SGXException(SGX_SERVER_FAILED_TO_START, "Http server could not start listening.");
     }
-    return 0;
+}
+
+int SGXWalletServer::exitServer() {
+  spdlog::info("Stoping sgx server");
+
+  if (server && !server->StopListening()) {
+      spdlog::error("Sgx server could not be stopped. Will forcefully terminate the app");
+  } else {
+      spdlog::info("Sgx server stopped");
+  }
+
+  return 0;
 }
 
 Json::Value
@@ -208,18 +246,19 @@ SGXWalletServer::importBLSKeyShareImpl(const string &_keyShare, const string &_k
         }
 
         if (!checkHex(hashTmp)) {
-            throw SGXException(BLS_IMPORT_INVALID_KEY_SHARE, string(__FUNCTION__) + ":Invalid BLS key share, please use hex");
+            throw SGXException(BLS_IMPORT_INVALID_KEY_SHARE,
+                               string(__FUNCTION__) + ":Invalid BLS key share, please use hex");
         }
 
         encryptedKeyShareHex = encryptBLSKeyShare2Hex(&errStatus, (char *) errMsg.data(), hashTmp.c_str());
 
         if (errStatus != 0) {
-            throw SGXException(errStatus, string(__FUNCTION__) + ":" +errMsg.data());
+            throw SGXException(errStatus, string(__FUNCTION__) + ":" + errMsg.data());
         }
 
         if (encryptedKeyShareHex.empty()) {
-            throw SGXException(BLS_IMPORT_EMPTY_ENCRYPTED_KEY_SHARE,string(__FUNCTION__) +
-                    ":Empty encrypted key share");
+            throw SGXException(BLS_IMPORT_EMPTY_ENCRYPTED_KEY_SHARE, string(__FUNCTION__) +
+                                                                     ":Empty encrypted key share");
         }
 
         result["encryptedKeyShare"] = encryptedKeyShareHex;
@@ -304,7 +343,6 @@ SGXWalletServer::blsSignMessageHashImpl(const string &_keyShareName, const strin
     result["signatureShare"] = string(signature.data());
 
 
-
     RETURN_SUCCESS(result);
 
 }
@@ -430,7 +468,7 @@ Json::Value SGXWalletServer::getPublicECDSAKeyImpl(const string &_keyName) {
     try {
         if (!checkECDSAKeyName(_keyName)) {
             throw SGXException(INVALID_ECDSA_GETPKEY_KEY_NAME, string(__FUNCTION__) +
-            ":Invalid ECDSA import key name");
+                                                               ":Invalid ECDSA import key name");
         }
         shared_ptr <string> keyStr = readFromDb(_keyName);
         publicKey = getECDSAPubKey(keyStr->c_str());
@@ -809,7 +847,7 @@ Json::Value SGXWalletServer::deleteBlsKeyImpl(const string &name) {
             result["deleted"] = true;
         } else {
             auto error_msg = "BLS key not found: " + name;
-            throw SGXException(DELETE_BLS_KEY_NOT_FOUND, string(__FUNCTION__)+ ":" + error_msg.c_str());
+            throw SGXException(DELETE_BLS_KEY_NOT_FOUND, string(__FUNCTION__) + ":" + error_msg.c_str());
         }
     } HANDLE_SGX_EXCEPTION(result)
 
@@ -898,8 +936,9 @@ Json::Value SGXWalletServer::dkgVerificationV2Impl(const string &_publicShares,
 }
 
 Json::Value
-SGXWalletServer::createBLSPrivateKeyV2Impl(const string &_blsKeyName, const string &_ethKeyName, const string &_polyName,
-                                         const string &_secretShare, int _t, int _n) {
+SGXWalletServer::createBLSPrivateKeyV2Impl(const string &_blsKeyName, const string &_ethKeyName,
+                                           const string &_polyName,
+                                           const string &_secretShare, int _t, int _n) {
     COUNT_STATISTICS
     spdlog::info("Entering {}", __FUNCTION__);
     INIT_RESULT(result)
@@ -1053,7 +1092,7 @@ SGXWalletServer::dkgVerificationV2(const string &_publicShares, const string &et
 
 Json::Value
 SGXWalletServer::createBLSPrivateKeyV2(const string &blsKeyName, const string &ethKeyName, const string &polyName,
-                                     const string &SecretShare, int t, int n) {
+                                       const string &SecretShare, int t, int n) {
     return createBLSPrivateKeyV2Impl(blsKeyName, ethKeyName, polyName, SecretShare, t, n);
 }
 
@@ -1061,6 +1100,7 @@ shared_ptr <string> SGXWalletServer::readFromDb(const string &name, const string
     auto dataStr = checkDataFromDb(prefix + name);
 
     if (dataStr == nullptr) {
+
         throw SGXException(KEY_SHARE_DOES_NOT_EXIST, string(__FUNCTION__) +  ":Data with this name does not exist: "
                                                                                                     + prefix + name);
     }

SGXWalletServer.hpp

@@ -39,8 +39,6 @@ using namespace std;
 #define TOSTRING(x) STRINGIFY(x)
 
 
-
-
 class SGXWalletServer : public AbstractStubServer {
     static shared_ptr<SGXWalletServer> server;
     static shared_ptr<HttpServer> httpServer;
@@ -55,7 +53,11 @@ class SGXWalletServer : public AbstractStubServer {
     static void checkForDuplicate(map <string, string> &_map, recursive_mutex &_m, const string &_key,
     const string &_value);
 
+
 public:
+
+    static bool verifyCert(string& _certFileName);
+
     static const char* getVersion() {
         return TOSTRING(SGXWALLET_VERSION);
     }
@@ -174,9 +176,13 @@ public:
 
     static void printDB();
 
-    static int initHttpServer();
+    static void initHttpServer();
+
+    static void initHttpsServer(bool _checkCerts);
+
+    static int exitServer();
 
-    static int initHttpsServer(bool _checkCerts);
+    static void createCertsIfNeeded();
 };
 
 #endif //SGXWALLET_SGXWALLETSERVER_HPP

ServerInit.cpp

@@ -42,7 +42,7 @@
 #include <unistd.h>
 
 
-
+#include "ExitHandler.h"
 #include "BLSPrivateKeyShareSGX.h"
 #include "sgxwallet_common.h"
 #include "third_party/intel/create_enclave.h"
@@ -68,16 +68,16 @@ using namespace std;
 void systemHealthCheck() {
     string ulimit;
     try {
-        ulimit = exec( "/bin/bash -c \"ulimit -n\"" );
-    } catch ( ... ) {
+        ulimit = exec("/bin/bash -c \"ulimit -n\"");
+    } catch (...) {
         spdlog::error("Execution of '/bin/bash -c ulimit -n' failed");
-        exit(-15);
+        throw SGXException(EXECUTION_ULIMIT_FAILED, "Execution of '/bin/bash -c ulimit -n' failed.");
     }
-    int noFiles = strtol( ulimit.c_str(), NULL, 10 );
+    int noFiles = strtol(ulimit.c_str(), NULL, 10);
 
-    auto noUlimitCheck = getenv( "NO_ULIMIT_CHECK" ) != nullptr;
+    auto noUlimitCheck = getenv("NO_ULIMIT_CHECK") != nullptr;
 
-    if ( noFiles < 65535 && !noUlimitCheck) {
+    if (noFiles < 65535 && !noUlimitCheck) {
         string errStr =
                 "sgxwallet requires setting Linux file descriptor limit to at least 65535 "
                 "You current limit (ulimit -n) is less than 65535. \n Please set it to 65535:"
@@ -85,13 +85,10 @@ void systemHealthCheck() {
                 "and setting 'DefaultLimitNOFILE=65535'\n"
                 "After that, restart sgxwallet";
         spdlog::error(errStr);
-        exit(-16);
+        throw SGXException(WRONG_ULIMIT, errStr);
     }
 }
 
-static ZMQServer* zmqServer = nullptr;
-atomic<bool> exiting(false);
-
 void initUserSpace() {
 
     libff::inhibit_profiling_counters = true;
@@ -104,26 +101,8 @@ void initUserSpace() {
     systemHealthCheck();
 #endif
 
-#ifdef EXPERIMENTAL_ZMQ_SERVER
-    zmqServer = new ZMQServer();
-    static std::thread serverThread(std::bind(&ZMQServer::run, zmqServer));
-#endif
 }
 
-void exitZMQServer() {
-#ifdef EXPERIMENTAL_ZMQ_SERVER
-
-    auto doExit = !exiting.exchange(true);
-
-    if (doExit) {
-        spdlog::info("Exiting zmq server ...");
-        delete zmqServer;
-        spdlog::info("Exited zmq server ...");
-        zmqServer = nullptr;
-    }
-
-#endif
-}
 
 uint64_t initEnclave() {
 
@@ -133,7 +112,7 @@ uint64_t initEnclave() {
     support = get_sgx_support();
     if (!SGX_OK(support)) {
         sgx_support_perror(support);
-        exit(-17);
+        throw SGXException(COULD_NOT_INIT_ENCLAVE, "SGX is not supported or not enabled");
     }
 #endif
 
@@ -164,11 +143,11 @@ uint64_t initEnclave() {
             } else {
                 spdlog::error("sgx_create_enclave_search failed {} {}", ENCLAVE_NAME, status);
             }
-            exit(-21);
+            throw SGXException(COULD_NOT_INIT_ENCLAVE, "Error initing enclave. Please re-check your enviroment.");
         }
 
         spdlog::info("Enclave created and started successfully");
-        
+
         status = trustedEnclaveInit(eid, enclaveLogLevel);
     }
 
@@ -183,8 +162,8 @@ uint64_t initEnclave() {
 }
 
 
-
-void initAll(uint32_t _logLevel, bool _checkCert, bool _autoSign, bool _generateTestKeys) {
+void initAll(uint32_t _logLevel, bool _checkCert,
+             bool _checkZMQSig, bool _autoSign, bool _generateTestKeys) {
 
 
     static atomic<bool> sgxServerInited(false);
@@ -206,9 +185,9 @@ void initAll(uint32_t _logLevel, bool _checkCert, bool _autoSign, bool _generate
         uint64_t counter = 0;
 
         uint64_t initResult = 0;
-        while ((initResult = initEnclave()) != 0 && counter < 10){
+        while ((initResult = initEnclave()) != 0 && counter < 10) {
             sleep(1);
-            counter ++;
+            counter++;
         }
 
         if (initResult != 0) {
@@ -218,27 +197,45 @@ void initAll(uint32_t _logLevel, bool _checkCert, bool _autoSign, bool _generate
         initUserSpace();
         initSEK();
 
+        SGXWalletServer::createCertsIfNeeded();
+
         if (useHTTPS) {
+            spdlog::info("Initing JSON-RPC server over HTTPS");
+            spdlog::info("Check client cert: {}", _checkCert);
             SGXWalletServer::initHttpsServer(_checkCert);
-            SGXRegistrationServer::initRegistrationServer(_autoSign);
-            CSRManagerServer::initCSRManagerServer();
+            spdlog::info("Inited JSON-RPC server over HTTPS");
         } else {
+            spdlog::info("Initing JSON-RPC server over HTTP");
             SGXWalletServer::initHttpServer();
+            spdlog::info("Inited JSON-RPC server over HTTP");
         }
+
+        SGXRegistrationServer::initRegistrationServer(_autoSign);
+        CSRManagerServer::initCSRManagerServer();
         SGXInfoServer::initInfoServer(_logLevel, _checkCert, _autoSign, _generateTestKeys);
+        ZMQServer::initZMQServer(_checkZMQSig);
 
         sgxServerInited = true;
     } catch (SGXException &_e) {
         spdlog::error(_e.getMessage());
-        exit(-18);
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_initing_user_space);
     } catch (exception &_e) {
         spdlog::error(_e.what());
-        exit(-19);
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_initing_user_space);
     }
     catch (...) {
         exception_ptr p = current_exception();
         printf("Exception %s \n", p.__cxa_exception_type()->name());
         spdlog::error("Unknown exception");
-        exit(-22);
+        ExitHandler::exitHandler(SIGTERM, ExitHandler::ec_initing_user_space);
     }
 };
+
+void exitAll() {
+    SGXWalletServer::exitServer();
+    SGXRegistrationServer::exitServer();
+    CSRManagerServer::exitServer();
+    SGXInfoServer::exitServer();
+    ZMQServer::exitZMQServer();
+
+}

ServerInit.h

@@ -32,7 +32,9 @@
 #define EXTERNC
 #endif
 
-EXTERNC void initAll(uint32_t  _logLevel, bool _checkCert, bool _autoSign, bool _generateTestKeys);
+EXTERNC void initAll(uint32_t  _logLevel, bool _checkCert, bool _checkZMQSig, bool _autoSign, bool _generateTestKeys);
+
+void exitAll();
 
 EXTERNC void initUserSpace();
 

ServerWorker.cpp

-//
-// Created by kladko on 14.12.20.
-//
+/*
+    Copyright (C) 2019-Present SKALE Labs
+
+    This file is part of sgxwallet.
+
+    sgxwallet is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    sgxwallet is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with sgxwallet.  If not, see <https://www.gnu.org/licenses/>.
+
+    @file ServerWorker.cpp
+    @author Stan Kladko
+    @date 2021
+*/
+
+
 #include "common.h"
+#include "sgxwallet_common.h"
 #include <json/writer.h>
+
+
+#include <zmq.hpp>
+#include "zhelpers.hpp"
+
+#include "Log.h"
 #include "ZMQMessage.h"
+
 #include "ServerWorker.h"
 
+std::atomic <uint64_t>  ServerWorker::workerCount(1);
 
-ServerWorker::ServerWorker(zmq::context_t &ctx, int sock_type) : ctx_(ctx),
-                                                                 worker_(ctx_, sock_type) {};
+ServerWorker::ServerWorker(zmq::context_t &_ctx, int sock_type, bool _checkSignature,
+                           const string& _caCert ) :            checkSignature(_checkSignature),
+                           caCert(_caCert),
+                                                                 isExitRequested(false) {
+
+    worker = make_shared<zmq::socket_t>(_ctx, sock_type);
+
+    if (checkSignature) {
+        CHECK_STATE(!caCert.empty())
+    }
+
+    index = workerCount.fetch_add(1);
+    int linger = 0;
+    zmq_setsockopt(*worker, ZMQ_LINGER, &linger, sizeof(linger));
+};
+
+
+
+void ServerWorker::doOneServerLoop() noexcept {
+    string replyStr;
+
+    Json::Value result;
+    result["status"] = ZMQ_SERVER_ERROR;
+    result["errorMessage"] = "";
+
+    zmq::message_t identity;
+    zmq::message_t identit2;
+    zmq::message_t copied_id;
 
-void ServerWorker::work() {
-    worker_.connect("inproc://backend");
     try {
-        while (true) {
-            zmq::message_t msg;
-            zmq::message_t copied_msg;
-            worker_.recv(&msg);
 
-            vector<uint8_t> msgData(msg.size() + 1, 0);
 
-            memcpy(msgData.data(), msg.data(), msg.size());
+        zmq_pollitem_t items[1];
+        items[0].socket = *worker;
+        items[0].events = ZMQ_POLLIN;
 
-            auto parsedMsg = ZMQMessage::parse(msgData);
+        int pollResult = 0;
 
-            CHECK_STATE(parsedMsg);
+        do {
+            pollResult = zmq_poll(items, 1, 1000);
+            if (isExitRequested) {
+                return;
+            }
+        } while (pollResult == 0);
 
-            auto reply  = parsedMsg->process();
 
-            Json::FastWriter fastWriter;
-            std::string replyStr = fastWriter.write(reply);
+        zmq::message_t msg;
+        zmq::message_t copied_msg;
+        worker->recv(&identity);
+        copied_id.copy(&identity);
+        worker->recv(&msg);
 
-            zmq::message_t replyMsg(replyStr.c_str(),replyStr.size() + 1);
-            worker_.send(replyMsg);
-        }
+        int64_t more;
+        size_t more_size = sizeof(more);
+        auto rc = zmq_getsockopt(*worker, ZMQ_RCVMORE, &more, &more_size);
+
+        CHECK_STATE2(rc == 0, ZMQ_COULD_NOT_GET_SOCKOPT);
+
+        vector <uint8_t> msgData(msg.size() + 1, 0);
+
+        memcpy(msgData.data(), msg.data(), msg.size());
+
+        CHECK_STATE2(msg.size() > 5 || msgData.at(0) == '{' || msgData[msg.size()] == '}',
+        ZMQ_INVALID_MESSAGE);
+
+        memcpy(msgData.data(), msg.data(), msg.size());
+
+        auto parsedMsg = ZMQMessage::parse(
+                (const char *) msgData.data(), msg.size(), true, checkSignature);
+
+        CHECK_STATE2(parsedMsg, ZMQ_COULD_NOT_PARSE);
+
+        result = parsedMsg->process();
+
+    } catch (SGXException &e) {
+        result["status"] = e.getStatus();
+        result["errorMessage"] = e.what();
+        spdlog::error("Exception in zmq server worker:{}", e.what());
     }
     catch (std::exception &e) {
-        spdlog::info("Exiting zmq server worker:{}", e.what());
-        return;
+        if (isExitRequested) {
+            return;
+        }
+        result["errorMessage"] = string(e.what());
+        spdlog::error("Exception in zmq server worker:{}", e.what());
     } catch (...) {
+        if (isExitRequested) {
+            return;
+        }
         spdlog::error("Error in zmq server worker");
-        return;
+        result["errorMessage"] = "Error in zmq server worker";
+    }
+
+    try {
+
+        Json::FastWriter fastWriter;
+
+        replyStr = fastWriter.write(result);
+        replyStr = replyStr.substr(0, replyStr.size() - 1);
+
+        CHECK_STATE(replyStr.size() > 2);
+        CHECK_STATE(replyStr.front() == '{');
+        CHECK_STATE(replyStr.back() == '}');
+        zmq::message_t replyMsg(replyStr.c_str(), replyStr.size() + 1);
+
+        worker->send(copied_id, ZMQ_SNDMORE);
+        worker->send(replyMsg);
+
+    } catch (std::exception &e) {
+        if (isExitRequested) {
+            return;
+        }
+        spdlog::error("Exception in zmq server worker send :{}", e.what());
+    } catch (...) {
+        if (isExitRequested) {
+            return;
+        }
+        spdlog::error("Unklnown exception in zmq server worker send");
     }
+}
+
+void ServerWorker::work() {
+    worker->connect("inproc://backend");
 
+
+    while (!isExitRequested) {
+        try {
+            doOneServerLoop();
+        } catch (...) {
+            spdlog::error("doOneServerLoop threw exception. This should never happen!");
+        }
+    }
+
+    spdlog::info("Exited worker thread {}", index);
 }
 
 
+void ServerWorker::requestExit() {
+    isExitRequested.exchange(true);
+    spdlog::info("Closed worker socket {}", index);
+}
+

ServerWorker.h

-//
-// Created by kladko on 14.12.20.
-//
+/*
+    Copyright (C) 2019-Present SKALE Labs
 
+    This file is part of sgxwallet.
+
+    sgxwallet is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    sgxwallet is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with sgxwallet.  If not, see <https://www.gnu.org/licenses/>.
+
+    @file ServerWorker.h
+    @author Stan Kladko
+    @date 2021
+*/
 #ifndef SGXWALLET_SERVERWORKER_H
 #define SGXWALLET_SERVERWORKER_H
 
@@ -19,16 +37,26 @@
 
 class ServerWorker {
 
+    bool checkSignature = true;
+    string caCert = "";
+
 public:
-    ServerWorker(zmq::context_t &ctx, int sock_type );
+    ServerWorker(zmq::context_t &ctx, int sock_type, bool _checkSignature, const string& _caCert );
 
 
     void work();
 
+    void requestExit();
 
 private:
-    zmq::context_t &ctx_;
-    zmq::socket_t worker_;
+    shared_ptr<zmq::socket_t> worker;
+
+    std::atomic<bool> isExitRequested;
+
+    void doOneServerLoop() noexcept;
+
+    static std::atomic<uint64_t> workerCount;
+    uint64_t index;
 };
 
 

TestUtils.cpp

@@ -42,6 +42,7 @@
 #include "SGXWalletServer.hpp"
 
 #include "catch.hpp"
+#include "ZMQClient.h"
 #include "BLSSigShare.h"
 #include "BLSSigShareSet.h"
 #include "BLSPublicKeyShare.h"
@@ -398,6 +399,96 @@ void TestUtils::sendRPCRequestV2() {
     sigShareSet.merge();
 }
 
+
+void TestUtils::sendRPCRequestZMQ() {
+    HttpClient client(RPC_ENDPOINT);
+    StubClient c(client, JSONRPC_CLIENT_V2);
+
+
+
+
+    int n = 16, t = 16;
+    Json::Value ethKeys[n];
+    Json::Value verifVects[n];
+    Json::Value pubEthKeys;
+    Json::Value secretShares[n];
+    Json::Value pubBLSKeys[n];
+    Json::Value blsSigShares[n];
+    vector <string> pubShares(n);
+    vector <string> polyNames(n);
+
+    static atomic<int> counter(1);
+
+    int schainID = counter.fetch_add(1);
+    int dkgID = counter.fetch_add(1);
+    for (uint8_t i = 0; i < n; i++) {
+        ethKeys[i] = c.generateECDSAKey();
+        CHECK_STATE(ethKeys[i]["status"] == 0);
+        string polyName =
+                "POLY:SCHAIN_ID:" + to_string(schainID) + ":NODE_ID:" + to_string(i) + ":DKG_ID:" + to_string(dkgID);
+        auto response = c.generateDKGPoly(polyName, t);
+        CHECK_STATE(response["status"] == 0);
+        polyNames[i] = polyName;
+        verifVects[i] = c.getVerificationVector(polyName, t, n);
+        CHECK_STATE(verifVects[i]["status"] == 0);
+
+        pubEthKeys.append(ethKeys[i]["publicKey"]);
+    }
+
+    for (uint8_t i = 0; i < n; i++) {
+        secretShares[i] = c.getSecretShareV2(polyNames[i], pubEthKeys, t, n);
+        for (uint8_t k = 0; k < t; k++) {
+            for (uint8_t j = 0; j < 4; j++) {
+                string pubShare = verifVects[i]["verificationVector"][k][j].asString();
+                pubShares[i] += convertDecToHex(pubShare);
+            }
+        }
+    }
+
+    vector <string> secShares(n);
+
+    for (int i = 0; i < n; i++)
+        for (int j = 0; j < n; j++) {
+            string secretShare = secretShares[i]["secretShare"].asString().substr(192 * j, 192);
+            secShares[i] += secretShares[j]["secretShare"].asString().substr(192 * i, 192);
+            Json::Value verif = c.dkgVerificationV2(pubShares[i], ethKeys[j]["keyName"].asString(), secretShare, t, n, j);
+            CHECK_STATE(verif["status"] == 0);
+        }
+
+    BLSSigShareSet sigShareSet(t, n);
+
+    string hash = SAMPLE_HASH;
+
+    auto hash_arr = make_shared < array < uint8_t, 32 >> ();
+    uint64_t binLen;
+    if (!hex2carray(hash.c_str(), &binLen, hash_arr->data(), 32)) {
+        throw SGXException(TEST_INVALID_HEX, "Invalid hash");
+    }
+
+    map <size_t, shared_ptr<BLSPublicKeyShare>> coeffs_pkeys_map;
+
+    Json::Value publicShares;
+    for (int i = 0; i < n; ++i) {
+        publicShares["publicShares"][i] = pubShares[i];
+    }
+
+    Json::Value blsPublicKeys = c.calculateAllBLSPublicKeys(publicShares, t, n);
+    CHECK_STATE(blsPublicKeys["status"] == 0);
+
+    for (int i = 0; i < t; i++) {
+        string blsName = "BLS_KEY" + polyNames[i].substr(4);
+        string hash = SAMPLE_HASH;
+        blsSigShares[i] = c.blsSignMessageHash(blsName, hash, t, n);
+        CHECK_STATE(blsSigShares[i]["status"] == 0);
+
+        shared_ptr <string> sig_share_ptr = make_shared<string>(blsSigShares[i]["signatureShare"].asString());
+        BLSSigShare sig(sig_share_ptr, i + 1, t, n);
+        sigShareSet.addSigShare(make_shared<BLSSigShare>(sig));
+    }
+
+    sigShareSet.merge();
+}
+
 void TestUtils::destroyEnclave() {
     if (eid != 0) {
         sgx_destroy_enclave(eid);
@@ -687,6 +778,146 @@ void TestUtils::doDKGV2(StubClient &c, int n, int t,
         cerr << i << endl;
 }
 
+
+void TestUtils::doZMQBLS(shared_ptr<ZMQClient> _zmqClient, StubClient &c, int n, int t,
+                        vector<string>& _ecdsaKeyNames, vector<string>& _blsKeyNames,
+                        int schainID, int dkgID) {
+    Json::Value ethKeys[n];
+    Json::Value verifVects[n];
+    Json::Value pubEthKeys;
+    Json::Value secretShares[n];
+    Json::Value pubBLSKeys[n];
+    Json::Value blsSigShares[n];
+    vector<string> pubShares(n);
+    vector<string> polyNames(n);
+
+    _ecdsaKeyNames.clear();
+    _blsKeyNames.clear();
+
+    for (uint8_t i = 0; i < n; i++) {
+        ethKeys[i] = c.generateECDSAKey();
+
+        CHECK_STATE(ethKeys[i]["status"] == 0);
+
+        auto keyName = ethKeys[i]["keyName"].asString();
+        CHECK_STATE(keyName.size() == ECDSA_KEY_NAME_SIZE);
+
+        _ecdsaKeyNames.push_back(keyName);
+
+        string polyName =
+                "POLY:SCHAIN_ID:" + to_string(schainID) + ":NODE_ID:" + to_string(i) + ":DKG_ID:" + to_string(dkgID);
+
+        Json::Value response = c.generateDKGPoly(polyName, t);
+        CHECK_STATE(response["status"] == 0);
+        polyNames[i] = polyName;
+        verifVects[i] = c.getVerificationVector(polyName, t, n);
+        CHECK_STATE(verifVects[i]["status"] == 0);
+        pubEthKeys.append(ethKeys[i]["publicKey"]);
+    }
+
+    for (uint8_t i = 0; i < n; i++) {
+        secretShares[i] = c.getSecretShareV2(polyNames[i], pubEthKeys, t, n);
+        CHECK_STATE(secretShares[i]["status"] == 0);
+        for (uint8_t k = 0; k < t; k++) {
+            for (uint8_t j = 0; j < 4; j++) {
+                string pubShare = verifVects[i]["verificationVector"][k][j].asString();
+                CHECK_STATE(pubShare.length() > 60);
+                pubShares[i] += TestUtils::convertDecToHex(pubShare);
+            }
+        }
+    }
+
+    int k = 0;
+
+    vector<string> secShares(n);
+
+    vector<string> pSharesBad(pubShares);
+
+    for (int i = 0; i < n; i++)
+        for (int j = 0; j < n; j++) {
+            string secretShare = secretShares[i]["secretShare"].asString().substr(192 * j, 192);
+            secShares[i] += secretShares[j]["secretShare"].asString().substr(192 * i, 192);
+            Json::Value response = c.dkgVerificationV2(pubShares[i], ethKeys[j]["keyName"].asString(), secretShare, t, n,
+                                                       j);
+            CHECK_STATE(response["status"] == 0);
+
+            bool res = response["result"].asBool();
+            CHECK_STATE(res);
+
+            k++;
+
+            pSharesBad[i][0] = 'q';
+            Json::Value wrongVerif = c.dkgVerificationV2(pSharesBad[i], ethKeys[j]["keyName"].asString(), secretShare, t,
+                                                         n, j);
+            res = wrongVerif["result"].asBool();
+            CHECK_STATE(!res);
+        }
+
+    BLSSigShareSet sigShareSet(t, n);
+
+    string hash = SAMPLE_HASH;
+
+    auto hash_arr = make_shared<array<uint8_t, 32 >>();
+    uint64_t binLen;
+    if (!hex2carray(hash.c_str(), &binLen, hash_arr->data(), 32)) {
+        throw SGXException(TEST_INVALID_HEX, "Invalid hash");
+    }
+
+    map<size_t, shared_ptr<BLSPublicKeyShare>> pubKeyShares;
+
+    for (int i = 0; i < n; i++) {
+        string endName = polyNames[i].substr(4);
+        string blsName = "BLS_KEY" + polyNames[i].substr(4);
+        _blsKeyNames.push_back(blsName);
+        string secretShare = secretShares[i]["secretShare"].asString();
+
+        auto response = c.createBLSPrivateKeyV2(blsName, ethKeys[i]["keyName"].asString(), polyNames[i], secShares[i], t,
+                                                n);
+        CHECK_STATE(response["status"] == 0);
+        pubBLSKeys[i] = c.getBLSPublicKeyShare(blsName);
+        CHECK_STATE(pubBLSKeys[i]["status"] == 0);
+    }
+
+    for (int i = 0; i < t; i++) {
+        vector<string> pubKeyVect;
+        for (uint8_t j = 0; j < 4; j++) {
+            pubKeyVect.push_back(pubBLSKeys[i]["blsPublicKeyShare"][j].asString());
+        }
+        BLSPublicKeyShare pubKey(make_shared<vector<string >>(pubKeyVect), t, n);
+
+        pubKeyShares[i + 1] = make_shared<BLSPublicKeyShare>(pubKey);
+    }
+
+    // create pub key
+
+    BLSPublicKey blsPublicKey(make_shared<map<size_t, shared_ptr<BLSPublicKeyShare >>>(pubKeyShares), t,
+            n);
+
+    // sign verify a sample sig
+
+    for (int i = 0; i < t; i++) {
+
+        string blsName = "BLS_KEY" + polyNames[i].substr(4);
+        auto sigShare =  make_shared<string>(_zmqClient->blsSignMessageHash(blsName, hash, t, n));
+        BLSSigShare sig(sigShare, i + 1, t, n);
+        sigShareSet.addSigShare(make_shared<BLSSigShare>(sig));
+
+        auto pubKey = pubKeyShares[i+1];
+
+        CHECK_STATE(pubKey->VerifySigWithHelper(hash_arr, make_shared<BLSSigShare>(sig), t, n));
+    }
+
+    shared_ptr<BLSSignature> commonSig = sigShareSet.merge();
+
+    CHECK_STATE(blsPublicKey.VerifySigWithHelper(hash_arr, commonSig, t, n));
+
+    for (auto&& i : _ecdsaKeyNames)
+        cerr << i << endl;
+
+    for (auto&& i : _blsKeyNames)
+        cerr << i << endl;
+}
+
 int sessionKeyRecoverDH(const char *skey_str, const char *sshare, char *common_key) {
 
     int ret = -1;

TestUtils.h

@@ -41,6 +41,7 @@
 #include <sgx_tcrypto.h>
 #include "stubclient.h"
 #include <jsonrpccpp/server/connectors/httpserver.h>
+#include "ZMQClient.h"
 #include "abstractstubserver.h"
 
 using namespace std;
@@ -81,6 +82,13 @@ public:
     static void doDKGV2(StubClient &c, int n, int t,
                                  vector<string>& _ecdsaKeyNames, vector<string>& _blsKeyNames,
                                  int schainID, int dkgID);
+
+    static void doZMQBLS(shared_ptr<ZMQClient> _zmqClient, StubClient &c, int n, int t,
+                        vector<string>& _ecdsaKeyNames, vector<string>& _blsKeyNames,
+                        int schainID, int dkgID);
+
+    static void sendRPCRequestZMQ();
+
 };
 
 int sessionKeyRecoverDH(const char *skey_str, const char *sshare, char *common_key);

ZMQClient.cpp

+/*
+    Copyright (C) 2018-2019 SKALE Labs
+
+    This file is part of skale-consensus.
+
+    skale-consensus is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    skale-consensus is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with skale-consensus.  If not, see <https://www.gnu.org/licenses/>.
+
+    @file ZMQClient.cpp
+    @author Stan Kladko
+    @date 2020
+*/
+
+#include "sys/random.h"
+#include <sys/types.h>
+#include <sys/syscall.h>
+
+
+#include <fstream>
+#include <streambuf>
+#include <regex>
+
+#include "sgxwallet_common.h"
+#include "common.h"
+#include "BLSCrypto.h"
+#include "BLSSignReqMessage.h"
+#include "BLSSignRspMessage.h"
+#include "ECDSASignReqMessage.h"
+#include "ECDSASignRspMessage.h"
+#include "ZMQClient.h"
+
+
+shared_ptr <ZMQMessage> ZMQClient::doRequestReply(Json::Value &_req) {
+
+    Json::FastWriter fastWriter;
+
+    if (sign) {
+        CHECK_STATE(!certificate.empty());
+        CHECK_STATE(!key.empty());
+
+        _req["cert"] = certificate;
+
+        string msgToSign = fastWriter.write(_req);
+
+        _req["msgSig"] = signString(pkey, msgToSign);
+    }
+
+    string reqStr = fastWriter.write(_req);
+
+
+    reqStr = reqStr.substr(0, reqStr.size() - 1);
+    CHECK_STATE(reqStr.front() == '{');
+    CHECK_STATE(reqStr.at(reqStr.size() - 1) == '}');
+
+
+    auto resultStr = doZmqRequestReply(reqStr);
+
+    try {
+
+        CHECK_STATE(resultStr.size() > 5)
+        CHECK_STATE(resultStr.front() == '{')
+        CHECK_STATE(resultStr.back() == '}')
+
+
+        return ZMQMessage::parse(resultStr.c_str(), resultStr.size(), false, false);
+    } catch (std::exception &e) {
+        spdlog::error(string("Error in doRequestReply:") + e.what());
+        throw;
+    } catch (...) {
+        spdlog::error("Error in doRequestReply");
+        throw;
+    }
+
+
+}
+
+
+string ZMQClient::doZmqRequestReply(string &_req) {
+
+    stringstream request;
+
+    shared_ptr <zmq::socket_t> clientSocket = nullptr;
+
+    {
+        lock_guard <recursive_mutex> m(mutex);
+        if (!clientSockets.count(getProcessID()))
+            reconnect();
+        clientSocket = clientSockets.at(getProcessID());
+        CHECK_STATE(clientSocket);
+    }
+    CHECK_STATE(clientSocket);
+
+    spdlog::debug("ZMQ client sending: \n {}", _req);
+
+    s_send(*clientSocket, _req);
+
+    while (true) {
+        //  Poll socket for a reply, with timeout
+        zmq::pollitem_t items[] = {
+                {static_cast<void *>(*clientSocket), 0, ZMQ_POLLIN, 0}};
+        zmq::poll(&items[0], 1, REQUEST_TIMEOUT);
+        //  If we got a reply, process it
+        if (items[0].revents & ZMQ_POLLIN) {
+            string reply = s_recv(*clientSocket);
+
+            CHECK_STATE(reply.size() > 5);
+            reply = reply.substr(0, reply.size() - 1);
+            spdlog::debug("ZMQ client received reply:{}", reply);
+            CHECK_STATE(reply.front() == '{');
+            CHECK_STATE(reply.back() == '}');
+
+            return reply;
+        } else {
+            spdlog::error("W: no response from server, retrying...");
+            reconnect();
+            //  Send request again, on new socket
+            s_send(*clientSocket, _req);
+        }
+    }
+}
+
+
+string ZMQClient::readFileIntoString(const string &_fileName) {
+    ifstream t(_fileName);
+    string str((istreambuf_iterator<char>(t)), istreambuf_iterator<char>());
+    return str;
+}
+
+
+
+void ZMQClient::verifySig(EVP_PKEY* _pubkey, const string& _str, const string& _sig) {
+
+    CHECK_STATE(_pubkey);
+    CHECK_STATE(!_str.empty());
+
+    static std::regex r("\\s+");
+    auto msgToSign = std::regex_replace(_str, r, "");
+
+    vector<uint8_t> binSig(256,0);
+
+    uint64_t binLen = 0;
+
+    CHECK_STATE2(hex2carray(_sig.c_str(), &binLen, binSig.data(), binSig.size()),
+                 ZMQ_COULD_NOT_PARSE);
+
+    CHECK_STATE(binLen > 0);
+
+    EVP_MD_CTX *mdctx = NULL;
+    int ret = 0;
+
+    size_t slen = 0;
+
+    CHECK_STATE(mdctx = EVP_MD_CTX_create());
+
+    CHECK_STATE((EVP_DigestVerifyInit(mdctx, NULL, EVP_sha256(), NULL, _pubkey) == 1));
+
+    CHECK_STATE(EVP_DigestVerifyUpdate(mdctx, msgToSign.c_str(), msgToSign.size()) == 1);
+
+/* First call EVP_DigestSignFinal with a NULL sig parameter to obtain the length of the
+ * signature. Length is returned in slen */
+
+
+
+    CHECK_STATE2(EVP_DigestVerifyFinal(mdctx, binSig.data(), binLen) == 1,
+                 ZMQ_COULD_NOT_VERIFY_SIG);
+
+    if (mdctx) EVP_MD_CTX_destroy(mdctx);
+
+    return;
+}
+
+
+string ZMQClient::signString(EVP_PKEY* _pkey, const string& _str) {
+
+    CHECK_STATE(_pkey);
+    CHECK_STATE(!_str.empty());
+
+    static std::regex r("\\s+");
+    auto msgToSign = std::regex_replace(_str, r, "");
+
+
+
+    EVP_MD_CTX *mdctx = NULL;
+    int ret = 0;
+    unsigned char *signature = NULL;
+    auto sig = &signature;
+    size_t slen = 0;
+
+    CHECK_STATE(mdctx = EVP_MD_CTX_create());
+
+
+    CHECK_STATE((EVP_DigestSignInit(mdctx, NULL, EVP_sha256(), NULL, _pkey) == 1));
+
+
+    CHECK_STATE(EVP_DigestSignUpdate(mdctx, msgToSign.c_str(), msgToSign.size()) == 1);
+
+/* First call EVP_DigestSignFinal with a NULL sig parameter to obtain the length of the
+ * signature. Length is returned in slen */
+
+    CHECK_STATE(EVP_DigestSignFinal(mdctx, NULL, &slen) == 1);
+    signature = (unsigned char *) OPENSSL_malloc(sizeof(unsigned char) * slen);
+    CHECK_STATE(signature);
+    CHECK_STATE(EVP_DigestSignFinal(mdctx, signature, &slen) == 1);
+
+    auto hexSig = carray2Hex(signature, slen);
+
+    string hexStringSig(hexSig.begin(), hexSig.end());
+
+    /* Clean up */
+    if (signature) OPENSSL_free(signature);
+    if (mdctx) EVP_MD_CTX_destroy(mdctx);
+
+    return hexStringSig;
+}
+
+pair<EVP_PKEY*, X509*> ZMQClient::readPublicKeyFromCertStr(const string& _certStr) {
+
+    CHECK_STATE(!_certStr.empty())
+
+    BIO *bo = BIO_new(BIO_s_mem());
+    CHECK_STATE(bo);
+    BIO_write(bo, _certStr.c_str(), _certStr.size());
+
+    X509* cert = nullptr;
+    PEM_read_bio_X509(bo, &cert, 0, 0);
+    CHECK_STATE(cert);
+    auto key = X509_get_pubkey(cert);
+    BIO_free(bo);
+    CHECK_STATE(key);
+    return {key, cert};
+};
+
+ZMQClient::ZMQClient(const string &ip, uint16_t port, bool _sign, const string &_certFileName,
+                     const string &_certKeyName) : ctx(1), sign(_sign),
+                                                   certKeyName(_certKeyName), certFileName(_certFileName) {
+
+    spdlog::info("Initing ZMQClient. Sign:{} ", _sign);
+
+    if (sign) {
+        CHECK_STATE(!_certFileName.empty());
+        CHECK_STATE(!_certKeyName.empty());
+
+        certificate = readFileIntoString(_certFileName);
+        CHECK_STATE(!certificate.empty());
+
+        key = readFileIntoString(_certKeyName);
+        CHECK_STATE(!key.empty());
+
+        BIO *bo = BIO_new(BIO_s_mem());
+        CHECK_STATE(bo);
+        BIO_write(bo, key.c_str(), key.size());
+
+        PEM_read_bio_PrivateKey(bo, &pkey, 0, 0);
+        CHECK_STATE(pkey);
+        BIO_free(bo);
+
+        auto pubKeyStr = readFileIntoString(_certFileName);
+        CHECK_STATE(!pubKeyStr.empty());
+
+        tie(pubkey, x509Cert) = readPublicKeyFromCertStr(pubKeyStr);
+
+        auto sig = signString(pkey, "sample");
+        verifySig(pubkey, "sample", sig);
+
+    } else {
+        CHECK_STATE(_certFileName.empty());
+        CHECK_STATE(_certKeyName.empty());
+    }
+
+    certFileName = _certFileName;
+    certKeyName = _certKeyName;
+
+
+    url = "tcp://" + ip + ":" + to_string(port);
+}
+
+void ZMQClient::reconnect() {
+
+    lock_guard <recursive_mutex> lock(mutex);
+
+    auto pid = getProcessID();
+
+    if (clientSockets.count(pid) > 0) {
+        clientSockets.erase(pid);
+    }
+
+
+    char identity[10];
+    getrandom(identity, 10, 0);
+    auto clientSocket = make_shared<zmq::socket_t>(ctx, ZMQ_DEALER);
+    clientSocket->setsockopt(ZMQ_IDENTITY, identity, 10);
+    //  Configure socket to not wait at close time
+    int linger = 0;
+    clientSocket->setsockopt(ZMQ_LINGER, &linger, sizeof(linger));
+    clientSocket->connect(url);
+    clientSockets.insert({pid, clientSocket});
+}
+
+
+string ZMQClient::blsSignMessageHash(const std::string &keyShareName, const std::string &messageHash, int t, int n) {
+    Json::Value p;
+    p["type"] = ZMQMessage::BLS_SIGN_REQ;
+    p["keyShareName"] = keyShareName;
+    p["messageHash"] = messageHash;
+    p["n"] = n;
+    p["t"] = t;
+    auto result = dynamic_pointer_cast<BLSSignRspMessage>(doRequestReply(p));
+    CHECK_STATE(result);
+    CHECK_STATE(result->getStatus() == 0);
+
+    return result->getSigShare();
+}
+
+string ZMQClient::ecdsaSignMessageHash(int base, const std::string &keyName, const std::string &messageHash) {
+    Json::Value p;
+    p["type"] = ZMQMessage::ECDSA_SIGN_REQ;
+    p["base"] = base;
+    p["keyName"] = keyName;
+    p["messageHash"] = messageHash;
+    auto result = dynamic_pointer_cast<ECDSASignRspMessage>(doRequestReply(p));
+    CHECK_STATE(result);
+    CHECK_STATE(result->getStatus() == 0);
+    return result->getSignature();
+}
+
+
+uint64_t ZMQClient::getProcessID() {
+    return syscall(__NR_gettid);
+}
\ No newline at end of file

ZMQClient.h

+/*
+    Copyright (C) 2018-2019 SKALE Labs
+
+    This file is part of skale-consensus.
+
+    skale-consensus is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    skale-consensus is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with skale-consensus.  If not, see <https://www.gnu.org/licenses/>.
+
+    @file ZMQClient.h
+    @author Stan Kladko
+    @date 2021
+*/
+
+
+
+
+#ifndef SGXWALLET_ZMQCLIENT_H
+#define SGXWALLET_ZMQCLIENT_H
+
+#include <openssl/pem.h>
+#include <openssl/evp.h>
+#include <openssl/err.h>
+#include <openssl/sha.h>
+#include <openssl/rand.h>
+
+#include "third_party/spdlog/spdlog.h"
+
+#include <zmq.hpp>
+#include "zhelpers.hpp"
+#include <jsonrpccpp/client.h>
+#include "ZMQMessage.h"
+
+
+
+
+#define REQUEST_TIMEOUT     10000    //  msecs, (> 1000!)
+
+class ZMQClient {
+
+
+private:
+
+    EVP_PKEY* pkey = 0;
+    EVP_PKEY* pubkey = 0;
+    X509* x509Cert = 0;
+
+
+    bool sign = true;
+    string certFileName = "";
+    string certKeyName = "";
+    string certificate = "";
+    string key = "";
+
+
+    recursive_mutex mutex;
+
+    zmq::context_t ctx;
+
+    string url;
+
+    // generate random identity
+
+    map<uint64_t , shared_ptr <zmq::socket_t>> clientSockets;
+
+    shared_ptr <ZMQMessage> doRequestReply(Json::Value &_req);
+
+    string doZmqRequestReply(string &_req);
+
+    uint64_t getProcessID();
+
+    static string readFileIntoString(const string& _fileName);
+
+public:
+
+
+    ZMQClient(const string &ip, uint16_t port, bool _sign, const string&  _certPathName,
+              const string& _certKeyName);
+
+    void reconnect() ;
+
+    static pair<EVP_PKEY*, X509*>  readPublicKeyFromCertStr(const string& _cert);
+
+    static string signString(EVP_PKEY* _pkey, const string& _str);
+
+    static void   verifySig(EVP_PKEY* _pubkey, const string& _str, const string& _sig);
+
+    string blsSignMessageHash(const std::string &keyShareName, const std::string &messageHash, int t, int n);
+
+    string ecdsaSignMessageHash(int base, const std::string &keyName, const std::string &messageHash);
+
+};
+
+
+
+#endif //SGXWALLET_ZMQCLIENT_H

ZMQMessage.cpp

@@ -22,17 +22,24 @@
 */
 
 #include "common.h"
+#include "sgxwallet_common.h"
+#include <third_party/cryptlite/sha256.h>
+#include <iostream>
+#include <fstream>
 
+#include "ZMQClient.h"
+#include "SGXWalletServer.hpp"
 #include "BLSSignReqMessage.h"
+#include "BLSSignRspMessage.h"
 #include "ECDSASignReqMessage.h"
+#include "ECDSASignRspMessage.h"
 #include "ZMQMessage.h"
 
 
-
 uint64_t ZMQMessage::getUint64Rapid(const char *_name) {
     CHECK_STATE(_name);
     CHECK_STATE(d->HasMember(_name));
-    const rapidjson::Value& a = (*d)[_name];
+    const rapidjson::Value &a = (*d)[_name];
     CHECK_STATE(a.IsUint64());
     return a.GetUint64();
 };
@@ -44,31 +51,120 @@ string ZMQMessage::getStringRapid(const char *_name) {
     return (*d)[_name].GetString();
 };
 
-shared_ptr<ZMQMessage> ZMQMessage::parse(vector<uint8_t>& _msg) {
 
-    CHECK_STATE(_msg.at(_msg.size() - 1) == 0);
+
+
+shared_ptr <ZMQMessage> ZMQMessage::parse(const char *_msg,
+                                          size_t _size, bool _isRequest,
+                                          bool _verifySig) {
+
+    CHECK_STATE(_msg);
+    CHECK_STATE2(_size > 5, ZMQ_INVALID_MESSAGE_SIZE);
+    // CHECK NULL TERMINATED
+    CHECK_STATE(_msg[_size] == 0);
+    CHECK_STATE2(_msg[_size - 1] == '}', ZMQ_INVALID_MESSAGE);
+    CHECK_STATE2(_msg[0] == '{', ZMQ_INVALID_MESSAGE);
 
     auto d = make_shared<rapidjson::Document>();
 
-    d->Parse((const char*) _msg.data());
+    d->Parse(_msg);
+
+    CHECK_STATE2(!d->HasParseError(), ZMQ_COULD_NOT_PARSE);
+    CHECK_STATE2(d->IsObject(), ZMQ_COULD_NOT_PARSE);
+
+    CHECK_STATE2(d->HasMember("type"), ZMQ_NO_TYPE_IN_MESSAGE);
+    CHECK_STATE2((*d)["type"].IsString(), ZMQ_NO_TYPE_IN_MESSAGE);
+    string type = (*d)["type"].GetString();
+
+    if (_verifySig) {
+        CHECK_STATE2(d->HasMember("cert"),ZMQ_NO_CERT_IN_MESSAGE);
+        CHECK_STATE2(d->HasMember("msgSig"), ZMQ_NO_SIG_IN_MESSAGE);
+        CHECK_STATE2((*d)["cert"].IsString(), ZMQ_NO_CERT_IN_MESSAGE);
+        CHECK_STATE2((*d)["msgSig"].IsString(), ZMQ_NO_SIG_IN_MESSAGE);
+
+        auto cert = make_shared<string>((*d)["cert"].GetString());
+        string hash = cryptlite::sha256::hash_hex(*cert);
+
+        auto filepath = "/tmp/sgx_wallet_cert_hash_" + hash;
+
+        std::ofstream outFile(filepath);
+
+        outFile << *cert;
+
+        outFile.close();
+
+        static recursive_mutex m;
+
+        EVP_PKEY *publicKey = nullptr;
+
+        {
+            lock_guard <recursive_mutex> lock(m);
+
+            if (!verifiedCerts.exists(*cert)) {
+                CHECK_STATE(SGXWalletServer::verifyCert(filepath));
+                auto handles = ZMQClient::readPublicKeyFromCertStr(*cert);
+                CHECK_STATE(handles.first);
+                CHECK_STATE(handles.second);
+                verifiedCerts.put(*cert, handles);
+                remove(cert->c_str());
+            }
+
+            publicKey = verifiedCerts.get(*cert).first;
 
-    CHECK_STATE(!d->HasParseError());
-    CHECK_STATE(d->IsObject())
+            CHECK_STATE(publicKey);
 
-    CHECK_STATE(d->HasMember("type"));
-    CHECK_STATE((*d)["type"].IsString());
-    auto type =  (*d)["type"].GetString();
+            auto msgSig = make_shared<string>((*d)["msgSig"].GetString());
 
-    shared_ptr<ZMQMessage> result;
+            d->RemoveMember("msgSig");
 
-    if (type == ZMQMessage::BLS_SIGN_REQ) {
-        result = make_shared<BLSSignReqMessage>(d);
-    } else if (type == ZMQMessage::ECDSA_SIGN_REQ) {
-        result = make_shared<ECDSASignReqMessage>(d);
+            rapidjson::StringBuffer buffer;
+
+            rapidjson::Writer<rapidjson::StringBuffer> w(buffer);
+
+            d->Accept(w);
+
+            auto msgToVerify = buffer.GetString();
+
+            ZMQClient::verifySig(publicKey,msgToVerify, *msgSig );
+
+        }
+    }
+
+
+    shared_ptr <ZMQMessage> result;
+
+    if (_isRequest) {
+        return buildRequest(type, d);
     } else {
-        throw SGXException(-301, "Incorrect zmq message type: " + string(type));
+        return buildResponse(type, d);
     }
+}
 
-    return result;
+shared_ptr <ZMQMessage> ZMQMessage::buildRequest(string &_type, shared_ptr <rapidjson::Document> _d) {
+    if (_type == ZMQMessage::BLS_SIGN_REQ) {
+        return make_shared<BLSSignReqMessage>(_d);
+    } else if (_type == ZMQMessage::ECDSA_SIGN_REQ) {
+        return
+                make_shared<ECDSASignReqMessage>(_d);
+    } else {
+        BOOST_THROW_EXCEPTION(SGXException(-301, "Incorrect zmq message type: " +
+                                                 string(_type)));
+    }
+}
 
+shared_ptr <ZMQMessage> ZMQMessage::buildResponse(string &_type, shared_ptr <rapidjson::Document> _d) {
+    if (_type == ZMQMessage::BLS_SIGN_RSP) {
+        return
+                make_shared<BLSSignRspMessage>(_d);
+    } else if (_type == ZMQMessage::ECDSA_SIGN_RSP) {
+        return
+                make_shared<ECDSASignRspMessage>(_d);
+    } else {
+        BOOST_THROW_EXCEPTION(InvalidStateException("Incorrect zmq message request type: " + string(_type),
+                                                    __CLASS_NAME__)
+        );
+    }
 }
+
+cache::lru_cache<string, pair < EVP_PKEY * , X509 *>>
+ZMQMessage::verifiedCerts(256);
\ No newline at end of file

ZMQMessage.h

@@ -26,10 +26,21 @@
 
 #include <memory>
 #include <vector>
-#include "abstractstubserver.h"
 
+#include <openssl/pem.h>
+#include <openssl/evp.h>
+#include <openssl/err.h>
+#include <openssl/sha.h>
+#include <openssl/rand.h>
+
+#include "third_party/lrucache.hpp"
+
+#include "abstractstubserver.h"
 
 #include "document.h"
+#include "stringbuffer.h"
+#include "writer.h"
+
 #include "SGXException.h"
 
 using namespace std;
@@ -38,16 +49,20 @@ class ZMQMessage {
 
     shared_ptr<rapidjson::Document> d;
 
-    static constexpr const char *BLS_SIGN_REQ = "BLSSignReq";
-    static constexpr const char *BLS_SIGN_RSP = "BLSSignRsp";
-    static constexpr const char *ECDSA_SIGN_REQ = "ECDSASignReq";
-    static constexpr const char *ECDSA_SIGN_RSP = "ECDSASignRsp";
+
+    static cache::lru_cache<string, pair<EVP_PKEY*, X509*>> verifiedCerts;
 
 protected:
 
 
 public:
 
+
+    static constexpr const char *BLS_SIGN_REQ = "BLSSignReq";
+    static constexpr const char *BLS_SIGN_RSP = "BLSSignRsp";
+    static constexpr const char *ECDSA_SIGN_REQ = "ECDSASignReq";
+    static constexpr const char *ECDSA_SIGN_RSP = "ECDSASignRsp";
+
     explicit ZMQMessage(shared_ptr<rapidjson::Document> &_d) : d(_d) {
     };
 
@@ -55,7 +70,15 @@ public:
 
     uint64_t getUint64Rapid(const char *_name);
 
-    static shared_ptr<ZMQMessage> parse(vector<uint8_t> &_msg);
+    uint64_t getStatus() {
+        return getUint64Rapid("status");
+    }
+
+    static shared_ptr <ZMQMessage> parse(const char* _msg, size_t _size, bool _isRequest,
+                                         bool _verifySig);
+
+    static shared_ptr<ZMQMessage> buildRequest(string& type, shared_ptr<rapidjson::Document> _d);
+    static shared_ptr<ZMQMessage> buildResponse(string& type, shared_ptr<rapidjson::Document> _d);
 
     virtual Json::Value process() = 0;
 

ZMQServer.cpp

@@ -21,68 +21,190 @@
     @date 2019
 */
 
+#include <fstream>
+#include <streambuf>
 
 
 #include "third_party/spdlog/spdlog.h"
 
+#include "common.h"
+
+#include "SGXException.h"
 #include "ZMQServer.h"
 #include "sgxwallet_common.h"
 
 using namespace std;
 
-ZMQServer::ZMQServer()
-        : isExitRequested(false), ctx_(1),
-          frontend_(ctx_, ZMQ_ROUTER),
-          backend_(ctx_, ZMQ_DEALER) {}
+shared_ptr <ZMQServer> ZMQServer::zmqServer = nullptr;
+
+ZMQServer::ZMQServer(bool _checkSignature, const string &_caCertFile)
+        : checkSignature(_checkSignature),
+          caCertFile(_caCertFile), ctx_(make_shared<zmq::context_t>(1)) {
+
+
+    frontend = make_shared<zmq::socket_t>(*ctx_, ZMQ_ROUTER);
+    backend = make_shared<zmq::socket_t>(*ctx_, ZMQ_DEALER);
+
+    //workerThreads = 2 * thread::hardware_concurrency();
+
+
+
+    workerThreads = 4; // do four threads for now
+
+    if (_checkSignature) {
+        CHECK_STATE(!_caCertFile.empty());
+        ifstream t(_caCertFile);
+        string str((istreambuf_iterator<char>(t)), istreambuf_iterator<char>());
+        caCert = str;
+        CHECK_STATE(!caCert.empty())
+    }
+
+    int linger = 0;
+
+
+    zmq_setsockopt(*frontend, ZMQ_LINGER, &linger, sizeof(linger));
+    zmq_setsockopt(*backend, ZMQ_LINGER, &linger, sizeof(linger));
+
+}
 
 
 void ZMQServer::run() {
 
-    auto port = BASE_PORT + 4;
+    auto port = BASE_PORT + 5;
 
-    spdlog::info("Starting zmq server ...");
+    spdlog::info("Starting zmq server on port {} ...", port);
 
     try {
-        frontend_.bind("tcp://*:" + to_string(BASE_PORT + 4));
+        CHECK_STATE(frontend);
+        frontend->bind("tcp://*:" + to_string(port));
     } catch (...) {
         spdlog::error("Server task could not bind to port:{}", port);
-        exit(-100);
+        throw SGXException(ZMQ_COULD_NOT_BIND_FRONT_END, "Server task could not bind.");
     }
 
     spdlog::info("Bound port ...");
 
     try {
-        backend_.bind("inproc://backend");
+        CHECK_STATE(backend);
+        backend->bind("inproc://backend");
     } catch (exception &e) {
         spdlog::error("Could not bind to zmq backend: {}", e.what());
-        exit(-101);
+        throw SGXException(ZMQ_COULD_NOT_BIND_BACK_END, "Could not bind to zmq backend.");
     }
 
 
-    spdlog::info("Creating {} zmq server workers ...", kMaxThread);
+    spdlog::info("Creating {} zmq server workers ...", workerThreads);
 
     try {
-        for (int i = 0; i < kMaxThread; ++i) {
-            worker.push_back(new ServerWorker(ctx_, ZMQ_DEALER));
-            worker_thread.push_back(new std::thread(std::bind(&ServerWorker::work, worker[i])));
+        for (int i = 0; i < workerThreads; ++i) {
+            workers.push_back(make_shared<ServerWorker>(*ctx_, ZMQ_DEALER,
+                                                        this->checkSignature, this->caCert));
+            auto th = make_shared<std::thread>(std::bind(&ServerWorker::work, workers[i]));
+            worker_threads.push_back(th);
         }
     } catch (std::exception &e) {
         spdlog::error("Could not create zmq server workers:{} ", e.what());
-        exit(-102);
-    }
+        throw SGXException(ZMQ_COULD_NOT_CREATE_WORKERS, "Could not create zmq server workers.");
+    };
 
 
     try {
-        zmq::proxy(static_cast<void *>(frontend_), static_cast<void *>(backend_), nullptr);
-    } catch (exception& _e) {
-        spdlog::info("Exiting zmq server {}", _e.what());
+        zmq::proxy(static_cast<void *>(*frontend), static_cast<void *>(*backend), nullptr);
+        spdlog::info("Exited zmq proxy");
+    } catch (exception &_e) {
+        if (isExitRequested) {
+            spdlog::info("Exited ZMQServer main thread");
+            return;
+        }
+        spdlog::info("Error, exiting zmq server ... {}", _e.what());
         return;
     } catch (...) {
-        spdlog::info("Exiting zmq server");
-        return;
+        if (isExitRequested) {
+            spdlog::info("Exited ZMQServer main thread");
+            return;
+        }
+        spdlog::info("Error, exiting zmq server ...");
+        throw SGXException(ZMQ_COULD_NOT_CREATE_PROXY, "Error, exiting zmq server.");
+    }
+}
+
+
+void ZMQServer::exitAll() {
+
+    spdlog::info("Exiting zmq server workers ...");
+
+    for (auto &&worker : workers) {
+        worker->requestExit();
+    }
+
+    for (auto &&workerThread : worker_threads) {
+        workerThread->join();
+    }
+
+    spdlog::info("Exited zmq server workers ...");
+
+}
+
+std::atomic<bool> ZMQServer::isExitRequested(false);
+
+void ZMQServer::exitZMQServer() {
+    auto doExit = !isExitRequested.exchange(true);
+    if (doExit) {
+
+        zmqServer->exitAll();
+
+        spdlog::info("deleting zmq server");
+        zmqServer = nullptr;
+        spdlog::info("deleted zmq server ");
     }
 }
 
+void ZMQServer::initZMQServer(bool _checkSignature) {
+    static bool initedServer = false;
+    CHECK_STATE(!initedServer)
+    initedServer = true;
+
+    spdlog::info("Initing zmq server. checkSignature is set to {}", _checkSignature);
+
+    string rootCAPath = "";
+
+    if (_checkSignature) {
+
+        rootCAPath = string(SGXDATA_FOLDER) + "cert_data/rootCA.pem";
+        spdlog::info("Reading root CA from {}", rootCAPath);
+        CHECK_STATE(access(rootCAPath.c_str(), F_OK) == 0);
+    };
+
+    zmqServer = make_shared<ZMQServer>(_checkSignature, rootCAPath);
+    serverThread = make_shared<thread>(std::bind(&ZMQServer::run, ZMQServer::zmqServer));
+    serverThread->detach();
+
+    spdlog::info("Inited zmq server ...");
+}
+
+shared_ptr <std::thread> ZMQServer::serverThread = nullptr;
+
+ZMQServer::~ZMQServer() {
+
+    spdlog::info("Deleting worker threads");
+    worker_threads.clear();
+    spdlog::info("Deleted worker threads");
 
+    spdlog::info("Deleting workers ...");
+    workers.clear();
+    spdlog::info("Deleted workers ...");
 
+    spdlog::info("Deleting front end and back end");
+    frontend = nullptr;
+    backend = nullptr;
+    spdlog::info("Deleted front end and back end");
 
+    spdlog::info("Deleting server thread");
+    ZMQServer::serverThread = nullptr;
+    spdlog::info("Deleted server thread");
+
+
+    spdlog::info("Deleting ZMQ context");
+    ctx_ = nullptr;
+    spdlog::info("Deleted ZMQ context");
+}

ZMQServer.h

@@ -42,26 +42,44 @@ using namespace std;
 
 
 class ZMQServer {
+
+    uint64_t workerThreads;
+
 public:
-    ZMQServer();
 
+    bool checkSignature = false;
+    string caCertFile = "";
+    string caCert = "";
+
+    static shared_ptr<ZMQServer> zmqServer;
 
-    atomic<bool> isExitRequested;
+    static shared_ptr<std::thread> serverThread;
 
-    enum {
-        kMaxThread = 5
-    };
+    ZMQServer(bool _checkSignature, const string& _caCertFile);
+
+    ~ZMQServer();
 
     void run();
 
+    void exitAll();
+
+    static void initZMQServer(bool _checkSignature);
+    static void exitZMQServer();
+
+
+
 private:
-    zmq::context_t ctx_;
-    zmq::socket_t frontend_;
-    zmq::socket_t backend_;
+    shared_ptr<zmq::context_t> ctx_;
+    shared_ptr<zmq::socket_t> frontend;
+    shared_ptr<zmq::socket_t> backend;
+
+    std::vector<shared_ptr<ServerWorker> > workers;
+    std::vector<shared_ptr<std::thread>> worker_threads;
 
-    std::vector<ServerWorker *> worker;
-    std::vector<std::thread *> worker_thread;
+    static std::atomic<bool> isExitRequested;
 
 };
 
+
+
 #endif //SGXWALLET_ZMQServer_H

cert/verify_client_cert

+#!/bin/bash
+
+openssl verify -CAfile $1 $2
\ No newline at end of file

common.h

@@ -61,7 +61,7 @@ inline std::string className(const std::string &prettyFunction) {
 
 #include <execinfo.h>
 
-inline void print_stack() {
+inline void print_stack(int _line) {
     void *array[10];
     size_t size;
 
@@ -69,9 +69,8 @@ inline void print_stack() {
     size = backtrace(array, 10);
 
     // print out all the frames to stderr
-    fprintf(stderr, "Error: signal \n");
+    fprintf(stderr, "Backtrace on line %d:   \n", _line);
     backtrace_symbols_fd(array, size, STDERR_FILENO);
-    exit(-1);
 }
 
 inline int parseLine(char* line) {
@@ -104,9 +103,17 @@ inline int getValue() { //Note: this value is in KB!
 #define CHECK_STATE(_EXPRESSION_) \
     if (!(_EXPRESSION_)) { \
         auto __msg__ = std::string("State check failed::") + #_EXPRESSION_ +  " " + std::string(__FILE__) + ":" + std::to_string(__LINE__); \
-        print_stack();                                \
+        print_stack(__LINE__);            \
+        \
         BOOST_THROW_EXCEPTION(SGXException(-100, string(__CLASS_NAME__) +  ":" + __msg__));}
 
+#define CHECK_STATE2(_EXPRESSION_, __STATUS__) \
+    if (!(_EXPRESSION_)) { \
+        auto __msg__ = std::string("State check failed::") + #_EXPRESSION_ +  " " + std::string(__FILE__) + ":" + std::to_string(__LINE__); \
+        print_stack(__LINE__);            \
+        \
+        BOOST_THROW_EXCEPTION(SGXException(__STATUS__, string(__CLASS_NAME__) +  ":" + __msg__));}
+
 
 #define HANDLE_TRUSTED_FUNCTION_ERROR(__STATUS__, __ERR_STATUS__, __ERR_MSG__) \
 if (__STATUS__ != SGX_SUCCESS) { \

docs/admin-guide.md

@@ -10,6 +10,7 @@
     -   [Run sgxwallet in secure mode](run-in-hardware-mode.md#run-sgxwallet-in-secure-mode)
     -   [Start, stop and upgrade sgxwallet containers](run-in-hardware-mode.md#start-stop-and-upgrade-sgxwallet-containers)
     -   [Logging](run-in-hardware-mode.md#logging)
+-   [Check that your SGXWallet is working correctly](healthchecks.md)
 -   [Backup and recover sgxwallet](backup-procedure.md)
 
 ## Community

docs/backup-procedure.md

@@ -39,3 +39,35 @@ docker-compose up -d
 ```
 
 5.  Edit the `docker-compose.yml` file, remove the `-b` flag.
+
+## Upgrade SGXWallet
+
+To upgrade SGXWallet to the version with different enclave code you need to backup your data first and then start SGXWallet in backup mode. To do this please follow the instructions:
+
+Ensure you have your backup key.
+
+1.  Stop the container:
+
+```bash
+docker-compose down
+```
+
+2.  Copy the entire `sgx_data` directory.
+
+3.  Edit the `docker-compose.yml` and add the `-b` flag to recover from backup and change image name.
+
+```yaml
+image: skalenetwork/<NEW_IMAGE>
+
+command: -s -y -d -b
+```
+
+4.  Copy the backed up `sgx_data` directory to the recovery `sgx_data` directory.
+5.  Create file `sgx_data/sgxwallet_backup_key.txt` in the recovery directory and write the backup key into it.
+6.  Execute:
+
+```bash
+docker-compose up -d
+```
+
+7.  Edit the `docker-compose.yml` file, remove the `-b` flag.
\ No newline at end of file

docs/healthchecks.md

+<!-- SPDX-License-Identifier: (AGPL-3.0-only OR CC-BY-4.0) -->
+
+# SGXServer healthchecks
+
+-   [Check JSON-RPC server](#check-json-rpc-server)
+-   [Check Secure Enclave part](#check-secure-enclave-part)
+
+## Check JSON-RPC server
+
+To verify JSON-RPC server inside SGXWallet is up running execute one of the following commands:
+
+```bash
+curl --cert PATH_TO_CERTS/file.crt --key PATH_TO_CERTS/file.key -X POST --data '{"jsonrpc":"2.0","id":1,"method":"getServerStatus","params":{}}' -H 'content-type:application/json;' YOUR_SGX_SERVER_URL -k
+```
+
+```bash
+curl --cert PATH_TO_CERTS/file.crt --key PATH_TO_CERTS/file.key -X POST --data '{"jsonrpc":"2.0","id":2,"method":"getServerVersion","params":{}}' -H 'content-type:application/json;' YOUR_SGX_SERVER_URL -k
+```
+
+If server does not respond or response contains error message than you should restart your SGXWallet.
+
+## Check Secure Enclave part
+
+To verify Secure Enclave part of SGXWallet is configured and initialized in a proper way run following commands:
+
+1. 
+```bash
+curl --cert PATH_TO_CERTS/file.crt --key PATH_TO_CERTS/file.key -X POST --data '{"jsonrpc":"2.0","id":3,"method":"importBLSKeyShare","params":{"keyShare":"0xe632f7fde2c90a073ec43eaa90dca7b82476bf28815450a11191484934b9c3f", "keyShareName":"BLS_KEY:SCHAIN_ID:123456789:NODE_ID:0:DKG_ID:0"}}' -H 'content-type:application/json;' YOUR_SGX_SERVER_URL -k
+```
+
+```bash
+curl --cert PATH_TO_CERTS/file.crt --key PATH_TO_CERTS/file.key -X POST --data '{"jsonrpc":"2.0","id":4,"method":"blsSignMessageHash","params":{"keyShareName":"BLS_KEY:SCHAIN_ID:123456789:NODE_ID:0:DKG_ID:0", "t":1, "n":1, "messageHash":"09c6137b97cdf159b9950f1492ee059d1e2b10eaf7d51f3a97d61f2eee2e81db"}}' -H 'content-type:application/json;' YOUR_SGX_SERVER_URL -k
+```
+
+2. 
+```bash
+curl --cert PATH_TO_CERTS/file.crt --key PATH_TO_CERTS/file.key -X POST --data '{"jsonrpc":"2.0","id":5,"method":"importECDSAKey","params":{"key":"0xe632f7fde2c90a073ec43eaa90dca7b82476bf28815450a11191484934b9c3f", "keyName":"NEK:abcdef"}}' -H 'content-type:application/json;' YOUR_SGX_SERVER_URL -k
+```
+
+```bash
+curl --cert PATH_TO_CERTS/file.crt --key PATH_TO_CERTS/file.key -X POST --data '{"jsonrpc":"2.0","id":6,"method":"ecdsaSignMessageHash","params":{"keyName":"NEK:abcdef", "base":16, "messageHash":"09c6137b97cdf159b9950f1492ee059d1e2b10eaf7d51f3a97d61f2eee2e81db"}}' -H 'content-type:application/json;' YOUR_SGX_SERVER_URL -k
+```
+
+Any error during one of the calls means that SGXWallet is misconfigured and will not work as you expect. Please try to run SGXWallet in backup mode.

run_sgx/docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 services:
   sgxwallet:
     image: skalenetwork/sgxwallet_signed:latest
-    restart: always
+    restart: unless-stopped
     ports:
       - "1026:1026"
       - "1027:1027"
@@ -20,7 +20,6 @@ services:
       options:
         max-size: "10m"
         max-file: "4"
-    restart: unless-stopped
     command: -s -y -d
     healthcheck:
       test: ["CMD", "ls", "/dev/isgx", "/dev/mei0"]

run_sgx_sim/docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 services:
   sgxwallet:
     image: skalenetwork/sgxwallet_sim:develop-latest
-    restart: always
+    restart: unless-stopped
     ports:
       - "1026:1026"
       - "1027:1027"
@@ -17,5 +17,4 @@ services:
       options:
         max-size: "10m"
         max-file: "4"
-    restart: unless-stopped
     command: -s -y

scripts/install_packages.sh

 #!/bin/bash
-sudo dpkg -i *.deb
 sudo apt install -y build-essential make gcc g++ yasm  python libprotobuf10 flex bison automake
 sudo apt install -y ccache cmake ccache autoconf texinfo libgcrypt20-dev libgnutls28-dev libtool pkg-config

secure_enclave/Makefile.in

@@ -320,7 +320,7 @@ SGX_EDGER8R = $(SGXSDK_BINDIR)/sgx_edger8r
 SGX_SIGN = $(SGXSDK_BINDIR)/sgx_sign
 SGXSSL_BINDIR = @SGXSSL_BINDIR@
 AM_CFLAGS = @SGX_ENCLAVE_CFLAGS@
-AM_CPPFLAGS = @SGX_ENCLAVE_CPPFLAGS@ -Wall \
+AM_CPPFLAGS = @SGX_ENCLAVE_CPPFLAGS@ -O2 -Wall \
 	-Wno-implicit-function-declaration $(TGMP_CPPFLAGS) \
 	-I./third_party/SCIPR -I../third_party/SCIPR \
 	-I../sgx-sdk-build/sgxsdk/include/libcxx

sgxwall.cpp

@@ -21,15 +21,17 @@
     @date 2020
 */
 
+#include <csignal>
 #include <stdbool.h>
 
+#include "ExitHandler.h"
+
 #include "BLSCrypto.h"
 #include "ServerInit.h"
 
 #include "SEKManager.h"
 #include "SGXWalletServer.h"
 
-
 #include <fstream>
 
 #include "TestUtils.h"
@@ -41,11 +43,6 @@
 #include "sgxwallet.h"
 
 
-void SGXWallet::usage() {
-    cerr << "usage: sgxwallet\n";
-    exit(-21);
-}
-
 void SGXWallet::printUsage() {
     cerr << "\nAvailable flags:\n";
     cerr << "\nDebug flags:\n\n";
@@ -54,10 +51,10 @@ void SGXWallet::printUsage() {
     cerr << "\nBackup, restore, update flags:\n\n";
     cerr << "   -b  filename Restore from back up or software update. You will need to put backup key into a file in sgx_data dir. \n";
     cerr << "   -y  Do not ask user to acknowledge receipt of the backup key \n";
-    cerr << "\nHTTPS flags:\n\n";
-    cerr << "   -n  Launch sgxwallet using http. Default is to use https with a selg-signed server cert.  \n";
-    cerr << "   -c  Do not verify SSL client certs\n";
-    cerr << "   -s  Sign SSL client certs without human confirmation \n";
+    cerr << "\nSecurity flags flags:\n\n";
+    cerr << "   -n  Use http instead of https. Default is to use https with a selg-signed server cert.  Insecure! \n";
+    cerr << "   -c  Disable client authentication using certificates. Insecure!\n";
+    cerr << "   -s  Sign client certificates without human confirmation. Insecure! \n";
 }
 
 
@@ -88,6 +85,11 @@ void SGXWallet::serializeKeys(const vector<string>& _ecdsaKeyNames, const vector
     fs.close();
 }
 
+void SGXWallet::signalHandler( int signalNo ) {
+    spdlog::info("Received exit signal {}.", signalNo);
+    ExitHandler::exitHandler( signalNo );
+}
+
 
 int main(int argc, char *argv[]) {
     bool enterBackupKeyOption  = false;
@@ -99,18 +101,20 @@ int main(int argc, char *argv[]) {
     bool autoSignClientCertOption = false;
     bool generateTestKeys = false;
 
+    std::signal(SIGABRT, SGXWallet::signalHandler);
+
     int opt;
 
     if (argc > 1 && strlen(argv[1]) == 1) {
         SGXWallet::printUsage();
-        exit(-22);
+        exit(-21);
     }
 
     while ((opt = getopt(argc, argv, "cshd0abyvVnT")) != -1) {
         switch (opt) {
             case 'h':
                 SGXWallet::printUsage();
-                exit(-24);
+                exit(-22);
             case 'c':
                 checkClientCertOption = false;
                 break;
@@ -174,11 +178,11 @@ int main(int argc, char *argv[]) {
         enclaveLogLevel = L_TRACE;
     }
 
-    initAll(enclaveLogLevel, checkClientCertOption, autoSignClientCertOption, generateTestKeys);
+    initAll(enclaveLogLevel, checkClientCertOption, checkClientCertOption, autoSignClientCertOption, generateTestKeys);
 
     ifstream is("sgx_data/4node.json");
 
-    if (generateTestKeys && !is.good()) {
+    if (generateTestKeys && !is.good() && !!ExitHandler::shouldExit()) {
         cerr << "Generating test keys ..." << endl;
 
         HttpClient client(RPC_ENDPOINT);
@@ -206,9 +210,14 @@ int main(int argc, char *argv[]) {
 
 
 
-    while (true) {
+    while ( !ExitHandler::shouldExit() ) {
         sleep(10);
     }
 
-    return 0;
+    ExitHandler::exit_code_t exitCode = ExitHandler::requestedExitCode();
+    int signal = ExitHandler::getSignal();
+    spdlog::info("Will exit with exit code {}", exitCode);
+    exitAll();
+    spdlog::info("Exiting with exit code {} and signal", exitCode, signal);
+    return exitCode;
 }

sgxwall.h

@@ -26,7 +26,8 @@ class SGXWallet {
 
 public:
 
-    static void usage();
+    static void signalHandler( int signalNo );
+
     static void printUsage();
 
     static void serializeKeys( const vector<string>& _ecdsaKeyNames,

sgxwallet_common.h

@@ -159,6 +159,31 @@ extern bool autoconfirm;
 #define ECDSA_SIGN_INVALID_KEY_HEX -86
 #define SET_SEK_INVALID_SEK_HEX -87
 #define TEST_INVALID_HEX -88
+#define ZMQ_SERVER_ERROR -89
+#define ZMQ_COULD_NOT_PARSE -90
+#define ZMQ_INVALID_MESSAGE -91
+#define ZMQ_COULD_NOT_GET_SOCKOPT -92
+#define ZMQ_INVALID_MESSAGE_SIZE -93
+#define ZMQ_NO_TYPE_IN_MESSAGE -94
+#define ZMQ_NO_SIG_IN_MESSAGE -95
+#define ZMQ_NO_CERT_IN_MESSAGE -96
+#define ZMQ_COULD_NOT_VERIFY_SIG -97
+#define ZMQ_COULD_NOT_BIND_FRONT_END -98
+#define ZMQ_COULD_NOT_BIND_BACK_END -99
+#define ZMQ_COULD_NOT_CREATE_WORKERS -100
+#define ZMQ_COULD_NOT_CREATE_PROXY -101
+#define REGISTRATION_SERVER_FAILED_TO_START -102
+#define CSR_MANAGER_SERVER_FAILED_TO_START -103
+#define SGX_INFO_SERVER_FAILED_TO_START -104
+#define COULD_NOT_GET_WORKING_DIRECTORY -105
+#define ERROR_CREATING_SGX_DATA_FOLDER -106
+#define EXECUTION_ULIMIT_FAILED -107
+#define WRONG_ULIMIT -108
+#define COULD_NOT_INIT_ENCLAVE -109
+#define FAIL_TO_VERIFY_CERTIFICATE -110
+#define SGX_SERVER_FAILED_TO_START -111
+#define CORRUPT_DATABASE -112
+#define INVALID_SEK -113
 
 #define SGX_ENCLAVE_ERROR -666
 
@@ -166,7 +191,7 @@ extern bool autoconfirm;
 
 #define BASE_PORT 1026
 
-#define WALLETDB_NAME  "sgxwallet.db"//"test_sgxwallet.db"
+#define WALLETDB_NAME "sgxwallet.db"
 #define ENCLAVE_NAME "secure_enclave.signed.so"
 #define SGXDATA_FOLDER "sgx_data/"
 

testw.h

@@ -33,6 +33,10 @@
 
 #define SAMPLE_POLY_NAME "POLY:SCHAIN_ID:1:NODE_ID:1:DKG_ID:1"
 #define RPC_ENDPOINT  "http://localhost:1029"
+#define RPC_ENDPOINT_HTTPS  "https://localhost:1026"
+#define ZMQ_IP "127.0.0.1"
+#define ZMQ_PORT 1031
+
 #define SAMPLE_PUBLIC_KEY_B "c0152c48bf640449236036075d65898fded1e242c00acb45519ad5f788ea7cbf9a5df1559e7fc87932eee5478b1b9023de19df654395574a690843988c3ff475"
 
 #define SAMPLE_DKG_PUB_KEY_1 "505f55a38f9c064da744f217d1cb993a17705e9839801958cda7c884e08ab4dad7fd8d22953d3ac7f0913de24fd67d7ed36741141b8a3da152d7ba954b0f14e2"

testw.py

@@ -28,7 +28,9 @@ username = getpass.getuser()
 
 topDir = os.getcwd() + "/sgxwallet"
 print("Top directory is:" + topDir)
-testList = ["[first-run]",
+testList = [ "[zmq-ecdsa]",
+            "[dkgzmqbls]",
+             "[first-run]",
             "[second-run]",
             "[many-threads-crypto]",
             "[many-threads-crypto-v2]",

third_party/lrucache.hpp

+/*
+ * File:   lrucache.hpp
+ * Author: Alexander Ponomarev
+ *
+ * Created on June 20, 2013, 5:09 PM
+ */
+
+
+#ifndef _LRUCACHE_HPP_INCLUDED_
+#define	_LRUCACHE_HPP_INCLUDED_
+
+#include <mutex>
+#include <shared_mutex>
+
+
+#include <any>
+
+#include <unordered_map>
+#include <list>
+#include <string>
+#include <cstddef>
+#include <stdexcept>
+
+/*
+ * Synchronized LRU cache
+ */
+
+using namespace std;
+
+namespace cache {
+
+
+    template<typename key_t, typename value_t>
+    class lru_cache {
+
+        std::recursive_mutex m;
+
+    public:
+        typedef typename std::pair<key_t, value_t> key_value_pair_t;
+        typedef typename std::list<key_value_pair_t>::iterator list_iterator_t;
+
+        lru_cache(size_t max_size) :
+                _max_size(max_size) {
+        }
+
+
+
+
+        void putIfDoesNotExist(const key_t& key, const value_t& value) {
+            lock_guard<recursive_mutex> l(m);
+            if (!exists(key))
+                put(key, value);
+        }
+
+        void put(const key_t& key, const value_t& value) {
+
+            lock_guard<recursive_mutex> l(m);
+
+            auto it = _cache_items_map.find(key);
+            _cache_items_list.push_front(key_value_pair_t(key, value));
+            if (it != _cache_items_map.end()) {
+                _cache_items_list.erase(it->second);
+                _cache_items_map.erase(it);
+            }
+            _cache_items_map[key] = _cache_items_list.begin();
+
+            if (_cache_items_map.size() > _max_size) {
+                auto last = _cache_items_list.end();
+                last--;
+                _cache_items_map.erase(last->first);
+                _cache_items_list.pop_back();
+            }
+        }
+
+
+
+
+        const value_t& get(const key_t& key) {
+
+            lock_guard<recursive_mutex> l(m);
+
+            auto it = _cache_items_map.find(key);
+            if (it == _cache_items_map.end()) {
+                throw std::range_error("There is no such key in cache");
+            } else {
+                _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+                return it->second->second;
+            }
+        }
+
+
+
+
+
+        bool exists(const key_t& key)  {
+
+            lock_guard<recursive_mutex> l(m);
+
+            return _cache_items_map.find(key) != _cache_items_map.end();
+        }
+
+        size_t size() const {
+            lock_guard<recursive_mutex> l(m);
+            return _cache_items_map.size();
+        }
+
+    private:
+        std::list<key_value_pair_t> _cache_items_list;
+        std::unordered_map<key_t, list_iterator_t> _cache_items_map;
+        size_t _max_size;
+    };
+
+} // namespace cache
+
+#endif	/* _LRUCACHE_HPP_INCLUDED_ */