Merge pull request #265 from skalenetwork/SKALE-3751-enable-zeromq

Skale 3751 enable zeromq

BLSCrypto.cpp

@@ -78,6 +78,7 @@ int char2int(char _input) {
 vector<char> carray2Hex(const unsigned char *d, uint64_t _len) {
 
     CHECK_STATE(d);
+
     vector<char> _hexArray( 2 * _len + 1);
 
     char hexval[16] = {'0', '1', '2', '3', '4', '5', '6', '7',

BLSSignReqMessage.cpp

@@ -7,9 +7,11 @@
 
 
 Json::Value BLSSignReqMessage::process() {
-    auto keyName = getStringRapid("kn");
-    auto hash = getStringRapid("mh");
+    auto keyName = getStringRapid("keyShareName");
+    auto hash = getStringRapid("messageHash");
     auto t = getUint64Rapid("t");
     auto n = getUint64Rapid("n");
-    return SGXWalletServer::blsSignMessageHashImpl(keyName, hash, t, n);
+    auto result =  SGXWalletServer::blsSignMessageHashImpl(keyName, hash, t, n);
+    result["type"] = ZMQMessage::BLS_SIGN_RSP;
+    return result;
 }
\ No newline at end of file

BLSSignRspMessage.cpp

+/*
+  Copyright (C) 2018-2019 SKALE Labs
+
+  This file is part of libBLS.
+
+  libBLS is free software: you can redistribute it and/or modify
+  it under the terms of the GNU Affero General Public License as published
+  by the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  libBLS is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU Affero General Public License for more details.
+
+  You should have received a copy of the GNU Affero General Public License
+  along with libBLS.  If not, see <https://www.gnu.org/licenses/>.
+
+  @file BLSRspSignMessage.h
+  @author Stan Kladko
+  @date 2020
+*/
+
+#include "BLSSignRspMessage.h"
+#include "SGXWalletServer.hpp"
+
+
+Json::Value BLSSignRspMessage::process() {
+    assert(false);
+}
\ No newline at end of file

BLSSignRspMessage.h

+/*
+  Copyright (C) 2018-2019 SKALE Labs
+
+  This file is part of libBLS.
+
+  libBLS is free software: you can redistribute it and/or modify
+  it under the terms of the GNU Affero General Public License as published
+  by the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  libBLS is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU Affero General Public License for more details.
+
+  You should have received a copy of the GNU Affero General Public License
+  along with libBLS.  If not, see <https://www.gnu.org/licenses/>.
+
+  @file BLSRspSignMessage.h
+  @author Stan Kladko
+  @date 2020
+*/
+
+#ifndef SGXWALLET_BLSSIGNRSPMSG_H
+#define SGXWALLET_BLSSIGNRSPMSG_H
+
+#include "ZMQMessage.h"
+
+class BLSSignRspMessage : public ZMQMessage {
+public:
+
+    BLSSignRspMessage(shared_ptr<rapidjson::Document>& _d) : ZMQMessage(_d) {};
+
+    virtual Json::Value process();
+
+    string getSigShare() {
+        return getStringRapid("signatureShare");
+    }
+
+};
+
+
+#endif //SGXWALLET_BLSSIGNRSPMSG_H

CMakeLists.txt

@@ -82,4 +82,4 @@ add_executable(sgxwallet
         SGXWalletServer.hpp
         stubclient.cpp
         stubclient.h
-        testw.cpp)
+        testw.cpp ZMQClient.cpp ZMQClient.h)

CSRManagerServer.cpp

@@ -115,6 +115,8 @@ int CSRManagerServer::initCSRManagerServer() {
     hs3->BindLocalhost();
     cs = make_shared<CSRManagerServer>(*hs3, JSONRPC_SERVER_V2); // server (json-rpc 2.0)
 
+    spdlog::info("Starting csr manager server on port {} ...", BASE_PORT + 2);
+
     if (!cs->StartListening()) {
         spdlog::info("CSR manager server could not start listening");
         exit(-1);

ECDSASignReqMessage.cpp

-//
-// Created by kladko on 15.12.20.
-//
+/*
+  Copyright (C) 2018-2019 SKALE Labs
+
+  This file is part of libBLS.
+
+  libBLS is free software: you can redistribute it and/or modify
+  it under the terms of the GNU Affero General Public License as published
+  by the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  libBLS is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU Affero General Public License for more details.
+
+  You should have received a copy of the GNU Affero General Public License
+  along with libBLS.  If not, see <https://www.gnu.org/licenses/>.
+
+  @file ECDSASignReqMessage.cpp
+  @author Stan Kladko
+  @date 2020
+*/
 
 
 #include "SGXWalletServer.hpp"
@@ -10,8 +29,10 @@
 
 
 Json::Value ECDSASignReqMessage::process() {
-    auto base = getUint64Rapid("bs");
-    auto keyName = getStringRapid("kn");
-    auto hash = getStringRapid("mh");
-    return SGXWalletServer::ecdsaSignMessageHashImpl(base, keyName, hash);
+    auto base = getUint64Rapid("base");
+    auto keyName = getStringRapid("keyName");
+    auto hash = getStringRapid("messageHash");
+    auto result =  SGXWalletServer::ecdsaSignMessageHashImpl(base, keyName, hash);
+    result["type"] = ZMQMessage::ECDSA_SIGN_RSP;
+    return result;
 }
\ No newline at end of file

ECDSASignReqMessage.h

 /*
-  Copyright (C) 2018-2019 SKALE Labs
+  Copyright (C) 2018- SKALE Labs
 
   This file is part of libBLS.
 
@@ -33,6 +33,7 @@ public:
 
     virtual Json::Value process();
 
+
 };
 
 

ECDSASignRspMessage.cpp

+/*
+  Copyright (C) 2018- SKALE Labs
+
+  This file is part of libBLS.
+
+  libBLS is free software: you can redistribute it and/or modify
+  it under the terms of the GNU Affero General Public License as published
+  by the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  libBLS is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU Affero General Public License for more details.
+
+  You should have received a copy of the GNU Affero General Public License
+  along with libBLS.  If not, see <https://www.gnu.org/licenses/>.
+
+  @file ECDSARspSignMessage.cpp
+  @author Stan Kladko
+  @date 2020
+*/
+
+#include "SGXWalletServer.hpp"
+
+#include "ECDSASignRspMessage.h"
+
+
+
+Json::Value ECDSASignRspMessage::process() {
+    // never called
+    assert(false);
+}
+
+string ECDSASignRspMessage::getSignature() {
+
+
+
+    string r = getStringRapid( "signature_r" );
+    string v = getStringRapid( "signature_v" );
+    string s = getStringRapid("signature_s" );
+
+    auto ret = v + ":" + r.substr( 2 ) + ":" + s.substr( 2 );
+
+    return ret;
+}

ECDSASignRspMessage.h

+/*
+  Copyright (C) 2018- SKALE Labs
+
+  This file is part of libBLS.
+
+  libBLS is free software: you can redistribute it and/or modify
+  it under the terms of the GNU Affero General Public License as published
+  by the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  libBLS is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU Affero General Public License for more details.
+
+  You should have received a copy of the GNU Affero General Public License
+  along with libBLS.  If not, see <https://www.gnu.org/licenses/>.
+
+  @file ECDSARspSignMessage.h
+  @author Stan Kladko
+  @date 2020
+*/
+
+#ifndef SGXWALLET_ECDSASIGNRSPMESSAGE_H
+#define SGXWALLET_ECDSASIGNRSPMESSAGE_H
+
+#include "ZMQMessage.h"
+
+class ECDSASignRspMessage : public ZMQMessage {
+public:
+
+    ECDSASignRspMessage(shared_ptr <rapidjson::Document> &_d) : ZMQMessage(_d) {};
+
+    virtual Json::Value process();
+
+    string getSignature();
+
+
+};
+
+
+#endif //SGXWALLET_ECDSASIGNRSPMESSAGE_H

Makefile.am

@@ -70,11 +70,11 @@ bin_PROGRAMS = sgxwallet testw sgx_util
 ## have to be explicitly listed
 
 
-COMMON_SRC = ECDSASignReqMessage.cpp BLSSignReqMessage.cpp ZMQMessage.cpp ZMQServer.cpp ServerWorker.cpp InvalidStateException.cpp  Exception.cpp InvalidArgumentException.cpp Log.cpp  \
+COMMON_SRC = SGXException.cpp ZMQClient.cpp BLSSignRspMessage.cpp ECDSASignRspMessage.cpp ECDSASignReqMessage.cpp BLSSignReqMessage.cpp ZMQMessage.cpp ZMQServer.cpp ServerWorker.cpp InvalidStateException.cpp  Exception.cpp InvalidArgumentException.cpp Log.cpp  \
              SGXWalletServer.cpp SGXRegistrationServer.cpp CSRManagerServer.cpp BLSCrypto.cpp \
              DKGCrypto.cpp ServerInit.cpp BLSPrivateKeyShareSGX.cpp LevelDB.cpp ServerDataChecker.cpp SEKManager.cpp \
              third_party/intel/sgx_stub.c third_party/intel/sgx_detect_linux.c third_party/intel/create_enclave.c third_party/intel/oc_alloc.c \
-             ECDSAImpl.c TestUtils.cpp sgxwallet.c SGXInfoServer.cpp ECDSACrypto.cpp 
+             ECDSAImpl.c TestUtils.cpp sgxwallet.c SGXInfoServer.cpp ECDSACrypto.cpp
 COMMON_ENCLAVE_SRC = secure_enclave_u.c secure_enclave_u.h
 
 sgxwallet_SOURCES = sgxwall.cpp $(COMMON_SRC)
@@ -116,7 +116,7 @@ nodist_testw_SOURCES=${nodist_sgxwallet_SOURCES}
 EXTRA_testw_DEPENDENCIES=${EXTRA_sgxwallet_DEPENDENCIES}
 testw_LDADD= ${sgxwallet_LDADD}
 
-sgx_util_SOURCES= InvalidStateException.cpp  Exception.cpp InvalidArgumentException.cpp Log.cpp sgx_util.cpp stubclient.cpp LevelDB.cpp SGXRegistrationServer.cpp CSRManagerServer.cpp
+sgx_util_SOURCES= SGXException.cpp InvalidStateException.cpp  Exception.cpp InvalidArgumentException.cpp Log.cpp sgx_util.cpp stubclient.cpp LevelDB.cpp SGXRegistrationServer.cpp CSRManagerServer.cpp
 
 sgx_util_LDADD=-LlibBLS/deps/deps_inst/x86_or_x64/lib -Lleveldb/build -LlibBLS/build \
                    -LlibBLS/build/libff/libff \

SGXException.cpp

+/*
+    Copyright (C) 2021-Present SKALE Labs
+
+    This file is part of sgxwallet.
+
+    sgxwallet is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    sgxwallet is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with sgxwallet.  If not, see <https://www.gnu.org/licenses/>.
+
+    @file SGXException.cpp
+    @author Stan Kladko
+    @date 2021
+*/
+
+#include "SGXException.h"
+
+const char* SGXException::what()  const noexcept {
+    return errString.c_str();
+}
+
+

SGXException.h

@@ -16,13 +16,13 @@
     You should have received a copy of the GNU Affero General Public License
     along with sgxwallet.  If not, see <https://www.gnu.org/licenses/>.
 
-    @file RPCException.h
+    @file SGXException.h
     @author Stan Kladko
     @date 2019
 */
 
-#ifndef SGXD_RPCEXCEPTION_H
-#define SGXD_RPCEXCEPTION_H
+#ifndef SGXD_SGXEXCEPTION_H
+#define SGXD_SGXEXCEPTION_H
 
 #include <string>
 #include <exception>
@@ -46,11 +46,11 @@ public:
         return errString;
     }
 
+    const char* what() const noexcept override;
+
     const int32_t getStatus() const {
         return status;
     }
-
-
 };
 
-#endif //SGXD_RPCEXCEPTION_H
+#endif //SGXD_SGXEXCEPTION_H

SGXInfoServer.cpp

@@ -111,6 +111,8 @@ int SGXInfoServer::initInfoServer(uint32_t _logLevel, bool _autoSign, bool _chec
     httpServer = make_shared<HttpServer>(BASE_PORT + 4);
     server = make_shared<SGXInfoServer>(*httpServer, JSONRPC_SERVER_V2, _logLevel, _autoSign, _checkCerts, _generateTestKeys); // hybrid server (json-rpc 1.0 & 2.0)
 
+    spdlog::info("Starting info server on port {} ...", BASE_PORT + 4);
+
     if (!server->StartListening()) {
         spdlog::error("Info server could not start listening on port {}", BASE_PORT + 4);
         exit(-10);

SGXRegistrationServer.cpp

@@ -168,6 +168,8 @@ int SGXRegistrationServer::initRegistrationServer(bool _autoSign) {
                                                 JSONRPC_SERVER_V2,
                                                 _autoSign); // hybrid server (json-rpc 1.0 & 2.0)
 
+    spdlog::info("Starting registration server on port {} ...", BASE_PORT + 1);
+
     if (!server->StartListening()) {
         spdlog::error("Registration server could not start listening on port {}", BASE_PORT + 1);
         exit(-10);

SGXWalletServer.cpp

@@ -118,12 +118,14 @@ void SGXWalletServer::printDB() {
 #define NUM_THREADS 200
 #endif
 
+bool SGXWalletServer::verifyCert(string &_certFileName) {
+    string rootCAPath = string(SGXDATA_FOLDER) + "cert_data/rootCA.pem";
+    string verifyCert = "cert/verify_client_cert " + rootCAPath + " " + _certFileName;
+    return system(verifyCert.c_str()) == 0;
+}
 
-int SGXWalletServer::initHttpsServer(bool _checkCerts) {
-    COUNT_STATISTICS
-    spdlog::info("Entering {}", __FUNCTION__);
 
-    spdlog::info("Initing server, number of threads: {}", NUM_THREADS);
+void SGXWalletServer::createCertsIfNeeded() {
 
     string rootCAPath = string(SGXDATA_FOLDER) + "cert_data/rootCA.pem";
     string keyCAPath = string(SGXDATA_FOLDER) + "cert_data/rootCA.key";
@@ -158,12 +160,39 @@ int SGXWalletServer::initHttpsServer(bool _checkCerts) {
         }
     }
 
+    spdlog::info("Verifying  server cert");
+
+    if (verifyCert(certPath)) {
+        spdlog::info("SERVER CERTIFICATE IS SUCCESSFULLY VERIFIED");
+    } else {
+        spdlog::info("SERVER CERTIFICATE VERIFICATION FAILED");
+        exit(-12);
+    }
+}
+
+
+int SGXWalletServer::initHttpsServer(bool _checkCerts) {
+    COUNT_STATISTICS
+    spdlog::info("Entering {}", __FUNCTION__);
+    spdlog::info("Initing server, number of threads: {}", NUM_THREADS);
+
+
+
+
+
+    string certPath = string(SGXDATA_FOLDER) + "cert_data/SGXServerCert.crt";
+    string keyPath = string(SGXDATA_FOLDER) + "cert_data/SGXServerCert.key";
+    string rootCAPath = string(SGXDATA_FOLDER) + "cert_data/rootCA.pem";
+    string keyCAPath = string(SGXDATA_FOLDER) + "cert_data/rootCA.key";
+
     httpServer = make_shared<HttpServer>(BASE_PORT, certPath, keyPath, rootCAPath, _checkCerts,
                                          NUM_THREADS);
 
     server = make_shared<SGXWalletServer>(*httpServer,
                                           JSONRPC_SERVER_V2); // hybrid server (json-rpc 1.0 & 2.0)
 
+    spdlog::info("Starting sgx server on port {} ...", BASE_PORT);
+
     if (!server->StartListening()) {
         spdlog::error("SGX Server could not start listening");
         exit(-13);
@@ -176,6 +205,9 @@ int SGXWalletServer::initHttpsServer(bool _checkCerts) {
 int SGXWalletServer::initHttpServer() { //without ssl
     COUNT_STATISTICS
     spdlog::info("Entering {}", __FUNCTION__);
+
+    spdlog::info("Starting sgx http server on port {} ...", BASE_PORT + 3);
+
     httpServer = make_shared<HttpServer>(BASE_PORT + 3, "", "", "", false,
                                          NUM_THREADS);
     server = make_shared<SGXWalletServer>(*httpServer,
@@ -208,18 +240,19 @@ SGXWalletServer::importBLSKeyShareImpl(const string &_keyShare, const string &_k
         }
 
         if (!checkHex(hashTmp)) {
-            throw SGXException(BLS_IMPORT_INVALID_KEY_SHARE, string(__FUNCTION__) + ":Invalid BLS key share, please use hex");
+            throw SGXException(BLS_IMPORT_INVALID_KEY_SHARE,
+                               string(__FUNCTION__) + ":Invalid BLS key share, please use hex");
         }
 
         encryptedKeyShareHex = encryptBLSKeyShare2Hex(&errStatus, (char *) errMsg.data(), hashTmp.c_str());
 
         if (errStatus != 0) {
-            throw SGXException(errStatus, string(__FUNCTION__) + ":" +errMsg.data());
+            throw SGXException(errStatus, string(__FUNCTION__) + ":" + errMsg.data());
         }
 
         if (encryptedKeyShareHex.empty()) {
-            throw SGXException(BLS_IMPORT_EMPTY_ENCRYPTED_KEY_SHARE,string(__FUNCTION__) +
-                    ":Empty encrypted key share");
+            throw SGXException(BLS_IMPORT_EMPTY_ENCRYPTED_KEY_SHARE, string(__FUNCTION__) +
+                                                                     ":Empty encrypted key share");
         }
 
         result["encryptedKeyShare"] = encryptedKeyShareHex;
@@ -304,7 +337,6 @@ SGXWalletServer::blsSignMessageHashImpl(const string &_keyShareName, const strin
     result["signatureShare"] = string(signature.data());
 
 
-
     RETURN_SUCCESS(result);
 
 }
@@ -430,7 +462,7 @@ Json::Value SGXWalletServer::getPublicECDSAKeyImpl(const string &_keyName) {
     try {
         if (!checkECDSAKeyName(_keyName)) {
             throw SGXException(INVALID_ECDSA_GETPKEY_KEY_NAME, string(__FUNCTION__) +
-            ":Invalid ECDSA import key name");
+                                                               ":Invalid ECDSA import key name");
         }
         shared_ptr <string> keyStr = readFromDb(_keyName);
         publicKey = getECDSAPubKey(keyStr->c_str());
@@ -809,7 +841,7 @@ Json::Value SGXWalletServer::deleteBlsKeyImpl(const string &name) {
             result["deleted"] = true;
         } else {
             auto error_msg = "BLS key not found: " + name;
-            throw SGXException(DELETE_BLS_KEY_NOT_FOUND, string(__FUNCTION__)+ ":" + error_msg.c_str());
+            throw SGXException(DELETE_BLS_KEY_NOT_FOUND, string(__FUNCTION__) + ":" + error_msg.c_str());
         }
     } HANDLE_SGX_EXCEPTION(result)
 
@@ -898,8 +930,9 @@ Json::Value SGXWalletServer::dkgVerificationV2Impl(const string &_publicShares,
 }
 
 Json::Value
-SGXWalletServer::createBLSPrivateKeyV2Impl(const string &_blsKeyName, const string &_ethKeyName, const string &_polyName,
-                                         const string &_secretShare, int _t, int _n) {
+SGXWalletServer::createBLSPrivateKeyV2Impl(const string &_blsKeyName, const string &_ethKeyName,
+                                           const string &_polyName,
+                                           const string &_secretShare, int _t, int _n) {
     COUNT_STATISTICS
     spdlog::info("Entering {}", __FUNCTION__);
     INIT_RESULT(result)
@@ -1053,7 +1086,7 @@ SGXWalletServer::dkgVerificationV2(const string &_publicShares, const string &et
 
 Json::Value
 SGXWalletServer::createBLSPrivateKeyV2(const string &blsKeyName, const string &ethKeyName, const string &polyName,
-                                     const string &SecretShare, int t, int n) {
+                                       const string &SecretShare, int t, int n) {
     return createBLSPrivateKeyV2Impl(blsKeyName, ethKeyName, polyName, SecretShare, t, n);
 }
 
@@ -1061,6 +1094,7 @@ shared_ptr <string> SGXWalletServer::readFromDb(const string &name, const string
     auto dataStr = checkDataFromDb(prefix + name);
 
     if (dataStr == nullptr) {
+
         throw SGXException(KEY_SHARE_DOES_NOT_EXIST, string(__FUNCTION__) +  ":Data with this name does not exist: "
                                                                                                     + prefix + name);
     }

SGXWalletServer.hpp

@@ -55,7 +55,11 @@ class SGXWalletServer : public AbstractStubServer {
     static void checkForDuplicate(map <string, string> &_map, recursive_mutex &_m, const string &_key,
     const string &_value);
 
+
 public:
+
+    static bool verifyCert(string& _certFileName);
+
     static const char* getVersion() {
         return TOSTRING(SGXWALLET_VERSION);
     }
@@ -177,6 +181,8 @@ public:
     static int initHttpServer();
 
     static int initHttpsServer(bool _checkCerts);
+
+    static void createCertsIfNeeded();
 };
 
 #endif //SGXWALLET_SGXWALLETSERVER_HPP

ServerInit.cpp

@@ -42,7 +42,6 @@
 #include <unistd.h>
 
 
-
 #include "BLSPrivateKeyShareSGX.h"
 #include "sgxwallet_common.h"
 #include "third_party/intel/create_enclave.h"
@@ -68,16 +67,16 @@ using namespace std;
 void systemHealthCheck() {
     string ulimit;
     try {
-        ulimit = exec( "/bin/bash -c \"ulimit -n\"" );
-    } catch ( ... ) {
+        ulimit = exec("/bin/bash -c \"ulimit -n\"");
+    } catch (...) {
         spdlog::error("Execution of '/bin/bash -c ulimit -n' failed");
         exit(-15);
     }
-    int noFiles = strtol( ulimit.c_str(), NULL, 10 );
+    int noFiles = strtol(ulimit.c_str(), NULL, 10);
 
-    auto noUlimitCheck = getenv( "NO_ULIMIT_CHECK" ) != nullptr;
+    auto noUlimitCheck = getenv("NO_ULIMIT_CHECK") != nullptr;
 
-    if ( noFiles < 65535 && !noUlimitCheck) {
+    if (noFiles < 65535 && !noUlimitCheck) {
         string errStr =
                 "sgxwallet requires setting Linux file descriptor limit to at least 65535 "
                 "You current limit (ulimit -n) is less than 65535. \n Please set it to 65535:"
@@ -89,8 +88,8 @@ void systemHealthCheck() {
     }
 }
 
-static ZMQServer* zmqServer = nullptr;
-atomic<bool> exiting(false);
+
+
 
 void initUserSpace() {
 
@@ -104,27 +103,11 @@ void initUserSpace() {
     systemHealthCheck();
 #endif
 
-#ifdef EXPERIMENTAL_ZMQ_SERVER
-    zmqServer = new ZMQServer();
-    static std::thread serverThread(std::bind(&ZMQServer::run, zmqServer));
-#endif
-}
-
-void exitZMQServer() {
-#ifdef EXPERIMENTAL_ZMQ_SERVER
-
-    auto doExit = !exiting.exchange(true);
 
-    if (doExit) {
-        spdlog::info("Exiting zmq server ...");
-        delete zmqServer;
-        spdlog::info("Exited zmq server ...");
-        zmqServer = nullptr;
-    }
 
-#endif
 }
 
+
 uint64_t initEnclave() {
 
 
@@ -168,7 +151,7 @@ uint64_t initEnclave() {
         }
 
         spdlog::info("Enclave created and started successfully");
-        
+
         status = trustedEnclaveInit(eid, enclaveLogLevel);
     }
 
@@ -183,8 +166,8 @@ uint64_t initEnclave() {
 }
 
 
-
-void initAll(uint32_t _logLevel, bool _checkCert, bool _autoSign, bool _generateTestKeys) {
+void initAll(uint32_t _logLevel, bool _checkCert,
+             bool _checkZMQSig, bool _autoSign, bool _generateTestKeys) {
 
 
     static atomic<bool> sgxServerInited(false);
@@ -206,9 +189,9 @@ void initAll(uint32_t _logLevel, bool _checkCert, bool _autoSign, bool _generate
         uint64_t counter = 0;
 
         uint64_t initResult = 0;
-        while ((initResult = initEnclave()) != 0 && counter < 10){
+        while ((initResult = initEnclave()) != 0 && counter < 10) {
             sleep(1);
-            counter ++;
+            counter++;
         }
 
         if (initResult != 0) {
@@ -218,14 +201,23 @@ void initAll(uint32_t _logLevel, bool _checkCert, bool _autoSign, bool _generate
         initUserSpace();
         initSEK();
 
+        SGXWalletServer::createCertsIfNeeded();
+
         if (useHTTPS) {
+            spdlog::info("Initing JSON-RPC server over HTTPS");
+            spdlog::info("Check client cert: {}", _checkCert);
             SGXWalletServer::initHttpsServer(_checkCert);
-            SGXRegistrationServer::initRegistrationServer(_autoSign);
-            CSRManagerServer::initCSRManagerServer();
+            spdlog::info("Inited JSON-RPC server over HTTPS");
         } else {
+            spdlog::info("Initing JSON-RPC server over HTTP");
             SGXWalletServer::initHttpServer();
+            spdlog::info("Inited JSON-RPC server over HTTP");
         }
+
+        SGXRegistrationServer::initRegistrationServer(_autoSign);
+        CSRManagerServer::initCSRManagerServer();
         SGXInfoServer::initInfoServer(_logLevel, _checkCert, _autoSign, _generateTestKeys);
+        ZMQServer::initZMQServer(_checkZMQSig);
 
         sgxServerInited = true;
     } catch (SGXException &_e) {

ServerInit.h

@@ -32,7 +32,7 @@
 #define EXTERNC
 #endif
 
-EXTERNC void initAll(uint32_t  _logLevel, bool _checkCert, bool _autoSign, bool _generateTestKeys);
+EXTERNC void initAll(uint32_t  _logLevel, bool _checkCert, bool _checkZMQSig, bool _autoSign, bool _generateTestKeys);
 
 EXTERNC void initUserSpace();
 

ServerWorker.cpp

-//
-// Created by kladko on 14.12.20.
-//
+/*
+    Copyright (C) 2019-Present SKALE Labs
+
+    This file is part of sgxwallet.
+
+    sgxwallet is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    sgxwallet is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with sgxwallet.  If not, see <https://www.gnu.org/licenses/>.
+
+    @file ServerWorker.cpp
+    @author Stan Kladko
+    @date 2021
+*/
+
+
 #include "common.h"
+#include "sgxwallet_common.h"
 #include <json/writer.h>
+
+
+#include <zmq.hpp>
+#include "zhelpers.hpp"
+
+#include "Log.h"
 #include "ZMQMessage.h"
+
 #include "ServerWorker.h"
 
+std::atomic <uint64_t>  ServerWorker::workerCount(1);
 
-ServerWorker::ServerWorker(zmq::context_t &ctx, int sock_type) : ctx_(ctx),
-                                                                 worker_(ctx_, sock_type) {};
+ServerWorker::ServerWorker(zmq::context_t &_ctx, int sock_type, bool _checkSignature,
+                           const string& _caCert ) :            checkSignature(_checkSignature),
+                           caCert(_caCert),
+                                                                 isExitRequested(false) {
+
+    worker = make_shared<zmq::socket_t>(_ctx, sock_type);
+
+    if (checkSignature) {
+        CHECK_STATE(!caCert.empty())
+    }
+
+    index = workerCount.fetch_add(1);
+    int linger = 0;
+    zmq_setsockopt(*worker, ZMQ_LINGER, &linger, sizeof(linger));
+};
+
+
+
+void ServerWorker::doOneServerLoop() noexcept {
+    string replyStr;
+
+    Json::Value result;
+    result["status"] = ZMQ_SERVER_ERROR;
+    result["errorMessage"] = "";
+
+    zmq::message_t identity;
+    zmq::message_t identit2;
+    zmq::message_t copied_id;
 
-void ServerWorker::work() {
-    worker_.connect("inproc://backend");
     try {
-        while (true) {
-            zmq::message_t msg;
-            zmq::message_t copied_msg;
-            worker_.recv(&msg);
 
-            vector<uint8_t> msgData(msg.size() + 1, 0);
 
-            memcpy(msgData.data(), msg.data(), msg.size());
+        zmq_pollitem_t items[1];
+        items[0].socket = *worker;
+        items[0].events = ZMQ_POLLIN;
 
-            auto parsedMsg = ZMQMessage::parse(msgData);
+        int pollResult = 0;
 
-            CHECK_STATE(parsedMsg);
+        do {
+            pollResult = zmq_poll(items, 1, 1000);
+            if (isExitRequested) {
+                return;
+            }
+        } while (pollResult == 0);
 
-            auto reply  = parsedMsg->process();
 
-            Json::FastWriter fastWriter;
-            std::string replyStr = fastWriter.write(reply);
+        zmq::message_t msg;
+        zmq::message_t copied_msg;
+        worker->recv(&identity);
+        copied_id.copy(&identity);
+        worker->recv(&msg);
 
-            zmq::message_t replyMsg(replyStr.c_str(),replyStr.size() + 1);
-            worker_.send(replyMsg);
-        }
+        int64_t more;
+        size_t more_size = sizeof(more);
+        auto rc = zmq_getsockopt(*worker, ZMQ_RCVMORE, &more, &more_size);
+
+        CHECK_STATE2(rc == 0, ZMQ_COULD_NOT_GET_SOCKOPT);
+
+        vector <uint8_t> msgData(msg.size() + 1, 0);
+
+        memcpy(msgData.data(), msg.data(), msg.size());
+
+        CHECK_STATE2(msg.size() > 5 || msgData.at(0) == '{' || msgData[msg.size()] == '}',
+        ZMQ_INVALID_MESSAGE);
+
+        memcpy(msgData.data(), msg.data(), msg.size());
+
+        auto parsedMsg = ZMQMessage::parse(
+                (const char *) msgData.data(), msg.size(), true, checkSignature);
+
+        CHECK_STATE2(parsedMsg, ZMQ_COULD_NOT_PARSE);
+
+        result = parsedMsg->process();
+
+    } catch (SGXException &e) {
+        result["status"] = e.getStatus();
+        result["errorMessage"] = e.getMessage();
+        spdlog::error("Exception in zmq server worker:{}", e.what());
     }
     catch (std::exception &e) {
-        spdlog::info("Exiting zmq server worker:{}", e.what());
-        return;
+        if (isExitRequested) {
+            return;
+        }
+        result["errorMessage"] = string(e.what());
+        spdlog::error("Exception in zmq server worker:{}", e.what());
     } catch (...) {
+        if (isExitRequested) {
+            return;
+        }
         spdlog::error("Error in zmq server worker");
-        return;
+        result["errorMessage"] = "Error in zmq server worker";
+    }
+
+    try {
+
+        Json::FastWriter fastWriter;
+
+        replyStr = fastWriter.write(result);
+        replyStr = replyStr.substr(0, replyStr.size() - 1);
+
+        CHECK_STATE(replyStr.size() > 2);
+        CHECK_STATE(replyStr.front() == '{');
+        CHECK_STATE(replyStr.back() == '}');
+        zmq::message_t replyMsg(replyStr.c_str(), replyStr.size() + 1);
+
+        worker->send(copied_id, ZMQ_SNDMORE);
+        worker->send(replyMsg);
+
+    } catch (std::exception &e) {
+        if (isExitRequested) {
+            return;
+        }
+        spdlog::error("Exception in zmq server worker send :{}", e.what());
+    } catch (...) {
+        if (isExitRequested) {
+            return;
+        }
+        spdlog::error("Unklnown exception in zmq server worker send");
     }
+}
+
+void ServerWorker::work() {
+    worker->connect("inproc://backend");
 
+
+    while (!isExitRequested) {
+        try {
+            doOneServerLoop();
+        } catch (...) {
+            spdlog::error("doOneServerLoop threw exception. This should never happen!");
+        }
+    }
+
+    spdlog::info("Exited worker thread {}", index);
 }
 
 
+void ServerWorker::requestExit() {
+    isExitRequested.exchange(true);
+    spdlog::info("Closed worker socket {}", index);
+}
+

ServerWorker.h

-//
-// Created by kladko on 14.12.20.
-//
+/*
+    Copyright (C) 2019-Present SKALE Labs
 
+    This file is part of sgxwallet.
+
+    sgxwallet is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    sgxwallet is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with sgxwallet.  If not, see <https://www.gnu.org/licenses/>.
+
+    @file ServerWorker.h
+    @author Stan Kladko
+    @date 2021
+*/
 #ifndef SGXWALLET_SERVERWORKER_H
 #define SGXWALLET_SERVERWORKER_H
 
@@ -19,16 +37,26 @@
 
 class ServerWorker {
 
+    bool checkSignature = true;
+    string caCert = "";
+
 public:
-    ServerWorker(zmq::context_t &ctx, int sock_type );
+    ServerWorker(zmq::context_t &ctx, int sock_type, bool _checkSignature, const string& _caCert );
 
 
     void work();
 
+    void requestExit();
 
 private:
-    zmq::context_t &ctx_;
-    zmq::socket_t worker_;
+    shared_ptr<zmq::socket_t> worker;
+
+    std::atomic<bool> isExitRequested;
+
+    void doOneServerLoop() noexcept;
+
+    static std::atomic<uint64_t> workerCount;
+    uint64_t index;
 };
 
 

TestUtils.cpp

@@ -42,6 +42,7 @@
 #include "SGXWalletServer.hpp"
 
 #include "catch.hpp"
+#include "ZMQClient.h"
 #include "BLSSigShare.h"
 #include "BLSSigShareSet.h"
 #include "BLSPublicKeyShare.h"
@@ -398,6 +399,96 @@ void TestUtils::sendRPCRequestV2() {
     sigShareSet.merge();
 }
 
+
+void TestUtils::sendRPCRequestZMQ() {
+    HttpClient client(RPC_ENDPOINT);
+    StubClient c(client, JSONRPC_CLIENT_V2);
+
+
+
+
+    int n = 16, t = 16;
+    Json::Value ethKeys[n];
+    Json::Value verifVects[n];
+    Json::Value pubEthKeys;
+    Json::Value secretShares[n];
+    Json::Value pubBLSKeys[n];
+    Json::Value blsSigShares[n];
+    vector <string> pubShares(n);
+    vector <string> polyNames(n);
+
+    static atomic<int> counter(1);
+
+    int schainID = counter.fetch_add(1);
+    int dkgID = counter.fetch_add(1);
+    for (uint8_t i = 0; i < n; i++) {
+        ethKeys[i] = c.generateECDSAKey();
+        CHECK_STATE(ethKeys[i]["status"] == 0);
+        string polyName =
+                "POLY:SCHAIN_ID:" + to_string(schainID) + ":NODE_ID:" + to_string(i) + ":DKG_ID:" + to_string(dkgID);
+        auto response = c.generateDKGPoly(polyName, t);
+        CHECK_STATE(response["status"] == 0);
+        polyNames[i] = polyName;
+        verifVects[i] = c.getVerificationVector(polyName, t, n);
+        CHECK_STATE(verifVects[i]["status"] == 0);
+
+        pubEthKeys.append(ethKeys[i]["publicKey"]);
+    }
+
+    for (uint8_t i = 0; i < n; i++) {
+        secretShares[i] = c.getSecretShareV2(polyNames[i], pubEthKeys, t, n);
+        for (uint8_t k = 0; k < t; k++) {
+            for (uint8_t j = 0; j < 4; j++) {
+                string pubShare = verifVects[i]["verificationVector"][k][j].asString();
+                pubShares[i] += convertDecToHex(pubShare);
+            }
+        }
+    }
+
+    vector <string> secShares(n);
+
+    for (int i = 0; i < n; i++)
+        for (int j = 0; j < n; j++) {
+            string secretShare = secretShares[i]["secretShare"].asString().substr(192 * j, 192);
+            secShares[i] += secretShares[j]["secretShare"].asString().substr(192 * i, 192);
+            Json::Value verif = c.dkgVerificationV2(pubShares[i], ethKeys[j]["keyName"].asString(), secretShare, t, n, j);
+            CHECK_STATE(verif["status"] == 0);
+        }
+
+    BLSSigShareSet sigShareSet(t, n);
+
+    string hash = SAMPLE_HASH;
+
+    auto hash_arr = make_shared < array < uint8_t, 32 >> ();
+    uint64_t binLen;
+    if (!hex2carray(hash.c_str(), &binLen, hash_arr->data(), 32)) {
+        throw SGXException(TEST_INVALID_HEX, "Invalid hash");
+    }
+
+    map <size_t, shared_ptr<BLSPublicKeyShare>> coeffs_pkeys_map;
+
+    Json::Value publicShares;
+    for (int i = 0; i < n; ++i) {
+        publicShares["publicShares"][i] = pubShares[i];
+    }
+
+    Json::Value blsPublicKeys = c.calculateAllBLSPublicKeys(publicShares, t, n);
+    CHECK_STATE(blsPublicKeys["status"] == 0);
+
+    for (int i = 0; i < t; i++) {
+        string blsName = "BLS_KEY" + polyNames[i].substr(4);
+        string hash = SAMPLE_HASH;
+        blsSigShares[i] = c.blsSignMessageHash(blsName, hash, t, n);
+        CHECK_STATE(blsSigShares[i]["status"] == 0);
+
+        shared_ptr <string> sig_share_ptr = make_shared<string>(blsSigShares[i]["signatureShare"].asString());
+        BLSSigShare sig(sig_share_ptr, i + 1, t, n);
+        sigShareSet.addSigShare(make_shared<BLSSigShare>(sig));
+    }
+
+    sigShareSet.merge();
+}
+
 void TestUtils::destroyEnclave() {
     if (eid != 0) {
         sgx_destroy_enclave(eid);
@@ -687,6 +778,146 @@ void TestUtils::doDKGV2(StubClient &c, int n, int t,
         cerr << i << endl;
 }
 
+
+void TestUtils::doZMQBLS(shared_ptr<ZMQClient> _zmqClient, StubClient &c, int n, int t,
+                        vector<string>& _ecdsaKeyNames, vector<string>& _blsKeyNames,
+                        int schainID, int dkgID) {
+    Json::Value ethKeys[n];
+    Json::Value verifVects[n];
+    Json::Value pubEthKeys;
+    Json::Value secretShares[n];
+    Json::Value pubBLSKeys[n];
+    Json::Value blsSigShares[n];
+    vector<string> pubShares(n);
+    vector<string> polyNames(n);
+
+    _ecdsaKeyNames.clear();
+    _blsKeyNames.clear();
+
+    for (uint8_t i = 0; i < n; i++) {
+        ethKeys[i] = c.generateECDSAKey();
+
+        CHECK_STATE(ethKeys[i]["status"] == 0);
+
+        auto keyName = ethKeys[i]["keyName"].asString();
+        CHECK_STATE(keyName.size() == ECDSA_KEY_NAME_SIZE);
+
+        _ecdsaKeyNames.push_back(keyName);
+
+        string polyName =
+                "POLY:SCHAIN_ID:" + to_string(schainID) + ":NODE_ID:" + to_string(i) + ":DKG_ID:" + to_string(dkgID);
+
+        Json::Value response = c.generateDKGPoly(polyName, t);
+        CHECK_STATE(response["status"] == 0);
+        polyNames[i] = polyName;
+        verifVects[i] = c.getVerificationVector(polyName, t, n);
+        CHECK_STATE(verifVects[i]["status"] == 0);
+        pubEthKeys.append(ethKeys[i]["publicKey"]);
+    }
+
+    for (uint8_t i = 0; i < n; i++) {
+        secretShares[i] = c.getSecretShareV2(polyNames[i], pubEthKeys, t, n);
+        CHECK_STATE(secretShares[i]["status"] == 0);
+        for (uint8_t k = 0; k < t; k++) {
+            for (uint8_t j = 0; j < 4; j++) {
+                string pubShare = verifVects[i]["verificationVector"][k][j].asString();
+                CHECK_STATE(pubShare.length() > 60);
+                pubShares[i] += TestUtils::convertDecToHex(pubShare);
+            }
+        }
+    }
+
+    int k = 0;
+
+    vector<string> secShares(n);
+
+    vector<string> pSharesBad(pubShares);
+
+    for (int i = 0; i < n; i++)
+        for (int j = 0; j < n; j++) {
+            string secretShare = secretShares[i]["secretShare"].asString().substr(192 * j, 192);
+            secShares[i] += secretShares[j]["secretShare"].asString().substr(192 * i, 192);
+            Json::Value response = c.dkgVerificationV2(pubShares[i], ethKeys[j]["keyName"].asString(), secretShare, t, n,
+                                                       j);
+            CHECK_STATE(response["status"] == 0);
+
+            bool res = response["result"].asBool();
+            CHECK_STATE(res);
+
+            k++;
+
+            pSharesBad[i][0] = 'q';
+            Json::Value wrongVerif = c.dkgVerificationV2(pSharesBad[i], ethKeys[j]["keyName"].asString(), secretShare, t,
+                                                         n, j);
+            res = wrongVerif["result"].asBool();
+            CHECK_STATE(!res);
+        }
+
+    BLSSigShareSet sigShareSet(t, n);
+
+    string hash = SAMPLE_HASH;
+
+    auto hash_arr = make_shared<array<uint8_t, 32 >>();
+    uint64_t binLen;
+    if (!hex2carray(hash.c_str(), &binLen, hash_arr->data(), 32)) {
+        throw SGXException(TEST_INVALID_HEX, "Invalid hash");
+    }
+
+    map<size_t, shared_ptr<BLSPublicKeyShare>> pubKeyShares;
+
+    for (int i = 0; i < n; i++) {
+        string endName = polyNames[i].substr(4);
+        string blsName = "BLS_KEY" + polyNames[i].substr(4);
+        _blsKeyNames.push_back(blsName);
+        string secretShare = secretShares[i]["secretShare"].asString();
+
+        auto response = c.createBLSPrivateKeyV2(blsName, ethKeys[i]["keyName"].asString(), polyNames[i], secShares[i], t,
+                                                n);
+        CHECK_STATE(response["status"] == 0);
+        pubBLSKeys[i] = c.getBLSPublicKeyShare(blsName);
+        CHECK_STATE(pubBLSKeys[i]["status"] == 0);
+    }
+
+    for (int i = 0; i < t; i++) {
+        vector<string> pubKeyVect;
+        for (uint8_t j = 0; j < 4; j++) {
+            pubKeyVect.push_back(pubBLSKeys[i]["blsPublicKeyShare"][j].asString());
+        }
+        BLSPublicKeyShare pubKey(make_shared<vector<string >>(pubKeyVect), t, n);
+
+        pubKeyShares[i + 1] = make_shared<BLSPublicKeyShare>(pubKey);
+    }
+
+    // create pub key
+
+    BLSPublicKey blsPublicKey(make_shared<map<size_t, shared_ptr<BLSPublicKeyShare >>>(pubKeyShares), t,
+            n);
+
+    // sign verify a sample sig
+
+    for (int i = 0; i < t; i++) {
+
+        string blsName = "BLS_KEY" + polyNames[i].substr(4);
+        auto sigShare =  make_shared<string>(_zmqClient->blsSignMessageHash(blsName, hash, t, n));
+        BLSSigShare sig(sigShare, i + 1, t, n);
+        sigShareSet.addSigShare(make_shared<BLSSigShare>(sig));
+
+        auto pubKey = pubKeyShares[i+1];
+
+        CHECK_STATE(pubKey->VerifySigWithHelper(hash_arr, make_shared<BLSSigShare>(sig), t, n));
+    }
+
+    shared_ptr<BLSSignature> commonSig = sigShareSet.merge();
+
+    CHECK_STATE(blsPublicKey.VerifySigWithHelper(hash_arr, commonSig, t, n));
+
+    for (auto&& i : _ecdsaKeyNames)
+        cerr << i << endl;
+
+    for (auto&& i : _blsKeyNames)
+        cerr << i << endl;
+}
+
 int sessionKeyRecoverDH(const char *skey_str, const char *sshare, char *common_key) {
 
     int ret = -1;

TestUtils.h

@@ -41,6 +41,7 @@
 #include <sgx_tcrypto.h>
 #include "stubclient.h"
 #include <jsonrpccpp/server/connectors/httpserver.h>
+#include "ZMQClient.h"
 #include "abstractstubserver.h"
 
 using namespace std;
@@ -81,6 +82,13 @@ public:
     static void doDKGV2(StubClient &c, int n, int t,
                                  vector<string>& _ecdsaKeyNames, vector<string>& _blsKeyNames,
                                  int schainID, int dkgID);
+
+    static void doZMQBLS(shared_ptr<ZMQClient> _zmqClient, StubClient &c, int n, int t,
+                        vector<string>& _ecdsaKeyNames, vector<string>& _blsKeyNames,
+                        int schainID, int dkgID);
+
+    static void sendRPCRequestZMQ();
+
 };
 
 int sessionKeyRecoverDH(const char *skey_str, const char *sshare, char *common_key);

VERSION

-1.66.1
+1.68.1

ZMQClient.cpp

+/*
+    Copyright (C) 2018-2019 SKALE Labs
+
+    This file is part of skale-consensus.
+
+    skale-consensus is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    skale-consensus is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with skale-consensus.  If not, see <https://www.gnu.org/licenses/>.
+
+    @file ZMQClient.cpp
+    @author Stan Kladko
+    @date 2020
+*/
+
+#include "sys/random.h"
+#include <sys/types.h>
+#include <sys/syscall.h>
+
+
+#include <fstream>
+#include <streambuf>
+#include <regex>
+
+#include "sgxwallet_common.h"
+#include "common.h"
+#include "BLSCrypto.h"
+#include "BLSSignReqMessage.h"
+#include "BLSSignRspMessage.h"
+#include "ECDSASignReqMessage.h"
+#include "ECDSASignRspMessage.h"
+#include "ZMQClient.h"
+
+
+shared_ptr <ZMQMessage> ZMQClient::doRequestReply(Json::Value &_req) {
+
+    Json::FastWriter fastWriter;
+
+    if (sign) {
+        CHECK_STATE(!certificate.empty());
+        CHECK_STATE(!key.empty());
+
+        _req["cert"] = certificate;
+
+        string msgToSign = fastWriter.write(_req);
+
+        _req["msgSig"] = signString(pkey, msgToSign);
+    }
+
+    string reqStr = fastWriter.write(_req);
+
+
+    reqStr = reqStr.substr(0, reqStr.size() - 1);
+    CHECK_STATE(reqStr.front() == '{');
+    CHECK_STATE(reqStr.at(reqStr.size() - 1) == '}');
+
+
+    auto resultStr = doZmqRequestReply(reqStr);
+
+    try {
+
+        CHECK_STATE(resultStr.size() > 5)
+        CHECK_STATE(resultStr.front() == '{')
+        CHECK_STATE(resultStr.back() == '}')
+
+
+        return ZMQMessage::parse(resultStr.c_str(), resultStr.size(), false, false);
+    } catch (std::exception &e) {
+        spdlog::error(string("Error in doRequestReply:") + e.what());
+        throw;
+    } catch (...) {
+        spdlog::error("Error in doRequestReply");
+        throw;
+    }
+
+
+}
+
+
+string ZMQClient::doZmqRequestReply(string &_req) {
+
+    stringstream request;
+
+    shared_ptr <zmq::socket_t> clientSocket = nullptr;
+
+    {
+        lock_guard <recursive_mutex> m(mutex);
+        if (!clientSockets.count(getProcessID()))
+            reconnect();
+        clientSocket = clientSockets.at(getProcessID());
+        CHECK_STATE(clientSocket);
+    }
+    CHECK_STATE(clientSocket);
+
+    spdlog::debug("ZMQ client sending: \n {}", _req);
+
+    s_send(*clientSocket, _req);
+
+    while (true) {
+        //  Poll socket for a reply, with timeout
+        zmq::pollitem_t items[] = {
+                {static_cast<void *>(*clientSocket), 0, ZMQ_POLLIN, 0}};
+        zmq::poll(&items[0], 1, REQUEST_TIMEOUT);
+        //  If we got a reply, process it
+        if (items[0].revents & ZMQ_POLLIN) {
+            string reply = s_recv(*clientSocket);
+
+            CHECK_STATE(reply.size() > 5);
+            reply = reply.substr(0, reply.size() - 1);
+            spdlog::debug("ZMQ client received reply:{}", reply);
+            CHECK_STATE(reply.front() == '{');
+            CHECK_STATE(reply.back() == '}');
+
+            return reply;
+        } else {
+            spdlog::error("W: no response from server, retrying...");
+            reconnect();
+            //  Send request again, on new socket
+            s_send(*clientSocket, _req);
+        }
+    }
+}
+
+
+string ZMQClient::readFileIntoString(const string &_fileName) {
+    ifstream t(_fileName);
+    string str((istreambuf_iterator<char>(t)), istreambuf_iterator<char>());
+    return str;
+}
+
+
+
+void ZMQClient::verifySig(EVP_PKEY* _pubkey, const string& _str, const string& _sig) {
+
+    CHECK_STATE(_pubkey);
+    CHECK_STATE(!_str.empty());
+
+    static std::regex r("\\s+");
+    auto msgToSign = std::regex_replace(_str, r, "");
+
+    vector<uint8_t> binSig(256,0);
+
+    uint64_t binLen = 0;
+
+    CHECK_STATE2(hex2carray(_sig.c_str(), &binLen, binSig.data(), binSig.size()),
+                 ZMQ_COULD_NOT_PARSE);
+
+    CHECK_STATE(binLen > 0);
+
+    EVP_MD_CTX *mdctx = NULL;
+    int ret = 0;
+
+    size_t slen = 0;
+
+    CHECK_STATE(mdctx = EVP_MD_CTX_create());
+
+    CHECK_STATE((EVP_DigestVerifyInit(mdctx, NULL, EVP_sha256(), NULL, _pubkey) == 1));
+
+    CHECK_STATE(EVP_DigestVerifyUpdate(mdctx, msgToSign.c_str(), msgToSign.size()) == 1);
+
+/* First call EVP_DigestSignFinal with a NULL sig parameter to obtain the length of the
+ * signature. Length is returned in slen */
+
+
+
+    CHECK_STATE2(EVP_DigestVerifyFinal(mdctx, binSig.data(), binLen) == 1,
+                 ZMQ_COULD_NOT_VERIFY_SIG);
+
+    if (mdctx) EVP_MD_CTX_destroy(mdctx);
+
+    return;
+}
+
+
+string ZMQClient::signString(EVP_PKEY* _pkey, const string& _str) {
+
+    CHECK_STATE(_pkey);
+    CHECK_STATE(!_str.empty());
+
+    static std::regex r("\\s+");
+    auto msgToSign = std::regex_replace(_str, r, "");
+
+
+
+    EVP_MD_CTX *mdctx = NULL;
+    int ret = 0;
+    unsigned char *signature = NULL;
+    auto sig = &signature;
+    size_t slen = 0;
+
+    CHECK_STATE(mdctx = EVP_MD_CTX_create());
+
+
+    CHECK_STATE((EVP_DigestSignInit(mdctx, NULL, EVP_sha256(), NULL, _pkey) == 1));
+
+
+    CHECK_STATE(EVP_DigestSignUpdate(mdctx, msgToSign.c_str(), msgToSign.size()) == 1);
+
+/* First call EVP_DigestSignFinal with a NULL sig parameter to obtain the length of the
+ * signature. Length is returned in slen */
+
+    CHECK_STATE(EVP_DigestSignFinal(mdctx, NULL, &slen) == 1);
+    signature = (unsigned char *) OPENSSL_malloc(sizeof(unsigned char) * slen);
+    CHECK_STATE(signature);
+    CHECK_STATE(EVP_DigestSignFinal(mdctx, signature, &slen) == 1);
+
+    auto hexSig = carray2Hex(signature, slen);
+
+    string hexStringSig(hexSig.begin(), hexSig.end());
+
+    /* Clean up */
+    if (signature) OPENSSL_free(signature);
+    if (mdctx) EVP_MD_CTX_destroy(mdctx);
+
+    return hexStringSig;
+}
+
+pair<EVP_PKEY*, X509*> ZMQClient::readPublicKeyFromCertStr(const string& _certStr) {
+
+    CHECK_STATE(!_certStr.empty())
+
+    BIO *bo = BIO_new(BIO_s_mem());
+    CHECK_STATE(bo);
+    BIO_write(bo, _certStr.c_str(), _certStr.size());
+
+    X509* cert = nullptr;
+    PEM_read_bio_X509(bo, &cert, 0, 0);
+    CHECK_STATE(cert);
+    auto key = X509_get_pubkey(cert);
+    BIO_free(bo);
+    CHECK_STATE(key);
+    return {key, cert};
+};
+
+ZMQClient::ZMQClient(const string &ip, uint16_t port, bool _sign, const string &_certFileName,
+                     const string &_certKeyName) : ctx(1), sign(_sign),
+                                                   certKeyName(_certKeyName), certFileName(_certFileName) {
+
+    spdlog::info("Initing ZMQClient. Sign:{} ", _sign);
+
+    if (sign) {
+        CHECK_STATE(!_certFileName.empty());
+        CHECK_STATE(!_certKeyName.empty());
+
+        certificate = readFileIntoString(_certFileName);
+        CHECK_STATE(!certificate.empty());
+
+        key = readFileIntoString(_certKeyName);
+        CHECK_STATE(!key.empty());
+
+        BIO *bo = BIO_new(BIO_s_mem());
+        CHECK_STATE(bo);
+        BIO_write(bo, key.c_str(), key.size());
+
+        PEM_read_bio_PrivateKey(bo, &pkey, 0, 0);
+        CHECK_STATE(pkey);
+        BIO_free(bo);
+
+        auto pubKeyStr = readFileIntoString(_certFileName);
+        CHECK_STATE(!pubKeyStr.empty());
+
+        tie(pubkey, x509Cert) = readPublicKeyFromCertStr(pubKeyStr);
+
+        auto sig = signString(pkey, "sample");
+        verifySig(pubkey, "sample", sig);
+
+    } else {
+        CHECK_STATE(_certFileName.empty());
+        CHECK_STATE(_certKeyName.empty());
+    }
+
+    certFileName = _certFileName;
+    certKeyName = _certKeyName;
+
+
+    url = "tcp://" + ip + ":" + to_string(port);
+}
+
+void ZMQClient::reconnect() {
+
+    lock_guard <recursive_mutex> lock(mutex);
+
+    auto pid = getProcessID();
+
+    if (clientSockets.count(pid) > 0) {
+        clientSockets.erase(pid);
+    }
+
+
+    char identity[10];
+    getrandom(identity, 10, 0);
+    auto clientSocket = make_shared<zmq::socket_t>(ctx, ZMQ_DEALER);
+    clientSocket->setsockopt(ZMQ_IDENTITY, identity, 10);
+    //  Configure socket to not wait at close time
+    int linger = 0;
+    clientSocket->setsockopt(ZMQ_LINGER, &linger, sizeof(linger));
+    clientSocket->connect(url);
+    clientSockets.insert({pid, clientSocket});
+}
+
+
+string ZMQClient::blsSignMessageHash(const std::string &keyShareName, const std::string &messageHash, int t, int n) {
+    Json::Value p;
+    p["type"] = ZMQMessage::BLS_SIGN_REQ;
+    p["keyShareName"] = keyShareName;
+    p["messageHash"] = messageHash;
+    p["n"] = n;
+    p["t"] = t;
+    auto result = dynamic_pointer_cast<BLSSignRspMessage>(doRequestReply(p));
+    CHECK_STATE(result);
+    CHECK_STATE(result->getStatus() == 0);
+
+    return result->getSigShare();
+}
+
+string ZMQClient::ecdsaSignMessageHash(int base, const std::string &keyName, const std::string &messageHash) {
+    Json::Value p;
+    p["type"] = ZMQMessage::ECDSA_SIGN_REQ;
+    p["base"] = base;
+    p["keyName"] = keyName;
+    p["messageHash"] = messageHash;
+    auto result = dynamic_pointer_cast<ECDSASignRspMessage>(doRequestReply(p));
+    CHECK_STATE(result);
+    CHECK_STATE(result->getStatus() == 0);
+    return result->getSignature();
+}
+
+
+uint64_t ZMQClient::getProcessID() {
+    return syscall(__NR_gettid);
+}
\ No newline at end of file

ZMQClient.h

+/*
+    Copyright (C) 2018-2019 SKALE Labs
+
+    This file is part of skale-consensus.
+
+    skale-consensus is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    skale-consensus is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with skale-consensus.  If not, see <https://www.gnu.org/licenses/>.
+
+    @file ZMQClient.h
+    @author Stan Kladko
+    @date 2021
+*/
+
+
+
+
+#ifndef SGXWALLET_ZMQCLIENT_H
+#define SGXWALLET_ZMQCLIENT_H
+
+#include <openssl/pem.h>
+#include <openssl/evp.h>
+#include <openssl/err.h>
+#include <openssl/sha.h>
+#include <openssl/rand.h>
+
+#include "third_party/spdlog/spdlog.h"
+
+#include <zmq.hpp>
+#include "zhelpers.hpp"
+#include <jsonrpccpp/client.h>
+#include "ZMQMessage.h"
+
+
+
+
+#define REQUEST_TIMEOUT     10000    //  msecs, (> 1000!)
+
+class ZMQClient {
+
+
+private:
+
+    EVP_PKEY* pkey = 0;
+    EVP_PKEY* pubkey = 0;
+    X509* x509Cert = 0;
+
+
+    bool sign = true;
+    string certFileName = "";
+    string certKeyName = "";
+    string certificate = "";
+    string key = "";
+
+
+    recursive_mutex mutex;
+
+    zmq::context_t ctx;
+
+    string url;
+
+    // generate random identity
+
+    map<uint64_t , shared_ptr <zmq::socket_t>> clientSockets;
+
+    shared_ptr <ZMQMessage> doRequestReply(Json::Value &_req);
+
+    string doZmqRequestReply(string &_req);
+
+    uint64_t getProcessID();
+
+    static string readFileIntoString(const string& _fileName);
+
+public:
+
+
+    ZMQClient(const string &ip, uint16_t port, bool _sign, const string&  _certPathName,
+              const string& _certKeyName);
+
+    void reconnect() ;
+
+    static pair<EVP_PKEY*, X509*>  readPublicKeyFromCertStr(const string& _cert);
+
+    static string signString(EVP_PKEY* _pkey, const string& _str);
+
+    static void   verifySig(EVP_PKEY* _pubkey, const string& _str, const string& _sig);
+
+    string blsSignMessageHash(const std::string &keyShareName, const std::string &messageHash, int t, int n);
+
+    string ecdsaSignMessageHash(int base, const std::string &keyName, const std::string &messageHash);
+
+};
+
+
+
+#endif //SGXWALLET_ZMQCLIENT_H

ZMQMessage.cpp

@@ -22,17 +22,24 @@
 */
 
 #include "common.h"
+#include "sgxwallet_common.h"
+#include <third_party/cryptlite/sha256.h>
+#include <iostream>
+#include <fstream>
 
+#include "ZMQClient.h"
+#include "SGXWalletServer.hpp"
 #include "BLSSignReqMessage.h"
+#include "BLSSignRspMessage.h"
 #include "ECDSASignReqMessage.h"
+#include "ECDSASignRspMessage.h"
 #include "ZMQMessage.h"
 
 
-
 uint64_t ZMQMessage::getUint64Rapid(const char *_name) {
     CHECK_STATE(_name);
     CHECK_STATE(d->HasMember(_name));
-    const rapidjson::Value& a = (*d)[_name];
+    const rapidjson::Value &a = (*d)[_name];
     CHECK_STATE(a.IsUint64());
     return a.GetUint64();
 };
@@ -44,31 +51,120 @@ string ZMQMessage::getStringRapid(const char *_name) {
     return (*d)[_name].GetString();
 };
 
-shared_ptr<ZMQMessage> ZMQMessage::parse(vector<uint8_t>& _msg) {
 
-    CHECK_STATE(_msg.at(_msg.size() - 1) == 0);
+
+
+shared_ptr <ZMQMessage> ZMQMessage::parse(const char *_msg,
+                                          size_t _size, bool _isRequest,
+                                          bool _verifySig) {
+
+    CHECK_STATE(_msg);
+    CHECK_STATE2(_size > 5, ZMQ_INVALID_MESSAGE_SIZE);
+    // CHECK NULL TERMINATED
+    CHECK_STATE(_msg[_size] == 0);
+    CHECK_STATE2(_msg[_size - 1] == '}', ZMQ_INVALID_MESSAGE);
+    CHECK_STATE2(_msg[0] == '{', ZMQ_INVALID_MESSAGE);
 
     auto d = make_shared<rapidjson::Document>();
 
-    d->Parse((const char*) _msg.data());
+    d->Parse(_msg);
+
+    CHECK_STATE2(!d->HasParseError(), ZMQ_COULD_NOT_PARSE);
+    CHECK_STATE2(d->IsObject(), ZMQ_COULD_NOT_PARSE);
+
+    CHECK_STATE2(d->HasMember("type"), ZMQ_NO_TYPE_IN_MESSAGE);
+    CHECK_STATE2((*d)["type"].IsString(), ZMQ_NO_TYPE_IN_MESSAGE);
+    string type = (*d)["type"].GetString();
+
+    if (_verifySig) {
+        CHECK_STATE2(d->HasMember("cert"),ZMQ_NO_CERT_IN_MESSAGE);
+        CHECK_STATE2(d->HasMember("msgSig"), ZMQ_NO_SIG_IN_MESSAGE);
+        CHECK_STATE2((*d)["cert"].IsString(), ZMQ_NO_CERT_IN_MESSAGE);
+        CHECK_STATE2((*d)["msgSig"].IsString(), ZMQ_NO_SIG_IN_MESSAGE);
+
+        auto cert = make_shared<string>((*d)["cert"].GetString());
+        string hash = cryptlite::sha256::hash_hex(*cert);
+
+        auto filepath = "/tmp/sgx_wallet_cert_hash_" + hash;
+
+        std::ofstream outFile(filepath);
+
+        outFile << *cert;
+
+        outFile.close();
+
+        static recursive_mutex m;
+
+        EVP_PKEY *publicKey = nullptr;
+
+        {
+            lock_guard <recursive_mutex> lock(m);
+
+            if (!verifiedCerts.exists(*cert)) {
+                CHECK_STATE(SGXWalletServer::verifyCert(filepath));
+                auto handles = ZMQClient::readPublicKeyFromCertStr(*cert);
+                CHECK_STATE(handles.first);
+                CHECK_STATE(handles.second);
+                verifiedCerts.put(*cert, handles);
+                remove(cert->c_str());
+            }
+
+            publicKey = verifiedCerts.get(*cert).first;
 
-    CHECK_STATE(!d->HasParseError());
-    CHECK_STATE(d->IsObject())
+            CHECK_STATE(publicKey);
 
-    CHECK_STATE(d->HasMember("type"));
-    CHECK_STATE((*d)["type"].IsString());
-    auto type =  (*d)["type"].GetString();
+            auto msgSig = make_shared<string>((*d)["msgSig"].GetString());
 
-    shared_ptr<ZMQMessage> result;
+            d->RemoveMember("msgSig");
 
-    if (type == ZMQMessage::BLS_SIGN_REQ) {
-        result = make_shared<BLSSignReqMessage>(d);
-    } else if (type == ZMQMessage::ECDSA_SIGN_REQ) {
-        result = make_shared<ECDSASignReqMessage>(d);
+            rapidjson::StringBuffer buffer;
+
+            rapidjson::Writer<rapidjson::StringBuffer> w(buffer);
+
+            d->Accept(w);
+
+            auto msgToVerify = buffer.GetString();
+
+            ZMQClient::verifySig(publicKey,msgToVerify, *msgSig );
+
+        }
+    }
+
+
+    shared_ptr <ZMQMessage> result;
+
+    if (_isRequest) {
+        return buildRequest(type, d);
     } else {
-        throw SGXException(-301, "Incorrect zmq message type: " + string(type));
+        return buildResponse(type, d);
     }
+}
 
-    return result;
+shared_ptr <ZMQMessage> ZMQMessage::buildRequest(string &_type, shared_ptr <rapidjson::Document> _d) {
+    if (_type == ZMQMessage::BLS_SIGN_REQ) {
+        return make_shared<BLSSignReqMessage>(_d);
+    } else if (_type == ZMQMessage::ECDSA_SIGN_REQ) {
+        return
+                make_shared<ECDSASignReqMessage>(_d);
+    } else {
+        BOOST_THROW_EXCEPTION(SGXException(-301, "Incorrect zmq message type: " +
+                                                 string(_type)));
+    }
+}
 
+shared_ptr <ZMQMessage> ZMQMessage::buildResponse(string &_type, shared_ptr <rapidjson::Document> _d) {
+    if (_type == ZMQMessage::BLS_SIGN_RSP) {
+        return
+                make_shared<BLSSignRspMessage>(_d);
+    } else if (_type == ZMQMessage::ECDSA_SIGN_RSP) {
+        return
+                make_shared<ECDSASignRspMessage>(_d);
+    } else {
+        BOOST_THROW_EXCEPTION(InvalidStateException("Incorrect zmq message request type: " + string(_type),
+                                                    __CLASS_NAME__)
+        );
+    }
 }
+
+cache::lru_cache<string, pair < EVP_PKEY * , X509 *>>
+ZMQMessage::verifiedCerts(256);
\ No newline at end of file

ZMQMessage.h

@@ -26,10 +26,21 @@
 
 #include <memory>
 #include <vector>
-#include "abstractstubserver.h"
 
+#include <openssl/pem.h>
+#include <openssl/evp.h>
+#include <openssl/err.h>
+#include <openssl/sha.h>
+#include <openssl/rand.h>
+
+#include "third_party/lrucache.hpp"
+
+#include "abstractstubserver.h"
 
 #include "document.h"
+#include "stringbuffer.h"
+#include "writer.h"
+
 #include "SGXException.h"
 
 using namespace std;
@@ -38,16 +49,20 @@ class ZMQMessage {
 
     shared_ptr<rapidjson::Document> d;
 
-    static constexpr const char *BLS_SIGN_REQ = "BLSSignReq";
-    static constexpr const char *BLS_SIGN_RSP = "BLSSignRsp";
-    static constexpr const char *ECDSA_SIGN_REQ = "ECDSASignReq";
-    static constexpr const char *ECDSA_SIGN_RSP = "ECDSASignRsp";
+
+    static cache::lru_cache<string, pair<EVP_PKEY*, X509*>> verifiedCerts;
 
 protected:
 
 
 public:
 
+
+    static constexpr const char *BLS_SIGN_REQ = "BLSSignReq";
+    static constexpr const char *BLS_SIGN_RSP = "BLSSignRsp";
+    static constexpr const char *ECDSA_SIGN_REQ = "ECDSASignReq";
+    static constexpr const char *ECDSA_SIGN_RSP = "ECDSASignRsp";
+
     explicit ZMQMessage(shared_ptr<rapidjson::Document> &_d) : d(_d) {
     };
 
@@ -55,7 +70,15 @@ public:
 
     uint64_t getUint64Rapid(const char *_name);
 
-    static shared_ptr<ZMQMessage> parse(vector<uint8_t> &_msg);
+    uint64_t getStatus() {
+        return getUint64Rapid("status");
+    }
+
+    static shared_ptr <ZMQMessage> parse(const char* _msg, size_t _size, bool _isRequest,
+                                         bool _verifySig);
+
+    static shared_ptr<ZMQMessage> buildRequest(string& type, shared_ptr<rapidjson::Document> _d);
+    static shared_ptr<ZMQMessage> buildResponse(string& type, shared_ptr<rapidjson::Document> _d);
 
     virtual Json::Value process() = 0;
 

ZMQServer.cpp

@@ -21,29 +21,61 @@
     @date 2019
 */
 
+#include <fstream>
+#include <streambuf>
 
 
 #include "third_party/spdlog/spdlog.h"
 
+#include "common.h"
+
 #include "ZMQServer.h"
 #include "sgxwallet_common.h"
 
 using namespace std;
 
-ZMQServer::ZMQServer()
-        : isExitRequested(false), ctx_(1),
-          frontend_(ctx_, ZMQ_ROUTER),
-          backend_(ctx_, ZMQ_DEALER) {}
+shared_ptr <ZMQServer> ZMQServer::zmqServer = nullptr;
+
+ZMQServer::ZMQServer(bool _checkSignature, const string &_caCertFile)
+        : checkSignature(_checkSignature),
+          caCertFile(_caCertFile), ctx_(make_shared<zmq::context_t>(1)) {
+
+
+    frontend = make_shared<zmq::socket_t>(*ctx_, ZMQ_ROUTER);
+    backend = make_shared<zmq::socket_t>(*ctx_, ZMQ_DEALER);
+
+    workerThreads = 2 * thread::hardware_concurrency();
+
+    if (workerThreads == 0) {
+        workerThreads = 8;
+    }
+
+    if (_checkSignature) {
+        CHECK_STATE(!_caCertFile.empty());
+        ifstream t(_caCertFile);
+        string str((istreambuf_iterator<char>(t)), istreambuf_iterator<char>());
+        caCert = str;
+        CHECK_STATE(!caCert.empty())
+    }
+
+    int linger = 0;
+
+
+    zmq_setsockopt(*frontend, ZMQ_LINGER, &linger, sizeof(linger));
+    zmq_setsockopt(*backend, ZMQ_LINGER, &linger, sizeof(linger));
+
+}
 
 
 void ZMQServer::run() {
 
-    auto port = BASE_PORT + 4;
+    auto port = BASE_PORT + 5;
 
-    spdlog::info("Starting zmq server ...");
+    spdlog::info("Starting zmq server  on port {} ...", port);
 
     try {
-        frontend_.bind("tcp://*:" + to_string(BASE_PORT + 4));
+        CHECK_STATE(frontend);
+        frontend->bind("tcp://*:" + to_string(port));
     } catch (...) {
         spdlog::error("Server task could not bind to port:{}", port);
         exit(-100);
@@ -52,37 +84,127 @@ void ZMQServer::run() {
     spdlog::info("Bound port ...");
 
     try {
-        backend_.bind("inproc://backend");
+        CHECK_STATE(backend);
+        backend->bind("inproc://backend");
     } catch (exception &e) {
         spdlog::error("Could not bind to zmq backend: {}", e.what());
         exit(-101);
     }
 
 
-    spdlog::info("Creating {} zmq server workers ...", kMaxThread);
+    spdlog::info("Creating {} zmq server workers ...", workerThreads);
 
     try {
-        for (int i = 0; i < kMaxThread; ++i) {
-            worker.push_back(new ServerWorker(ctx_, ZMQ_DEALER));
-            worker_thread.push_back(new std::thread(std::bind(&ServerWorker::work, worker[i])));
+        for (int i = 0; i < workerThreads; ++i) {
+            workers.push_back(make_shared<ServerWorker>(*ctx_, ZMQ_DEALER,
+                                                        this->checkSignature, this->caCert));
+            auto th = make_shared<std::thread>(std::bind(&ServerWorker::work, workers[i]));
+            worker_threads.push_back(th);
         }
     } catch (std::exception &e) {
         spdlog::error("Could not create zmq server workers:{} ", e.what());
         exit(-102);
-    }
+    };
 
 
     try {
-        zmq::proxy(static_cast<void *>(frontend_), static_cast<void *>(backend_), nullptr);
-    } catch (exception& _e) {
-        spdlog::info("Exiting zmq server {}", _e.what());
+        zmq::proxy(static_cast<void *>(*frontend), static_cast<void *>(*backend), nullptr);
+        spdlog::info("Exited zmq proxy");
+    } catch (exception &_e) {
+        if (isExitRequested) {
+            spdlog::info("Exited ZMQServer main thread");
+            return;
+        }
+        spdlog::info("Error, exiting zmq server ... {}", _e.what());
         return;
     } catch (...) {
-        spdlog::info("Exiting zmq server");
+        if (isExitRequested) {
+            spdlog::info("Exited ZMQServer main thread");
+            return;
+        }
+        spdlog::info("Error, exiting zmq server ...");
         return;
     }
 }
 
 
+void ZMQServer::exitAll() {
+
+    spdlog::info("Exiting zmq server workers ...");
+
+    for (auto &&worker : workers) {
+        worker->requestExit();
+    }
+
+    for (auto &&workerThread : worker_threads) {
+        workerThread->join();
+    }
+
+    spdlog::info("Exited zmq server workers ...");
+
+}
+
+std::atomic<bool> ZMQServer::isExitRequested(false);
+
+void ZMQServer::exitZMQServer() {
+    auto doExit = !isExitRequested.exchange(true);
+    if (doExit) {
+
+        zmqServer->exitAll();
+
+
+        spdlog::info("deleting zmq server");
+        zmqServer = nullptr;
+        spdlog::info("deleted zmq server ");
+    }
+}
+
+void ZMQServer::initZMQServer(bool _checkSignature) {
+    static bool initedServer = false;
+    CHECK_STATE(!initedServer)
+    initedServer = true;
+
+    spdlog::info("Initing zmq server. checkSignature is set to {}", _checkSignature);
+
+    string rootCAPath = "";
+
+    if (_checkSignature) {
+
+        rootCAPath = string(SGXDATA_FOLDER) + "cert_data/rootCA.pem";
+        spdlog::info("Reading root CA from {}", rootCAPath);
+        CHECK_STATE(access(rootCAPath.c_str(), F_OK) == 0);
+    };
+
+    zmqServer = make_shared<ZMQServer>(_checkSignature, rootCAPath);
+    serverThread = make_shared<thread>(std::bind(&ZMQServer::run, ZMQServer::zmqServer));
+    serverThread->detach();
+
+    spdlog::info("Inited zmq server ...");
+}
+
+shared_ptr <std::thread> ZMQServer::serverThread = nullptr;
+
+ZMQServer::~ZMQServer() {
+
+    spdlog::info("Deleting worker threads");
+    worker_threads.clear();
+    spdlog::info("Deleted worker threads");
+
+    spdlog::info("Deleting workers ...");
+    workers.clear();
+    spdlog::info("Deleted workers ...");
+
+    spdlog::info("Deleting front end and back end");
+    frontend = nullptr;
+    backend = nullptr;
+    spdlog::info("Deleted front end and back end");
+
+    spdlog::info("Deleting server thread");
+    ZMQServer::serverThread = nullptr;
+    spdlog::info("Deleted server thread");
 
 
+    spdlog::info("Deleting ZMQ context");
+    ctx_ = nullptr;
+    spdlog::info("Deleted ZMQ context");
+}
\ No newline at end of file

ZMQServer.h

@@ -42,26 +42,44 @@ using namespace std;
 
 
 class ZMQServer {
+
+    uint64_t workerThreads;
+
 public:
-    ZMQServer();
 
+    bool checkSignature = false;
+    string caCertFile = "";
+    string caCert = "";
+
+    static shared_ptr<ZMQServer> zmqServer;
 
-    atomic<bool> isExitRequested;
+    static shared_ptr<std::thread> serverThread;
 
-    enum {
-        kMaxThread = 5
-    };
+    ZMQServer(bool _checkSignature, const string& _caCertFile);
+
+    ~ZMQServer();
 
     void run();
 
+    void exitAll();
+
+    static void initZMQServer(bool _checkSignature);
+    static void exitZMQServer();
+
+
+
 private:
-    zmq::context_t ctx_;
-    zmq::socket_t frontend_;
-    zmq::socket_t backend_;
+    shared_ptr<zmq::context_t> ctx_;
+    shared_ptr<zmq::socket_t> frontend;
+    shared_ptr<zmq::socket_t> backend;
+
+    std::vector<shared_ptr<ServerWorker> > workers;
+    std::vector<shared_ptr<std::thread>> worker_threads;
 
-    std::vector<ServerWorker *> worker;
-    std::vector<std::thread *> worker_thread;
+    static std::atomic<bool> isExitRequested;
 
 };
 
+
+
 #endif //SGXWALLET_ZMQServer_H

cert/verify_client_cert

+#!/bin/bash
+
+openssl verify -CAfile $1 $2
\ No newline at end of file

common.h

@@ -56,7 +56,7 @@ inline std::string className(const std::string &prettyFunction) {
 
 #include <execinfo.h>
 
-inline void print_stack() {
+inline void print_stack(int _line) {
     void *array[10];
     size_t size;
 
@@ -64,18 +64,25 @@ inline void print_stack() {
     size = backtrace(array, 10);
 
     // print out all the frames to stderr
-    fprintf(stderr, "Error: signal \n");
+    fprintf(stderr, "Backtrace on line %d:   \n", _line);
     backtrace_symbols_fd(array, size, STDERR_FILENO);
-    exit(-1);
 }
 
 
 #define CHECK_STATE(_EXPRESSION_) \
     if (!(_EXPRESSION_)) { \
         auto __msg__ = std::string("State check failed::") + #_EXPRESSION_ +  " " + std::string(__FILE__) + ":" + std::to_string(__LINE__); \
-        print_stack();                                \
+        print_stack(__LINE__);            \
+        \
         BOOST_THROW_EXCEPTION(SGXException(-100, string(__CLASS_NAME__) +  ":" + __msg__));}
 
+#define CHECK_STATE2(_EXPRESSION_, __STATUS__) \
+    if (!(_EXPRESSION_)) { \
+        auto __msg__ = std::string("State check failed::") + #_EXPRESSION_ +  " " + std::string(__FILE__) + ":" + std::to_string(__LINE__); \
+        print_stack(__LINE__);            \
+        \
+        BOOST_THROW_EXCEPTION(SGXException(__STATUS__, string(__CLASS_NAME__) +  ":" + __msg__));}
+
 
 #define HANDLE_TRUSTED_FUNCTION_ERROR(__STATUS__, __ERR_STATUS__, __ERR_MSG__) \
 if (__STATUS__ != SGX_SUCCESS) { \

scripts/install_packages.sh

 #!/bin/bash
-sudo dpkg -i *.deb
 sudo apt install -y build-essential make gcc g++ yasm  python libprotobuf10 flex bison automake
 sudo apt install -y ccache cmake ccache autoconf texinfo libgcrypt20-dev libgnutls28-dev libtool pkg-config

secure_enclave/Makefile.in

@@ -320,7 +320,7 @@ SGX_EDGER8R = $(SGXSDK_BINDIR)/sgx_edger8r
 SGX_SIGN = $(SGXSDK_BINDIR)/sgx_sign
 SGXSSL_BINDIR = @SGXSSL_BINDIR@
 AM_CFLAGS = @SGX_ENCLAVE_CFLAGS@
-AM_CPPFLAGS = @SGX_ENCLAVE_CPPFLAGS@ -Wall \
+AM_CPPFLAGS = @SGX_ENCLAVE_CPPFLAGS@ -O2 -Wall \
 	-Wno-implicit-function-declaration $(TGMP_CPPFLAGS) \
 	-I./third_party/SCIPR -I../third_party/SCIPR \
 	-I../sgx-sdk-build/sgxsdk/include/libcxx

sgxwall.cpp

@@ -174,7 +174,7 @@ int main(int argc, char *argv[]) {
         enclaveLogLevel = L_TRACE;
     }
 
-    initAll(enclaveLogLevel, checkClientCertOption, autoSignClientCertOption, generateTestKeys);
+    initAll(enclaveLogLevel, checkClientCertOption, checkClientCertOption, autoSignClientCertOption, generateTestKeys);
 
     ifstream is("sgx_data/4node.json");
 

sgxwallet_common.h

@@ -159,6 +159,16 @@ extern bool autoconfirm;
 #define ECDSA_SIGN_INVALID_KEY_HEX -86
 #define SET_SEK_INVALID_SEK_HEX -87
 #define TEST_INVALID_HEX -88
+#define ZMQ_SERVER_ERROR -89
+#define ZMQ_COULD_NOT_PARSE -90
+#define ZMQ_INVALID_MESSAGE -91
+#define ZMQ_COULD_NOT_GET_SOCKOPT -92
+#define ZMQ_INVALID_MESSAGE_SIZE -93
+#define ZMQ_NO_TYPE_IN_MESSAGE -94
+#define ZMQ_NO_SIG_IN_MESSAGE -95
+#define ZMQ_NO_CERT_IN_MESSAGE -96
+#define ZMQ_COULD_NOT_VERIFY_SIG -97
+
 
 #define SGX_ENCLAVE_ERROR -666
 

testw.h

@@ -33,6 +33,10 @@
 
 #define SAMPLE_POLY_NAME "POLY:SCHAIN_ID:1:NODE_ID:1:DKG_ID:1"
 #define RPC_ENDPOINT  "http://localhost:1029"
+#define RPC_ENDPOINT_HTTPS  "https://localhost:1026"
+#define ZMQ_IP "127.0.0.1"
+#define ZMQ_PORT 1031
+
 #define SAMPLE_PUBLIC_KEY_B "c0152c48bf640449236036075d65898fded1e242c00acb45519ad5f788ea7cbf9a5df1559e7fc87932eee5478b1b9023de19df654395574a690843988c3ff475"
 
 #define SAMPLE_DKG_PUB_KEY_1 "505f55a38f9c064da744f217d1cb993a17705e9839801958cda7c884e08ab4dad7fd8d22953d3ac7f0913de24fd67d7ed36741141b8a3da152d7ba954b0f14e2"

testw.py

@@ -28,7 +28,9 @@ username = getpass.getuser()
 
 topDir = os.getcwd() + "/sgxwallet"
 print("Top directory is:" + topDir)
-testList = ["[first-run]",
+testList = [ "[zmq-ecdsa]",
+            "[dkgzmqbls]",
+             "[first-run]",
             "[second-run]",
             "[many-threads-crypto]",
             "[many-threads-crypto-v2]",

third_party/lrucache.hpp

+/*
+ * File:   lrucache.hpp
+ * Author: Alexander Ponomarev
+ *
+ * Created on June 20, 2013, 5:09 PM
+ */
+
+
+#ifndef _LRUCACHE_HPP_INCLUDED_
+#define	_LRUCACHE_HPP_INCLUDED_
+
+#include <mutex>
+#include <shared_mutex>
+
+
+#include <any>
+
+#include <unordered_map>
+#include <list>
+#include <string>
+#include <cstddef>
+#include <stdexcept>
+
+/*
+ * Synchronized LRU cache
+ */
+
+using namespace std;
+
+namespace cache {
+
+
+    template<typename key_t, typename value_t>
+    class lru_cache {
+
+        std::recursive_mutex m;
+
+    public:
+        typedef typename std::pair<key_t, value_t> key_value_pair_t;
+        typedef typename std::list<key_value_pair_t>::iterator list_iterator_t;
+
+        lru_cache(size_t max_size) :
+                _max_size(max_size) {
+        }
+
+
+
+
+        void putIfDoesNotExist(const key_t& key, const value_t& value) {
+            lock_guard<recursive_mutex> l(m);
+            if (!exists(key))
+                put(key, value);
+        }
+
+        void put(const key_t& key, const value_t& value) {
+
+            lock_guard<recursive_mutex> l(m);
+
+            auto it = _cache_items_map.find(key);
+            _cache_items_list.push_front(key_value_pair_t(key, value));
+            if (it != _cache_items_map.end()) {
+                _cache_items_list.erase(it->second);
+                _cache_items_map.erase(it);
+            }
+            _cache_items_map[key] = _cache_items_list.begin();
+
+            if (_cache_items_map.size() > _max_size) {
+                auto last = _cache_items_list.end();
+                last--;
+                _cache_items_map.erase(last->first);
+                _cache_items_list.pop_back();
+            }
+        }
+
+
+
+
+        const value_t& get(const key_t& key) {
+
+            lock_guard<recursive_mutex> l(m);
+
+            auto it = _cache_items_map.find(key);
+            if (it == _cache_items_map.end()) {
+                throw std::range_error("There is no such key in cache");
+            } else {
+                _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+                return it->second->second;
+            }
+        }
+
+
+
+
+
+        bool exists(const key_t& key)  {
+
+            lock_guard<recursive_mutex> l(m);
+
+            return _cache_items_map.find(key) != _cache_items_map.end();
+        }
+
+        size_t size() const {
+            lock_guard<recursive_mutex> l(m);
+            return _cache_items_map.size();
+        }
+
+    private:
+        std::list<key_value_pair_t> _cache_items_list;
+        std::unordered_map<key_t, list_iterator_t> _cache_items_map;
+        size_t _max_size;
+    };
+
+} // namespace cache
+
+#endif	/* _LRUCACHE_HPP_INCLUDED_ */