!18568 added certificate verify

Merge pull request !18568 from anancds/ssl

mindspore/ccsrc/ps/core/communicator/http_request_handler.cc

@@ -26,11 +26,18 @@ bool HttpRequestHandler::Initialize(int fd, const std::unordered_map<std::string
   MS_EXCEPTION_IF_NULL(http);
 
   if (PSContext::instance()->enable_ssl()) {
+    MS_LOG(INFO) << "Enable ssl support.";
     SSL_CTX_set_options(SSLWrapper::GetInstance().GetSSLCtx(),
                         SSL_OP_SINGLE_DH_USE | SSL_OP_SINGLE_ECDH_USE | SSL_OP_NO_SSLv2);
     EC_KEY *ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1);
     MS_EXCEPTION_IF_NULL(ecdh);
 
+    X509 *cert = SSLWrapper::GetInstance().ReadCertFromFile(kCertificateChain);
+    if (!SSLWrapper::GetInstance().VerifyCertTime(cert)) {
+      MS_LOG(INFO) << "Verify cert time failed.";
+      return false;
+    }
+
     if (!SSL_CTX_use_certificate_chain_file(SSLWrapper::GetInstance().GetSSLCtx(), kCertificateChain)) {
       MS_LOG(ERROR) << "SSL use certificate chain file failed!";
       return false;

mindspore/ccsrc/ps/core/communicator/ssl_wrapper.cc

@@ -17,10 +17,29 @@
 
 #include "ps/core/communicator/ssl_wrapper.h"
 
+#include <sys/time.h>
+#include <openssl/pem.h>
+#include <openssl/sha.h>
+
+#include <cstdio>
+#include <cstring>
+#include <cstdlib>
+#include <vector>
+#include <iomanip>
+#include <sstream>
+
 namespace mindspore {
 namespace ps {
 namespace core {
-SSLWrapper::SSLWrapper() : ssl_ctx_(nullptr), client_ssl_ctx_(nullptr) { InitSSL(); }
+SSLWrapper::SSLWrapper()
+    : ssl_ctx_(nullptr),
+      client_ssl_ctx_(nullptr),
+      rootFirstCA_(nullptr),
+      rootSecondCA_(nullptr),
+      rootFirstCrl_(nullptr),
+      rootSecondCrl_(nullptr) {
+  InitSSL();
+}
 
 SSLWrapper::~SSLWrapper() { CleanSSL(); }
 
@@ -64,6 +83,161 @@ SSL_CTX *SSLWrapper::GetSSLCtx(bool is_server) {
     return client_ssl_ctx_;
   }
 }
+
+X509 *SSLWrapper::ReadCertFromFile(const std::string &certPath) {
+  BIO *bio = BIO_new_file(certPath.c_str(), "r");
+  return PEM_read_bio_X509(bio, nullptr, nullptr, nullptr);
+}
+
+X509 *SSLWrapper::ReadCertFromPerm(std::string cert) {
+  BIO *bio = BIO_new_mem_buf(reinterpret_cast<void *>(cert.data()), -1);
+  return PEM_read_bio_X509(bio, nullptr, nullptr, nullptr);
+}
+
+X509_CRL *SSLWrapper::ReadCrlFromFile(const std::string &crlPath) {
+  BIO *bio = BIO_new_file(crlPath.c_str(), "r");
+  return PEM_read_bio_X509_CRL(bio, nullptr, nullptr, nullptr);
+}
+
+void SSLWrapper::InitRootCertAndCRL(const std::string rootFirstCaFilePath, const std::string rootSecondCaFilePath,
+                                    const std::string crlFirstFilePath, const std::string crlSecondFilePath) {
+  if (rootFirstCaFilePath.empty() || rootSecondCaFilePath.empty() || crlFirstFilePath.empty() ||
+      crlSecondFilePath.empty()) {
+    return;
+  }
+  rootFirstCA_ = SSLWrapper::ReadCertFromFile(rootFirstCaFilePath);
+  rootSecondCA_ = SSLWrapper::ReadCertFromFile(rootSecondCaFilePath);
+  MS_LOG(INFO) << "Root first ca serialNumber: " << X509_get_serialNumber(rootFirstCA_)->data;
+  MS_LOG(INFO) << "Root second ca serialNumber: " << X509_get_serialNumber(rootSecondCA_)->data;
+
+  rootFirstCrl_ = SSLWrapper::ReadCrlFromFile(crlFirstFilePath);
+  rootSecondCrl_ = SSLWrapper::ReadCrlFromFile(crlSecondFilePath);
+
+  MS_LOG(INFO) << "Root first crl version: " << X509_CRL_get_version(rootFirstCrl_);
+  MS_LOG(INFO) << "Root second crl version: " << X509_CRL_get_version(rootSecondCrl_);
+}
+
+bool SSLWrapper::VerifyCertTime(const X509 *cert) {
+  ASN1_TIME *start = X509_getm_notBefore(cert);
+  ASN1_TIME *end = X509_getm_notAfter(cert);
+
+  int day = 0;
+  int sec = 0;
+  ASN1_TIME_diff(&day, &sec, start, NULL);
+
+  if (day < 0 || sec < 0) {
+    MS_LOG(INFO) << "Cert start time is later than now time.";
+    return false;
+  }
+  day = 0;
+  sec = 0;
+  ASN1_TIME_diff(&day, &sec, NULL, end);
+  if (day < 0 || sec < 0) {
+    MS_LOG(INFO) << "Cert end time is sooner than now time.";
+    return false;
+  }
+
+  return true;
+}
+
+bool SSLWrapper::VerifyCAChain(const std::string &keyAttestation, const std::string &equipCert,
+                               const std::string &equipCACert, std::string) {
+  X509 *keyAttestationCertObj = ReadCertFromPerm(keyAttestation);
+  X509 *equipCertObj = ReadCertFromPerm(equipCert);
+  X509 *equipCACertObj = ReadCertFromPerm(equipCACert);
+
+  if (!VerifyCertTime(keyAttestationCertObj) || !VerifyCertTime(equipCertObj) || !VerifyCertTime(equipCACertObj)) {
+    return false;
+  }
+
+  EVP_PKEY *equipPubKey = X509_get_pubkey(equipCertObj);
+  EVP_PKEY *equipCAPubKey = X509_get_pubkey(equipCACertObj);
+
+  EVP_PKEY *rootFirstPubKey = X509_get_pubkey(rootFirstCA_);
+  EVP_PKEY *rootSecondPubKey = X509_get_pubkey(rootSecondCA_);
+
+  int ret = 0;
+  ret = X509_verify(keyAttestationCertObj, equipPubKey);
+  if (ret != 1) {
+    MS_LOG(INFO) << "keyAttestationCert verify is failed";
+    return false;
+  }
+  ret = X509_verify(equipCertObj, equipCAPubKey);
+  if (ret != 1) {
+    MS_LOG(INFO) << "Equip cert verify is failed";
+    return false;
+  }
+  int ret_first = X509_verify(equipCACertObj, rootFirstPubKey);
+  int ret_second = X509_verify(equipCACertObj, rootSecondPubKey);
+  if (ret_first != 1 && ret_second != 1) {
+    MS_LOG(INFO) << "Equip ca cert verify is failed";
+    return false;
+  }
+  MS_LOG(INFO) << "VerifyCAChain success.";
+
+  EVP_PKEY_free(equipPubKey);
+  EVP_PKEY_free(equipCAPubKey);
+  EVP_PKEY_free(rootFirstPubKey);
+  EVP_PKEY_free(rootSecondPubKey);
+  return true;
+}
+
+bool SSLWrapper::VerifyCRL(const std::string &equipCert) {
+  X509 *equipCertObj = ReadCertFromPerm(equipCert);
+  if (rootFirstCrl_ == nullptr && rootSecondCrl_ == nullptr) {
+    MS_LOG(INFO) << "RootFirstCrl && rootSecondCrl is nullptr.";
+    return false;
+  }
+
+  EVP_PKEY *evp_pkey = X509_get_pubkey(equipCertObj);
+  int ret = X509_CRL_verify(rootFirstCrl_, evp_pkey);
+  if (ret == 1) {
+    MS_LOG(INFO) << "Equip cert in root first crl, verify failed";
+    return false;
+  }
+  ret = X509_CRL_verify(rootSecondCrl_, evp_pkey);
+  if (ret == 1) {
+    MS_LOG(INFO) << "Equip cert in root second crl, verify failed";
+    return false;
+  }
+  MS_LOG(INFO) << "VerifyCRL success.";
+  return true;
+}
+
+bool SSLWrapper::VerifyRSAKey(const std::string &keyAttestation, const unsigned char *srcData,
+                              const unsigned char *signData, int srcDataLen) {
+  if (keyAttestation.empty() || srcData == nullptr || signData == nullptr) {
+    MS_LOG(INFO) << "KeyAttestation or srcData or signData is empty.";
+    return false;
+  }
+
+  X509 *keyAttestationCertObj = ReadCertFromPerm(keyAttestation);
+
+  EVP_PKEY *pubKey = X509_get_pubkey(keyAttestationCertObj);
+  RSA *pRSAPublicKey = EVP_PKEY_get0_RSA(pubKey);
+  if (pRSAPublicKey == nullptr) {
+    MS_LOG(INFO) << "Get rsa public key failed.";
+    return false;
+  }
+
+  int pubKeyLen = RSA_size(pRSAPublicKey);
+  int ret = RSA_verify(NID_sha256, srcData, srcDataLen, signData, pubKeyLen, pRSAPublicKey);
+  if (ret != 1) {
+    MS_LOG(WARNING) << "Verify error.";
+    int64_t ulErr = ERR_get_error();
+    char szErrMsg[1024] = {0};
+    MS_LOG(WARNING) << "Error number: " << ulErr;
+    ERR_error_string(ulErr, szErrMsg);
+    MS_LOG(INFO) << "Error message:" << szErrMsg;
+    return false;
+  }
+  RSA_free(pRSAPublicKey);
+  X509_free(keyAttestationCertObj);
+  CRYPTO_cleanup_all_ex_data();
+
+  MS_LOG(INFO) << "VerifyRSAKey success.";
+  return true;
+}
 }  // namespace core
 }  // namespace ps
 }  // namespace mindspore

mindspore/ccsrc/ps/core/communicator/ssl_wrapper.h

@@ -20,6 +20,11 @@
 #include <openssl/ssl.h>
 #include <openssl/rand.h>
 #include <openssl/err.h>
+#include <openssl/evp.h>
+#include <assert.h>
+
+#include <iostream>
+#include <string>
 
 #include "utils/log_adapter.h"
 
@@ -34,6 +39,32 @@ class SSLWrapper {
   }
   SSL_CTX *GetSSLCtx(bool is_server = true);
 
+  void InitRootCertAndCRL(const std::string rootFirstCaFilePath, const std::string rootSecondCaFilePath,
+                          const std::string crlFirstFilePath, const std::string crlSecondFilePath);
+
+  // read certificate from file path
+  X509 *ReadCertFromFile(const std::string &certPath);
+
+  // read Certificate Revocation List from file absolute path
+  X509_CRL *ReadCrlFromFile(const std::string &crlPath);
+
+  // read certificate from pem string
+  X509 *ReadCertFromPerm(std::string cert);
+
+  // verify valid of certificate time
+  bool VerifyCertTime(const X509 *cert);
+
+  // verify valid of certificate chain
+  bool VerifyCAChain(const std::string &keyAttestation, const std::string &equipCert, const std::string &equipCACert,
+                     std::string rootCert);
+
+  // verify valid of sign data
+  bool VerifyRSAKey(const std::string &keyAttestation, const unsigned char *srcData, const unsigned char *signData,
+                    int srcDataLen);
+
+  // verify valid of equip certificate with CRL
+  bool VerifyCRL(const std::string &equipCert);
+
  private:
   SSLWrapper();
   virtual ~SSLWrapper();
@@ -45,6 +76,15 @@ class SSLWrapper {
 
   SSL_CTX *ssl_ctx_;
   SSL_CTX *client_ssl_ctx_;
+
+  // The firset root ca certificate.
+  X509 *rootFirstCA_;
+  // The second root ca certificate.
+  X509 *rootSecondCA_;
+  // The firset root revocation list
+  X509_CRL *rootFirstCrl_;
+  // The second root revocation list
+  X509_CRL *rootSecondCrl_;
 };
 }  // namespace core
 }  // namespace ps

mindspore/ccsrc/ps/core/communicator/tcp_client.cc

@@ -109,7 +109,15 @@ void TcpClient::Init() {
   if (!PSContext::instance()->enable_ssl()) {
     buffer_event_ = bufferevent_socket_new(event_base_, -1, BEV_OPT_CLOSE_ON_FREE | BEV_OPT_THREADSAFE);
   } else {
+    MS_LOG(INFO) << "Enable ssl support.";
+
     SSL *ssl = SSL_new(SSLWrapper::GetInstance().GetSSLCtx(false));
+
+    X509 *cert = SSLWrapper::GetInstance().ReadCertFromFile(kCertificateChain);
+    if (!SSLWrapper::GetInstance().VerifyCertTime(cert)) {
+      MS_LOG(EXCEPTION) << "Verify cert time failed.";
+    }
+
     if (!SSL_CTX_use_certificate_chain_file(SSLWrapper::GetInstance().GetSSLCtx(false), kCertificateChain)) {
       MS_LOG(EXCEPTION) << "SSL use certificate chain file failed!";
     }

mindspore/ccsrc/ps/core/communicator/tcp_server.cc

@@ -287,8 +287,14 @@ void TcpServer::ListenerCallback(struct evconnlistener *, evutil_socket_t fd, st
   if (!PSContext::instance()->enable_ssl()) {
     bev = bufferevent_socket_new(base, fd, BEV_OPT_CLOSE_ON_FREE | BEV_OPT_THREADSAFE);
   } else {
+    MS_LOG(INFO) << "Enable ssl support.";
     SSL *ssl = SSL_new(SSLWrapper::GetInstance().GetSSLCtx());
 
+    X509 *cert = SSLWrapper::GetInstance().ReadCertFromFile(kCertificateChain);
+    if (!SSLWrapper::GetInstance().VerifyCertTime(cert)) {
+      MS_LOG(EXCEPTION) << "Verify cert time failed.";
+    }
+
     if (!SSL_CTX_use_certificate_chain_file(SSLWrapper::GetInstance().GetSSLCtx(), kCertificateChain)) {
       MS_LOG(EXCEPTION) << "SSL use certificate chain file failed!";
     }