$NetBSD: patch-src_SelfSignedCertificate.cpp,v 1.1 2024/03/21 10:34:10 nia Exp $

From f4cea62ed95e4967d8591f25e903f5e8fc2e2a30 Mon Sep 17 00:00:00 2001
From: Terry Geng <terry@terriex.com>
Date: Mon, 6 Dec 2021 10:45:11 -0500
Subject: [PATCH] BUILD(crypto): Migrate to OpenSSL 3.0-compatible API

OpenSSL 3.0 deprecated several low-level APIs and the usage of them
caused errors/warnings that prevent the binary from being built against
OpenSSL 3.0.
Some primitive efforts have been made in #5317 but were incomplete.
This commit follows https://www.openssl.org/docs/man3.0/man7/migration_guide.html,
https://code.woboq.org/qt6/qtopcua/src/opcua/x509/qopcuakeypair_openssl.cpp.html,
and clears all errors/warnings related to the usage of deprecated APIs.

--- src/SelfSignedCertificate.cpp.orig	2022-09-13 17:24:40.000000000 +0000
+++ src/SelfSignedCertificate.cpp
@@ -5,8 +5,6 @@
 
 #include "SelfSignedCertificate.h"
 
-#include <openssl/x509v3.h>
-
 #define SSL_STRING(x) QString::fromLatin1(x).toUtf8().data()
 
 static int add_ext(X509 *crt, int nid, char *value) {
@@ -28,108 +26,86 @@ static int add_ext(X509 *crt, int nid, c
 	return 1;
 }
 
-bool SelfSignedCertificate::generate(CertificateType certificateType, QString clientCertName, QString clientCertEmail,
-									 QSslCertificate &qscCert, QSslKey &qskKey) {
-	bool ok                    = true;
-	X509 *x509                 = nullptr;
-	EVP_PKEY *pkey             = nullptr;
-	RSA *rsa                   = nullptr;
-	BIGNUM *e                  = nullptr;
-	X509_NAME *name            = nullptr;
-	ASN1_INTEGER *serialNumber = nullptr;
-	ASN1_TIME *notBefore       = nullptr;
-	ASN1_TIME *notAfter        = nullptr;
-	QString commonName;
-	bool isServerCert = certificateType == CertificateTypeServerCertificate;
-
-	if (CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON) == -1) {
-		ok = false;
-		goto out;
+EVP_PKEY *SelfSignedCertificate::generate_rsa_keypair() {
+	EVP_PKEY *pkey = EVP_PKEY_new();
+	if (!pkey) {
+		return nullptr;
 	}
 
-	x509 = X509_new();
-	if (!x509) {
-		ok = false;
-		goto out;
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+	EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, nullptr);
+	if (!ctx) {
+		return nullptr;
 	}
-
-	pkey = EVP_PKEY_new();
-	if (!pkey) {
-		ok = false;
-		goto out;
+	if (EVP_PKEY_keygen_init(ctx) <= 0) {
+		return nullptr;
 	}
-
-	rsa = RSA_new();
+	if (EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, 2048) <= 0) {
+		return nullptr;
+	}
+	if (EVP_PKEY_keygen(ctx, &pkey) <= 0) {
+		return nullptr;
+	}
+	EVP_PKEY_CTX_free(ctx);
+#else
+	RSA *rsa  = RSA_new();
+	BIGNUM *e = BN_new();
 	if (!rsa) {
-		ok = false;
-		goto out;
+		return nullptr;
 	}
-
-	e = BN_new();
 	if (!e) {
-		ok = false;
-		goto out;
+		return nullptr;
 	}
 	if (BN_set_word(e, 65537) == 0) {
-		ok = false;
-		goto out;
+		return nullptr;
 	}
-
 	if (RSA_generate_key_ex(rsa, 2048, e, nullptr) == 0) {
-		ok = false;
-		goto out;
+		return nullptr;
 	}
-
 	if (EVP_PKEY_assign_RSA(pkey, rsa) == 0) {
-		ok = false;
-		goto out;
-	}
-
-	if (X509_set_version(x509, 2) == 0) {
-		ok = false;
-		goto out;
-	}
-
-	serialNumber = X509_get_serialNumber(x509);
-	if (!serialNumber) {
-		ok = false;
-		goto out;
-	}
-	if (ASN1_INTEGER_set(serialNumber, 1) == 0) {
-		ok = false;
-		goto out;
+		return nullptr;
 	}
+	BN_free(e);
+	RSA_free(rsa);
+#endif
+	return pkey;
+}
 
-	notBefore = X509_get_notBefore(x509);
-	if (!notBefore) {
-		ok = false;
-		goto out;
-	}
-	if (!X509_gmtime_adj(notBefore, 0)) {
-		ok = false;
-		goto out;
+#define CHECK(statement) \
+	if (!(statement)) {  \
+		ok = false;      \
+		goto out;        \
 	}
 
-	notAfter = X509_get_notAfter(x509);
-	if (!notAfter) {
-		ok = false;
-		goto out;
-	}
-	if (!X509_gmtime_adj(notAfter, 60 * 60 * 24 * 365 * 20)) {
-		ok = false;
-		goto out;
-	}
 
-	if (X509_set_pubkey(x509, pkey) == 0) {
-		ok = false;
-		goto out;
-	}
+bool SelfSignedCertificate::generate(CertificateType certificateType, QString clientCertName, QString clientCertEmail,
+									 QSslCertificate &qscCert, QSslKey &qskKey) {
+	bool ok                    = true;
+	EVP_PKEY *pkey             = nullptr;
+	X509 *x509                 = nullptr;
+	X509_NAME *name            = nullptr;
+	ASN1_INTEGER *serialNumber = nullptr;
+	ASN1_TIME *notBefore       = nullptr;
+	ASN1_TIME *notAfter        = nullptr;
+	QString commonName;
+	bool isServerCert = certificateType == CertificateTypeServerCertificate;
 
-	name = X509_get_subject_name(x509);
-	if (!name) {
-		ok = false;
-		goto out;
-	}
+	// In Qt 5.15, a class was added to wrap up the procedures of generating a self-signed certificate.
+	// See https://doc.qt.io/qt-5/qopcuax509certificatesigningrequest.html.
+	// We should consider migrating to this class after switching to Qt 5.15.
+
+	CHECK(pkey = generate_rsa_keypair());
+
+	CHECK(x509 = X509_new());
+	CHECK(X509_set_version(x509, 2));
+	CHECK(serialNumber = X509_get_serialNumber(x509));
+	CHECK(ASN1_INTEGER_set(serialNumber, 1));
+	CHECK(notBefore = X509_get_notBefore(x509));
+	CHECK(X509_gmtime_adj(notBefore, 0));
+	CHECK(notAfter = X509_get_notAfter(x509));
+	CHECK(X509_gmtime_adj(notAfter, 60 * 60 * 24 * 365 * 20))
+	CHECK(X509_set_pubkey(x509, pkey));
+	CHECK(name = X509_get_subject_name(x509));
 
 	if (isServerCert) {
 		commonName = QLatin1String("Murmur Autogenerated Certificate v2");
@@ -141,120 +117,63 @@ bool SelfSignedCertificate::generate(Cer
 		}
 	}
 
-	if (X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_UTF8,
-								   reinterpret_cast< unsigned char * >(commonName.toUtf8().data()), -1, -1, 0)
-		== 0) {
-		ok = false;
-		goto out;
-	}
+	CHECK(X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_UTF8,
+									 reinterpret_cast< unsigned char * >(commonName.toUtf8().data()), -1, -1, 0));
 
-	if (X509_set_issuer_name(x509, name) == 0) {
-		ok = false;
-		goto out;
-	}
+	CHECK(X509_set_issuer_name(x509, name));
 
-	if (add_ext(x509, NID_basic_constraints, SSL_STRING("critical,CA:FALSE")) == 0) {
-		ok = false;
-		goto out;
-	}
+	CHECK(add_ext(x509, NID_basic_constraints, SSL_STRING("critical,CA:FALSE")));
 
 	if (isServerCert) {
-		if (add_ext(x509, NID_ext_key_usage, SSL_STRING("serverAuth,clientAuth")) == 0) {
-			ok = false;
-			goto out;
-		}
+		CHECK(add_ext(x509, NID_ext_key_usage, SSL_STRING("serverAuth,clientAuth")))
 	} else {
-		if (add_ext(x509, NID_ext_key_usage, SSL_STRING("clientAuth")) == 0) {
-			ok = false;
-			goto out;
-		}
+		CHECK(add_ext(x509, NID_ext_key_usage, SSL_STRING("clientAuth")));
 	}
 
-	if (add_ext(x509, NID_subject_key_identifier, SSL_STRING("hash")) == 0) {
-		ok = false;
-		goto out;
-	}
+	CHECK(add_ext(x509, NID_subject_key_identifier, SSL_STRING("hash")));
 
 	if (isServerCert) {
-		if (add_ext(x509, NID_netscape_comment, SSL_STRING("Generated from murmur")) == 0) {
-			ok = false;
-			goto out;
-		}
+		CHECK(add_ext(x509, NID_netscape_comment, SSL_STRING("Generated from murmur")));
 	} else {
-		if (add_ext(x509, NID_netscape_comment, SSL_STRING("Generated by Mumble")) == 0) {
-			ok = false;
-			goto out;
-		}
+		CHECK(add_ext(x509, NID_netscape_comment, SSL_STRING("Generated by Mumble")));
 	}
 
 	if (!isServerCert) {
 		if (!clientCertEmail.trimmed().isEmpty()) {
-			if (add_ext(x509, NID_subject_alt_name,
-						QString::fromLatin1("email:%1").arg(clientCertEmail).toUtf8().data())
-				== 0) {
-				ok = false;
-				goto out;
-			}
+			CHECK(add_ext(x509, NID_subject_alt_name,
+						  QString::fromLatin1("email:%1").arg(clientCertEmail).toUtf8().data()));
 		}
 	}
 
-	if (X509_sign(x509, pkey, EVP_sha1()) == 0) {
-		ok = false;
-		goto out;
-	}
+	CHECK(X509_sign(x509, pkey, EVP_sha1()));
 
 	{
 		QByteArray crt;
 		int len = i2d_X509(x509, nullptr);
-		if (len <= 0) {
-			ok = false;
-			goto out;
-		}
+		CHECK(len > 0);
 		crt.resize(len);
 
 		unsigned char *dptr = reinterpret_cast< unsigned char * >(crt.data());
-		if (i2d_X509(x509, &dptr) != len) {
-			ok = false;
-			goto out;
-		}
+		CHECK(i2d_X509(x509, &dptr) == len);
 
 		qscCert = QSslCertificate(crt, QSsl::Der);
-		if (qscCert.isNull()) {
-			ok = false;
-			goto out;
-		}
+		CHECK(!qscCert.isNull());
 	}
 
 	{
 		QByteArray key;
 		int len = i2d_PrivateKey(pkey, nullptr);
-		if (len <= 0) {
-			ok = false;
-			goto out;
-		}
+		CHECK(len > 0);
 		key.resize(len);
 
 		unsigned char *dptr = reinterpret_cast< unsigned char * >(key.data());
-		if (i2d_PrivateKey(pkey, &dptr) != len) {
-			ok = false;
-			goto out;
-		}
+		CHECK(i2d_PrivateKey(pkey, &dptr) == len);
 
 		qskKey = QSslKey(key, QSsl::Rsa, QSsl::Der);
-		if (qskKey.isNull()) {
-			ok = false;
-			goto out;
-		}
+		CHECK(!qskKey.isNull());
 	}
 
 out:
-	if (e) {
-		BN_free(e);
-	}
-	// We only need to free the pkey pointer,
-	// not the RSA pointer. We have assigned
-	// our RSA key to pkey, and it will be freed
-	// once we free pkey.
 	if (pkey) {
 		EVP_PKEY_free(pkey);
 	}
