[vbox-dev] [PATCH] virtualbox 5.1.4: get it built against openssl-1.1
Sebastian Andrzej Siewior
vbox-dev at ml.breakpoint.cc
Tue Sep 13 07:08:44 GMT 2016
From: Sebastian Andrzej Siewior <sebastian at breakpoint.cc>
Built tested. Please double check it, especially that part where
required_pkey_type vanished.
Signed-off-by: Sebastian Andrzej Siewior <sebastian at breakpoint.cc>
---
src/VBox/RDP/client-1.8.3/ssl.c | 70 +++++++++++++++++---------
src/VBox/Runtime/VBox/VBoxRTDeps.cpp | 14 +++---
src/VBox/Runtime/common/crypto/pkix-verify.cpp | 46 +++++++++++------
3 files changed, 83 insertions(+), 47 deletions(-)
diff --git a/src/VBox/RDP/client-1.8.3/ssl.c b/src/VBox/RDP/client-1.8.3/ssl.c
index dac5511..bd207d5 100644
--- a/src/VBox/RDP/client-1.8.3/ssl.c
+++ b/src/VBox/RDP/client-1.8.3/ssl.c
@@ -97,7 +97,7 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
uint8 * exponent)
{
BN_CTX *ctx;
- BIGNUM mod, exp, x, y;
+ BIGNUM *mod, *exp, *x, *y;
uint8 inr[SEC_MAX_MODULUS_SIZE];
int outlen;
@@ -107,24 +107,24 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
reverse(inr, len);
ctx = BN_CTX_new();
- BN_init(&mod);
- BN_init(&exp);
- BN_init(&x);
- BN_init(&y);
-
- BN_bin2bn(modulus, modulus_size, &mod);
- BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
- BN_bin2bn(inr, len, &x);
- BN_mod_exp(&y, &x, &exp, &mod, ctx);
- outlen = BN_bn2bin(&y, out);
+ mod = BN_new();
+ exp = BN_new();
+ x = BN_new();
+ y = BN_new();
+
+ BN_bin2bn(modulus, modulus_size, mod);
+ BN_bin2bn(exponent, SEC_EXPONENT_SIZE, exp);
+ BN_bin2bn(inr, len, x);
+ BN_mod_exp(y, x, exp, mod, ctx);
+ outlen = BN_bn2bin(y, out);
reverse(out, outlen);
if (outlen < (int) modulus_size)
memset(out + outlen, 0, modulus_size - outlen);
- BN_free(&y);
- BN_clear_free(&x);
- BN_free(&exp);
- BN_free(&mod);
+ BN_free(y);
+ BN_clear_free(x);
+ BN_free(exp);
+ BN_free(mod);
BN_CTX_free(ctx);
}
@@ -149,18 +149,25 @@ rdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len)
EVP_PKEY *epk = NULL;
RDSSL_RKEY *lkey;
int nid;
+ X509_PUBKEY *x509_pk;
+ X509_ALGOR *algor;
+ const ASN1_OBJECT *alg_obj;
/* By some reason, Microsoft sets the OID of the Public RSA key to
the oid for "MD5 with RSA Encryption" instead of "RSA Encryption"
Kudos to Richard Levitte for the following (. intiutive .)
lines of code that resets the OID and let's us extract the key. */
- nid = OBJ_obj2nid(cert->cert_info->key->algor->algorithm);
+
+ x509_pk = X509_get_X509_PUBKEY(cert);
+ X509_PUBKEY_get0_param(NULL, NULL, NULL, &algor, x509_pk);
+ X509_ALGOR_get0(&alg_obj, NULL, NULL, algor);
+
+ nid = OBJ_obj2nid(alg_obj);
if ((nid == NID_md5WithRSAEncryption) || (nid == NID_shaWithRSAEncryption))
{
DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n"));
- ASN1_OBJECT_free(cert->cert_info->key->algor->algorithm);
- cert->cert_info->key->algor->algorithm = OBJ_nid2obj(NID_rsaEncryption);
+ X509_ALGOR_set0(algor, OBJ_nid2obj(NID_rsaEncryption), 0, NULL);
}
epk = X509_get_pubkey(cert);
if (NULL == epk)
@@ -203,21 +210,37 @@ rdssl_rkey_free(RDSSL_RKEY * rkey)
RSA_free(rkey);
}
+#if OPENSSL_VERSION_NUMBER < 0x10100000
+static inline void RSA_get0_key(const RSA *r, const BIGNUM **n,
+ const BIGNUM **e, const BIGNUM **d)
+{
+ if (n != NULL)
+ *n = r->n;
+ if (e != NULL)
+ *e = r->e;
+ if (d != NULL)
+ *d = r->d;
+}
+#endif
+
/* returns error */
int
rdssl_rkey_get_exp_mod(RDSSL_RKEY * rkey, uint8 * exponent, uint32 max_exp_len, uint8 * modulus,
uint32 max_mod_len)
{
int len;
+ const BIGNUM *e, *n;
+
+ RSA_get0_key(rkey, &n, &e, NULL);
- if ((BN_num_bytes(rkey->e) > (int) max_exp_len) ||
- (BN_num_bytes(rkey->n) > (int) max_mod_len))
+ if ((BN_num_bytes(e) > (int) max_exp_len) ||
+ (BN_num_bytes(n) > (int) max_mod_len))
{
return 1;
}
- len = BN_bn2bin(rkey->e, exponent);
+ len = BN_bn2bin(e, exponent);
reverse(exponent, len);
- len = BN_bn2bin(rkey->n, modulus);
+ len = BN_bn2bin(n, modulus);
reverse(modulus, len);
return 0;
}
@@ -238,8 +261,5 @@ void
rdssl_hmac_md5(const void *key, int key_len, const unsigned char *msg, int msg_len,
unsigned char *md)
{
- HMAC_CTX ctx;
- HMAC_CTX_init(&ctx);
HMAC(EVP_md5(), key, key_len, msg, msg_len, md, NULL);
- HMAC_CTX_cleanup(&ctx);
}
diff --git a/src/VBox/Runtime/VBox/VBoxRTDeps.cpp b/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
index ddfccec..fa19fa7 100644
--- a/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
+++ b/src/VBox/Runtime/VBox/VBoxRTDeps.cpp
@@ -76,26 +76,28 @@ PFNRT g_VBoxRTDeps[] =
(PFNRT)i2d_X509,
(PFNRT)i2d_X509,
(PFNRT)i2d_PublicKey,
+#if OPENSSL_VERSION_NUMBER < 0x10100000
(PFNRT)RSA_generate_key,
- (PFNRT)RSA_generate_key_ex,
(PFNRT)DH_generate_parameters,
- (PFNRT)DH_generate_parameters_ex,
- (PFNRT)RAND_load_file,
(PFNRT)CRYPTO_set_dynlock_create_callback,
(PFNRT)CRYPTO_set_dynlock_lock_callback,
(PFNRT)CRYPTO_set_dynlock_destroy_callback,
+ (PFNRT)SSL_library_init,
+ (PFNRT)SSL_load_error_strings,
+ (PFNRT)TLSv1_server_method,
+#endif
+ (PFNRT)RSA_generate_key_ex,
+ (PFNRT)DH_generate_parameters_ex,
+ (PFNRT)RAND_load_file,
(PFNRT)RTAssertShouldPanic,
(PFNRT)ASMAtomicReadU64,
(PFNRT)ASMAtomicCmpXchgU64,
(PFNRT)ASMBitFirstSet,
(PFNRT)RTBldCfgRevision,
(PFNRT)SSL_free,
- (PFNRT)SSL_library_init,
- (PFNRT)SSL_load_error_strings,
(PFNRT)SSL_CTX_free,
(PFNRT)SSL_CTX_use_certificate_file,
(PFNRT)SSLv23_method,
- (PFNRT)TLSv1_server_method,
NULL
};
diff --git a/src/VBox/Runtime/common/crypto/pkix-verify.cpp b/src/VBox/Runtime/common/crypto/pkix-verify.cpp
index ba537d5..924005b 100644
--- a/src/VBox/Runtime/common/crypto/pkix-verify.cpp
+++ b/src/VBox/Runtime/common/crypto/pkix-verify.cpp
@@ -124,28 +124,39 @@ RTDECL(int) RTCrPkixPubKeyVerifySignature(PCRTASN1OBJID pAlgorithm, PCRTASN1DYNT
"EVP_get_digestbyname failed on %s (%s)", pszAlogSn, pAlgorithm->szObjId);
/* Initialize the EVP message digest context. */
- EVP_MD_CTX EvpMdCtx;
- EVP_MD_CTX_init(&EvpMdCtx);
- if (!EVP_VerifyInit_ex(&EvpMdCtx, pEvpMdType, NULL /*engine*/))
+ EVP_MD_CTX *EvpMdCtx;
+
+ EvpMdCtx = EVP_MD_CTX_create();
+
+ if (!EvpMdCtx)
+ return RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_CIPHER_ALOG_INIT_FAILED,
+ "EVP_MD_CTX_create failed");
+
+ if (!EVP_VerifyInit_ex(EvpMdCtx, pEvpMdType, NULL /*engine*/)) {
+ EVP_MD_CTX_destroy(EvpMdCtx);
return RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_CIPHER_ALOG_INIT_FAILED,
"EVP_VerifyInit_ex failed (algorithm type is %s / %s)", pszAlogSn, pAlgorithm->szObjId);
+ }
/* Create an EVP public key. */
int rcOssl;
EVP_PKEY *pEvpPublicKey = EVP_PKEY_new();
if (pEvpPublicKey)
{
- pEvpPublicKey->type = EVP_PKEY_type(pEvpMdType->required_pkey_type[0]);
- if (pEvpPublicKey->type != NID_undef)
+ int key_type;
+
+ EVP_PKEY_set_type(pEvpPublicKey, iAlgoNid);
+ key_type = EVP_PKEY_base_id(pEvpPublicKey);
+ if (key_type != NID_undef)
{
const unsigned char *puchPublicKey = RTASN1BITSTRING_GET_BIT0_PTR(pPublicKey);
- if (d2i_PublicKey(pEvpPublicKey->type, &pEvpPublicKey, &puchPublicKey, RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
+ if (d2i_PublicKey(key_type, &pEvpPublicKey, &puchPublicKey, RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
{
/* Digest the data. */
- EVP_VerifyUpdate(&EvpMdCtx, pvData, cbData);
+ EVP_VerifyUpdate(EvpMdCtx, pvData, cbData);
/* Verify the signature. */
- if (EVP_VerifyFinal(&EvpMdCtx,
+ if (EVP_VerifyFinal(EvpMdCtx,
RTASN1BITSTRING_GET_BIT0_PTR(pSignatureValue),
RTASN1BITSTRING_GET_BYTE_SIZE(pSignatureValue),
pEvpPublicKey) > 0)
@@ -158,13 +169,13 @@ RTDECL(int) RTCrPkixPubKeyVerifySignature(PCRTASN1OBJID pAlgorithm, PCRTASN1DYNT
}
else
rcOssl = RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_EVP_PKEY_TYPE_ERROR,
- "EVP_PKEY_type(%d) failed", pEvpMdType->required_pkey_type[0]);
+ "EVP_PKEY_type(%d) failed", iAlgoNid);
/* Cleanup and return.*/
EVP_PKEY_free(pEvpPublicKey);
}
else
- rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d) failed", pEvpMdType->required_pkey_type[0]);
- EVP_MD_CTX_cleanup(&EvpMdCtx);
+ rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d) failed", iAlgoNid);
+ EVP_MD_CTX_destroy(EvpMdCtx);
/*
* Check the result.
@@ -261,11 +272,14 @@ RTDECL(int) RTCrPkixPubKeyVerifySignedDigest(PCRTASN1OBJID pAlgorithm, PCRTASN1D
EVP_PKEY *pEvpPublicKey = EVP_PKEY_new();
if (pEvpPublicKey)
{
- pEvpPublicKey->type = EVP_PKEY_type(pEvpMdType->required_pkey_type[0]);
- if (pEvpPublicKey->type != NID_undef)
+ int key_type;
+
+ EVP_PKEY_set_type(pEvpPublicKey, iAlgoNid);
+ key_type = EVP_PKEY_base_id(pEvpPublicKey);
+ if (key_type != NID_undef)
{
const unsigned char *puchPublicKey = RTASN1BITSTRING_GET_BIT0_PTR(pPublicKey);
- if (d2i_PublicKey(pEvpPublicKey->type, &pEvpPublicKey, &puchPublicKey, RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
+ if (d2i_PublicKey(key_type, &pEvpPublicKey, &puchPublicKey, RTASN1BITSTRING_GET_BYTE_SIZE(pPublicKey)))
{
/* Create an EVP public key context we can use to validate the digest. */
EVP_PKEY_CTX *pEvpPKeyCtx = EVP_PKEY_CTX_new(pEvpPublicKey, NULL);
@@ -306,12 +320,12 @@ RTDECL(int) RTCrPkixPubKeyVerifySignedDigest(PCRTASN1OBJID pAlgorithm, PCRTASN1D
}
else
rcOssl = RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_OSSL_EVP_PKEY_TYPE_ERROR,
- "EVP_PKEY_type(%d) failed", pEvpMdType->required_pkey_type[0]);
+ "EVP_PKEY_type(%d) failed", iAlgoNid);
/* Cleanup and return.*/
EVP_PKEY_free(pEvpPublicKey);
}
else
- rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d) failed", pEvpMdType->required_pkey_type[0]);
+ rcOssl = RTErrInfoSetF(pErrInfo, VERR_NO_MEMORY, "EVP_PKEY_new(%d) failed", iAlgoNid);
/*
* Check the result.
--
2.9.3
More information about the vbox-dev
mailing list