Index: /trunk/include/iprt/crypto/pkcs7.h =================================================================== --- /trunk/include/iprt/crypto/pkcs7.h (revision 84247) +++ /trunk/include/iprt/crypto/pkcs7.h (revision 84248) @@ -596,6 +596,6 @@ RTDECL(int) RTCrPkcs7SimpleSignSignedData(uint32_t fFlags, PCRTCRX509CERTIFICATE pSigner, RTCRKEY hPrivateKey, - void const *pvData, size_t cbData, RTCRSTORE hAdditionalCerts, - void *pvResult, size_t *pcbResult, PRTERRINFO pErrInfo); + void const *pvData, size_t cbData, RTDIGESTTYPE enmDigestType, + RTCRSTORE hAdditionalCerts, void *pvResult, size_t *pcbResult, PRTERRINFO pErrInfo); /** @} */ Index: /trunk/include/iprt/crypto/pkix.h =================================================================== --- /trunk/include/iprt/crypto/pkix.h (revision 84247) +++ /trunk/include/iprt/crypto/pkix.h (revision 84248) @@ -35,4 +35,5 @@ RT_C_DECLS_BEGIN +struct RTCRX509CERTIFICATE; struct RTCRX509SUBJECTPUBLICKEYINFO; @@ -93,4 +94,15 @@ RTCRDIGEST hDigest, PRTERRINFO pErrInfo); +/** + * Checks if the hash size can be handled by the given public key. + */ +RTDECL(bool) RTCrPkixPubKeyCanHandleDigestType(struct RTCRX509SUBJECTPUBLICKEYINFO const *pPublicKeyInfo, + RTDIGESTTYPE enmDigestType, PRTERRINFO pErrInfo); + +/** + * Checks if the hash size can be handled by the given certificate's public key. + */ +RTDECL(bool) RTCrPkixCanCertHandleDigestType(struct RTCRX509CERTIFICATE const *pCertificate, + RTDIGESTTYPE enmDigestType, PRTERRINFO pErrInfo); /** Index: /trunk/include/iprt/crypto/rsa.h =================================================================== --- /trunk/include/iprt/crypto/rsa.h (revision 84247) +++ /trunk/include/iprt/crypto/rsa.h (revision 84248) @@ -62,4 +62,7 @@ RTASN1TYPE_STANDARD_PROTOTYPES(RTCRRSAPUBLICKEY, RTDECL, RTCrRsaPublicKey, SeqCore.Asn1Core); +RTDECL(bool) RTCrRsaPublicKey_CanHandleDigestType(PCRTCRRSAPUBLICKEY pRsaPublicKey, RTDIGESTTYPE enmDigestType, + PRTERRINFO pErrInfo); + /** @@ -124,4 +127,7 @@ /** @} */ +RTDECL(bool) RTCrRsaPrivateKey_CanHandleDigestType(PCRTCRRSAPRIVATEKEY pRsaPrivateKey, RTDIGESTTYPE enmDigestType, + PRTERRINFO pErrInfo); + /** Index: /trunk/include/iprt/crypto/x509.h =================================================================== --- /trunk/include/iprt/crypto/x509.h (revision 84247) +++ /trunk/include/iprt/crypto/x509.h (revision 84248) @@ -159,13 +159,15 @@ * as such. * @{ */ -#define RTCRX509ALGORITHMIDENTIFIERID_RSA "1.2.840.113549.1.1.1" -#define RTCRX509ALGORITHMIDENTIFIERID_MD2_WITH_RSA "1.2.840.113549.1.1.2" -#define RTCRX509ALGORITHMIDENTIFIERID_MD4_WITH_RSA "1.2.840.113549.1.1.3" -#define RTCRX509ALGORITHMIDENTIFIERID_MD5_WITH_RSA "1.2.840.113549.1.1.4" -#define RTCRX509ALGORITHMIDENTIFIERID_SHA1_WITH_RSA "1.2.840.113549.1.1.5" -#define RTCRX509ALGORITHMIDENTIFIERID_SHA256_WITH_RSA "1.2.840.113549.1.1.11" -#define RTCRX509ALGORITHMIDENTIFIERID_SHA384_WITH_RSA "1.2.840.113549.1.1.12" -#define RTCRX509ALGORITHMIDENTIFIERID_SHA512_WITH_RSA "1.2.840.113549.1.1.13" -#define RTCRX509ALGORITHMIDENTIFIERID_SHA224_WITH_RSA "1.2.840.113549.1.1.14" +#define RTCRX509ALGORITHMIDENTIFIERID_RSA "1.2.840.113549.1.1.1" +#define RTCRX509ALGORITHMIDENTIFIERID_MD2_WITH_RSA "1.2.840.113549.1.1.2" +#define RTCRX509ALGORITHMIDENTIFIERID_MD4_WITH_RSA "1.2.840.113549.1.1.3" +#define RTCRX509ALGORITHMIDENTIFIERID_MD5_WITH_RSA "1.2.840.113549.1.1.4" +#define RTCRX509ALGORITHMIDENTIFIERID_SHA1_WITH_RSA "1.2.840.113549.1.1.5" +#define RTCRX509ALGORITHMIDENTIFIERID_SHA256_WITH_RSA "1.2.840.113549.1.1.11" +#define RTCRX509ALGORITHMIDENTIFIERID_SHA384_WITH_RSA "1.2.840.113549.1.1.12" +#define RTCRX509ALGORITHMIDENTIFIERID_SHA512_WITH_RSA "1.2.840.113549.1.1.13" +#define RTCRX509ALGORITHMIDENTIFIERID_SHA224_WITH_RSA "1.2.840.113549.1.1.14" +#define RTCRX509ALGORITHMIDENTIFIERID_SHA512T224_WITH_RSA "1.2.840.113549.1.1.15" +#define RTCRX509ALGORITHMIDENTIFIERID_SHA512T256_WITH_RSA "1.2.840.113549.1.1.16" /** @} */ Index: /trunk/include/iprt/mangling.h =================================================================== --- /trunk/include/iprt/mangling.h (revision 84247) +++ /trunk/include/iprt/mangling.h (revision 84248) @@ -3342,5 +3342,7 @@ # define RTCrRsaOtherPrimeInfos_CheckSanity RT_MANGLER(RTCrRsaOtherPrimeInfos_CheckSanity) # define RTCrRsaPrivateKey_CheckSanity RT_MANGLER(RTCrRsaPrivateKey_CheckSanity) +# define RTCrRsaPrivateKey_CanHandleDigestType RT_MANGLER(RTCrRsaPrivateKey_CanHandleDigestType) # define RTCrRsaPublicKey_CheckSanity RT_MANGLER(RTCrRsaPublicKey_CheckSanity) +# define RTCrRsaPublicKey_CanHandleDigestType RT_MANGLER(RTCrRsaPublicKey_CanHandleDigestType) # define RTCrPemFindFirstSectionInContent RT_MANGLER(RTCrPemFindFirstSectionInContent) # define RTCrPemFreeSections RT_MANGLER(RTCrPemFreeSections) @@ -3463,4 +3465,6 @@ # define RTCrPkixPubKeyVerifySignedDigest RT_MANGLER(RTCrPkixPubKeyVerifySignedDigest) # define RTCrPkixPubKeyVerifySignedDigestByCertPubKeyInfo RT_MANGLER(RTCrPkixPubKeyVerifySignedDigestByCertPubKeyInfo) +# define RTCrPkixPubKeyCanHandleDigestType RT_MANGLER(RTCrPkixPubKeyCanHandleDigestType) +# define RTCrPkixCanCertHandleDigestType RT_MANGLER(RTCrPkixCanCertHandleDigestType) # define RTCrRandBytes RT_MANGLER(RTCrRandBytes) # define RTCrSpcAttributeTypeAndOptionalValue_DecodeAsn1 RT_MANGLER(RTCrSpcAttributeTypeAndOptionalValue_DecodeAsn1) Index: /trunk/src/VBox/Runtime/common/crypto/digest-core.cpp =================================================================== --- /trunk/src/VBox/Runtime/common/crypto/digest-core.cpp (revision 84247) +++ /trunk/src/VBox/Runtime/common/crypto/digest-core.cpp (revision 84248) @@ -426,4 +426,6 @@ case RTDIGESTTYPE_SHA384: return RTCRX509ALGORITHMIDENTIFIERID_SHA384; case RTDIGESTTYPE_SHA512: return RTCRX509ALGORITHMIDENTIFIERID_SHA512; + case RTDIGESTTYPE_SHA512T224: return RTCRX509ALGORITHMIDENTIFIERID_SHA512T224; + case RTDIGESTTYPE_SHA512T256: return RTCRX509ALGORITHMIDENTIFIERID_SHA512T256; default: return NULL; } Index: /trunk/src/VBox/Runtime/common/crypto/iprt-openssl.cpp =================================================================== --- /trunk/src/VBox/Runtime/common/crypto/iprt-openssl.cpp (revision 84247) +++ /trunk/src/VBox/Runtime/common/crypto/iprt-openssl.cpp (revision 84248) @@ -36,4 +36,5 @@ # include # include +# include # include "internal/iprt-openssl.h" @@ -146,4 +147,26 @@ } + +DECLHIDDEN(const void /*EVP_MD*/ *) rtCrOpenSslConvertDigestType(RTDIGESTTYPE enmDigestType, PRTERRINFO pErrInfo) +{ + const char *pszAlgoObjId = RTCrDigestTypeToAlgorithmOid(enmDigestType); + AssertReturnStmt(pszAlgoObjId, RTErrInfoSetF(pErrInfo, VERR_INVALID_PARAMETER, "Invalid type: %d", enmDigestType), NULL); + + int iAlgoNid = OBJ_txt2nid(pszAlgoObjId); + AssertReturnStmt(iAlgoNid != NID_undef, + RTErrInfoSetF(pErrInfo, VERR_CR_DIGEST_OSSL_DIGEST_INIT_ERROR, + "OpenSSL does not know: %s (%s)", pszAlgoObjId, RTCrDigestTypeToName(enmDigestType)), + NULL); + + const char *pszAlgoSn = OBJ_nid2sn(iAlgoNid); + const EVP_MD *pEvpMdType = EVP_get_digestbyname(pszAlgoSn); + AssertReturnStmt(pEvpMdType, + RTErrInfoSetF(pErrInfo, VERR_CR_DIGEST_OSSL_DIGEST_INIT_ERROR, "OpenSSL/EVP does not know: %d (%s; %s; %s)", + iAlgoNid, pszAlgoSn, pszAlgoSn, RTCrDigestTypeToName(enmDigestType)), + NULL); + + return pEvpMdType; +} + #endif /* IPRT_WITH_OPENSSL */ Index: /trunk/src/VBox/Runtime/common/crypto/pkcs7-sign.cpp =================================================================== --- /trunk/src/VBox/Runtime/common/crypto/pkcs7-sign.cpp (revision 84247) +++ /trunk/src/VBox/Runtime/common/crypto/pkcs7-sign.cpp (revision 84248) @@ -85,12 +85,22 @@ RTDECL(int) RTCrPkcs7SimpleSignSignedData(uint32_t fFlags, PCRTCRX509CERTIFICATE pSigner, RTCRKEY hPrivateKey, - void const *pvData, size_t cbData, RTCRSTORE hAdditionalCerts, - void *pvResult, size_t *pcbResult, PRTERRINFO pErrInfo) + void const *pvData, size_t cbData, RTDIGESTTYPE enmDigestType, + RTCRSTORE hAdditionalCerts, void *pvResult, size_t *pcbResult, PRTERRINFO pErrInfo) { size_t const cbResultBuf = *pcbResult; *pcbResult = 0; AssertReturn(!(fFlags & ~RTCRPKCS7SIGN_SD_F_VALID_MASK), VERR_INVALID_FLAGS); -#if defined(IPRT_WITH_OPENSSL) +#ifdef IPRT_WITH_OPENSSL AssertReturn((int)cbData >= 0 && (unsigned)cbData == cbData, VERR_TOO_MUCH_DATA); + + /* + * Resolve the digest type. + */ + const EVP_MD *pEvpMd = NULL; + if (enmDigestType != RTDIGESTTYPE_UNKNOWN) + { + pEvpMd = (const EVP_MD *)rtCrOpenSslConvertDigestType(enmDigestType, pErrInfo); + AssertReturn(pEvpMd, pErrInfo ? pErrInfo->rc : VERR_INVALID_PARAMETER); + } /* @@ -125,46 +135,57 @@ * Do the signing. */ - unsigned int fOsslSign = CMS_BINARY; + unsigned int fOsslSign = CMS_BINARY | CMS_PARTIAL; if (fFlags & RTCRPKCS7SIGN_SD_F_DEATCHED) fOsslSign |= CMS_DETACHED; if (fFlags & RTCRPKCS7SIGN_SD_F_NO_SMIME_CAP) fOsslSign |= CMS_NOSMIMECAP; - CMS_ContentInfo *pCms = CMS_sign(pOsslSigner, pEvpPrivateKey, pOsslAdditionalCerts, pOsslData, fOsslSign); - if (pCms) + CMS_ContentInfo *pCms = CMS_sign(NULL, NULL, pOsslAdditionalCerts, NULL, fOsslSign); + if (pCms != NULL) { - /* - * Get the output and copy it into the result buffer. - */ - BIO *pOsslResult = BIO_new(BIO_s_mem()); - if (pOsslResult) + if (CMS_add1_signer(pCms, pOsslSigner, pEvpPrivateKey, pEvpMd, fOsslSign) != NULL) { - rc = i2d_CMS_bio(pOsslResult, pCms); + rc = CMS_final(pCms, pOsslData, NULL /*dcont*/, fOsslSign); if (rc > 0) { - BUF_MEM *pBuf = NULL; - rc = (int)BIO_get_mem_ptr(pOsslResult, &pBuf); - if (rc > 0) + /* + * Get the output and copy it into the result buffer. + */ + BIO *pOsslResult = BIO_new(BIO_s_mem()); + if (pOsslResult) { - AssertPtr(pBuf); - size_t const cbResult = pBuf->length; - if ( cbResultBuf >= cbResult - && pvResult != NULL) + rc = i2d_CMS_bio(pOsslResult, pCms); + if (rc > 0) { - memcpy(pvResult, pBuf->data, cbResult); - rc = VINF_SUCCESS; + BUF_MEM *pBuf = NULL; + rc = (int)BIO_get_mem_ptr(pOsslResult, &pBuf); + if (rc > 0) + { + AssertPtr(pBuf); + size_t const cbResult = pBuf->length; + if ( cbResultBuf >= cbResult + && pvResult != NULL) + { + memcpy(pvResult, pBuf->data, cbResult); + rc = VINF_SUCCESS; + } + else + rc = VERR_BUFFER_OVERFLOW; + *pcbResult = cbResult; + } + else + rc = RTErrInfoSet(pErrInfo, VERR_GENERAL_FAILURE, "BIO_get_mem_ptr"); } else - rc = VERR_BUFFER_OVERFLOW; - *pcbResult = cbResult; + rc = RTErrInfoSet(pErrInfo, VERR_GENERAL_FAILURE, "i2d_CMS_bio"); + BIO_free(pOsslResult); } else - rc = RTErrInfoSet(pErrInfo, VERR_GENERAL_FAILURE, "BIO_get_mem_ptr"); + rc = RTErrInfoSet(pErrInfo, VERR_NO_MEMORY, "BIO_new/BIO_s_mem"); } else - rc = RTErrInfoSet(pErrInfo, VERR_GENERAL_FAILURE, "i2d_CMS_bio"); - BIO_free(pOsslResult); + rc = RTErrInfoSet(pErrInfo, VERR_GENERAL_FAILURE, "CMS_final"); } else - rc = RTErrInfoSet(pErrInfo, VERR_NO_MEMORY, "BIO_new/BIO_s_mem"); + rc = RTErrInfoSet(pErrInfo, VERR_GENERAL_FAILURE, "CMS_add1_signer"); CMS_ContentInfo_free(pCms); } @@ -180,5 +201,5 @@ return rc; #else - RT_NOREF(fFlags, pSigner, hPrivateKey, pvData, cbData, hAdditionalCerts, pvResult, pErrInfo, cbResultBuf); + RT_NOREF(fFlags, pSigner, hPrivateKey, pvData, cbData, enmDigestType, hAdditionalCerts, pvResult, pErrInfo, cbResultBuf); *pcbResult = 0; return VERR_NOT_IMPLEMENTED; Index: /trunk/src/VBox/Runtime/common/crypto/pkix-signature-rsa.cpp =================================================================== --- /trunk/src/VBox/Runtime/common/crypto/pkix-signature-rsa.cpp (revision 84247) +++ /trunk/src/VBox/Runtime/common/crypto/pkix-signature-rsa.cpp (revision 84248) @@ -81,34 +81,42 @@ * @{ */ static const uint8_t g_abMd2[] = -{/* { { 1.2.840.113549.2.2 (MD2), NULL }, hash octet-string } */ - 0x30,0x20, 0x30,0x0c, 0x06,0x08,0x2a,0x86,0x48,0x86,0xf7,0x0d,0x02,0x02, 0x05,0x00, 0x04,0x10 +{/* { { 1.2.840.113549.2.2 (MD2), NULL }, hash octet-string } */ + 0x30,0x20, 0x30,0x0c, 0x06,0x08,0x2a,0x86,0x48,0x86,0xf7,0x0d,0x02,0x02, 0x05,0x00, 0x04,0x10 }; static const uint8_t g_abMd4[] = -{/* { { 1.2.840.113549.2.4 (MD4), NULL }, hash octet-string } */ - 0x30,0x20, 0x30,0x0c, 0x06,0x08,0x2a,0x86,0x48,0x86,0xf7,0x0d,0x02,0x04, 0x05,0x00, 0x04,0x10 +{/* { { 1.2.840.113549.2.4 (MD4), NULL }, hash octet-string } */ + 0x30,0x20, 0x30,0x0c, 0x06,0x08,0x2a,0x86,0x48,0x86,0xf7,0x0d,0x02,0x04, 0x05,0x00, 0x04,0x10 }; static const uint8_t g_abMd5[] = -{/* { { 1.2.840.113549.2.5 (MD5), NULL }, hash octet-string } */ - 0x30,0x20, 0x30,0x0c, 0x06,0x08,0x2a,0x86,0x48,0x86,0xf7,0x0d,0x02,0x05, 0x05,0x00, 0x04,0x10 +{/* { { 1.2.840.113549.2.5 (MD5), NULL }, hash octet-string } */ + 0x30,0x20, 0x30,0x0c, 0x06,0x08,0x2a,0x86,0x48,0x86,0xf7,0x0d,0x02,0x05, 0x05,0x00, 0x04,0x10 }; static const uint8_t g_abSha1[] = -{/* { { 1.3.14.3.2.26 (SHA-1), NULL }, hash octet-string } */ - 0x30,0x21, 0x30,0x09, 0x06,0x05,0x2b,0x0e,0x03,0x02,0x1a, 0x05,0x00, 0x04,0x14 +{/* { { 1.3.14.3.2.26 (SHA-1), NULL }, hash octet-string } */ + 0x30,0x21, 0x30,0x09, 0x06,0x05,0x2b,0x0e,0x03,0x02,0x1a, 0x05,0x00, 0x04,0x14 }; static const uint8_t g_abSha256[] = -{/* { { 2.16.840.1.101.3.4.2.1 (SHA-256), NULL }, hash octet-string } */ - 0x30,0x31, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x01, 0x05,0x00, 0x04,0x20 +{/* { { 2.16.840.1.101.3.4.2.1 (SHA-256), NULL }, hash octet-string } */ + 0x30,0x31, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x01, 0x05,0x00, 0x04,0x20 }; static const uint8_t g_abSha384[] = -{/* { { 2.16.840.1.101.3.4.2.2 (SHA-384), NULL }, hash octet-string } */ - 0x30,0x41, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x02, 0x05,0x00, 0x04,0x30 +{/* { { 2.16.840.1.101.3.4.2.2 (SHA-384), NULL }, hash octet-string } */ + 0x30,0x41, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x02, 0x05,0x00, 0x04,0x30 }; static const uint8_t g_abSha512[] = -{/* { { 2.16.840.1.101.3.4.2.3 (SHA-512), NULL }, hash octet-string } */ - 0x30,0x51, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x03, 0x05,0x00, 0x04,0x40 +{/* { { 2.16.840.1.101.3.4.2.3 (SHA-512), NULL }, hash octet-string } */ + 0x30,0x51, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x03, 0x05,0x00, 0x04,0x40 }; static const uint8_t g_abSha224[] = -{/* { { 2.16.840.1.101.3.4.2.4 (SHA-224), NULL }, hash octet-string } */ - 0x30,0x2d, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x04, 0x05,0x00, 0x04,0x1c +{/* { { 2.16.840.1.101.3.4.2.4 (SHA-224), NULL }, hash octet-string } */ + 0x30,0x2d, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x04, 0x05,0x00, 0x04,0x1c +}; +static const uint8_t g_abSha512T224[] = +{/* { { 2.16.840.1.101.3.4.2.5 (SHA-512T224), NULL }, hash octet-string } */ + 0x30,0x2d, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x05, 0x05,0x00, 0x04,0x1c +}; +static const uint8_t g_abSha512T256[] = +{/* { { 2.16.840.1.101.3.4.2.6 (SHA-512T256), NULL }, hash octet-string } */ + 0x30,0x31, 0x30,0x0d, 0x06,0x09,0x60,0x86,0x48,0x01,0x65,0x03,0x04,0x02,0x06, 0x05,0x00, 0x04,0x20 }; /** @} */ @@ -122,12 +130,14 @@ } const g_aDigestInfos[] = { - { RTDIGESTTYPE_MD2, g_abMd2, sizeof(g_abMd2) }, - { RTDIGESTTYPE_MD4, g_abMd4, sizeof(g_abMd4) }, - { RTDIGESTTYPE_MD5, g_abMd5, sizeof(g_abMd5) }, - { RTDIGESTTYPE_SHA1, g_abSha1, sizeof(g_abSha1) }, - { RTDIGESTTYPE_SHA256, g_abSha256, sizeof(g_abSha256) }, - { RTDIGESTTYPE_SHA384, g_abSha384, sizeof(g_abSha384) }, - { RTDIGESTTYPE_SHA512, g_abSha512, sizeof(g_abSha512) }, - { RTDIGESTTYPE_SHA224, g_abSha224, sizeof(g_abSha224) }, + { RTDIGESTTYPE_SHA1, g_abSha1, sizeof(g_abSha1) }, + { RTDIGESTTYPE_SHA256, g_abSha256, sizeof(g_abSha256) }, + { RTDIGESTTYPE_SHA512, g_abSha512, sizeof(g_abSha512) }, + { RTDIGESTTYPE_MD2, g_abMd2, sizeof(g_abMd2) }, + { RTDIGESTTYPE_MD4, g_abMd4, sizeof(g_abMd4) }, + { RTDIGESTTYPE_MD5, g_abMd5, sizeof(g_abMd5) }, + { RTDIGESTTYPE_SHA384, g_abSha384, sizeof(g_abSha384) }, + { RTDIGESTTYPE_SHA224, g_abSha224, sizeof(g_abSha224) }, + { RTDIGESTTYPE_SHA512T224, g_abSha512T224, sizeof(g_abSha512T224)}, + { RTDIGESTTYPE_SHA512T256, g_abSha512T256, sizeof(g_abSha512T256)}, }; @@ -464,2 +474,52 @@ }; + +/** + * Worker for RTCrRsaPublicKey_CanHandleDigestType and + * RTCrRsaPrivateKey_CanHandleDigestType. + * + * We implement these two functions here because we've already got the + * DigestInfo sizes nicely lined up here. + */ +static bool rtCrRsa_CanHandleDigestType(int32_t cModulusBits, RTDIGESTTYPE enmDigestType, PRTERRINFO pErrInfo) +{ + /* + * ASSUME EMSA-PKCS1-v1_5 padding scheme (RFC-8017 section 9.2): + * - 11 byte padding prefix (00, 01, 8 times ff) + * - digest info der sequence for rsaWithXxxxEncryption + * - the hash value. + */ + for (uint32_t i = 0; i < RT_ELEMENTS(g_aDigestInfos); i++) + if (g_aDigestInfos[i].enmDigest == enmDigestType) + { + size_t const cbHash = RTCrDigestTypeToHashSize(enmDigestType); + AssertBreak(cbHash > 0); + + size_t cbMsg = 11 + g_aDigestInfos[i].cb + cbHash; + if ((ssize_t)cbMsg <= cModulusBits / 8) + return true; + RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_INVALID_SIGNATURE_LENGTH, "cModulusBits=%d cbMsg=%u", cModulusBits, cbMsg); + return false; + } + RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_UNKNOWN_DIGEST_TYPE, "%s", RTCrDigestTypeToName(enmDigestType)); + return false; +} + + +RTDECL(bool) RTCrRsaPublicKey_CanHandleDigestType(PCRTCRRSAPUBLICKEY pRsaPublicKey, RTDIGESTTYPE enmDigestType, + PRTERRINFO pErrInfo) +{ + if (RTCrRsaPublicKey_IsPresent(pRsaPublicKey)) + return rtCrRsa_CanHandleDigestType(RTAsn1Integer_UnsignedLastBit(&pRsaPublicKey->Modulus) + 1, enmDigestType, pErrInfo); + return false; +} + + +RTDECL(bool) RTCrRsaPrivateKey_CanHandleDigestType(PCRTCRRSAPRIVATEKEY pRsaPrivateKey, RTDIGESTTYPE enmDigestType, + PRTERRINFO pErrInfo) +{ + if (RTCrRsaPrivateKey_IsPresent(pRsaPrivateKey)) + return rtCrRsa_CanHandleDigestType(RTAsn1Integer_UnsignedLastBit(&pRsaPrivateKey->Modulus) + 1, enmDigestType, pErrInfo); + return false; +} + Index: /trunk/src/VBox/Runtime/common/crypto/pkix-util.cpp =================================================================== --- /trunk/src/VBox/Runtime/common/crypto/pkix-util.cpp (revision 84247) +++ /trunk/src/VBox/Runtime/common/crypto/pkix-util.cpp (revision 84248) @@ -32,6 +32,9 @@ #include -#include +#include +#include +#include #include +#include #ifdef IPRT_WITH_OPENSSL @@ -92,2 +95,49 @@ } + +RTDECL(bool) RTCrPkixPubKeyCanHandleDigestType(PCRTCRX509SUBJECTPUBLICKEYINFO pPublicKeyInfo, RTDIGESTTYPE enmDigestType, + PRTERRINFO pErrInfo) +{ + bool fRc = false; + if (RTCrX509SubjectPublicKeyInfo_IsPresent(pPublicKeyInfo)) + { + void const * const pvKeyBits = RTASN1BITSTRING_GET_BIT0_PTR(&pPublicKeyInfo->SubjectPublicKey); + uint32_t const cbKeyBits = RTASN1BITSTRING_GET_BYTE_SIZE(&pPublicKeyInfo->SubjectPublicKey); + RTASN1CURSORPRIMARY PrimaryCursor; + union + { + RTCRRSAPUBLICKEY RsaPublicKey; + } u; + + if (RTAsn1ObjId_CompareWithString(&pPublicKeyInfo->Algorithm.Algorithm, RTCR_PKCS1_RSA_OID) == 0) + { + /* + * RSA. + */ + RTAsn1CursorInitPrimary(&PrimaryCursor, pvKeyBits, cbKeyBits, pErrInfo, &g_RTAsn1DefaultAllocator, + RTASN1CURSOR_FLAGS_DER, "rsa"); + + RT_ZERO(u.RsaPublicKey); + int rc = RTCrRsaPublicKey_DecodeAsn1(&PrimaryCursor.Cursor, 0, &u.RsaPublicKey, "PublicKey"); + if (RT_SUCCESS(rc)) + fRc = RTCrRsaPublicKey_CanHandleDigestType(&u.RsaPublicKey, enmDigestType, pErrInfo); + RTCrRsaPublicKey_Delete(&u.RsaPublicKey); + } + else + { + RTErrInfoSetF(pErrInfo, VERR_CR_PKIX_CIPHER_ALGO_NOT_KNOWN, "%s", pPublicKeyInfo->Algorithm.Algorithm.szObjId); + AssertMsgFailed(("unknown key algorithm: %s\n", pPublicKeyInfo->Algorithm.Algorithm.szObjId)); + fRc = true; + } + } + return fRc; +} + + +RTDECL(bool) RTCrPkixCanCertHandleDigestType(PCRTCRX509CERTIFICATE pCertificate, RTDIGESTTYPE enmDigestType, PRTERRINFO pErrInfo) +{ + if (RTCrX509Certificate_IsPresent(pCertificate)) + return RTCrPkixPubKeyCanHandleDigestType(&pCertificate->TbsCertificate.SubjectPublicKeyInfo, enmDigestType, pErrInfo); + return false; +} + Index: /trunk/src/VBox/Runtime/common/crypto/x509-core.cpp =================================================================== --- /trunk/src/VBox/Runtime/common/crypto/x509-core.cpp (revision 84247) +++ /trunk/src/VBox/Runtime/common/crypto/x509-core.cpp (revision 84248) @@ -170,4 +170,14 @@ { if (!strcmp(pszEncryptedDigestOid, RTCRX509ALGORITHMIDENTIFIERID_SHA224_WITH_RSA)) + return 0; + } + else if (!strcmp(pszDigestOid, RTCRX509ALGORITHMIDENTIFIERID_SHA512T224)) + { + if (!strcmp(pszEncryptedDigestOid, RTCRX509ALGORITHMIDENTIFIERID_SHA512T224_WITH_RSA)) + return 0; + } + else if (!strcmp(pszDigestOid, RTCRX509ALGORITHMIDENTIFIERID_SHA512T256)) + { + if (!strcmp(pszEncryptedDigestOid, RTCRX509ALGORITHMIDENTIFIERID_SHA512T256_WITH_RSA)) return 0; } @@ -219,4 +229,10 @@ || !strcmp(pszDigestOid, RTCRX509ALGORITHMIDENTIFIERID_SHA224_WITH_RSA)) return RTCRX509ALGORITHMIDENTIFIERID_SHA224_WITH_RSA; + if ( !strcmp(pszDigestOid, RTCRX509ALGORITHMIDENTIFIERID_SHA512T224) + || !strcmp(pszDigestOid, RTCRX509ALGORITHMIDENTIFIERID_SHA512T224_WITH_RSA)) + return RTCRX509ALGORITHMIDENTIFIERID_SHA512T224_WITH_RSA; + if ( !strcmp(pszDigestOid, RTCRX509ALGORITHMIDENTIFIERID_SHA512T256) + || !strcmp(pszDigestOid, RTCRX509ALGORITHMIDENTIFIERID_SHA512T256_WITH_RSA)) + return RTCRX509ALGORITHMIDENTIFIERID_SHA512T256_WITH_RSA; /* if (!strcmp(pszDigestOid, RTCRX509ALGORITHMIDENTIFIERID_WHIRLPOOL)) Index: /trunk/src/VBox/Runtime/include/internal/iprt-openssl.h =================================================================== --- /trunk/src/VBox/Runtime/include/internal/iprt-openssl.h (revision 84247) +++ /trunk/src/VBox/Runtime/include/internal/iprt-openssl.h (revision 84248) @@ -42,4 +42,5 @@ DECLHIDDEN(void) rtCrOpenSslFreeConvertedX509Cert(void *pvOsslCert); DECLHIDDEN(int) rtCrOpenSslAddX509CertToStack(void *pvOsslStack, PCRTCRX509CERTIFICATE pCert, PRTERRINFO pErrInfo); +DECLHIDDEN(const void /*EVP_MD*/ *) rtCrOpenSslConvertDigestType(RTDIGESTTYPE enmDigestType, PRTERRINFO pErrInfo); DECLHIDDEN(int) rtCrKeyToOpenSslKey(RTCRKEY hKey, bool fNeedPublic, void /*EVP_PKEY*/ **ppEvpKey, PRTERRINFO pErrInfo);